Files

201 lines
13 KiB
JavaScript

/** Verify runtime accounting and controls against the installed native Harness. */
import assert from 'node:assert/strict';
import {mkdtemp,readFile,writeFile,mkdir} from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {createRequire} from 'node:module';
import {pathToFileURL,fileURLToPath} from 'node:url';
import {mock} from 'node:test';
import {registerRuntimeUsage} from '../integration/dsh/task-plugin/runtime_usage.mjs';
import {bootRuntime} from '../integration/dsh/start.mjs';
const repo=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..');
const output=await mkdtemp(path.join(os.tmpdir(),'dsh-runtime-usage-'));
const reportPath=path.join(repo,'build/dsh-native-usage-verification/report.json');
process.env.DX_DSH_HOME=path.join(output,'home');
process.env.DX_DSH_CONTEXT_FILE=path.join(output,'context.json');
await writeFile(process.env.DX_DSH_CONTEXT_FILE,JSON.stringify({task_id:'usage-fixture',revision:'usage-r1'}));
// The small lifecycle fixture checks bounded retention and unknown values. All
// provider accounting, authentication and cancellation below use the native CLI.
{
const listeners=new Map(),agents=new Map();
const fixture={on(name,fn){listeners.set(name,fn);return()=>listeners.delete(name);},get(){return undefined;},
agents:{get:id=>agents.get(id),list:()=>[...agents.values()]},
webServer:{register(){return()=>{};}},sessionController:{cancel:()=>({accepted:true})}};
const historicalUsage={inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24};
const historicalEvents=[{type:'assistant/message',time:1000,data:{usage:historicalUsage,message:{content:[{type:'text',text:'PRIVATE_HISTORY_TEXT'}]}}},
{type:'assistant/message',time:2000,data:{message:{content:[]}}},
{type:'compaction/summary',time:3000,data:{llmStreamCall:true,usage:historicalUsage}},
{type:'assistant/attempt',time:4000,data:{stream:[{type:'chunk',time:3999,chunk:{type:'usage',usage:historicalUsage}}]}}];
const restored={id:'restored-session',session:{snapshotEvents:()=>historicalEvents}};
agents.set(restored.id,restored);
const runtime=await registerRuntimeUsage(fixture,{admit:()=>true});
let restoredRow=runtime.snapshot().sessions.find(row=>row.session_id===restored.id);
assert.equal(restoredRow.input_tokens,36);assert.equal(restoredRow.output_tokens,12);
assert.equal(restoredRow.cached_tokens,24);assert.equal(restoredRow.total_tokens,72);
assert.equal(restoredRow.usage_complete,false);assert.equal(restoredRow.response_count,4);
assert.equal(restoredRow.elapsed_seconds,null);
assert.ok(!JSON.stringify(restoredRow).includes('PRIVATE_HISTORY_TEXT'));
// Re-announcement and a fresh response must not recharge historical events.
await listeners.get('agent/created')?.({agent:restored});
for await(const chunk of listeners.get('llm/stream')({sessionId:restored.id,signal:new AbortController().signal},async function*(){
yield {type:'usage',usage:historicalUsage};yield {type:'finish',reason:{kind:'stop'}};
})){}
restoredRow=runtime.snapshot().sessions.find(row=>row.session_id===restored.id);
assert.equal(restoredRow.total_tokens,96);
const created={id:'reopened-session',session:{snapshotEvents:()=>historicalEvents}};
agents.set(created.id,created);await listeners.get('agent/created')?.({agent:created});
assert.equal(runtime.snapshot().sessions.find(row=>row.session_id===created.id).total_tokens,72);
for(let i=0;i<65;i++){
const agent={id:'known-session-'+i};agents.set(agent.id,agent);
await listeners.get('agent/created')?.({agent});
}
const snapshot=runtime.snapshot();
assert.equal(snapshot.sessions.length,50);
assert.ok(snapshot.sessions.every(row=>row.input_tokens===null&&row.price===null));
assert.equal((await runtime.cancel('known-session-64')).accepted,true);
await assert.rejects(runtime.cancel('unknown-session'));
const frozenRequest=Object.freeze({sessionId:'known-session-64',signal:new AbortController().signal});
for await(const chunk of listeners.get('llm/stream')(frozenRequest,async function*(){
yield {type:'finish',reason:{kind:'error',failure:{code:'PRIVATE_VERIFICATION_SECRET',message:'Bearer privateVerificationSecret'}}};
})){}
assert.ok(!JSON.stringify(runtime.snapshot()).includes('VERIFICATION_SECRET'));
await assert.rejects(async()=>{
for await(const chunk of listeners.get('llm/stream')(frozenRequest,()=>{throw Error('privateVerificationSecret');})){}
});
assert.equal(runtime.snapshot().sessions[0].status,'error');
await Promise.all([runtime.configure({response_timeout_seconds:120}),runtime.configure({model_context_limit:8000})]);
assert.deepEqual(runtime.snapshot().settings,{response_timeout_seconds:120,model_context_limit:8000});
await runtime.configure({response_timeout_seconds:90,model_context_limit:16000});
runtime.dispose();assert.equal(listeners.size,0);
}
if(process.argv.includes('--unit-only')){console.log('usage lifecycle boundary checks passed');process.exit(0);}
await mkdir(path.dirname(reportPath),{recursive:true});
await writeFile(reportPath,JSON.stringify({status:'running',nativeHarness:true,externalModel:false},null,2));
console.error('usage verification: boot');
const {ctx,shutdown}=await bootRuntime({announce:false});
console.error('usage verification: routes');
let usage;
try{
const base=`http://127.0.0.1:${ctx.webServer.port}`;
if((await fetch(base+'/dx/runtime')).status===404){
usage=await registerRuntimeUsage(ctx,{admit(req,res){const result=ctx.connection.admit(req);
if('rejection'in result){res.writeHead(result.rejection);res.end();return false;}return true;}});
}
assert.equal((await fetch(base+'/dx/runtime')).status,401);
const login=await fetch(ctx.connection.authenticatedUrl(base+'/'),{redirect:'manual'});
const cookie=login.headers.getSetCookie().map(row=>row.split(';')[0]).join('; ');
const headers={Cookie:cookie,'Content-Type':'application/json'};
const get=async()=>{const response=await fetch(base+'/dx/runtime',{headers});assert.equal(response.status,200);return response.json();};
const control=body=>fetch(base+'/dx/runtime/control',{method:'POST',headers,body:JSON.stringify(body)});
assert.equal((await get()).settings.response_timeout_seconds,90);
for(const body of [{action:'configure',response_timeout_seconds:14},
{action:'configure',response_timeout_seconds:true},{action:'configure',model_context_limit:128001},
{action:'configure',model_context_limit:3000,secret:'rejected'},
{action:'cancel',session_id:'not-loaded'},{action:'stop_task',session_id:'not-loaded'}]){
assert.ok((await control(body)).status>=400);
}
assert.equal((await control({action:'configure',response_timeout_seconds:15,model_context_limit:4000})).status,200);
assert.deepEqual(JSON.parse(await readFile(path.join(output,'assistant-settings.json'),'utf8')),
{response_timeout_seconds:15,model_context_limit:4000});
const require=createRequire(path.join(repo,'integration/dsh/package.json'));
const {LlmAdapter}=await import(pathToFileURL(require.resolve('@deepseek-ai/dsh-llm')).href);
let mode='usage',entered,compactionRequests=0;
let enterResolve;
const waiting=()=>{entered=new Promise(resolve=>{enterResolve=resolve;});};
class Controlled extends LlmAdapter{
providerInfo(provider){return {id:provider,name:'Controlled verification'};}
async listModels(provider){return [{provider,id:'fixture',name:'Fixture'}];}
async *stream(options){
enterResolve?.();
if(options.purpose==='compaction')compactionRequests++;
if(mode==='hang'){
await new Promise((resolve,reject)=>{
options.signal?.addEventListener('abort',()=>reject(options.signal.reason),{once:true});
});
return;
}
if(mode==='error'){
yield {type:'finish',reason:{kind:'error',failure:{code:'PRIVATE_VERIFICATION_SECRET',message:'Bearer privateVerificationSecret'}}};return;
}
yield {type:'block-start',index:0,blockType:'text'};
yield {type:'text-delta',index:0,text:'已完成核对。'};
yield {type:'block-end',index:0,block:{type:'text',text:'已完成核对。'}};
if(mode==='usage'){
yield {type:'usage',usage:{inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24}};
// Providers may emit cumulative usage more than once; charge this response once.
yield {type:'usage',usage:{inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24}};
}
yield {type:'finish',reason:{kind:'stop'}};
}
}
const unregister=ctx.llm.registerAdapter(['usage-fixture'],new Controlled());
await mkdir(path.join(output,'workspace'),{recursive:true});
const workspace=await ctx.workspaceRegistry.create(path.join(output,'workspace'),'Usage verification');
const {sessionId}=await ctx.sessionController.create({workspaceId:workspace.id,agentPreset:'dx-task'});
await ctx.sessionController.selectModel({sessionId,provider:'usage-fixture',model:'fixture'});
const agent=ctx.agents.get(sessionId);
const prompt=async(text='控制适配器核验,无外部请求。')=>{
await ctx.sessionController.commands.prompt({sessionId,requestId:crypto.randomUUID(),mode:'queue',
content:[{type:'text',text}]}, {kind:'user',rpcId:crypto.randomUUID()});
};
console.error('usage verification: first turn');
await prompt();await agent.whenIdle();
let row=(await get()).sessions.find(row=>row.session_id===sessionId);
assert.equal(row.input_tokens,12);assert.equal(row.cached_tokens,8);
assert.equal(row.output_tokens,4);assert.equal(row.total_tokens,24);assert.equal(row.price,null);
assert.ok(row.elapsed_seconds>=0);
assert.equal(row.status,'completed');
// Time spent after the model finishes (including confirmed task tools) has no response timer.
mock.timers.enable({apis:['setTimeout','Date']});
mock.timers.tick(20000);
mock.timers.reset();
assert.equal((await get()).sessions.find(row=>row.session_id===sessionId).status,'completed');
mode='error';await prompt();await agent.whenIdle();
row=(await get()).sessions.find(row=>row.session_id===sessionId);
assert.equal(row.status,'error');assert.ok(!JSON.stringify(row).includes('privateVerificationSecret'));
assert.ok(!JSON.stringify(row).includes('PRIVATE_VERIFICATION_SECRET'));
mode='hang';waiting();await prompt();await entered;
assert.equal((await control({action:'cancel',session_id:sessionId})).status,200);
await agent.whenIdle();
assert.equal((await get()).sessions.find(row=>row.session_id===sessionId).status,'cancelled');
waiting();
mock.timers.enable({apis:['setTimeout','Date']});
await prompt();await entered;
mock.timers.tick(15001);
await agent.whenIdle();
mock.timers.reset();
row=(await get()).sessions.find(row=>row.session_id===sessionId);
assert.equal(row.status,'timeout');
assert.equal(row.input_tokens,12); // unknown usage is never invented as zero or doubled
assert.equal(row.total_tokens,24);
mode='unknown';
for(let i=0;i<12;i++){await prompt('历史核验数据。'.repeat(130));await agent.whenIdle();}
const compactionMode=(await get()).context_limit_mode;
if(compactionMode==='native_compaction_threshold'){
await prompt();await agent.whenIdle();
row=(await get()).sessions.find(row=>row.session_id===sessionId);
assert.ok(compactionRequests>0,'configured native context threshold must request compaction: '+JSON.stringify({
state:row.context_limit_state,tokens:ctx.get('tokenMeter')?.measure(agent.session)?.totalTokens}));
assert.ok(row.context_compactions>0,'native compaction must complete');
assert.ok(agent.session.snapshotEvents().some(event=>event.type==='compaction/end'));
}
// Recreate the observer against actual native persisted event schema without
// prompting again; this is the same initialization used when an Agent reopens.
const rehydrated=await registerRuntimeUsage({agents:ctx.agents,sessionController:ctx.sessionController,
get:name=>ctx.get(name),on(){return()=>{};},webServer:{register(){return()=>{};}}},{admit:()=>true});
const original=(await get()).sessions.find(row=>row.session_id===sessionId);
const recovered=rehydrated.snapshot().sessions.find(row=>row.session_id===sessionId);
for(const field of ['input_tokens','output_tokens','cached_tokens','total_tokens'])assert.equal(recovered[field],original[field],field+' must survive observer reload');
assert.equal(recovered.elapsed_seconds,null);
rehydrated.dispose();
unregister();
const report={status:'passed',nativeHarness:true,controlledAdapter:true,externalModel:false,
authenticatedRoutes:true,providerUsage:true,duplicateUsageNotDoubleCounted:true,
unknownPriceUnavailable:true,nativeCancel:true,perStreamTimeout:true,settingsDurable:true,
context_limit_mode:compactionMode,nativeContextCompaction:compactionRequests>0,nativeHistoryUsageRestored:true};
await mkdir(path.join(repo,'build/dsh-native-usage-verification'),{recursive:true});
await writeFile(reportPath,JSON.stringify(report,null,2));
console.log(JSON.stringify(report));
}finally{mock.timers.reset();usage?.dispose();await shutdown.shutdown(0);}