201 lines
13 KiB
JavaScript
201 lines
13 KiB
JavaScript
/** Verify runtime accounting and controls against the installed native Harness. */
|
|
import assert from 'node:assert/strict';
|
|
import {mkdtemp,readFile,writeFile,mkdir} from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import {createRequire} from 'node:module';
|
|
import {pathToFileURL,fileURLToPath} from 'node:url';
|
|
import {mock} from 'node:test';
|
|
import {registerRuntimeUsage} from '../integration/dsh/task-plugin/runtime_usage.mjs';
|
|
import {bootRuntime} from '../integration/dsh/start.mjs';
|
|
|
|
const repo=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..');
|
|
const output=await mkdtemp(path.join(os.tmpdir(),'dsh-runtime-usage-'));
|
|
const reportPath=path.join(repo,'build/dsh-native-usage-verification/report.json');
|
|
process.env.DX_DSH_HOME=path.join(output,'home');
|
|
process.env.DX_DSH_CONTEXT_FILE=path.join(output,'context.json');
|
|
await writeFile(process.env.DX_DSH_CONTEXT_FILE,JSON.stringify({task_id:'usage-fixture',revision:'usage-r1'}));
|
|
// The small lifecycle fixture checks bounded retention and unknown values. All
|
|
// provider accounting, authentication and cancellation below use the native CLI.
|
|
{
|
|
const listeners=new Map(),agents=new Map();
|
|
const fixture={on(name,fn){listeners.set(name,fn);return()=>listeners.delete(name);},get(){return undefined;},
|
|
agents:{get:id=>agents.get(id),list:()=>[...agents.values()]},
|
|
webServer:{register(){return()=>{};}},sessionController:{cancel:()=>({accepted:true})}};
|
|
const historicalUsage={inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24};
|
|
const historicalEvents=[{type:'assistant/message',time:1000,data:{usage:historicalUsage,message:{content:[{type:'text',text:'PRIVATE_HISTORY_TEXT'}]}}},
|
|
{type:'assistant/message',time:2000,data:{message:{content:[]}}},
|
|
{type:'compaction/summary',time:3000,data:{llmStreamCall:true,usage:historicalUsage}},
|
|
{type:'assistant/attempt',time:4000,data:{stream:[{type:'chunk',time:3999,chunk:{type:'usage',usage:historicalUsage}}]}}];
|
|
const restored={id:'restored-session',session:{snapshotEvents:()=>historicalEvents}};
|
|
agents.set(restored.id,restored);
|
|
const runtime=await registerRuntimeUsage(fixture,{admit:()=>true});
|
|
let restoredRow=runtime.snapshot().sessions.find(row=>row.session_id===restored.id);
|
|
assert.equal(restoredRow.input_tokens,36);assert.equal(restoredRow.output_tokens,12);
|
|
assert.equal(restoredRow.cached_tokens,24);assert.equal(restoredRow.total_tokens,72);
|
|
assert.equal(restoredRow.usage_complete,false);assert.equal(restoredRow.response_count,4);
|
|
assert.equal(restoredRow.elapsed_seconds,null);
|
|
assert.ok(!JSON.stringify(restoredRow).includes('PRIVATE_HISTORY_TEXT'));
|
|
// Re-announcement and a fresh response must not recharge historical events.
|
|
await listeners.get('agent/created')?.({agent:restored});
|
|
for await(const chunk of listeners.get('llm/stream')({sessionId:restored.id,signal:new AbortController().signal},async function*(){
|
|
yield {type:'usage',usage:historicalUsage};yield {type:'finish',reason:{kind:'stop'}};
|
|
})){}
|
|
restoredRow=runtime.snapshot().sessions.find(row=>row.session_id===restored.id);
|
|
assert.equal(restoredRow.total_tokens,96);
|
|
const created={id:'reopened-session',session:{snapshotEvents:()=>historicalEvents}};
|
|
agents.set(created.id,created);await listeners.get('agent/created')?.({agent:created});
|
|
assert.equal(runtime.snapshot().sessions.find(row=>row.session_id===created.id).total_tokens,72);
|
|
for(let i=0;i<65;i++){
|
|
const agent={id:'known-session-'+i};agents.set(agent.id,agent);
|
|
await listeners.get('agent/created')?.({agent});
|
|
}
|
|
const snapshot=runtime.snapshot();
|
|
assert.equal(snapshot.sessions.length,50);
|
|
assert.ok(snapshot.sessions.every(row=>row.input_tokens===null&&row.price===null));
|
|
assert.equal((await runtime.cancel('known-session-64')).accepted,true);
|
|
await assert.rejects(runtime.cancel('unknown-session'));
|
|
const frozenRequest=Object.freeze({sessionId:'known-session-64',signal:new AbortController().signal});
|
|
for await(const chunk of listeners.get('llm/stream')(frozenRequest,async function*(){
|
|
yield {type:'finish',reason:{kind:'error',failure:{code:'PRIVATE_VERIFICATION_SECRET',message:'Bearer privateVerificationSecret'}}};
|
|
})){}
|
|
assert.ok(!JSON.stringify(runtime.snapshot()).includes('VERIFICATION_SECRET'));
|
|
await assert.rejects(async()=>{
|
|
for await(const chunk of listeners.get('llm/stream')(frozenRequest,()=>{throw Error('privateVerificationSecret');})){}
|
|
});
|
|
assert.equal(runtime.snapshot().sessions[0].status,'error');
|
|
await Promise.all([runtime.configure({response_timeout_seconds:120}),runtime.configure({model_context_limit:8000})]);
|
|
assert.deepEqual(runtime.snapshot().settings,{response_timeout_seconds:120,model_context_limit:8000});
|
|
await runtime.configure({response_timeout_seconds:90,model_context_limit:16000});
|
|
runtime.dispose();assert.equal(listeners.size,0);
|
|
}
|
|
if(process.argv.includes('--unit-only')){console.log('usage lifecycle boundary checks passed');process.exit(0);}
|
|
await mkdir(path.dirname(reportPath),{recursive:true});
|
|
await writeFile(reportPath,JSON.stringify({status:'running',nativeHarness:true,externalModel:false},null,2));
|
|
console.error('usage verification: boot');
|
|
const {ctx,shutdown}=await bootRuntime({announce:false});
|
|
console.error('usage verification: routes');
|
|
let usage;
|
|
try{
|
|
const base=`http://127.0.0.1:${ctx.webServer.port}`;
|
|
if((await fetch(base+'/dx/runtime')).status===404){
|
|
usage=await registerRuntimeUsage(ctx,{admit(req,res){const result=ctx.connection.admit(req);
|
|
if('rejection'in result){res.writeHead(result.rejection);res.end();return false;}return true;}});
|
|
}
|
|
assert.equal((await fetch(base+'/dx/runtime')).status,401);
|
|
const login=await fetch(ctx.connection.authenticatedUrl(base+'/'),{redirect:'manual'});
|
|
const cookie=login.headers.getSetCookie().map(row=>row.split(';')[0]).join('; ');
|
|
const headers={Cookie:cookie,'Content-Type':'application/json'};
|
|
const get=async()=>{const response=await fetch(base+'/dx/runtime',{headers});assert.equal(response.status,200);return response.json();};
|
|
const control=body=>fetch(base+'/dx/runtime/control',{method:'POST',headers,body:JSON.stringify(body)});
|
|
assert.equal((await get()).settings.response_timeout_seconds,90);
|
|
for(const body of [{action:'configure',response_timeout_seconds:14},
|
|
{action:'configure',response_timeout_seconds:true},{action:'configure',model_context_limit:128001},
|
|
{action:'configure',model_context_limit:3000,secret:'rejected'},
|
|
{action:'cancel',session_id:'not-loaded'},{action:'stop_task',session_id:'not-loaded'}]){
|
|
assert.ok((await control(body)).status>=400);
|
|
}
|
|
assert.equal((await control({action:'configure',response_timeout_seconds:15,model_context_limit:4000})).status,200);
|
|
assert.deepEqual(JSON.parse(await readFile(path.join(output,'assistant-settings.json'),'utf8')),
|
|
{response_timeout_seconds:15,model_context_limit:4000});
|
|
const require=createRequire(path.join(repo,'integration/dsh/package.json'));
|
|
const {LlmAdapter}=await import(pathToFileURL(require.resolve('@deepseek-ai/dsh-llm')).href);
|
|
let mode='usage',entered,compactionRequests=0;
|
|
let enterResolve;
|
|
const waiting=()=>{entered=new Promise(resolve=>{enterResolve=resolve;});};
|
|
class Controlled extends LlmAdapter{
|
|
providerInfo(provider){return {id:provider,name:'Controlled verification'};}
|
|
async listModels(provider){return [{provider,id:'fixture',name:'Fixture'}];}
|
|
async *stream(options){
|
|
enterResolve?.();
|
|
if(options.purpose==='compaction')compactionRequests++;
|
|
if(mode==='hang'){
|
|
await new Promise((resolve,reject)=>{
|
|
options.signal?.addEventListener('abort',()=>reject(options.signal.reason),{once:true});
|
|
});
|
|
return;
|
|
}
|
|
if(mode==='error'){
|
|
yield {type:'finish',reason:{kind:'error',failure:{code:'PRIVATE_VERIFICATION_SECRET',message:'Bearer privateVerificationSecret'}}};return;
|
|
}
|
|
yield {type:'block-start',index:0,blockType:'text'};
|
|
yield {type:'text-delta',index:0,text:'已完成核对。'};
|
|
yield {type:'block-end',index:0,block:{type:'text',text:'已完成核对。'}};
|
|
if(mode==='usage'){
|
|
yield {type:'usage',usage:{inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24}};
|
|
// Providers may emit cumulative usage more than once; charge this response once.
|
|
yield {type:'usage',usage:{inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24}};
|
|
}
|
|
yield {type:'finish',reason:{kind:'stop'}};
|
|
}
|
|
}
|
|
const unregister=ctx.llm.registerAdapter(['usage-fixture'],new Controlled());
|
|
await mkdir(path.join(output,'workspace'),{recursive:true});
|
|
const workspace=await ctx.workspaceRegistry.create(path.join(output,'workspace'),'Usage verification');
|
|
const {sessionId}=await ctx.sessionController.create({workspaceId:workspace.id,agentPreset:'dx-task'});
|
|
await ctx.sessionController.selectModel({sessionId,provider:'usage-fixture',model:'fixture'});
|
|
const agent=ctx.agents.get(sessionId);
|
|
const prompt=async(text='控制适配器核验,无外部请求。')=>{
|
|
await ctx.sessionController.commands.prompt({sessionId,requestId:crypto.randomUUID(),mode:'queue',
|
|
content:[{type:'text',text}]}, {kind:'user',rpcId:crypto.randomUUID()});
|
|
};
|
|
console.error('usage verification: first turn');
|
|
await prompt();await agent.whenIdle();
|
|
let row=(await get()).sessions.find(row=>row.session_id===sessionId);
|
|
assert.equal(row.input_tokens,12);assert.equal(row.cached_tokens,8);
|
|
assert.equal(row.output_tokens,4);assert.equal(row.total_tokens,24);assert.equal(row.price,null);
|
|
assert.ok(row.elapsed_seconds>=0);
|
|
assert.equal(row.status,'completed');
|
|
// Time spent after the model finishes (including confirmed task tools) has no response timer.
|
|
mock.timers.enable({apis:['setTimeout','Date']});
|
|
mock.timers.tick(20000);
|
|
mock.timers.reset();
|
|
assert.equal((await get()).sessions.find(row=>row.session_id===sessionId).status,'completed');
|
|
mode='error';await prompt();await agent.whenIdle();
|
|
row=(await get()).sessions.find(row=>row.session_id===sessionId);
|
|
assert.equal(row.status,'error');assert.ok(!JSON.stringify(row).includes('privateVerificationSecret'));
|
|
assert.ok(!JSON.stringify(row).includes('PRIVATE_VERIFICATION_SECRET'));
|
|
mode='hang';waiting();await prompt();await entered;
|
|
assert.equal((await control({action:'cancel',session_id:sessionId})).status,200);
|
|
await agent.whenIdle();
|
|
assert.equal((await get()).sessions.find(row=>row.session_id===sessionId).status,'cancelled');
|
|
waiting();
|
|
mock.timers.enable({apis:['setTimeout','Date']});
|
|
await prompt();await entered;
|
|
mock.timers.tick(15001);
|
|
await agent.whenIdle();
|
|
mock.timers.reset();
|
|
row=(await get()).sessions.find(row=>row.session_id===sessionId);
|
|
assert.equal(row.status,'timeout');
|
|
assert.equal(row.input_tokens,12); // unknown usage is never invented as zero or doubled
|
|
assert.equal(row.total_tokens,24);
|
|
mode='unknown';
|
|
for(let i=0;i<12;i++){await prompt('历史核验数据。'.repeat(130));await agent.whenIdle();}
|
|
const compactionMode=(await get()).context_limit_mode;
|
|
if(compactionMode==='native_compaction_threshold'){
|
|
await prompt();await agent.whenIdle();
|
|
row=(await get()).sessions.find(row=>row.session_id===sessionId);
|
|
assert.ok(compactionRequests>0,'configured native context threshold must request compaction: '+JSON.stringify({
|
|
state:row.context_limit_state,tokens:ctx.get('tokenMeter')?.measure(agent.session)?.totalTokens}));
|
|
assert.ok(row.context_compactions>0,'native compaction must complete');
|
|
assert.ok(agent.session.snapshotEvents().some(event=>event.type==='compaction/end'));
|
|
}
|
|
// Recreate the observer against actual native persisted event schema without
|
|
// prompting again; this is the same initialization used when an Agent reopens.
|
|
const rehydrated=await registerRuntimeUsage({agents:ctx.agents,sessionController:ctx.sessionController,
|
|
get:name=>ctx.get(name),on(){return()=>{};},webServer:{register(){return()=>{};}}},{admit:()=>true});
|
|
const original=(await get()).sessions.find(row=>row.session_id===sessionId);
|
|
const recovered=rehydrated.snapshot().sessions.find(row=>row.session_id===sessionId);
|
|
for(const field of ['input_tokens','output_tokens','cached_tokens','total_tokens'])assert.equal(recovered[field],original[field],field+' must survive observer reload');
|
|
assert.equal(recovered.elapsed_seconds,null);
|
|
rehydrated.dispose();
|
|
unregister();
|
|
const report={status:'passed',nativeHarness:true,controlledAdapter:true,externalModel:false,
|
|
authenticatedRoutes:true,providerUsage:true,duplicateUsageNotDoubleCounted:true,
|
|
unknownPriceUnavailable:true,nativeCancel:true,perStreamTimeout:true,settingsDurable:true,
|
|
context_limit_mode:compactionMode,nativeContextCompaction:compactionRequests>0,nativeHistoryUsageRestored:true};
|
|
await mkdir(path.join(repo,'build/dsh-native-usage-verification'),{recursive:true});
|
|
await writeFile(reportPath,JSON.stringify(report,null,2));
|
|
console.log(JSON.stringify(report));
|
|
}finally{mock.timers.reset();usage?.dispose();await shutdown.shutdown(0);}
|