/** Verify runtime accounting and controls against the installed native Harness. */ import assert from 'node:assert/strict'; import {mkdtemp,readFile,writeFile,mkdir} from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import {createRequire} from 'node:module'; import {pathToFileURL,fileURLToPath} from 'node:url'; import {mock} from 'node:test'; import {registerRuntimeUsage} from '../integration/dsh/task-plugin/runtime_usage.mjs'; import {bootRuntime} from '../integration/dsh/start.mjs'; const repo=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..'); const output=await mkdtemp(path.join(os.tmpdir(),'dsh-runtime-usage-')); const reportPath=path.join(repo,'build/dsh-native-usage-verification/report.json'); process.env.DX_DSH_HOME=path.join(output,'home'); process.env.DX_DSH_CONTEXT_FILE=path.join(output,'context.json'); await writeFile(process.env.DX_DSH_CONTEXT_FILE,JSON.stringify({task_id:'usage-fixture',revision:'usage-r1'})); // The small lifecycle fixture checks bounded retention and unknown values. All // provider accounting, authentication and cancellation below use the native CLI. { const listeners=new Map(),agents=new Map(); const fixture={on(name,fn){listeners.set(name,fn);return()=>listeners.delete(name);},get(){return undefined;}, agents:{get:id=>agents.get(id),list:()=>[...agents.values()]}, webServer:{register(){return()=>{};}},sessionController:{cancel:()=>({accepted:true})}}; const historicalUsage={inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24}; const historicalEvents=[{type:'assistant/message',time:1000,data:{usage:historicalUsage,message:{content:[{type:'text',text:'PRIVATE_HISTORY_TEXT'}]}}}, {type:'assistant/message',time:2000,data:{message:{content:[]}}}, {type:'compaction/summary',time:3000,data:{llmStreamCall:true,usage:historicalUsage}}, {type:'assistant/attempt',time:4000,data:{stream:[{type:'chunk',time:3999,chunk:{type:'usage',usage:historicalUsage}}]}}]; const restored={id:'restored-session',session:{snapshotEvents:()=>historicalEvents}}; agents.set(restored.id,restored); const runtime=await registerRuntimeUsage(fixture,{admit:()=>true}); assert.deepEqual(runtime.snapshot().settings,{response_timeout_seconds:0,model_context_limit:0}); assert.equal(runtime.snapshot().context_limit_mode,'native_model'); // Deep reasoning can exceed 90 seconds without the observer cancelling it. const longRequest=Object.freeze({sessionId:restored.id,signal:new AbortController().signal}); mock.timers.enable({apis:['setTimeout','Date']}); let finished=false; const longResponse=(async()=>{ for await(const chunk of listeners.get('llm/stream')(longRequest,async function*(){ await new Promise(resolve=>setTimeout(resolve,120000)); yield {type:'finish',reason:{kind:'stop'}}; })){} finished=true; })(); mock.timers.tick(90001);await Promise.resolve(); assert.equal(finished,false); assert.equal(runtime.snapshot().sessions[0].status,'generating'); mock.timers.tick(30000);await longResponse; assert.equal(runtime.snapshot().sessions[0].status,'completed'); mock.timers.reset(); let restoredRow=runtime.snapshot().sessions.find(row=>row.session_id===restored.id); assert.equal(restoredRow.input_tokens,36);assert.equal(restoredRow.output_tokens,12); assert.equal(restoredRow.cached_tokens,24);assert.equal(restoredRow.total_tokens,72); assert.equal(restoredRow.usage_complete,false);assert.equal(restoredRow.response_count,5); assert.equal(restoredRow.elapsed_seconds,null); assert.ok(!JSON.stringify(restoredRow).includes('PRIVATE_HISTORY_TEXT')); // Re-announcement and a fresh response must not recharge historical events. await listeners.get('agent/created')?.({agent:restored}); for await(const chunk of listeners.get('llm/stream')({sessionId:restored.id,signal:new AbortController().signal},async function*(){ yield {type:'usage',usage:historicalUsage};yield {type:'finish',reason:{kind:'stop'}}; })){} restoredRow=runtime.snapshot().sessions.find(row=>row.session_id===restored.id); assert.equal(restoredRow.total_tokens,96); const created={id:'reopened-session',session:{snapshotEvents:()=>historicalEvents}}; agents.set(created.id,created);await listeners.get('agent/created')?.({agent:created}); assert.equal(runtime.snapshot().sessions.find(row=>row.session_id===created.id).total_tokens,72); for(let i=0;i<65;i++){ const agent={id:'known-session-'+i};agents.set(agent.id,agent); await listeners.get('agent/created')?.({agent}); } const snapshot=runtime.snapshot(); assert.equal(snapshot.sessions.length,50); assert.ok(snapshot.sessions.every(row=>row.input_tokens===null&&row.price===null)); assert.equal((await runtime.cancel('known-session-64')).accepted,true); await assert.rejects(runtime.cancel('unknown-session')); const frozenRequest=Object.freeze({sessionId:'known-session-64',signal:new AbortController().signal}); for await(const chunk of listeners.get('llm/stream')(frozenRequest,async function*(){ yield {type:'finish',reason:{kind:'error',failure:{code:'PRIVATE_VERIFICATION_SECRET',message:'Bearer privateVerificationSecret'}}}; })){} assert.ok(!JSON.stringify(runtime.snapshot()).includes('VERIFICATION_SECRET')); await assert.rejects(async()=>{ for await(const chunk of listeners.get('llm/stream')(frozenRequest,()=>{throw Error('privateVerificationSecret');})){} }); assert.equal(runtime.snapshot().sessions[0].status,'error'); await Promise.all([runtime.configure({response_timeout_seconds:120}),runtime.configure({model_context_limit:8000})]); assert.deepEqual(runtime.snapshot().settings,{response_timeout_seconds:120,model_context_limit:8000}); await runtime.configure({response_timeout_seconds:0,model_context_limit:0}); runtime.dispose();assert.equal(listeners.size,0); } if(process.argv.includes('--unit-only')){console.log('usage lifecycle boundary checks passed');process.exit(0);} await mkdir(path.dirname(reportPath),{recursive:true}); await writeFile(reportPath,JSON.stringify({status:'running',nativeHarness:true,externalModel:false},null,2)); console.error('usage verification: boot'); const {ctx,shutdown}=await bootRuntime({announce:false}); console.error('usage verification: routes'); let usage; try{ const base=`http://127.0.0.1:${ctx.webServer.port}`; if((await fetch(base+'/dx/runtime')).status===404){ usage=await registerRuntimeUsage(ctx,{admit(req,res){const result=ctx.connection.admit(req); if('rejection'in result){res.writeHead(result.rejection);res.end();return false;}return true;}}); } assert.equal((await fetch(base+'/dx/runtime')).status,401); const login=await fetch(ctx.connection.authenticatedUrl(base+'/'),{redirect:'manual'}); const cookie=login.headers.getSetCookie().map(row=>row.split(';')[0]).join('; '); const headers={Cookie:cookie,'Content-Type':'application/json'}; const get=async()=>{const response=await fetch(base+'/dx/runtime',{headers});assert.equal(response.status,200);return response.json();}; const control=body=>fetch(base+'/dx/runtime/control',{method:'POST',headers,body:JSON.stringify(body)}); assert.equal((await get()).settings.response_timeout_seconds,0); for(const body of [{action:'configure',response_timeout_seconds:14}, {action:'configure',response_timeout_seconds:true},{action:'configure',model_context_limit:128001}, {action:'configure',model_context_limit:3000,secret:'rejected'}, {action:'cancel',session_id:'not-loaded'},{action:'stop_task',session_id:'not-loaded'}]){ assert.ok((await control(body)).status>=400); } assert.equal((await control({action:'configure',response_timeout_seconds:15,model_context_limit:4000})).status,200); assert.deepEqual(JSON.parse(await readFile(path.join(output,'assistant-settings.json'),'utf8')), {response_timeout_seconds:15,model_context_limit:4000}); const require=createRequire(path.join(repo,'integration/dsh/package.json')); const {LlmAdapter}=await import(pathToFileURL(require.resolve('@deepseek-ai/dsh-llm')).href); let mode='usage',entered,compactionRequests=0; let enterResolve; const waiting=()=>{entered=new Promise(resolve=>{enterResolve=resolve;});}; class Controlled extends LlmAdapter{ providerInfo(provider){return {id:provider,name:'Controlled verification'};} async listModels(provider){return [{provider,id:'fixture',name:'Fixture'}];} async *stream(options){ enterResolve?.(); if(options.purpose==='compaction')compactionRequests++; if(mode==='hang'){ await new Promise((resolve,reject)=>{ options.signal?.addEventListener('abort',()=>reject(options.signal.reason),{once:true}); }); return; } if(mode==='error'){ yield {type:'finish',reason:{kind:'error',failure:{code:'PRIVATE_VERIFICATION_SECRET',message:'Bearer privateVerificationSecret'}}};return; } yield {type:'block-start',index:0,blockType:'text'}; yield {type:'text-delta',index:0,text:'已完成核对。'}; yield {type:'block-end',index:0,block:{type:'text',text:'已完成核对。'}}; if(mode==='usage'){ yield {type:'usage',usage:{inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24}}; // Providers may emit cumulative usage more than once; charge this response once. yield {type:'usage',usage:{inputTokens:12,outputTokens:4,cacheReadTokens:8,totalTokens:24}}; } yield {type:'finish',reason:{kind:'stop'}}; } } const unregister=ctx.llm.registerAdapter(['usage-fixture'],new Controlled()); await mkdir(path.join(output,'workspace'),{recursive:true}); const workspace=await ctx.workspaceRegistry.create(path.join(output,'workspace'),'Usage verification'); const {sessionId}=await ctx.sessionController.create({workspaceId:workspace.id,agentPreset:'dx-task'}); await ctx.sessionController.selectModel({sessionId,provider:'usage-fixture',model:'fixture'}); const agent=ctx.agents.get(sessionId); const prompt=async(text='控制适配器核验,无外部请求。')=>{ await ctx.sessionController.commands.prompt({sessionId,requestId:crypto.randomUUID(),mode:'queue', content:[{type:'text',text}]}, {kind:'user',rpcId:crypto.randomUUID()}); }; console.error('usage verification: first turn'); await prompt();await agent.whenIdle(); let row=(await get()).sessions.find(row=>row.session_id===sessionId); assert.equal(row.input_tokens,12);assert.equal(row.cached_tokens,8); assert.equal(row.output_tokens,4);assert.equal(row.total_tokens,24);assert.equal(row.price,null); assert.ok(row.elapsed_seconds>=0); assert.equal(row.status,'completed'); // Time spent after the model finishes (including confirmed task tools) has no response timer. mock.timers.enable({apis:['setTimeout','Date']}); mock.timers.tick(20000); mock.timers.reset(); assert.equal((await get()).sessions.find(row=>row.session_id===sessionId).status,'completed'); mode='error';await prompt();await agent.whenIdle(); row=(await get()).sessions.find(row=>row.session_id===sessionId); assert.equal(row.status,'error');assert.ok(!JSON.stringify(row).includes('privateVerificationSecret')); assert.ok(!JSON.stringify(row).includes('PRIVATE_VERIFICATION_SECRET')); mode='hang';waiting();await prompt();await entered; assert.equal((await control({action:'cancel',session_id:sessionId})).status,200); await agent.whenIdle(); assert.equal((await get()).sessions.find(row=>row.session_id===sessionId).status,'cancelled'); waiting(); mock.timers.enable({apis:['setTimeout','Date']}); await prompt();await entered; mock.timers.tick(15001); await agent.whenIdle(); mock.timers.reset(); row=(await get()).sessions.find(row=>row.session_id===sessionId); assert.equal(row.status,'timeout'); assert.equal(row.input_tokens,12); // unknown usage is never invented as zero or doubled assert.equal(row.total_tokens,24); mode='unknown'; for(let i=0;i<12;i++){await prompt('历史核验数据。'.repeat(130));await agent.whenIdle();} const compactionMode=(await get()).context_limit_mode; if(compactionMode==='native_compaction_threshold'){ await prompt();await agent.whenIdle(); row=(await get()).sessions.find(row=>row.session_id===sessionId); assert.ok(compactionRequests>0,'configured native context threshold must request compaction: '+JSON.stringify({ state:row.context_limit_state,tokens:ctx.get('tokenMeter')?.measure(agent.session)?.totalTokens})); assert.ok(row.context_compactions>0,'native compaction must complete'); assert.ok(agent.session.snapshotEvents().some(event=>event.type==='compaction/end')); } // Recreate the observer against actual native persisted event schema without // prompting again; this is the same initialization used when an Agent reopens. const rehydrated=await registerRuntimeUsage({agents:ctx.agents,sessionController:ctx.sessionController, get:name=>ctx.get(name),on(){return()=>{};},webServer:{register(){return()=>{};}}},{admit:()=>true}); const original=(await get()).sessions.find(row=>row.session_id===sessionId); const recovered=rehydrated.snapshot().sessions.find(row=>row.session_id===sessionId); for(const field of ['input_tokens','output_tokens','cached_tokens','total_tokens'])assert.equal(recovered[field],original[field],field+' must survive observer reload'); assert.equal(recovered.elapsed_seconds,null); rehydrated.dispose(); unregister(); const report={status:'passed',nativeHarness:true,controlledAdapter:true,externalModel:false, authenticatedRoutes:true,providerUsage:true,duplicateUsageNotDoubleCounted:true, unknownPriceUnavailable:true,nativeCancel:true,perStreamTimeout:true,settingsDurable:true, context_limit_mode:compactionMode,nativeContextCompaction:compactionRequests>0,nativeHistoryUsageRestored:true}; await mkdir(path.join(repo,'build/dsh-native-usage-verification'),{recursive:true}); await writeFile(reportPath,JSON.stringify(report,null,2)); console.log(JSON.stringify(report)); }finally{mock.timers.reset();usage?.dispose();await shutdown.shutdown(0);}