226 lines
12 KiB
JavaScript
226 lines
12 KiB
JavaScript
/** Application-owned, bounded observations around the official model stream. */
|
|
import {mkdir,readFile,writeFile,rename,unlink} from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import {randomUUID} from 'node:crypto';
|
|
|
|
const DEFAULTS=Object.freeze({response_timeout_seconds:90,model_context_limit:16000});
|
|
const MAX_SESSIONS=50;
|
|
const count=value=>Number.isSafeInteger(value)&&value>=0?value:null;
|
|
const SAFE_ERRORS=new Set(['AUTH','RATE_LIMIT','QUOTA_EXCEEDED','ACCOUNT_QUOTA_EXCEEDED','INVALID_CREDENTIAL',
|
|
'NETWORK','NO_ADAPTER','CONTEXT_WINDOW_EXCEEDED','EMPTY_RESPONSE','TIMEOUT','ABORTED']);
|
|
const code=value=>SAFE_ERRORS.has(value)?value:'MODEL_ERROR';
|
|
|
|
function settingsValue(raw,previous=DEFAULTS){
|
|
if(!raw||typeof raw!=='object'||Array.isArray(raw)||Object.keys(raw).some(key=>!Object.hasOwn(DEFAULTS,key)))throw Error('invalid_settings');
|
|
const next={...previous,...raw};
|
|
if(!Number.isInteger(next.response_timeout_seconds)||next.response_timeout_seconds<15||next.response_timeout_seconds>600||
|
|
!Number.isInteger(next.model_context_limit)||next.model_context_limit<2000||next.model_context_limit>128000)throw Error('invalid_settings');
|
|
return next;
|
|
}
|
|
|
|
function addUsage(row,usage){
|
|
if(!usage){row.usage_complete=false;return;}
|
|
const mapping={input_tokens:'inputTokens',output_tokens:'outputTokens',cache_read_tokens:'cacheReadTokens',
|
|
cache_write_tokens:'cacheWriteTokens',total_tokens:'totalTokens'};
|
|
for(const [field,source]of Object.entries(mapping)){
|
|
const value=count(usage[source]);
|
|
if(value!==null)row[field]=(row[field]??0)+value;
|
|
}
|
|
const read=count(usage.cacheReadTokens),write=count(usage.cacheWriteTokens);
|
|
if(read!==null||write!==null)row.cached_tokens=(row.cached_tokens??0)+(read??0)+(write??0);
|
|
if(count(usage.inputTokens)===null||count(usage.outputTokens)===null||count(usage.totalTokens)===null)row.usage_complete=false;
|
|
}
|
|
|
|
function restoreUsage(row,agent){
|
|
const events=agent?.session?.snapshotEvents?.();
|
|
if(!Array.isArray(events))return;
|
|
for(const event of events){
|
|
const data=event.data;
|
|
if(!data||!['assistant/message','assistant/attempt','compaction/summary'].includes(event.type))continue;
|
|
if(event.type==='compaction/summary'&&data.llmStreamCall!==true)continue;
|
|
let usage=data.usage;
|
|
if(!usage&&Array.isArray(data.stream)){
|
|
// Native attempt streams retain raw usage chunks; repeated chunks are
|
|
// cumulative for that call, so only the last one is authoritative.
|
|
usage=data.stream.findLast(record=>record.type==='chunk'&&record.chunk?.type==='usage')?.chunk.usage;
|
|
}
|
|
addUsage(row,usage);row.response_count++;
|
|
if(event.type==='compaction/summary')row.context_compactions++;
|
|
}
|
|
if(row.response_count){
|
|
row.elapsed_seconds=null;row.duration_complete=false;
|
|
row.usage_source='native_session_history';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Required services: llm, agents, sessionController, webServer. Optional native
|
|
* compaction/tokenMeter are resolved dynamically. No task execution API is used.
|
|
*/
|
|
export async function registerRuntimeUsage(ctx,{admit}){
|
|
if(typeof admit!=='function'||!process.env.DX_DSH_HOME)throw Error('runtime_usage_configuration');
|
|
const settingsPath=path.join(path.dirname(process.env.DX_DSH_HOME),'assistant-settings.json');
|
|
let settings={...DEFAULTS};
|
|
try{settings=settingsValue(JSON.parse(await readFile(settingsPath,'utf8')));}catch{}
|
|
const rows=new Map(),active=new Set(),timers=new Map(),releases=[];
|
|
let writeTail=Promise.resolve();
|
|
const service=name=>typeof ctx.get==='function'?ctx.get(name):ctx[name];
|
|
const nativeCompaction=agent=>(agent&&service('agentPresets')?.serviceFor?.(agent,'compaction'))??agent?.ctx?.get?.('compaction')??service('compaction');
|
|
const tokenMeter=()=>service('tokenMeter');
|
|
const on=(name,fn)=>{const release=ctx.on(name,fn);if(typeof release==='function')releases.push(release);};
|
|
const known=id=>typeof id==='string'&&id.length>0&&id.length<=256&&ctx.agents.get(id);
|
|
const entry=id=>{
|
|
let row=rows.get(id);
|
|
if(row)return row;
|
|
if(rows.size>=MAX_SESSIONS){
|
|
const oldest=[...rows.keys()].find(key=>!active.has(key));
|
|
if(oldest===undefined)return null;
|
|
rows.delete(oldest);
|
|
}
|
|
row={session_id:id,status:'idle',elapsed_seconds:0,input_tokens:null,output_tokens:null,
|
|
cached_tokens:null,cache_read_tokens:null,cache_write_tokens:null,total_tokens:null,
|
|
price:null,usage_complete:true,response_count:0,last_started_at:null,last_finished_at:null,
|
|
duration_complete:true,usage_source:'native_model_stream',context_compactions:0,context_limit_state:'not_measured'};
|
|
restoreUsage(row,known(id));
|
|
rows.set(id,row);return row;
|
|
};
|
|
const snapshot=()=>({sessions:[...rows.values()].map(row=>{
|
|
const value={...row};
|
|
if(row._started!==undefined&&value.elapsed_seconds!==null)value.elapsed_seconds+=Math.max(0,(Date.now()-row._started)/1000);
|
|
delete value._started;return value;
|
|
}).reverse().slice(0,MAX_SESSIONS),settings:{...settings},
|
|
context_limit_mode:tokenMeter()&&(nativeCompaction()||ctx.agents.list().some(agent=>nativeCompaction(agent)))?'native_compaction_threshold':'unavailable',
|
|
pricing_status:'unavailable'});
|
|
const configure=raw=>{
|
|
settingsValue(raw,settings);
|
|
const fields={...raw};
|
|
const operation=writeTail.then(async()=>{
|
|
const next=settingsValue(fields,settings);
|
|
const temporary=path.join(path.dirname(settingsPath),'settings-'+randomUUID()+'.tmp');
|
|
await mkdir(path.dirname(settingsPath),{recursive:true});
|
|
try{await writeFile(temporary,JSON.stringify(next,null,2),'utf8');await rename(temporary,settingsPath);settings=next;}
|
|
finally{await unlink(temporary).catch(()=>{});}
|
|
return {...settings};
|
|
});
|
|
writeTail=operation.catch(()=>{});return operation;
|
|
};
|
|
const cancel=async id=>{
|
|
if(!known(id))throw Error('unknown_session');
|
|
const row=entry(id);
|
|
// This cancels the native conversation turn. It never calls task-stop, CAD,
|
|
// or the task-center lifecycle bridge; an admitted task has its own lifetime.
|
|
const result=await ctx.sessionController.cancel({sessionId:id});
|
|
if(row&&row.status!=='timeout')row.status='cancelled';
|
|
return {status:'cancellation_requested',session_id:id,accepted:result?.accepted===true};
|
|
};
|
|
|
|
for(const agent of ctx.agents.list())entry(agent.id);
|
|
on('agent/created',({agent})=>{entry(agent.id);});
|
|
|
|
on('llm/stream',async function*(options,next){
|
|
const id=options.sessionId;
|
|
if(!known(id)){yield* next();return;}
|
|
const row=entry(id);
|
|
if(!row||active.has(id)){yield* next();return;}
|
|
active.add(id);
|
|
const started=Date.now();
|
|
row._started=started;row.last_started_at=new Date(started).toISOString();
|
|
row.status='generating';row.response_count++;delete row.error;
|
|
let timer,latestUsage=null,finish=null,timedOut=false,completed=false;
|
|
let timeoutResolve;
|
|
const timeout=new Promise(resolve=>{timeoutResolve=resolve;});
|
|
timer=setTimeout(()=>{
|
|
timedOut=true;row.status='timeout';row.error='RESPONSE_TIMEOUT';
|
|
// Called only while a provider stream is pending, never during a tool wait.
|
|
Promise.resolve().then(()=>ctx.sessionController.cancel({sessionId:id})).catch(()=>{});
|
|
timeoutResolve({done:false,value:{type:'finish',reason:{kind:'aborted',failure:{
|
|
code:'RESPONSE_TIMEOUT',message:'Model response timeout'}}}});
|
|
},settings.response_timeout_seconds*1000);
|
|
timers.set(id,timer);
|
|
timer.unref?.();
|
|
let iterator;
|
|
try{
|
|
iterator=next()[Symbol.asyncIterator]();
|
|
while(true){
|
|
const result=await Promise.race([iterator.next(),timeout]);
|
|
if(result.done){completed=true;break;}
|
|
const chunk=result.value;
|
|
if(chunk.type==='usage')latestUsage=chunk.usage;
|
|
if(chunk.type==='finish'){
|
|
finish=chunk.reason;
|
|
// Provider completion ends its response timer before any tool starts.
|
|
clearTimeout(timer);
|
|
}
|
|
yield chunk;
|
|
if(timedOut)break;
|
|
}
|
|
if(!timedOut){
|
|
row.status=finish?.kind==='error'?'error':finish?.kind==='aborted'||options.signal?.aborted?'cancelled':
|
|
finish?.kind==='tool-calls'?'waiting_for_tool':finish?'completed':'error';
|
|
if(row.status==='error')row.error=code(finish?.failure?.code);
|
|
}
|
|
}catch(error){
|
|
if(!timedOut){row.status=options.signal?.aborted?'cancelled':'error';row.error=options.signal?.aborted?'ABORTED':code(error?.code);}
|
|
throw error;
|
|
}finally{
|
|
clearTimeout(timer);timers.delete(id);active.delete(id);
|
|
if(!completed){try{Promise.resolve(iterator?.return?.()).catch(()=>{});}catch{}}
|
|
if(row.elapsed_seconds!==null)row.elapsed_seconds+=Math.max(0,(Date.now()-started)/1000);delete row._started;
|
|
row.last_finished_at=new Date().toISOString();
|
|
addUsage(row,latestUsage);
|
|
}
|
|
});
|
|
on('agent/status',({agent,status})=>{
|
|
const row=rows.get(agent.id);
|
|
if(row&&status==='idle'&&row.status==='waiting_for_tool')row.status='completed';
|
|
});
|
|
on('agent/pre-step',async({agent,signal},next)=>{
|
|
const compaction=nativeCompaction(agent),meter=tokenMeter();
|
|
if(compaction&&meter&&!signal.aborted){
|
|
const row=entry(agent.id);
|
|
try{
|
|
const measured=count(meter.measure(agent.session)?.totalTokens);
|
|
if(row)row.context_limit_state=measured===null?'unavailable':measured<settings.model_context_limit?'within_limit':'threshold_reached';
|
|
if(measured!==null&&measured>=settings.model_context_limit){
|
|
// Native selection preserves balanced tool pairs and reconstructable
|
|
// system/identity records. Never truncate the frozen request envelope.
|
|
const result=await compaction.compactIfNeeded(agent,'context-overflow',signal);
|
|
if(row){row.context_compactions+=result?1:0;row.context_limit_state=result?'compacted':'no_safe_range';}
|
|
}
|
|
}catch(error){
|
|
if(signal.aborted)throw error;
|
|
if(row)row.context_limit_state='compaction_unavailable';
|
|
}
|
|
}
|
|
return next();
|
|
});
|
|
|
|
const respond=(res,status,value)=>{res.writeHead(status,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store'});res.end(JSON.stringify(value));};
|
|
const register=route=>{const release=ctx.webServer.register(route);if(typeof release==='function')releases.push(release);};
|
|
register({kind:'exact',path:'/dx/runtime',handler:async(req,res)=>{
|
|
if(!admit(req,res))return;
|
|
if(req.method!=='GET'){respond(res,405,{error:'method_not_allowed'});return;}
|
|
respond(res,200,snapshot());
|
|
}});
|
|
register({kind:'exact',path:'/dx/runtime/control',handler:async(req,res)=>{
|
|
if(!admit(req,res))return;
|
|
if(req.method!=='POST'){respond(res,405,{error:'method_not_allowed'});return;}
|
|
let raw='',size=0;
|
|
for await(const chunk of req){size+=Buffer.byteLength(chunk);if(size>2048){respond(res,413,{error:'request_too_large'});return;}raw+=chunk;}
|
|
try{
|
|
const value=JSON.parse(raw);
|
|
if(!value||typeof value!=='object'||Array.isArray(value))throw Error('invalid_control');
|
|
if(value.action==='configure'){
|
|
const {action,...fields}=value;
|
|
respond(res,200,{status:'configured',settings:await configure(fields)});return;
|
|
}
|
|
if(value.action==='cancel'&&Object.keys(value).every(key=>['action','session_id'].includes(key))){
|
|
respond(res,200,await cancel(value.session_id));return;
|
|
}
|
|
throw Error('invalid_control');
|
|
}catch(error){respond(res,error.message==='unknown_session'?409:400,{error:error.message==='unknown_session'?'unknown_session':'invalid_control'});}
|
|
}});
|
|
const dispose=()=>{for(const timer of timers.values())clearTimeout(timer);timers.clear();for(const release of releases.splice(0).reverse())release();};
|
|
return {snapshot,configure,cancel,dispose};
|
|
}
|