Core regression / Python core and release configuration (push) Successful in 18s
Core regression / JavaScript syntax (push) Successful in 3s
Core regression / Python core and release configuration (pull_request) Successful in 15s
Core regression / JavaScript syntax (pull_request) Successful in 3s
201 lines
13 KiB
JavaScript
201 lines
13 KiB
JavaScript
/** Real Harness requests must keep each provider endpoint, model and credential together. */
|
|
import assert from 'node:assert/strict';
|
|
import {createServer} from 'node:http';
|
|
import {spawn} from 'node:child_process';
|
|
import {mkdir,mkdtemp,readFile,writeFile,readdir} from 'node:fs/promises';
|
|
import {createRequire} from 'node:module';
|
|
import path from 'node:path';
|
|
import {fileURLToPath} from 'node:url';
|
|
import {bootRuntime,overlay,prepareProfileDirectory} from '../integration/dsh/start.mjs';
|
|
|
|
const script=fileURLToPath(import.meta.url);
|
|
const require=createRequire(new URL('../integration/dsh/start.mjs',import.meta.url));
|
|
const {load}=require('js-yaml');
|
|
const appKeys=['DSH_API_BASE','DSH_MODEL','DSH_API_KEY','DX_DSH_MODELS','DSH_HOME','DX_DSH_HOME','DX_DSH_CONTEXT_FILE','DX_DSH_PARENT_PIPE'];
|
|
const selection=ctx=>ctx.agentDefaultModel.currentSelection();
|
|
|
|
async function runChild(specPath,env){
|
|
return new Promise(resolve=>{
|
|
const child=spawn(process.execPath,[script,'--child',specPath],{env,stdio:['ignore','pipe','pipe'],windowsHide:true});
|
|
let stdout='',stderr='';child.stdout.on('data',x=>stdout+=x);child.stderr.on('data',x=>stderr+=x);
|
|
const timeout=setTimeout(()=>child.kill(),60000);
|
|
child.on('close',code=>{clearTimeout(timeout);resolve({code,stdout,stderr});});
|
|
});
|
|
}
|
|
|
|
async function stream(ctx,provider,model){
|
|
const chunks=[];
|
|
for await(const chunk of ctx.llm.stream({provider,model,messages:[{role:'user',content:[{type:'text',text:'fixture request'}]}],
|
|
maxTokens:8,signal:AbortSignal.timeout(15000)}))chunks.push(chunk);
|
|
assert.equal(chunks.find(x=>x.type==='finish')?.reason.kind,'stop','owned fixture stream must finish successfully');
|
|
assert.equal(chunks.filter(x=>x.type==='text-delta').map(x=>x.text).join(''),'ok');
|
|
}
|
|
|
|
if(process.argv[2]==='--child'){
|
|
const spec=JSON.parse(await readFile(process.argv[3],'utf8'));
|
|
if(spec.orphanKey){
|
|
await assert.rejects(bootRuntime({announce:false}),/Stored DSH_API_KEY has no provider binding|Duplicate .* prevents verifying model binding|Application model route did not match/);
|
|
console.log('DX_SWITCH_REPORT '+JSON.stringify({status:'passed',unboundCredentialRefused:true}));
|
|
process.exit(0);
|
|
}
|
|
const {ctx,shutdown}=await bootRuntime({announce:false});
|
|
try{
|
|
if(spec.hostAbsent){
|
|
assert.ok(!ctx.llm.listProviders().some(x=>x.id==='dx'));
|
|
await stream(ctx,'custom','custom-model');
|
|
console.log('DX_SWITCH_REPORT '+JSON.stringify({status:'passed',customWorksWithoutHost:true}));
|
|
}else{
|
|
// Send even when the model catalogue is stale: detect the actual HTTP credential destination.
|
|
const dxModels=(await ctx.llm.listModels('dx')).map(x=>x.id);
|
|
await stream(ctx,'dx',dxModels[0]);
|
|
if(spec.custom){
|
|
assert.deepEqual((await ctx.llm.listModels('custom')).map(x=>x.id),['custom-model']);
|
|
await stream(ctx,'custom','custom-model');
|
|
}
|
|
assert.deepEqual(dxModels,spec.customDx?['custom-dx-model']:['new-model','alternate-model']);
|
|
assert.deepEqual(selection(ctx),spec.expectedDefault);
|
|
console.log('DX_SWITCH_REPORT '+JSON.stringify({status:'passed',dxModels,selection:selection(ctx)}));
|
|
}
|
|
}finally{await shutdown.shutdown(0);}
|
|
}else{
|
|
const output=path.resolve('build/dsh-provider-switch-verification');await mkdir(output,{recursive:true});
|
|
const base=await mkdtemp(path.join(output,'run-'));
|
|
const requests=[];
|
|
async function endpoint(name){
|
|
const server=createServer(async(req,res)=>{
|
|
let text='';for await(const chunk of req)text+=chunk;
|
|
const body=text?JSON.parse(text):{};
|
|
requests.push({server:name,path:req.url,authorization:req.headers.authorization,model:body.model});
|
|
res.writeHead(200,{'Content-Type':'text/event-stream'});
|
|
res.end('data: '+JSON.stringify({id:'fixture',object:'chat.completion.chunk',created:1,model:body.model,
|
|
choices:[{index:0,delta:{role:'assistant',content:'ok'},finish_reason:null}]})+'\n\n'+
|
|
'data: '+JSON.stringify({id:'fixture',object:'chat.completion.chunk',created:1,model:body.model,
|
|
choices:[{index:0,delta:{},finish_reason:'stop'}]})+'\n\ndata: [DONE]\n\n');
|
|
});
|
|
await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));
|
|
return {server,url:`http://127.0.0.1:${server.address().port}/v1`};
|
|
}
|
|
const old=await endpoint('old'),next=await endpoint('new'),custom=await endpoint('custom');
|
|
const cleanEnv={...process.env};for(const key of appKeys)delete cleanEnv[key];
|
|
const nextEnv={DSH_API_BASE:next.url,DSH_MODEL:'new-model',DSH_API_KEY:'fixture-new-key',DX_DSH_MODELS:'["new-model","alternate-model"]'};
|
|
const cases=[
|
|
{name:'legacy-switch',legacy:true},
|
|
{name:'legacy-custom-default',legacy:true,custom:true,expectedDefault:{provider:'custom',model:'custom-model'}},
|
|
{name:'already-migrated-switch',custom:true},
|
|
{name:'native-custom-provider',custom:true,omitDx:true},
|
|
{name:'native-dx-selection',expectedDefault:{provider:'dx',model:'alternate-model'},selected:'alternate-model'},
|
|
{name:'native-independent-dx',customDx:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
|
|
{name:'native-stored-dx-key',customDx:true,storedHostKey:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
|
|
{name:'legacy-stored-dx-key',legacy:true,customDx:true,storedHostKey:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
|
|
{name:'native-key-only-edit',customDx:true,storedHostKey:true,keyOnly:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
|
|
{name:'unbound-stored-key',storedHostKey:true,orphanKey:true},
|
|
{name:'ignored-name-binding',storedHostKey:true,orphanKey:true,ignoredName:true},
|
|
{name:'legacy-ignored-name-binding',legacy:true,storedHostKey:true,orphanKey:true,ignoredName:true},
|
|
{name:'insert-is-not-a-binding',storedHostKey:true,orphanKey:true,insertOnly:true},
|
|
{name:'alternate-unbound-key',alternateCredentials:true,storedHostKey:true,orphanKey:true},
|
|
{name:'alternate-bound-key',alternateCredentials:true,customDx:true,storedHostKey:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
|
|
{name:'duplicate-credentials',storedHostKey:true,orphanKey:true,duplicate:'credentials'},
|
|
{name:'duplicate-model-adapter',orphanKey:true,duplicate:'llm-pi-ai'},
|
|
];
|
|
const reports=[];
|
|
try{
|
|
for(const spec of cases){
|
|
if(process.argv[2]==='--case'&&process.argv[3]!==spec.name)continue;
|
|
const home=path.join(base,spec.name),dir=path.join(home,'profiles/dx-task');await mkdir(dir,{recursive:true});
|
|
const credentialsFile=path.join(home,spec.alternateCredentials?'alternate-credentials.yaml':'.credentials.yaml');
|
|
Object.assign(process.env,{DSH_API_BASE:old.url,DSH_MODEL:'old-model',DSH_API_KEY:'fixture-old-key',DX_DSH_MODELS:'["old-model"]'});
|
|
let rows=overlay();
|
|
if(!spec.legacy)rows=rows.filter(x=>['llm-pi-ai','agent-default-model'].includes(x.id));
|
|
const llm=rows.find(x=>x.id==='llm-pi-ai');
|
|
if(spec.omitDx)delete llm.config.providers.dx;
|
|
if(spec.custom)llm.config.providers.custom={api:'openai-completions',baseURL:custom.url,apiKeyEnv:'FIXTURE_CUSTOM_KEY',models:[{id:'custom-model'}]};
|
|
if(spec.customDx)llm.config.providers.dx={api:'openai-completions',baseURL:custom.url,
|
|
apiKeyEnv:spec.storedHostKey?'DSH_API_KEY':'FIXTURE_CUSTOM_KEY',models:[{id:'custom-dx-model'}]};
|
|
if(spec.keyOnly){
|
|
rows=[];
|
|
Object.assign(process.env,{DSH_API_BASE:custom.url,DSH_MODEL:'custom-dx-model',DX_DSH_MODELS:'["custom-dx-model"]'});
|
|
}
|
|
if(spec.orphanKey&&!spec.ignoredName&&!spec.insertOnly&&!spec.duplicate)rows=[];
|
|
if(spec.ignoredName)llm.name='@deepseek-ai/previous-llm-name';
|
|
if(spec.insertOnly)llm.insert=[];
|
|
if(spec.alternateCredentials)rows.push({id:'credentials',config:{path:credentialsFile}});
|
|
if(spec.duplicate)rows.push({insert:[{id:spec.duplicate,
|
|
name:spec.duplicate==='credentials'?'@deepseek-ai/dsh-credentials-local':'@deepseek-ai/dsh-llm-pi-ai',
|
|
config:spec.duplicate==='credentials'?{path:path.join(home,'empty-store.yaml')}:structuredClone(llm.config)}]});
|
|
const defaultRow=rows.find(x=>x.id==='agent-default-model');
|
|
if(defaultRow)defaultRow.config=spec.expectedDefault??{provider:'dx',model:spec.selected??'old-model'};
|
|
const patch=path.join(dir,'cordis.patch.yml');
|
|
if(spec.keyOnly){
|
|
await writeFile(patch,'[]');await prepareProfileDirectory(dir);
|
|
rows=load(await readFile(patch,'utf8'));
|
|
assert.equal(rows.find(x=>x.id==='llm-pi-ai').config.providers.dx.baseURL,custom.url,
|
|
'host route must be materialized before the official UI can save a key-only edit');
|
|
}
|
|
const original=JSON.stringify(rows,null,2);
|
|
await writeFile(patch,original);
|
|
const refs={FIXTURE_CUSTOM_KEY:'fixture-custom-key',...spec.storedHostKey?{DSH_API_KEY:'fixture-custom-key'}:{}};
|
|
const credentials=JSON.stringify({version:1,refs});
|
|
await writeFile(credentialsFile,credentials,{mode:0o600});
|
|
const context=path.join(home,'context.json');await writeFile(context,'{"status":"observe","revision":"switch-fixture"}');
|
|
spec.expectedDefault??={provider:'dx',model:'new-model'};
|
|
const specPath=path.join(home,'spec.json');await writeFile(specPath,JSON.stringify(spec));
|
|
const env={...cleanEnv,...nextEnv,DX_DSH_HOME:home,DX_DSH_CONTEXT_FILE:context};
|
|
const firstRequest=requests.length;
|
|
const result=await runChild(specPath,env);
|
|
const calls=requests.slice(firstRequest);
|
|
if(spec.orphanKey){
|
|
assert.deepEqual(calls,[],'unbound stored key must not make requests');
|
|
assert.equal(result.code,0,spec.name+': '+result.stderr);
|
|
assert.equal(await readFile(patch,'utf8'),original,'refused profile must stay unchanged');
|
|
reports.push({name:spec.name,status:'passed',unboundCredentialRefused:true});
|
|
continue;
|
|
}
|
|
assert.equal(calls.length,spec.custom?2:1,spec.name+': request count; '+result.stderr);
|
|
const dx=calls[0];
|
|
assert.equal(dx.server,spec.customDx?'custom':'new',spec.name+': wrong server received the current key');
|
|
assert.equal(dx.authorization,spec.customDx?'Bearer fixture-custom-key':'Bearer fixture-new-key',spec.name+': credential source');
|
|
assert.equal(dx.model,spec.customDx?'custom-dx-model':'new-model');
|
|
assert.equal(dx.path,'/v1/chat/completions');
|
|
if(spec.custom)assert.deepEqual(calls[1],{server:'custom',path:'/v1/chat/completions',authorization:'Bearer fixture-custom-key',model:'custom-model'});
|
|
assert.equal(result.code,0,spec.name+': '+result.stderr+'\n'+result.stdout);
|
|
// Harness itself adds a browser-session record; model references must survive unchanged.
|
|
assert.deepEqual(load(await readFile(credentialsFile,'utf8')).refs,
|
|
refs,'stored provider credentials must be untouched');
|
|
Object.assign(process.env,nextEnv);
|
|
const migrated=await readFile(patch,'utf8');
|
|
const credentialsAfterBoot=await readFile(credentialsFile,'utf8');
|
|
await prepareProfileDirectory(dir);
|
|
assert.equal(await readFile(credentialsFile,'utf8'),credentialsAfterBoot,'profile preparation must not rewrite credentials');
|
|
assert.equal(await readFile(patch,'utf8'),migrated,'second boot must not rewrite the same profile');
|
|
if(spec.legacy){
|
|
const backup=(await readdir(dir)).find(x=>x.includes('.legacy-'));
|
|
assert.equal(await readFile(path.join(dir,backup),'utf8'),original);
|
|
}
|
|
reports.push({name:spec.name,status:'passed',calls:calls.map(({authorization,...rest})=>({...rest,credentialMatched:true}))});
|
|
}
|
|
// Removing host configuration must not leave a stale application route backed by a new key later.
|
|
if(process.argv[2]!=='--case'){
|
|
const dir=path.join(base,'already-migrated-switch/profiles/dx-task');
|
|
for(const key of ['DSH_API_BASE','DSH_API_KEY','DSH_MODEL','DX_DSH_MODELS'])delete process.env[key];
|
|
await prepareProfileDirectory(dir);
|
|
const cleared=load(await readFile(path.join(dir,'cordis.patch.yml'),'utf8'));
|
|
assert.equal(cleared.find(x=>x.id==='llm-pi-ai')?.config.providers.dx,undefined);
|
|
assert.ok(cleared.find(x=>x.id==='llm-pi-ai').config.providers.custom);
|
|
assert.equal(cleared.find(x=>x.id==='agent-default-model'),undefined);
|
|
const home=path.resolve(dir,'../..'),credentialsFile=path.join(home,'.credentials.yaml');
|
|
const stored=load(await readFile(credentialsFile,'utf8'));stored.refs.DSH_API_KEY='fixture-orphan-key';
|
|
await writeFile(credentialsFile,JSON.stringify(stored));
|
|
const specPath=path.join(home,'no-host-spec.json');await writeFile(specPath,'{"hostAbsent":true}');
|
|
const count=requests.length;
|
|
const result=await runChild(specPath,{...cleanEnv,DX_DSH_HOME:home,DX_DSH_CONTEXT_FILE:path.join(home,'context.json')});
|
|
assert.equal(result.code,0,'independent provider must work without host configuration: '+result.stderr);
|
|
assert.deepEqual(requests.slice(count),[{server:'custom',path:'/v1/chat/completions',authorization:'Bearer fixture-custom-key',model:'custom-model'}]);
|
|
reports.push({name:'no-host-with-unused-stored-key',status:'passed',calls:[{server:'custom',path:'/v1/chat/completions',model:'custom-model',credentialMatched:true}]});
|
|
}
|
|
const report={status:'passed',realHarness:true,externalModelCalls:false,cases:reports,hostRemoval:process.argv[2]!=='--case'};
|
|
await writeFile(path.join(base,'report.json'),JSON.stringify(report,null,2));console.log(JSON.stringify(report));
|
|
}finally{
|
|
for(const {server} of [old,next,custom]){server.closeAllConnections();await new Promise(resolve=>server.close(resolve));}
|
|
}
|
|
}
|