Files
dxdesigner-part-replacement/tools/verify_dsh_provider_switch.mjs
阿白叔 af46ee1617
Core regression / Python core and release configuration (push) Successful in 18s
Core regression / JavaScript syntax (push) Successful in 3s
Core regression / Python core and release configuration (pull_request) Successful in 15s
Core regression / JavaScript syntax (pull_request) Successful in 3s
fix(dsh): bind migrated model routes to credential sources
2026-11-03 17:28:22 +08:00

201 lines
13 KiB
JavaScript

/** Real Harness requests must keep each provider endpoint, model and credential together. */
import assert from 'node:assert/strict';
import {createServer} from 'node:http';
import {spawn} from 'node:child_process';
import {mkdir,mkdtemp,readFile,writeFile,readdir} from 'node:fs/promises';
import {createRequire} from 'node:module';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {bootRuntime,overlay,prepareProfileDirectory} from '../integration/dsh/start.mjs';
const script=fileURLToPath(import.meta.url);
const require=createRequire(new URL('../integration/dsh/start.mjs',import.meta.url));
const {load}=require('js-yaml');
const appKeys=['DSH_API_BASE','DSH_MODEL','DSH_API_KEY','DX_DSH_MODELS','DSH_HOME','DX_DSH_HOME','DX_DSH_CONTEXT_FILE','DX_DSH_PARENT_PIPE'];
const selection=ctx=>ctx.agentDefaultModel.currentSelection();
async function runChild(specPath,env){
return new Promise(resolve=>{
const child=spawn(process.execPath,[script,'--child',specPath],{env,stdio:['ignore','pipe','pipe'],windowsHide:true});
let stdout='',stderr='';child.stdout.on('data',x=>stdout+=x);child.stderr.on('data',x=>stderr+=x);
const timeout=setTimeout(()=>child.kill(),60000);
child.on('close',code=>{clearTimeout(timeout);resolve({code,stdout,stderr});});
});
}
async function stream(ctx,provider,model){
const chunks=[];
for await(const chunk of ctx.llm.stream({provider,model,messages:[{role:'user',content:[{type:'text',text:'fixture request'}]}],
maxTokens:8,signal:AbortSignal.timeout(15000)}))chunks.push(chunk);
assert.equal(chunks.find(x=>x.type==='finish')?.reason.kind,'stop','owned fixture stream must finish successfully');
assert.equal(chunks.filter(x=>x.type==='text-delta').map(x=>x.text).join(''),'ok');
}
if(process.argv[2]==='--child'){
const spec=JSON.parse(await readFile(process.argv[3],'utf8'));
if(spec.orphanKey){
await assert.rejects(bootRuntime({announce:false}),/Stored DSH_API_KEY has no provider binding|Duplicate .* prevents verifying model binding|Application model route did not match/);
console.log('DX_SWITCH_REPORT '+JSON.stringify({status:'passed',unboundCredentialRefused:true}));
process.exit(0);
}
const {ctx,shutdown}=await bootRuntime({announce:false});
try{
if(spec.hostAbsent){
assert.ok(!ctx.llm.listProviders().some(x=>x.id==='dx'));
await stream(ctx,'custom','custom-model');
console.log('DX_SWITCH_REPORT '+JSON.stringify({status:'passed',customWorksWithoutHost:true}));
}else{
// Send even when the model catalogue is stale: detect the actual HTTP credential destination.
const dxModels=(await ctx.llm.listModels('dx')).map(x=>x.id);
await stream(ctx,'dx',dxModels[0]);
if(spec.custom){
assert.deepEqual((await ctx.llm.listModels('custom')).map(x=>x.id),['custom-model']);
await stream(ctx,'custom','custom-model');
}
assert.deepEqual(dxModels,spec.customDx?['custom-dx-model']:['new-model','alternate-model']);
assert.deepEqual(selection(ctx),spec.expectedDefault);
console.log('DX_SWITCH_REPORT '+JSON.stringify({status:'passed',dxModels,selection:selection(ctx)}));
}
}finally{await shutdown.shutdown(0);}
}else{
const output=path.resolve('build/dsh-provider-switch-verification');await mkdir(output,{recursive:true});
const base=await mkdtemp(path.join(output,'run-'));
const requests=[];
async function endpoint(name){
const server=createServer(async(req,res)=>{
let text='';for await(const chunk of req)text+=chunk;
const body=text?JSON.parse(text):{};
requests.push({server:name,path:req.url,authorization:req.headers.authorization,model:body.model});
res.writeHead(200,{'Content-Type':'text/event-stream'});
res.end('data: '+JSON.stringify({id:'fixture',object:'chat.completion.chunk',created:1,model:body.model,
choices:[{index:0,delta:{role:'assistant',content:'ok'},finish_reason:null}]})+'\n\n'+
'data: '+JSON.stringify({id:'fixture',object:'chat.completion.chunk',created:1,model:body.model,
choices:[{index:0,delta:{},finish_reason:'stop'}]})+'\n\ndata: [DONE]\n\n');
});
await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));
return {server,url:`http://127.0.0.1:${server.address().port}/v1`};
}
const old=await endpoint('old'),next=await endpoint('new'),custom=await endpoint('custom');
const cleanEnv={...process.env};for(const key of appKeys)delete cleanEnv[key];
const nextEnv={DSH_API_BASE:next.url,DSH_MODEL:'new-model',DSH_API_KEY:'fixture-new-key',DX_DSH_MODELS:'["new-model","alternate-model"]'};
const cases=[
{name:'legacy-switch',legacy:true},
{name:'legacy-custom-default',legacy:true,custom:true,expectedDefault:{provider:'custom',model:'custom-model'}},
{name:'already-migrated-switch',custom:true},
{name:'native-custom-provider',custom:true,omitDx:true},
{name:'native-dx-selection',expectedDefault:{provider:'dx',model:'alternate-model'},selected:'alternate-model'},
{name:'native-independent-dx',customDx:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
{name:'native-stored-dx-key',customDx:true,storedHostKey:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
{name:'legacy-stored-dx-key',legacy:true,customDx:true,storedHostKey:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
{name:'native-key-only-edit',customDx:true,storedHostKey:true,keyOnly:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
{name:'unbound-stored-key',storedHostKey:true,orphanKey:true},
{name:'ignored-name-binding',storedHostKey:true,orphanKey:true,ignoredName:true},
{name:'legacy-ignored-name-binding',legacy:true,storedHostKey:true,orphanKey:true,ignoredName:true},
{name:'insert-is-not-a-binding',storedHostKey:true,orphanKey:true,insertOnly:true},
{name:'alternate-unbound-key',alternateCredentials:true,storedHostKey:true,orphanKey:true},
{name:'alternate-bound-key',alternateCredentials:true,customDx:true,storedHostKey:true,expectedDefault:{provider:'dx',model:'custom-dx-model'}},
{name:'duplicate-credentials',storedHostKey:true,orphanKey:true,duplicate:'credentials'},
{name:'duplicate-model-adapter',orphanKey:true,duplicate:'llm-pi-ai'},
];
const reports=[];
try{
for(const spec of cases){
if(process.argv[2]==='--case'&&process.argv[3]!==spec.name)continue;
const home=path.join(base,spec.name),dir=path.join(home,'profiles/dx-task');await mkdir(dir,{recursive:true});
const credentialsFile=path.join(home,spec.alternateCredentials?'alternate-credentials.yaml':'.credentials.yaml');
Object.assign(process.env,{DSH_API_BASE:old.url,DSH_MODEL:'old-model',DSH_API_KEY:'fixture-old-key',DX_DSH_MODELS:'["old-model"]'});
let rows=overlay();
if(!spec.legacy)rows=rows.filter(x=>['llm-pi-ai','agent-default-model'].includes(x.id));
const llm=rows.find(x=>x.id==='llm-pi-ai');
if(spec.omitDx)delete llm.config.providers.dx;
if(spec.custom)llm.config.providers.custom={api:'openai-completions',baseURL:custom.url,apiKeyEnv:'FIXTURE_CUSTOM_KEY',models:[{id:'custom-model'}]};
if(spec.customDx)llm.config.providers.dx={api:'openai-completions',baseURL:custom.url,
apiKeyEnv:spec.storedHostKey?'DSH_API_KEY':'FIXTURE_CUSTOM_KEY',models:[{id:'custom-dx-model'}]};
if(spec.keyOnly){
rows=[];
Object.assign(process.env,{DSH_API_BASE:custom.url,DSH_MODEL:'custom-dx-model',DX_DSH_MODELS:'["custom-dx-model"]'});
}
if(spec.orphanKey&&!spec.ignoredName&&!spec.insertOnly&&!spec.duplicate)rows=[];
if(spec.ignoredName)llm.name='@deepseek-ai/previous-llm-name';
if(spec.insertOnly)llm.insert=[];
if(spec.alternateCredentials)rows.push({id:'credentials',config:{path:credentialsFile}});
if(spec.duplicate)rows.push({insert:[{id:spec.duplicate,
name:spec.duplicate==='credentials'?'@deepseek-ai/dsh-credentials-local':'@deepseek-ai/dsh-llm-pi-ai',
config:spec.duplicate==='credentials'?{path:path.join(home,'empty-store.yaml')}:structuredClone(llm.config)}]});
const defaultRow=rows.find(x=>x.id==='agent-default-model');
if(defaultRow)defaultRow.config=spec.expectedDefault??{provider:'dx',model:spec.selected??'old-model'};
const patch=path.join(dir,'cordis.patch.yml');
if(spec.keyOnly){
await writeFile(patch,'[]');await prepareProfileDirectory(dir);
rows=load(await readFile(patch,'utf8'));
assert.equal(rows.find(x=>x.id==='llm-pi-ai').config.providers.dx.baseURL,custom.url,
'host route must be materialized before the official UI can save a key-only edit');
}
const original=JSON.stringify(rows,null,2);
await writeFile(patch,original);
const refs={FIXTURE_CUSTOM_KEY:'fixture-custom-key',...spec.storedHostKey?{DSH_API_KEY:'fixture-custom-key'}:{}};
const credentials=JSON.stringify({version:1,refs});
await writeFile(credentialsFile,credentials,{mode:0o600});
const context=path.join(home,'context.json');await writeFile(context,'{"status":"observe","revision":"switch-fixture"}');
spec.expectedDefault??={provider:'dx',model:'new-model'};
const specPath=path.join(home,'spec.json');await writeFile(specPath,JSON.stringify(spec));
const env={...cleanEnv,...nextEnv,DX_DSH_HOME:home,DX_DSH_CONTEXT_FILE:context};
const firstRequest=requests.length;
const result=await runChild(specPath,env);
const calls=requests.slice(firstRequest);
if(spec.orphanKey){
assert.deepEqual(calls,[],'unbound stored key must not make requests');
assert.equal(result.code,0,spec.name+': '+result.stderr);
assert.equal(await readFile(patch,'utf8'),original,'refused profile must stay unchanged');
reports.push({name:spec.name,status:'passed',unboundCredentialRefused:true});
continue;
}
assert.equal(calls.length,spec.custom?2:1,spec.name+': request count; '+result.stderr);
const dx=calls[0];
assert.equal(dx.server,spec.customDx?'custom':'new',spec.name+': wrong server received the current key');
assert.equal(dx.authorization,spec.customDx?'Bearer fixture-custom-key':'Bearer fixture-new-key',spec.name+': credential source');
assert.equal(dx.model,spec.customDx?'custom-dx-model':'new-model');
assert.equal(dx.path,'/v1/chat/completions');
if(spec.custom)assert.deepEqual(calls[1],{server:'custom',path:'/v1/chat/completions',authorization:'Bearer fixture-custom-key',model:'custom-model'});
assert.equal(result.code,0,spec.name+': '+result.stderr+'\n'+result.stdout);
// Harness itself adds a browser-session record; model references must survive unchanged.
assert.deepEqual(load(await readFile(credentialsFile,'utf8')).refs,
refs,'stored provider credentials must be untouched');
Object.assign(process.env,nextEnv);
const migrated=await readFile(patch,'utf8');
const credentialsAfterBoot=await readFile(credentialsFile,'utf8');
await prepareProfileDirectory(dir);
assert.equal(await readFile(credentialsFile,'utf8'),credentialsAfterBoot,'profile preparation must not rewrite credentials');
assert.equal(await readFile(patch,'utf8'),migrated,'second boot must not rewrite the same profile');
if(spec.legacy){
const backup=(await readdir(dir)).find(x=>x.includes('.legacy-'));
assert.equal(await readFile(path.join(dir,backup),'utf8'),original);
}
reports.push({name:spec.name,status:'passed',calls:calls.map(({authorization,...rest})=>({...rest,credentialMatched:true}))});
}
// Removing host configuration must not leave a stale application route backed by a new key later.
if(process.argv[2]!=='--case'){
const dir=path.join(base,'already-migrated-switch/profiles/dx-task');
for(const key of ['DSH_API_BASE','DSH_API_KEY','DSH_MODEL','DX_DSH_MODELS'])delete process.env[key];
await prepareProfileDirectory(dir);
const cleared=load(await readFile(path.join(dir,'cordis.patch.yml'),'utf8'));
assert.equal(cleared.find(x=>x.id==='llm-pi-ai')?.config.providers.dx,undefined);
assert.ok(cleared.find(x=>x.id==='llm-pi-ai').config.providers.custom);
assert.equal(cleared.find(x=>x.id==='agent-default-model'),undefined);
const home=path.resolve(dir,'../..'),credentialsFile=path.join(home,'.credentials.yaml');
const stored=load(await readFile(credentialsFile,'utf8'));stored.refs.DSH_API_KEY='fixture-orphan-key';
await writeFile(credentialsFile,JSON.stringify(stored));
const specPath=path.join(home,'no-host-spec.json');await writeFile(specPath,'{"hostAbsent":true}');
const count=requests.length;
const result=await runChild(specPath,{...cleanEnv,DX_DSH_HOME:home,DX_DSH_CONTEXT_FILE:path.join(home,'context.json')});
assert.equal(result.code,0,'independent provider must work without host configuration: '+result.stderr);
assert.deepEqual(requests.slice(count),[{server:'custom',path:'/v1/chat/completions',authorization:'Bearer fixture-custom-key',model:'custom-model'}]);
reports.push({name:'no-host-with-unused-stored-key',status:'passed',calls:[{server:'custom',path:'/v1/chat/completions',model:'custom-model',credentialMatched:true}]});
}
const report={status:'passed',realHarness:true,externalModelCalls:false,cases:reports,hostRemoval:process.argv[2]!=='--case'};
await writeFile(path.join(base,'report.json'),JSON.stringify(report,null,2));console.log(JSON.stringify(report));
}finally{
for(const {server} of [old,next,custom]){server.closeAllConnections();await new Promise(resolve=>server.close(resolve));}
}
}