Core regression / Python 3.12 core, intervention UI and release configuration (push) Failing after 20s
Core regression / JavaScript syntax (push) Successful in 4s
Core regression / Python 3.12 core, intervention UI and release configuration (pull_request) Failing after 20s
Core regression / JavaScript syntax (pull_request) Successful in 3s
93 lines
5.8 KiB
JavaScript
93 lines
5.8 KiB
JavaScript
/** Offline fixtures: real Harness composition, no runtime boot or model requests. */
|
|
import assert from 'node:assert/strict';
|
|
import {mkdtemp,mkdir,writeFile,readFile,readdir,rm} from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import {checkModelAvailability} from '../integration/dsh/model_availability.mjs';
|
|
|
|
const home=await mkdtemp(path.join(os.tmpdir(),'dx-model-availability-'));
|
|
const directory=path.join(home,'profiles/dx-task');
|
|
await mkdir(directory,{recursive:true});
|
|
const names=['DSH_API_BASE','DSH_MODEL','DSH_API_KEY','DX_DSH_MODELS','FIXTURE_NATIVE_KEY','DEEPSEEK_API_KEY'];
|
|
const saved=Object.fromEntries(names.map(key=>[key,process.env[key]]));
|
|
for(const key of names)delete process.env[key];
|
|
const route={api:'openai-completions',baseURL:'https://fixture.invalid/v1',
|
|
apiKeyEnv:'FIXTURE_NATIVE_KEY',models:[{id:'fixture-model'}]};
|
|
const rows=profile=>[{id:'llm-pi-ai',config:{providers:{custom:profile}}},
|
|
{id:'agent-default-model',config:{provider:'custom',model:'fixture-model'}}];
|
|
let cases=0;
|
|
|
|
async function snapshot(directory){
|
|
const entries=await readdir(directory,{withFileTypes:true});
|
|
const result={};
|
|
for(const entry of entries){
|
|
const filename=path.join(directory,entry.name);
|
|
result[entry.name]=entry.isDirectory()?await snapshot(filename):(await readFile(filename)).toString('base64');
|
|
}
|
|
return result;
|
|
}
|
|
|
|
async function verify(patch,credentials,code,environmentRefs){
|
|
await writeFile(path.join(directory,'cordis.patch.yml'),JSON.stringify(patch));
|
|
await writeFile(path.join(home,'.credentials.yaml'),typeof credentials==='string'?credentials:JSON.stringify(credentials),{mode:0o600});
|
|
const before=await snapshot(home);
|
|
const started=performance.now();
|
|
const checked=await checkModelAvailability(home);
|
|
assert.equal(checked.code,code);
|
|
assert.equal(checked.available,code==='available');
|
|
if(checked.available){
|
|
assert.ok(['native','host'].includes(checked.source));
|
|
assert.ok(Array.isArray(checked.environment_refs));
|
|
assert.ok(checked.environment_refs.every(name=>/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)));
|
|
if(environmentRefs!==undefined)assert.deepEqual(checked.environment_refs,environmentRefs);
|
|
}
|
|
assert.deepEqual(await snapshot(home),before,'status check must not change files');
|
|
assert.ok(performance.now()-started<3000,'native read-only check must fit desktop worker deadline');
|
|
cases++;
|
|
}
|
|
|
|
try{
|
|
await verify([],{},'missing_credential');
|
|
await verify([],{version:1,refs:{DEEPSEEK_API_KEY:'fake-official-native-key'}},'available',[]);
|
|
assert.equal((await checkModelAvailability(home)).source,'native');
|
|
assert.deepEqual((await checkModelAvailability(home)).environment_refs,[]);
|
|
await verify([{id:'llm-deepseek',config:{apiKeyEnv:'FIXTURE_NATIVE_KEY'}}],
|
|
{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-custom-official-key'}},'available',[]);
|
|
await verify(rows(route),{},'missing_credential');
|
|
await verify(rows(route),{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-offline-key'}},'available',[]);
|
|
// Native settings save a provider without editing agent-default-model.
|
|
await verify(rows(route).slice(0,1),{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-offline-key'}},'available',[]);
|
|
assert.deepEqual((await checkModelAvailability(home)).model_selection,{provider:'custom',model:'fixture-model'});
|
|
await verify(rows(route).slice(0,1),{},'missing_credential');
|
|
await verify([{id:'llm-pi-ai',config:{providers:{custom:route,other:{...route,apiKeyEnv:undefined}}}},
|
|
rows(route)[1]],{},'missing_credential');
|
|
await verify(rows(route),{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-offline-key'},records:[]},'invalid_configuration');
|
|
await verify(rows(route),{FIXTURE_NATIVE_KEY:'fake-flat-offline-key'},'available',[]);
|
|
await verify(rows(route),'%YAML 1.2\n---\nFIXTURE_NATIVE_KEY: fake-flat-offline-key\n','invalid_configuration');
|
|
await verify(rows(route),'version: 1\nrefs: [fake-secret', 'invalid_configuration');
|
|
await verify(rows(route),{version:99,refs:{FIXTURE_NATIVE_KEY:'fake-offline-key'}},'invalid_configuration');
|
|
process.env.FIXTURE_NATIVE_KEY='fake-environment-key';
|
|
await verify(rows(route),{},'available',['FIXTURE_NATIVE_KEY']);
|
|
await verify(rows(route),{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-stored-wins'}},'available',[]);
|
|
await verify([{id:'llm-deepseek',config:{apiKeyEnv:'FIXTURE_NATIVE_KEY'}}],{},'available',['FIXTURE_NATIVE_KEY']);
|
|
delete process.env.FIXTURE_NATIVE_KEY;
|
|
await verify(rows({...route,apiKeyEnv:undefined}),{},'available');
|
|
const ignored=rows(route);ignored[0].name='@deepseek-ai/not-the-configured-plugin';
|
|
await verify(ignored,{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-offline-key'}},'missing_configuration');
|
|
await verify([...rows(route),{insert:[{id:'llm-pi-ai',name:'@deepseek-ai/dsh-llm-pi-ai'}]}],
|
|
{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-offline-key'}},'invalid_configuration');
|
|
Object.assign(process.env,{DSH_API_BASE:'https://host.invalid/v1',DSH_MODEL:'host-model',DSH_API_KEY:'fake-host-key'});
|
|
await verify([],{},'available',['DSH_API_KEY']);
|
|
await verify(rows(route),{version:1,refs:{FIXTURE_NATIVE_KEY:'fake-offline-key'}},'available');
|
|
await verify([],{version:1,refs:{DSH_API_KEY:'fake-unbound-key'}},'invalid_configuration');
|
|
// Independently stored native dx refs are valid even with no host provider.
|
|
for(const key of ['DSH_API_BASE','DSH_MODEL','DSH_API_KEY'])delete process.env[key];
|
|
await verify([{id:'llm-pi-ai',config:{providers:{dx:{...route,apiKeyEnv:'DSH_API_KEY'}}}},
|
|
{id:'agent-default-model',config:{provider:'dx',model:'fixture-model'}}],
|
|
{version:1,refs:{DSH_API_KEY:'fake-bound-native-key'}},'available',[]);
|
|
console.log(JSON.stringify({status:'passed',cases,realHarnessComposition:true,readOnly:true,remoteModelCalls:false}));
|
|
}finally{
|
|
for(const key of names){if(saved[key]===undefined)delete process.env[key];else process.env[key]=saved[key];}
|
|
await rm(home,{recursive:true,force:true});
|
|
}
|