Files
dxdesigner-part-replacement/tests/packaging/test_acceptance_execution.py

135 lines
7.9 KiB
Python

import json
from pathlib import Path
import shutil
import subprocess
import pytest
def test_successful_shell_job_receipt_without_nonzero_exit_marker():
node=shutil.which('node')
if not node: pytest.skip('Node runtime not installed')
module=(Path(__file__).parents[2]/'tools/extract_acceptance_execution.mjs').as_uri()
events=[{'type':'tool/call','data':{'name':'pwsh','callId':'x','arguments':json.dumps({'command':'python fixed.py'})}},
{'type':'job/settled','data':{'toolCall':{'callId':'x'},'status':'completed','detail':'exit code: 0'}},
{'type':'tool/result','data':{'message':{'toolCallId':'x','isError':False,'content':[{'type':'text','text':'{"sum":10,"count":2}'}]}}}]
code=f'import {{executionReceipts}} from {json.dumps(module)}; console.log(JSON.stringify(executionReceipts({json.dumps(events)})));'
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
receipt=json.loads(result.stdout)[0]
assert receipt['exit_code']==0
assert receipt['output_json']==[{'sum':10,'count':2}]
events[1]['data']['detail']='exit code: 1'
code=f'import {{executionReceipts}} from {json.dumps(module)}; console.log(JSON.stringify(executionReceipts({json.dumps(events)})));'
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
assert json.loads(result.stdout)[0]['exit_code']==1
def test_preflight_checks_new_active_session_not_historical_working_directory():
node=shutil.which('node')
if not node: pytest.skip('Node runtime not installed')
module=(Path(__file__).parents[2]/'tools/acceptance_runtime_state.mjs').as_uri()
code=f'import {{workspaceCandidates,verifyWorkspaces}} from {json.dumps(module)};'
code+='''
const old={id:'old',cwd:'/owned/root/previous-case',sandbox:{mode:'workspace-write',workspaceRoot:'/owned/root/previous-case'}};
const active={id:'new',cwd:'/owned/root',sandbox:{mode:'workspace-write',workspaceRoot:'/owned/root'}};
const candidates=workspaceCandidates({sessions:[old,active]},new Set(['old']),'/owned/root');
verifyWorkspaces(candidates,'/owned/root');
if(candidates.length!==1||candidates[0].id!=='new')throw Error('History leaked into active preflight');
active.sandbox.workspaceRoot='/wrong';
let rejected=false;try{verifyWorkspaces(candidates,'/owned/root');}catch{rejected=true;}
if(!rejected)throw Error('Wrong actual sandbox root accepted');
console.log('verified');
'''
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
assert result.stdout.strip()=='verified'
def test_session_directory_before_log_publication_waits_without_aborting(tmp_path):
node=shutil.which('node')
if not node: pytest.skip('Node runtime not installed')
module=(Path(__file__).parents[2]/'tools/harness_session_evidence.mjs').as_uri()
code=f'import {{evidenceFor,readSession}} from {json.dumps(module)};'
code+='''
import {mkdir,writeFile,unlink} from 'node:fs/promises';
import path from 'node:path';
import {zstdCompressSync} from 'node:zlib';
const home=process.argv[1],directory=path.join(home,'sessions','workspace','new-session');
await mkdir(directory,{recursive:true});
const filename=path.join(directory,'session.v4.jsonl.zstd');
const pending=await evidenceFor(home,'owned prompt');
if(pending.complete!==false)throw Error('Unpublished log accepted as complete');
const events=[{seq:1,type:'turn/start',data:{}},{seq:2,type:'user/message',data:{content:[{text:'owned prompt'}]}},
{seq:3,type:'turn/end',data:{reason:{kind:'completed'}}}];
await writeFile(filename,zstdCompressSync(Buffer.from(events.map(e=>JSON.stringify(e)).join('\\n')+'\\n')));
const done=await evidenceFor(home,'owned prompt');
if(!done.complete||done.filename!==filename)throw Error('Published completed turn was not found');
await unlink(filename);
const raw=filename.slice(0,-5),lines=events.map(e=>JSON.stringify(e)).join('\\n')+'\\n';
await writeFile(raw,lines);
if(!(await evidenceFor(home,'owned prompt',filename)).complete)throw Error('Zstd to raw transition lost live session');
await unlink(raw);
if((await evidenceFor(home,'owned prompt',filename)).complete)throw Error('Missing live session became completed');
await writeFile(filename,zstdCompressSync(Buffer.from(lines)));
if(!(await evidenceFor(home,'owned prompt',raw)).complete)throw Error('Raw to zstd transition lost live session');
await writeFile(filename,Buffer.from('permanently corrupt log'));
let corrupt=false;try{await evidenceFor(home,'owned prompt',filename);}catch{corrupt=true;}
if(!corrupt)throw Error('Permanent compression corruption was treated as pending');
await unlink(filename);
await writeFile(raw,'{malformed complete record}\\n');
corrupt=false;try{await readSession(raw);}catch{corrupt=true;}
if(!corrupt)throw Error('Permanent JSON corruption was treated as pending');
await writeFile(raw,lines+JSON.stringify({seq:4,type:'partial'}).slice(0,8));
if((await readSession(raw)).length!==3)throw Error('Incomplete append was not retained as pending prefix');
await mkdir(path.join(home,'not-a-file'));
let rejected=false;try{await readSession(path.join(home,'not-a-file'));}catch{rejected=true;}
if(!rejected)throw Error('Non-ENOENT observer error was silently ignored');
console.log('verified');
'''
result=subprocess.run([node,'--input-type=module','-e',code,str(tmp_path)],capture_output=True,text=True,check=True)
assert result.stdout.strip()=='verified'
def test_owned_runtime_origin_waits_for_navigation_but_rejects_another_runtime():
node=shutil.which('node')
if not node: pytest.skip('Node runtime not installed')
module=(Path(__file__).parents[2]/'tools/dsh_cdp.mjs').as_uri()
code=f'import {{awaitRuntimeOrigin}} from {json.dumps(module)};'
code+='''
const expected='http://127.0.0.1:10168',origins=['null','null',expected];
await awaitRuntimeOrigin({evaluate:async()=>origins.shift()},expected,{delayMs:0});
let rejected=false;
try{await awaitRuntimeOrigin({evaluate:async()=> 'http://127.0.0.1:9999'},expected,{delayMs:0});}catch{rejected=true;}
if(!rejected)throw Error('Another runtime was accepted');
rejected=false;
try{await awaitRuntimeOrigin({evaluate:async()=> 'null'},expected,{timeoutMs:5,delayMs:0});}catch{rejected=true;}
if(!rejected)throw Error('Permanent blank navigation was accepted');
console.log('verified');
'''
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
assert result.stdout.strip()=='verified'
def test_paired_summary_preserves_manifest_purpose_and_rejects_mismatch(tmp_path):
node=shutil.which('node')
if not node: pytest.skip('Node runtime not installed')
script=Path(__file__).parents[2]/'tools/summarize_paired_samples.mjs'
for arm in ('native','harness'):
directory=tmp_path/arm
directory.mkdir()
manifest=dict(seed='1',purpose='holdout',cases=[],data=str(directory),
workspace=str(directory/'dsh/harness-home/workspace'),source_files={},coverage=[],runtime=arm)
(directory/'manifest.json').write_text(json.dumps(manifest),encoding='utf-8')
(directory/'report.json').write_text(json.dumps({'samples':[]}),encoding='utf-8')
(directory/'semantic-score.json').write_text(json.dumps({'samples':[],'passed':0,'failed':0}),encoding='utf-8')
command=[node,str(script),str(tmp_path),'--summary-only']
subprocess.run(command,capture_output=True,text=True,check=True)
summary=tmp_path/'paired-summary.json'
assert json.loads(summary.read_text())['purpose']=='holdout'
before=summary.read_bytes()
manifest=tmp_path/'harness/manifest.json'
changed=json.loads(manifest.read_text());changed['purpose']='exploration'
manifest.write_text(json.dumps(changed),encoding='utf-8')
result=subprocess.run(command,capture_output=True,text=True)
assert result.returncode!=0 and 'Paired manifest purposes differ' in result.stderr
assert summary.read_bytes()==before