135 lines
7.9 KiB
Python
135 lines
7.9 KiB
Python
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
|
|
def test_successful_shell_job_receipt_without_nonzero_exit_marker():
|
|
node=shutil.which('node')
|
|
if not node: pytest.skip('Node runtime not installed')
|
|
module=(Path(__file__).parents[2]/'tools/extract_acceptance_execution.mjs').as_uri()
|
|
events=[{'type':'tool/call','data':{'name':'pwsh','callId':'x','arguments':json.dumps({'command':'python fixed.py'})}},
|
|
{'type':'job/settled','data':{'toolCall':{'callId':'x'},'status':'completed','detail':'exit code: 0'}},
|
|
{'type':'tool/result','data':{'message':{'toolCallId':'x','isError':False,'content':[{'type':'text','text':'{"sum":10,"count":2}'}]}}}]
|
|
code=f'import {{executionReceipts}} from {json.dumps(module)}; console.log(JSON.stringify(executionReceipts({json.dumps(events)})));'
|
|
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
|
|
receipt=json.loads(result.stdout)[0]
|
|
assert receipt['exit_code']==0
|
|
assert receipt['output_json']==[{'sum':10,'count':2}]
|
|
events[1]['data']['detail']='exit code: 1'
|
|
code=f'import {{executionReceipts}} from {json.dumps(module)}; console.log(JSON.stringify(executionReceipts({json.dumps(events)})));'
|
|
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
|
|
assert json.loads(result.stdout)[0]['exit_code']==1
|
|
|
|
|
|
def test_preflight_checks_new_active_session_not_historical_working_directory():
|
|
node=shutil.which('node')
|
|
if not node: pytest.skip('Node runtime not installed')
|
|
module=(Path(__file__).parents[2]/'tools/acceptance_runtime_state.mjs').as_uri()
|
|
code=f'import {{workspaceCandidates,verifyWorkspaces}} from {json.dumps(module)};'
|
|
code+='''
|
|
const old={id:'old',cwd:'/owned/root/previous-case',sandbox:{mode:'workspace-write',workspaceRoot:'/owned/root/previous-case'}};
|
|
const active={id:'new',cwd:'/owned/root',sandbox:{mode:'workspace-write',workspaceRoot:'/owned/root'}};
|
|
const candidates=workspaceCandidates({sessions:[old,active]},new Set(['old']),'/owned/root');
|
|
verifyWorkspaces(candidates,'/owned/root');
|
|
if(candidates.length!==1||candidates[0].id!=='new')throw Error('History leaked into active preflight');
|
|
active.sandbox.workspaceRoot='/wrong';
|
|
let rejected=false;try{verifyWorkspaces(candidates,'/owned/root');}catch{rejected=true;}
|
|
if(!rejected)throw Error('Wrong actual sandbox root accepted');
|
|
console.log('verified');
|
|
'''
|
|
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
|
|
assert result.stdout.strip()=='verified'
|
|
|
|
|
|
def test_session_directory_before_log_publication_waits_without_aborting(tmp_path):
|
|
node=shutil.which('node')
|
|
if not node: pytest.skip('Node runtime not installed')
|
|
module=(Path(__file__).parents[2]/'tools/harness_session_evidence.mjs').as_uri()
|
|
code=f'import {{evidenceFor,readSession}} from {json.dumps(module)};'
|
|
code+='''
|
|
import {mkdir,writeFile,unlink} from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import {zstdCompressSync} from 'node:zlib';
|
|
const home=process.argv[1],directory=path.join(home,'sessions','workspace','new-session');
|
|
await mkdir(directory,{recursive:true});
|
|
const filename=path.join(directory,'session.v4.jsonl.zstd');
|
|
const pending=await evidenceFor(home,'owned prompt');
|
|
if(pending.complete!==false)throw Error('Unpublished log accepted as complete');
|
|
const events=[{seq:1,type:'turn/start',data:{}},{seq:2,type:'user/message',data:{content:[{text:'owned prompt'}]}},
|
|
{seq:3,type:'turn/end',data:{reason:{kind:'completed'}}}];
|
|
await writeFile(filename,zstdCompressSync(Buffer.from(events.map(e=>JSON.stringify(e)).join('\\n')+'\\n')));
|
|
const done=await evidenceFor(home,'owned prompt');
|
|
if(!done.complete||done.filename!==filename)throw Error('Published completed turn was not found');
|
|
await unlink(filename);
|
|
const raw=filename.slice(0,-5),lines=events.map(e=>JSON.stringify(e)).join('\\n')+'\\n';
|
|
await writeFile(raw,lines);
|
|
if(!(await evidenceFor(home,'owned prompt',filename)).complete)throw Error('Zstd to raw transition lost live session');
|
|
await unlink(raw);
|
|
if((await evidenceFor(home,'owned prompt',filename)).complete)throw Error('Missing live session became completed');
|
|
await writeFile(filename,zstdCompressSync(Buffer.from(lines)));
|
|
if(!(await evidenceFor(home,'owned prompt',raw)).complete)throw Error('Raw to zstd transition lost live session');
|
|
await writeFile(filename,Buffer.from('permanently corrupt log'));
|
|
let corrupt=false;try{await evidenceFor(home,'owned prompt',filename);}catch{corrupt=true;}
|
|
if(!corrupt)throw Error('Permanent compression corruption was treated as pending');
|
|
await unlink(filename);
|
|
await writeFile(raw,'{malformed complete record}\\n');
|
|
corrupt=false;try{await readSession(raw);}catch{corrupt=true;}
|
|
if(!corrupt)throw Error('Permanent JSON corruption was treated as pending');
|
|
await writeFile(raw,lines+JSON.stringify({seq:4,type:'partial'}).slice(0,8));
|
|
if((await readSession(raw)).length!==3)throw Error('Incomplete append was not retained as pending prefix');
|
|
await mkdir(path.join(home,'not-a-file'));
|
|
let rejected=false;try{await readSession(path.join(home,'not-a-file'));}catch{rejected=true;}
|
|
if(!rejected)throw Error('Non-ENOENT observer error was silently ignored');
|
|
console.log('verified');
|
|
'''
|
|
result=subprocess.run([node,'--input-type=module','-e',code,str(tmp_path)],capture_output=True,text=True,check=True)
|
|
assert result.stdout.strip()=='verified'
|
|
|
|
|
|
def test_owned_runtime_origin_waits_for_navigation_but_rejects_another_runtime():
|
|
node=shutil.which('node')
|
|
if not node: pytest.skip('Node runtime not installed')
|
|
module=(Path(__file__).parents[2]/'tools/dsh_cdp.mjs').as_uri()
|
|
code=f'import {{awaitRuntimeOrigin}} from {json.dumps(module)};'
|
|
code+='''
|
|
const expected='http://127.0.0.1:10168',origins=['null','null',expected];
|
|
await awaitRuntimeOrigin({evaluate:async()=>origins.shift()},expected,{delayMs:0});
|
|
let rejected=false;
|
|
try{await awaitRuntimeOrigin({evaluate:async()=> 'http://127.0.0.1:9999'},expected,{delayMs:0});}catch{rejected=true;}
|
|
if(!rejected)throw Error('Another runtime was accepted');
|
|
rejected=false;
|
|
try{await awaitRuntimeOrigin({evaluate:async()=> 'null'},expected,{timeoutMs:5,delayMs:0});}catch{rejected=true;}
|
|
if(!rejected)throw Error('Permanent blank navigation was accepted');
|
|
console.log('verified');
|
|
'''
|
|
result=subprocess.run([node,'--input-type=module','-e',code],capture_output=True,text=True,check=True)
|
|
assert result.stdout.strip()=='verified'
|
|
|
|
|
|
def test_paired_summary_preserves_manifest_purpose_and_rejects_mismatch(tmp_path):
|
|
node=shutil.which('node')
|
|
if not node: pytest.skip('Node runtime not installed')
|
|
script=Path(__file__).parents[2]/'tools/summarize_paired_samples.mjs'
|
|
for arm in ('native','harness'):
|
|
directory=tmp_path/arm
|
|
directory.mkdir()
|
|
manifest=dict(seed='1',purpose='holdout',cases=[],data=str(directory),
|
|
workspace=str(directory/'dsh/harness-home/workspace'),source_files={},coverage=[],runtime=arm)
|
|
(directory/'manifest.json').write_text(json.dumps(manifest),encoding='utf-8')
|
|
(directory/'report.json').write_text(json.dumps({'samples':[]}),encoding='utf-8')
|
|
(directory/'semantic-score.json').write_text(json.dumps({'samples':[],'passed':0,'failed':0}),encoding='utf-8')
|
|
command=[node,str(script),str(tmp_path),'--summary-only']
|
|
subprocess.run(command,capture_output=True,text=True,check=True)
|
|
summary=tmp_path/'paired-summary.json'
|
|
assert json.loads(summary.read_text())['purpose']=='holdout'
|
|
before=summary.read_bytes()
|
|
manifest=tmp_path/'harness/manifest.json'
|
|
changed=json.loads(manifest.read_text());changed['purpose']='exploration'
|
|
manifest.write_text(json.dumps(changed),encoding='utf-8')
|
|
result=subprocess.run(command,capture_output=True,text=True)
|
|
assert result.returncode!=0 and 'Paired manifest purposes differ' in result.stderr
|
|
assert summary.read_bytes()==before
|