Files

71 lines
4.0 KiB
Python

import importlib.util
import hashlib
import json
from pathlib import Path
spec=importlib.util.spec_from_file_location('semantics',Path(__file__).parents[2]/'tools/score_acceptance_semantics.py')
oracle=importlib.util.module_from_spec(spec);spec.loader.exec_module(oracle)
def test_required_fields_allow_extra_keys_but_not_wrong_values():
expected={'status':'ambiguous','candidate_count':3,'selected_path':None}
assert oracle.matches(dict(expected,project_name='Board-X'),expected)
assert not oracle.matches(dict(expected,candidate_count=4),expected)
assert not oracle.matches({'status':'ambiguous'},expected)
assert not oracle.matches({'count':True},{'count':1})
def test_arrays_require_exact_membership_counts_and_requested_order():
assert oracle.matches(['R2','R1'],['R1','R2'])
assert not oracle.matches(['R2','R1'],['R1','R2'],ordered=True)
assert not oracle.matches(['R1','R1'],['R1','R2'])
assert not oracle.matches(['R1','R2','R3'],['R1','R2'])
assert oracle.matches([{'x':1,'y':2},{'x':1}], [{'x':1},{'x':1,'y':2}])
def test_preservation_contract_requires_exact_recursive_schema():
expected={'options':{'enabled':True},'note':'keep'}
assert oracle.matches(expected,expected,exclusive=True)
assert not oracle.matches(dict(expected,extra=3),expected,exclusive=True)
assert not oracle.matches({'options':{'enabled':True,'extra':3},'note':'keep'},expected,exclusive=True)
def test_artifact_rescore_preserves_strict_report_and_detects_input_mutation(tmp_path):
workspace=tmp_path/'workspace';workspace.mkdir()
original=workspace/'catalog.json';original.write_text('[]',encoding='utf-8')
(workspace/'result.json').write_text('{"status":"ambiguous","count":2,"note":"extra"}',encoding='utf-8')
manifest={'seed':'123','cases':[{'id':'case','stratum':'ambiguity','directory':str(workspace),
'expected':{'result.json':{'status':'ambiguous','count':2}},
'input_hashes':{'catalog.json':hashlib.sha256(b'[]').hexdigest()}}]}
report={'samples':[{'id':'case','status':'failed','turn_end':{'kind':'completed'}}]}
(tmp_path/'manifest.json').write_text(json.dumps(manifest),encoding='utf-8')
report_path=tmp_path/'report.json';report_path.write_text(json.dumps(report),encoding='utf-8')
protected=report_path.read_bytes()
result=oracle.score(tmp_path)
assert result['passed']==1
assert result['samples'][0]['strict_original_status']=='failed'
assert report_path.read_bytes()==protected
(workspace/'result.json').write_text('{"status":"ambiguous","count":2,"extra":NaN}',encoding='utf-8')
assert oracle.score(tmp_path)['failed']==1
(workspace/'result.json').write_text('{"status":"ambiguous","count":2}',encoding='utf-8')
original.write_text('["mutated"]',encoding='utf-8')
assert oracle.score(tmp_path)['failed']==1
def test_execution_instruction_requires_receipt_and_zero_exit_not_just_right_artifact(tmp_path):
workspace=tmp_path/'work';workspace.mkdir()
(workspace/'result.json').write_text('{"sum":10,"count":2}',encoding='utf-8')
(workspace/'fixed.py').write_text('print("result")',encoding='utf-8')
case={'id':'run','stratum':'repair-run','directory':str(workspace),
'expected':{'result.json':{'sum':10,'count':2}},'input_hashes':{}}
(tmp_path/'manifest.json').write_text(json.dumps({'seed':'7','cases':[case]}),encoding='utf-8')
(tmp_path/'report.json').write_text(json.dumps({'samples':[{'id':'run','status':'passed','turn_end':{'kind':'completed'}}]}),encoding='utf-8')
assert oracle.score(tmp_path)['failed']==1
receipt={'exit_code':0,'tool_error':False,'output_json':[{'sum':10,'count':2}]}
evidence={'samples':[{'id':'run','receipts':[receipt]}]}
(tmp_path/'execution-evidence.json').write_text(json.dumps(evidence),encoding='utf-8')
assert oracle.score(tmp_path)['passed']==1
receipt['exit_code']=1
(tmp_path/'execution-evidence.json').write_text(json.dumps(evidence),encoding='utf-8')
assert oracle.score(tmp_path)['failed']==1