71 lines
4.0 KiB
Python
71 lines
4.0 KiB
Python
import importlib.util
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
spec=importlib.util.spec_from_file_location('semantics',Path(__file__).parents[2]/'tools/score_acceptance_semantics.py')
|
|
oracle=importlib.util.module_from_spec(spec);spec.loader.exec_module(oracle)
|
|
|
|
|
|
def test_required_fields_allow_extra_keys_but_not_wrong_values():
|
|
expected={'status':'ambiguous','candidate_count':3,'selected_path':None}
|
|
assert oracle.matches(dict(expected,project_name='Board-X'),expected)
|
|
assert not oracle.matches(dict(expected,candidate_count=4),expected)
|
|
assert not oracle.matches({'status':'ambiguous'},expected)
|
|
assert not oracle.matches({'count':True},{'count':1})
|
|
|
|
|
|
def test_arrays_require_exact_membership_counts_and_requested_order():
|
|
assert oracle.matches(['R2','R1'],['R1','R2'])
|
|
assert not oracle.matches(['R2','R1'],['R1','R2'],ordered=True)
|
|
assert not oracle.matches(['R1','R1'],['R1','R2'])
|
|
assert not oracle.matches(['R1','R2','R3'],['R1','R2'])
|
|
assert oracle.matches([{'x':1,'y':2},{'x':1}], [{'x':1},{'x':1,'y':2}])
|
|
|
|
|
|
def test_preservation_contract_requires_exact_recursive_schema():
|
|
expected={'options':{'enabled':True},'note':'keep'}
|
|
assert oracle.matches(expected,expected,exclusive=True)
|
|
assert not oracle.matches(dict(expected,extra=3),expected,exclusive=True)
|
|
assert not oracle.matches({'options':{'enabled':True,'extra':3},'note':'keep'},expected,exclusive=True)
|
|
|
|
|
|
def test_artifact_rescore_preserves_strict_report_and_detects_input_mutation(tmp_path):
|
|
workspace=tmp_path/'workspace';workspace.mkdir()
|
|
original=workspace/'catalog.json';original.write_text('[]',encoding='utf-8')
|
|
(workspace/'result.json').write_text('{"status":"ambiguous","count":2,"note":"extra"}',encoding='utf-8')
|
|
manifest={'seed':'123','cases':[{'id':'case','stratum':'ambiguity','directory':str(workspace),
|
|
'expected':{'result.json':{'status':'ambiguous','count':2}},
|
|
'input_hashes':{'catalog.json':hashlib.sha256(b'[]').hexdigest()}}]}
|
|
report={'samples':[{'id':'case','status':'failed','turn_end':{'kind':'completed'}}]}
|
|
(tmp_path/'manifest.json').write_text(json.dumps(manifest),encoding='utf-8')
|
|
report_path=tmp_path/'report.json';report_path.write_text(json.dumps(report),encoding='utf-8')
|
|
protected=report_path.read_bytes()
|
|
result=oracle.score(tmp_path)
|
|
assert result['passed']==1
|
|
assert result['samples'][0]['strict_original_status']=='failed'
|
|
assert report_path.read_bytes()==protected
|
|
(workspace/'result.json').write_text('{"status":"ambiguous","count":2,"extra":NaN}',encoding='utf-8')
|
|
assert oracle.score(tmp_path)['failed']==1
|
|
(workspace/'result.json').write_text('{"status":"ambiguous","count":2}',encoding='utf-8')
|
|
original.write_text('["mutated"]',encoding='utf-8')
|
|
assert oracle.score(tmp_path)['failed']==1
|
|
|
|
|
|
def test_execution_instruction_requires_receipt_and_zero_exit_not_just_right_artifact(tmp_path):
|
|
workspace=tmp_path/'work';workspace.mkdir()
|
|
(workspace/'result.json').write_text('{"sum":10,"count":2}',encoding='utf-8')
|
|
(workspace/'fixed.py').write_text('print("result")',encoding='utf-8')
|
|
case={'id':'run','stratum':'repair-run','directory':str(workspace),
|
|
'expected':{'result.json':{'sum':10,'count':2}},'input_hashes':{}}
|
|
(tmp_path/'manifest.json').write_text(json.dumps({'seed':'7','cases':[case]}),encoding='utf-8')
|
|
(tmp_path/'report.json').write_text(json.dumps({'samples':[{'id':'run','status':'passed','turn_end':{'kind':'completed'}}]}),encoding='utf-8')
|
|
assert oracle.score(tmp_path)['failed']==1
|
|
receipt={'exit_code':0,'tool_error':False,'output_json':[{'sum':10,'count':2}]}
|
|
evidence={'samples':[{'id':'run','receipts':[receipt]}]}
|
|
(tmp_path/'execution-evidence.json').write_text(json.dumps(evidence),encoding='utf-8')
|
|
assert oracle.score(tmp_path)['passed']==1
|
|
receipt['exit_code']=1
|
|
(tmp_path/'execution-evidence.json').write_text(json.dumps(evidence),encoding='utf-8')
|
|
assert oracle.score(tmp_path)['failed']==1
|