154 lines
7.3 KiB
Python
154 lines
7.3 KiB
Python
from pathlib import Path
|
|
from types import SimpleNamespace as NS
|
|
from threading import Event
|
|
import json
|
|
import zipfile
|
|
|
|
from dx_productivity.dsh.incidents import IncidentRunner, decision
|
|
|
|
|
|
class Service:
|
|
def __init__(self, *, interrupted=False, fixed=True):
|
|
self.calls = []
|
|
self.interrupted = interrupted
|
|
self.fixed = fixed
|
|
self.progress = NS(completed_items=0, total_items=1, saved_items=0, failed_items=1,
|
|
no_match_items=0, queue_complete=True, next_project_path=None)
|
|
|
|
def task_context(self, task_id):
|
|
return NS(task=NS(task_id=task_id, status=NS(value='running'), copy_part_number_to_value=False),
|
|
progress=self.progress, interrupted=self.interrupted, phase='retry', rules=(1,))
|
|
|
|
def prepare_failed_quick_run(self, task_id):
|
|
self.calls.append('prepare')
|
|
self.progress = NS(**(vars(self.progress) | {'next_project_path': Path('board.prj'),
|
|
'next_old_part_number': 'OLD', 'next_new_part_number': 'NEW', 'queue_complete': False}))
|
|
|
|
def execute_quick_run_queue(self, task_id, *, expected_item, stop_event):
|
|
self.calls.append('execute')
|
|
self.progress = NS(**(vars(self.progress) | {'queue_complete': True, 'failed_items': 0 if self.fixed else 1,
|
|
'completed_items': 1, 'saved_items': 1 if self.fixed else 0}))
|
|
return NS(paused=False, progress=self.progress)
|
|
|
|
|
|
def analyze(*args):
|
|
return {'reply': json.dumps({'summary': '已核对失败记录', 'action': 'retry_failed', 'unknowns': ['需执行后核验']}),
|
|
'session_id': 'native-session', 'transcript': [{'type': 'tool/result', 'name': 'dx_task_snapshot'}]}
|
|
|
|
|
|
def make_runner(tmp_path, service=None, analyzer=analyze, publisher=None):
|
|
def publish(record, archive):
|
|
assert archive.is_file()
|
|
return {'status': 'submitted', 'url': 'https://git.example/owner/repo/issues/1', 'attachment_status': 'uploaded'}
|
|
return IncidentRunner(tmp_path, service or Service(), analyzer, publisher or publish)
|
|
|
|
|
|
def test_verified_repair_runs_once_and_duplicate_does_not_repeat(tmp_path):
|
|
service = Service()
|
|
runner = make_runner(tmp_path, service)
|
|
first = runner.run('t', 'worker_error', 'failure', run_id='run1', execution_authorized=True)
|
|
assert first['status'] == 'resolved'
|
|
assert service.calls == ['prepare', 'execute']
|
|
second = make_runner(tmp_path, service).run('t', 'worker_error', 'failure', run_id='run1', execution_authorized=True)
|
|
assert second['incident_id'] == first['incident_id']
|
|
assert service.calls == ['prepare', 'execute']
|
|
assert first['verification']['failed_items'] == 0
|
|
|
|
|
|
def test_uncertain_results_never_replay_even_when_model_requests_retry(tmp_path):
|
|
service = Service(interrupted=True)
|
|
result = make_runner(tmp_path, service).run('t', 'worker_error', 'failure', execution_authorized=True)
|
|
assert not service.calls
|
|
assert result['status'] == 'escalated'
|
|
assert result['ticket']['status'] == 'submitted'
|
|
|
|
|
|
def test_failed_retry_is_not_reported_as_resolved_and_reflection_is_saved(tmp_path):
|
|
service = Service(fixed=False)
|
|
phases = []
|
|
def model(incident, snapshot, evidence, phase):
|
|
phases.append(phase)
|
|
return analyze()
|
|
result = make_runner(tmp_path, service, model).run('t', 'worker_error', 'failure', execution_authorized=True)
|
|
assert service.calls == ['prepare', 'execute']
|
|
assert phases == ['diagnose', 'reflect']
|
|
assert result['status'] == 'escalated'
|
|
assert (Path(result['evidence_dir']) / 'reflect.json').is_file()
|
|
|
|
|
|
def test_unknown_error_or_unapproved_run_never_executes(tmp_path):
|
|
for code, authorized in [('unknown_error', True), ('worker_error', False)]:
|
|
service = Service()
|
|
result = make_runner(tmp_path / code, service).run('t', code, 'failure', execution_authorized=authorized)
|
|
assert not service.calls
|
|
assert result['status'] == 'escalated'
|
|
|
|
|
|
def test_model_failure_and_git_failure_preserve_evidence_and_outbox(tmp_path):
|
|
def model(*args): raise TimeoutError('model timed out')
|
|
def publish(*args): raise OSError('network down')
|
|
result = make_runner(tmp_path, analyzer=model, publisher=publish).run('t', 'worker_error', 'failure')
|
|
assert result['status'] == 'pending_issue'
|
|
directory = Path(result['evidence_dir'])
|
|
assert (directory / 'issue.md').is_file()
|
|
assert (directory / 'evidence.zip').is_file()
|
|
manifest = json.loads((directory / 'manifest.json').read_text(encoding='utf8'))
|
|
assert manifest['sha256']
|
|
assert result['ticket']['status'] == 'failed'
|
|
|
|
|
|
def test_model_cannot_invent_actions_or_claim_resolution(tmp_path):
|
|
def model(*args): return {'reply': '{"summary":"solved","action":"shell","resolved":true}'}
|
|
service = Service()
|
|
result = make_runner(tmp_path, service, model).run('t', 'worker_error', 'failure', execution_authorized=True)
|
|
assert not service.calls
|
|
assert result['status'] == 'escalated'
|
|
|
|
|
|
def test_stop_request_prevents_retry_and_sensitive_text_is_redacted(tmp_path):
|
|
stop = Event(); stop.set()
|
|
service = Service()
|
|
result = make_runner(tmp_path, service).run('t', 'worker_error',
|
|
'Authorization: Bearer exampleCredential123 sk-testSecretCredential123', execution_authorized=True, stop_event=stop)
|
|
assert not service.calls
|
|
with zipfile.ZipFile(Path(result['evidence_dir']) / 'evidence.zip') as archive:
|
|
text = b'\n'.join(archive.read(n) for n in archive.namelist())
|
|
assert b'exampleCredential123' not in text
|
|
assert b'sk-testSecretCredential123' not in text
|
|
|
|
|
|
def test_publish_pending_reuses_existing_incident_without_ai_or_execution(tmp_path):
|
|
result = make_runner(tmp_path, publisher=lambda *a: {'status': 'unavailable'}).run('t', 'unknown', 'failure')
|
|
assert result['status'] == 'pending_issue'
|
|
runner = make_runner(tmp_path, analyzer=lambda *a: (_ for _ in ()).throw(AssertionError('must not rediagnose')))
|
|
retried = runner.publish_pending(result['incident_id'])
|
|
assert retried['status'] == 'escalated'
|
|
|
|
|
|
def test_incomplete_persisted_attempt_is_not_reexecuted(tmp_path):
|
|
result = make_runner(tmp_path).run('t', 'worker_error', 'failure', run_id='run1')
|
|
path = Path(result['evidence_dir']) / 'incident.json'
|
|
row = json.loads(path.read_text(encoding='utf8')); row['status'] = 'retrying'
|
|
path.write_text(json.dumps(row), encoding='utf8')
|
|
service = Service()
|
|
same = make_runner(tmp_path, service).run('t', 'worker_error', 'failure', run_id='run1', execution_authorized=True)
|
|
assert not service.calls
|
|
assert same['status'] == 'pending_issue'
|
|
|
|
|
|
def test_native_human_readable_diagnosis_retains_validated_decision():
|
|
value = decision('## 结论\n建议按规则重试。\n\n```json\n{"action":"retry_failed"}\n```')
|
|
assert value['action'] == 'retry_failed'
|
|
assert '建议按规则重试' in value['summary']
|
|
|
|
|
|
def test_different_error_codes_share_one_retry_budget_for_original_run(tmp_path):
|
|
service = Service(fixed=False)
|
|
runner = make_runner(tmp_path, service)
|
|
first = runner.run('t', 'worker_error', 'first failure', execution_authorized=True, run_id='original')
|
|
assert first['attempts'] == 1
|
|
service.progress = NS(**(vars(service.progress) | {'queue_complete': True}))
|
|
second = runner.run('t', 'execution_failed', 'same run, different code', execution_authorized=True, run_id='original')
|
|
assert second['attempts'] == 0
|
|
assert service.calls == ['prepare', 'execute']
|