diff --git a/docs/user/dsh-ai-supervision.md b/docs/user/dsh-ai-supervision.md index f54106a..eca06d5 100644 --- a/docs/user/dsh-ai-supervision.md +++ b/docs/user/dsh-ai-supervision.md @@ -69,7 +69,7 @@ Harness 使用任务中心数据目录下独立的 `dsh/harness-home` 保存会 最近一次执行回执独立保留,随后查看任务列表不会覆盖它。执行历史根据本地持久记录和任务证据判断可否恢复;重启只展示记录,不自动重放。点击 **核对并恢复原队列** 后仍须原生确认。计划变化、缺少或损坏检查点、未核验中断、失败项等会阻断恢复;待核验结果从任务中心原流程处理。 -已结束运行的保存、失败和未命中合计来自持久回执;任一任务缺少某项计数时,该项合计仍显示“未报告”,不会把未知当作零。恢复过程报错时保留上一份任务证据和队列基线,供下一次核对。 +已结束运行的保存、失败和未命中合计来自持久回执;任一任务缺少某项计数时,该项合计仍显示“未报告”,不会把未知当作零。恢复过程报错时保留上一份任务证据和队列基线。若已产生核验通过的进度,仅在异常收尾时完整核对通过后更新进度基线并留存差异,以便再次确认接续原队列。旧终态变化、失败、未核验结果、计数回退或核对读取失败时不更新基线;停机后新增的证据变化仍会阻断恢复。 恢复执行后若在新回执写入前退出,原队列中新增的已核验进度可继续接续,不会因旧暂停回执的计数较小而重复执行或永久阻断。原先已收尾任务的证据变化、计数回退,以及完成检查点缺少匹配完成证据,仍会阻断。执行后核验只在队列完成且失败为零时报告就绪;未完成或有失败时明确显示待核验,实际保存与未命中数量分别保留。 diff --git a/integration/dsh/task-plugin/client.js b/integration/dsh/task-plugin/client.js index d76c522..19707aa 100644 --- a/integration/dsh/task-plugin/client.js +++ b/integration/dsh/task-plugin/client.js @@ -87,7 +87,7 @@ window.__ModuleLoader__.load({id:'@dx/dsh-task',factory:require=>{ const border='1px solid var(--dsw-alias-border-l2)'; const buttonStyle={border,background:'var(--dsw-alias-bg-base)',color:'var(--dsw-alias-label-primary)',borderRadius:8,padding:'7px 10px',font:'inherit',fontSize:12,cursor:'pointer'}; const stateNames={...names,pending:'待执行',claimed:'已领取',interrupted:'执行中断',cancelled:'已取消',synced:'已同步',not_synced:'尚未同步',unavailable:'暂不可用',open:'处理中',closed:'已关闭',idle:'待命',generating:'生成中',timeout:'响应超时',error:'出错',finished:'已结束'}; - const blockers={queue_missing:'原队列检查点不存在',queue_invalid:'原队列证据无效',interrupted:'存在待核验步骤',failed_items:'仍有失败项',plan_changed:'保存计划已变化',task_unavailable:'任务不可用',task_paused:'任务已手动暂停,请先核对并接续原执行',not_due:'尚未到执行时间',worker_busy:'执行器忙碌',already_claimed:'排期已被领取',finished:'排期已结束',preflight_blocked:'执行预检未通过',preflight_unavailable:'执行预检不可用',dependency_incomplete:'依赖任务尚未完成并通过核验'}; + const blockers={queue_missing:'原队列检查点不存在',queue_invalid:'原队列证据无效',interrupted:'存在待核验步骤',failed_items:'仍有失败项',plan_changed:'保存计划已变化',evidence_changed:'执行证据已变化,请核验后再恢复',task_unavailable:'任务不可用',task_paused:'任务已手动暂停,请先核对并接续原执行',not_due:'尚未到执行时间',worker_busy:'执行器忙碌',already_claimed:'排期已被领取',finished:'排期已结束',preflight_blocked:'执行预检未通过',preflight_unavailable:'执行预检不可用',dependency_incomplete:'依赖任务尚未完成并通过核验'}; const display=value=>value===null||value===undefined||value===''?'未报告':String(value); const status=value=>stateNames[value]||display(value); const count=value=>Number.isFinite(value)?value.toLocaleString():'未报告'; diff --git a/src/dx_productivity/dsh/execution_runs.py b/src/dx_productivity/dsh/execution_runs.py index 1be75bf..0ba0378 100644 --- a/src/dx_productivity/dsh/execution_runs.py +++ b/src/dx_productivity/dsh/execution_runs.py @@ -9,6 +9,31 @@ from ..trial_task_center.errors import TaskCenterError from ..trial_task_center.task_queue import execute_task_queue +def _failure_evidence(service, journal, run_id, baseline): + evidence = {key: baseline[key] for key in ('tasks', 'entries') if key in baseline} + if not baseline: + return evidence + try: + # Inspect while the journal still records this authorized attempt as + # running. Only its verified, monotonic progress may advance the old + # baseline; terminal changes, failed/uncertain results and rollbacks + # must remain visible to the next inspection. + inspected = journal.inspect(service, run_id) + changes = inspected['diff']['evidence_changes'] + if (inspected['status'] != 'running' or not inspected['diff']['safe'] + or not changes or not all(row.get('expected_progress') is True for row in changes)): + return evidence + tasks = {row['task_id']: dict(row) for row in baseline['tasks']} + for change in changes: + tasks[change['task_id']][change['field']] = change['current'] + evidence.update(tasks=list(tasks.values()), verified_progress=changes) + except Exception: + # Readback failure must neither hide the original execution error nor + # replace the last durable evidence with an incomplete snapshot. + pass + return evidence + + def execute_confirmed_run(service, root, task_ids, report, stop, run_id, *, resume=None, schedule=None, automatic=False): prior = RunJournal(root).get(run_id) if resume else None schedule_id = schedule['schedule_id'] if schedule else (prior or {}).get('schedule_id') @@ -47,7 +72,7 @@ def _execute_confirmed_run(service, root, task_ids, report, stop, run_id, *, res queue_id=run_id if resume and resume['queue_available'] else None, checkpoint_id=None if resume and resume['queue_available'] else run_id) except Exception as error: - failure = {**{key: baseline[key] for key in ('tasks', 'entries') if key in baseline}, + failure = {**(_failure_evidence(service, journal, run_id, baseline) if initialized else {}), 'status': 'interrupted', 'run_id': run_id, 'queue_id': run_id, 'code': getattr(error, 'code', 'execution_interrupted'), 'message': '执行未完成,请按原队列证据核验后接续。'} if initialized: diff --git a/tests/dsh/test_execution_recovery.py b/tests/dsh/test_execution_recovery.py index 3a5d45a..c68c4b9 100644 --- a/tests/dsh/test_execution_recovery.py +++ b/tests/dsh/test_execution_recovery.py @@ -12,6 +12,121 @@ from test_run_journal import project_service from test_run_journal import checkpoint, evidence +def _pause_then_fail_completed_checkpoint(project_service, monkeypatch, *, before_failure=None): + from dx_productivity.trial_task_center import task_queue + service, _, data, ids, _ = project_service + calls = [] + stop = Event() + + def worker(command, timeout): + queue, job = Path(command[3]), command[-1] + calls.append(next(key for key in ids if key in queue.parts)) + with (queue / 'status.tsv').open('a', encoding='utf8') as stream: + stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n') + if len(calls) == 1: + stop.set() + return subprocess.CompletedProcess(command, 0, '', '') + + for task_id in ids: + preparer = service.for_project_task(task_id)._quick_run_preparer + preparer._process_runner = worker + preparer._cscript_resolver = lambda: Path('cscript.exe') + run_id = 'f' * 32 + first = execute_confirmed_run(service, data, ids, preflight(service, ids), stop, run_id) + assert first['status'] == 'paused' and calls == ids[:1] + original = RunJournal(data).inspect(service, run_id) + save = task_queue._save + + def fail_completed(path, state): + if state['entries'][ids[1]]['state'] == 'completed': + if before_failure: + before_failure(service, ids) + raise OSError('controlled completed checkpoint publication failure') + save(path, state) + + with monkeypatch.context() as patch: + patch.setattr(task_queue, '_save', fail_completed) + with pytest.raises(OSError, match='checkpoint publication failure'): + execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True), + Event(), run_id, resume=original) + assert calls == ids[:2] + return run_id, first, calls + + +def test_resume_after_verified_progress_and_checkpoint_io_failure_runs_each_task_once(project_service, monkeypatch): + service, _, data, ids, _ = project_service + run_id, first, calls = _pause_then_fail_completed_checkpoint(project_service, monkeypatch) + restarted = RunJournal(data).inspect(service, run_id) + assert restarted['status'] == 'interrupted' + assert restarted['checkpoint_states'][ids[1]] == 'running' + assert restarted['resume_ready'], restarted['blockers'] + result = restarted['result'] + old_terminal = next(row for row in result['tasks'] if row['task_id'] == ids[0]) + assert old_terminal == next(row for row in first['tasks'] if row['task_id'] == ids[0]) + advanced = next(row for row in result['tasks'] if row['task_id'] == ids[1]) + assert advanced['no_match_items'] == 1 and advanced['queue_complete'] is True + last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True), + Event(), run_id, resume=restarted) + assert last['status'] == 'completed' and last['queue_id'] == run_id + assert calls == ids + assert all(row['no_match_items'] == 1 and row['failed_items'] == 0 for row in last['tasks']) + + +@pytest.mark.parametrize('task_index', [0, 1, 2]) +def test_soft_failure_snapshot_does_not_allow_evidence_changes_while_stopped(project_service, monkeypatch, task_index): + service, _, data, ids, _ = project_service + run_id, _, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch) + if task_index == 2: + evidence(service, ids[task_index]) + else: + root = service.for_project_task(ids[task_index])._quick_run_preparer.data_root / 'quick-run' + (root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8') + restarted = RunJournal(data).inspect(service, run_id) + assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers'] + + +@pytest.mark.parametrize('fault', ['old_terminal', 'failed_result', 'uncertain_result']) +def test_checkpoint_failure_does_not_adopt_unsafe_evidence_as_new_baseline(project_service, monkeypatch, fault): + service, _, data, ids, _ = project_service + + def corrupt(service, ids): + if fault == 'old_terminal': + root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run' + (root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8') + else: + evidence(service, ids[1], 'batch_failed' if fault == 'failed_result' else 'batch_saved_unverified', '') + + run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch, before_failure=corrupt) + restarted = RunJournal(data).inspect(service, run_id) + assert not restarted['resume_ready'] + assert restarted['result']['tasks'] == first['tasks'] + assert 'evidence_changed' in restarted['blockers'] + + +def test_failed_exception_readback_keeps_original_error_and_baseline(project_service, monkeypatch): + service, _, data, _, _ = project_service + inspect = RunJournal.inspect + failing = False + + def unavailable(self, *args, **kwargs): + if failing: + raise RuntimeError('controlled readback failure') + return inspect(self, *args, **kwargs) + + def start_failing(_service, _ids): + nonlocal failing + failing = True + + with monkeypatch.context() as patch: + patch.setattr(RunJournal, 'inspect', unavailable) + run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch, + before_failure=start_failing) + restarted = RunJournal(data).inspect(service, run_id) + assert restarted['result']['tasks'] == first['tasks'] + assert restarted['result']['entries'] == first['entries'] + assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers'] + + def test_second_resume_after_fatal_exit_preserves_verified_progress_and_runs_each_task_once(project_service, monkeypatch): service, _, data, ids, _ = project_service calls = []