fix(dsh): retain verified resume progress on checkpoint failure

This commit is contained in:
阿白叔 committed 2026-11-01 18:23:40 +08:00
1 parent ec7eab89fb
commit 7c29f72ff7
4 files changed
+143 -3

No files matched your search

+1 -1
View File
@@ -69,7 +69,7 @@ Harness 使用任务中心数据目录下独立的 `dsh/harness-home` 保存会
最近一次执行回执独立保留,随后查看任务列表不会覆盖它。执行历史根据本地持久记录和任务证据判断可否恢复;重启只展示记录,不自动重放。点击 **核对并恢复原队列** 后仍须原生确认。计划变化、缺少或损坏检查点、未核验中断、失败项等会阻断恢复;待核验结果从任务中心原流程处理。
已结束运行的保存、失败和未命中合计来自持久回执;任一任务缺少某项计数时,该项合计仍显示“未报告”,不会把未知当作零。恢复过程报错时保留上一份任务证据和队列基线,供下一次核对。
已结束运行的保存、失败和未命中合计来自持久回执;任一任务缺少某项计数时,该项合计仍显示“未报告”,不会把未知当作零。恢复过程报错时保留上一份任务证据和队列基线。若已产生核验通过的进度,仅在异常收尾时完整核对通过后更新进度基线并留存差异,以便再次确认接续原队列。旧终态变化、失败、未核验结果、计数回退或核对读取失败时不更新基线;停机后新增的证据变化仍会阻断恢复。
恢复执行后若在新回执写入前退出,原队列中新增的已核验进度可继续接续,不会因旧暂停回执的计数较小而重复执行或永久阻断。原先已收尾任务的证据变化、计数回退,以及完成检查点缺少匹配完成证据,仍会阻断。执行后核验只在队列完成且失败为零时报告就绪;未完成或有失败时明确显示待核验,实际保存与未命中数量分别保留。
+1 -1
View File
@@ -87,7 +87,7 @@ window.__ModuleLoader__.load({id:'@dx/dsh-task',factory:require=>{
const border='1px solid var(--dsw-alias-border-l2)';
const buttonStyle={border,background:'var(--dsw-alias-bg-base)',color:'var(--dsw-alias-label-primary)',borderRadius:8,padding:'7px 10px',font:'inherit',fontSize:12,cursor:'pointer'};
const stateNames={...names,pending:'待执行',claimed:'已领取',interrupted:'执行中断',cancelled:'已取消',synced:'已同步',not_synced:'尚未同步',unavailable:'暂不可用',open:'处理中',closed:'已关闭',idle:'待命',generating:'生成中',timeout:'响应超时',error:'出错',finished:'已结束'};
const blockers={queue_missing:'原队列检查点不存在',queue_invalid:'原队列证据无效',interrupted:'存在待核验步骤',failed_items:'仍有失败项',plan_changed:'保存计划已变化',task_unavailable:'任务不可用',task_paused:'任务已手动暂停,请先核对并接续原执行',not_due:'尚未到执行时间',worker_busy:'执行器忙碌',already_claimed:'排期已被领取',finished:'排期已结束',preflight_blocked:'执行预检未通过',preflight_unavailable:'执行预检不可用',dependency_incomplete:'依赖任务尚未完成并通过核验'};
const blockers={queue_missing:'原队列检查点不存在',queue_invalid:'原队列证据无效',interrupted:'存在待核验步骤',failed_items:'仍有失败项',plan_changed:'保存计划已变化',evidence_changed:'执行证据已变化,请核验后再恢复',task_unavailable:'任务不可用',task_paused:'任务已手动暂停,请先核对并接续原执行',not_due:'尚未到执行时间',worker_busy:'执行器忙碌',already_claimed:'排期已被领取',finished:'排期已结束',preflight_blocked:'执行预检未通过',preflight_unavailable:'执行预检不可用',dependency_incomplete:'依赖任务尚未完成并通过核验'};
const display=value=>value===null||value===undefined||value===''?'未报告':String(value);
const status=value=>stateNames[value]||display(value);
const count=value=>Number.isFinite(value)?value.toLocaleString():'未报告';
+26 -1
View File
@@ -9,6 +9,31 @@ from ..trial_task_center.errors import TaskCenterError
from ..trial_task_center.task_queue import execute_task_queue
def _failure_evidence(service, journal, run_id, baseline):
evidence = {key: baseline[key] for key in ('tasks', 'entries') if key in baseline}
if not baseline:
return evidence
try:
# Inspect while the journal still records this authorized attempt as
# running. Only its verified, monotonic progress may advance the old
# baseline; terminal changes, failed/uncertain results and rollbacks
# must remain visible to the next inspection.
inspected = journal.inspect(service, run_id)
changes = inspected['diff']['evidence_changes']
if (inspected['status'] != 'running' or not inspected['diff']['safe']
or not changes or not all(row.get('expected_progress') is True for row in changes)):
return evidence
tasks = {row['task_id']: dict(row) for row in baseline['tasks']}
for change in changes:
tasks[change['task_id']][change['field']] = change['current']
evidence.update(tasks=list(tasks.values()), verified_progress=changes)
except Exception:
# Readback failure must neither hide the original execution error nor
# replace the last durable evidence with an incomplete snapshot.
pass
return evidence
def execute_confirmed_run(service, root, task_ids, report, stop, run_id, *, resume=None, schedule=None, automatic=False):
prior = RunJournal(root).get(run_id) if resume else None
schedule_id = schedule['schedule_id'] if schedule else (prior or {}).get('schedule_id')
@@ -47,7 +72,7 @@ def _execute_confirmed_run(service, root, task_ids, report, stop, run_id, *, res
queue_id=run_id if resume and resume['queue_available'] else None,
checkpoint_id=None if resume and resume['queue_available'] else run_id)
except Exception as error:
failure = {**{key: baseline[key] for key in ('tasks', 'entries') if key in baseline},
failure = {**(_failure_evidence(service, journal, run_id, baseline) if initialized else {}),
'status': 'interrupted', 'run_id': run_id, 'queue_id': run_id,
'code': getattr(error, 'code', 'execution_interrupted'), 'message': '执行未完成,请按原队列证据核验后接续。'}
if initialized:
+115
View File
@@ -12,6 +12,121 @@ from test_run_journal import project_service
from test_run_journal import checkpoint, evidence
def _pause_then_fail_completed_checkpoint(project_service, monkeypatch, *, before_failure=None):
from dx_productivity.trial_task_center import task_queue
service, _, data, ids, _ = project_service
calls = []
stop = Event()
def worker(command, timeout):
queue, job = Path(command[3]), command[-1]
calls.append(next(key for key in ids if key in queue.parts))
with (queue / 'status.tsv').open('a', encoding='utf8') as stream:
stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n')
if len(calls) == 1:
stop.set()
return subprocess.CompletedProcess(command, 0, '', '')
for task_id in ids:
preparer = service.for_project_task(task_id)._quick_run_preparer
preparer._process_runner = worker
preparer._cscript_resolver = lambda: Path('cscript.exe')
run_id = 'f' * 32
first = execute_confirmed_run(service, data, ids, preflight(service, ids), stop, run_id)
assert first['status'] == 'paused' and calls == ids[:1]
original = RunJournal(data).inspect(service, run_id)
save = task_queue._save
def fail_completed(path, state):
if state['entries'][ids[1]]['state'] == 'completed':
if before_failure:
before_failure(service, ids)
raise OSError('controlled completed checkpoint publication failure')
save(path, state)
with monkeypatch.context() as patch:
patch.setattr(task_queue, '_save', fail_completed)
with pytest.raises(OSError, match='checkpoint publication failure'):
execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
Event(), run_id, resume=original)
assert calls == ids[:2]
return run_id, first, calls
def test_resume_after_verified_progress_and_checkpoint_io_failure_runs_each_task_once(project_service, monkeypatch):
service, _, data, ids, _ = project_service
run_id, first, calls = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
restarted = RunJournal(data).inspect(service, run_id)
assert restarted['status'] == 'interrupted'
assert restarted['checkpoint_states'][ids[1]] == 'running'
assert restarted['resume_ready'], restarted['blockers']
result = restarted['result']
old_terminal = next(row for row in result['tasks'] if row['task_id'] == ids[0])
assert old_terminal == next(row for row in first['tasks'] if row['task_id'] == ids[0])
advanced = next(row for row in result['tasks'] if row['task_id'] == ids[1])
assert advanced['no_match_items'] == 1 and advanced['queue_complete'] is True
last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
Event(), run_id, resume=restarted)
assert last['status'] == 'completed' and last['queue_id'] == run_id
assert calls == ids
assert all(row['no_match_items'] == 1 and row['failed_items'] == 0 for row in last['tasks'])
@pytest.mark.parametrize('task_index', [0, 1, 2])
def test_soft_failure_snapshot_does_not_allow_evidence_changes_while_stopped(project_service, monkeypatch, task_index):
service, _, data, ids, _ = project_service
run_id, _, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
if task_index == 2:
evidence(service, ids[task_index])
else:
root = service.for_project_task(ids[task_index])._quick_run_preparer.data_root / 'quick-run'
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
restarted = RunJournal(data).inspect(service, run_id)
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
@pytest.mark.parametrize('fault', ['old_terminal', 'failed_result', 'uncertain_result'])
def test_checkpoint_failure_does_not_adopt_unsafe_evidence_as_new_baseline(project_service, monkeypatch, fault):
service, _, data, ids, _ = project_service
def corrupt(service, ids):
if fault == 'old_terminal':
root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run'
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
else:
evidence(service, ids[1], 'batch_failed' if fault == 'failed_result' else 'batch_saved_unverified', '')
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch, before_failure=corrupt)
restarted = RunJournal(data).inspect(service, run_id)
assert not restarted['resume_ready']
assert restarted['result']['tasks'] == first['tasks']
assert 'evidence_changed' in restarted['blockers']
def test_failed_exception_readback_keeps_original_error_and_baseline(project_service, monkeypatch):
service, _, data, _, _ = project_service
inspect = RunJournal.inspect
failing = False
def unavailable(self, *args, **kwargs):
if failing:
raise RuntimeError('controlled readback failure')
return inspect(self, *args, **kwargs)
def start_failing(_service, _ids):
nonlocal failing
failing = True
with monkeypatch.context() as patch:
patch.setattr(RunJournal, 'inspect', unavailable)
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch,
before_failure=start_failing)
restarted = RunJournal(data).inspect(service, run_id)
assert restarted['result']['tasks'] == first['tasks']
assert restarted['result']['entries'] == first['entries']
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
def test_second_resume_after_fatal_exit_preserves_verified_progress_and_runs_each_task_once(project_service, monkeypatch):
service, _, data, ids, _ = project_service
calls = []