fix(dsh): retain verified resume progress on checkpoint failure
This commit is contained in:
1 parent
ec7eab89fb
commit
7c29f72ff7
4 files changed
+143
-3
No files matched your search
@@ -69,7 +69,7 @@ Harness 使用任务中心数据目录下独立的 `dsh/harness-home` 保存会
|
||||
|
||||
最近一次执行回执独立保留,随后查看任务列表不会覆盖它。执行历史根据本地持久记录和任务证据判断可否恢复;重启只展示记录,不自动重放。点击 **核对并恢复原队列** 后仍须原生确认。计划变化、缺少或损坏检查点、未核验中断、失败项等会阻断恢复;待核验结果从任务中心原流程处理。
|
||||
|
||||
已结束运行的保存、失败和未命中合计来自持久回执;任一任务缺少某项计数时,该项合计仍显示“未报告”,不会把未知当作零。恢复过程报错时保留上一份任务证据和队列基线,供下一次核对。
|
||||
已结束运行的保存、失败和未命中合计来自持久回执;任一任务缺少某项计数时,该项合计仍显示“未报告”,不会把未知当作零。恢复过程报错时保留上一份任务证据和队列基线。若已产生核验通过的进度,仅在异常收尾时完整核对通过后更新进度基线并留存差异,以便再次确认接续原队列。旧终态变化、失败、未核验结果、计数回退或核对读取失败时不更新基线;停机后新增的证据变化仍会阻断恢复。
|
||||
|
||||
恢复执行后若在新回执写入前退出,原队列中新增的已核验进度可继续接续,不会因旧暂停回执的计数较小而重复执行或永久阻断。原先已收尾任务的证据变化、计数回退,以及完成检查点缺少匹配完成证据,仍会阻断。执行后核验只在队列完成且失败为零时报告就绪;未完成或有失败时明确显示待核验,实际保存与未命中数量分别保留。
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ window.__ModuleLoader__.load({id:'@dx/dsh-task',factory:require=>{
|
||||
const border='1px solid var(--dsw-alias-border-l2)';
|
||||
const buttonStyle={border,background:'var(--dsw-alias-bg-base)',color:'var(--dsw-alias-label-primary)',borderRadius:8,padding:'7px 10px',font:'inherit',fontSize:12,cursor:'pointer'};
|
||||
const stateNames={...names,pending:'待执行',claimed:'已领取',interrupted:'执行中断',cancelled:'已取消',synced:'已同步',not_synced:'尚未同步',unavailable:'暂不可用',open:'处理中',closed:'已关闭',idle:'待命',generating:'生成中',timeout:'响应超时',error:'出错',finished:'已结束'};
|
||||
const blockers={queue_missing:'原队列检查点不存在',queue_invalid:'原队列证据无效',interrupted:'存在待核验步骤',failed_items:'仍有失败项',plan_changed:'保存计划已变化',task_unavailable:'任务不可用',task_paused:'任务已手动暂停,请先核对并接续原执行',not_due:'尚未到执行时间',worker_busy:'执行器忙碌',already_claimed:'排期已被领取',finished:'排期已结束',preflight_blocked:'执行预检未通过',preflight_unavailable:'执行预检不可用',dependency_incomplete:'依赖任务尚未完成并通过核验'};
|
||||
const blockers={queue_missing:'原队列检查点不存在',queue_invalid:'原队列证据无效',interrupted:'存在待核验步骤',failed_items:'仍有失败项',plan_changed:'保存计划已变化',evidence_changed:'执行证据已变化,请核验后再恢复',task_unavailable:'任务不可用',task_paused:'任务已手动暂停,请先核对并接续原执行',not_due:'尚未到执行时间',worker_busy:'执行器忙碌',already_claimed:'排期已被领取',finished:'排期已结束',preflight_blocked:'执行预检未通过',preflight_unavailable:'执行预检不可用',dependency_incomplete:'依赖任务尚未完成并通过核验'};
|
||||
const display=value=>value===null||value===undefined||value===''?'未报告':String(value);
|
||||
const status=value=>stateNames[value]||display(value);
|
||||
const count=value=>Number.isFinite(value)?value.toLocaleString():'未报告';
|
||||
|
||||
@@ -9,6 +9,31 @@ from ..trial_task_center.errors import TaskCenterError
|
||||
from ..trial_task_center.task_queue import execute_task_queue
|
||||
|
||||
|
||||
def _failure_evidence(service, journal, run_id, baseline):
|
||||
evidence = {key: baseline[key] for key in ('tasks', 'entries') if key in baseline}
|
||||
if not baseline:
|
||||
return evidence
|
||||
try:
|
||||
# Inspect while the journal still records this authorized attempt as
|
||||
# running. Only its verified, monotonic progress may advance the old
|
||||
# baseline; terminal changes, failed/uncertain results and rollbacks
|
||||
# must remain visible to the next inspection.
|
||||
inspected = journal.inspect(service, run_id)
|
||||
changes = inspected['diff']['evidence_changes']
|
||||
if (inspected['status'] != 'running' or not inspected['diff']['safe']
|
||||
or not changes or not all(row.get('expected_progress') is True for row in changes)):
|
||||
return evidence
|
||||
tasks = {row['task_id']: dict(row) for row in baseline['tasks']}
|
||||
for change in changes:
|
||||
tasks[change['task_id']][change['field']] = change['current']
|
||||
evidence.update(tasks=list(tasks.values()), verified_progress=changes)
|
||||
except Exception:
|
||||
# Readback failure must neither hide the original execution error nor
|
||||
# replace the last durable evidence with an incomplete snapshot.
|
||||
pass
|
||||
return evidence
|
||||
|
||||
|
||||
def execute_confirmed_run(service, root, task_ids, report, stop, run_id, *, resume=None, schedule=None, automatic=False):
|
||||
prior = RunJournal(root).get(run_id) if resume else None
|
||||
schedule_id = schedule['schedule_id'] if schedule else (prior or {}).get('schedule_id')
|
||||
@@ -47,7 +72,7 @@ def _execute_confirmed_run(service, root, task_ids, report, stop, run_id, *, res
|
||||
queue_id=run_id if resume and resume['queue_available'] else None,
|
||||
checkpoint_id=None if resume and resume['queue_available'] else run_id)
|
||||
except Exception as error:
|
||||
failure = {**{key: baseline[key] for key in ('tasks', 'entries') if key in baseline},
|
||||
failure = {**(_failure_evidence(service, journal, run_id, baseline) if initialized else {}),
|
||||
'status': 'interrupted', 'run_id': run_id, 'queue_id': run_id,
|
||||
'code': getattr(error, 'code', 'execution_interrupted'), 'message': '执行未完成,请按原队列证据核验后接续。'}
|
||||
if initialized:
|
||||
|
||||
@@ -12,6 +12,121 @@ from test_run_journal import project_service
|
||||
from test_run_journal import checkpoint, evidence
|
||||
|
||||
|
||||
def _pause_then_fail_completed_checkpoint(project_service, monkeypatch, *, before_failure=None):
|
||||
from dx_productivity.trial_task_center import task_queue
|
||||
service, _, data, ids, _ = project_service
|
||||
calls = []
|
||||
stop = Event()
|
||||
|
||||
def worker(command, timeout):
|
||||
queue, job = Path(command[3]), command[-1]
|
||||
calls.append(next(key for key in ids if key in queue.parts))
|
||||
with (queue / 'status.tsv').open('a', encoding='utf8') as stream:
|
||||
stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n')
|
||||
if len(calls) == 1:
|
||||
stop.set()
|
||||
return subprocess.CompletedProcess(command, 0, '', '')
|
||||
|
||||
for task_id in ids:
|
||||
preparer = service.for_project_task(task_id)._quick_run_preparer
|
||||
preparer._process_runner = worker
|
||||
preparer._cscript_resolver = lambda: Path('cscript.exe')
|
||||
run_id = 'f' * 32
|
||||
first = execute_confirmed_run(service, data, ids, preflight(service, ids), stop, run_id)
|
||||
assert first['status'] == 'paused' and calls == ids[:1]
|
||||
original = RunJournal(data).inspect(service, run_id)
|
||||
save = task_queue._save
|
||||
|
||||
def fail_completed(path, state):
|
||||
if state['entries'][ids[1]]['state'] == 'completed':
|
||||
if before_failure:
|
||||
before_failure(service, ids)
|
||||
raise OSError('controlled completed checkpoint publication failure')
|
||||
save(path, state)
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(task_queue, '_save', fail_completed)
|
||||
with pytest.raises(OSError, match='checkpoint publication failure'):
|
||||
execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
||||
Event(), run_id, resume=original)
|
||||
assert calls == ids[:2]
|
||||
return run_id, first, calls
|
||||
|
||||
|
||||
def test_resume_after_verified_progress_and_checkpoint_io_failure_runs_each_task_once(project_service, monkeypatch):
|
||||
service, _, data, ids, _ = project_service
|
||||
run_id, first, calls = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
|
||||
restarted = RunJournal(data).inspect(service, run_id)
|
||||
assert restarted['status'] == 'interrupted'
|
||||
assert restarted['checkpoint_states'][ids[1]] == 'running'
|
||||
assert restarted['resume_ready'], restarted['blockers']
|
||||
result = restarted['result']
|
||||
old_terminal = next(row for row in result['tasks'] if row['task_id'] == ids[0])
|
||||
assert old_terminal == next(row for row in first['tasks'] if row['task_id'] == ids[0])
|
||||
advanced = next(row for row in result['tasks'] if row['task_id'] == ids[1])
|
||||
assert advanced['no_match_items'] == 1 and advanced['queue_complete'] is True
|
||||
last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
||||
Event(), run_id, resume=restarted)
|
||||
assert last['status'] == 'completed' and last['queue_id'] == run_id
|
||||
assert calls == ids
|
||||
assert all(row['no_match_items'] == 1 and row['failed_items'] == 0 for row in last['tasks'])
|
||||
|
||||
|
||||
@pytest.mark.parametrize('task_index', [0, 1, 2])
|
||||
def test_soft_failure_snapshot_does_not_allow_evidence_changes_while_stopped(project_service, monkeypatch, task_index):
|
||||
service, _, data, ids, _ = project_service
|
||||
run_id, _, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch)
|
||||
if task_index == 2:
|
||||
evidence(service, ids[task_index])
|
||||
else:
|
||||
root = service.for_project_task(ids[task_index])._quick_run_preparer.data_root / 'quick-run'
|
||||
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
|
||||
restarted = RunJournal(data).inspect(service, run_id)
|
||||
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fault', ['old_terminal', 'failed_result', 'uncertain_result'])
|
||||
def test_checkpoint_failure_does_not_adopt_unsafe_evidence_as_new_baseline(project_service, monkeypatch, fault):
|
||||
service, _, data, ids, _ = project_service
|
||||
|
||||
def corrupt(service, ids):
|
||||
if fault == 'old_terminal':
|
||||
root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run'
|
||||
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
|
||||
else:
|
||||
evidence(service, ids[1], 'batch_failed' if fault == 'failed_result' else 'batch_saved_unverified', '')
|
||||
|
||||
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch, before_failure=corrupt)
|
||||
restarted = RunJournal(data).inspect(service, run_id)
|
||||
assert not restarted['resume_ready']
|
||||
assert restarted['result']['tasks'] == first['tasks']
|
||||
assert 'evidence_changed' in restarted['blockers']
|
||||
|
||||
|
||||
def test_failed_exception_readback_keeps_original_error_and_baseline(project_service, monkeypatch):
|
||||
service, _, data, _, _ = project_service
|
||||
inspect = RunJournal.inspect
|
||||
failing = False
|
||||
|
||||
def unavailable(self, *args, **kwargs):
|
||||
if failing:
|
||||
raise RuntimeError('controlled readback failure')
|
||||
return inspect(self, *args, **kwargs)
|
||||
|
||||
def start_failing(_service, _ids):
|
||||
nonlocal failing
|
||||
failing = True
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(RunJournal, 'inspect', unavailable)
|
||||
run_id, first, _ = _pause_then_fail_completed_checkpoint(project_service, monkeypatch,
|
||||
before_failure=start_failing)
|
||||
restarted = RunJournal(data).inspect(service, run_id)
|
||||
assert restarted['result']['tasks'] == first['tasks']
|
||||
assert restarted['result']['entries'] == first['entries']
|
||||
assert not restarted['resume_ready'] and 'evidence_changed' in restarted['blockers']
|
||||
|
||||
|
||||
def test_second_resume_after_fatal_exit_preserves_verified_progress_and_runs_each_task_once(project_service, monkeypatch):
|
||||
service, _, data, ids, _ = project_service
|
||||
calls = []
|
||||
|
||||
Reference in new issue
Block a user