fix(dsh): reconcile resumed progress with checkpoint evidence
This commit is contained in:
1 parent
d3bdff81e5
commit
5cb8b6ac83
8 files changed
+187
-3
No files matched your search
@@ -0,0 +1,27 @@
|
||||
# PR #11 自查修复
|
||||
|
||||
## 范围
|
||||
|
||||
以 `9552f1e` 为起点,仅审查 PR #11 的增量,并在 `codex/dsh` 同步其当前目标基线 `6f9f820` 解决冲突。窗口同时保留 DSH 监管和资源状态的初始化、刷新、原生确认门禁与关闭处理。
|
||||
|
||||
## 发现与修复
|
||||
|
||||
1. 恢复运行后,旧暂停回执的计数会使正常新增进度被判为 evidence_changed,阻断再次续跑。现在仅在运行状态、有效原检查点、旧非终态任务、当前无失败/中断且计数单调增长时识别为 expected_progress;旧终态证据变化、暂停期间变化、计数回退仍阻断。
|
||||
2. 检查点已标 completed、当前实际证据却未完成时,原实现可能直接跳过该任务。现在必须有匹配的完整、零失败、无中断证据,否则返回 checkpoint_evidence_mismatch。反向的业务结果已落盘、检查点尚为 running 的退出窗口允许重新核对并续跑。
|
||||
3. 执行后核验适配器只检查中断标记,失败项或不完整队列也可能返回 ready。现在有失败、失败计数未知或队列未完成都返回 uncertain,保留真实数量及独立阻断原因。底层已知执行回执不因补充读回结果而被覆盖。
|
||||
|
||||
## 专项验证
|
||||
|
||||
真实 TaskCenterService、原队列和 SQLite 的 25 项恢复/核验/运行记录测试通过,独立复审同样复跑通过。新增回归先复现失败后验证修复。
|
||||
|
||||
端到端队列回归使用隔离工程 fixture 和受控 cscript process_runner:A 完成后暂停,恢复完成 B 后在最终 journal.finish 前注入 BaseException 模拟退出,重新创建 RunJournal 并恢复 C。断言原 run/queue 不变,A/B/C 各执行一次,实际未命中计数保留,没有重复工程写入。
|
||||
|
||||
CLI 官方运行时门禁、模型结果裁剪和 dock 启动检查通过。未操作真实用户工程、未部署 broker、未构建 EXE。
|
||||
|
||||
DSH/UI/v1_1/打包回归:1255 passed,1 warning(已有 pywinauto STA COM 提示);耗时 382.79 秒。包含 integration 的更大范围探索运行已停止,未计为通过。最终恢复/核验 4 模块另由独立 reviewer 复跑 25 passed,覆盖最后补充的检查点证据一致性门禁。
|
||||
|
||||
## 桌面点击验收
|
||||
|
||||
两进程真实 Qt/CDP 验收通过,报告:build/dsh-pr11-self-review/20261031-144814/report.json。自然语言调用、原生确认、安全暂停、退出后原队列续跑、授权排期、审计损坏隔离及已有工单只读 GET 同步均通过。受控执行器总调用 3 次且步骤唯一,fixture 哈希及损坏审计原字节不变,两个应用进程均正常退出。realCadStarted=false,remoteIssuesCreated=false。未验证真实用户 CAD 写入。
|
||||
|
||||
PR 增量及本轮文件使用本机两份已知凭据逐字节扫描:85 文件、0 匹配,没有输出凭据。
|
||||
@@ -67,6 +67,8 @@ Harness 使用任务中心数据目录下独立的 `dsh/harness-home` 保存会
|
||||
|
||||
最近一次执行回执独立保留,随后查看任务列表不会覆盖它。执行历史根据本地持久记录和任务证据判断可否恢复;重启只展示记录,不自动重放。点击 **核对并恢复原队列** 后仍须原生确认。计划变化、缺少或损坏检查点、未核验中断、失败项等会阻断恢复;待核验结果从任务中心原流程处理。
|
||||
|
||||
恢复执行后若在新回执写入前退出,原队列中新增的已核验进度可继续接续,不会因旧暂停回执的计数较小而重复执行或永久阻断。原先已收尾任务的证据变化、计数回退,以及完成检查点缺少匹配完成证据,仍会阻断。执行后核验只在队列完成且失败为零时报告就绪;未完成或有失败时明确显示待核验,实际保存与未命中数量分别保留。
|
||||
|
||||
原生任务中心的当前任务执行,以及不超过 20 个任务的原生批次,也写入这套运行记录。超过 20 个任务的原生分组继续使用原分组检查点,不加入 DSH 合并运行历史;不要把该分组当作可以从本页恢复的合并运行。
|
||||
|
||||
### 影响范围
|
||||
|
||||
@@ -51,6 +51,10 @@ class CadVerificationAdapter:
|
||||
'interrupted': bool(getattr(context, 'interrupted', True))}
|
||||
if row['interrupted']:
|
||||
blockers.append('interrupted')
|
||||
if type(row['failed_items']) is not int or row['failed_items'] != 0:
|
||||
blockers.append('failed_items')
|
||||
if row['queue_complete'] is not True:
|
||||
blockers.append('queue_incomplete')
|
||||
except Exception as error:
|
||||
row = {'task_id': task_id, 'saved_items': None, 'failed_items': None,
|
||||
'no_match_items': None, 'queue_complete': None, 'interrupted': True,
|
||||
|
||||
@@ -24,16 +24,35 @@ def resume_diff(service, inspected_run, *, current_plan_hashes=None):
|
||||
remaining = list(inspected_run.get('remaining_task_ids', ids))
|
||||
completed = [key for key in ids if key not in remaining]
|
||||
baseline = {row['task_id']: row for row in (inspected_run.get('result') or {}).get('tasks', [])}
|
||||
previous_entries = {row['task_id']: row.get('state')
|
||||
for row in (inspected_run.get('result') or {}).get('entries', [])}
|
||||
evidence_changes = []
|
||||
for task_id, context in contexts.items():
|
||||
saved = baseline.get(task_id, {})
|
||||
actual = {'interrupted': context.interrupted,
|
||||
**{field: getattr(context.progress, field, None) for field in
|
||||
('saved_items', 'failed_items', 'no_match_items', 'queue_complete')}}
|
||||
# A resumed run can crash after verified progress has landed, before
|
||||
# its final receipt. The old pause receipt remains a useful baseline,
|
||||
# but is immutable only for tasks that were already terminal then.
|
||||
progressing = (
|
||||
inspected_run.get('status') == 'running'
|
||||
and inspected_run.get('queue_available') is True
|
||||
and previous_entries.get(task_id) in {'pending', 'paused', 'running', 'blocked'}
|
||||
and inspected_run.get('checkpoint_states', {}).get(task_id) in {'running', 'paused', 'completed'}
|
||||
and saved.get('interrupted') is False and actual['interrupted'] is False
|
||||
and saved.get('queue_complete') is False and type(actual['queue_complete']) is bool
|
||||
and type(saved.get('failed_items')) is int and saved['failed_items'] == 0
|
||||
and type(actual['failed_items']) is int and actual['failed_items'] == 0
|
||||
and all(type(saved.get(key)) is int and type(actual[key]) is int
|
||||
and 0 <= saved[key] <= actual[key] for key in ('saved_items', 'no_match_items')))
|
||||
for field, value in actual.items():
|
||||
if saved.get(field) is not None and value != saved[field]:
|
||||
evidence_changes.append({'task_id': task_id, 'field': field, 'saved': saved[field], 'current': value})
|
||||
if evidence_changes:
|
||||
change = {'task_id': task_id, 'field': field, 'saved': saved[field], 'current': value}
|
||||
if progressing:
|
||||
change['expected_progress'] = True
|
||||
evidence_changes.append(change)
|
||||
if any(not row.get('expected_progress') for row in evidence_changes):
|
||||
blockers.append('evidence_changed')
|
||||
blockers = list(dict.fromkeys(blockers))
|
||||
return {'safe': bool(inspected_run.get('resume_ready')) and not blockers,
|
||||
|
||||
@@ -193,6 +193,11 @@ class RunJournal:
|
||||
blockers.append("interrupted")
|
||||
if getattr(progress, "failed_items", 0):
|
||||
blockers.append("failed_items")
|
||||
if queue and queue['entries'][task_id]['state'] == 'completed' and not (
|
||||
context.has_evidence and progress.queue_complete is True
|
||||
and type(progress.failed_items) is int and progress.failed_items == 0
|
||||
and context.interrupted is False):
|
||||
blockers.append('checkpoint_evidence_mismatch')
|
||||
from ..trial_task_center.task_queue import _plan_hash
|
||||
current_hashes[task_id] = _plan_hash(service.for_project_task(task_id), task_id)
|
||||
if current_hashes[task_id] != row["plan_hashes"][task_id]:
|
||||
@@ -210,7 +215,7 @@ class RunJournal:
|
||||
if remaining:
|
||||
try:
|
||||
from .task_lifecycle import preflight
|
||||
if not preflight(service, remaining)["ready"]:
|
||||
if not preflight(service, remaining, allow_completed=True)["ready"]:
|
||||
blockers.append("preflight_blocked")
|
||||
except Exception:
|
||||
blockers.append("preflight_unavailable")
|
||||
@@ -218,6 +223,7 @@ class RunJournal:
|
||||
blockers.append("resources_pending")
|
||||
blockers = list(dict.fromkeys(blockers))
|
||||
value = {**row, "queue_available": queue is not None, "tasks": tasks, "counters": counters,
|
||||
"checkpoint_states": {key: entry['state'] for key, entry in queue['entries'].items()} if queue else {},
|
||||
"remaining_task_ids": remaining, "blockers": blockers, "resume_ready": not blockers,
|
||||
"requires_confirmation": True,
|
||||
"cad_execution_started": False, "original_scope": {"task_ids": row["task_ids"], "plan_hashes": row["plan_hashes"]}}
|
||||
|
||||
@@ -4,6 +4,7 @@ from types import SimpleNamespace as NS
|
||||
import pytest
|
||||
|
||||
from dx_productivity.dsh.cad_verification import CadVerificationAdapter
|
||||
from test_run_journal import project_service, evidence
|
||||
|
||||
|
||||
def service(tmp_path, *, probe=True, interrupted=False):
|
||||
@@ -43,3 +44,24 @@ def test_adapter_rejects_missing_task_without_inventing_counts(tmp_path):
|
||||
result = CadVerificationAdapter(svc).preflight(['missing'])
|
||||
assert result['status'] == 'blocked'
|
||||
assert result['projects'][0]['component_count'] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('state,blocker', [('batch_failed', 'failed_items'), (None, 'queue_incomplete')])
|
||||
def test_after_run_requires_successful_complete_evidence(project_service, state, blocker):
|
||||
service, _, _, ids, _ = project_service
|
||||
if state:
|
||||
evidence(service, ids[0], state)
|
||||
context = service.task_context(ids[0])
|
||||
assert context.progress.failed_items == (1 if state else 0)
|
||||
assert not context.interrupted
|
||||
result = CadVerificationAdapter(service).after_run(ids[:1])
|
||||
assert result['status'] == 'uncertain'
|
||||
assert blocker in result['blockers']
|
||||
|
||||
|
||||
def test_after_run_keeps_verified_no_match_distinct_from_saved(project_service):
|
||||
service, _, _, ids, _ = project_service
|
||||
evidence(service, ids[0])
|
||||
result = CadVerificationAdapter(service).after_run(ids[:1])
|
||||
assert result['status'] == 'ready' and result['blockers'] == []
|
||||
assert result['tasks'][0]['saved_items'] == 0 and result['tasks'][0]['no_match_items'] == 1
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Original queue recovery across a fatal receipt-write boundary."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from threading import Event
|
||||
|
||||
import pytest
|
||||
|
||||
from dx_productivity.dsh.execution_runs import execute_confirmed_run
|
||||
from dx_productivity.dsh.run_journal import RunJournal
|
||||
from dx_productivity.dsh.task_lifecycle import preflight
|
||||
from test_run_journal import project_service
|
||||
|
||||
|
||||
def test_second_resume_after_fatal_exit_preserves_verified_progress_and_runs_each_task_once(project_service, monkeypatch):
|
||||
service, _, data, ids, _ = project_service
|
||||
calls = []
|
||||
active = {'stop': Event(), 'pause': True}
|
||||
def worker(command, timeout):
|
||||
queue, job = Path(command[3]), command[-1]
|
||||
calls.append(next(key for key in ids if key in queue.parts))
|
||||
with (queue / 'status.tsv').open('a', encoding='utf8') as stream:
|
||||
stream.write(f'{job}\tno_match\tbatch_closed\tControlled verified no match\n')
|
||||
if active['pause']:
|
||||
active['stop'].set()
|
||||
return subprocess.CompletedProcess(command, 0, '', '')
|
||||
for task_id in ids:
|
||||
preparer = service.for_project_task(task_id)._quick_run_preparer
|
||||
preparer._process_runner = worker
|
||||
preparer._cscript_resolver = lambda: Path('cscript.exe')
|
||||
run_id = 'd' * 32
|
||||
first = execute_confirmed_run(service, data, ids, preflight(service, ids), active['stop'], run_id)
|
||||
assert first['status'] == 'paused' and calls == ids[:1]
|
||||
original = RunJournal(data).inspect(service, run_id)
|
||||
active['stop'] = Event()
|
||||
class FatalExit(BaseException):
|
||||
pass
|
||||
with monkeypatch.context() as patch:
|
||||
def fatal_finish(*args, **kwargs):
|
||||
raise FatalExit('exit after checkpoint, before final receipt')
|
||||
patch.setattr(RunJournal, 'finish', fatal_finish)
|
||||
with pytest.raises(FatalExit):
|
||||
execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
||||
active['stop'], run_id, resume=original)
|
||||
assert calls == ids[:2]
|
||||
restarted = RunJournal(data).inspect(service, run_id)
|
||||
assert restarted['status'] == 'running' and restarted['resume_ready'], restarted['blockers']
|
||||
assert restarted['remaining_task_ids'] == ids[2:]
|
||||
active.update(stop=Event(), pause=False)
|
||||
last = execute_confirmed_run(service, data, ids, preflight(service, ids, allow_completed=True),
|
||||
active['stop'], run_id, resume=restarted)
|
||||
assert last['status'] == 'completed' and last['queue_id'] == run_id
|
||||
assert calls == ids
|
||||
assert all(task['saved_items'] == 0 and task['no_match_items'] == 1 for task in last['tasks'])
|
||||
@@ -1,6 +1,9 @@
|
||||
from dx_productivity.dsh.resume_diff import resume_diff
|
||||
from dx_productivity.dsh.run_journal import RunJournal
|
||||
from test_run_journal import project_service, checkpoint, evidence
|
||||
from dx_productivity.trial_task_center.task_queue import _plan_hash
|
||||
from dx_productivity.trial_task_center.models import TaskStatus
|
||||
import pytest
|
||||
|
||||
|
||||
def test_real_service_plan_changes_are_reported(project_service):
|
||||
@@ -46,3 +49,51 @@ def test_missing_current_hash_cannot_be_silently_ignored(project_service):
|
||||
'queue_available': True, 'resume_ready': True, 'tasks': []})
|
||||
assert value['safe'] is False
|
||||
assert value['changed'] == ids
|
||||
|
||||
|
||||
def test_completed_checkpoint_requires_matching_current_evidence(project_service):
|
||||
service, _, data, ids, hashes = project_service
|
||||
journal = RunJournal(data)
|
||||
journal.begin('c' * 32, ids, hashes)
|
||||
checkpoint(data, 'c' * 32, ids, hashes, {ids[0]: 'completed'})
|
||||
report = journal.inspect(service, 'c' * 32)
|
||||
assert not report['resume_ready']
|
||||
assert 'checkpoint_evidence_mismatch' in report['blockers']
|
||||
evidence(service, ids[0])
|
||||
assert journal.inspect(service, 'c' * 32)['resume_ready']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('checkpoint_state', ['running', 'completed'])
|
||||
def test_crash_during_resumed_run_accepts_verified_progress_but_not_old_terminal_changes(project_service, checkpoint_state):
|
||||
service, store, data, ids, _ = project_service
|
||||
hashes = {key: _plan_hash(service.for_project_task(key), key) for key in ids}
|
||||
journal = RunJournal(data)
|
||||
journal.begin('b' * 32, ids, hashes)
|
||||
evidence(service, ids[0])
|
||||
for state in (TaskStatus.READY, TaskStatus.RUNNING, TaskStatus.COMPLETED):
|
||||
store.set_task_status(ids[0], state)
|
||||
checkpoint(data, 'b' * 32, ids, hashes, {ids[0]: 'completed', ids[1]: 'paused'})
|
||||
baseline = []
|
||||
for task_id in ids:
|
||||
context = service.task_context(task_id)
|
||||
baseline.append({'task_id': task_id, 'interrupted': context.interrupted,
|
||||
**{key: getattr(context.progress, key) for key in
|
||||
('saved_items', 'failed_items', 'no_match_items', 'queue_complete')}})
|
||||
journal.finish('b' * 32, {'status': 'paused', 'tasks': baseline,
|
||||
'entries': [{'task_id': key, 'state': 'completed' if key == ids[0] else 'paused' if key == ids[1] else 'pending'} for key in ids]})
|
||||
assert journal.inspect(service, 'b' * 32)['resume_ready']
|
||||
journal.resume('b' * 32)
|
||||
evidence(service, ids[1])
|
||||
for state in (TaskStatus.READY, TaskStatus.RUNNING, TaskStatus.COMPLETED):
|
||||
store.set_task_status(ids[1], state)
|
||||
checkpoint(data, 'b' * 32, ids, hashes, {ids[0]: 'completed', ids[1]: checkpoint_state})
|
||||
report = journal.inspect(service, 'b' * 32)
|
||||
assert report['resume_ready'], report['blockers']
|
||||
assert report['diff']['evidence_changes']
|
||||
assert all(row['expected_progress'] for row in report['diff']['evidence_changes'])
|
||||
# The run cannot rewrite or lose evidence for a task already terminal in
|
||||
# the last durable receipt, even while later tasks legitimately advance.
|
||||
root = service.for_project_task(ids[0])._quick_run_preparer.data_root / 'quick-run'
|
||||
(root / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\n', encoding='utf8')
|
||||
changed = journal.inspect(service, 'b' * 32)
|
||||
assert not changed['resume_ready'] and 'evidence_changed' in changed['blockers']
|
||||
Reference in new issue
Block a user