From 4db5f05531cbae7b9723ddf27ecb7a77ddf943e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E7=99=BD=E5=8F=94?= <11794527+absfan@user.noreply.gitee.com> Date: Fri, 6 Nov 2026 13:29:56 +0800 Subject: [PATCH] fix: isolate component failures and recover owned CAD resources --- .gitea/workflows/ci.yml | 13 + ...2026-11-06-nonblocking-repair-checklist.md | 71 +++ docs/testing/2026-11-06-component-scope.md | 52 ++ docs/testing/2026-11-06-nonblocking-ai.md | 54 ++ .../2026-11-06-nonblocking-bom-values.md | 38 ++ ...026-11-06-nonblocking-repair-acceptance.md | 79 +++ .../2026-11-06-nonblocking-resources.md | 90 +++ docs/testing/2026-11-06-project-health.md | 55 ++ docs/user/nonblocking-item-execution.md | 13 + .../dxdesigner795/NativeSnapshotReader.vbs | 5 +- .../dxdesigner795/QuickPartReplace.vbs | 122 +++- src/dx_productivity/dsh/execution_runs.py | 2 +- src/dx_productivity/dsh/incidents.py | 103 ++-- src/dx_productivity/dsh/resume_diff.py | 3 +- src/dx_productivity/dsh/run_journal.py | 64 +- .../trial_task_center/batch.py | 47 +- .../trial_task_center/batch_resources.py | 184 +++++- .../trial_task_center/batch_runtime.py | 173 +++++- .../trial_task_center/batch_supervisor.py | 154 ++++- .../trial_task_center/batch_worker.py | 65 +- .../trial_task_center/bom_batch.py | 12 + .../trial_task_center/bom_import_report.py | 95 ++- .../trial_task_center/bom_import_state.py | 8 +- .../trial_task_center/bom_plan.py | 46 +- .../trial_task_center/cad_dialogs.py | 29 +- .../trial_task_center/cell_catalog.py | 57 +- .../trial_task_center/component_scope.py | 165 +++++ .../trial_task_center/diagnostics_bundle.py | 12 +- .../trial_task_center/discovery_cache.py | 25 +- .../trial_task_center/execution_journal.py | 183 +++++- .../trial_task_center/execution_problems.py | 100 +++ .../trial_task_center/execution_proof.py | 10 + .../trial_task_center/inventory.py | 14 +- .../trial_task_center/isolated_cad.py | 71 ++- .../trial_task_center/isolated_desktop.py | 18 + .../native_inventory_protocol.py | 5 +- .../trial_task_center/problem_pass.py | 61 ++ .../project_session_health.py | 235 +++++++ .../trial_task_center/quick_run.py | 215 ++++++- .../trial_task_center/read_recovery.py | 33 + .../trial_task_center/service.py | 57 +- .../trial_task_center/store_read_model.py | 3 +- .../trial_task_center/store_schema.py | 3 + .../trial_task_center/store_target_plan.py | 39 +- .../trial_task_center/supervisor_result.py | 8 +- .../trial_task_center/target_plan.py | 5 + .../trial_task_center/task_queue.py | 26 +- .../ui/cad_dialog_operations.py | 82 +++ .../trial_task_center/ui/detail_windows.py | 4 +- .../trial_task_center/ui/dsh_automation.py | 124 +++- .../trial_task_center/ui/dsh_operations.py | 43 +- .../ui/dsh_task_lifecycle.py | 3 +- .../trial_task_center/ui/main_window.py | 38 +- .../trial_task_center/ui/messages.py | 15 +- .../trial_task_center/ui/project_plan.py | 55 +- .../trial_task_center/value_readback.py | 166 +++++ .../trial_task_center/worker_monitor.py | 132 +++- .../trial_task_center/workspace_bom.py | 24 +- .../trial_task_center/workspace_execution.py | 202 ++++++ tests/dsh/test_incident_admission.py | 96 +++ tests/dsh/test_run_journal.py | 60 ++ .../trial_task_center/test_worker_monitor.py | 8 +- tests/ui/test_dsh_ai_nonblocking.py | 240 ++++++++ tests/ui/test_dsh_operations.py | 7 +- tests/ui/test_nonblocking_controls.py | 38 ++ tests/v1_1/test_accuracy_vbs_guards.py | 11 +- tests/v1_1/test_batch_runtime.py | 21 +- tests/v1_1/test_bom_actual_metadata.py | 145 +++++ tests/v1_1/test_cad_dialog_operations.py | 26 + tests/v1_1/test_component_scope.py | 254 ++++++++ tests/v1_1/test_deferred_problem_pass.py | 93 +++ tests/v1_1/test_diagnostics_bundle.py | 25 + tests/v1_1/test_inventory_extension.py | 2 +- tests/v1_1/test_manual_dialog_native.py | 33 + tests/v1_1/test_missing_uid_inventory.py | 31 +- tests/v1_1/test_native_snapshot_reader.py | 2 +- tests/v1_1/test_nonblocking_acceptance.py | 80 +++ tests/v1_1/test_nonblocking_bom_values.py | 184 ++++++ tests/v1_1/test_nonblocking_items.py | 145 +++++ tests/v1_1/test_nonblocking_resources.py | 574 ++++++++++++++++++ tests/v1_1/test_nonblocking_resume.py | 73 +++ tests/v1_1/test_nonblocking_value_vbs.py | 170 ++++++ tests/v1_1/test_project_session_health.py | 271 +++++++++ tests/v1_1/test_retained_write_owners.py | 332 ++++++++++ tests/v1_1/test_supervisor_result.py | 12 + tests/v1_1/test_value_readback.py | 87 +++ tests/v1_1/test_workspace_execution.py | 159 +++++ 87 files changed, 6688 insertions(+), 301 deletions(-) create mode 100644 docs/superpowers/plans/2026-11-06-nonblocking-repair-checklist.md create mode 100644 docs/testing/2026-11-06-component-scope.md create mode 100644 docs/testing/2026-11-06-nonblocking-ai.md create mode 100644 docs/testing/2026-11-06-nonblocking-bom-values.md create mode 100644 docs/testing/2026-11-06-nonblocking-repair-acceptance.md create mode 100644 docs/testing/2026-11-06-nonblocking-resources.md create mode 100644 docs/testing/2026-11-06-project-health.md create mode 100644 docs/user/nonblocking-item-execution.md create mode 100644 src/dx_productivity/trial_task_center/component_scope.py create mode 100644 src/dx_productivity/trial_task_center/execution_problems.py create mode 100644 src/dx_productivity/trial_task_center/problem_pass.py create mode 100644 src/dx_productivity/trial_task_center/project_session_health.py create mode 100644 src/dx_productivity/trial_task_center/read_recovery.py create mode 100644 src/dx_productivity/trial_task_center/ui/cad_dialog_operations.py create mode 100644 src/dx_productivity/trial_task_center/value_readback.py create mode 100644 src/dx_productivity/trial_task_center/workspace_execution.py create mode 100644 tests/dsh/test_incident_admission.py create mode 100644 tests/ui/test_dsh_ai_nonblocking.py create mode 100644 tests/ui/test_nonblocking_controls.py create mode 100644 tests/v1_1/test_bom_actual_metadata.py create mode 100644 tests/v1_1/test_cad_dialog_operations.py create mode 100644 tests/v1_1/test_component_scope.py create mode 100644 tests/v1_1/test_deferred_problem_pass.py create mode 100644 tests/v1_1/test_manual_dialog_native.py create mode 100644 tests/v1_1/test_nonblocking_acceptance.py create mode 100644 tests/v1_1/test_nonblocking_bom_values.py create mode 100644 tests/v1_1/test_nonblocking_items.py create mode 100644 tests/v1_1/test_nonblocking_resources.py create mode 100644 tests/v1_1/test_nonblocking_resume.py create mode 100644 tests/v1_1/test_nonblocking_value_vbs.py create mode 100644 tests/v1_1/test_project_session_health.py create mode 100644 tests/v1_1/test_retained_write_owners.py create mode 100644 tests/v1_1/test_value_readback.py create mode 100644 tests/v1_1/test_workspace_execution.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 43551ab..3e9e1d1 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -43,6 +43,19 @@ jobs: tests/trial_task_center/test_bom_import_review.py tests/v1_1/test_cad_dialogs.py tests/v1_1/test_lxb_blocking_recovery.py + tests/v1_1/test_nonblocking_items.py + tests/v1_1/test_component_scope.py + tests/v1_1/test_deferred_problem_pass.py + tests/v1_1/test_nonblocking_acceptance.py + tests/v1_1/test_nonblocking_resume.py + tests/v1_1/test_project_session_health.py + tests/v1_1/test_bom_actual_metadata.py + tests/v1_1/test_manual_dialog_native.py + tests/v1_1/test_workspace_execution.py + tests/v1_1/test_nonblocking_bom_values.py + tests/v1_1/test_value_readback.py + tests/dsh/test_run_journal.py + tests/dsh/test_incident_admission.py --junitxml=build/ci-results.xml --tb=short - name: Windows release scope run: | diff --git a/docs/superpowers/plans/2026-11-06-nonblocking-repair-checklist.md b/docs/superpowers/plans/2026-11-06-nonblocking-repair-checklist.md new file mode 100644 index 0000000..0a29a91 --- /dev/null +++ b/docs/superpowers/plans/2026-11-06-nonblocking-repair-checklist.md @@ -0,0 +1,71 @@ +# 逐位号非阻塞修复与独立验收清单 + +**Goal:** 一个明确失败位号不挡住可安全执行的独立位号;工程不确定时隔离该工程,确认资源释放后继续其他工程。 + +**Architecture:** 复用现有执行账本、保存回读、恢复事务和进程归属证明。新增位号粒度、问题轮次与工程隔离;AI 等待退出执行通道。工作树基线为 8b4e8ee。 + +**Tech Stack:** Python 3.12、SQLite、PySide6、Windows 自有 Job/私有桌面、现有 VBS 原生执行器。 + +**Spec:** 本清单落实 2026-11-06 用户确认的筛查结论;排除独立校验工作台及其分支。 + +## 约束和默认选择 + +- 不确定写入不自动重放。自动请求安全退出;资源确认释放后可继续其他独立工程;卡死时强制结束需要操作者明确确认。 +- 保留强制备份、UID/目标范围、保存与回读、旧尝试拒绝、原子恢复发布和源表来源信息。 +- 每工程正常项复用会话,工程间串行切换;禁止任意多开 CAD,禁止清理外部 CAD。 +- 新计划按位号执行;旧冻结整组回执保持原义,未决组必须先核验,不通过改 key 获得重放权限。 +- Block/Symbol 属性来源尚未确认;仅加证据、歧义拒绝和样本,不推断层级、不以 Value 相同合并身份。 +- BOM 允许显式接受通过的工程;默认仍整批确认。已是目标必须同时满足 Value/Cell 要求,否则不能假报成功。 +- 自动问题重试限原 task/round/目标意图一次;首轮后进行;未知/确定性/不确定写入不自动重试。AI 无权放宽门禁。 +- 仅新增功能的独立验证和必要受影响回归;用受控原生边界、真实 SQLite/队列、一次性进程/弹窗验证,不重复真实 CAD/PCB。 + +## 修复与验收 list + +| ID | 需要修复/补齐 | 修复完成的独立验收标准 | +| --- | --- | --- | +| R01 | 新计划逐位号冻结和派发 | 同规则 R1/R2 各有单目标命令、独立回执;Value 相同仍为两目标 | +| R02 | 旧组回执兼容 | 已成功/未决组不被拆分重放;旧队列恢复保留冻结身份 | +| R03 | 同工程安全失败继续 | R1 明确失败、R2 独立成功;成功项只派发一次 | +| R04 | 位号继续健康准入 | 写入不确定、目标重叠、基线变化不能凭重启 CAD 解除 | +| R05 | 问题身份与分类 | 重绑 job/attempt、重启不生成新逻辑问题或新额度 | +| R06 | 持久首轮/问题轮次 | 首轮所有可运行项先执行;之后仅合规问题项最多重试一次 | +| R07 | 重试统一授权 | DSH 与问题调度竞争同一额度,实际仅派发一次;暂停不清额度 | +| R08 | 逐工程监督与检查点 | A 异常后记录 A;确认释放后 B 继续,冻结范围/顺序不变 | +| R09 | 工程独立 CAD 生命周期 | A/B 不共享 CAD;同工程正常项复用;无并行窗口累积 | +| R10 | 不确定状态与资源占用分离 | 进程确认退出可结束等待,但工程仍待核验且不能自动重放 | +| R11 | 业务进展监控 | 仅心跳不延长业务停滞;慢保存不被自动强杀 | +| R12 | 只读失败完整释放 | 读取超时/取消后关闭整代,确认后跳过;不重复打开失败工程 | +| R13 | 所有者失联及重启恢复 | 父进程退出只回收证明归属的只读残留;PID 复用/写操作拒绝 | +| R14 | 未知弹窗人工入口 | 能识别工程、显示窗口内容和待处理原因;提供受控处理入口并复核 | +| R15 | 退出观察覆盖所有等待路径 | worker/runtime/CAD 都可因自有进程确实退出结束等待;不清除 uncertainty | +| R16 | 资源残留再准入 | 有进程/桌面释放证明才解除资源挡板;坏证据仍阻断且可诊断 | +| R17 | AI 分析与网络等待解耦 | 模型等待/反思/工单发布时可执行独立任务;实际变更临时独占 | +| R18 | 问题结果汇总 | 成功/失败/暂缓/待核验分开;批次结束不误报全部成功 | +| R19 | BOM 按工程显式部分接收 | 用户明确选择通过工程后仅保存该范围;失败行和来源完整保留 | +| R20 | 已是目标识别 | 目标料号且 Value/Cell 均正确时跳过;属性不符明确报告、不能假完成 | +| R21 | 中心库目标预检 | 真实连接业务的只读预检在写入前发现目标/关联问题;缺工具不谎报已查 | +| R22 | 缓存内容与释放后复核 | 元数据变化可复核;内容变化/不可读不复用;失败释放后不陷重复读取 | +| R23 | 实际 Value 报告 | 报告绑定当前 job/UID/回执,取保存后实际值;旧/外国报告拒绝 | +| R24 | Block/Symbol 歧义证据 | 同值不去重;属性来源可记录;未知来源不猜写;样本明确当前支持边界 | +| R25 | 混合批次闭环 | 100 项混合故障核对真实派发/账本;成功不重做、不确定不重试、独立项完成 | + +## 执行单元与验证入口 + +- [x] T1:R01~R07、R18。修改 target_plan、quick_run、batch、execution_journal、task_queue、service;新建 execution_problems/problem_pass(需要时)。先用真实账本/受控 worker 复现 R1 失败拖住 R2,再实现逐位号、旧回执兼容、持久问题轮次。测试 test_nonblocking_items、test_deferred_problem_pass 和受影响旧组。 +- [x] T2:R08~R16、R22。修改 batch_runtime、batch_resources、batch_supervisor、batch_worker、worker_monitor、isolated_cad、discovery_cache;添加工程监督/所有者关联最小模块。先写资源归属、退出观察和逐工程超时样本,再实现;不以固定超时授权杀写进程。测试 test_nonblocking_resources 和必要 monitor/runtime/只读恢复组。 +- [x] T3:R17。移植 PR19 的 AI 等待/变更准入分离,保留当前身份与取消保护;仅修改 dsh_automation、dsh_operations、dsh/incidents 和相关 UI 测试。模型永久未返回时独立任务仍可派发;真实 mutation 必须重新检查准入。 +- [x] T4:R19~R21、R23~R24。修改 BOM 匹配/预览/保存合同与现有导入 UI,增加实际 Value 报告和来源证据。默认保持原子整批;显式部分接收必须绑定原预览与源表版本。先用 Excel/报告样本失败复现,测试真实预览/事务及 VBS 边界。 +- [x] T5:R25 与集成审阅。整合各独立结果,补混合批次和崩溃恢复;按 R01~R25 逐项记录证据,运行一次必要集成集合;独立审阅并修复真实发现。 + +每单元遵循:失败复现 → 最小实现 → 对应功能通过 → 必要邻接回归 → 更新清单证据。未有证据的项不勾选完成。 + +## 验收记录 + +- 基线:test_task_queue + test_safe_cbb_continue,9 passed(保留旧行为对照)。 +- 修复证据由各单元追加到 docs/testing/2026-11-06-nonblocking-repair-acceptance.md;结果不累加不同版本的重复测试数。 + +## 完成记录 + +R01~R25 已完成本轮开发、独立样本验证及限定代码复核。逐项证据见 [验收索引](../../testing/2026-11-06-nonblocking-repair-acceptance.md)。最后集成集合26 passed;独立连接额度竞争及中断恢复2 passed;BOM最后缺元数据与身份保护8 passed。不同阶段结果不累计。 + +真实 CAD/PCB 未重复执行,安装包和远程 CI 未在本轮运行;Block/Symbol 所属层级保持未知,交付的是证据保留与歧义写入保护。无法证明归属或释放的遗留资源仍保留人工核验门禁,不能把清理强制化作为完成标准。 diff --git a/docs/testing/2026-11-06-component-scope.md b/docs/testing/2026-11-06-component-scope.md new file mode 100644 index 0000000..c65e306 --- /dev/null +++ b/docs/testing/2026-11-06-component-scope.md @@ -0,0 +1,52 @@ +# R01: freeze new raw all-match rules into per-UID commands + +New native `ensure_batch_handoff` calls request `collect_components=True` for raw rules as well as target rules. Validated read-only discovery expands each matching component into a separate exact UID command, including components with equal Values. The existing discovery cache, native/COM reader validation, reader-release checks, runtime forwarding and `release_idle_for_fingerprint` hook remain in use. + +`component-scope.json` records the ordered original rule/options signature, base Value policy, project/sheet scope and captured component identities. Snapshot validation regenerates commands from that frozen inventory and the original rules. Resume and retry do not rediscover successful targets whose Part Numbers have already changed. Original custom Value, NC and Cell option changes are rejected. + +An unmatched source rule produces a separate `execution_unit=inventory_empty`, empty-selector record with a preterminal `batch_skipped / inventory_empty` manual issue. It never reports a saved replacement. Other matched UIDs continue using their own commands. The dispatch boundary independently rejects inventory-empty/empty-target commands even if their terminal status disappears. No UID is invented. Missing or inconsistent actual component manifests fail closed rather than generating all-match mutations. + +Existing unscoped raw queues and grouped target queues retain their original commands. Retry preparation inherits both the frozen scope and recorded split/group shape through `prepare(..., component_scope_from=original)`. If a converted raw queue loses its sidecar, retry refuses to turn it back into an all-match queue. + +Parent integration hooks are implemented outside this unit: service passes `component_scope_from=original`; `execution_proof.capture_result` calls `component_scope.validate_empty_command(root, row)`, copies/hashes the sidecar into immutable proof, and marks the proof as inventory-empty; journal continuation accepts that manual no-mutation failure only with nonempty intact proof. The helper validates the entire active regenerated command queue against the sidecar. + +## Test-first evidence + +```text +python -m pytest tests/v1_1/test_component_scope.py -q --basetemp=F:/dx-nb-exp-1 -o cache_dir=build/pytest-component-scope-1 +8 failed, 1 passed in 3.58s +``` + +The initial failures showed raw discovery requesting no components, grouped commands, changed original options silently rebuilding, absent frozen resume, missing retry inheritance and incomplete inventories falling back to all-match behavior. The passing case established the legacy queue boundary. + +After the minimum frozen-scope implementation: + +```text +python -m pytest tests/v1_1/test_component_scope.py -q --basetemp=F:/dx-nb-exp-2 -o cache_dir=build/pytest-component-scope-2 +9 passed in 1.35s +``` + +Additional boundary cases were red before their fixes: + +```text +python -m pytest tests/v1_1/test_component_scope.py -q -k "missing_terminal or empty_command_proof" --basetemp=F:/dx-nb-exp-3 -o cache_dir=build/pytest-component-scope-3 +2 failed, 9 deselected in 2.05s + +python -m pytest tests/v1_1/test_component_scope.py::test_retry_missing_frozen_sidecar_cannot_become_an_all_match_queue -q --basetemp=F:/dx-nb-exp-8 -o cache_dir=F:/dx-nb-exp-cache-8 +1 failed in 1.73s +``` + +The first two exposed empty-selector dispatch after loss of terminal status, and a proof helper accepting one empty row despite changed inventory elsewhere in its queue. The last exposed a missing-sidecar retry reverting to all-match commands. + +Actual service/SQLite/journal checks confirmed that `ABSENT + OLD` dispatches exactly `uid-1, uid-2`, retains one manual issue and two saved receipts, and never redispatches on queue reopen. A controlled prewrite failure on `uid-1` followed by successful `uid-2`, shared DSH retry reservation and retry preparation dispatches exactly `uid-1, uid-2, uid-1` while retaining the original sidecar. Initial integration fixture failures were corrected by providing a valid DxD project and keeping data outside the backed-up project directory. + +## Affected verification + +```text +python -m pytest tests/v1_1/test_component_scope.py tests/v1_1/test_target_execution.py tests/v1_1/test_retry_recovery_transactions.py tests/v1_1/test_nonblocking_items.py tests/trial_task_center/test_quick_run_snapshot.py tests/v1_1/test_discovery_cache.py::test_handoff_reuses_collected_inventory_without_second_cad_open tests/v1_1/test_discovery_cache.py::test_failed_worker_invalidates_before_it_runs tests/v1_1/test_nonblocking_resources.py::test_cache_hashes_after_verified_reader_release -q --basetemp=F:/dx-nb-exp-9 -o cache_dir=F:/dx-nb-exp-cache-9 +55 passed in 13.97s +git diff --check +exit 0 (no output) +``` + +No real CAD/PCB, external AI, network tickets, or redundant full suites were run. Native interaction uses controlled worker/discovery boundaries; actual filesystem queues, SQLite receipts, backup/publication paths and dispatch counts were exercised. An earlier cache path under the synchronized workspace produced one Windows access warning; subsequent verification uses unique independent paths on `F:`. No source rules or existing grouped receipts were rewritten. diff --git a/docs/testing/2026-11-06-nonblocking-ai.md b/docs/testing/2026-11-06-nonblocking-ai.md new file mode 100644 index 0000000..831a580 --- /dev/null +++ b/docs/testing/2026-11-06-nonblocking-ai.md @@ -0,0 +1,54 @@ +# R17: AI waits and execution admission + +Scope: T3 in `2026-11-06-nonblocking-repair-checklist.md`. Adapt the applicable PR19 admission/pool changes without importing its model-readiness UI or replacing current protections. + +## Behavior + +- Diagnosis, reflection, issue publication/resubmission, operations history/health/experience/impact reads, ticket synchronization, and schedule preparation use application-owned independent Qt pools. They do not enter the window execution worker set. +- Operations still publish their own activity/busy state, while native execution confirmation can proceed during an operations read. Schedule preparation rechecks GUI activity before opening a confirmation or dispatching work. +- Actual retry obtains GUI admission, checks cancellation, current run identity, dirty plans, open confirmations, loading and existing execution. The runner then compares the current task context against immutable pre-model evidence before reserving any retry. +- Admission spans preparation, actual execution and verification. Explicit GUI release acknowledgement precedes reflection/publication, including when GUI release takes longer than five seconds. A timed-out acquisition cannot acquire later. +- Finishing an old diagnosis preserves a newer manual run's stop object, owner and live state. Close cancels model waiters and waits for the AI worker before releasing application resources; cancelling the window close keeps subsequent automation available. Evidence remains pending and cancellation does not start a ticket submission. +- New service integration: `reserve_problem_retry(task_id, source='dsh')` returns the exact authorized operation keys, forwarded as `prepare_failed_quick_run(task_id, operation_keys=reserved)`. An empty reservation denies execution. The old exclusive retry-budget file is used only by legacy adapters without the shared API. + +## Red/green evidence + +Initial test-first command: + +```text +python -m pytest tests/dsh/test_incident_admission.py tests/ui/test_dsh_ai_nonblocking.py -q +10 failed, 3 errors in 22.83s +``` + +The expected failures showed rejected admission still executing, changed model-time context still executing, retry scope missing, cancellation unsupported, shared retry denial ignored, operations reads reserving the lane, old diagnosis clearing a new manual run, and close not waking the waiter. The errors exposed baseline Qt widget teardown during blocked diagnosis plus one fixture ordering error; fixture order and cleanup were corrected. + +Initial focused green after minimal implementation: + +```text +python -m pytest tests/dsh/test_incident_admission.py tests/ui/test_dsh_ai_nonblocking.py -q +13 passed in 5.58s +``` + +Shared reservation forwarding was independently red (`[None] != [('uid-operation',)]`), then passed in the affected regressions: + +```text +python -m pytest tests/dsh/test_incidents.py tests/dsh/test_incident_admission.py tests/ui/test_dsh_ai_nonblocking.py tests/ui/test_dsh_operations.py tests/ui/test_dsh_native_menu.py tests/ui/test_dsh_task_lifecycle.py -q +79 passed in 55.62s +``` + +Final concurrency review added targeted red cases: empty shared reservation incorrectly left `retry_budget_available=True`; delayed GUI release let reflection start before acknowledgement (`1 failed in 7.73s`); cancelled close left idle/finished automation permanently closing (`2 failed`). These justify only the subsequent focused rerun, not repeating the 79-test regression set. + +Final focused verification after those fixes and Qt fixture cleanup: + +```text +python -m pytest tests/dsh/test_incident_admission.py tests/ui/test_dsh_ai_nonblocking.py -q +17 passed in 17.85s +git diff --check +exit 0 (no output) +``` + +## Limits + +Tests use real Qt worker scheduling, controlled model waiters and service doubles, plus the existing UI SQLite/service fixtures. No real AI, remote ticket creation, CAD, PCB or redundant full suites were run. The shared retry API is supplied by T1 and needs the parent task's integration verification with actual per-operation journals. A ticket request already in progress must finish or hit its existing network timeout before close completes; it does not reserve task execution while waiting. + +`src/dx_productivity/dsh/task_lifecycle.py` and the UI lifecycle controller were not changed. diff --git a/docs/testing/2026-11-06-nonblocking-bom-values.md b/docs/testing/2026-11-06-nonblocking-bom-values.md new file mode 100644 index 0000000..7145c6b --- /dev/null +++ b/docs/testing/2026-11-06-nonblocking-bom-values.md @@ -0,0 +1,38 @@ +# T4:BOM 部分导入、目标库与实际 Value 证据 + +工作目录:`.worktrees/codex-nonblocking-items`。验证全部使用临时 Excel、真实 SQLite、COM 形状假对象和 Windows `cscript.exe` 执行生产 VBS 例程;未打开真实 CAD、PCB、Library Manager 或 AI。 + +## 行为与边界 + +- **R19**:默认整批校验仍为原子操作,Excel 前置错误不会启动采集。显式勾选“允许部分导入通过工程”后只核验没有前置错误的工程;返回报告不会自动暂存部分计划,必须在“选择通过工程…”中主动勾选。多来源的公式(包括未映射额外列中的公式)、数值 PN、非法 Value mode 保留为各自工程的失败行,不阻止其他完整通过工程;默认模式仍在任何采集前拒绝。只接收完整通过工程,失败/未检查行、原工作簿和行号保留。部分接收保存时要求原预览行完全一致,重新验证所有原工作簿 SHA-256、source-map SHA-256、工程成员和原库存;任何不符均拒绝。临时归一化工作簿删除后,绑定的是原工作簿。损坏工作簿、错误表头、相对工程路径或无法唯一归属的来源仍拒绝进入部分采集。 +- **R20**:首次 BOM 采集同时请求源、目标 PN;普通源 PN 计划可复用已保存库存,但任何启用的“已到目标”跳过判定必须再次经过只读发现,不能把数据库里的旧读数当作当前证据。发现缓存仅可凭当前源内容指纹复用。`ComponentTarget` 增加可选实际 Part Label / Cell;官方 FLAT 生效报告解析其有效值,CAD 只读采集输出扩展 9 列,旧 7 列仍按未知接收。缺少所需证据时失效发现缓存并重新只读采集一次,仍不支持则明确报问题。实际 PN 已为目标时,四种 Value 策略及指定 Cell 都按实际读数判定,匹配生成禁用跳过行,不符拒绝,绝不使用计划 Cell 冒充实际 Cell。SQLite 可空列迁移保留未知状态,未知到已知的补证保留用户编辑和原 UID/ref/PN/Value 基线,已知证据冲突拒绝并记为工程失败行。真实 `QPCaptureAndSelect` 同样识别精确 UID 已到目标,实际 Value/Cell 符合时返回 `no_match / already_target`;不符或属性歧义直接拒绝。冻结旧分组中混有源和目标器件时明确要求逐器件处理,不声称整组完成。 +- **R21**:`preflight_command_target` 调用现有物理库克隆、只读隔离读取和清理证明链;即使没有指定 Cell 也读取目标 PN,指定 Cell 必须属于该目标。源库内容/依赖指纹绑定缓存证据。缺件、歧义或关联错误在证明资源释放后转为 `target_cell_catalog`;没有正面桌面/Job 释放证据则保留 `batch_resources_pending`,不当作可继续的业务失败。主流程已接在 journal claim 后、原生启动/修改前,安全目标错误记独立失败回执。 +- **R23**:原生保存重开核验、人工恢复核验和只读恢复核验后,写每 job 的 v2 `value-readback-.tsv`。绑定 task/job/终态、完整 PN/Value/NC/Cell/UID 选择、当前快照原文;记录实际 PN、Value 存在性、实际 Value 和 Cell。启动前清除同 job 旧观测;Python 依据当前终态、精确 UID 集和当前快照验证,再原子保存 JSON。后续读取复核命令、终态、原始报告与快照哈希;旧/外国/不完整报告返回 rejected,缺报告返回 unavailable,不填计划值。显示表增加原 Value / 实际 Value 列,缺证据明确提示。 +- **R24**:所有枚举属性按 UID 和 ordinal 保留原始 `Attr.Value` / `Attr.InstanceValue`,包括相同名称、相同文本的两个观察。层级固定记录 `unknown`,不把任一属性 API 等同于 Block 或 Symbol。选中器件写前和恢复事务前检查同名属性;歧义原始观察写入 sidecar 后拒绝修改。原有单一组件属性 InstanceValue 行为保留;**未证明 Block/Symbol 所有权,也未新增任何推测性的层级写入功能**。 + +## 验证记录 + +按失败用例先行验证了缺失的部分接收 API、严格原行/来源校验、目标库执行适配错误分类、已到目标的 BOM 匹配、实际 Value 报告、报告表格列和原生属性守卫,再最小实现。 + +- 新增定向组:`test_nonblocking_bom_values.py`、`test_value_readback.py`、`test_nonblocking_value_vbs.py`,连同 BOM 状态机:**34 passed**。 +- BOM 既有行为、来源映射、迁移、每项 Value、原子确认:**90 passed**(加入最后两项定向来源/Cell 用例前的受影响组)。 +- 原生守卫、生成的原生只读脚本和目标库克隆证明:**131 passed**。 +- Value 可见报告与历史日志:**13 passed**。 + +在后续补齐实际 Part Label / Cell 前,完整受影响组合为 **219 passed**。补齐后:实际元数据、BOM、库存、原生解析/读取、服务/后端身份保护组合 **140 passed**;真实只读采集 VBS 入口、实际元数据、BOM 新例及完整 VBS 编译 **36 passed**。最终 BOM 来源/每项 Value 组合结果由本次末尾追加;不把重叠统计相加。测试使用独立 `--basetemp`,避免多个协作进程共享 pytest 临时目录保留清理。修正了旧 VBS 假环境缺失 `QP_Fso` 初始化及新增报告依赖桩;真实生产例程仍由 cscript 执行。 + +最终 BOM 组合:`test_bom_batch`、`test_bom_import_report`、`test_bom_plan`、`test_bom_import_review`、`test_item_value_and_bom_sources`、`test_bom_actual_metadata`、`test_nonblocking_bom_values`、`test_value_readback`:**115 passed in 51.47s**。`git diff --check` 无空白错误。 + +最后补测来源行隔离及空 Part Label:先观察公式/数值 PN/非法模式/额外列公式 4 个服务端到端用例失败,再接通显式部分来源读取;空 Part Label 必须拒绝,不能以空 Value 相等误报通过。最后 `test_nonblocking_bom_values`、`test_bom_actual_metadata`、`test_bom_import_review`:**42 passed in 27.71s**。库存合并可读性整理后的存储/扩展/报告小组:**39 passed**。新增源错误隔离后没有再次执行较早的 219 项完整组。 + +独立审查补修:数据库已有目标 PN 和完整 Label/Cell 仍可能过期。新增真实 SQLite/XLSX 用例,在数据库保持 `NEW / LABEL / CELL` 时修改实际源文件,读取器分别返回变化后的 Label、Cell、Value;4 项均先因读取次数为 0 而失败。修复后,所有已到目标判定都经过当前只读发现,源与已保存基线冲突时只产生失败报告、不生成禁用成功行、不修改数据库;当前源匹配时允许跳过。窄回归 `test_bom_actual_metadata`、既有普通源库存复用用例、`test_bom_import_report`、`test_nonblocking_bom_values`:**61 passed in 34.51s**,没有真实 CAD 调用或完整 VBS 重跑。 + +最终窄审补修:本轮已读取但 `actual_part_label` 或指定 `actual_cell` 返回 `None` 时,普通加法库存合并会保留旧已知值;该历史补值不能参与“已到目标”策略核验。匹配器现在独立接收本轮原始观察,按精确工程/图页/UID 核验启用目标的 Value/Label/Cell,库存合并继续保持原有语义。新增“历史已知、本轮未知 Label/Cell”2 项先失败,再修复为失败报告、无预览、无跳过,数据库原记录不变;当前观察匹配仍允许跳过。 + +最终精确回归选择当前源观察 6 项(匹配、Label/Cell/Value 改变、Label/Cell 未知)及普通库存扩展基线保护 2 项:**8 passed, 27 deselected in 7.06s**。普通源计划仍保留原 `target_identity` 异常语义;仅已到目标判定的基线冲突转为失败报告。未重跑广泛套件,`git diff --check` 无空白错误。 + +## 集成说明 + +主流程调用 `clear_value_readback`、`capture_value_readback` 和库预检;service 的来源标注调用 `bind_source_report`,保留原文件与 source-map 绑定。资源或异常退出包装器必须同时清除 `preview` / `passed_preview`。事件日志通过 `value_readback_details(result)` 读取已验证持久报告,不能从请求 Value 推导实际结果。 + +未进行真实工程回放。旧清单没有可选 Part Label / Cell 读数时会重新采集,仍未知则采用显式问题;实际有效属性证据不能视作已实现完整 Block/Symbol 语义。 diff --git a/docs/testing/2026-11-06-nonblocking-repair-acceptance.md b/docs/testing/2026-11-06-nonblocking-repair-acceptance.md new file mode 100644 index 0000000..93f411d --- /dev/null +++ b/docs/testing/2026-11-06-nonblocking-repair-acceptance.md @@ -0,0 +1,79 @@ +# 非阻塞修复逐项验收记录 + +基线:8b4e8ee;工作树:codex/nonblocking-items。范围排除独立校验工作台。 + +## 本轮直接运行的集成证据 + +| 场景 | 方法和真实执行边界 | 已观察结果 | +| --- | --- | --- | +| 原缺陷与新粒度 | test_nonblocking_items 首次运行真实队列/SQLite,只有原生 worker 被替换 | 5 项先失败:R1/R2 被合并、无问题额度 API、无旧格式准备参数;修复后 5 passed | +| 同工程继续与旧保护 | execution_journal/task_queue/safe_cbb_continue/nonblocking_items/deferred_problem_pass 受影响集合 | 29 passed;保留旧无位号组的暂停语义与成功回执防重放 | +| 首轮后问题处理 | deferred_problem_pass 首轮四项均完成,再派发原失败位号 | 4 项先失败,修复后覆盖主轮次优先、持久一次额度、暂停收尾与 DSH 竞争 | +| 准入边界 | 未完成 prepared 尝试后领取另 UID、同 UID 不同命令同时绑定 | 两项先失败(未抛拒绝),加共享账本检查后 2 passed | +| 100 位号混合批次 | test_nonblocking_acceptance:真实工作区调度、子队列、备份、SQLite、问题轮次;仅进程传输与原生 CAD 用受控执行器 | 93 verified、5 failed、2 uncertain;100 次实际派发,各一次;恢复原批次零新增派发;关闭重开 DB 计数保留 | +| 额度扣减后中断 | 实际 SQLite 已保留额度,准备重试前注入 OSError,再恢复同队列 | 与 100 项样本一起 2 passed;恢复后仅原 R1 再派发,R2 不重复,额度仍为 1 | +| 新工作区检查点兼容 | RunJournal.inspect 读取工作区冻结队列,保留旧 task-queues;拒绝双重身份 | 新用例先失败,修复后 run_journal 13 passed | +| 隔离工程后的真实状态 | execute_confirmed_run 对已收尾但仍有 blocked 工程的结果 | 新用例先误报 completed,改为 blocked;与人工弹窗 UI 入口一起 2 passed | +| 活动任务中的人工弹窗入口 | 真实 Qt 窗口在 execution worker busy 时,复用原资源释放按钮 | 先被隐藏而失败;接线后可打开受控弹窗页面,禁止自动选择保存/许可按钮 | +| 部分导入安全撤销 | cleanup/shutdown 异常时不可保留 passed_preview | 两项先失败;修复后 supervisor_result + diagnostics_bundle 23 passed | +| 新证据诊断导出 | 工作区检查点、owner-loss、只读释放记录进入脱敏包;完整组件输入仍排除 | 先仅导出 manifest 而失败;修复后上述 23 项集合通过 | + +## 分组证据 + +- [逐位号冻结与旧队列兼容](2026-11-06-component-scope.md):普通规则与位号计划均生成新逐 UID 命令;旧冻结组保留;对应 55 项受影响验证。 +- [AI 执行入口解耦](2026-11-06-nonblocking-ai.md):模型/工单不占执行入口,真实变更临时准入,退出/释放确认;79 项邻接与最终 17 项定向验证(存在重叠,不相加)。 +- [BOM 与实际 Value](2026-11-06-nonblocking-bom-values.md):临时 Excel、真实 SQLite、原生 VBS 函数样本与来源/报告绑定。 +- [资源生命周期与独立审阅修复](2026-11-06-nonblocking-resources.md):涵盖主进程已退但后代仍活、Quit helper 保留、监测器异常/无控制对象、非交互 worker 寿命及显式强制入口;资源限定复核通过。 +- [失败后的肯定健康证明](2026-11-06-project-health.md):12 个定向用例覆盖缺证据、源变化、过期尝试、篡改和成功基线推进;限定复核通过。 + +## 逐项验收索引 + +| ID | 验收证据(生产逻辑 + 受控原生边界) | 状态 | +| --- | --- | --- | +| R01 | component_scope / nonblocking_items:同规则同值两个 UID 各一命令、普通规则采集冻结 | 通过 | +| R02 | component_scope / execution_journal:旧整组回执保留、重试侧车继承、成功不拆分重做 | 通过 | +| R03 | nonblocking_items / nonblocking_acceptance:明确失败后 R2 和其他工程继续 | 通过 | +| R04 | project_session_health:空证明、旧尝试、待核验、源变化、篡改均拒绝;最新源检查点可推进 | 通过 | +| R05 | execution_journal / deferred_problem_pass:重绑 job、重启和原轮次问题身份 | 通过 | +| R06 | deferred_problem_pass / nonblocking_resume:首轮优先、问题轮次暂停、中断准备后恢复 | 通过 | +| R07 | 两个独立 SQLite 连接同时竞争 DSH/problems,只有一个额度;重启不重置 | 通过 | +| R08 | workspace_execution / nonblocking_acceptance:隔离两个不确定工程且独立工程完成 | 通过 | +| R09 | batch_runtime:工程间串行不同 CAD,正常同工程复用,异常先释放 | 通过 | +| R10 | resources / retained_write_owners:资源消失可解除等待,uncertainty 保留 | 通过 | +| R11 | resources:心跳不重置业务停滞;无控制对象的停滞同样不强杀写入 | 通过 | +| R12 | resources:只读失败关闭整代,不重复失败工程 | 通过 | +| R13 | owner-loss 身份/请求检查和重启资源复核,拒绝 PID 复用/未知写入回收 | 通过 | +| R14 | manual_dialog_native + nonblocking_controls:真实未知 MessageBox 仅按明确选择关闭,释放无强杀 | 通过 | +| R15 | retained_write_owners:成功 Quit 仍有后代、Quit helper 存活、监测异常等均保留完整所有权 | 通过 | +| R16 | retained_write_owners:活跃/保留 Job 均参与准入;确认/身份/保护记录缺失不得强制 | 通过 | +| R17 | dsh_ai_nonblocking / incident_admission:模型等待不占执行入口,真实重试临时准入 | 通过 | +| R18 | run_journal / nonblocking_resume / Value 可见报告:成功、失败、隔离分别呈现,恢复范围正确 | 通过 | +| R19 | nonblocking_bom_values:显式部分选择、原表 SHA/行/source-map、错误来源工程隔离、异常撤销预览 | 通过 | +| R20 | bom_actual_metadata:当前实际 PN/Value/Label/Cell;旧库存不可证明新鲜,缺当前元数据不得回填判通过 | 通过 | +| R21 | cell_catalog / nonblocking_bom_values:真实执行适配接通只读目标库预检,错误与清理失败区分 | 通过 | +| R22 | resources / discovery_cache:内容而非单纯元数据判别、释放后复核、未知实际属性保留 | 通过 | +| R23 | value_readback / nonblocking_value_vbs / report UI:真实读数持久绑定当前终态与快照,旧报告拒绝 | 通过 | +| R24 | native VBS 属性观察/重复属性守卫:全部原始观察保留,层级未知不猜写,同值不合并 UID | 通过 | +| R25 | 100 项真实调度/账本样本:93 verified + 5 failed + 2 uncertain;派发100次、恢复无重放 | 通过 | + +## 最后集成与审阅 + +在最新肯定健康证明、实时源摘要和所有权修复后运行的集成集合: + +```text +python -m pytest tests/v1_1/test_nonblocking_acceptance.py tests/v1_1/test_nonblocking_resume.py tests/v1_1/test_deferred_problem_pass.py tests/v1_1/test_manual_dialog_native.py tests/ui/test_nonblocking_controls.py tests/v1_1/test_supervisor_result.py -q --tb=short -o cache_dir=F:/dx-nb-final-cache --basetemp=F:/dx-nb-final-integration +``` + +结果:**26 passed,26.36 秒**。随后额度双连接竞争与准备中断恢复两个定向场景 **2 passed**。旧 DSH 恢复兼容出现的 11 个失败定位为临时更改了通用空文件证明语义;恢复旧兼容、仅在失败放行处要求肯定证明后,仅重跑失败集合 **11 passed、12 deselected**。 + +核心/健康和资源两个独立限定审阅均通过;审阅期间发现的暂停恢复、源证书新鲜性、所有者后代、旧库存跳过问题均有单独复现和修复证据。没有用重复大套件代替功能验收。 + +BOM 最后补充的两种当前 Label/Cell 缺失情况先复现失败,再与当前值变化、匹配及普通库存身份保护共 **8 passed、27 deselected**;限定复核通过。策略核验直接使用本轮对应 UID 的原始属性读数,不能由库存合并回填的旧值证明。至此 R01~R25 全部具有独立验收索引。 + +## 验收边界 + +没有重复运行真实 CAD 换料、PCB、真实模型请求、远程工单或发布安装包。一次性 Windows 进程、私有桌面弹窗和 cscript 样本属于本轮验证。 + +Block/Symbol 的属性所属层级仍未知:验收的是保留原始观察、不以相同 Value 合并 UID、歧义写入前拒绝;不宣称自动选择 Block 或 Symbol 已实现。 + +本文件中的测试数是不同阶段的证据,不能累计为互不重复的测试总数。最终逐项状态以修复清单和独立审阅收尾记录为准。 diff --git a/docs/testing/2026-11-06-nonblocking-resources.md b/docs/testing/2026-11-06-nonblocking-resources.md new file mode 100644 index 0000000..2d6db6f --- /dev/null +++ b/docs/testing/2026-11-06-nonblocking-resources.md @@ -0,0 +1,90 @@ +# T2 — nonblocking project resources, 2026-11-06 + +Scope: R08–R16 and R22 in `docs/superpowers/plans/2026-11-06-nonblocking-repair-checklist.md`. All changes stay in the shared `codex-nonblocking-items` worktree. No commit, real CAD/PCB execution, AI call, network send, or user environment-variable change was performed. + +## Result and production wiring + +- `workspace_execution.execute_workspace_projects` freezes ordered task IDs, plan hashes, parent phase, deterministic child checkpoint IDs, release flags, and problem-pass completion in `workspace-queues/.json`. Each project calls the existing supervised task queue separately. A released uncertain A remains blocked with durable uncertainty while B can run. An unproved release pauses the parent. Resume skips blocked A after manual physical reconciliation; it never replays A. All primary projects precede deferred problem passes; paused/running problem passes resume the same child checkpoint. +- `BatchCadRuntime` reuses CAD within one project, then closes its complete owned generation before another project starts. Rebuilds still share the existing one-per-project retry budget; changing projects does not replenish it. Old leases retain their original generation. +- `reset_project_after_safe_failure(project, evidence_root, *, job_id=None)` requires the current owned close checkpoint and returns a persisted health proof. It binds the exact selected failed job to its current attempt/generation, hashes that attempt record, and checks `can_retry_attempt` for no mutation. It records a fresh after-release source fingerprint, whether the whole source fingerprint stayed unchanged, an immutable release snapshot/hash, checkpoint hash, and immutable per-job health file. A closed project alone is not a no-write certificate. Callers must require affirmative no-mutation/source/release proof or combine a verified native rollback baseline for that same job. +- Read provenance is fixed at admission (`bom_import` and `collect_target_inventory`). Failed/cancelled CAD reads release all owned workers, CAD, and desktop before raising `ReleasedProjectReadError`. Proven native read failure uses the same release path without spending a retry or opening another desktop. Execution-provenance native prewrite recovery retains its existing strict source/identity proof and one-shot budget. +- `batch_worker` observes the admitted GUI parent PID plus creation time, persists owner-loss evidence, and requests cancellation. Read workers use bounded cooperative cancellation. Owner loss never grants permission to kill a possible write. +- `ProgressWatch` tracks worker liveness and business progress separately. Repeated identical stdout/native counts cannot extend the business idle deadline. Unsafe write stalls request safe stop and retain the process. `BusinessProgressWatch` separately ignores supervisor heartbeats and unchanged state. +- Every retained worker/runtime/CAD/supervisor wait has an exact owned-process exit observer. Native observers check both root handles and Job descendants. A completed root cannot hide live descendants. Confirmed physical exit ends the wait while uncertainty remains. +- No-control write sessions retain their handles when orderly shutdown fails. They do not fall through to terminate-capable desktop cleanup. Raw retained CAD owners remain in a live registry, appear in pending resource status, block another CAD launch, and reconcile normally only once every retained owned tree has exited. +- `force_pending_owned_resources(root, *, confirmed=False)` is an in-process GUI API. Default/false refuses without action. Explicit true validates live retained handle identities against durable records, writes uncertainty/protection and force intent, then terminates only those exact owners. It supports GUI supervisor Jobs, batch runtimes, and raw retained CAD sessions. Corrupt/reused PID identity and denied protection both prevent termination. Ordinary resource cleanup refuses to force a retained write Job. +- Unknown/save/license dialogs retain their text and actual buttons. The worker accepts only one-shot requests bound to supervisor batch/attempt, generation, and CAD session token. The desktop re-enumerates and compares the complete dialog fingerprint before an explicit user's selected button is sent. Existing automatic acknowledgement remains restricted to a complete recognized VBScript compilation-error prompt with one permitted OK button. Startup CAD PID ownership is published before GUID binding, so startup prompts can also be handled manually. +- `ui.cad_dialog_operations.show_cad_dialogs(window)` displays the active project, original prompt text, and explicit original button choices. Root integration repurposes the existing resource cleanup button while the supervisor reports a blocking prompt. No global input, desktop switching, save decision, license decision, or automatic unknown-dialog click was introduced. +- Persistent/process-local discovery caches refresh metadata-only differences while hashing every file's content on lookup. File replacement, changed content, unreadable files, changed inventories, and unknown identities remain misses. Identity comparisons include optional actual Part Label/Cell metadata and preserve `None` versus an empty string. A verified reader-release hook runs before the after-discovery fingerprint to release file locks. + +## Red/green evidence + +Commands used Python 3.12 and pytest with unique short `F:/dx-nb-res-*` basetemp paths and `F:/dx-nb-res-cache`; no test output directories are in the repository. + +| Regression introduced first | Observed red | Observed green | +| --- | --- | --- | +| Initial resource boundaries | 9 failures: CAD identity skipped, legacy CAD released without identity, a released label bypassed evidence, metadata-only changes reread, missing exit observer/business clock/read provenance/reset/manual action | Initial set expanded to 14 passing cases | +| Read generation cleanup and project switch | Unsupported read-only session flag and old CAD reused for another project | Read failure closes one generation without retry; next project starts only after previous CAD closes | +| Owner loss and worker exception wait | Missing recovery module and missing `resolved` observer | Verified parent identity evidence; unsafe worker exception resolves on owned exit | +| Frozen workspace queue | 5 missing-module failures | Ordered A/B isolation, no release proof holds B, exact resume IDs and stale hashes rejected | +| Manual transport/UI | Missing one-shot consumer and missing UI module | Generation/token mismatch rejected, one action only; actual prompt text shown, no action until button click | +| Failed cleanup ownership | CAD reference dropped and desktop closed after an active write owner cleanup failure | CAD/desktop retained; Job descendants keep exit observer false | +| Native read no replay | Read-only native failure created another session instead of raising released read error | One generation released, no retry budget spent | +| Identical heartbeat progress | Both stdout and native repeated count frames hid the business deadline | Both request safe stop then wait indefinitely for a possible write | +| Finished supervisor label | `finished` skipped an original live process | Live original remains pending | +| Crashed supervisor with harmless 6-second child | Existing five-second descendant grace automatically killed the possible-write Job (`forced_termination=True`) | Child exits naturally; `forced_termination=False`, durable uncertainty remains, resources released | +| No-control possible write | Harmless owned Python sleep process killed on failed Quit | Process remains alive and retained; only explicit confirmed force releases it | +| Explicit retained Job force | API absent | No confirmation, mismatched creation time and denied protection all refuse to kill; exact protected confirmed Job is terminated | +| Health/metadata proof | Reset returned a boolean; actual metadata unknown/empty compared equal | Current attempt/source/release hashes are persisted; unknown differs from empty | +| Paused deferred problem pass | Resume skipped paused project and falsely marked parent finished | Same child checkpoint resumes in the persisted problems phase | +| Raw owner natural exit | Exited raw session remained permanently pending | All owned trees observed exited; ordinary cleanup releases handles, retains uncertainty, never force-kills | +| Completed CAD root with active descendants/repeated force | Root exit permitted desktop cleanup; repeated force returned early from root PID alone | Tree-level proof is required; explicit force still reaches the retained Job | + +## Targeted verification performed + +- Initial necessary resource regression set (`test_batch_runtime`, `test_batch_supervisor`, `test_isolated_cad`, `test_isolated_desktop`, `test_cad_dialogs`, `test_discovery_cache`, `test_persistent_discovery_cache`, `test_worker_monitor`): **218 passed, 3 failed**. Two old assertions assumed CAD reuse between distinct projects; those assertions now verify required serial project isolation while retaining same-project reuse. The third was a native harmless modal probe's five-second transient timeout. Its evidence showed the recognized compilation prompt and recorded acknowledgement; the exact isolated rerun passed **1/1** in 1.48 seconds. No production workaround or timeout weakening was added. +- After the project-lifecycle assertions: runtime + initial new resource/workspace/Qt tests passed **59/59**. +- Native read resource change: native/resource/runtime focused set passed **55/55**. +- Business/liveness separation: resources + worker monitor passed **55/55**. +- Affected integration set (`resources`, `workspace`, Qt, supervisor, runtime integration, native reader and project discovery): **93 passed, 1 failed**. The failure was a previously supported injected `runtime=object()` without the new release hook. Root changed the caller to require a callable hook. The exact failed discovery case then passed **1/1**; the 93-test set was not repeated. +- Resources/workspace/Qt/runtime/isolated-CAD after no-control ownership and proof changes passed **106/106**. Additional real harmless raw/runtime force variants passed **2/2**. +- Final affected safety set (`test_nonblocking_resources`, `test_workspace_execution`, `test_cad_dialog_operations`, `test_batch_runtime`, `test_isolated_cad`, `test_batch_supervisor`, `test_safe_user_stop`): **150 passed** in 109.07 seconds. This includes all final retained-owner/descendant/force changes. All 12 edited/new resource modules parse successfully; `git diff --check` passes. No broad repository suite is claimed. + +## Limits and review boundaries + +Native behavior was verified through owned disposable Python processes, Windows Jobs/desktops, and a harmless modal dialog. Real Viewdraw/COM/CAD saving, actual license dialogs, and production design files were not exercised. An unverified or legacy CAD record without a PID creation-time identity remains pending rather than guessing ownership. After restart, reconciliation observes disappearance only; the force API never opens arbitrary PIDs or attaches to a foreign application. An unavailable source fingerprint fails closed. Business uncertainty is never cleared by resource release or force. + +Root owns task-queue admission, receipts, rollback/native baselines, service/UI entry wiring, and integration acceptance. This T2 record does not claim those separate caller changes solely from helper availability; their own tests/review must verify the real admission path. + +## Independent review fixes: complete owned write trees + +The read-only review reproduced four additional ownership gaps, then the parent explicitly assigned their implementation to the reviewer. No real CAD/PCB or AI was used. + +- Successful Quit now checks every owned CAD/helper/worker tree before desktop cleanup. Root exit code zero cannot authorize killing surviving descendants. Runtime cleanup retains the CAD owner when any retained Quit-helper or desktop process remains. +- Unsafe monitor exceptions, unsuccessful resolution observers, normal root completion with live children, and a stalled write without an interactive controller retain the exact process/Job handles. `RETAINED_WORKERS` exposes these owners through pending-resource status and explicit confirmed force. Force verifies creation identity against durable evidence and protects uncertainty before terminating. Ordinary cleanup only closes trees observed exited. +- A durable `retained-worker-.json` record continues to block after registry/process restart. Without the original Job handle, root PID disappearance cannot prove descendant exit and cannot authorize force. Natural exit and explicit force in the owning process write a released tree proof; uncertainty is retained. +- Reconciliation consults both retained and active supervisor handles, so a finished/root-exited journal cannot hide a live owned Job. +- Every supervised worker uses `StopControl`, including noninteractive callers. Its finalization observes retained write owners before process exit can close Windows kill-on-close Jobs. A protected active supervisor is reachable through the existing explicit service force API: the API validates identity, durably records `force-request.json`, and signals its owning loop; the loop performs protected termination. It does not race the owner's handle cleanup or bypass the supervisor lease for direct termination. +- Diagnostic exports now include these two narrowly named sidecars; component-scope/input tables remain excluded. + +### Focused evidence + +`tests/v1_1/test_retained_write_owners.py` contains 18 cases. The first six ownership cases failed before their fixes. Additional red cases covered noninteractive worker lifetime, active owner reconciliation, resolution-error finalization, restart protection, and a no-controller business stall. Tests use mocked CAD boundaries and real harmless Python children, Windows Jobs, and private desktops. + +The necessary affected suite was run once: + +```text +python -m pytest tests/v1_1/test_retained_write_owners.py tests/v1_1/test_isolated_cad.py tests/v1_1/test_batch_runtime.py tests/v1_1/test_batch_supervisor.py tests/trial_task_center/test_worker_monitor.py -q --basetemp=F:/dx-review-resource-affected -o cache_dir=F:/dx-review-owner-cache +``` + +Result: **136 passed, 1 failed** in 65.62 seconds. The new natural-exit test incorrectly expected Job accounting to reach zero immediately when its root handle became signaled. The assertion was corrected to wait for actual full-tree exit. The exact natural/force variants then passed **2/2** (`-k retained_native_worker`, `F:/dx-review-owner-natural-final`); no production timeout or ownership guard was weakened and the affected suite was not repeated. + +Afterward, the real noninteractive worker lifetime/active-force variants passed **2/2** (`-k noninteractive_worker`, `F:/dx-review-owner-supervisor-force`): a natural child wrote its completion marker, while the explicit-force variant retained it until the separate service call requested force and returned `user_forced_stop` with uncertainty evidence. + +The final narrow previous-regression plus no-controller-stall command passed **9/9**: + +```text +python -m pytest tests/v1_1/test_retained_write_owners.py tests/v1_1/test_nonblocking_resources.py -q -k 'unsafe_stall_without or unsafe_worker_exception_wait or no_control or force_pending or repeat_explicit_force or owned_exit_observer' --basetemp=F:/dx-review-owner-stall-green -o cache_dir=F:/dx-review-owner-cache +``` + +Diagnostic sidecar inclusion was red first, then `test_diagnostics_bundle.py` passed **9/9** (`F:/dx-review-owner-diagnostic-green`). Production modules parse and `git diff --check` passes. Counts overlap and are not cumulative. The earlier 150-case resource suite and the broad repository suite were not rerun. diff --git a/docs/testing/2026-11-06-project-health.md b/docs/testing/2026-11-06-project-health.md new file mode 100644 index 0000000..2c3843a --- /dev/null +++ b/docs/testing/2026-11-06-project-health.md @@ -0,0 +1,55 @@ +# Affirmative per-component failure health — 2026-11-06 + +This follow-up closes the admission gap where `proof_intact({})` and an ordinary failed receipt allowed the next UID. Generic legacy success/import proof behavior is preserved. Component failure admission now requires an explicit, nonempty, durable health certificate. + +## Production path + +`quick_run.execute_step` records the logical journal attempt before dispatch. For a native failed result, an immutable dispatch certificate binds the frozen command and its journal task/round/operation/job/attempt/generation to the actual runtime session/attempt/generation and native attempt file. A later health proof from another native session or retry cannot certify that dispatch, even if the job ID is reused. + +After a safe component failure, `batch.execute_batch` asks `reset_project_after_safe_failure(..., job_id=metric['job_id'])` for the owned generation's release proof, then calls `journal.certify_safe_failure(task_id, row, health)` before another UID can be claimed. Certification only updates the already-failed, exact current SQL operation. It verifies native no-mutation state, the physical released generation record, matching project-close/worker-close checkpoint, session/generation/job identities, and all artifact hashes. Evidence is copied into immutable execution-proof files rather than relying on mutable active handoff or session files. + +A native current-attempt no-mutation proof can continue when the full source fingerprint is unchanged. When a retained worker's initial source fingerprint predates legitimate earlier successful steps, continuation additionally requires the intact last verified semantic baseline for the exact failed project's sheet. A baseline from another sheet, empty evidence, a changed baseline, missing release evidence, a different native dispatch, or a possibly mutated current native attempt cannot authorize continuation. A rollback marker's text alone is not upgraded to healthy execution; existing explicit rollback/recovery protections remain in effect. + +Custom synchronous `process_runner` adapters use the same production gate. The preparer captures a full source fingerprint before dispatch. A correlated terminal failure gets a durable certificate only after a matching nonempty after-failure fingerprint and proof that the related runtime/retained CAD resources are absent. The certificate includes the frozen command, failed outcome, exact journal identity, source digest, no-mutation flag, and completion/release flags. A source change or retained owned resource becomes uncertainty and blocks remaining UIDs. This behavior applies to all synchronous adapters, without detecting tests or granting them an exception. Code 2/3 rejection receipts accept captured evidence; native rejection receipts carry the actual native dispatch binding. + +`journal.independent_continuation_ready(task_id)` provides the UI/DSH with a read-only answer for the first pending item in frozen plan order. It uses the same affirmative gate, returns false on missing or changed evidence and unresolved attempts, and never chooses a later UID to skip a blocked first pending item. + +## Red/green evidence + +All commands use Python 3.12, pytest, short unique `F:/dx-nb-health-*` basetemp paths, and the existing short test cache. + +The first four tests were run before implementation and failed for the intended defects: + +- A bare `journal.finish(..., 'batch_failed', proof=None)` let another UID claim execution. +- Empty proof passed the existing generic integrity function and was incorrectly treated as admission evidence. The fix is local to affirmative component health; legacy success/import compatibility remains intact. +- An actual controlled synchronous runner changed the project source after R1 failed and R2 still ran. +- Failed receipts had no immutable failure-health artifact to protect or validate on resume. + +Additional tests verify actual runtime reset proof and native/journal binding using the real runtime/journal logic with an injected CAD/desktop boundary: + +- Matching current prewrite native health permits the next UID. +- A hashed release record containing only a `released` label without release evidence is rejected. This first reproduced as a failure, then passed after structural physical proof validation was added. +- A changed source with no verified sheet baseline and a mismatched native job are rejected. +- The exact intact last verified sheet baseline supports a current prewrite failure; tampering with that baseline revokes readiness. +- Reusing old native health for a new journal/native dispatch is rejected. The immutable dispatch-binding implementation was added after a red test exposed its absence. +- A safely paused R1 failure reports independent continuation readiness; certificate tampering and a bare failed receipt report false. These readiness tests failed before the API existed, then passed. + +Focused acceptance and necessary adjacent regressions: + +`test_project_session_health.py`, `test_nonblocking_items.py`, `test_deferred_problem_pass.py`, `test_component_scope.py`, `test_nonblocking_acceptance.py`, `test_execution_journal.py`, and `test_execution_proof.py`: **54 passed** in 24.21 seconds after the native dispatch binding changes. After adding readable frozen command fields to the certificates, all **8 health tests passed** in 4.41 seconds. The five affected modules parse successfully and `git diff --check` passes. No broad repository suite was run. + +## Limits + +These checks exercise real SQLite receipts, actual handoff/status parsing, controlled synchronous dispatch, source fingerprint changes, immutable evidence files, and the actual runtime health/reset code. CAD/desktop boundaries in these health tests are injected; no real CAD/PCB, AI, or network operation was performed. Native health certificates authorize only affirmative current-attempt no-mutation. Possible native mutation remains held for the established verified rollback/recovery path rather than being inferred from a terminal error or a marker's text. + +Parent-owned service/DSH wiring and the separate review fixes for retained process cleanup require their own integration checks. No commit or index change was made. + +## Stale-source follow-up from independent review + +Review found that immutable certificate integrity alone did not establish the current source: after a safely failed UID and pause, a later edit to another project file still left readiness true. The actual pause/failure/edit regression was run first and failed. A second new case showed the same bypass after a later successful semantic baseline. + +Certificates now record `source_digest_algorithm`, the exact `source_project_path`, and `source_excluded_root`. Every component continuation readiness/claim compares the current full project fingerprint with certified source. A newer successful same-sheet semantic baseline may supersede an older failure certificate only through an immutable source checkpoint that explicitly names that exact failure identity and certificate SHA. Its current full-source digest must match too. A pre-existing baseline or a certificate from another sheet cannot waive a source change. + +Actual controlled-runner tests demonstrate that a legal verified R2 save creates the newer checkpoint and R3 remains executable, while an unrelated edit after that checkpoint revokes readiness. Native post-failure success gracefully finishes the retained worker and releases its verified generation before source checkpoint publication. The checkpoint reuses the released native source digest bound to the matching native attempt and release snapshot, avoiding a second full hash; the dedicated test rejects any duplicate source fingerprint call. + +The focused health set now has **12 passing tests**. The earlier 54-test set was not repeated for this narrow review fix. Tests still use injected CAD boundaries and controlled local files; real CAD was not run. diff --git a/docs/user/nonblocking-item-execution.md b/docs/user/nonblocking-item-execution.md new file mode 100644 index 0000000..49b907a --- /dev/null +++ b/docs/user/nonblocking-item-execution.md @@ -0,0 +1,13 @@ +# 逐位号执行与问题处理 + +新建自动队列先冻结工程中的位号身份,每个位号单独执行和记账。同名或相同 Value 不会让不同 UID 合并成一个执行项。旧队列保留原先冻结的整组范围,不因升级重新派发已经成功的位号。 + +明确失败的位号进入问题记录。当前工程可证明安全时,其余独立位号继续执行;工程状态不确定时暂缓同工程余项。资源确认释放后继续其他工程。所有可运行项完成后,明确列入安全临时错误规则的问题项最多自动重试一次。暂停、重启、换 job 或 AI 建议都不会增加额度。 + +保存或回读不确定不表示成功,也不会自动再次换料。程序先请求安全退出;仍卡住时需要你明确选择强制结束。强制结束仅作用于程序仍持有并能证明归属的进程,工程仍保留待核验状态。 + +后台出现未知弹窗时,原资源释放按钮会显示为“处理阻塞弹窗”。页面显示当前工程、原始弹窗内容及按钮,由你选择。程序不会自动替你选择保存、放弃或许可操作。 + +BOM 默认仍要求整批通过。只有明确启用部分导入并选择通过工程,才保存对应范围;失败行和原表来源继续保留。预览后原 Excel 或来源映射变化需要重新导入。资源释放异常会撤销所有可保存预览。 + +替换报告区分原 Value 与保存后实际 Value。缺少可验证报告时显示缺少证据,不能把计划值当成实际值。Block/Symbol 层级未确认的同名属性会保留诊断并拒绝歧义写入,不能用“Value 相同”推断应该修改哪一层。 diff --git a/integration/dxdesigner795/NativeSnapshotReader.vbs b/integration/dxdesigner795/NativeSnapshotReader.vbs index ec6b0ee..e8e4b05 100644 --- a/integration/dxdesigner795/NativeSnapshotReader.vbs +++ b/integration/dxdesigner795/NativeSnapshotReader.vbs @@ -108,7 +108,10 @@ Sub QPValidateNativeSelectedState(ByVal State, ByVal Uid) If UBound(Fields) <> 11 Then QPNativeReadInvalid "invalid selected attribute fields" If Fields(0) <> "A" Or QPDecode(Fields(1)) <> Uid Then QPNativeReadInvalid "selected attribute owner mismatch" Name = QPDecode(Fields(2)) - If Len(Name) = 0 Or Names.Exists(Name) Then QPNativeReadInvalid "duplicate or empty selected attribute name" + ' Raw A rows remain in the native frame even when values are identical. + ' Value/InstanceValue do not prove Block/Symbol hierarchy ownership. + Name = LCase(Name) + If Len(Name) = 0 Or Names.Exists(Name) Then QPNativeReadInvalid "ambiguous selected attribute ownership; raw occurrences retained; no layer writes" Names.Add Name, True Next End Sub diff --git a/integration/dxdesigner795/QuickPartReplace.vbs b/integration/dxdesigner795/QuickPartReplace.vbs index 9da4ed5..65af852 100644 --- a/integration/dxdesigner795/QuickPartReplace.vbs +++ b/integration/dxdesigner795/QuickPartReplace.vbs @@ -726,7 +726,8 @@ Sub QPDiscoverInventory(ByRef Documents) Text = "project_path" & vbTab & "schematic_name" & vbTab & "sheet_number" & vbTab & "old_part" & vbCrLf ComponentMode = QP_Fso.FileExists(QP_Fso.BuildPath(QP_Root, "component-inventory.request")) ComponentText = "project_path" & vbTab & "schematic_name" & vbTab & "sheet_number" & vbTab & _ - "uid" & vbTab & "refdes" & vbTab & "old_part" & vbTab & "old_value" & vbCrLf + "uid" & vbTab & "refdes" & vbTab & "old_part" & vbTab & "old_value" & vbTab & _ + "actual_part_label" & vbTab & "actual_cell" & vbCrLf Count = 0 Set Schematics = Documents.GetAvailableSchematics() For Each Schematic In Schematics @@ -763,13 +764,16 @@ Sub QPDiscoverInventory(ByRef Documents) PartKey = LCase(CStr(PartNumber)) If Requested.Exists(PartKey) Then If ComponentMode Then + QPRequireUniqueAttributeNames Comp ValueText = QPEffectiveComponentAttribute(Comp, "Value") If Len(Trim(CStr(Uid))) = 0 Or Len(Trim(CStr(Comp.Refdes))) = 0 Then _ Err.Raise vbObjectError + 872, "QuickPartReplace.Discovery", "component identity is incomplete" ComponentText = ComponentText & QPEncode(QPProjectPath()) & vbTab & _ QPEncode(CStr(Schematic)) & vbTab & QPEncode(CStr(Sheet)) & vbTab & _ QPEncode(Uid) & vbTab & QPEncode(CStr(Comp.Refdes)) & vbTab & _ - QPEncode(CStr(PartNumber)) & vbTab & QPEncode(CStr(ValueText)) & vbCrLf + QPEncode(CStr(PartNumber)) & vbTab & QPEncode(CStr(ValueText)) & vbTab & _ + QPEncode(QPEffectiveComponentAttribute(Comp, "Part Label")) & vbTab & _ + QPEncode(QPEffectiveComponentAttribute(Comp, "Cell Name")) & vbCrLf End If HitKey = QPSheetKey(Schematic, Sheet) & Chr(30) & PartKey If Not Seen.Exists(HitKey) Then @@ -979,7 +983,7 @@ Sub QPDiscardFailedBatchProject(ByRef ErrorDescription) End Sub Sub QPFinishBatchItem() - Dim Command, SnapshotPath, Records, Plans, ValueStates, Index, Record, Candidate, TargetPath, FinalState + Dim Command, SnapshotPath, Records, Plans, ValueStates, Index, Record, Candidate, TargetPath, FinalState, ValueReportState If QP_BatchState <> "restored_ok" And QP_BatchState <> "no_match" Then _ Err.Raise vbObjectError + 871, "QuickPartReplace.Batch", "worker has no verified terminal result" FinalState = QP_BatchState @@ -1025,6 +1029,9 @@ Sub QPFinishBatchItem() Next QPVerifyRestoredPlans Command, Records, Plans, ValueStates QPWriteVerifiedState Command + ValueReportState = "saved_ok" + If QP_Fso.FileExists(QP_Fso.BuildPath(QP_Root, "recovery.request")) Then ValueReportState = "restored_ok" + QPWriteValueReadback Command, Records, Plans, ValueReportState If Not QP_SessionMode Then QPCloseBatchProject TargetPath, False FinalState = "saved_ok" QPLog QP_Root, "INFO", "saved_readback_verified job=" & QP_CurrentJobId @@ -1295,7 +1302,7 @@ Sub QPCaptureAndSelect(ByRef Command, ByVal SnapshotPath) Dim PartNumber Dim Snapshot Dim Index - Dim Selectors, Selector, Uid, ValueAttr, ValueText, MatchedUids, UntouchedCount, NativeRead + Dim Selectors, Selector, Uid, ValueAttr, ValueText, MatchedUids, UntouchedCount, NativeRead, AlreadyTargets QP_FallbackIdentityUsed = False Set QP_FallbackIdentityWarnings = CreateObject("Scripting.Dictionary") @@ -1305,6 +1312,7 @@ Sub QPCaptureAndSelect(ByRef Command, ByVal SnapshotPath) QPInvalidateComponentIndex QPProgress "query_components", 0, 0 Set Matches = CreateObject("Scripting.Dictionary") + Set AlreadyTargets = CreateObject("Scripting.Dictionary") QP_Stage = "query_components" QPLog QP_Root, "INFO", "stage=query_components" Set Components = QP_App.ActiveView.Query(VDM_COMP, VD_ALL) @@ -1318,22 +1326,35 @@ Sub QPCaptureAndSelect(ByRef Command, ByVal SnapshotPath) If StrComp(PartNumber, CStr(Command("old_part_number")), _ vbTextCompare) = 0 Then Matches.Add CStr(Matches.Count), Comp ElseIf Selectors.Exists(Uid) Then + QPRequireUniqueAttributeNames Comp PartNumber = QPRequiredAttributeValue(Comp, "Part Number") Selector = Selectors.Item(Uid) ValueText = "" Set ValueAttr = QPFindCurrentAttribute(Comp, "Value") If Not ValueAttr Is Nothing Then ValueText = QPEffectiveAttributeValue(ValueAttr) + If StrComp(PartNumber, CStr(Command("new_part_number")), vbTextCompare) = 0 Then + QPRequireAlreadyTargetPolicy Comp, Command, Selector + AlreadyTargets.Add Uid, True + Else If CStr(Comp.Refdes) <> CStr(Selector(0)) Or _ StrComp(PartNumber, CStr(Command("old_part_number")), vbTextCompare) <> 0 Or _ ValueText <> CStr(Selector(1)) Then _ Err.Raise vbObjectError + 873, "QuickPartReplace.Target", "selected component changed since inventory" Matches.Add CStr(Matches.Count), Comp + End If End If QPProgress "find_components", Matches.Count, 0 Next If Not Selectors Is Nothing Then - If Matches.Count <> Selectors.Count Then _ + If Matches.Count + AlreadyTargets.Count <> Selectors.Count Then _ Err.Raise vbObjectError + 873, "QuickPartReplace.Target", "selected component is missing" + If AlreadyTargets.Count > 0 Then + If Matches.Count > 0 Then Err.Raise vbObjectError + 880, "QuickPartReplace.Target", _ + "legacy group mixes source and already-target components; explicit per-component repair required" + QPWriteStatus QP_Root, CStr(Command("job_id")), "no_match", "already_target", _ + "Selected target UID already has requested Part Number and verified Value/Cell policy; skipped without writes." + Exit Sub + End If End If If Matches.Count = 0 Then If Not Selectors Is Nothing Then _ @@ -1362,6 +1383,7 @@ Sub QPCaptureAndSelect(ByRef Command, ByVal SnapshotPath) Set MatchedUids = CreateObject("Scripting.Dictionary") For Index = 0 To Matches.Count - 1 Set Comp = Matches.Item(CStr(Index)) + QPRequireUniqueAttributeNames Comp MatchedUids.Add QPComponentIdentityKey(Comp), True Next Set NativeRead = Nothing @@ -1901,6 +1923,7 @@ Sub QPRestoreAfterOperatorReplace(ByRef Command, ByVal SnapshotPath) " Target Cell verified: " & QPTargetCell(Command) & "." If Len(QP_NcSyncSummary) > 0 Then SuccessMessage = SuccessMessage & _ " NC synced: " & QP_NcSyncSummary + If Not QP_BatchMode Then QPWriteValueReadback Command, Records, Plans, "restored_ok" QPWriteStatus QP_Root, CStr(Command("job_id")), "restored_ok", "", SuccessMessage QPLog QP_Root, "INFO", "verified replacement count=" & Plans.Count & " value_mode=" & QPValueMode(Command) End Sub @@ -2065,11 +2088,13 @@ Sub QPRecoverSavedJob(ByRef Command, ByVal SnapshotPath) ' Closing without saving avoids a second save/reopen cycle for no writes. QPRequireCommandSnapshot Command, SnapshotPath QPWriteVerifiedState Command + QPWriteValueReadback Command, Records, Plans, "restored_ok" QPCloseBatchProject CStr(Command("project_path")), False QP_BatchMode = False QPWriteStatus QP_Root, QP_CurrentJobId, "restored_ok", "recovery_saved_verified", _ "Saved replacement verified from disk and owned project closed without additional saving." Else + QPWriteValueReadback Command, Records, Plans, "restored_ok" QPWriteStatus QP_Root, QP_CurrentJobId, "restored_ok", "recovery_verified", "Saved replacement already passes all restoration checks; no CAD writes." End If Else @@ -2122,6 +2147,7 @@ Sub QPRestorePreflight(ByRef Comp, ByRef ComponentRecord, _ Dim CurrentConnections Dim SavedConnections + QPRequireUniqueAttributeNames Comp If QPComponentIdentityKey(Comp) <> CStr(ComponentRecord("uid")) Or _ CStr(Comp.Refdes) <> CStr(ComponentRecord("refdes")) Then _ Err.Raise vbObjectError + 831, "QuickPartReplace", "replacement component UID or Refdes differs from snapshot; no restoration writes allowed" @@ -2814,6 +2840,92 @@ Function QPFindCurrentAttribute(ByRef Comp, ByVal Name) Set QPFindCurrentAttribute = Attr End Function +Function QPAttributeObservations(ByRef Comp) + Dim Attr, Ordinal, Text, Uid + Uid = QPComponentIdentityKey(Comp) + Text = "" + Ordinal = 0 + ' API properties are observations, not proven Block/Symbol ownership. + For Each Attr In Comp.Attributes + Text = Text & "E" & vbTab & QPEncode(Uid) & vbTab & CStr(Ordinal) & vbTab & _ + QPEncode(CStr(Attr.Name)) & vbTab & QPEncode(CStr(Attr.Value)) & vbTab & _ + QPEncode(CStr(Attr.InstanceValue)) & vbTab & "unknown" & vbCrLf + Ordinal = Ordinal + 1 + Next + QPAttributeObservations = Text +End Function + +Sub QPRequireUniqueAttributeNames(ByRef Comp) + Dim Attr, Names, Name, Evidence, Ambiguous + Evidence = QPAttributeObservations(Comp) + Evidence = "O" & vbTab & QPEncode(CStr(QP_CurrentJobId)) & vbTab & _ + QPEncode(QPComponentIdentityKey(Comp)) & vbTab & QPEncode(CStr(QP_Stage)) & vbTab & CStr(Timer) & vbCrLf & Evidence + QPAppendAscii QP_Fso.BuildPath(QP_Root, "attribute-observations-" & CStr(QP_CurrentJobId) & ".tsv"), Evidence + Set Names = CreateObject("Scripting.Dictionary") + Ambiguous = False + For Each Attr In Comp.Attributes + Name = LCase(CStr(Attr.Name)) + If Names.Exists(Name) Then + Ambiguous = True + Else + Names.Add Name, True + End If + Next + If Ambiguous Then Err.Raise vbObjectError + 880, "QuickPartReplace.AttributeSource", _ + "attribute ownership ambiguous; every occurrence retained with unknown Block/Symbol layer; no speculative writes allowed" +End Sub + +Sub QPWriteValueReadback(ByRef Command, ByRef Records, ByRef Plans, ByVal State) + Dim Text, Index, Comp, Attr, Present, Value, Selectors, Schematic, Sheet, SnapshotPath + Selectors = "": Schematic = "": Sheet = "" + If Command.Exists("target_selectors") Then Selectors = QPDecode(CStr(Command("target_selectors"))) + If Command.Exists("schematic_name") Then Schematic = CStr(Command("schematic_name")) + If Command.Exists("sheet_number") Then Sheet = CStr(Command("sheet_number")) + SnapshotPath = QPJobSnapshotPath(CStr(Command("job_id"))) + Text = "V" & vbTab & "2" & vbTab & QPEncode(CStr(Command("task_id"))) & vbTab & _ + QPEncode(CStr(Command("job_id"))) & vbTab & QPEncode(State) & vbTab & _ + QPEncode(CStr(Command("project_path"))) & vbTab & QPEncode(Schematic) & vbTab & QPEncode(Sheet) & vbTab & _ + QPEncode(CStr(Command("old_part_number"))) & vbTab & QPEncode(CStr(Command("new_part_number"))) & vbTab & _ + QPEncode(QPValueMode(Command)) & vbTab & QPEncode(QPCustomValue(Command)) & vbTab & _ + QPBooleanText(QPSyncNc(Command)) & vbTab & QPEncode(QPTargetCell(Command)) & vbTab & _ + QPEncode(Selectors) & vbTab & QPEncode(QPReadUtf8(SnapshotPath)) & vbCrLf + For Index = 0 To Plans.Count - 1 + Set Comp = Plans.Item(CStr(Index)) + Set Attr = QPFindCurrentAttribute(Comp, "Value") + Present = Not (Attr Is Nothing) + Value = "" + If Present Then Value = QPEffectiveAttributeValue(Attr) + Text = Text & "C" & vbTab & QPEncode(QPComponentIdentityKey(Comp)) & vbTab & _ + QPEncode(CStr(Comp.Refdes)) & vbTab & QPEncode(QPEffectiveComponentAttribute(Comp, "Part Number")) & vbTab & _ + QPEncode(Value) & vbTab & QPBooleanText(Present) & vbTab & _ + QPEncode(QPEffectiveComponentAttribute(Comp, "Cell Name")) & vbCrLf + Text = Text & QPAttributeObservations(Comp) + Next + QPWriteUtf8 QP_Fso.BuildPath(QP_Root, "value-readback-" & CStr(Command("job_id")) & ".tsv"), Text +End Sub + +Sub QPRequireAlreadyTargetPolicy(ByRef Comp, ByRef Command, ByRef Selector) + Dim ExpectedValue, ActualValue, Mode, Cell + If CStr(Comp.Refdes) <> CStr(Selector(0)) Then _ + Err.Raise vbObjectError + 873, "QuickPartReplace.Target", "already-target Refdes differs from frozen selector" + Mode = QPValueMode(Command) + Select Case Mode + Case "preserve": ExpectedValue = CStr(Selector(1)) + Case "custom": ExpectedValue = QPCustomValue(Command) + Case "part_number": ExpectedValue = QPEffectiveComponentAttribute(Comp, "Part Number") + Case "part_label", "legacy_part_label" + ExpectedValue = QPEffectiveComponentAttribute(Comp, "Part Label") + If Len(ExpectedValue) = 0 Then Err.Raise vbObjectError + 880, "QuickPartReplace.Target", _ + "already-target Part Label is unavailable; Value policy cannot be proven" + Case Else: Err.Raise vbObjectError + 880, "QuickPartReplace.Target", "already-target Value policy cannot be proven" + End Select + If QPSyncNc(Command) And QPValueHasNcMarker(CStr(Selector(1))) Then ExpectedValue = QPAppendNcMarker(ExpectedValue) + ActualValue = QPEffectiveComponentAttribute(Comp, "Value") + Cell = QPTargetCell(Command) + If ActualValue <> ExpectedValue Or (Len(Cell) > 0 And QPEffectiveComponentAttribute(Comp, "Cell Name") <> Cell) Then _ + Err.Raise vbObjectError + 880, "QuickPartReplace.Target", "already-target Part Number has unresolved Value/Cell policy; no replacement or completion claimed" +End Sub + Function QPRequiredAttributeValue(ByRef Comp, ByVal Name) Dim Attr, ErrorNumber, ErrorSource, ErrorDescription On Error Resume Next diff --git a/src/dx_productivity/dsh/execution_runs.py b/src/dx_productivity/dsh/execution_runs.py index 0ba0378..a5f886a 100644 --- a/src/dx_productivity/dsh/execution_runs.py +++ b/src/dx_productivity/dsh/execution_runs.py @@ -81,7 +81,7 @@ def _execute_confirmed_run(service, root, task_ids, report, stop, run_id, *, res schedules.finish(schedule['schedule_id'], 'interrupted', run_id=run_id, receipt=failure) raise entries = [{'task_id': e.task_id, 'state': e.state, 'message': e.message} for e in result.entries] - status = ('blocked' if result.paused and any(e['state'] == 'blocked' for e in entries) + status = ('blocked' if any(e['state'] == 'blocked' for e in entries) else 'paused' if result.paused else 'failed' if any(e['state'] == 'failed' for e in entries) else 'completed') receipt = {'status': status, 'run_id': run_id, 'queue_id': result.queue_id, 'paused': result.paused, 'user_pause_requested': bool(stop.is_set()), 'awaiting_user_resume': bool(result.paused and stop.is_set()), diff --git a/src/dx_productivity/dsh/incidents.py b/src/dx_productivity/dsh/incidents.py index f526a4b..cd3382d 100644 --- a/src/dx_productivity/dsh/incidents.py +++ b/src/dx_productivity/dsh/incidents.py @@ -1,6 +1,7 @@ """Durable automatic diagnosis, bounded repair, verification and escalation.""" from __future__ import annotations +from contextlib import nullcontext from hashlib import sha256 import json import os @@ -34,10 +35,11 @@ def decision(reply): class IncidentRunner: - def __init__(self, data_root, service, analyze, publish, notify=lambda row: None): + def __init__(self, data_root, service, analyze, publish, notify=lambda row: None, *, action_scope=None): self.data_root, self.service = Path(data_root), service self.analyze, self.publish, self.notify = analyze, publish, notify self.decisions = DecisionLog(self.data_root) + self.action_scope = action_scope or (lambda _action: nullcontext()) def _record_decision(self, record, evidence, *, outcome, human_takeover=False): refs = [] @@ -60,12 +62,13 @@ class IncidentRunner: record['decision_log_status'] = 'unavailable' evidence.event('decision_log_unavailable', error=type(error).__name__) - def run(self, task_id, issue_code, message, *, run_id='observation', execution_authorized=False, stop_event=None): + def run(self, task_id, issue_code, message, *, run_id='observation', execution_authorized=False, stop_event=None, cancel_event=None): with _RUN_LOCK: return self._run(task_id, issue_code, message, run_id=run_id, - execution_authorized=execution_authorized, stop_event=stop_event or Event()) + execution_authorized=execution_authorized, stop_event=stop_event or Event(), + cancel_event=cancel_event or Event()) - def _run(self, task_id, issue_code, message, *, run_id, execution_authorized, stop_event): + def _run(self, task_id, issue_code, message, *, run_id, execution_authorized, stop_event, cancel_event): incident_id = sha256(f'{task_id}:{run_id}:{issue_code}'.encode()).hexdigest()[:24] evidence = IncidentEvidence(self.data_root, incident_id) state_path = evidence.directory / 'incident.json' @@ -88,10 +91,12 @@ class IncidentRunner: self.notify(dict(record)) update('collecting') snapshot, context = self._snapshot(task_id, issue_code, message) + context_fingerprint = repr(context) record['task_name'] = snapshot.get('task_name', task_id) budget_id = sha256(f'{task_id}:{run_id}'.encode()).hexdigest()[:24] budget_path = self.data_root / 'dsh' / 'retry-budget' / (budget_id + '.json') - budget_available = not budget_path.exists() + reserve_retry = getattr(self.service, 'reserve_problem_retry', None) + budget_available = reserve_retry is not None or not budget_path.exists() snapshot['intervention'] = {'original_execution_authorized': bool(execution_authorized), 'retry_attempts': 0 if budget_available else 1, 'phase': getattr(context, 'phase', 'unknown'), 'interrupted': getattr(context, 'interrupted', True), 'known_retryable_error': issue_code in RETRYABLE_CODES, @@ -112,7 +117,16 @@ class IncidentRunner: write_json(evidence.directory / (phase + '.json'), failure) evidence.event('analysis_unavailable', **failure) return None + def cancelled(): + update('pending_issue', cancellation='Automatic intervention cancelled; no further action or publication was started.') + self._draft(record, evidence) + evidence.pack(record) + return record + if cancel_event.is_set(): + return cancelled() suggestion = analyze('diagnose') + if cancel_event.is_set(): + return cancelled() policy = classify_failure(issue_code, getattr(context.task.status, 'value', '') if context is not None else 'unknown', interrupted=context is None or bool(getattr(context, 'interrupted', True)), authorized=bool(execution_authorized)) @@ -126,41 +140,48 @@ class IncidentRunner: write_json(evidence.directory / 'gates.json', gates) safe = all(value is True for key, value in gates.items() if key != 'failure_policy') and policy['retry_allowed'] if safe: - # Consume before entering a service mutation. A crash or changed error - # code must never buy another replay of the same authorized run. - budget_path.parent.mkdir(parents=True, exist_ok=True) try: - with budget_path.open('x', encoding='utf8') as stream: - json.dump({'incident_id': incident_id, 'task_id': task_id, 'run_id': run_id, 'attempts': 1}, stream) - stream.flush() - os.fsync(stream.fileno()) - except FileExistsError: - safe = False - gates['retry_budget_available'] = False - write_json(evidence.directory / 'gates.json', gates) - if safe: - update('retrying', attempts=1) - try: - # Re-read at the actual mutation boundary; the model never supplies parameters. - current = self.service.task_context(task_id) - if current.interrupted or current.phase != 'retry' or stop_event.is_set(): - raise ValueError('retry_gate_changed') - self.service.prepare_failed_quick_run(task_id) - prepared = self.service.task_context(task_id) - p = prepared.progress - expected = (str(p.next_project_path), p.next_old_part_number, p.next_new_part_number, - 'part_label' if prepared.task.copy_part_number_to_value else 'preserve') - if prepared.interrupted or not p.next_project_path or stop_event.is_set(): - raise ValueError('retry_execution_gate_changed') - outcome = self.service.execute_quick_run_queue(task_id, expected_item=expected, stop_event=stop_event) - update('verifying') - current = self.service.task_context(task_id) - p = current.progress - verification = {'queue_complete': p.queue_complete, 'failed_items': p.failed_items, - 'saved_items': p.saved_items, 'completed_items': p.completed_items, - 'interrupted': current.interrupted, 'paused': outcome.paused} - write_json(evidence.directory / 'verification.json', verification) - record['verification'] = verification + with self.action_scope('retry_failed'): + # Compare immutable evidence after the model wait and GUI admission. + current = self.service.task_context(task_id) + if (repr(current) != context_fingerprint or current.interrupted or current.phase != 'retry' + or stop_event.is_set() or cancel_event.is_set()): + raise ValueError('retry_gate_changed') + if reserve_retry is not None: + reserved = reserve_retry(task_id, source='dsh') + if not reserved: + gates['retry_budget_available'] = False + write_json(evidence.directory / 'gates.json', gates) + raise ValueError('retry_budget_spent') + else: + # Legacy service adapters use the original exclusive durable budget. + budget_path.parent.mkdir(parents=True, exist_ok=True) + with budget_path.open('x', encoding='utf8') as stream: + json.dump({'incident_id': incident_id, 'task_id': task_id, 'run_id': run_id, 'attempts': 1}, stream) + stream.flush() + os.fsync(stream.fileno()) + update('retrying', attempts=1) + if stop_event.is_set() or cancel_event.is_set(): + raise ValueError('retry_gate_changed') + if reserve_retry is not None: + self.service.prepare_failed_quick_run(task_id, operation_keys=reserved) + else: + self.service.prepare_failed_quick_run(task_id) + prepared = self.service.task_context(task_id) + p = prepared.progress + expected = (str(p.next_project_path), p.next_old_part_number, p.next_new_part_number, + 'part_label' if prepared.task.copy_part_number_to_value else 'preserve') + if prepared.interrupted or not p.next_project_path or stop_event.is_set() or cancel_event.is_set(): + raise ValueError('retry_execution_gate_changed') + outcome = self.service.execute_quick_run_queue(task_id, expected_item=expected, stop_event=stop_event) + update('verifying') + current = self.service.task_context(task_id) + p = current.progress + verification = {'queue_complete': p.queue_complete, 'failed_items': p.failed_items, + 'saved_items': p.saved_items, 'completed_items': p.completed_items, + 'interrupted': current.interrupted, 'paused': outcome.paused} + write_json(evidence.directory / 'verification.json', verification) + record['verification'] = verification if p.queue_complete and not p.failed_items and not current.interrupted and not outcome.paused: update('resolved') evidence.capture('diagnostics-after') @@ -173,6 +194,8 @@ class IncidentRunner: record['retry_error'] = scrub({'type': type(error).__name__, 'code': getattr(error, 'code', ''), 'detail': str(error)[:1000]}) else: record['retry_skipped'] = 'Policy, execution authorization, current evidence or model recommendation did not admit replay.' + if cancel_event.is_set(): + return cancelled() snapshot, _ = self._snapshot(task_id, issue_code, message) snapshot['intervention'] = {'retry_attempts': record['attempts'], 'gates': gates, 'verification': record.get('verification')} @@ -185,6 +208,8 @@ class IncidentRunner: original_excerpt = evidence.excerpt evidence.excerpt = lambda: original_excerpt() + '\nGATES:\n' + json.dumps(gates) + '\nACTIONS:\n' + (evidence.directory / 'actions.json').read_text(encoding='utf8') analyze('reflect') + if cancel_event.is_set(): + return cancelled() update('pending_issue') self._draft(record, evidence) archive = evidence.pack(record) diff --git a/src/dx_productivity/dsh/resume_diff.py b/src/dx_productivity/dsh/resume_diff.py index 2121f47..fcdc28f 100644 --- a/src/dx_productivity/dsh/resume_diff.py +++ b/src/dx_productivity/dsh/resume_diff.py @@ -9,12 +9,13 @@ def resume_diff(service, inspected_run, *, current_plan_hashes=None): current = {} if current_plan_hashes is None else dict(current_plan_hashes) blockers = list(inspected_run.get('blockers') or []) contexts = {} + isolated = set(inspected_run.get('isolated_task_ids') or ()) for task_id in ids: try: contexts[task_id] = service.task_context(task_id) if current_plan_hashes is None: current[task_id] = _plan_hash(service.for_project_task(task_id), task_id) - if contexts[task_id].interrupted: + if contexts[task_id].interrupted and task_id not in isolated: blockers.append('interrupted') except Exception: blockers.append('task_unavailable') diff --git a/src/dx_productivity/dsh/run_journal.py b/src/dx_productivity/dsh/run_journal.py index ba2e05e..de8c391 100644 --- a/src/dx_productivity/dsh/run_journal.py +++ b/src/dx_productivity/dsh/run_journal.py @@ -156,18 +156,34 @@ class RunJournal: raise TaskCenterError("run_missing", "run does not exist") blockers: list[str] = [] checkpoint = self.data_root / "task-queues" / f"{run_id}.json" + workspace_checkpoint = self.data_root / "workspace-queues" / f"{run_id}.json" + workspace = workspace_checkpoint.is_file() + conflicting = workspace and checkpoint.is_file() + if workspace: + checkpoint = workspace_checkpoint queue = None try: queue = json.loads(checkpoint.read_text(encoding="utf-8")) + if conflicting: + raise ValueError('ambiguous checkpoint ownership') if queue.get("version") != 1 or queue.get("task_ids") != row["task_ids"] or queue.get("plan_hashes") != row["plan_hashes"]: raise ValueError entries = queue["entries"] if set(entries) != set(row["task_ids"]) or any( - not isinstance(entry, dict) or set(entry) != {"state", "message"} or + not isinstance(entry, dict) or (not {"state", "message", "released", "problems_done", "checkpoint_id"} <= set(entry) + or set(entry) - {"state", "message", "released", "problems_done", "checkpoint_id", "supervisor_batch_id"} + if workspace else set(entry) != {"state", "message"}) or entry["state"] not in {"pending", "running", "paused", "blocked", "completed", "failed", "skipped"} or not isinstance(entry["message"], str) for entry in entries.values() ): raise ValueError + if workspace and (queue.get('phase') not in {'primary', 'problems', 'finished'} or any( + type(entry['released']) is not bool or type(entry['problems_done']) is not bool + or not isinstance(entry['checkpoint_id'], str) or not _ID.fullmatch(entry['checkpoint_id']) + or 'supervisor_batch_id' in entry and (not isinstance(entry['supervisor_batch_id'], str) + or not _ID.fullmatch(entry['supervisor_batch_id'])) + for entry in entries.values())): + raise ValueError('invalid workspace checkpoint') except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError): queue = None blockers.append("queue_missing" if not checkpoint.exists() else "queue_invalid") @@ -175,10 +191,17 @@ class RunJournal: tasks = [] has_evidence = False current_hashes = {} + released_primary = {key for key, entry in (queue['entries'].items() if workspace and queue else ()) + if entry['released'] and entry['state'] in {'completed', 'failed', 'skipped', 'blocked'}} + isolated = {key for key in released_primary if queue['entries'][key]['state'] == 'blocked'} + safe_continuations = set() for task_id in row["task_ids"]: item = {"task_id": task_id, "status": "unknown", "saved_items": 0, "failed_items": 0, "no_match_items": 0} try: context = service.task_context(task_id) + if (queue and queue['entries'][task_id]['state'] in {'paused', 'running'} + and not context.interrupted and service.independent_continuation_ready(task_id)): + safe_continuations.add(task_id) has_evidence = has_evidence or context.has_evidence progress = context.progress item.update({"status": context.task.status.value, "interrupted": bool(context.interrupted), @@ -189,9 +212,9 @@ class RunJournal: for key in ("saved_items", "failed_items", "no_match_items"): counters[key] += item[key] counters["total"] += item["total_items"] - if context.interrupted: + if context.interrupted and task_id not in isolated: blockers.append("interrupted") - if getattr(progress, "failed_items", 0): + if getattr(progress, "failed_items", 0) and task_id not in released_primary | safe_continuations: blockers.append("failed_items") if queue and queue['entries'][task_id]['state'] == 'completed' and not ( context.has_evidence and progress.queue_complete is True @@ -208,21 +231,31 @@ class RunJournal: if queue: counters["completed"] = sum(queue["entries"][task_id]["state"] in {"completed", "failed", "skipped"} for task_id in row["task_ids"]) remaining = [task_id for task_id in row["task_ids"] if not queue or queue["entries"][task_id]["state"] not in {"completed", "failed", "skipped"}] + if workspace and queue: + primary_exhausted = all(entry['state'] in {'completed', 'failed', 'skipped', 'blocked'} for entry in queue['entries'].values()) + remaining = [key for key in row['task_ids'] if key not in isolated and ( + queue['entries'][key]['state'] not in {'completed', 'failed', 'skipped'} or + (queue['phase'] == 'problems' or queue['phase'] == 'primary' and primary_exhausted) + and not queue['entries'][key]['problems_done'])] if "queue_missing" in blockers and not has_evidence: # The receipt may have committed before the executor created its checkpoint. # A native reconfirmation can safely create the original queue; no auto replay. blockers.remove("queue_missing") - if remaining: + runnable_remaining = [key for key in remaining if key not in released_primary] + if runnable_remaining: try: from .task_lifecycle import preflight - if not preflight(service, remaining, allow_completed=True)["ready"]: + if not preflight(service, runnable_remaining, allow_completed=True)["ready"]: blockers.append("preflight_blocked") except Exception: blockers.append("preflight_unavailable") - elif service.pending_batch_resources(): + if service.pending_batch_resources(): blockers.append("resources_pending") blockers = list(dict.fromkeys(blockers)) value = {**row, "queue_available": queue is not None, "tasks": tasks, "counters": counters, + 'isolated_task_ids': [key for key in row['task_ids'] if key in isolated], + 'released_primary_task_ids': [key for key in row['task_ids'] if key in released_primary], + 'safe_continuation_task_ids': [key for key in row['task_ids'] if key in safe_continuations], "checkpoint_states": {key: entry['state'] for key, entry in queue['entries'].items()} if queue else {}, "remaining_task_ids": remaining, "blockers": blockers, "resume_ready": not blockers, "requires_confirmation": True, @@ -233,3 +266,22 @@ class RunJournal: value['blockers'] = value['diff']['blockers'] value['resume_ready'] = False return value + + +def resume_preflight(service, inspected_run): + """Retain the original scope while displaying tasks held out of this resume.""" + from .task_lifecycle import preflight + if not inspected_run.get('resume_ready') or not inspected_run.get('diff', {}).get('safe'): + raise TaskCenterError('run_blocked', '原队列范围或证据尚未通过恢复准入。') + report = preflight(service, inspected_run['task_ids'], allow_completed=True) + if report['plan_hashes'] != inspected_run['plan_hashes']: + raise TaskCenterError('assistant_stale', '原执行计划已变化,请重新核对。') + held = set(inspected_run.get('released_primary_task_ids') or ()) + isolated = set(inspected_run.get('isolated_task_ids') or ()) + for task in report['tasks']: + if task['task_id'] in held: + task['blockers'] = [] + task['isolated'] = task['task_id'] in isolated + report['ready'] = all(not task['blockers'] for task in report['tasks']) + report['resume_scope'] = list(inspected_run['remaining_task_ids']) + return report diff --git a/src/dx_productivity/trial_task_center/batch.py b/src/dx_productivity/trial_task_center/batch.py index 901b53e..58f280a 100644 --- a/src/dx_productivity/trial_task_center/batch.py +++ b/src/dx_productivity/trial_task_center/batch.py @@ -22,6 +22,7 @@ from .retention import maintain_quick_run_storage from .schematic_backup import create_schematic_backup from .execution_metrics import measure_phase from .batch_backup import BatchBackupCoordinator, _write_backup_state +from .value_readback import value_readback_details @dataclass(frozen=True) @@ -46,7 +47,8 @@ def execute_batch( ) -> BatchRunResult: preparer = service._quick_run_preparer journal = getattr(preparer, 'journal', None) - if (journal is not None and journal.requires_retry(task_id)) or service._retryable_quick_run(task_id): + if ((journal is not None and journal.requires_retry(task_id)) or service._retryable_quick_run(task_id)) and not ( + journal is not None and journal.has_pending_components(task_id)): raise TaskCenterError('quick_run_retry_required', '本工程有失败项,普通继续不会重试;请准备失败项重试。') retention = maintain_quick_run_storage(preparer.data_root) if retention.removed_entries or retention.skipped_entries: @@ -149,6 +151,30 @@ def execute_batch( **({'runtime': runtime} if runtime is not None else {})) metric['state'] = last_result.state metric['completed_units'] = int(last_result.state in {'saved_ok', 'restored_ok', 'no_match'}) + except TaskCenterError as error: + current_row = session.journal_rows.get(metric['job_id'], {}) + if error.code not in {'quick_run_session', 'target_cell_catalog'} or current_row.get('execution_unit') != 'component': + raise + # Only an already committed safe failure may be consumed here. + issues = service.execution_problems(task_id) + from .execution_journal import command_identity, _hash + key = _hash(command_identity(current_row)) + if not any(issue['operation_key'] == key and issue['classification'] != 'uncertain' for issue in issues): + raise + candidate = preparer.progress(task_id, projects, rules, + copy_part_number_to_value=task.copy_part_number_to_value) + if candidate.queue_complete: + session.progress = candidate + else: + session = preparer.begin_batch_session(task_id, projects, rules, + copy_part_number_to_value=task.copy_part_number_to_value, + **({'runtime': runtime} if runtime is not None else {})) + candidate = session.progress + last_result = QuickRunStepResult(session.handoff.script_path, session.handoff.command_path, + session.handoff.status_path, session.handoff.log_path, BATCH_FAILED_STATE, error.message, + candidate.next_project_path, candidate.next_old_part_number, candidate.next_new_part_number, + candidate.next_schematic_name, candidate.next_sheet_number, candidate) + metric['state'] = BATCH_FAILED_STATE finally: worker_progress = session.handoff.command_path.parent / 'worker-progress.json' try: @@ -190,6 +216,7 @@ def execute_batch( append_task_event(service._logging, level=event_level, task_id=task_id, project_id=project_id, operation='quick_run_executed', message=last_result.message, details={'state': last_result.state, 'continue_queue': not updated.queue_complete, + **value_readback_details(last_result), 'timing': metric, 'project_path': str(current), 'schematic_name': progress.next_schematic_name, @@ -201,6 +228,24 @@ def execute_batch( task_id, last_result.state, updated.completed_items, updated.total_items) progress = updated if task.primary_project_id and last_result.state in {BATCH_FAILED_STATE, 'batch_skipped'}: + row = session.journal_rows.get(metric['job_id'], {}) + if row.get('execution_unit') == 'component': + # A failed component is terminal; rebuild its session before + # admitting another UID against the last verified baseline. + if project_session.worker is not None: + project_session.worker.close() + project_session.worker = None + from .quick_run import _run_process + if journal is None: + raise TaskCenterError('quick_run_uncertain', '缺少失败回执及工程健康证明,暂缓剩余位号。') + if not journal.failure_certified(task_id, row): + if preparer._process_runner is not _run_process or runtime is None: + raise TaskCenterError('quick_run_uncertain', '失败后缺少明确的未写入与资源释放证明,暂缓剩余位号。') + health = runtime.reset_project_after_safe_failure(current, + session.handoff.command_path.parent, job_id=metric['job_id']) + journal.certify_safe_failure(task_id, row, health) + service._batch_feedback = (task_id, progress, '问题位号已记录,继续独立位号。') + continue # execute_step has already rejected possible unverified native # mutation. Leave this engineering task's remaining work held, # but permit the outer serial queue to run the next project. diff --git a/src/dx_productivity/trial_task_center/batch_resources.py b/src/dx_productivity/trial_task_center/batch_resources.py index f470b5c..6305e1f 100644 --- a/src/dx_productivity/trial_task_center/batch_resources.py +++ b/src/dx_productivity/trial_task_center/batch_resources.py @@ -72,13 +72,55 @@ def pending_resources(root): record = json.loads(internal_path(path).read_text(encoding='utf-8')) if not _valid_record(record, path): raise ValueError('invalid resource ownership record') - if record.get('state') != 'released': + if record.get('state') != 'released' or not _released_evidence(record): pending.append(dict(record, record_path=str(path))) except (OSError, ValueError, TypeError) as error: pending.append({'state': 'blocked', 'record_path': str(path), 'error': str(error)}) + import sys + sessions = getattr(sys.modules.get(__package__ + '.isolated_cad'), 'RETAINED_CAD_SESSIONS', {}) + for session in sessions.values(): + if session.root.is_relative_to(canonical_path(root)): + pending.append({'state': 'blocked', 'kind': 'retained_cad', 'cad_pid': session.cad.pid, + 'desktop': session.desktop.name, 'record_path': str(session.root / 'private-desktop.json')}) + workers = getattr(sys.modules.get(__package__ + '.worker_monitor'), 'RETAINED_WORKERS', {}) + worker_paths = set() + for entry in tuple(workers.values()): + if entry['root'].is_relative_to(canonical_path(root)): + worker_paths.add(canonical_path(entry['path'])) + pending.append({'state': 'blocked', 'kind': 'retained_worker', + 'record_path': str(entry['path']), 'worker_pid': entry['process'].pid}) + for path in internal_path(canonical_path(root)).rglob('retained-worker-*.json'): + if canonical_path(path) in worker_paths: + continue + try: + record = json.loads(path.read_text(encoding='utf-8')) + if (record.get('version') == 1 and record.get('state') == 'released' + and record.get('processes_exited') is True + and path.name == 'retained-worker-' + record.get('token', '') + '.json'): + continue + except (OSError, ValueError, TypeError, AttributeError): + pass + # After restart there is no live Job handle to prove descendant exit. + pending.append({'state': 'blocked', 'kind': 'retained_worker', 'record_path': str(canonical_path(path))}) return tuple(pending) +def _released_evidence(record): + """A state label alone must never reopen the physical resource gate.""" + proof = record.get('release_proof', {}) + if (isinstance(proof, dict) and proof.get('generation') == record['generation'] + and proof.get('desktop_released') is True and proof.get('processes_exited') is True): + return True + releases = record.get('generation_releases', ()) + if not isinstance(releases, list) or not releases: + return False + last = releases[-1] + cleanup = last.get('cleanup') or {} + return (last.get('generation') == record['generation'] and last.get('desktop') == record['desktop'] + and cleanup.get('desktop_handle_closed') is True and not cleanup.get('errors') + and (cleanup.get('desktop_confirmation') or {}).get('released') is True) + + def guard_pending_resources(root): if pending_resources(root): raise BatchResourceCleanupError('上一批资源尚未确认释放;请先重试释放资源,再开始新批次。') @@ -86,7 +128,7 @@ def guard_pending_resources(root): def process_identity(process): result = {'pid': process.pid, 'creation_time': None} - handle = getattr(process, 'handle', None) + handle = getattr(process, 'handle', getattr(process, '_handle', None)) if handle is not None: import win32process result['creation_time'] = str(win32process.GetProcessTimes(handle)['CreationTime']) @@ -117,6 +159,17 @@ def _process_gone(identity): def cleanup_pending_resources(root, *, logging_context=None): from .isolated_desktop import _Win32Api from .batch_release_log import write_release_event + from .isolated_cad import RETAINED_CAD_SESSIONS + from .read_recovery import process_tree_exited + from .worker_monitor import cleanup_retained_workers + cleanup_retained_workers(root) + for session in tuple(RETAINED_CAD_SESSIONS.values()): + if (session.root.is_relative_to(canonical_path(root)) + and all(process_tree_exited(process) for process in session._retained_processes)): + try: + session.__exit__(None, None, None) + except Exception: + pass # Business uncertainty remains; only confirmed handle release removes the owner. for record in pending_resources(root): path = Path(record['record_path']) live = LIVE_RESOURCES.get(str(path)) @@ -145,16 +198,27 @@ def cleanup_pending_resources(root, *, logging_context=None): if not record.get('session_id'): continue # Corrupt evidence cannot authorize resource actions. log('started') + identities = list(record.get('processes', ())) + cad_identity = record.get('cad_process') + if record.get('cad_pid') is not None: + if (not isinstance(cad_identity, dict) or cad_identity.get('pid') != record['cad_pid'] + or not cad_identity.get('creation_time')): + log('failed', 'CAD 进程身份缺失,不能仅凭桌面消失解除资源保护') + continue + if cad_identity not in identities: + identities.append(cad_identity) + if not all(_process_gone(item) for item in identities): + log('failed', '记录中的进程仍未确认退出') + continue desktop = record.get('desktop') if desktop: _Win32Api().confirm_desktop_released(desktop) - if not all(_process_gone(item) for item in record.get('processes', ())): - log('failed', '记录中的进程仍未确认退出') - continue # After restart only confirm disappearance. Never attach to COM, # close a project, or terminate a PID whose handles we no longer own. record.pop('record_path', None) - record = dict(record, state='released', reconciled=True, released_at=time.time()) + record = dict(record, state='released', reconciled=True, released_at=time.time(), + release_proof={'generation': record['generation'], 'desktop_released': True, + 'processes_exited': True, 'processes': identities, 'method': 'restart_observation'}) write_record(path, record) log('succeeded') except Exception as error: @@ -188,3 +252,111 @@ def project_checkpoint_released(root): and record['checkpoints'][key] == checkpoint['attempt_id']) except (OSError, ValueError, TypeError, KeyError): return False + + +def _verify_retained_handles(processes, recorded): + from .read_recovery import process_tree_exited + for process in processes: + if process_tree_exited(process): + continue + identity = process_identity(process) + if not identity.get('creation_time') or identity not in recorded: + raise TaskCenterError('target_identity', '保留进程的创建身份与证据不符,拒绝强制结束。') + + +def force_pending_owned_resources(root, *, confirmed=False): + """Explicit GUI action: protect first, terminate only exact retained handles.""" + if confirmed is not True: + raise TaskCenterError('force_confirmation_required', '必须明确确认强制结束可能写入的进程。') + from . import batch_supervisor, isolated_cad + from .read_recovery import process_tree_exited + from .stop_control import StopControl, protect_interruption + root = canonical_path(root) + active = [(path, process, control) for path, (_, process, control) + in tuple(batch_supervisor._ACTIVE_SUPERVISORS.items()) + if canonical_path(path).parent.parent == root / 'batch-resources' + and control.waiting_for_resolution] + if active: + # Signal the owning supervision loop; never race its process/Job close. + for path, process, control in active: + journal = json.loads(internal_path(path).read_text(encoding='utf-8')) + identity = process_identity(process) + if (not batch_supervisor.valid_journal(journal, path.parent.name) + or not identity.get('creation_time') or identity != journal.get('process')): + raise TaskCenterError('target_identity', '活动监督进程身份不符,拒绝强制结束。') + protect_interruption(path.parent, 'user_confirmed_force_pending_owned_resources') + write_record(path.parent / 'force-request.json', {'version': 1, 'batch_id': journal['batch_id'], + 'attempt_id': journal['attempt_id'], 'process': identity, 'confirmed': True}) + control.request_force() + return tuple({'state': 'blocked', 'record_path': str(path), 'force_requested': True} + for path, _, _ in active) + with batch_supervisor.supervisor_lease(root): + for path, (api, process) in tuple(batch_supervisor._LIVE_SUPERVISORS.items()): + if canonical_path(path).parent.parent != root / 'batch-resources': + continue + journal = json.loads(internal_path(path).read_text(encoding='utf-8')) + if (not batch_supervisor.valid_journal(journal, path.parent.name) + or process_identity(process) != journal.get('process')): + raise TaskCenterError('target_identity', '监督进程身份已变化,拒绝强制结束。') + batch_supervisor._force_owned_supervisor(root, journal, process) + process.close() + api.close_pending() + batch_supervisor._LIVE_SUPERVISORS.pop(path, None) + for path, runtime in tuple(LIVE_RESOURCES.items()): + if canonical_path(path).parent.parent != root / 'batch-resources': + continue + record = json.loads(internal_path(path).read_text(encoding='utf-8')) + if (not _valid_record(record, Path(path)) or record['session_id'] != runtime.session_id + or record['generation'] != runtime.generation + or record['desktop'] != getattr(runtime.desktop, 'name', None)): + raise TaskCenterError('target_identity', '批次资源会话身份已变化,拒绝强制结束。') + processes = list(getattr(runtime.desktop, '_processes', getattr(runtime.desktop, 'processes', ()))) + recorded = list(record['processes']) + if record.get('cad_process'): + recorded.append(record['cad_process']) + _verify_retained_handles(processes, recorded) + evidence = runtime._active_evidence or runtime.path.parent + protect_interruption(evidence, 'user_confirmed_force_pending_owned_resources') + runtime._save(force_state='requested', force_confirmed=True, force_requested_at=time.time()) + control = StopControl() + control.request_force() + runtime.stop_event = control + if runtime.cad is not None: + runtime.cad.stop_control = control + for process in processes: + if not process_tree_exited(process): + process.terminate(reason='user_confirmed_pending_resource_force') + try: + runtime.close(reason='manual_retry') + except Exception as error: + runtime.record['force_cleanup_error'] = str(error) + runtime._save() + for key, session in tuple(isolated_cad.RETAINED_CAD_SESSIONS.items()): + if not session.root.is_relative_to(root): + continue + record = json.loads(internal_path(session.root / 'private-desktop.json').read_text(encoding='utf-8')) + processes = session._retained_processes + if (record.get('token') != session.token or record.get('desktop') != session.desktop.name + or record.get('cad_pid') != session.cad.pid or not record.get('retained_for_manual')): + raise TaskCenterError('target_identity', '保留 CAD 会话身份已变化,拒绝强制结束。') + _verify_retained_handles(processes, record.get('retained_processes', ())) + protect_interruption(session.root, 'user_confirmed_force_pending_owned_resources') + session._record(force_state='requested', force_confirmed=True, force_requested_at=time.time()) + control = StopControl() + control.request_force() + session.stop_control = control + for process in processes: + if not process_tree_exited(process): + process.terminate(reason='user_confirmed_pending_resource_force') + try: + session.__exit__(None, None, None) + except Exception: + if not all(process_tree_exited(process) for process in processes): + raise + from .worker_monitor import cleanup_retained_workers + cleanup_retained_workers(root, force=True) + pending = cleanup_pending_resources(root) + supervisors = batch_supervisor.reconcile_supervisors(root) + retained = tuple({'state': 'blocked', 'record_path': str(session.root / 'private-desktop.json')} + for session in isolated_cad.RETAINED_CAD_SESSIONS.values() if session.root.is_relative_to(root)) + return (*pending, *supervisors, *retained) diff --git a/src/dx_productivity/trial_task_center/batch_runtime.py b/src/dx_productivity/trial_task_center/batch_runtime.py index 63d93f6..37fc607 100644 --- a/src/dx_productivity/trial_task_center/batch_runtime.py +++ b/src/dx_productivity/trial_task_center/batch_runtime.py @@ -75,6 +75,17 @@ class _DesktopLease: raise WorkerMonitorError('stale or unavailable batch dialog owner') self.desktop.acknowledge_dialog(dialog) + def manual_dialog_action(self, dialog, button_hwnd): + if (self.desktop is None or self.generation != self.owner.generation or + not any(process.pid == dialog['pid'] and process.poll() is None for process in self.processes)): + raise WorkerMonitorError('stale or unavailable batch dialog owner') + self.desktop.manual_dialog_action(dialog, button_hwnd) + + def register_cad_process(self, process, token): + if self.generation != self.owner.generation or not any(item is process for item in self.processes): + raise WorkerMonitorError('CAD startup process does not belong to this desktop lease') + self.owner._save(cad_pid=process.pid, cad_process=process_identity(process), cad_token=token) + def close(self): errors, remaining, records = [], [], [] for process in reversed(self.processes): @@ -101,6 +112,7 @@ class BatchCadRuntime: def __init__(self, root, kind, stop_event=None, *, desktop_factory=None, cad_factory=None, session_id=None): self.root = canonical_path(root) self.kind, self.stop_event = kind, stop_event + self.read_only = kind in {'bom_import', 'collect_target_inventory'} self.session_id = uuid4().hex if session_id is None else UUID(str(session_id)).hex self.path = self.root / 'batch-resources' / self.session_id / 'session.json' self.desktop_factory = desktop_factory or PrivateDesktop @@ -109,6 +121,7 @@ class BatchCadRuntime: self.generation = 0 self.result = None self.record = {'version': 1, 'session_id': self.session_id, 'kind': kind, 'state': 'new', + 'read_only': self.read_only, 'generation': 0, 'desktop': None, 'cad_pid': None, 'cad_guid': None, 'processes': [], 'checkpoints': {}, 'retries': {}, 'timings': [], 'native_operations': {}} self._retry_counts = {} @@ -116,6 +129,8 @@ class BatchCadRuntime: self._logging = None self._active_evidence = None self._active_attempts = [] + self._current_attempts = [] + self._attempt_sources = {} self._close_failure = None self._close_failure_rebuild = False @@ -182,7 +197,7 @@ class BatchCadRuntime: name = 'dx-' + uuid4().hex self._save(state='starting', generation=self.generation, desktop=name, processes=[], cad_pid=None, cad_guid=None, checkpoints={}, native_operations={}, - cad_shutdown=None, cad_start_requested=False, cleanup=None) + cad_process=None, cad_token=None, cad_shutdown=None, cad_start_requested=False, cleanup=None) LIVE_RESOURCES[str(self.path)] = self self.desktop = self.desktop_factory(name, enabled=True) started = time.monotonic() @@ -212,6 +227,96 @@ class BatchCadRuntime: def native_scope(self, evidence): return _DesktopLease(self, evidence, native=True) + def owned_processes_exited(self): + """Observe exact retained handles, never infer exit from a timeout.""" + processes = list(getattr(self.desktop, '_processes', getattr(self.desktop, 'processes', ()))) + cad_process = getattr(self.cad, 'cad', None) + if cad_process is not None and cad_process not in processes: + processes.append(cad_process) + try: + from .read_recovery import process_tree_exited + return bool(processes) and all(process_tree_exited(process) for process in processes) + except Exception: + return False + + def reset_project_after_safe_failure(self, project, evidence_root, *, job_id=None): + """Discard a checkpointed project session before another admitted item.""" + root = canonical_path(evidence_root) + try: + checkpoint = json.loads(internal_path(root / 'project-checkpoint.json').read_text(encoding='utf-8')) + safe = (self.record['state'] == 'ready' and self.cad is not None + and canonical_path(checkpoint['project_path']) == canonical_path(project) + and canonical_path(checkpoint['resource_record']) == canonical_path(self.path) + and checkpoint['generation'] == self.generation and project_checkpoint_released(root)) + except (OSError, ValueError, TypeError, KeyError): + safe = False + if not safe: + raise BatchResourceCleanupError('缺少当前工程关闭检查点证明,不能继续执行。') + from .batch_attempt import can_retry_attempt + from .discovery_cache import project_fingerprint + attempt_id = checkpoint['attempt_id'] + job_id = self.record.get('active_job') if job_id is None else job_id + selected = [(path, expected) for path, expected in self._current_attempts + if expected.get('attempt_id') == attempt_id and expected.get('job_id') == job_id + and expected.get('session_id') == self.session_id and expected.get('generation') == self.generation + and canonical_path(expected['project_path']) == canonical_path(project)] + attempt_path = selected[0][0] if len(selected) == 1 else None + no_mutation = bool(len(selected) == 1 and can_retry_attempt(*selected[0])) + before = self._attempt_sources.get(attempt_id) + checkpoint_digest = hashlib.sha256(internal_path(root / 'project-checkpoint.json').read_bytes()).hexdigest() + self.close(rebuilding=True, reason='safe_project_failure') + self.desktop = self.cad = None + self._active_evidence = None + self._active_attempts = [] + after = project_fingerprint(canonical_path(project), self.root) + if after is None: + raise BatchResourceCleanupError('资源已释放,但无法核对工程源文件,不能证明安全继续。') + release_path = self.path.parent / ('generation-release-' + attempt_id + '.json') + write_record(release_path, self.record) + job_token = hashlib.sha256(str(job_id).encode()).hexdigest()[:12] + proof_path = root / ('project-health-' + attempt_id + '-' + job_token + '.json') + proof = {'version': 1, 'session_id': self.session_id, 'generation': self.generation, + 'project_path': str(canonical_path(project)), 'job_id': job_id, 'attempt_id': attempt_id, + 'project_closed': True, 'resources_released': True, 'no_mutation': no_mutation, + 'attempt_record_path': str(attempt_path) if attempt_path is not None else None, + 'attempt_record_sha256': hashlib.sha256(internal_path(attempt_path).read_bytes()).hexdigest() if attempt_path else None, + 'source_sha256': hashlib.sha256(repr(after).encode()).hexdigest(), + 'source_unchanged': before is not None and before == after, + 'resource_record': str(release_path), 'resource_record_sha256': hashlib.sha256(internal_path(release_path).read_bytes()).hexdigest(), + 'session_record': str(self.path), + 'checkpoint_sha256': checkpoint_digest, 'proof_path': str(proof_path)} + write_record(proof_path, proof) + return proof + + def release_idle_for_fingerprint(self, project): + if not self.read_only or self.cad is None or self._active_evidence is None: + return False + if canonical_path(self.record.get('active_project', '')) != canonical_path(project): + return False + self.reset_project_after_safe_failure(project, self._active_evidence) + return True + + def dialog_state(self): + guard = getattr(self.cad, '_dialogs', None) + return guard.summary() if guard is not None else None + + def business_progress(self): + if self._active_evidence is None: + return None + try: + path = internal_path(self._active_evidence / 'worker-progress.json') + value = json.loads(path.read_text(encoding='utf-8')) + if not isinstance(value, dict): + return None + return {key: value.get(key) for key in ('job_id', 'stage', 'done', 'total', 'safe_stop')} + except (OSError, ValueError, TypeError): + return None + + def manual_dialog_action(self, dialog, button_hwnd): + if self.cad is None or self.cad.cad.poll() is not None: + raise WorkerMonitorError('owned CAD dialog session is unavailable') + self.cad._dialogs.manual_action(dialog, button_hwnd) + def _ensure_cad(self, abort_event=None): if self.cad is None: from .isolated_cad import IsolatedCadSession @@ -221,6 +326,7 @@ class BatchCadRuntime: self._save(cad_start_requested=True) cad = factory(self.path.parent / f'generation-{self.generation + (self.desktop is None)}', abort_event=abort_event, desktop=lease, + **({'read_only': True} if self.read_only else {}), **({'stop_control': self.stop_event} if hasattr(self.stop_event, 'force_event') else {})) self.cad = cad # Retain cleanup ownership even if startup fails. try: @@ -233,7 +339,8 @@ class BatchCadRuntime: self.cad = None raise self._timing('cad_start', started) - self._save(state='ready', cad_pid=cad.binding.pid, cad_guid=cad.binding.guid) + self._save(state='ready', cad_pid=cad.binding.pid, cad_guid=cad.binding.guid, + cad_process=process_identity(cad.cad) if getattr(cad, 'cad', None) is not None else None) return self.cad def _prepare_attempts(self, root, command, attempt_id, generation): @@ -255,6 +362,7 @@ class BatchCadRuntime: task_id=row[0], job_id=row[1], project_path=str(project), attempt_id=attempt_id) expected.append((path, record)) self._project_attempts.setdefault(str(project).casefold(), []).extend(expected) + self._current_attempts = list(expected) return project, legacy_path(directory), expected def _can_retry(self, project, expected): @@ -278,7 +386,19 @@ class BatchCadRuntime: from .discovery_cache import project_fingerprint from .isolated_cad import PrivateCadStartupError, is_identity_conflict from .worker_monitor import run_monitored_process + cooperative_abort = cooperative_abort or self.read_only + if self.read_only and abort_event is None: + abort_event = self.stop_event root = canonical_path(command[3]) + if self.cad is not None and self.record.get('active_project'): + from .quick_run import _read_commands + selected = [row for row in _read_commands(internal_path(root / 'command.tsv')) if row[1] == command[-1]] + if len(selected) != 1: + raise TaskCenterError('target_identity', '批次工作命令身份缺失或重复,已停止。') + if canonical_path(selected[0][2]) != canonical_path(self.record['active_project']): + if self._active_evidence is None: + raise BatchResourceCleanupError('缺少上一工程关闭检查点,不能启动下一工程。') + self.reset_project_after_safe_failure(self.record['active_project'], self._active_evidence) self._active_evidence = root while True: check_cancelled(abort_event or self.stop_event) @@ -298,6 +418,7 @@ class BatchCadRuntime: # this task rather than a stale completed task from session.json. self._save(active_task=self.record['active_task'], active_attempt=attempt_id) before = project_fingerprint(project, self.root) + self._attempt_sources[attempt_id] = before status = internal_path(root / 'status.tsv') status_before = status.read_bytes() if status.exists() else None started = time.monotonic() @@ -311,7 +432,10 @@ class BatchCadRuntime: return cad.start_worker(launch, extra_environment=environment, **options) result = run_monitored_process(command, timeout, abort_event=abort_event, cooperative_abort=cooperative_abort, process_factory=start_worker, + **({'owner_loss_path': self.path.parent / 'owner-loss.json'} if self.read_only else {}), **({'force_control': self.stop_event} if hasattr(self.stop_event, 'force_event') else {})) + if result.returncode and self.read_only: + raise WorkerMonitorError(result.stderr or 'read-only worker failed') if result.returncode: self._retain_uncertain(root, project, expected, WorkerMonitorError(result.stderr or 'worker failed after possible mutation')) @@ -324,6 +448,19 @@ class BatchCadRuntime: finally: self._timing('project_worker', started, project=str(project), attempt_id=attempt_id, task_id=self.record['active_task'], job_id=self.record['active_job']) + if self.read_only: + self.close(reason='read_failure') + write_record(root / 'project-read-release.json', {'version': 1, + 'session_id': self.session_id, 'generation': self.generation, + 'project_path': str(project), 'resource_record': str(self.path), + 'resources_released': True, 'error': str(failure)}) + self.desktop = self.cad = None + self._active_evidence = None + self._active_attempts = [] + check_cancelled(abort_event or self.stop_event) + from .project_discovery import ReleasedProjectReadError + raise ReleasedProjectReadError('quick_run_sheet_discovery', + f'工程读取失败,读取会话已释放,可继续其他工程:{project.name};{failure}') from failure # Missing proof, changed inputs/status, cancellation or a previous # write anywhere in this project can never authorize replay. after = project_fingerprint(project, self.root) @@ -346,11 +483,13 @@ class BatchCadRuntime: self.desktop = self.cad = None def _retain_uncertain(self, root, project, expected, error): + if self.read_only: + return from .batch_attempt import can_retry_attempt from .stop_control import wait_for_resolution attempts = self._project_attempts.get(str(project).casefold(), expected) if attempts and not all(can_retry_attempt(path, record) for path, record in attempts): - wait_for_resolution(self.stop_event, root, error) + wait_for_resolution(self.stop_event, root, error, resolved=self.owned_processes_exited) def recover_native_failure(self, project, evidence, reason): """One read-only recovery, based on durable source and worker proof.""" @@ -379,6 +518,17 @@ class BatchCadRuntime: return False except (OSError, ValueError, KeyError, TypeError): return False + if self.read_only: + self.close(reason='read_failure') + write_record(evidence / 'project-read-release.json', {'version': 1, + 'session_id': self.session_id, 'generation': self.generation, + 'project_path': str(project), 'resource_record': str(self.path), + 'resources_released': True, 'error': str(reason)}) + self.desktop = self.cad = None + self._active_evidence = None + from .project_discovery import ReleasedProjectReadError + raise ReleasedProjectReadError('quick_run_sheet_discovery', + f'原生工程读取失败,读取会话已释放,可继续其他工程:{project.name};{reason}') self._retry_counts[key] = 1 self.record['retries'][key] = {'count': 1, 'reason': str(reason), 'backend': 'native_cli'} self._save(state='recycling') @@ -428,14 +578,16 @@ class BatchCadRuntime: def _close_resources(self, *, rebuilding=False): started = time.monotonic() errors = [] + retained_cad = False try: self._save(state='closing') except Exception as error: errors.append('resource journal: ' + str(error)) if self.cad is not None: - cad, self.cad = self.cad, None + cad = self.cad try: cad.__exit__(None, None, None) + self.cad = None except Exception as error: if rebuilding: self.record['shutdown_warning'] = str(error) @@ -443,6 +595,15 @@ class BatchCadRuntime: # an unconfirmed shutdown. Keep the warning for diagnostics, # but retain the failure in the resource gate as well. errors.append(str(error)) + from .read_recovery import process_tree_exited + process = getattr(cad, 'cad', None) + processes = [*getattr(cad, '_retained_processes', ()), + *getattr(self.desktop, '_processes', getattr(self.desktop, 'processes', ()))] + if process is not None: + processes.append(process) + retained_cad = not processes or not all(process_tree_exited(item) for item in processes) + if not retained_cad: + self.cad = None finally: cad_record = getattr(cad, 'record', {}) cad_record = cad_record if isinstance(cad_record, dict) else {} @@ -450,8 +611,8 @@ class BatchCadRuntime: 'state', 'clean', 'cad_exit_code', 'quit_helper_exit_code', 'cleanup_errors', 'shutdown_seconds', 'orderly_shutdown', 'diagnostic_errors', 'forced_termination', 'forced_root_termination', 'forced_descendant_pids', 'termination_reason', - 'dialog_supervision') if key in cad_record} - if self.desktop is not None: + 'dialog_supervision', 'read_only_released') if key in cad_record} + if self.desktop is not None and not retained_cad: try: self.desktop.close() except Exception as error: diff --git a/src/dx_productivity/trial_task_center/batch_supervisor.py b/src/dx_productivity/trial_task_center/batch_supervisor.py index c8d5073..466bea3 100644 --- a/src/dx_productivity/trial_task_center/batch_supervisor.py +++ b/src/dx_productivity/trial_task_center/batch_supervisor.py @@ -21,9 +21,56 @@ from .supervisor_protocol import VERSION, MessageGate, valid_journal, encode, de _WORKER_ROOT = None _LAUNCH_LOCK = threading.Lock() _LIVE_SUPERVISORS = {} +_ACTIVE_SUPERVISORS = {} _WRITE_OPERATIONS = frozenset({'execute_task_queue', 'execute_quick_run_queue', 'execute_quick_run', 'recover_quick_run'}) +class BusinessProgressWatch: + """Control-channel liveness cannot certify forward business progress.""" + def __init__(self, started, idle_timeout=150): + self.last_progress = started + self.idle_timeout = idle_timeout + self.signatures = {} + + def observe(self, kind, payload, now): + if kind == 'heartbeat': + return + if kind == 'state': + payload = {key: payload.get(key) for key in ('phase', 'queue', 'batch', 'business_progress', 'active_project')} + if kind not in {'state', 'progress', 'phase'}: + return + signature = json.dumps([kind, payload], ensure_ascii=True, sort_keys=True, default=str) + if signature != self.signatures.get(kind): + self.last_progress = now + self.signatures[kind] = signature + + def stalled(self, now): + return now - self.last_progress >= self.idle_timeout + + +def request_manual_dialog_action(root, binding, dialog, button_hwnd): + """Publish one explicit UI choice for the current supervisor generation.""" + from .batch_resources import write_record, _valid_record, _process_gone + from uuid import UUID + batch_id, attempt_id = binding['batch_id'], binding['attempt_id'] + if UUID(batch_id).hex != batch_id or UUID(attempt_id).hex != attempt_id: + raise TaskCenterError('target_identity', '弹窗监督进程身份无效。') + directory = canonical_path(root) / 'batch-resources' / batch_id + journal = json.loads(internal_path(directory / 'supervisor.json').read_text(encoding='utf-8')) + resource = json.loads(internal_path(directory / 'session.json').read_text(encoding='utf-8')) + if (not valid_journal(journal, batch_id) or journal['attempt_id'] != attempt_id + or journal['state'] not in {'running', 'blocked'} or not journal.get('process') + or _process_gone(journal['process']) or not _valid_record(resource, directory / 'session.json') + or binding.get('generation') != resource['generation'] or not binding.get('session_token') + or binding['session_token'] != resource.get('cad_token') + or dialog.get('pid') != resource.get('cad_pid') or type(button_hwnd) is not int): + raise TaskCenterError('target_identity', '弹窗或工程会话已变化,请刷新后重新处理。') + request_id = uuid4().hex + write_record(directory / ('manual-dialog-' + request_id + '.json'), dict(binding, + version=VERSION, request_id=request_id, dialog=dialog, button_hwnd=button_hwnd)) + return request_id + + def _decode_process_output(value): if not isinstance(value, bytes): return str(value) @@ -70,9 +117,10 @@ def _worker_context(root): def _force_owned_supervisor(root, journal, process): """Persist uncertainty before terminating the exact process/Job we own.""" from .batch_resources import write_record + from .read_recovery import process_tree_exited path = Path(root) / 'batch-resources' / journal['batch_id'] / 'supervisor.json' if (journal.get('force_state') in {'termination_requested', 'confirmed'} - and process.poll() is not None): + and process_tree_exited(process)): return journal['stop_mode'] = 'forced' journal.setdefault('force_requested_at', time.time()) @@ -138,7 +186,11 @@ def _retain_supervisor_after_control_error(root, journal, process, control, erro return True try: process.wait(timeout=.2) - return False + from .read_recovery import process_tree_exited + if process_tree_exited(process): + control.waiting_for_resolution = False + return False + threading.Event().wait(.2) except subprocess.TimeoutExpired: pass except Exception as failure: @@ -196,15 +248,24 @@ def mutation_lease(root): def pending_supervisors(root): from .batch_resources import _process_gone + from .read_recovery import process_tree_exited pending = [] for path in (internal_path(root) / 'batch-resources').glob('*/supervisor.json'): try: value = json.loads(path.read_text(encoding='utf-8')) if not valid_journal(value, path.parent.name): raise ValueError('invalid supervisor ownership record') - if value.get('state') == 'finished': + owner = (_LIVE_SUPERVISORS.get(canonical_path(path)) + or _ACTIVE_SUPERVISORS.get(canonical_path(path))) + if owner is not None and not process_tree_exited(owner[1]): + pending.append(dict(value, record_path=str(canonical_path(path)), state='blocked')) continue identity = value.get('process') or value.get('parent_process') + if value.get('state') == 'finished': + if value.get('process') is None or _process_gone(value['process']): + continue + pending.append(dict(value, record_path=str(canonical_path(path)), state='blocked')) + continue if (value.get('state') in {'starting', 'running'} and identity and not _process_gone(identity) and value.get('parent_process') and not _process_gone(value['parent_process'])): continue @@ -217,8 +278,13 @@ def pending_supervisors(root): def reconcile_supervisors(root): """Recheck orphan ownership only; this never launches or replays a job.""" from .batch_resources import _process_gone, pending_resources, write_record + from .read_recovery import process_tree_exited physical = pending_resources(root) for record in pending_supervisors(root): + owner = (_LIVE_SUPERVISORS.get(canonical_path(record['record_path'])) + or _ACTIVE_SUPERVISORS.get(canonical_path(record['record_path']))) + if owner is not None and not process_tree_exited(owner[1]): + continue identity = record.get('process') if not record.get('batch_id') or (identity and not _process_gone(identity)): continue @@ -263,6 +329,9 @@ def _finish_response(root, journal, response, context): write_record(Path(root) / 'batch-resources' / journal['batch_id'] / 'supervisor.json', journal) if related: raise resource_pending_error(journal['operation'], result, error, primary) + for failure in (error, primary): + if failure is not None: + failure.resources_released = True resource_path = internal_path(Path(root) / 'batch-resources' / journal['batch_id'] / 'session.json') def preserve_shutdown_failure(detail): try: @@ -271,6 +340,10 @@ def _finish_response(root, journal, response, context): context.logger.error('supervisor_shutdown_record_failed batch_id=%s error=%s', journal['batch_id'], record_error) journal['shutdown_record_error'] = str(record_error) + def shutdown_failure(): + failure = application_shutdown_error(result, error, primary) + failure.resources_released = True + return failure try: resource = json.loads(resource_path.read_text(encoding='utf-8')) except FileNotFoundError: @@ -278,7 +351,7 @@ def _finish_response(root, journal, response, context): except (OSError, ValueError, TypeError) as failure: journal.update(state='blocked', shutdown_record_error=str(failure)) preserve_shutdown_failure(failure) - raise application_shutdown_error(result, error, primary) from failure + raise shutdown_failure() from failure shutdown = resource.get('cad_shutdown') if isinstance(resource, dict) else None if shutdown is None and isinstance(resource, dict): shutdown = {} @@ -287,11 +360,11 @@ def _finish_response(root, journal, response, context): or any(not isinstance(item, dict) for item in shutdown.get('cleanup_errors', ()))): journal.update(state='blocked', shutdown_record_error='invalid shutdown record shape') preserve_shutdown_failure('invalid shutdown record shape') - raise application_shutdown_error(result, error, primary) + raise shutdown_failure() if (shutdown.get('orderly_shutdown') is False or shutdown.get('forced_termination') or shutdown.get('diagnostic_errors') or any(item.get('stage') != 'desktop_release' for item in shutdown.get('cleanup_errors', ()))): - raise application_shutdown_error(result, error, primary) + raise shutdown_failure() if response.get('business_succeeded'): if response.get('error'): context.logger.info('supervisor_resource_recovery_passed batch_id=%s business_replayed=false', journal['batch_id']) @@ -310,18 +383,25 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres from .isolated_desktop import _Win32Api from .stop_control import StopControl root = canonical_path(service._logging.root) + force_control = stop_event if isinstance(stop_event, StopControl) else StopControl() if stop_event is not None and stop_event.is_set(): raise TaskCenterError('operation_cancelled', '本批操作已取消,未启动监督进程。') with supervisor_lease(root): if operation == 'cleanup_batch_resources': for key, (owned_api, owned_process) in tuple(_LIVE_SUPERVISORS.items()): if key.parent.parent == root / 'batch-resources': + from .read_recovery import process_tree_exited + retained = json.loads(internal_path(key).read_text(encoding='utf-8')) + if retained.get('operation') in _WRITE_OPERATIONS and not process_tree_exited(owned_process): + raise TaskCenterError('batch_resources_pending', + '保留的写入进程尚未全部退出;普通资源清理不能强制结束,请先人工处理或明确强制结束。') owned_process.close() owned_api.close_pending() _LIVE_SUPERVISORS.pop(key) if operation != 'cleanup_batch_resources' and pending_supervisors(root): raise TaskCenterError('batch_resources_pending', '上一监督进程尚未完成资源复核,请先重新清理资源。') batch_id, attempt_id = uuid4().hex, uuid4().hex + service._last_supervisor_batch_id = batch_id directory = root / 'batch-resources' / batch_id request = directory / 'request.json' journal = {'version': VERSION, 'batch_id': batch_id, 'attempt_id': attempt_id, @@ -351,6 +431,8 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres stderr_reader = None stdout_tail, stderr_tail = deque(maxlen=65536), deque(maxlen=65536) sent_cancel, last_heartbeat = False, time.monotonic() + business_watch = BusinessProgressWatch(last_heartbeat) + stall_reported = False transport_error = None ownership_cleanup_error = None force_accepted = False @@ -388,18 +470,27 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres _environment(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) journal.update(state='running', process=process_identity(process)) write_record(directory / 'supervisor.json', journal) + _ACTIVE_SUPERVISORS[directory / 'supervisor.json'] = (api, process, force_control) reader = threading.Thread(target=receive, daemon=True) reader.start() stderr_reader = threading.Thread(target=receive_stderr, daemon=True) stderr_reader.start() while process.poll() is None or not messages.empty(): - force = getattr(stop_event, 'force_event', None) + stalled = business_watch.stalled(time.monotonic()) + if stalled and not stall_reported: + journal['business_stalled'] = True + journal['business_stalled_at'] = time.time() + write_record(directory / 'supervisor.json', journal) + service._logging.logger.warning('supervisor_business_stalled batch_id=%s; owned processes retained', batch_id) + stall_reported = True + service._supervisor_business_stalled = stalled + force = force_control.force_event if force is not None and force.is_set() and process.poll() is None: try: _force_owned_supervisor(root, journal, process) except Exception as error: force.clear() - stop_event.control_error = '强制结束未执行或未确认;请保留窗口和诊断:' + str(error) + force_control.control_error = '强制结束未执行或未确认;请保留窗口和诊断:' + str(error) service._logging.logger.error('force_stop_failed batch_id=%s error=%s', batch_id, error) else: force_accepted = True @@ -425,16 +516,24 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres continue last_message = value last_heartbeat = time.monotonic() - if value['type'] == 'progress' and on_progress is not None: - on_progress(value['payload']) - elif value['type'] == 'phase' and on_phase is not None: + if value['type'] == 'progress': + business_watch.observe('progress', value['payload'], time.monotonic()) + if on_progress is not None: + on_progress(value['payload']) + elif value['type'] == 'phase': payload = decode(value['payload']) - on_phase(*payload) + business_watch.observe('phase', payload, time.monotonic()) + if on_phase is not None: + on_phase(*payload) elif value['type'] == 'state': state = decode(value['payload']) - service._task_queue_feedback = state['queue'] + business_watch.observe('state', state, time.monotonic()) + service._task_queue_feedback = getattr(service, '_workspace_queue_context', None) or state['queue'] service._batch_feedback = state['batch'] - service._supervisor_waiting = bool(state.get('waiting_for_resolution')) + service._supervisor_waiting = bool(state.get('waiting_for_resolution')) or stalled + force_control.waiting_for_resolution = service._supervisor_waiting + service._supervisor_dialog_state = {'project': state.get('active_project'), + 'binding': state.get('dialog_binding'), 'supervision': state.get('dialog_supervision')} if on_phase is not None and state['phase'] in {'recycling', 'closing'}: from .bom_import_state import BomImportPhase on_phase(BomImportPhase.RECOVERING if state['phase'] == 'recycling' else BomImportPhase.RELEASING, @@ -442,14 +541,23 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres process.wait(timeout=15) except Exception as error: transport_error = error - if process is not None and isinstance(stop_event, StopControl): - force_accepted = _retain_supervisor_after_control_error(root, journal, process, stop_event, error) + if process is not None and operation in _WRITE_OPERATIONS: + force_accepted = _retain_supervisor_after_control_error(root, journal, process, force_control, error) finally: service._task_queue_feedback = None service._batch_feedback = None service._supervisor_waiting = False + service._supervisor_business_stalled = False + service._supervisor_dialog_state = None try: if process is not None: + from .read_recovery import process_tree_exited + if (operation in _WRITE_OPERATIONS and not force_accepted + and not process_tree_exited(process)): + retained_error = TaskCenterError('supervisor_interrupted', + '监督进程已结束但所属进程仍在运行;保留可能写入,等待人工处理。') + force_accepted = _retain_supervisor_after_control_error( + root, journal, process, force_control, retained_error) process.close() # Waits, and confines termination to the owned Job. api.close_pending() except Exception as error: @@ -458,6 +566,8 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres _LIVE_SUPERVISORS[directory / 'supervisor.json'] = (api, process) transport_error = error if transport_error is None else TaskCenterError( 'batch_resources_pending', f'监督进程错误:{transport_error};收尾错误:{error}') + finally: + _ACTIVE_SUPERVISORS.pop(directory / 'supervisor.json', None) if reader is not None: reader.join(3) if stderr_reader is not None: @@ -543,11 +653,14 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres # Recheck physical resources even when final business results are # missing; this never authorizes repeating the interrupted job. from .batch_resources import cleanup_pending_resources - cleanup_pending_resources(root, logging_context=service._logging) + outstanding = cleanup_pending_resources(root, logging_context=service._logging) failure = cause if isinstance(cause, TaskCenterError) else TaskCenterError( 'supervisor_interrupted', '批次监督进程未留下可信结果;保留保护记录,不自动重试。' + (f' 首发原因:{transport_error}' if transport_error else '')) failure.primary_error = transport_error or error + from .read_recovery import process_tree_exited + failure.resources_released = (not outstanding and ownership_cleanup_error is None + and (process is None or process_tree_exited(process))) raise failure from (transport_error or error) if transport_error is not None: if response['payload']['business_succeeded']: @@ -555,5 +668,10 @@ def run_supervised(service, operation, arguments, *, stop_event=None, on_progres write_record(directory / 'supervisor.json', journal) else: _mark_interrupted(root, journal, transport_error) - raise resource_pending_error(operation, decoded_result, transport_error, decoded_primary) + failure = resource_pending_error(operation, decoded_result, transport_error, decoded_primary) + from .batch_resources import pending_resources + from .read_recovery import process_tree_exited + failure.resources_released = (ownership_cleanup_error is None and not pending_resources(root) + and process is not None and process_tree_exited(process)) + raise failure return _finish_response(root, journal, response['payload'], service._logging) diff --git a/src/dx_productivity/trial_task_center/batch_worker.py b/src/dx_productivity/trial_task_center/batch_worker.py index 200e4f9..4b4b3f2 100644 --- a/src/dx_productivity/trial_task_center/batch_worker.py +++ b/src/dx_productivity/trial_task_center/batch_worker.py @@ -59,6 +59,57 @@ def _admit_request(directory, request, request_bytes): 'process': own}, stream) stream.flush() os.fsync(stream.fileno()) + return journal + + +def consume_manual_dialog_actions(directory, runtime, batch_id, attempt_id): + from .batch_resources import write_record + for path in list(internal_path(directory).glob('manual-dialog-*.json'))[:8]: + claimed = path.with_suffix('.claimed') + try: + path.replace(claimed) # Claim before executing; never replay after a crash. + except FileNotFoundError: + continue + result = {'request_file': path.name, 'succeeded': False} + try: + request = json.loads(claimed.read_text(encoding='utf-8')) + if (request.get('version') != VERSION or request.get('batch_id') != batch_id + or request.get('attempt_id') != attempt_id or type(request.get('generation')) is not int + or request['generation'] != runtime.generation + or request.get('session_token') != getattr(runtime.cad, 'token', None) + or path.name != 'manual-dialog-' + UUID(request['request_id']).hex + '.json'): + raise ValueError('manual dialog request belongs to a different owned session') + runtime.manual_dialog_action(request['dialog'], request['button_hwnd']) + result['succeeded'] = True + except Exception as error: + result['error'] = str(error) + write_record(claimed.with_suffix('.result'), result) + + +def _wait_for_owned_writes(runtime, control): + """Process exit would close kill-on-close Jobs, so it is also a write boundary.""" + if runtime.read_only: + return + from .isolated_cad import RETAINED_CAD_SESSIONS + from .read_recovery import process_tree_exited + from .worker_monitor import RETAINED_WORKERS, _worker_tree_exited, cleanup_retained_workers + from .stop_control import wait_for_resolution + def exited(): + if (runtime.record['state'] != 'released' and (runtime.desktop is not None or runtime.cad is not None) + and not runtime.owned_processes_exited()): + return False + if any(session.root.is_relative_to(runtime.root) + and not all(process_tree_exited(process) for process in session._owned_processes()) + for session in tuple(RETAINED_CAD_SESSIONS.values())): + return False + return all(_worker_tree_exited(entry['process'], entry['owned']) + for entry in tuple(RETAINED_WORKERS.values()) if entry['root'].is_relative_to(runtime.root)) + if not exited(): + wait_for_resolution(control, runtime.path.parent, + 'supervisor retains possible-write Jobs until all owned processes exit', resolved=exited) + cleanup_retained_workers(runtime.root) + if runtime.desktop is not None or runtime.cad is not None: + runtime.close(reason='manual_retry') def run(request_path): @@ -78,7 +129,7 @@ def run(request_path): evidence = LifecycleEvidence(directory) if request_path != directory / 'request.json': raise ValueError('Supervisor request ownership path mismatch') - _admit_request(directory, request, request_bytes) + admission = _admit_request(directory, request, request_bytes) evidence.try_append('worker_started', pid=os.getpid()) from .batch_runtime import BatchCadRuntime from .batch_supervisor import _worker_context @@ -87,7 +138,7 @@ def run(request_path): from .store import TaskStore _worker_context(root) from .stop_control import StopControl - stop = StopControl() if request.get('interactive_stop', False) else threading.Event() + stop = StopControl() finished, output_lock = threading.Event(), threading.Lock() kind = {'inspect_bom_batch': 'bom_import', 'preview_bom_batch': 'bom_import', 'preview_bom_sources': 'bom_import', 'execute_task_queue': 'task_queue', 'execute_quick_run_queue': 'quick_run_queue'}.get(request['operation'], request['operation']) @@ -110,15 +161,24 @@ def run(request_path): def monitor(): while not finished.is_set(): try: + from .read_recovery import check_owner_loss + if check_owner_loss(directory, admission, read_only=runtime.read_only): + stop.set() cancel = internal_path(directory / 'cancel.json') if cancel.is_file(): value = json.loads(cancel.read_text(encoding='utf-8')) if value == {'version': VERSION, 'batch_id': batch_id, 'attempt_id': attempt_id}: stop.set() + consume_manual_dialog_actions(directory, runtime, batch_id, attempt_id) emit('heartbeat') feedback = getattr(service, '_task_queue_feedback', None) emit('state', encode({'phase': runtime.record['state'], 'queue': feedback, 'batch': service.batch_feedback(), + 'active_project': runtime.record.get('active_project'), + 'business_progress': runtime.business_progress(), + 'dialog_supervision': runtime.dialog_state(), + 'dialog_binding': {'batch_id': batch_id, 'attempt_id': attempt_id, + 'generation': runtime.generation, 'session_token': getattr(runtime.cad, 'token', None)}, 'waiting_for_resolution': getattr(stop, 'waiting_for_resolution', False), 'resolution_error': getattr(stop, 'resolution_error', '')})) except Exception: @@ -187,6 +247,7 @@ def run(request_path): context.report_exception('batch_supervisor', error) response.update(status='error', error=encode_error(error)) finally: + _wait_for_owned_writes(runtime, stop) evidence.try_append('worker_closing', status=response.get('status'), business_succeeded=response.get('business_succeeded')) finished.set() thread.join(3) diff --git a/src/dx_productivity/trial_task_center/bom_batch.py b/src/dx_productivity/trial_task_center/bom_batch.py index d8c8f0f..aa66e15 100644 --- a/src/dx_productivity/trial_task_center/bom_batch.py +++ b/src/dx_productivity/trial_task_center/bom_batch.py @@ -1,5 +1,6 @@ """Project routing, staged previews and atomic confirmation for a batch BOM.""" from dataclasses import dataclass +import hashlib from pathlib import Path from .bom_plan import write_bom_template @@ -14,6 +15,8 @@ class BomProjectPreview: baseline: tuple inventory: tuple rows: tuple + source_hashes: tuple[tuple[str, str], ...] = () + strict_rows: bool = False @dataclass(frozen=True) @@ -80,6 +83,15 @@ def save_batch(service, preview, rows_by_task): raise TaskCenterError('project_binding', '本批工程范围已改变,请重新导入 BOM') for project in preview.projects: _assert_editable(service, project.task_id) + if project.strict_rows and tuple(rows_by_task[project.task_id]) != project.rows: + raise TaskCenterError('bom_import', '部分导入必须使用原预览行,请重新导入后确认') + for source, digest in project.source_hashes: + try: + actual = hashlib.sha256(Path(source).read_bytes()).hexdigest() + except OSError as error: + raise TaskCenterError('bom_import', 'BOM 来源已不可读,请重新导入') from error + if actual != digest: + raise TaskCenterError('bom_import', 'BOM 来源在预览后已改变,请重新导入') if tuple(store.list_target_plan(project.task_id)) != project.baseline: raise TaskCenterError('target_identity', '工程位号计划在 BOM 预览后已改变,请重新导入') for project in preview.projects: diff --git a/src/dx_productivity/trial_task_center/bom_import_report.py b/src/dx_productivity/trial_task_center/bom_import_report.py index 95f4990..4f82675 100644 --- a/src/dx_productivity/trial_task_center/bom_import_report.py +++ b/src/dx_productivity/trial_task_center/bom_import_report.py @@ -1,5 +1,6 @@ """Read-only, exhaustive BOM validation with an all-or-nothing preview.""" import inspect +import hashlib from dataclasses import dataclass, replace from pathlib import Path @@ -43,6 +44,7 @@ class BomImportReport: preview: BomBatchPreview | None = None errors: tuple[str, ...] = () source_map_path: Path | None = None + passed_preview: BomBatchPreview | None = None @property def total_rows(self): @@ -128,7 +130,29 @@ def _check_record_conflicts(rows, records, indices): duplicate_rows=tuple(rows[item].excel_row for item in dict.fromkeys(claimed))) -def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, on_phase=None, runtime=None): +def select_passed_projects(report, task_ids): + """Explicitly accept whole passed projects from this immutable inspection.""" + task_ids = tuple(task_ids) + preview = report.passed_preview + if (preview is None or report.errors or not task_ids or len(task_ids) != len(set(task_ids)) + or not set(task_ids) <= {p.task_id for p in preview.projects}): + raise TaskCenterError('bom_import', '只能选择本次预览中校验通过的工程') + return replace(preview, projects=tuple(replace(p, strict_rows=True) for p in preview.projects + if p.task_id in task_ids)) + + +def bind_source_report(report, source_hashes, source_map_path): + """Replace temporary normalized input binding with original workbook proof.""" + bindings = tuple(source_hashes) + ((str(source_map_path), hashlib.sha256( + Path(source_map_path).read_bytes()).hexdigest()),) + def bind(preview): + return None if preview is None else replace(preview, projects=tuple( + replace(p, source_hashes=bindings) for p in preview.projects)) + return replace(report, preview=bind(report.preview), passed_preview=bind(report.passed_preview)) + + +def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, on_phase=None, runtime=None, + allow_partial=False): """Validate all independent rows; never persist inventory or return partial plans. Operational failures (including cancellation and native cleanup) propagate so @@ -141,16 +165,16 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, with BatchCadRuntime(service._logging.root, 'bom_import', stop_event) as owned_runtime: try: report = inspect_batch(service, task_id, path, stop_event=stop_event, - on_progress=on_progress, on_phase=on_phase, runtime=owned_runtime) + on_progress=on_progress, on_phase=on_phase, runtime=owned_runtime, allow_partial=allow_partial) except BomInspectionError as error: report = error.bom_report raise owned_runtime.result = report return report except BatchResourceCleanupError as error: - error.bom_report = replace(report, preview=None) if report is not None else None + error.bom_report = replace(report, preview=None, passed_preview=None) if report is not None else None if isinstance(error.completed_result, BomImportReport): - error.completed_result = replace(error.completed_result, preview=None) + error.completed_result = replace(error.completed_result, preview=None, passed_preview=None) raise def check_cancelled(): if stop_event is not None and stop_event.is_set(): @@ -171,6 +195,7 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, if len(owners) != len(scope): raise TaskCenterError('project_binding', '同批任务存在重复工程') rows, records, grouped = [], {}, {} + source_hash = hashlib.sha256(Path(path).read_bytes()).hexdigest() if Path(path).is_file() else '' workbook = None try: workbook = load_workbook(Path(path), read_only=True, data_only=False) @@ -251,26 +276,52 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, except TaskCenterError as error: for index in indices: rows[index] = replace(rows[index], status='failed', reason=error.message) - if any(row.status == 'failed' for row in rows): + if any(row.status == 'failed' for row in rows) and not allow_partial: return _report(rows) notify(f'BOM 表格校验通过,共 {len(rows)} 行、涉及 {len(grouped)} 个工程;开始核验工程位号。') previews = [] for position, (key, indices) in enumerate(grouped.items(), 1): check_cancelled() task, project = owners[key] + # One invalid row rejects its whole project, including parsed rows which + # were excluded from grouped because they failed Excel validation. + if any(row.status == 'failed' and row.project_path == str(project.path) for row in rows): + continue task_default_value_mode = 'part_label' if task.copy_part_number_to_value else 'preserve' def progress(message, phase=None): notify(f'工程 {position}/{len(grouped)} · {message}', phase) progress(f'正在核验「{task.name}」的位号库存…', BomImportPhase.INVENTORY) baseline = tuple(service._store.list_target_plan(task.task_id)) - requested = tuple(dict.fromkeys(records[index].old for index in indices)) + requested = tuple(dict.fromkeys(part for index in indices + for part in (records[index].old, records[index].new))) known = tuple(dict.fromkeys(row.target.old_part_number for row in baseline)) - if baseline and {part.casefold() for part in requested} <= {part.casefold() for part in known}: + known_parts = {part.casefold() for part in known} + project_rules = service.rules(task.task_id) + defaults = {(rule.old_part_number.casefold(), rule.new_part_number): rule.target_cell + for rule in project_rules if rule.enabled} + missing_metadata = False + already_target_inventory = False + for index in indices: + record = records[index] + mode = task_default_value_mode if record.value_mode == 'inherit' else record.value_mode + for row in baseline: + if (record.enabled and row.target.old_part_number.casefold() == record.new.casefold() + and row.target.refdes.casefold() in {ref.casefold() for ref in record.refs}): + already_target_inventory = True + cell = (row.target_cell if row.new_part_number == record.new else '') or defaults.get( + (record.old.casefold(), record.new), '') + missing_metadata |= ((mode == 'part_label' and row.target.actual_part_label is None) + or (bool(cell) and row.target.actual_cell is None)) + if baseline and all(records[index].old.casefold() in known_parts or records[index].new.casefold() in known_parts + for index in indices) and not already_target_inventory: progress(f'「{task.name}」复用已有位号库存,无需启动 CAD') inventory = tuple(row.target for row in baseline) else: progress(f'「{task.name}」正在获取位号库存,仅采集本表涉及的旧料号及已有计划料号') - collector = service.for_project_task(task.task_id)._quick_run_preparer.collect_inventory + preparer = service.for_project_task(task.task_id)._quick_run_preparer + if missing_metadata: + preparer.invalidate_discovery(project.path) + collector = preparer.collect_inventory controls = ({'runtime': runtime} if any( parameter.name == 'runtime' or parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in inspect.signature(collector).parameters.values()) else {}) @@ -300,9 +351,17 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, raise BomInspectionError(error, report) from error check_cancelled() progress(f'「{task.name}」正在匹配 {len(indices)} 行 BOM 与工程位号…', BomImportPhase.MATCHING) - merged = merge_target_inventory(baseline, inventory) + try: + merged = merge_target_inventory(baseline, inventory) + except TaskCenterError as error: + if not already_target_inventory: + raise + for index in indices: + rows[index] = replace(rows[index], status='failed', reason=error.message) + continue matched_rows, touched = {}, {} - matcher = BomMatcher(merged, service.rules(task.task_id)) if merged else None + matcher = BomMatcher(merged, project_rules, default_value_mode=task_default_value_mode, + current_inventory=inventory) if merged else None for index in indices: check_cancelled() try: @@ -314,7 +373,12 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, for target_key, row in matched.items(): touched.setdefault(target_key, []).append(index) matched_rows[target_key] = row - rows[index] = replace(rows[index], status='passed', reason='校验通过') + already = tuple(row.target.refdes for row in matched.values() + if records[index].enabled and + row.target.old_part_number.casefold() == records[index].new.casefold()) + reason = ('校验通过;已是目标料号且实际 Value/Cell 满足策略,跳过位号:' + ', '.join(already) + if already else '校验通过') + rows[index] = replace(rows[index], status='passed', reason=reason) except TaskCenterError as error: rows[index] = replace(rows[index], status='failed', reason=error.message) for target, indices_for_target in touched.items(): @@ -330,9 +394,14 @@ def inspect_batch(service, task_id, path, *, stop_event=None, on_progress=None, for index in indices: if rows[index].status == 'passed': rows[index] = replace(rows[index], status='failed', reason=error.message) - previews.append(BomProjectPreview(task.task_id, project.path, baseline, inventory, combined)) + if all(rows[index].status == 'passed' for index in indices): + previews.append(BomProjectPreview(task.task_id, project.path, baseline, inventory, combined, + ((str(Path(path).resolve()), source_hash),))) progress(f'「{task.name}」校验完成') check_cancelled() preview = None if any(row.status != 'passed' for row in rows) else BomBatchPreview( task_id, _identity(scope), tuple(previews)) - return _report(rows, preview) + if hashlib.sha256(Path(path).read_bytes()).hexdigest() != source_hash: + return _report(rows, errors=('BOM 来源在校验期间已改变,请重新导入',)) + passed = BomBatchPreview(task_id, _identity(scope), tuple(previews)) if allow_partial and previews else None + return replace(_report(rows, preview), passed_preview=passed) diff --git a/src/dx_productivity/trial_task_center/bom_import_state.py b/src/dx_productivity/trial_task_center/bom_import_state.py index e6ab8fb..87290b5 100644 --- a/src/dx_productivity/trial_task_center/bom_import_state.py +++ b/src/dx_productivity/trial_task_center/bom_import_state.py @@ -115,6 +115,8 @@ class BomImportMachine: def release_preview(self, *, saved: bool) -> None: report = self.state.report - if report is not None and report.preview is not None: - self.state = replace(self.state, report=replace(report, preview=None), notice= - '校验通过,位号计划已保存。' if saved else '本次导入预览已放弃,校验报告保留。') + if report is not None and (report.preview is not None or report.passed_preview is not None): + notice = ('校验通过,位号计划已保存。' if report.preview is not None else + '已保存选择的通过工程;失败和未检查行的报告保留。') + self.state = replace(self.state, report=replace(report, preview=None, passed_preview=None), notice= + notice if saved else '本次导入预览已放弃,校验报告保留。') diff --git a/src/dx_productivity/trial_task_center/bom_plan.py b/src/dx_productivity/trial_task_center/bom_plan.py index 9bd1993..a5130d7 100644 --- a/src/dx_productivity/trial_task_center/bom_plan.py +++ b/src/dx_productivity/trial_task_center/bom_plan.py @@ -178,15 +178,18 @@ def parse_bom_row(cells, positions): class BomMatcher: """Index one immutable inventory once, then match independent Excel rows.""" - def __init__(self, baseline, rules=()): + def __init__(self, baseline, rules=(), *, default_value_mode='preserve', current_inventory=None): if not baseline: raise TaskCenterError('target_inventory', '未采集到可匹配的工程位号') self.by_key = {} self.projects = set() self.by_identity = {} + self.default_value_mode = default_value_mode self.cell_defaults = {(rule.old_part_number.casefold(), rule.new_part_number): rule.target_cell for rule in rules if rule.enabled} self.identities = ProjectIdentityResolver() + self.current_inventory = (None if current_inventory is None else + {self.identities.target(target): target for target in current_inventory}) for row in baseline: target = row.target key = self.identities.target(target) @@ -201,25 +204,47 @@ class BomMatcher: raise TaskCenterError('bom_import', 'BOM 工程与当前任务不一致') matched = {} for refdes in record.refs: - candidates = [(key, target) for key, target in self.by_identity.get((record.old.casefold(), refdes.casefold()), ()) - if (not record.schematic or target.schematic_name.casefold() == record.schematic.casefold()) + identities = ((record.old.casefold(), refdes.casefold()), (record.new.casefold(), refdes.casefold())) + candidates = [(key, target) for identity in dict.fromkeys(identities) + for key, target in self.by_identity.get(identity, ()) + if (not record.project_path or self.identities.project(record.project_path) == key[0]) + and (not record.schematic or target.schematic_name.casefold() == record.schematic.casefold()) and (not record.sheet or target.sheet_number.casefold() == record.sheet.casefold()) and (not record.uid or target.uid == record.uid)] if len(candidates) != 1: raise TaskCenterError('bom_import', f'位号 {refdes} 无法唯一匹配当前采集清单') key, target = candidates[0] + if record.project_path and self.identities.project(record.project_path) != key[0]: + raise TaskCenterError('bom_import', f'位号 {refdes} 不属于本行工程') if key in matched: raise TaskCenterError('bom_import', f'位号 {refdes} 在 BOM 中重复') - matched[key] = TargetPlanRow(target, record.enabled, record.new, + cell = ((self.by_key[key].target_cell if self.by_key[key].new_part_number == record.new else '') + or self.cell_defaults.get((record.old.casefold(), record.new), '')) + already_target = target.old_part_number.casefold() == record.new.casefold() + if already_target and record.enabled: + current = target if self.current_inventory is None else self.current_inventory.get(key) + if current is None: + raise TaskCenterError('bom_import', f'位号 {refdes} 缺少本轮只读观察,不能确认已到目标') + mode = self.default_value_mode if record.value_mode == 'inherit' else record.value_mode + expected = {'preserve': current.old_value, 'custom': record.custom, 'part_number': current.old_part_number, + 'part_label': current.actual_part_label}.get(mode) + has_nc = lambda value: any(part.strip().casefold() == 'nc' for part in value.split('/')[1:]) + if expected is not None and record.sync_nc and has_nc(current.old_value) and not has_nc(expected): + expected = expected.rstrip() + ('NC' if expected.rstrip().endswith('/') else '/NC') + if expected is None or (mode == 'part_label' and not expected) or current.old_value != expected: + raise TaskCenterError('bom_import', f'位号 {refdes} 已是目标料号,但实际 Value 不符合策略或缺少 Part Label 证据;请核验后处理') + if cell and current.actual_cell != cell: + reason = '库存没有实际 Cell 回读证据' if current.actual_cell is None else '实际 Cell 不符合指定封装' + raise TaskCenterError('bom_import', f'位号 {refdes} 已是目标料号,但{reason},不能确认跳过') + matched[key] = TargetPlanRow(target, record.enabled and not already_target, record.new, record.value_mode == 'custom', record.custom, record.sync_nc, - (self.by_key[key].target_cell if self.by_key[key].new_part_number == record.new else '') - or self.cell_defaults.get((target.old_part_number.casefold(), record.new), ''), + cell, record.value_mode) return matched -def match_bom_records(baseline, records, rules=()) -> tuple[TargetPlanRow, ...]: - matcher = BomMatcher(baseline, rules) +def match_bom_records(baseline, records, rules=(), *, default_value_mode='preserve') -> tuple[TargetPlanRow, ...]: + matcher = BomMatcher(baseline, rules, default_value_mode=default_value_mode) matched = {} for record in records: incoming = matcher.match_record(record) @@ -227,14 +252,15 @@ def match_bom_records(baseline, records, rules=()) -> tuple[TargetPlanRow, ...]: raise TaskCenterError('bom_import', '位号在 BOM 中重复') matched.update(incoming) result = tuple(matched.get(key, TargetPlanRow(row.target)) for key, row in matcher.by_key.items()) - compile_target_rules(result) + compile_target_rules(result, default_value_mode=default_value_mode) return result def read_bom_plan(store, task_id: str, path: Path) -> tuple[TargetPlanRow, ...]: from .batch_rules import batch_rule_view return match_bom_records(store.list_target_plan(task_id), read_bom_records(path), - batch_rule_view(store, task_id, strict=True)) + batch_rule_view(store, task_id, strict=True), default_value_mode= + 'part_label' if store.get_task(task_id).copy_part_number_to_value else 'preserve') def _normal_headers(values): diff --git a/src/dx_productivity/trial_task_center/cad_dialogs.py b/src/dx_productivity/trial_task_center/cad_dialogs.py index fb9b779..a18d7a7 100644 --- a/src/dx_productivity/trial_task_center/cad_dialogs.py +++ b/src/dx_productivity/trial_task_center/cad_dialogs.py @@ -73,6 +73,20 @@ def acknowledge_owned_dialog(api, desktop_name, pid, expected): win32gui.PostMessage(current['buttons'][0]['hwnd'], 0x00F5, 0, 0) # BM_CLICK, no global input +def manually_act_on_owned_dialog(api, desktop_name, pid, expected, button_hwnd): + """Execute one explicit user choice only while the full prompt still matches.""" + import win32gui + current = next((row for row in inspect_owned_dialogs(api, desktop_name, pid) + if row == expected and row['pid'] == pid), None) + if current is None or current.get('complete') is not True or type(button_hwnd) is not int: + raise OSError('dialog changed before the manual action') + button = next((row for row in current['buttons'] + if row['hwnd'] == button_hwnd and row['enabled']), None) + if button is None: + raise OSError('manual dialog button is no longer available') + win32gui.PostMessage(button_hwnd, 0x00F5, 0, 0) + + def _script_error(row): title = row['title'].strip().casefold() body = row['text'].casefold() @@ -89,6 +103,7 @@ class CadDialogGuard: self.evidence_path, self.token = evidence_path, token self.records, self.seen = [], set() self.blocking = [] + self.current = [] self.failure = '' self._lock = RLock() @@ -103,6 +118,7 @@ class CadDialogGuard: return rows = [row for row in inspect(self.process.pid) if row['pid'] == self.process.pid and row['class'] == '#32770'] + self.current = rows self.blocking = [row['title'] or '未命名弹窗' for row in rows] for row in rows: key = json.dumps(row, ensure_ascii=False, sort_keys=True) @@ -125,4 +141,15 @@ class CadDialogGuard: def summary(self): with self._lock: return {'evidence_path': str(self.evidence_path), 'dialogs_seen': len(self.records), - 'blocked': bool(self.blocking), 'blocking_titles': list(self.blocking), 'error': self.failure} + 'blocked': bool(self.blocking), 'blocking_titles': list(self.blocking), + 'current_dialogs': list(self.current), 'error': self.failure} + + def manual_action(self, dialog, button_hwnd): + with self._lock: + self._poll('manual_review') + if dialog not in self.current: + raise OSError('manual dialog is no longer current') + self.records.append(dict(dialog, action='user_selected_button', button_hwnd=button_hwnd, + observed_at=time.time())) + self._save() + self.desktop.manual_dialog_action(dialog, button_hwnd) diff --git a/src/dx_productivity/trial_task_center/cell_catalog.py b/src/dx_productivity/trial_task_center/cell_catalog.py index 64dcb87..c88ab7f 100644 --- a/src/dx_productivity/trial_task_center/cell_catalog.py +++ b/src/dx_productivity/trial_task_center/cell_catalog.py @@ -153,13 +153,15 @@ def _catalog_from_evidence(evidence, library, part, partition, fingerprint, path def _require_cleanup(cleanup): + def pending(message): + raise TaskCenterError('batch_resources_pending', message) if not isinstance(cleanup, dict) or cleanup.get('errors') or not cleanup.get('desktop_handle_closed') or not cleanup.get('desktop_confirmation', {}).get('released'): - _fail('catalog private desktop cleanup was not proven') + pending('catalog private desktop cleanup was not proven') if not cleanup.get('processes'): - _fail('catalog owned process cleanup is missing') + pending('catalog owned process cleanup is missing') for process in cleanup['processes']: if process.get('errors') or process.get('job_active_processes') != 0 or not process.get('process_handle_closed') or not process.get('job_handle_closed') or process.get('unconfirmed_descendant_handles', 0): - _fail('catalog owned Job process cleanup was not proven') + pending('catalog owned Job process cleanup was not proven') def _dispose_clone(root): @@ -208,6 +210,7 @@ def read_library_cell_catalog(library_path: Path, part_number: str, evidence_roo root.mkdir(parents=True) evidence_path = root / 'catalog.json' reader_started = False + result = None try: clone = _clone_library(library, root, manifest) _write_json(root / 'source-before.json', manifest) @@ -235,9 +238,16 @@ def read_library_cell_catalog(library_path: Path, part_number: str, evidence_roo cleanup_path = root / 'desktop-cleanup.json' if not reader_started: _dispose_clone(root) + elif result is not None: + _require_cleanup(result.get('cleanup')) + _dispose_clone(root) elif cleanup_path.is_file(): _require_cleanup(json.loads(cleanup_path.read_text(encoding='utf-8'))) _dispose_clone(root) + elif getattr(error, 'catalog_resources_released', False): + _dispose_clone(root) + else: + raise TaskCenterError('batch_resources_pending', 'catalog reader cleanup/release evidence is missing') from error raise @@ -271,6 +281,41 @@ def read_project_cell_choices(project_paths: tuple[Path, ...], part_number: str, return tuple(sorted(choices)) +def preflight_target_libraries(project_paths, rules, evidence_root, *, allow_cold_read=True): + """Prove each requested target PN and Cell using the isolated readonly reader. + + An empty Cell choice still requires an authoritative associated cell list; + missing parts and ambiguous partitions are rejected by the reader itself. + """ + projects = tuple(project_paths) + if not projects: + _fail('no project libraries are available for target preflight') + requests = tuple(dict.fromkeys((rule.new_part_number, getattr(rule, 'target_cell', '')) + for rule in rules if getattr(rule, 'enabled', True))) + proof = [] + for library in dict.fromkeys(project_library_path(project) for project in projects): + for part, cell in requests: + catalog = read_library_cell_catalog(library, part, evidence_root, + allow_cold_read=allow_cold_read) + if cell and cell not in {choice.name for choice in catalog.cells}: + _fail(f'target Cell is not associated with requested Part Number: {part} / {cell}') + proof.append(catalog) + return tuple(proof) + + +def preflight_command_target(command, evidence_root, *, allow_cold_read=True): + """Execution adapter for one frozen command, before journal claim or CAD writes.""" + from types import SimpleNamespace + try: + return preflight_target_libraries((Path(command['project_path']),), + (SimpleNamespace(new_part_number=command['new_part'], target_cell=command.get('target_cell', '')),), + evidence_root, allow_cold_read=allow_cold_read) + except TaskCenterError as error: + if error.code != 'cell_catalog': + raise + raise TaskCenterError('target_cell_catalog', error.message) from error + + def _worker_command(request): if getattr(sys, 'frozen', False): return (sys.executable, '--cell-catalog-worker', str(request)) @@ -359,7 +404,11 @@ def _catalog_launch_lock(): def _run_isolated_reader(clone, part, partition, root): with _catalog_launch_lock(): - _require_no_existing_manager() + try: + _require_no_existing_manager() + except TaskCenterError as error: + error.catalog_resources_released = True + raise return _run_owned_reader(clone, part, partition, root) diff --git a/src/dx_productivity/trial_task_center/component_scope.py b/src/dx_productivity/trial_task_center/component_scope.py new file mode 100644 index 0000000..aba3745 --- /dev/null +++ b/src/dx_productivity/trial_task_center/component_scope.py @@ -0,0 +1,165 @@ +"""Frozen read-only UID expansion for newly created all-match queues.""" +from dataclasses import dataclass +import csv +from hashlib import sha256 +from io import StringIO +import json +from pathlib import Path +from types import SimpleNamespace + +from .errors import TaskCenterError +from .target_plan import ComponentTarget, TargetRule + + +@dataclass(frozen=True) +class InventoryEmptyRule: + old_part_number: str + new_part_number: str + inventory_empty_sheet: tuple[str, str, str] + custom_value_enabled: bool = False + custom_value: str = '' + sync_nc: bool = True + target_cell: str = '' + value_mode: str = 'preserve' + enabled: bool = True + scope: str = 'targets' + targets: tuple = () + + +def _error(detail): + return TaskCenterError('component_scope', 'Frozen component scope is invalid: ' + detail) + + +def _digest(value): + return sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf8')).hexdigest() + + +def _sources(rules, base_mode): + if any(getattr(rule, 'scope', 'all') != 'all' for rule in rules): + raise _error('source rules are not all-match rules') + return [{name: getattr(rule, name, default) for name, default in ( + ('rule_id', ''), ('task_id', ''), ('position', 0), ('old_part_number', ''), + ('new_part_number', ''), ('enabled', True), ('custom_value_enabled', False), + ('custom_value', ''), ('sync_nc', True), ('target_cell', ''), ('value_mode', base_mode))} + for rule in rules] + + +def freeze(projects, rules, sheets, discoveries, failures, base_mode): + """The discovery boundary certifies parser validation and reader release.""" + from .quick_run import _path_key + sources = _sources(rules, base_mode) + requested = {rule.old_part_number.casefold() for rule in rules} + inventories = [] + for project in projects: + key = _path_key(project.path) + discovery = discoveries.get(key) + components = [] + if key not in failures: + if discovery is None or set(discovery.sheets) != set(discovery.matched_old_parts): + raise _error('missing validated read-only component manifest') + seen, actual = set(), {sheet: set() for sheet in sheets[key]} + for target in discovery.components: + sheet = target.schematic_name, target.sheet_number + if _path_key(target.project_path) != key or sheet not in actual or target.key in seen: + raise _error('component inventory identity is not unique within the frozen sheet scope') + seen.add(target.key) + if target.old_part_number.casefold() in requested: + actual[sheet].add(target.old_part_number.casefold()) + components.append({ + 'schematic_name': target.schematic_name, 'sheet_number': target.sheet_number, + 'uid': target.uid, 'refdes': target.refdes, + 'old_part_number': target.old_part_number, 'old_value': target.old_value, + 'actual_part_label': target.actual_part_label, 'actual_cell': target.actual_cell}) + expected = {sheet: {part.casefold() for part in discovery.matched_old_parts[sheet]} & requested + for sheet in sheets[key]} + if actual != expected: + raise _error('Part Number inventory has no matching component identity manifest') + inventories.append({'project_path': str(project.path), 'sheets': [list(sheet) for sheet in sheets[key]], + 'components': components, 'observation': 'project_missing' if key in failures else 'validated_readonly_inventory'}) + return {'version': 1, 'base_value_mode': base_mode, 'source_rules': sources, + 'source_signature': _digest(sources), 'projects': inventories} + + +def load(path): + try: + record = json.loads(path.read_text(encoding='utf-8-sig')) + if (record['version'] != 1 or record['base_value_mode'] not in {'preserve', 'part_label', 'part_number'} + or not isinstance(record['source_rules'], list) or not record['source_rules'] + or record['source_signature'] != _digest(record['source_rules']) + or not isinstance(record['projects'], list) or not record['projects']): + raise ValueError('source signature or schema changed') + return record + except (OSError, UnicodeError, ValueError, KeyError, TypeError) as error: + raise _error('source signature, schema or inventory sidecar is missing or changed') from error + + +def expand(record, projects, rules, base_mode, sheets): + """Reconstruct exact commands from original rules, without querying changed CAD.""" + from .quick_run import _path_key, _rule_command_options + if record['base_value_mode'] != base_mode or record['source_rules'] != _sources(rules, base_mode): + raise _error('source rule options differ from the frozen source signature') + try: + inventories = {_path_key(item['project_path']): item for item in record['projects']} + if len(inventories) != len(record['projects']): + raise ValueError('duplicate project') + current, empty = [], [] + for project in projects: + key = _path_key(project.path) + item = inventories[key] + available = tuple(tuple(sheet) for sheet in item['sheets']) + if (available != sheets[key] or not available or len(set(available)) != len(available) + or any(len(sheet) != 2 or not all(isinstance(value, str) and value for value in sheet) for sheet in available) + or item['observation'] not in {'validated_readonly_inventory', 'project_missing'}): + raise ValueError('sheet scope or provenance changed') + components = tuple(ComponentTarget(Path(project.path), **target) for target in item['components']) + if (len({target.key for target in components}) != len(components) + or any((target.schematic_name, target.sheet_number) not in available for target in components) + or any(not isinstance(target.old_value, str) for target in components) + or (item['observation'] == 'project_missing' and components)): + raise ValueError('component inventory changed') + current.extend(components) + for rule in rules: + if not any(target.old_part_number.casefold() == rule.old_part_number.casefold() for target in components): + mode, custom, sync, _ = _rule_command_options(rule, base_mode) + empty.append(InventoryEmptyRule(rule.old_part_number, rule.new_part_number, + (key, *available[0]), mode == 'custom', custom, sync, + str(getattr(rule, 'target_cell', '')), mode)) + expanded = [] + for rule in rules: + targets = tuple(target for target in current if target.old_part_number.casefold() == rule.old_part_number.casefold()) + mode, custom, sync, _ = _rule_command_options(rule, base_mode) + if targets: + expanded.append(TargetRule(rule.old_part_number, rule.new_part_number, targets, + custom_value_enabled=mode == 'custom', custom_value=custom, sync_nc=sync, + target_cell=str(getattr(rule, 'target_cell', '')), value_mode=mode)) + expanded.extend(item for item in empty if (item.old_part_number, item.new_part_number) + == (rule.old_part_number, rule.new_part_number)) + return tuple(expanded) + except (ValueError, KeyError, TypeError) as error: + raise _error('frozen component identities, provenance or sheet scope changed') from error + + +def validate_empty_command(root, row): + """Bind a non-dispatchable no-match row to the frozen read-only inventory.""" + from .execution_journal import read_command_rows + from .path_access import internal_path + from .quick_run import _command_text, _decode_vbs_field, _path_key + try: + if row.get('execution_unit') != 'inventory_empty' or row.get('target_selectors') != '': + return False + scope = load(internal_path(root / 'component-scope.json')) + actual_rows = read_command_rows(root / 'command.tsv') + active = {_path_key(item['project_path']) for item in actual_rows} + selected = tuple(item for item in scope['projects'] if _path_key(item['project_path']) in active) + projects = tuple(SimpleNamespace(path=Path(item['project_path'])) for item in selected) + sheets = {_path_key(item.path): tuple(tuple(sheet) for sheet in saved['sheets']) + for item, saved in zip(projects, selected)} + rules = tuple(SimpleNamespace(**source) for source in scope['source_rules']) + expanded = expand(scope, projects, rules, scope['base_value_mode'], sheets) + expected = _command_text('expected', 'expected', projects, expanded, scope['base_value_mode'], sheet_targets=sheets) + expected_rows = tuple({key: _decode_vbs_field(value) for key, value in item.items()} + for item in csv.DictReader(StringIO(expected), delimiter='\t')) + identity = lambda item: {key: value for key, value in item.items() if key not in {'task_id', 'job_id'}} + return row in actual_rows and tuple(map(identity, actual_rows)) == tuple(map(identity, expected_rows)) + except (OSError, UnicodeError, ValueError, KeyError, TypeError, TaskCenterError): + return False diff --git a/src/dx_productivity/trial_task_center/diagnostics_bundle.py b/src/dx_productivity/trial_task_center/diagnostics_bundle.py index 5c2e1b8..88a3afa 100644 --- a/src/dx_productivity/trial_task_center/diagnostics_bundle.py +++ b/src/dx_productivity/trial_task_center/diagnostics_bundle.py @@ -15,10 +15,10 @@ MAX_BUNDLE_BYTES = 32 * 1024 * 1024 MAX_FILES = 512 _BATCH_NAMES = {'session.json', 'supervisor.json', 'request.json', 'result.json', 'business-result.json', 'supervisor.log', 'resource-release.log', 'cancel.json', - 'process.json', 'lifecycle.jsonl'} + 'process.json', 'lifecycle.jsonl', 'owner-loss.json', 'claim.json', 'force-request.json'} _QUICK_NAMES = {'private-desktop.json', 'checkpoint.json', 'status.json', 'uncertain.flag', 'resolved-uncertain.flag', 'protection.json', 'cleanup.json', - 'project-session.json', 'project-checkpoint.json', 'status.tsv'} + 'project-session.json', 'project-checkpoint.json', 'project-read-release.json', 'status.tsv'} _SECRET_KEY = re.compile(r'(?:password|passwd|(?:client[_-]?)?secret|credentials?|authorization|api[_-]?key|access[_-]?token|refresh[_-]?token|token)', re.I) _USER_KEY = re.compile(r'(?:user(?:[_-]?name)?|login[_-]?name)', re.I) _SECRET_TEXT = re.compile(r'''(?ix)(\b(?:password|passwd|(?:client[_-]?)?secret|credentials?|authorization|api[_-]?key|access[_-]?token|refresh[_-]?token|token)\b["']?\s*[=:]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;}]+)''') @@ -61,17 +61,17 @@ def _allowed(parts): if len(parts) == 2 and parts[0] == 'logs': return bool(re.fullmatch(r'[^/]+\.log(?:\.\d+)?', parts[-1])) if len(parts) == 3 and parts[0] == 'batch-resources': - return parts[-1] in _BATCH_NAMES + return parts[-1] in _BATCH_NAMES or bool(re.fullmatch(r'manual-dialog-[a-f0-9]{32}\.json', parts[-1])) if len(parts) >= 5 and parts[0] == 'batch-resources' and parts[2] == 'attempts': return parts[-1].endswith('.json') if len(parts) >= 2 and parts[0] == 'attempts': return parts[-1].endswith('.json') if len(parts) == 3 and parts[0] == 'tasks': return bool(re.fullmatch(r'events\.jsonl(?:\.\d+)?', parts[-1])) - if len(parts) >= 2 and parts[0] in {'queue', 'queues', 'task-queues'}: + if len(parts) >= 2 and parts[0] in {'queue', 'queues', 'task-queues', 'workspace-queues'}: return parts[-1].endswith('.json') if len(parts) >= 2 and any(value in parts for value in ('quick-run', 'quickrun', 'quick-run-history')): - return parts[-1] in _QUICK_NAMES + return parts[-1] in _QUICK_NAMES or bool(re.fullmatch(r'retained-worker-[a-f0-9]{32}\.json', parts[-1])) return False @@ -101,7 +101,7 @@ def export_diagnostic_bundle(data_root, destination: Path) -> Path: info = entry.lstat() if stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & 0x400: manifest['omissions'].append({'path': _scrub_text(relative), 'reason': 'link_or_reparse_point'}) - elif base == physical and name not in {'logs', 'batch-resources', 'attempts', 'tasks', 'queue', 'queues', 'task-queues', 'quick-run', 'quick-run-history'}: + elif base == physical and name not in {'logs', 'batch-resources', 'attempts', 'tasks', 'queue', 'queues', 'task-queues', 'workspace-queues', 'quick-run', 'quick-run-history'}: manifest['omissions'].append({'path': _scrub_text(relative), 'reason': 'outside_allowlist'}) else: safe_folders.append(name) diff --git a/src/dx_productivity/trial_task_center/discovery_cache.py b/src/dx_productivity/trial_task_center/discovery_cache.py index 2bd131a..bd0fbae 100644 --- a/src/dx_productivity/trial_task_center/discovery_cache.py +++ b/src/dx_productivity/trial_task_center/discovery_cache.py @@ -109,6 +109,14 @@ def _only_database_changed(before, after): return design is not None and before != after and design == _without_runtime_database(after) +def _content_fingerprint(stamp): + if stamp is None: + return None + # Hash every byte on each lookup. Metadata changes can be refreshed, but + # file replacement still invalidates identity even when bytes are equal. + return stamp[0], tuple((row[0], row[1], row[2], row[3], row[-1]) for row in stamp[1]) + + def _discovery_identity(discovery): sheets = discovery.sheets components = getattr(discovery, 'components', ()) @@ -116,7 +124,8 @@ def _discovery_identity(discovery): raise TaskCenterError('target_identity', '原生采集返回重复图页或位号身份,请核对工程') return (tuple(sorted(sheets)), tuple(sorted((sheet, tuple(sorted(parts))) for sheet, parts in discovery.matched_old_parts.items())), - tuple(sorted((row.key, row.refdes, row.old_part_number, row.old_value) for row in components))) + tuple(sorted((row.key, row.refdes, row.old_part_number, row.old_value, + getattr(row, 'actual_part_label', None), getattr(row, 'actual_cell', None)) for row in components))) class DiscoveryCache: @@ -158,7 +167,7 @@ class DiscoveryCache: payload = json.dumps(record['discovery'], ensure_ascii=True, sort_keys=True).encode('utf-8') if record['payload_sha256'] != hashlib.sha256(payload).hexdigest(): raise ValueError('invalid discovery cache') - if record['project'] != _key(project) or record['fingerprint'] != hashlib.sha256(repr(stamp).encode()).hexdigest(): + if record['project'] != _key(project) or record['fingerprint'] != hashlib.sha256(repr(_content_fingerprint(stamp)).encode()).hexdigest(): return None discovery = decode(record['discovery']) from .quick_run_protocol import ProjectDiscovery @@ -180,7 +189,7 @@ class DiscoveryCache: try: payload = encode(discovery) record = {'version': 2, 'project': _key(project), - 'fingerprint': hashlib.sha256(repr(stamp).encode()).hexdigest(), 'parts': sorted(requested), + 'fingerprint': hashlib.sha256(repr(_content_fingerprint(stamp)).encode()).hexdigest(), 'parts': sorted(requested), 'collect_components': components, 'discovery': payload, 'payload_sha256': hashlib.sha256(json.dumps(payload, ensure_ascii=True, sort_keys=True).encode()).hexdigest()} record['record_sha256'] = hashlib.sha256(json.dumps(record, ensure_ascii=True, sort_keys=True).encode()).hexdigest() @@ -191,7 +200,7 @@ class DiscoveryCache: except (OSError, ValueError, TypeError): pass # Cache persistence is optional, unlike execution write-ahead proof. - def discover(self, project, parts, collect_components, loader, *, stop_event=None): + def discover(self, project, parts, collect_components, loader, *, stop_event=None, release_idle=None): with self.lock: check_cancelled(stop_event) def fingerprint(): @@ -203,10 +212,10 @@ class DiscoveryCache: entry = self._entries.pop(key, None) or self._disk_entry(project, before) if entry is not None: stamp, covered, has_components, discovery = entry - if (before is not None and before == stamp and requested <= covered + if (before is not None and _content_fingerprint(before) == _content_fingerprint(stamp) and requested <= covered and (not collect_components or has_components) and (collect_components or getattr(discovery, 'complete_sheet_manifest', True))): - self._entries[key] = entry + self._entries[key] = (before, covered, has_components, discovery) check_cancelled(stop_event) return deepcopy(discovery) # Fingerprinting is an optimization: an initially unverifiable @@ -214,6 +223,8 @@ class DiscoveryCache: if before is None: discovery = loader() check_cancelled(stop_event) + if release_idle is not None: + release_idle() _discovery_identity(discovery) return discovery # Native opens may update database session metadata on every read. @@ -229,6 +240,8 @@ class DiscoveryCache: check_cancelled(stop_event) discovery = loader() check_cancelled(stop_event) + if release_idle is not None: + release_idle() identity = _discovery_identity(discovery) after = fingerprint() if before is not None and before == after: diff --git a/src/dx_productivity/trial_task_center/execution_journal.py b/src/dx_productivity/trial_task_center/execution_journal.py index 64587f1..7f19f4d 100644 --- a/src/dx_productivity/trial_task_center/execution_journal.py +++ b/src/dx_productivity/trial_task_center/execution_journal.py @@ -14,6 +14,8 @@ _TERMINAL={'saved_ok','no_match','restored_ok','batch_failed','batch_saved_unver def create_execution_schema(connection): + from .execution_problems import create_problem_schema + create_problem_schema(connection) connection.execute('''CREATE TABLE IF NOT EXISTS execution_transitions ( transition_id TEXT PRIMARY KEY, task_id TEXT NOT NULL)''') connection.execute('''CREATE TABLE IF NOT EXISTS execution_baselines ( @@ -72,6 +74,7 @@ class JournalAttempt: operation_key: str attempt_id: str generation: int + job_id: str | None = None class ExecutionJournal: @@ -99,7 +102,7 @@ class ExecutionJournal: ON CONFLICT(task_id) DO UPDATE SET round_id=excluded.round_id,plan_json='[]', retry_authorized=0,pause_requested=0''',(task_id,round_id)) - def authorize_retry(self,task_id): + def authorize_retry(self,task_id,*,operation_keys=None): with self.store.transaction(): current=self._round(task_id) if current: @@ -107,8 +110,131 @@ class ExecutionJournal: WHERE task_id=? AND round_id=? AND state IN ('prepared','uncertain') LIMIT 1''', (task_id,current['round_id'])).fetchone() if unsafe:raise TaskCenterError('quick_run_uncertain','原执行项尚待核验,不能准备重试。') + self.store._connection.execute('DELETE FROM execution_retry_selections WHERE task_id=?',(task_id,)) + if operation_keys is not None: + selected=tuple(operation_keys) + if not selected:raise TaskCenterError('problem_retry_scope','重试范围不能为空。') + valid={row[0] for row in self.store._connection.execute("SELECT operation_key FROM execution_operations WHERE task_id=? AND round_id=? AND state='failed'",(task_id,current['round_id']))} + if not set(selected)<=valid:raise TaskCenterError('problem_retry_scope','重试范围不属于原失败项。') + self.store._connection.execute('INSERT INTO execution_retry_selections VALUES (?,?,?)',(task_id,current['round_id'],_json(selected))) self.store._connection.execute('UPDATE execution_rounds SET retry_authorized=1 WHERE task_id=?',(task_id,)) + def has_pending_components(self,task_id): + with self.store._lock: + current=self._round(task_id) + if current is None:return False + rows=self.store._connection.execute("SELECT command_json FROM execution_operations WHERE task_id=? AND round_id=? AND state='pending'",(task_id,current['round_id'])) + return any(json.loads(row[0]).get('execution_unit')=='component' for row in rows) + + def independent_continuation_ready(self, task_id): + """Inspect the first pending frozen item; never select a later UID.""" + with self.store._lock: + try: + current = self._round(task_id) + if current is None:return False + records = {row['operation_key']: row for row in self.store._connection.execute( + 'SELECT operation_key,state,command_json FROM execution_operations WHERE task_id=? AND round_id=?', + (task_id, current['round_id']))} + for key in json.loads(current['plan_json']): + record = records.get(key) + if record is None or record['state'] in {'prepared', 'uncertain'}:return False + if record['state'] == 'pending': + command = json.loads(record['command_json']) + return bool(self.can_continue_component(task_id, command)) + return False + except (OSError, ValueError, KeyError, TypeError, AttributeError, TaskCenterError): + return False + + def can_continue_component(self,task_id,command): + from .execution_proof import proof_intact + if command.get('execution_unit')!='component' or len(_targets(command))!=1:return False + current=self._round(task_id) + if current is None:return False + baselines = self.expected_baselines(task_id, command['project_path']) + for row in self.store._connection.execute('SELECT * FROM execution_operations WHERE task_id=? AND round_id=?',(task_id,current['round_id'])): + prior=json.loads(row['command_json']) + if prior['project_path']!=command['project_path']:continue + if row['state'] in {'prepared','uncertain'}:return False + if row['state']=='failed': + receipt=json.loads(row['receipt_json']) + proof=receipt.get('proof',{}) + if not isinstance(proof, dict):return False + if prior.get('execution_unit')=='inventory_empty' and proof.get('inventory_empty') is True and proof.get('files') and proof_intact(proof): + continue + if prior.get('execution_unit')!='component' or set(_targets(prior))&set(_targets(command)):return False + from .project_session_health import affirmative + identity = {'task_id': task_id, 'round_id': current['round_id'], + 'operation_key': row['operation_key'], 'job_id': receipt.get('job_id'), + 'project_path': prior['project_path'], 'attempt_id': row['attempt_id'], + 'generation': row['generation']} + if not proof_intact(proof) or not affirmative(proof.get('failure_health', {}), identity, baselines=baselines):return False + return True + + def _failed_healths(self, task_id, project_path): + from .project_session_health import health_reference + current = self._round(task_id) + if current is None:return () + key = str(canonical_path(project_path)).casefold() + values = [] + for record in self.store._connection.execute('''SELECT command_json,receipt_json FROM execution_operations + WHERE task_id=? AND round_id=? AND state='failed' ''', (task_id, current['round_id'])): + command = json.loads(record['command_json']) + if command['project_path'] != key:continue + health = json.loads(record['receipt_json']).get('proof', {}).get('failure_health', {}) + if health_reference(health):values.append(health) + return tuple(values) + + def failure_health_references(self, task_id, project_path): + from .project_session_health import health_reference + with self.store._lock: + return tuple(health_reference(value) for value in self._failed_healths(task_id, project_path)) + + def certify_safe_failure(self, task_id, row, health): + from pathlib import Path + from .project_session_health import native_completion + key = _hash(command_identity(row)) + with self.store.transaction(): + current = self._round(task_id) + record = self.store._connection.execute('''SELECT * FROM execution_operations + WHERE task_id=? AND round_id=? AND operation_key=?''', (task_id, current['round_id'], key)).fetchone() if current else None + if record is None or record['state'] != 'failed' or row['task_id'] != task_id: + raise TaskCenterError('quick_run_uncertain', '健康证明只能绑定当前轮次已失败的原执行项。') + receipt = json.loads(record['receipt_json']) + if receipt.get('job_id') != row['job_id']: + raise TaskCenterError('quick_run_uncertain', '健康证明任务与原失败回执不一致。') + attempt = JournalAttempt(task_id, current['round_id'], key, record['attempt_id'], record['generation']) + bound = native_completion(Path(self.store.path).parent / 'task-evidence' / task_id, + row, attempt, health, self.expected_baselines(task_id, row['project_path']), + receipt.get('proof', {}).get('native_dispatch', {})) + proof = receipt.setdefault('proof', {'files': []}) + proof['failure_health'] = bound + proof.setdefault('files', []).extend(bound['files']) + self.store._connection.execute('''UPDATE execution_operations SET receipt_json=? + WHERE task_id=? AND round_id=? AND operation_key=?''', (_json(receipt), task_id, current['round_id'], key)) + + def failure_certified(self, task_id, row): + from .project_session_health import affirmative + from .execution_proof import proof_intact + with self.store._lock: + current = self._round(task_id) + if current is None:return False + key = _hash(command_identity(row)) + record = self.store._connection.execute('''SELECT * FROM execution_operations + WHERE task_id=? AND round_id=? AND operation_key=?''', (task_id, current['round_id'], key)).fetchone() + if record is None or record['state'] != 'failed':return False + receipt = json.loads(record['receipt_json']) + proof = receipt.get('proof', {}) + if not isinstance(proof, dict):return False + identity = {'task_id': task_id, 'round_id': current['round_id'], 'operation_key': key, + 'job_id': receipt.get('job_id'), 'project_path': command_identity(row)['project_path'], + 'attempt_id': record['attempt_id'], 'generation': record['generation']} + try: + baselines = self.expected_baselines(task_id, row['project_path']) + except TaskCenterError: + return False + return receipt.get('job_id') == row['job_id'] and proof_intact(proof) and affirmative( + proof.get('failure_health', {}), identity, baselines=baselines) + def pause(self,task_id): with self.store.transaction(): self.store._connection.execute('UPDATE execution_rounds SET pause_requested=1 WHERE task_id=?',(task_id,)) @@ -153,10 +279,20 @@ class ExecutionJournal: keys=[_hash(identity) for identity in identities] if not keys or len(keys)!=len(set(keys)): raise TaskCenterError('quick_run_uncertain','执行计划包含重复或空的逻辑项。') + component_targets=set() + for identity in identities: + if identity.get('execution_unit')!='component':continue + fields=identity.get('target_selectors','').split('\x1f') + targets=_targets(identity) + if len(targets)!=1 or len(fields)!=3 or not all(fields[:2]) or component_targets.intersection(targets): + raise TaskCenterError('quick_run_uncertain','逐位号执行范围重复或身份不完整。') + component_targets.update(targets) with self.store.transaction(): current=self._round(task_id,create=True) round_id=current['round_id'];connection=self.store._connection prior=json.loads(current['plan_json']); retry=bool(current['retry_authorized']) + selection=connection.execute('SELECT keys_json FROM execution_retry_selections WHERE task_id=? AND round_id=?',(task_id,round_id)).fetchone() + retry_keys=set(json.loads(selection[0])) if selection else None if prior and prior!=keys and not retry: raise TaskCenterError('quick_run_uncertain','执行计划与本轮已保存回执不一致,请核验原范围。') restored={} @@ -187,8 +323,14 @@ class ExecutionJournal: restored[row['job_id']]=tuple(json.loads(record['receipt_json'])[name] for name in ('native_state','code','message')) continue elif record['state']=='failed' and retry: + if retry_keys is not None and key not in retry_keys: + restored[row['job_id']]=tuple(json.loads(record['receipt_json'])[name] for name in ('native_state','code','message')) + continue connection.execute("UPDATE execution_operations SET state='pending',attempt_id=NULL WHERE task_id=? AND round_id=? AND operation_key=?",(task_id,round_id,key)) continue # The old failed TSV must not cancel explicit retry. + elif record['state']=='failed': + restored[row['job_id']]=tuple(json.loads(record['receipt_json'])[name] for name in ('native_state','code','message')) + continue outcome=latest.get(row['job_id']) if outcome and outcome[0] in _TERMINAL: record=connection.execute('''SELECT * FROM execution_operations @@ -216,13 +358,16 @@ class ExecutionJournal: # The current UI owns one CBB per bound task. Legacy unbound # multi-project API queues retain their existing failure policy. hold_project=bool(self.store.get_task(task_id).primary_project_id) - if record['state']=='failed' or (hold_project and not importing and self.requires_retry(task_id)): + if record['state']=='failed' or (hold_project and not importing and self.requires_retry(task_id) + and not self.can_continue_component(task_id,command_identity(row))): raise TaskCenterError('quick_run_retry_required','本工程有失败项,请准备失败项重试;不会重新执行成功项。') if record['state']!='pending':raise TaskCenterError('quick_run_uncertain','上次尝试尚未核验,禁止再次派发。') + if not importing and row.get('execution_unit')=='component' and not self.can_continue_component(task_id,command_identity(row)): + raise TaskCenterError('quick_run_uncertain','工程有未决尝试或健康证明不足,暂缓独立位号。') identifier=uuid4().hex;generation=record['generation']+1 self.store._connection.execute('''UPDATE execution_operations SET state='prepared',attempt_id=?,generation=? WHERE task_id=? AND round_id=? AND operation_key=?''',(identifier,generation,task_id,current['round_id'],key)) - return JournalAttempt(task_id,current['round_id'],key,identifier,generation) + return JournalAttempt(task_id,current['round_id'],key,identifier,generation,row['job_id']) def finish(self,attempt,native_state,code,message,*,progress=None,proof=None): if native_state not in _TERMINAL:raise TaskCenterError('quick_run_uncertain','不能提交未知的执行结果。') @@ -240,18 +385,44 @@ class ExecutionJournal: raise TaskCenterError('quick_run_uncertain','旧尝试的结果不能覆盖当前执行回执。') command=json.loads(record['command_json']) receipt={'native_state':native_state,'code':code,'message':message} + # job_id is outside command_identity and must be retained separately + # to bind failure health to this exact dispatch, including rebinding. + health_identity = (proof or {}).get('failure_health', {}).get('identity', {}) + if health_identity: + receipt['job_id'] = health_identity.get('job_id') + else: + receipt['job_id'] = getattr(attempt, 'job_id', None) if proof is not None: from .execution_proof import proof_intact - if not proof_intact(proof):raise TaskCenterError('quick_run_uncertain','执行证明不完整,不能提交回执。') - receipt['proof']=proof + if not proof_intact(proof): + if state != 'failed' or proof.get('files'): + raise TaskCenterError('quick_run_uncertain','执行证明不完整,不能提交回执。') + else: + receipt['proof']=proof + source_checkpoint = None + if state in _SUCCESS and proof and proof.get('baseline'): + healths = self._failed_healths(attempt.task_id, command['project_path']) + if healths: + from pathlib import Path + from .project_session_health import verified_source_checkpoint, health_reference + source_checkpoint = verified_source_checkpoint(Path(self.store.path).parent / 'task-evidence' / attempt.task_id, + dict(command, task_id=attempt.task_id, job_id=attempt.job_id), attempt, proof, + [health_reference(value) for value in healths], excluded_root=healths[0]['source_excluded_root'], + native_source=proof.get('verified_source_health')) + if source_checkpoint: + proof['files'].extend(source_checkpoint['files']) + proof['source_checkpoint'] = source_checkpoint connection.execute('''UPDATE execution_operations SET state=?,receipt_json=? WHERE task_id=? AND round_id=? AND operation_key=?''', (state,_json(receipt),attempt.task_id,attempt.round_id,attempt.operation_key)) + from .execution_problems import ExecutionProblems + ExecutionProblems(self.store).record(attempt,command,state,code,message) if state in _SUCCESS: if proof and proof.get('baseline'): baseline=dict(proof['baseline']) baseline['project_path']=command['project_path'] baseline['files']=proof.get('files',[]) + if source_checkpoint:baseline['source_checkpoint'] = source_checkpoint connection.execute('''INSERT INTO execution_baselines VALUES (?,?,?,?,?,?) ON CONFLICT(task_id,round_id,project_path,schematic_name,sheet_number) DO UPDATE SET baseline_json=excluded.baseline_json''', @@ -264,6 +435,8 @@ class ExecutionJournal: history_receipt={key:value for key,value in receipt.items() if key!='proof'} if proof: history_receipt['proof_file_count']=len(proof.get('files',())) + if 'value_readback' in proof: + history_receipt['value_readback']=proof['value_readback'] HistoryRepository(self.store).append(task_id=attempt.task_id,task_name=task.name, project_path=command['project_path'],operation='器件替换',state=state,summary=message, details={**command,**history_receipt,'round_id':attempt.round_id,'attempt_id':attempt.attempt_id, diff --git a/src/dx_productivity/trial_task_center/execution_problems.py b/src/dx_productivity/trial_task_center/execution_problems.py new file mode 100644 index 0000000..c2156d8 --- /dev/null +++ b/src/dx_productivity/trial_task_center/execution_problems.py @@ -0,0 +1,100 @@ +"""Durable per-target issues and a single automatic replay allowance.""" +import hashlib +import json + +from .errors import TaskCenterError + + +RULE_VERSION = 'safe-prewrite-v1' +RETRYABLE_CODES = frozenset({'cad_attachment_failed', 'worker_timeout_before_write'}) + + +def create_problem_schema(connection): + connection.execute('''CREATE TABLE IF NOT EXISTS execution_problems ( + task_id TEXT NOT NULL, round_id TEXT NOT NULL, problem_key TEXT NOT NULL, + operation_key TEXT NOT NULL, classification TEXT NOT NULL, state TEXT NOT NULL, + code TEXT NOT NULL, message TEXT NOT NULL, auto_retries INTEGER NOT NULL DEFAULT 0, + reservation_source TEXT, rule_version TEXT NOT NULL, + PRIMARY KEY(task_id,round_id,problem_key))''') + connection.execute('''CREATE TABLE IF NOT EXISTS execution_retry_selections ( + task_id TEXT PRIMARY KEY, round_id TEXT NOT NULL, keys_json TEXT NOT NULL)''') + connection.execute('''CREATE TABLE IF NOT EXISTS execution_problem_passes ( + queue_id TEXT NOT NULL, task_id TEXT NOT NULL, round_id TEXT NOT NULL, + generations_json TEXT NOT NULL, state TEXT NOT NULL, + PRIMARY KEY(queue_id,task_id))''') + + +def problem_key(command): + # Policy/part revisions of the same original target never earn a new budget. + targets = sorted(selector.split('\x1f')[0] for selector in + command.get('target_selectors', '').split('\x1e') if selector) + scope = [command.get(name, '') for name in ('project_path', 'schematic_name', 'sheet_number')] + scope.append(targets or ['all', command.get('old_part', '')]) + return hashlib.sha256(json.dumps(scope, ensure_ascii=False).encode('utf-8')).hexdigest() + + +class ExecutionProblems: + def __init__(self, store): + self.store = store + + def record(self, attempt, command, state, code, message): + key = problem_key(command) + if state in {'verified', 'restored', 'no_match'}: + self.store._connection.execute('''UPDATE execution_problems SET state='resolved' + WHERE task_id=? AND round_id=? AND problem_key=?''', + (attempt.task_id, attempt.round_id, key)) + return + category = ('uncertain' if state == 'uncertain' else + 'retryable' if code in RETRYABLE_CODES else 'manual') + self.store._connection.execute('''INSERT INTO execution_problems + (task_id,round_id,problem_key,operation_key,classification,state,code,message,rule_version) + VALUES (?,?,?,?,?,'open',?,?,?) + ON CONFLICT(task_id,round_id,problem_key) DO UPDATE SET + operation_key=excluded.operation_key,classification=excluded.classification, + state='open',code=excluded.code,message=excluded.message''', + (attempt.task_id, attempt.round_id, key, attempt.operation_key, category, code, message, RULE_VERSION)) + + def list(self, task_id): + with self.store._lock: + return tuple(dict(row) for row in self.store._connection.execute('''SELECT p.* + FROM execution_problems p JOIN execution_rounds r + ON p.task_id=r.task_id AND p.round_id=r.round_id WHERE p.task_id=? ORDER BY p.rowid''', (task_id,))) + + def reserve(self, task_id, *, operation_keys=None, source='problems'): + if source not in {'problems', 'dsh'}: + raise TaskCenterError('problem_retry_scope', '未知自动重试来源。') + with self.store.transaction(): + current = self.store._connection.execute('SELECT round_id FROM execution_rounds WHERE task_id=?', (task_id,)).fetchone() + if current is None: + return () + round_id = current[0] + # Finish the primary pass first. A retained unknown write cannot be retried. + unfinished = self.store._connection.execute('''SELECT 1 FROM execution_operations + WHERE task_id=? AND round_id=? AND state IN ('pending','prepared','uncertain') LIMIT 1''', (task_id, round_id)).fetchone() + if unfinished: + return () + requested = set(operation_keys) if operation_keys is not None else None + rows = self.store._connection.execute('''SELECT p.problem_key,p.operation_key,o.command_json + FROM execution_problems p JOIN execution_operations o + ON p.task_id=o.task_id AND p.round_id=o.round_id AND p.operation_key=o.operation_key + WHERE p.task_id=? AND p.round_id=? AND p.classification='retryable' + AND p.state='open' AND p.auto_retries=0 AND o.state='failed' ORDER BY p.rowid''', (task_id, round_id)).fetchall() + keys = [] + for row in rows: + if json.loads(row['command_json']).get('execution_unit') != 'component': + continue + if requested is not None and row['operation_key'] not in requested: + continue + self.store._connection.execute('''UPDATE execution_problems SET auto_retries=1, + reservation_source=? WHERE task_id=? AND round_id=? AND problem_key=? AND auto_retries=0''', + (source, task_id, round_id, row['problem_key'])) + keys.append(row['operation_key']) + return tuple(keys) + + +def problem_projection(problems): + issues = tuple(row for row in problems if row['state'] != 'resolved') + return {'status': 'completed_with_issues' if issues else 'completed', + 'failed': sum(row['classification'] != 'uncertain' for row in issues), + 'uncertain': sum(row['classification'] == 'uncertain' for row in issues), + 'resolved': sum(row['state'] == 'resolved' for row in problems), 'items': issues} diff --git a/src/dx_productivity/trial_task_center/execution_proof.py b/src/dx_productivity/trial_task_center/execution_proof.py index bad7299..ff3da5d 100644 --- a/src/dx_productivity/trial_task_center/execution_proof.py +++ b/src/dx_productivity/trial_task_center/execution_proof.py @@ -25,9 +25,19 @@ def capture_result(data_root,root,row,outcome,*,strict=False,persist=True): paths=[] try: _safe_run_token(row['job_id']) + if row.get('execution_unit') == 'inventory_empty': + from .component_scope import validate_empty_command + if state != 'batch_skipped' or not validate_empty_command(root, row): + raise ValueError('empty inventory command is not certified') + manifest = internal_path(root / 'component-scope.json') + checked_manifest = manifest.read_bytes() + paths.append(manifest) + proof['inventory_empty'] = True if strict and state in {'saved_ok','no_match'} and code not in {'batch_closed','session_verified'}: raise ValueError('save/close was not verified') checked={} + if row.get('execution_unit') == 'inventory_empty': + checked[manifest] = checked_manifest baseline=internal_path(root/f'post-state-{row["job_id"]}.tsv') if baseline.is_file(): checked[baseline]=baseline.read_bytes() diff --git a/src/dx_productivity/trial_task_center/inventory.py b/src/dx_productivity/trial_task_center/inventory.py index 67dfdbb..8eaf915 100644 --- a/src/dx_productivity/trial_task_center/inventory.py +++ b/src/dx_productivity/trial_task_center/inventory.py @@ -11,6 +11,7 @@ from .target_plan import ComponentTarget, ProjectIdentityResolver HEADER = ("project_path", "schematic_name", "sheet_number", "uid", "refdes", "old_part", "old_value") +EFFECTIVE_HEADER = HEADER + ('actual_part_label', 'actual_cell') def validate_target_inventory(project_path, rules, components): @@ -25,7 +26,9 @@ def validate_target_inventory(project_path, rules, components): actual = current.get(identities.target(target)) if (actual is None or actual.refdes != target.refdes or actual.old_part_number.casefold() != target.old_part_number.casefold() - or actual.old_value != target.old_value): + or actual.old_value != target.old_value + or any(getattr(target, name) is not None and getattr(actual, name) != getattr(target, name) + for name in ('actual_part_label', 'actual_cell'))): raise TaskCenterError('target_identity', f'位号计划与当前工程不一致,请重新采集:{target.refdes}(UID={target.uid})') @@ -36,7 +39,8 @@ def read_component_inventory(path: Path, project_path: Path, try: lines = Path(path).read_text(encoding="utf-8-sig").splitlines() - if not lines or tuple(lines[0].split("\t")) != HEADER: + header = tuple(lines[0].split('\t')) if lines else () + if header not in {HEADER, EFFECTIVE_HEADER}: raise ValueError("component inventory header changed") result: list[ComponentTarget] = [] seen: set[tuple[str, str, str]] = set() @@ -44,15 +48,15 @@ def read_component_inventory(path: Path, project_path: Path, if not line: continue fields = tuple(_decode_vbs_field(field) for field in line.split("\t")) - if len(fields) != len(HEADER): + if len(fields) != len(header): raise ValueError("component inventory row is malformed") - project, schematic, sheet, uid, refdes, old_part, old_value = fields + project, schematic, sheet, uid, refdes, old_part, old_value = fields[:7] if (_path_key(project) != _path_key(project_path) or (schematic, sheet) not in sheets or not all((uid, refdes, old_part)) or (schematic, sheet, uid) in seen): raise ValueError("component inventory identity is invalid") seen.add((schematic, sheet, uid)) result.append(ComponentTarget(Path(project_path), schematic, sheet, uid, refdes, - old_part, old_value)) + old_part, old_value, *fields[7:])) return tuple(result) except (OSError, UnicodeError, ValueError) as error: raise TaskCenterError("target_inventory", str(error)) from error diff --git a/src/dx_productivity/trial_task_center/isolated_cad.py b/src/dx_productivity/trial_task_center/isolated_cad.py index bf7f4f8..1659d8f 100644 --- a/src/dx_productivity/trial_task_center/isolated_cad.py +++ b/src/dx_productivity/trial_task_center/isolated_cad.py @@ -16,6 +16,8 @@ from .worker_monitor import WorkerMonitorError from .path_access import internal_path, canonical_path, retry_sharing_violation from .cad_dialogs import CadDialogGuard +RETAINED_CAD_SESSIONS = {} + class PrivateCadStartupError(WorkerMonitorError): """The owned desktop was released before any project worker was launched.""" @@ -89,7 +91,7 @@ def _designer_executable(): class IsolatedCadSession: """The supervisor never attaches to COM on the user's input desktop.""" - def __init__(self, root, *, abort_event=None, desktop=None, stop_control=None): + def __init__(self, root, *, abort_event=None, desktop=None, stop_control=None, read_only=False): self.root = canonical_path(root) self.token = uuid4().hex self.borrowed_desktop = desktop is not None @@ -99,7 +101,9 @@ class IsolatedCadSession: self.started = time.monotonic() self.abort_event = abort_event self.stop_control = stop_control - self.record = {'desktop': self.desktop.name, 'token': self.token, 'state': 'starting'} + self.read_only = read_only + self.record = {'desktop': self.desktop.name, 'token': self.token, 'state': 'starting', + 'read_only': read_only} self._dialogs = None self._dialog_monitor_error = '' self._dialog_stop = Event() @@ -120,13 +124,19 @@ class IsolatedCadSession: _replace_record(pending, archived) def __enter__(self): + if RETAINED_CAD_SESSIONS: + raise WorkerMonitorError('an owned CAD session is retained for manual reconciliation') try: self._check_abort() self.desktop.__enter__() self.cad = self.desktop.start_process((str(_designer_executable()),)) + publish = getattr(self.desktop, 'register_cad_process', None) + if publish is not None: + publish(self.cad, self.token) self._dialogs = CadDialogGuard(self.desktop, self.cad, self.root / ('cad-dialogs-' + self.token + '.json'), self.token) - self._record(cad_pid=self.cad.pid) + from .batch_resources import process_identity + self._record(cad_pid=self.cad.pid, cad_process=process_identity(self.cad)) bootstrap = self.root / ('cad-binding-' + self.token + '.json') environment = dict(os.environ, DXPART_DESKTOP=self.desktop.name, DXPART_CAD_PID=str(self.cad.pid), DXPART_SESSION_TOKEN=self.token) @@ -266,6 +276,8 @@ class IsolatedCadSession: return result def __exit__(self, error_type, error, traceback): + if self.read_only: + return self._close_read_only() self._stop_dialog_monitor() closing_started = time.monotonic() cleanup_errors = [] @@ -334,20 +346,30 @@ class IsolatedCadSession: # Evidence failure must never prevent the identity-checked Quit # above or release of the handles we own below. persist_uncertain() - if self.stop_control is not None and cleanup_errors and self.cad is not None: + if self.cad is not None: cad_running = True try: - cad_running = self.cad.poll() is None + from .read_recovery import process_tree_exited + cad_running = not all(process_tree_exited(process) + for process in self._owned_processes(closing)) except Exception as failure: remember('cad_exit_check', failure) if cad_running: + if not cleanup_errors: + remember('owned_tree_active', WorkerMonitorError('owned processes remain after orderly Quit')) + persist_uncertain() + from .stop_control import StopControl + if not isinstance(self.stop_control, StopControl): + self._retain_for_manual(closing) + raise WorkerMonitorError('owned CAD is still running; handles retained until explicit force or exit') # A refused/timed-out Quit may still be saving or closing. # Keep both CAD and its helper owned and alive until the # caller confirms force or both retained processes exit. from .stop_control import wait_for_resolution def owned_processes_exited(): self._poll_dialogs('closing') - return self.cad.poll() is not None and (closing is None or closing.poll() is not None) + from .read_recovery import process_tree_exited + return all(process_tree_exited(process) for process in self._owned_processes(closing)) try: wait_for_resolution(self.stop_control, self.root, failure_message(), resolved=owned_processes_exited) @@ -363,6 +385,7 @@ class IsolatedCadSession: remember('quit_helper_release', failure) try: self.desktop.close() + RETAINED_CAD_SESSIONS.pop(str(self.root), None) except Exception as failure: remember('desktop_release', failure) desktop_cleanup = getattr(self.desktop, 'cleanup_record', None) or {} @@ -387,6 +410,40 @@ class IsolatedCadSession: if failures: raise WorkerMonitorError('private CAD cleanup requires reconciliation: ' + failure_message()) from failures[0] + def _owned_processes(self, closing=None): + processes = list(getattr(self.desktop, '_processes', getattr(self.desktop, 'processes', ()))) + for process in (*getattr(self, '_retained_processes', ()), self.cad, closing): + if process is not None and not any(item is process for item in processes): + processes.append(process) + return processes + + def _retain_for_manual(self, closing): + from .batch_resources import process_identity + processes = self._owned_processes(closing) + self._retained_processes = processes + if getattr(self.desktop, 'owner', None) is None: + RETAINED_CAD_SESSIONS[str(self.root)] = self + self._record(state='uncertain', retained_for_manual=True, + retained_processes=[process_identity(process) for process in processes]) + + def _close_read_only(self): + """Whole-generation cleanup is allowed only by admitted read provenance.""" + self._stop_dialog_monitor() + started = time.monotonic() + # Record the authority and exact ownership before any terminate-capable + # handle close. A failed evidence write retains the session for retry. + self._record(state='read_only_releasing', read_only_cleanup_authorized=True, + cad_pid=getattr(self.cad, 'pid', None)) + try: + self.desktop.close() + except Exception as error: + self._record(state='read_only_release_pending', clean=False, cleanup_error=str(error)) + raise WorkerMonitorError('read-only generation release was not confirmed: ' + str(error)) from error + self._record(state='session_closed' if self.borrowed_desktop else 'released', + clean=True, read_only_released=True, desktop_retained_by_batch=self.borrowed_desktop, + shutdown_seconds=time.monotonic() - started, + desktop_cleanup=getattr(self.desktop, 'cleanup_record', None)) + def _uncertain(self, message, **details): from .quick_run import _write_uncertain_flag _write_uncertain_flag(self.root / 'uncertain.flag', 'private_desktop: ' + message) @@ -399,7 +456,7 @@ def run_isolated_process(command, timeout, *, abort_event=None, cooperative_abor return runtime.run_worker(command, timeout, abort_event=abort_event, cooperative_abort=cooperative_abort) try: - with IsolatedCadSession(Path(command[3]), abort_event=abort_event) as session: + with IsolatedCadSession(Path(command[3]), abort_event=abort_event, read_only=cooperative_abort) as session: return run_monitored_process(command, timeout, abort_event=abort_event, process_factory=session.start_worker, **({'cooperative_abort': True} if cooperative_abort else {})) except CadIdentityConflict: diff --git a/src/dx_productivity/trial_task_center/isolated_desktop.py b/src/dx_productivity/trial_task_center/isolated_desktop.py index 56055ee..fcb6456 100644 --- a/src/dx_productivity/trial_task_center/isolated_desktop.py +++ b/src/dx_productivity/trial_task_center/isolated_desktop.py @@ -345,6 +345,17 @@ class _Win32Process: self.returncode = self._process_api.GetExitCodeProcess(self.handle) return self.returncode + def tree_exited(self): + if self.poll() is None: + return False + if self.job is not None and self.job_assigned: + active = self._job_api.QueryInformationJobObject( + self.job, self._job_api.JobObjectBasicAccountingInformation)['ActiveProcesses'] + if active: + return False + return all(self._event_api.WaitForSingleObject(handle, 0) == 0 + for handle in self._descendant_handles) + def wait(self, timeout=None): if self.returncode is not None: return self.returncode @@ -549,6 +560,13 @@ class PrivateDesktop: raise TaskCenterError('private_desktop', 'dialog process is no longer owned and alive') acknowledge_owned_dialog(self._api, self.name, pid, dialog) + def manual_dialog_action(self, dialog, button_hwnd): + from .cad_dialogs import manually_act_on_owned_dialog + pid = dialog['pid'] + if self._handle is None or not any(process.pid == pid and process.poll() is None for process in self._processes): + raise TaskCenterError('private_desktop', 'dialog process is no longer owned and alive') + manually_act_on_owned_dialog(self._api, self.name, pid, dialog, button_hwnd) + def _create(self) -> None: if os.name != "nt" and self._api is None: return diff --git a/src/dx_productivity/trial_task_center/native_inventory_protocol.py b/src/dx_productivity/trial_task_center/native_inventory_protocol.py index 31aecca..5507e00 100644 --- a/src/dx_productivity/trial_task_center/native_inventory_protocol.py +++ b/src/dx_productivity/trial_task_center/native_inventory_protocol.py @@ -14,7 +14,7 @@ class NativeInventoryUnsupported(ValueError): _UID = re.compile(r'^\$\d+I\d+$') -_PROPERTY = re.compile(r'\tProp (Ref Designator|Part Number|Value)(?: (.*))?') +_PROPERTY = re.compile(r'\tProp (Ref Designator|Part Number|Value|Part Label|Cell Name)(?: (.*))?') def read_native_text(path: Path) -> str: @@ -109,7 +109,8 @@ def _parse_scope(text, requested, effective, inconsistent_refs): ref, part = values.get('Ref Designator', ''), values['Part Number'] if not ref or 'Ref Designator' not in attributes: raise NativeInventoryUnsupported('全工程报告中的请求物料缺少位号') - target = block, sheet, uid, ref, part, values.get('Value', '') + target = (block, sheet, uid, ref, part, values.get('Value', ''), + values.get('Part Label'), values.get('Cell Name')) if target in targets: raise NativeInventoryUnsupported('全工程报告含重复器件身份') targets.add(target) diff --git a/src/dx_productivity/trial_task_center/problem_pass.py b/src/dx_productivity/trial_task_center/problem_pass.py new file mode 100644 index 0000000..30c41b3 --- /dev/null +++ b/src/dx_productivity/trial_task_center/problem_pass.py @@ -0,0 +1,61 @@ +"""Run only reserved safe issues, after all runnable primary items finish.""" +import json + +from .errors import TaskCenterError + + +def run_problem_pass(service, task_ids, queue_id, *, stop_event, runtime): + results = {} + store = service._store + for task_id in task_ids: + if stop_event.is_set(): + break + child = service.for_project_task(task_id) + if child.has_interrupted_quick_run(task_id): + continue + with store.transaction(): + record = store._connection.execute('SELECT * FROM execution_problem_passes WHERE queue_id=? AND task_id=?', + (queue_id, task_id)).fetchone() + current = store._connection.execute('SELECT round_id FROM execution_rounds WHERE task_id=?', (task_id,)).fetchone() + if record is None: + keys = child.reserve_problem_retry(task_id, source='problems') + if not keys: + continue + generations = {row['operation_key']: row['generation'] for row in store._connection.execute( + 'SELECT operation_key,generation FROM execution_operations WHERE task_id=? AND round_id=?', + (task_id, current[0])) if row['operation_key'] in keys} + store._connection.execute('INSERT INTO execution_problem_passes VALUES (?,?,?,?,?)', + (queue_id, task_id, current[0], json.dumps(generations, sort_keys=True), 'reserved')) + record = store._connection.execute('SELECT * FROM execution_problem_passes WHERE queue_id=? AND task_id=?', + (queue_id, task_id)).fetchone() + if current is None or current[0] != record['round_id']: + raise TaskCenterError('problem_retry_scope', '问题轮次与原执行轮次不一致。') + if record['state'] == 'finished': + continue + original = json.loads(record['generations_json']) + states = {row['operation_key']: row for row in store._connection.execute( + 'SELECT operation_key,generation,state FROM execution_operations WHERE task_id=? AND round_id=?', + (task_id, current[0])) if row['operation_key'] in original} + if set(states) != set(original): + raise TaskCenterError('problem_retry_scope', '原问题项已变化,不能扩大自动重试范围。') + if any(row['state'] in {'prepared', 'uncertain'} for row in states.values()): + raise TaskCenterError('quick_run_uncertain', '问题项上次尝试尚未核验,禁止自动重放。') + prepare_keys = tuple(key for key, row in states.items() + if row['state'] == 'failed' and row['generation'] == original[key]) + if prepare_keys: + child.prepare_failed_quick_run(task_id, runtime=runtime, operation_keys=prepare_keys) + pending = bool(prepare_keys) or any(row['state'] == 'pending' for row in states.values()) + if pending and not stop_event.is_set(): + progress = child.quick_run_progress(task_id) + task = child.get_task(task_id) + expected = (str(progress.next_project_path), progress.next_old_part_number, + progress.next_new_part_number, 'part_label' if task.copy_part_number_to_value else 'preserve') + result = child.execute_quick_run_queue(task_id, expected_item=expected, + stop_event=stop_event, runtime=runtime) + results[task_id] = result + if result.paused: + break + if not stop_event.is_set(): + with store.transaction(): + store._connection.execute("UPDATE execution_problem_passes SET state='finished' WHERE queue_id=? AND task_id=?", (queue_id, task_id)) + return results diff --git a/src/dx_productivity/trial_task_center/project_session_health.py b/src/dx_productivity/trial_task_center/project_session_health.py new file mode 100644 index 0000000..161c0e4 --- /dev/null +++ b/src/dx_productivity/trial_task_center/project_session_health.py @@ -0,0 +1,235 @@ +"""Affirmative, immutable failure health bound to one journal attempt.""" +import hashlib +import json +from pathlib import Path +from uuid import uuid4 + +from .batch_resources import write_record +from .errors import TaskCenterError +from .path_access import canonical_path, internal_path + +SOURCE_DIGEST_ALGORITHM = 'sha256-project-fingerprint-repr-v1' + + +def _source_digest(value): + from .discovery_cache import project_fingerprint + if value.get('source_digest_algorithm') != SOURCE_DIGEST_ALGORITHM: + return None + if not all(Path(value[key]).is_absolute() for key in ('source_project_path', 'source_excluded_root')): + return None + project, root = canonical_path(value['source_project_path']), canonical_path(value['source_excluded_root']) + stamp = project_fingerprint(project, root) + return hashlib.sha256(repr(stamp).encode()).hexdigest() if stamp is not None else None + + +def _source_parameters(project, root): + return {'source_project_path': str(canonical_path(project)), + 'source_excluded_root': str(canonical_path(root)), 'source_digest_algorithm': SOURCE_DIGEST_ALGORITHM} + + +def _intact_certificate(value): + from .execution_proof import proof_intact + return (isinstance(value, dict) and isinstance(value.get('files'), list) and bool(value['files']) and proof_intact(value) + and json.loads(internal_path(value['files'][0]['path']).read_text(encoding='utf-8')) == + {key: item for key, item in value.items() if key != 'files'}) + + +def health_reference(health): + try: + return {'identity': health['identity'], 'certificate_sha256': health['files'][0]['sha256']} if _intact_certificate(health) else None + except (OSError, ValueError, KeyError, TypeError): + return None + + +def _identity(row, attempt): + from .execution_journal import command_identity, _hash + return {'task_id': row['task_id'], 'round_id': attempt.round_id, + 'operation_key': _hash(command_identity(row)), 'job_id': row['job_id'], + 'project_path': str(canonical_path(row['project_path'])).casefold(), + 'attempt_id': attempt.attempt_id, 'generation': attempt.generation} + + +def _persist(data_root, value, evidence=()): + destination = internal_path(Path(data_root) / 'execution-proof' / uuid4().hex) + files = [] + for number, (path, digest) in enumerate(evidence): + data = internal_path(path).read_bytes() + if hashlib.sha256(data).hexdigest() != digest: + raise TaskCenterError('quick_run_uncertain', '工程健康证明文件已变化。') + target = destination / (str(number) + '-' + Path(path).name) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open('xb') as stream: + import os + stream.write(data); stream.flush(); os.fsync(stream.fileno()) + files.append({'path': str(canonical_path(target)), 'sha256': digest}) + certificate = destination / 'completion.json' + write_record(certificate, value) + files.insert(0, {'path': str(canonical_path(certificate)), + 'sha256': hashlib.sha256(certificate.read_bytes()).hexdigest()}) + return dict(value, files=files) + + +def synchronous_completion(data_root, row, attempt, outcome, before, *, runtime=None): + from .discovery_cache import project_fingerprint + from .batch_resources import LIVE_RESOURCES + from .isolated_cad import RETAINED_CAD_SESSIONS + from .execution_journal import command_identity + absent = runtime is None or (hasattr(runtime, 'desktop') and hasattr(runtime, 'cad') + and runtime.desktop is None and runtime.cad is None) + root = canonical_path(data_root) + def related(path): + return path.is_relative_to(root) or root.is_relative_to(path) + absent = absent and not any(related(session.root) for session in RETAINED_CAD_SESSIONS.values()) and not any( + owner.desktop is not None or owner.cad is not None for owner in LIVE_RESOURCES.values() if related(owner.root)) + value = {'version': 1, 'identity': _identity(row, attempt), 'command': command_identity(row), 'outcome': list(outcome), + 'kind': 'synchronous_completion', 'resources_released': absent, + 'project_closed': absent, 'no_mutation': False, 'source_unchanged': False} + if outcome[0] == 'batch_failed': + after = project_fingerprint(canonical_path(row['project_path']), canonical_path(data_root)) + if before is None or after is None or before != after or not absent: + raise TaskCenterError('quick_run_uncertain', '失败后无法证明工程源文件未变化且执行会话已释放。') + value.update(no_mutation=True, source_unchanged=True, + source_sha256=hashlib.sha256(repr(after).encode()).hexdigest()) + value.update(_source_parameters(row['project_path'], data_root)) + return _persist(data_root, value) + + +def runtime_dispatch_binding(data_root, row, attempt, runtime): + from .batch_attempt import read_attempt + from .execution_journal import command_identity + selected = [(path, expected) for path, expected in runtime._current_attempts + if expected.get('job_id') == row['job_id'] and expected.get('task_id') == row['task_id'] + and expected.get('attempt_id') == runtime.record.get('active_attempt') + and expected.get('session_id') == runtime.session_id and expected.get('generation') == runtime.generation] + if len(selected) != 1: + raise TaskCenterError('quick_run_uncertain', '当前原生派发身份无法与执行回执绑定。') + path, expected = selected[0] + actual = read_attempt(path) + if any(actual.get(key) != expected.get(key) for key in ('job_id', 'task_id', 'project_path', 'session_id', 'attempt_id', 'generation')): + raise TaskCenterError('quick_run_uncertain', '原生派发身份发生变化。') + return _persist(data_root, {'version': 1, 'kind': 'native_dispatch', 'identity': _identity(row, attempt), 'command': command_identity(row), + 'native_identity': {key: actual[key] for key in ('session_id', 'attempt_id', 'generation')}}, + [(path, hashlib.sha256(internal_path(path).read_bytes()).hexdigest())]) + + +def native_completion(data_root, row, attempt, health, baselines, dispatch): + from .execution_proof import proof_intact + from .execution_journal import command_identity + from .batch_resources import _valid_record, _released_evidence + try: + if (not dispatch.get('files') or not proof_intact(dispatch) + or dispatch.get('identity') != _identity(row, attempt) + or dispatch.get('native_identity') != {key: health.get(key) for key in ('session_id', 'attempt_id', 'generation')} + or json.loads(internal_path(dispatch['files'][0]['path']).read_text(encoding='utf-8')) != + {key: value for key, value in dispatch.items() if key != 'files'}): + raise ValueError('health belongs to another logical or native dispatch') + if (health.get('version') != 1 or health.get('job_id') != row['job_id'] + or canonical_path(health['project_path']) != canonical_path(row['project_path']) + or not all(health.get(key) is True for key in ('no_mutation', 'resources_released', 'project_closed')) + or not isinstance(health.get('source_sha256'), str) or len(health['source_sha256']) != 64): + raise ValueError('non-affirmative project health') + native_path = internal_path(health['proof_path']) + if json.loads(native_path.read_text(encoding='utf-8')) != health: + raise ValueError('health record changed') + release_path = canonical_path(health['resource_record']) + release = json.loads(internal_path(release_path).read_text(encoding='utf-8')) + checkpoint_path = native_path.parent / 'project-checkpoint.json' + checkpoint = json.loads(checkpoint_path.read_text(encoding='utf-8')) + if (not _valid_record(release, release_path) or release.get('state') != 'released' + or not _released_evidence(release) + or any(release.get(key) != health.get(key) for key in ('session_id', 'generation')) + or checkpoint.get('project_closed') is not True or checkpoint.get('worker_closed') is not True + or canonical_path(checkpoint['project_path']) != canonical_path(row['project_path']) + or any(checkpoint.get(key) != health.get(key) for key in ('session_id', 'generation', 'attempt_id')) + or (checkpoint.get('cad_pid'), checkpoint.get('cad_guid')) != (release['cad_pid'], release['cad_guid'])): + raise ValueError('physical generation release is not certified') + native = json.loads(internal_path(health['attempt_record_path']).read_text(encoding='utf-8')) + if (native.get('state') != 'prepared' or native.get('job_id') != row['job_id'] + or native.get('task_id') != row['task_id'] + or canonical_path(native['project_path']) != canonical_path(row['project_path']) + or any(native.get(key) != health.get(key) for key in ('session_id', 'attempt_id', 'generation'))): + raise ValueError('native attempt does not bind the failed job') + evidence = [(health['proof_path'], hashlib.sha256(native_path.read_bytes()).hexdigest()), + (health['attempt_record_path'], health['attempt_record_sha256']), + (health['resource_record'], health['resource_record_sha256']), + (checkpoint_path, health['checkpoint_sha256'])] + evidence.extend((item['path'], item['sha256']) for item in dispatch['files']) + baseline = None + if health.get('source_unchanged') is not True: + baseline = next((value for value in baselines + if value.get('project_path') == str(canonical_path(row['project_path'])).casefold() + and (value.get('schematic_name'), value.get('sheet_number')) == (row.get('schematic_name'), row.get('sheet_number')) + and value.get('state', '').startswith('S\tfull_sheet_v1\r\n') and proof_intact(value)), None) + if baseline is None: + raise ValueError('no intact last verified baseline for the failed sheet') + evidence.extend((item['path'], item['sha256']) for item in baseline['files']) + return _persist(data_root, {'version': 1, 'identity': _identity(row, attempt), 'command': command_identity(row), + 'kind': 'native_failure_health', 'resources_released': True, 'project_closed': True, + 'no_mutation': True, 'source_unchanged': health.get('source_unchanged') is True, + 'source_sha256': health['source_sha256'], 'verified_baseline': baseline, + **_source_parameters(health['project_path'], canonical_path(health['session_record']).parents[2]), + 'native_identity': {key: health[key] for key in ('session_id', 'attempt_id', 'generation')}}, evidence) + except (OSError, ValueError, KeyError, TypeError) as error: + raise TaskCenterError('quick_run_uncertain', '当前失败位号没有完整且匹配的工程健康证明。') from error + + +def verified_source_checkpoint(data_root, row, attempt, proof, references, *, excluded_root, native_source=None): + from .execution_proof import proof_intact + if not references or not proof.get('baseline') or not proof.get('files') or not proof_intact(proof): + return None + value = {'version': 1, 'kind': 'verified_source_checkpoint', 'identity': _identity(row, attempt), + 'baseline': proof['baseline'], 'supersedes_failure_health': references, + **_source_parameters(row['project_path'], excluded_root)} + evidence = [(item['path'], item['sha256']) for item in proof['files']] + if native_source is not None: + from .batch_resources import _valid_record, _released_evidence + health_path = internal_path(native_source['proof_path']) + release_path = canonical_path(native_source['resource_record']) + release = json.loads(internal_path(release_path).read_text(encoding='utf-8')) + if (json.loads(health_path.read_text(encoding='utf-8')) != native_source + or native_source.get('job_id') != row['job_id'] + or canonical_path(native_source['project_path']) != canonical_path(row['project_path']) + or any(native_source.get(key) is not True for key in ('resources_released', 'project_closed')) + or any(native_source.get(key) != proof.get('native_attempt', {}).get(key) + for key in ('session_id', 'attempt_id', 'generation')) + or not _valid_record(release, release_path) or release.get('state') != 'released' + or not _released_evidence(release) + or any(release.get(key) != native_source.get(key) for key in ('session_id', 'generation'))): + raise TaskCenterError('quick_run_uncertain', '已保存步骤的资源释放证明无法与原生回执绑定。') + value.update(_source_parameters(native_source['project_path'], canonical_path(native_source['session_record']).parents[2])) + digest = native_source['source_sha256'] + evidence.extend(((health_path, hashlib.sha256(health_path.read_bytes()).hexdigest()), + (release_path, native_source['resource_record_sha256']))) + else: + digest = _source_digest(value) + if not isinstance(digest, str) or len(digest) != 64 or any(char not in '0123456789abcdef' for char in digest): + raise TaskCenterError('quick_run_uncertain', '已保存步骤的工程源文件无法核对,不能放行后续位号。') + value['source_sha256'] = digest + return _persist(data_root, value, evidence) + + +def affirmative(health, identity, *, baselines=()): + from .execution_proof import proof_intact + try: + if not _intact_certificate(health) or health.get('identity') != identity: + return False + valid = (all(health.get(key) is True for key in ('no_mutation', 'resources_released', 'project_closed')) + and (health.get('source_unchanged') is True or health.get('kind') == 'native_failure_health' + and health.get('verified_baseline') and proof_intact(health['verified_baseline']))) + if not valid:return False + if _source_digest(health) == health.get('source_sha256') and health.get('source_sha256'): + return True + command = health['command'] + for baseline in baselines: + checkpoint = baseline.get('source_checkpoint', {}) + if ((baseline.get('project_path'), baseline.get('schematic_name'), baseline.get('sheet_number')) != + (command['project_path'], command.get('schematic_name'), command.get('sheet_number')) + or not baseline.get('files') or not proof_intact(baseline) or not _intact_certificate(checkpoint) + or checkpoint.get('baseline') != {key: baseline[key] for key in checkpoint.get('baseline', {})} + or health_reference(health) not in checkpoint.get('supersedes_failure_health', ())): + continue + if _source_digest(checkpoint) == checkpoint.get('source_sha256') and checkpoint.get('source_sha256'): + return True + return False + except (OSError, ValueError, KeyError, TypeError): + return False diff --git a/src/dx_productivity/trial_task_center/quick_run.py b/src/dx_productivity/trial_task_center/quick_run.py index 7f921f3..01c2885 100644 --- a/src/dx_productivity/trial_task_center/quick_run.py +++ b/src/dx_productivity/trial_task_center/quick_run.py @@ -103,6 +103,9 @@ class QuickRunPreparer: sheet_targets: dict[str, tuple[tuple[str, str], ...]] | None = None, discovery_matches: dict[str, dict[tuple[str, str], frozenset[str]]] | None = None, discovery_failures: dict[str, str] | None = None, + split_targets: bool = True, + component_scope: dict | None = None, + component_scope_from: Path | None = None, ) -> QuickRunResult: from .handoff_guard import assert_handoff_intact # Preserve the public preparation errors (and diagnostic entries) for @@ -118,6 +121,22 @@ class QuickRunPreparer: raise TaskCenterError("selected_project_required", "at least one available project must be selected") if not enabled_rules: raise TaskCenterError("enabled_rule_required", "at least one replacement rule must be enabled") + if component_scope_from is not None: + original = _io_path(component_scope_from) + original_header = (original / 'command.tsv').read_text(encoding='utf-8-sig').splitlines()[0].split('\t') + split_targets = 'execution_unit' in original_header + scope_path = original / 'component-scope.json' + if scope_path.exists(): + from .component_scope import load + component_scope = load(scope_path) + elif 'target_scope' in original_header and all(getattr(rule, 'scope', 'all') == 'all' for rule in enabled_rules): + raise TaskCenterError('component_scope', 'Frozen component scope sidecar is missing; refusing an all-match retry') + if component_scope is not None: + from .component_scope import expand + if sheet_targets is None: + raise TaskCenterError('component_scope', 'Frozen component scope requires its original sheet manifest') + enabled_rules = expand(component_scope, selected_projects, enabled_rules, + _value_mode(copy_part_number_to_value), sheet_targets) source = self._required_script_source() self._validate_fields(selected_projects, enabled_rules) _validate_target_scope(selected_projects, enabled_rules, sheet_targets) @@ -137,6 +156,11 @@ class QuickRunPreparer: _io_path(manifest_path).write_text(_sheet_manifest_text(selected_projects, sheet_targets), encoding="utf-8", newline="") else: (_io_path(root / "sheets.tsv")).unlink(missing_ok=True) + scope_path = _io_path(root / 'component-scope.json') + if component_scope is not None: + scope_path.write_text(json.dumps(component_scope, ensure_ascii=False, sort_keys=True), encoding='utf8') + else: + scope_path.unlink(missing_ok=True) _io_path(command_path).write_text( _command_text( task_id, @@ -145,6 +169,7 @@ class QuickRunPreparer: enabled_rules, _value_mode(copy_part_number_to_value), sheet_targets=sheet_targets, + split_targets=split_targets, ), encoding="utf-8", newline="", @@ -207,6 +232,7 @@ class QuickRunPreparer: {} if self._sheet_discoverer is None else None ) discovery_failures: dict[str, str] = {} + component_discoveries = {} targeted = all(getattr(rule, 'scope', 'all') == 'targets' for rule in enabled_rules) old_part_numbers = tuple(rule.old_part_number for rule in enabled_rules) for project in selected_projects: @@ -215,12 +241,13 @@ class QuickRunPreparer: before_sheet_discovery(project.path) if self._sheet_discoverer is None: discovery = self.discover_project(project.path, old_part_numbers, - **({'collect_components': True} if targeted else {}), + collect_components=True, **({'runtime': runtime} if runtime is not None else {})) if targeted: from .inventory import validate_target_inventory validate_target_inventory(project.path, enabled_rules, discovery.components) targets = discovery.sheets + component_discoveries[_path_key(project.path)] = discovery assert discovery_matches is not None discovery_matches[_path_key(project.path)] = discovery.matched_old_parts else: @@ -241,6 +268,11 @@ class QuickRunPreparer: discovery_matches[project_key] = {_MISSING_PROJECT_SHEET: frozenset()} _validate_sheet_targets(project.path, targets) sheet_targets[_path_key(project.path)] = targets + component_scope = None + if self._sheet_discoverer is None and not targeted: + from .component_scope import freeze + component_scope = freeze(selected_projects, enabled_rules, sheet_targets, + component_discoveries, discovery_failures, value_mode) return self.prepare( task_id, projects, @@ -249,6 +281,7 @@ class QuickRunPreparer: sheet_targets=sheet_targets, discovery_matches=discovery_matches, discovery_failures=discovery_failures, + component_scope=component_scope, ) def begin_batch_session( @@ -446,6 +479,7 @@ class QuickRunPreparer: selected_command = _next_unfinished_command(commands, before) if selected_command is None: raise TaskCenterError("quick_run_complete", "all replacement rules in this queue are complete") + _assert_dispatchable_component(handoff.command_path, selected_command[1], batch_session) actual_item = (*selected_command[2:], value_mode) if expected_item is not None and actual_item != expected_item: raise TaskCenterError("quick_run_selection_changed", "confirmed queue item or Value policy changed; confirm again") @@ -462,10 +496,26 @@ class QuickRunPreparer: batch_request = handoff.script_path.parent / "batch.request" journal = getattr(self, 'journal', None) journal_attempt = None + dispatch_fingerprint = None if journal is not None: row = (batch_session.journal_rows if batch_session is not None else journal_rows)[selected_command[1]] self.write_expected_baselines(task_id,handoff.command_path.parent,selected_command[2]) journal_attempt = journal.claim(task_id, row) + if self._process_runner is not _run_process: + from .discovery_cache import project_fingerprint + dispatch_fingerprint = project_fingerprint(Path(selected_command[2]), self._data_root) + if self._process_runner is _run_process: + try: + from .cell_catalog import preflight_command_target + preflight_command_target(row, self._data_root / 'cell-catalog') + except TaskCenterError as error: + if error.code == 'target_cell_catalog': + self._record_rejected_attempt(journal_attempt, handoff, selected_command, before, + projects, error.message, code='target_library_rejected', + row=row, runtime=runtime, dispatch_not_started=True) + raise + from .value_readback import clear_value_readback + clear_value_readback(handoff.command_path.parent, row) current_command_stamp = None if batch_mode: if batch_session is not None: @@ -530,7 +580,8 @@ class QuickRunPreparer: detail = completed.stderr.strip() or completed.stdout.strip() or "DxDesigner attachment failed" if journal_attempt is not None: self._record_rejected_attempt(journal_attempt, handoff, selected_command, before, - projects, detail) + projects, detail, code='cad_attachment_failed', row=row, + source_before=dispatch_fingerprint, runtime=runtime) raise TaskCenterError("quick_run_session", _session_failure_message(detail)) if completed.returncode == 3: if after == before: @@ -538,7 +589,8 @@ class QuickRunPreparer: detail = completed.stderr.strip() or completed.stdout.strip() or "DxDesigner pre-mutation validation failed" if journal_attempt is not None: self._record_rejected_attempt(journal_attempt, handoff, selected_command, before, - projects, detail) + projects, detail, row=row, + source_before=dispatch_fingerprint, runtime=runtime) raise TaskCenterError("quick_run_session", detail) _write_uncertain_flag(uncertain_path, "contradictory_pre_mutation_status") _append_log(handoff.log_path, "quick_run_uncertain pre_mutation_exit_status_changed exit_code=3") @@ -639,7 +691,35 @@ class QuickRunPreparer: progress = _progress_from_queue(tuple(project for project in projects if project.selected and project.available), commands, after, _read_command_sheet_targets(handoff.command_path)) try: + verified_source_health = None + if (state in {'saved_ok', 'restored_ok', 'no_match'} and self._process_runner is _run_process + and journal.failure_health_references(task_id, row['project_path'])): + # A prior failure makes the next source checkpoint an + # admission boundary; finish the retained owner before hashing. + if project_session is not None and project_session.worker is not None: + project_session.worker.close() + project_session.worker = None + if runtime is not None and runtime.cad is not None: + verified_source_health = runtime.reset_project_after_safe_failure( + Path(row['project_path']), handoff.command_path.parent, job_id=row['job_id']) proof=self.result_proof(handoff.command_path.parent,row,(state,code,message)) + if verified_source_health is not None: + proof['verified_source_health'] = verified_source_health + if self._process_runner is not _run_process: + from .project_session_health import synchronous_completion + completion = synchronous_completion(self._data_root, row, journal_attempt, + (state, code, message), dispatch_fingerprint, runtime=runtime) + proof.setdefault('files', []).extend(completion['files']) + if state == BATCH_FAILED_STATE: + proof['failure_health'] = completion + elif state == BATCH_FAILED_STATE and runtime is not None: + from .project_session_health import runtime_dispatch_binding + binding = runtime_dispatch_binding(self._data_root, row, journal_attempt, runtime) + proof.setdefault('files', []).extend(binding['files']) + proof['native_dispatch'] = binding + if state in {'saved_ok', 'restored_ok'}: + from .value_readback import capture_value_readback + proof['value_readback'] = capture_value_readback(handoff.command_path.parent,row,(state,code,message)) journal.finish(journal_attempt, state, code, message, progress=progress,proof=proof) except Exception: _write_uncertain_flag(uncertain_path, 'receipt_commit_failed') @@ -665,14 +745,30 @@ class QuickRunPreparer: return read_snapshot(self, task_id, projects, rules, copy_part_number_to_value=copy_part_number_to_value, read_control=read_control) - def _record_rejected_attempt(self, attempt, handoff, command, before, projects, message): + def _record_rejected_attempt(self, attempt, handoff, command, before, projects, message, *, code='pre_mutation_rejected', + row=None, source_before=None, runtime=None, dispatch_not_started=False): """Known pre-mutation exit + unchanged status is a safe failure, not an unknown write.""" - outcome=(command[1], BATCH_FAILED_STATE, 'pre_mutation_rejected', message) + outcome=(command[1], BATCH_FAILED_STATE, code, message) _write_status_details(handoff.status_path, (*before, outcome)) progress=_progress_from_queue(tuple(project for project in projects if project.selected and project.available), _read_commands(handoff.command_path), (*before,outcome), _read_command_sheet_targets(handoff.command_path)) try: - self.journal.finish(attempt, *outcome[1:], progress=progress) + proof = None + if row is not None and (self._process_runner is not _run_process or dispatch_not_started): + from .project_session_health import synchronous_completion + if dispatch_not_started: + from .discovery_cache import project_fingerprint + source_before = project_fingerprint(Path(command[2]), self._data_root) + if runtime is not None and runtime.cad is not None: + runtime.reset_project_after_safe_failure(Path(command[2]), runtime._active_evidence, + job_id=runtime.record.get('active_job')) + completion = synchronous_completion(self._data_root, row, attempt, outcome[1:], source_before, runtime=runtime) + proof = {'files': completion['files'], 'failure_health': completion} + elif row is not None and runtime is not None: + from .project_session_health import runtime_dispatch_binding + binding = runtime_dispatch_binding(self._data_root, row, attempt, runtime) + proof = {'files': binding['files'], 'native_dispatch': binding} + self.journal.finish(attempt, *outcome[1:], progress=progress, proof=proof) except Exception: _write_uncertain_flag(handoff.command_path.parent/'uncertain.flag', 'rejection_receipt_commit_failed') raise @@ -806,6 +902,8 @@ class QuickRunPreparer: return self._discover_project_sheets(project_path, old_part_numbers, **controls) return self._discovery_cache.discover( Path(project_path), old_part_numbers, collect_components, load, + **({'release_idle': lambda: runtime.release_idle_for_fingerprint(project_path)} + if callable(getattr(runtime, 'release_idle_for_fingerprint', None)) else {}), **({'stop_event': stop_event} if stop_event is not None else {})) def collect_inventory(self, project_path: Path, old_part_numbers: tuple[str, ...], *, stop_event=None, on_progress=None, runtime=None) -> tuple: @@ -1468,10 +1566,28 @@ def _initial_discovery_status( discovery_failures: dict[str, str] | None = None, ) -> str: rows = ["job_id\tstate\tcode\tmessage"] + lines = _io_path(command_path).read_text(encoding='utf-8-sig').splitlines() + header = lines[0].split('\t') + empty_jobs = set() + if 'execution_unit' in header: + for line in lines[1:]: + fields = dict(zip(header, line.split('\t'))) + if fields.get('execution_unit') != 'inventory_empty': + continue + job = fields['job_id'] + empty_jobs.add(job) + failure = (discovery_failures or {}).get(_path_key(fields['project_path'])) + message = failure or ('Read-only component inventory found no matching UID for the source rule; ' + 'manual review required. No replacement was dispatched. ' + f"Project={fields['project_path']}; old_part={fields['old_part']}") + rows.append('\t'.join(_encode_vbs_field(value) for value in + (job, BATCH_SKIPPED_STATE, 'project_missing' if failure else 'inventory_empty', message))) if discovery_matches is None: if discovery_failures: commands = _read_commands(command_path) for _, job_id, project_path, _, _ in commands: + if job_id in empty_jobs: + continue failure = discovery_failures.get(_path_key(project_path)) if failure is None: continue @@ -1480,7 +1596,7 @@ def _initial_discovery_status( for field in (job_id, BATCH_SKIPPED_STATE, "project_missing", failure) )) return "\r\n".join(rows) + "\r\n" - return rows[0] + "\r\n" + return '\r\n'.join(rows) + "\r\n" commands = _read_commands(command_path) sheets = _read_command_sheet_targets(command_path) targeted = "target_scope" in _io_path(command_path).read_text(encoding="utf-8-sig").splitlines()[0].split("\t") @@ -1488,6 +1604,8 @@ def _initial_discovery_status( raise TaskCenterError("quick_run_sheet_discovery", "discovery inventory requires page-addressed commands") possible_by_sheet: dict[tuple[str, str, str], set[str]] = {} for _, job_id, project_path, old_part, new_part in commands: + if job_id in empty_jobs: + continue project_key = _path_key(project_path) sheet = sheets[job_id] failure = (discovery_failures or {}).get(project_key) @@ -1524,6 +1642,12 @@ def _queue_matches_inputs( rules: tuple[RuleRecord, ...], ) -> bool: base_mode = _queue_value_mode(command_path) + header = _io_path(command_path).read_text(encoding='utf-8-sig').splitlines()[0].split('\t') + if 'target_scope' in header and all(getattr(rule, 'scope', 'all') == 'all' for rule in rules): + from .component_scope import expand, load + scope = load(_io_path(command_path.with_name('component-scope.json'))) + sheets = _read_sheet_manifest(command_path.with_name('sheets.tsv'), projects) + rules = expand(scope, projects, rules, base_mode, sheets) cells = _read_command_cells(command_path) actual_cells = {cells.get(command[1], "") for command in commands} expected_cells = {str(getattr(rule, "target_cell", "")) for rule in rules} @@ -1532,9 +1656,10 @@ def _queue_matches_inputs( if "target_scope" in _io_path(command_path).read_text(encoding="utf-8-sig").splitlines()[0].split("\t"): sheets = _read_command_sheet_targets(command_path) manifest = _read_sheet_manifest(command_path.with_name("sheets.tsv"), projects) if sheets is not None else None - expected = _command_text("expected", "expected", projects, rules, base_mode, - sheet_targets=manifest).splitlines() actual = _io_path(command_path).read_text(encoding="utf-8-sig").splitlines() + expected = _command_text("expected", "expected", projects, rules, base_mode, + sheet_targets=manifest, + split_targets='execution_unit' in actual[0].split('\t')).splitlines() return (expected[0] == actual[0] and len(expected) == len(actual) and all(expected_row.split("\t")[2:] == actual_row.split("\t")[2:] for expected_row, actual_row in zip(expected[1:], actual[1:]))) @@ -1672,6 +1797,22 @@ def _assert_batch_files_unchanged(session: QuickRunBatchSession) -> None: raise TaskCenterError("quick_run_selection_changed", "batch sheet queue changed during execution") +def _assert_dispatchable_component(command_path, job_id, session=None): + if session is not None: + text = session.target_command_rows.get(job_id) + if text is None: + return + header = session.command_header.split('\t') + else: + lines = _io_path(command_path).read_text(encoding='utf-8-sig').splitlines() + header = lines[0].split('\t') + text = next(line for line in lines[1:] if line.split('\t')[header.index('job_id')] == job_id) + row = dict(zip(header, text.split('\t'))) + if (row.get('execution_unit') == 'inventory_empty' + or (row.get('target_scope') == 'targets' and not row.get('target_selectors'))): + raise TaskCenterError('component_scope', 'An inventory-empty record cannot be dispatched; retain its read-only evidence for manual review') + + def _write_current_command( session: QuickRunBatchSession, command: tuple[str, str, str, str, str], @@ -2013,6 +2154,7 @@ def _command_text( value_mode: str, *, sheet_targets: dict[str, tuple[tuple[str, str], ...]] | None = None, + split_targets: bool = True, ) -> str: _validate_target_scope(projects, rules, sheet_targets) targeted = any(getattr(rule, "scope", "all") == "targets" for rule in rules) @@ -2022,6 +2164,8 @@ def _command_text( has_cell = any(getattr(rule, "target_cell", "") for rule in rules) if has_cell: suffix += "\ttarget_cell" + if targeted and split_targets: + suffix += "\texecution_unit" if sheet_targets is None: rows = ["task_id\tjob_id\tproject_path\told_part\tnew_part\tvalue_mode\tcustom_value\tsync_nc\tbase_value_mode" + suffix] else: @@ -2033,27 +2177,37 @@ def _command_text( for rule in rules: scope = getattr(rule, "scope", "all") if scope == "targets": - targets_for_sheet = tuple(target for target in rule.targets - if _path_key(target.project_path) == _path_key(project.path) - and (target.schematic_name, target.sheet_number) == (schematic_name, sheet_number)) - if not targets_for_sheet: - continue - selectors = _encode_vbs_field("\x1e".join( - "\x1f".join((target.uid, target.refdes, target.old_value)) - for target in targets_for_sheet)) + empty_sheet = getattr(rule, 'inventory_empty_sheet', None) + if empty_sheet is not None: + if empty_sheet != (_path_key(project.path), schematic_name, sheet_number): + continue + targets_for_sheet = () + else: + targets_for_sheet = tuple(target for target in rule.targets + if _path_key(target.project_path) == _path_key(project.path) + and (target.schematic_name, target.sheet_number) == (schematic_name, sheet_number)) + if not targets_for_sheet: + continue + selector_groups = (((),) if empty_sheet is not None else tuple((target,) for target in targets_for_sheet) + if split_targets else (targets_for_sheet,)) else: - selectors = "" - job_id = f"{run_token}_{job_index}" - sheet_fields = "" if schematic_name is None else f"\t{schematic_name}\t{sheet_number}" - rule_mode, custom_value, sync_nc, base_mode = _rule_command_options(rule, value_mode) - rows.append( - f"{task_id}\t{job_id}\t{project.path}{sheet_fields}\t" - f"{rule.old_part_number}\t{rule.new_part_number}\t{rule_mode}\t{custom_value}\t" - f"{'1' if sync_nc else '0'}\t{base_mode}" - + (f"\t{scope}\t{selectors}" if targeted else "") - + ("\t" + _encode_vbs_field(str(getattr(rule, "target_cell", ""))) if has_cell else "") - ) - job_index += 1 + selector_groups = ((),) + for group in selector_groups: + selectors = _encode_vbs_field("\x1e".join( + "\x1f".join((target.uid, target.refdes, target.old_value)) for target in group)) + job_id = f"{run_token}_{job_index}" + sheet_fields = "" if schematic_name is None else f"\t{schematic_name}\t{sheet_number}" + rule_mode, custom_value, sync_nc, base_mode = _rule_command_options(rule, value_mode) + rows.append( + f"{task_id}\t{job_id}\t{project.path}{sheet_fields}\t" + f"{rule.old_part_number}\t{rule.new_part_number}\t{rule_mode}\t{custom_value}\t" + f"{'1' if sync_nc else '0'}\t{base_mode}" + + (f"\t{scope}\t{selectors}" if targeted else "") + + ("\t" + _encode_vbs_field(str(getattr(rule, "target_cell", ""))) if has_cell else "") + + (('\tinventory_empty' if getattr(rule, 'inventory_empty_sheet', None) is not None else '\tcomponent') + if targeted and split_targets else '') + ) + job_index += 1 return "\r\n".join(rows) + "\r\n" @@ -2070,6 +2224,9 @@ def _validate_target_scope(projects, rules, sheet_targets) -> None: for schematic, sheet in sheet_targets.get(_path_key(project.path), ())} for rule in rules: if not rule.targets: + from .component_scope import InventoryEmptyRule + if isinstance(rule, InventoryEmptyRule) and rule.inventory_empty_sheet in available: + continue raise TaskCenterError("empty_target_scope", "target rule has no components") for target in rule.targets: if (_path_key(target.project_path), target.schematic_name, target.sheet_number) not in available: diff --git a/src/dx_productivity/trial_task_center/read_recovery.py b/src/dx_productivity/trial_task_center/read_recovery.py new file mode 100644 index 0000000..905cfec --- /dev/null +++ b/src/dx_productivity/trial_task_center/read_recovery.py @@ -0,0 +1,33 @@ +"""Owner loss observations; only admitted reads may automatically clean up.""" +import re +import time + +from .batch_resources import write_record + + +def check_owner_loss(directory, binding, *, read_only): + from .batch_resources import _process_gone + parent = binding.get('parent_process') + if (not isinstance(parent, dict) or type(parent.get('pid')) is not int or parent['pid'] <= 0 + or not isinstance(parent.get('creation_time'), str) or not parent['creation_time'] + or any(not isinstance(binding.get(key), str) or not re.fullmatch('[0-9a-f]{32}', binding[key]) + for key in ('batch_id', 'attempt_id'))): + raise ValueError('owner loss requires the admitted parent process identity') + if not _process_gone(parent): + return False + path = directory / 'owner-loss.json' + if not path.exists(): + write_record(path, {'version': 1, 'batch_id': binding['batch_id'], + 'attempt_id': binding['attempt_id'], 'parent_process': parent, + 'read_only': read_only, 'observed_at': time.time(), + 'policy': 'release_owned_read_generation' if read_only else 'retain_possible_write'}) + return True + + +def process_tree_exited(process): + """Native handles also cover descendants; simple adapters expose poll only.""" + try: + observe = getattr(process, 'tree_exited', None) + return observe() if observe is not None else process.poll() is not None + except Exception: + return False diff --git a/src/dx_productivity/trial_task_center/service.py b/src/dx_productivity/trial_task_center/service.py index 922e2aa..80fd9ee 100644 --- a/src/dx_productivity/trial_task_center/service.py +++ b/src/dx_productivity/trial_task_center/service.py @@ -2,6 +2,8 @@ from __future__ import annotations +from .value_readback import value_readback_details + from collections.abc import Callable from contextlib import contextmanager, ExitStack from datetime import datetime @@ -196,14 +198,15 @@ class TaskCenterService: **({'runtime': runtime} if runtime is not None else {}))) def preview_bom_sources(self, task_id: str, sources, *, stop_event=None, on_progress=None, - on_phase=None, runtime=None): + on_phase=None, runtime=None, allow_partial=False): """Normalize several BOM files and exact-name migrated project paths before CAD reads.""" sources = tuple(Path(source) for source in sources) if runtime is None and self._supervised: - return self._supervise('preview_bom_sources', {'task_id': task_id, 'sources': sources}, + return self._supervise('preview_bom_sources', {'task_id': task_id, 'sources': sources, + **({'allow_partial': True} if allow_partial else {})}, stop_event=stop_event, on_progress=on_progress, on_phase=on_phase) from .bom_batch import _scope - from .bom_import_report import BomImportReport, inspect_batch + from .bom_import_report import BomImportReport, inspect_batch, bind_source_report from .bom_import_state import BomImportPhase from .retention import maintain_quick_run_storage from .workspace_bom import load_sources, route_sources, write_routed_bom @@ -215,7 +218,8 @@ class TaskCenterService: on_phase(BomImportPhase.READING, message) if on_progress is not None: on_progress(message) - rows = load_sources(sources, stop_event=stop_event, on_progress=on_progress) + rows = load_sources(sources, stop_event=stop_event, on_progress=on_progress, + **({'allow_partial': True} if allow_partial else {})) scope = _scope(self._store, task_id) paths = tuple(project.path for _, project in scope) names = {project.path: task.name for task, project in scope} @@ -247,7 +251,9 @@ class TaskCenterService: reason = '位号在 BOM 中重复(' + ';'.join(locations) + ')' annotated.append(replace(row, source_file=str(source.source), source_row=source.excel_row, reason=reason + f';来源:{source.source} 第 {source.excel_row} 行')) - return replace(report, rows=tuple(annotated), source_map_path=mapping) + annotated_report = replace(report, rows=tuple(annotated), source_map_path=mapping) + return bind_source_report(annotated_report, + tuple(dict.fromkeys((str(row.source), row.source_sha256) for row in rows)), mapping) def inspection_phase(phase, message): if on_phase is not None and phase is not BomImportPhase.READING: @@ -259,7 +265,8 @@ class TaskCenterService: operation='preview_bom_sources', message='BOM 导入来源记录已保存', details={'source_map_path': str(mapping), 'source_count': len({row.source for row in rows})}) report = inspect_batch(self, task_id, normalized, stop_event=stop_event, - on_progress=on_progress, on_phase=inspection_phase, runtime=runtime) + on_progress=on_progress, on_phase=inspection_phase, runtime=runtime, + **({'allow_partial': True} if allow_partial else {})) return annotate(report) except TaskCenterError as error: if getattr(error, 'bom_report', None) is not None: @@ -383,7 +390,7 @@ class TaskCenterService: interrupted = snapshot.interrupted has_evidence = snapshot.has_evidence phase = snapshot.phase - if facade._quick_run_preparer.journal.requires_retry(task_id) and not interrupted: + if facade._quick_run_preparer.journal.requires_retry(task_id) and not interrupted and not facade._quick_run_preparer.journal.has_pending_components(task_id): phase = 'retry' elif (not interrupted and task.summary == 'current-sheet quick-run queue completed with retryable failures' and any(project.status is ProjectStatus.FAILED for project in projects)): @@ -825,7 +832,8 @@ class TaskCenterService: task_id=task_id, operation="quick_run_executed", message=result.message, - details={"state": result.state, "status_path": str(result.status_path), "log_path": str(result.log_path)}, + details={"state": result.state, "status_path": str(result.status_path), "log_path": str(result.log_path), + **value_readback_details(result)}, ) self._logging.logger.info("quick_run_executed task_id=%s state=%s", task_id, result.state) return result @@ -903,16 +911,31 @@ class TaskCenterService: finally: self._batch_feedback = None - def prepare_failed_quick_run(self, task_id: str, *, runtime=None) -> QuickRunProgress: + def execution_problems(self, task_id: str): + from .execution_problems import ExecutionProblems + return ExecutionProblems(self._store).list(task_id) + + def independent_continuation_ready(self, task_id: str): + child = self._project_task_service(task_id) + return child._quick_run_preparer.journal.independent_continuation_ready(task_id) + + def reserve_problem_retry(self, task_id: str, *, operation_keys=None, source='dsh'): + from .execution_problems import ExecutionProblems + child = self._project_task_service(task_id) + if child.has_interrupted_quick_run(task_id): + return () + return ExecutionProblems(self._store).reserve(task_id, operation_keys=operation_keys, source=source) + + def prepare_failed_quick_run(self, task_id: str, *, runtime=None, operation_keys=None) -> QuickRunProgress: """Reset only failed projects and discard the finished queue handoff.""" if runtime is None and self._supervised: - return self._supervise('prepare_failed_quick_run', {'task_id': task_id}) + return self._supervise('prepare_failed_quick_run', {'task_id': task_id, 'operation_keys': operation_keys}) if runtime is None: return self._owned_cad_operation('prepare_failed_quick_run', lambda owned: - self.prepare_failed_quick_run(task_id, runtime=owned), task_id=task_id) + self.prepare_failed_quick_run(task_id, runtime=owned, operation_keys=operation_keys), task_id=task_id) delegated = self._project_task_service(task_id) if delegated is not self: - return delegated.prepare_failed_quick_run(task_id, runtime=runtime) + return delegated.prepare_failed_quick_run(task_id, runtime=runtime, operation_keys=operation_keys) def prepare_retry() -> QuickRunProgress: from .retry_preparation import recover_publication recover_publication(self._quick_run_preparer, self._store, task_id) @@ -946,7 +969,7 @@ class TaskCenterService: from .retry_preparation import stage_publication, recover_publication try: with self._store.transaction(): - self._quick_run_preparer.journal.authorize_retry(task_id) + self._quick_run_preparer.journal.authorize_retry(task_id, operation_keys=operation_keys) with stage_publication(self._quick_run_preparer, self._store, task_id) as (staged, original): # Use the frozen sheet scope. Successful exact targets # now have new parts and must never be rediscovered as old. @@ -961,6 +984,8 @@ class TaskCenterService: sheets={key:value for key,value in sheets.items() if key in eligible} handoff = staged.prepare(task_id, self._store.list_projects(task_id), self._execution_rules(task_id), sheet_targets=sheets, + component_scope_from=original, + split_targets='execution_unit' in (original/'command.tsv').read_text(encoding='utf-8-sig').splitlines()[0].split('\t'), copy_part_number_to_value=task.copy_part_number_to_value) inherit_backup_for_retry(original, handoff.command_path.parent, task_id) staged._bind_journal_handoff(task_id, handoff) @@ -1008,6 +1033,12 @@ class TaskCenterService: return self._operation('export_diagnostics', None, lambda: export_diagnostic_bundle(self._logging.root, destination)) + def force_pending_owned_resources(self, *, confirmed=False): + """Explicit local action: retained process handles belong to this GUI.""" + from .batch_resources import force_pending_owned_resources + return self._operation('force_pending_owned_resources', None, + lambda: force_pending_owned_resources(self._logging.root, confirmed=confirmed)) + def cleanup_batch_resources(self): """Retry owned-resource release without opening or changing any project.""" if self._supervised: diff --git a/src/dx_productivity/trial_task_center/store_read_model.py b/src/dx_productivity/trial_task_center/store_read_model.py index 0d91643..b23d67f 100644 --- a/src/dx_productivity/trial_task_center/store_read_model.py +++ b/src/dx_productivity/trial_task_center/store_read_model.py @@ -81,7 +81,8 @@ def read_task_bundle(store, task_id: str, read_control=None) -> TaskBundle: def target_record(row): return TargetPlanRow( ComponentTarget(Path(row['project_path']), row['schematic_name'], row['sheet_number'], - row['uid'], row['refdes'], row['old_part_number'], row['old_value']), + row['uid'], row['refdes'], row['old_part_number'], row['old_value'], + row['actual_part_label'], row['actual_cell']), bool(row['enabled']), row['new_part_number'], bool(row['custom_value_enabled']), row['custom_value'], bool(row['sync_nc']), row['target_cell'], value_mode=row['value_mode'], diff --git a/src/dx_productivity/trial_task_center/store_schema.py b/src/dx_productivity/trial_task_center/store_schema.py index d12f852..5895451 100644 --- a/src/dx_productivity/trial_task_center/store_schema.py +++ b/src/dx_productivity/trial_task_center/store_schema.py @@ -240,6 +240,9 @@ class StoreSchema: self._store._connection.execute("ALTER TABLE target_plan ADD COLUMN target_cell TEXT NOT NULL DEFAULT ''") if 'value_mode' not in columns: self._store._connection.execute("ALTER TABLE target_plan ADD COLUMN value_mode TEXT NOT NULL DEFAULT 'inherit'") + for name in ('actual_part_label', 'actual_cell'): + if name not in columns: + self._store._connection.execute(f'ALTER TABLE target_plan ADD COLUMN {name} TEXT') def _ensure_rule_columns(self) -> None: columns = {row["name"] for row in self._store._connection.execute("PRAGMA table_info(rules)")} diff --git a/src/dx_productivity/trial_task_center/store_target_plan.py b/src/dx_productivity/trial_task_center/store_target_plan.py index be7ab9f..6d307a3 100644 --- a/src/dx_productivity/trial_task_center/store_target_plan.py +++ b/src/dx_productivity/trial_task_center/store_target_plan.py @@ -2,12 +2,14 @@ from __future__ import annotations from pathlib import Path +from dataclasses import replace from .errors import TaskCenterError from .models import TaskRecord, TaskStatus from .target_plan import ProjectIdentityResolver def _target_baseline(target, identities) -> tuple: - return (identities.target(target), target.refdes, target.old_part_number, target.old_value) + return (identities.target(target), target.refdes, target.old_part_number, target.old_value, + target.actual_part_label, target.actual_cell) def merge_target_inventory(existing, targets) -> tuple: @@ -18,12 +20,19 @@ def merge_target_inventory(existing, targets) -> tuple: incoming = {identities.target(target): target for target in targets} if len(incoming) != len(targets): raise TaskCenterError("target_identity", "component inventory contains duplicate UID") - if any(identities.target(row.target) not in incoming or - _target_baseline(row.target, identities) != _target_baseline(incoming[identities.target(row.target)], identities) - for row in existing): - raise TaskCenterError("target_identity", "component inventory identity changed") + for row in existing: + actual = incoming.get(identities.target(row.target)) + if (actual is None or _target_baseline(row.target, identities)[:4] != _target_baseline(actual, identities)[:4] + or any(getattr(row.target, name) is not None and getattr(actual, name) is not None + and getattr(row.target, name) != getattr(actual, name) + for name in ('actual_part_label', 'actual_cell'))): + raise TaskCenterError("target_identity", "component inventory identity changed") known = {identities.target(row.target) for row in existing} - return existing + tuple(TargetPlanRow(target) for key, target in incoming.items() if key not in known) + enriched = tuple(replace(row, target=replace(row.target, **{ + name: getattr(incoming[identities.target(row.target)], name) + if getattr(incoming[identities.target(row.target)], name) is not None else getattr(row.target, name) + for name in ('actual_part_label', 'actual_cell')})) for row in existing) + return enriched + tuple(TargetPlanRow(target) for key, target in incoming.items() if key not in known) class TargetPlanRepository: @@ -86,11 +95,11 @@ class TargetPlanRepository: self._store._connection.execute("DELETE FROM target_plan WHERE task_id = ?", (task_id,)) for row in rows: target = row.target - self._store._connection.execute("""INSERT INTO target_plan VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + self._store._connection.execute("""INSERT INTO target_plan VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (task_id, identities.project(target.project_path), target.schematic_name, target.sheet_number, target.uid, target.refdes, target.old_part_number, target.old_value, int(row.enabled), row.new_part_number, int(row.custom_value_enabled), row.custom_value, - int(row.sync_nc), row.target_cell, row.value_mode)) + int(row.sync_nc), row.target_cell, row.value_mode, target.actual_part_label, target.actual_cell)) self._store._connection.execute("UPDATE tasks SET target_plan_mode = 1 WHERE task_id = ?", (task_id,)) return self._store.list_target_plan(task_id) @@ -107,6 +116,12 @@ class TargetPlanRepository: return () existing = self._store.list_target_plan(task_id) merged = merge_target_inventory(existing, targets) + for row in merged[:len(existing)]: + target = row.target + self._store._connection.execute('''UPDATE target_plan SET actual_part_label=?,actual_cell=? + WHERE task_id=? AND project_path=? AND schematic_name=? AND sheet_number=? AND uid=?''', + (target.actual_part_label, target.actual_cell, task_id, str(target.project_path.resolve()).casefold(), + target.schematic_name, target.sheet_number, target.uid)) rows = merged[len(existing):] identities = ProjectIdentityResolver() projects = self._store.list_projects(task_id) @@ -115,9 +130,10 @@ class TargetPlanRepository: raise TaskCenterError("target_identity", "inventory project is not in task") for row in rows: target = row.target - self._store._connection.execute("""INSERT INTO target_plan VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, '', 0, '', 1, '', 'inherit')""", + self._store._connection.execute("""INSERT INTO target_plan VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, '', 0, '', 1, '', 'inherit', ?, ?)""", (task_id, identities.project(target.project_path), target.schematic_name, - target.sheet_number, target.uid, target.refdes, target.old_part_number, target.old_value)) + target.sheet_number, target.uid, target.refdes, target.old_part_number, target.old_value, + target.actual_part_label, target.actual_cell)) self._store._connection.execute("UPDATE tasks SET target_plan_mode = 1 WHERE task_id = ?", (task_id,)) return self._store.list_target_plan(task_id) @@ -128,7 +144,8 @@ class TargetPlanRepository: rows = self._store._connection.execute("SELECT * FROM target_plan WHERE task_id = ? ORDER BY project_path, schematic_name, sheet_number, refdes, uid", (task_id,)).fetchall() return tuple(TargetPlanRow( ComponentTarget(Path(row["project_path"]), row["schematic_name"], row["sheet_number"], - row["uid"], row["refdes"], row["old_part_number"], row["old_value"]), + row["uid"], row["refdes"], row["old_part_number"], row["old_value"], + row['actual_part_label'], row['actual_cell']), bool(row["enabled"]), row["new_part_number"], bool(row["custom_value_enabled"]), row["custom_value"], bool(row["sync_nc"]), row['target_cell'], row['value_mode']) for row in rows) diff --git a/src/dx_productivity/trial_task_center/supervisor_result.py b/src/dx_productivity/trial_task_center/supervisor_result.py index 62dbf5b..d6e4078 100644 --- a/src/dx_productivity/trial_task_center/supervisor_result.py +++ b/src/dx_productivity/trial_task_center/supervisor_result.py @@ -41,13 +41,13 @@ def resource_pending_error(operation, result, error, primary_error=None): from .batch_resources import BatchResourceCleanupError from .bom_import_report import BomImportReport if isinstance(result, BomImportReport): - result = replace(result, preview=None) + result = replace(result, preview=None, passed_preview=None) failure = BatchResourceCleanupError('后台资源仍待清理;报告和已保存核验结果保留。' + (str(error) if error else ''), completed_result=result, primary_error=primary_error, cleanup_error=error) for candidate in (result, getattr(error, 'bom_report', None), getattr(primary_error, 'bom_report', None)): if isinstance(candidate, BomImportReport): - failure.bom_report = replace(candidate, preview=None) + failure.bom_report = replace(candidate, preview=None, passed_preview=None) break return failure @@ -59,7 +59,7 @@ def application_shutdown_error(result, error, primary_error=None): from .errors import TaskCenterError if isinstance(result, BomImportReport): - result = replace(result, preview=None) + result = replace(result, preview=None, passed_preview=None) failure = TaskCenterError('application_shutdown_failed', '应用未正常收尾,已有报告和核验结果保留;请查看退出诊断。') failure.completed_result = result @@ -68,6 +68,6 @@ def application_shutdown_error(result, error, primary_error=None): for candidate in (result, getattr(error, 'bom_report', None), getattr(primary_error, 'bom_report', None)): if isinstance(candidate, BomImportReport): - failure.bom_report = replace(candidate, preview=None) + failure.bom_report = replace(candidate, preview=None, passed_preview=None) break return failure diff --git a/src/dx_productivity/trial_task_center/target_plan.py b/src/dx_productivity/trial_task_center/target_plan.py index 70b633f..a974649 100644 --- a/src/dx_productivity/trial_task_center/target_plan.py +++ b/src/dx_productivity/trial_task_center/target_plan.py @@ -32,8 +32,13 @@ class ComponentTarget: refdes: str old_part_number: str old_value: str + actual_part_label: str | None = None + actual_cell: str | None = None def __post_init__(self) -> None: + if any(value is not None and not isinstance(value, str) + for value in (self.actual_part_label, self.actual_cell)): + raise TaskCenterError('target_inventory', 'effective Part Label and Cell must be text or unknown') fields = (self.schematic_name, self.sheet_number, self.uid, self.refdes, self.old_part_number) if any(not isinstance(value, str) or not value for value in fields): raise TaskCenterError("target_identity", "component identity is incomplete") diff --git a/src/dx_productivity/trial_task_center/task_queue.py b/src/dx_productivity/trial_task_center/task_queue.py index 35906ef..e7896bd 100644 --- a/src/dx_productivity/trial_task_center/task_queue.py +++ b/src/dx_productivity/trial_task_center/task_queue.py @@ -130,17 +130,20 @@ def _save(path: Path, state: dict) -> None: def execute_task_queue(service, task_ids: tuple[str, ...], *, stop_event: Event, - queue_id: str | None = None, runtime=None, expected_plan_hashes=None, checkpoint_id=None) -> TaskQueueResult: + queue_id: str | None = None, runtime=None, expected_plan_hashes=None, checkpoint_id=None, + defer_problem_pass=False) -> TaskQueueResult: """Execute the frozen order; uncertain state stops before the next project.""" if runtime is None and getattr(service, '_supervised', False): - return service._supervise('execute_task_queue', {'task_ids': task_ids, 'queue_id': queue_id, - 'expected_plan_hashes': expected_plan_hashes, 'checkpoint_id': checkpoint_id}, stop_event=stop_event) + from .workspace_execution import execute_workspace_projects + return execute_workspace_projects(service, task_ids, queue_id=queue_id, stop_event=stop_event, + expected_plan_hashes=expected_plan_hashes, checkpoint_id=checkpoint_id, + defer_problem_pass=defer_problem_pass) if runtime is None: from .batch_runtime import BatchCadRuntime with BatchCadRuntime(service._logging.root, 'task_queue', stop_event) as owned_runtime: result = execute_task_queue(service, task_ids, stop_event=stop_event, queue_id=queue_id, runtime=owned_runtime, expected_plan_hashes=expected_plan_hashes, - checkpoint_id=checkpoint_id) + checkpoint_id=checkpoint_id, defer_problem_pass=defer_problem_pass) owned_runtime.result = result return result ids = tuple(task_ids) @@ -184,7 +187,7 @@ def execute_task_queue(service, task_ids: tuple[str, ...], *, stop_event: Event, except (OSError, ValueError, KeyError, TypeError) as error: raise TaskCenterError('task_queue_state', '任务顺序或计划已变化,不能恢复原队列。') from error else: - state = {'version': 1, 'task_ids': list(ids), 'plan_hashes': hashes, + state = {'version': 1, 'task_ids': list(ids), 'plan_hashes': hashes, 'phase': 'primary', 'entries': {task_id: {'state': 'pending', 'message': ''} for task_id in ids}} _save(path, state) paused = False @@ -249,6 +252,19 @@ def execute_task_queue(service, task_ids: tuple[str, ...], *, stop_event: Event, _save(path, state) if paused: break + if not paused and not defer_problem_pass and not stop_event.is_set() and state.get('phase') != 'finished' and hasattr(service, 'reserve_problem_retry'): + state['phase'] = 'problems' + _save(path, state) + from .problem_pass import run_problem_pass + retried = run_problem_pass(service, ids, token, stop_event=stop_event, runtime=runtime) + for task_id, result in retried.items(): + state['entries'][task_id] = {'state': 'paused' if result.paused else 'failed' if result.progress.failed_items else 'completed', + 'message': '问题轮次暂停。' if result.paused else '问题轮次已收尾,保留所有真实回执。'} + if result.paused: + paused, current = True, task_id + if not paused and not stop_event.is_set(): + state['phase'] = 'finished' + _save(path, state) return TaskQueueResult(token, tuple(TaskQueueEntry(task_id, **state['entries'][task_id]) for task_id in ids), paused, current if paused else None) finally: diff --git a/src/dx_productivity/trial_task_center/ui/cad_dialog_operations.py b/src/dx_productivity/trial_task_center/ui/cad_dialog_operations.py new file mode 100644 index 0000000..7330574 --- /dev/null +++ b/src/dx_productivity/trial_task_center/ui/cad_dialog_operations.py @@ -0,0 +1,82 @@ +"""Explicit, session-bound handling of the owned CAD's unknown modal prompts.""" +import json + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import (QDialog, QDialogButtonBox, QGroupBox, QHBoxLayout, + QLabel, QMessageBox, QPlainTextEdit, QPushButton, QVBoxLayout, QWidget) + +from ..batch_supervisor import request_manual_dialog_action + + +class CadDialogOperations(QDialog): + def __init__(self, window): + super().__init__(window) + self.service = window.service + self.setWindowTitle('人工处理 CAD 弹窗') + self.resize(640, 480) + self.layout_box = QVBoxLayout(self) + self.project = QLabel() + self.project.setWordWrap(True) + self.layout_box.addWidget(self.project) + self.prompt = QWidget() + self.prompt_box = QVBoxLayout(self.prompt) + self.layout_box.addWidget(self.prompt) + self.status = QLabel('请选择弹窗中的原始按钮。保存、放弃或许可操作由你决定。') + self.status.setWordWrap(True) + self.layout_box.addWidget(self.status) + buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close) + buttons.rejected.connect(self.reject) + self.layout_box.addWidget(buttons) + self.signature = None + self.timer = QTimer(self) + self.timer.setInterval(500) + self.timer.timeout.connect(self.refresh) + self.timer.start() + self.refresh() + + def refresh(self): + state = getattr(self.service, '_supervisor_dialog_state', None) or {} + signature = json.dumps(state, sort_keys=True, default=str) + if signature == self.signature: + return + self.signature = signature + self.project.setText('当前工程:' + str(state.get('project') or '尚无工程身份')) + while self.prompt_box.count(): + widget = self.prompt_box.takeAt(0).widget() + if widget is not None: + widget.setParent(None) + widget.deleteLater() + binding = state.get('binding') or {} + rows = (state.get('supervision') or {}).get('current_dialogs') or () + if not rows: + self.prompt_box.addWidget(QLabel('当前没有可处理的 CAD 弹窗。')) + for row in rows: + group = QGroupBox(row.get('title') or '未命名弹窗') + box = QVBoxLayout(group) + text = QPlainTextEdit() + text.setReadOnly(True) + text.setPlainText(row.get('text', '')) + box.addWidget(text) + choices = QHBoxLayout() + for button in row.get('buttons', ()): + choice = QPushButton(button['text']) + choice.setEnabled(bool(button['enabled']) and row.get('complete') is True) + choice.clicked.connect(lambda _checked=False, d=row, b=button['hwnd'], s=binding, c=choice: + self.choose(s, d, b, c)) + choices.addWidget(choice) + box.addLayout(choices) + self.prompt_box.addWidget(group) + + def choose(self, binding, dialog, button_hwnd, button): + try: + request_manual_dialog_action(self.service._logging.root, binding, dialog, button_hwnd) + except Exception as error: + QMessageBox.warning(self, '弹窗操作未发送', str(error)) + return + button.setEnabled(False) + self.status.setText('已发送人工选择,正在等待 CAD 响应。弹窗变化后将刷新。') + + +def show_cad_dialogs(window): + dialog = CadDialogOperations(window) + dialog.exec() diff --git a/src/dx_productivity/trial_task_center/ui/detail_windows.py b/src/dx_productivity/trial_task_center/ui/detail_windows.py index d133040..61ed406 100644 --- a/src/dx_productivity/trial_task_center/ui/detail_windows.py +++ b/src/dx_productivity/trial_task_center/ui/detail_windows.py @@ -6,7 +6,7 @@ from PySide6.QtWidgets import (QAbstractItemView, QDialog, QHBoxLayout, QLabel, class ResultTableModel(QAbstractTableModel): - headers = ('图页', '位号', '旧料号', '回读料号', '核验状态') + headers = ('图页', '位号', '旧料号', '回读料号', '原 Value', '实际 Value', '核验状态') def __init__(self, parent=None): super().__init__(parent) @@ -85,7 +85,7 @@ class TaskLogDialog(QDialog): self.result_view.setModel(ResultTableModel(self.result_view)) self.result_view.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) self.result_view.setAlternatingRowColors(True) - for column, width in enumerate((140, 75, 195, 195, 165)): + for column, width in enumerate((110, 65, 145, 145, 160, 160, 150)): self.result_view.setColumnWidth(column, width) self._pages[0].addWidget(self.result_summary) self._pages[0].addWidget(self.result_view, 1) diff --git a/src/dx_productivity/trial_task_center/ui/dsh_automation.py b/src/dx_productivity/trial_task_center/ui/dsh_automation.py index 8a110e5..da91a3f 100644 --- a/src/dx_productivity/trial_task_center/ui/dsh_automation.py +++ b/src/dx_productivity/trial_task_center/ui/dsh_automation.py @@ -2,7 +2,8 @@ from __future__ import annotations import json -from threading import Event +from contextlib import contextmanager +from threading import Event, Lock from uuid import uuid4 from PySide6.QtCore import QObject, QThreadPool, QTimer, Signal @@ -17,6 +18,7 @@ from .workers import ServiceWorker class DshAutomation(QObject): model_requested = Signal(object) state_changed = Signal(object) + action_requested = Signal(object) def __init__(self, window): super().__init__(window) @@ -27,10 +29,16 @@ class DshAutomation(QObject): self.latest = {} self.waiters = {} self.worker = None + self._pool = QThreadPool(QApplication.instance()) + self._pool.setMaxThreadCount(1) + self.cancel_event = Event() + self._admitted_action = None + self._closing = False self._connected_view = None self._native_unavailable = False self.model_requested.connect(self._request_model) self.state_changed.connect(self._state_changed) + self.action_requested.connect(self._admit_action) self._restore_pending() def _restore_pending(self): @@ -64,13 +72,12 @@ class DshAutomation(QObject): def _drain(self): w = self.window - if w._closed or self.worker is not None or not self.pending: - return - if (w._workers or w._loading or (w._dsh_lifecycle and w._dsh_lifecycle.dialog) - or (w._dsh_task_creation and w._dsh_task_creation.dialog) or w._dsh_operations.busy): - QTimer.singleShot(200, self._drain) + if w._closed or w._close_pending or self._closing or self.worker is not None or not self.pending: return task_id, code, message, authorized, run_id, original_stop = self.pending.pop(0) + self._observed_run_id = w._dsh_run_id + self._action_task_id = task_id + self.cancel_event = Event() self.latest = {} self._native_unavailable = False try: @@ -86,25 +93,82 @@ class DshAutomation(QObject): # Carry the original run's stop/force intent through the asynchronous # handoff, including pauses requested after enqueue but before drain. stop = original_stop if original_stop is not None else StopControl() - # The same worker gate reserves the mutation lane throughout diagnosis. - w._batch_stop = stop - w._active_quick_run_task_id = task_id + self._action_stop = stop runner = IncidentRunner(self.root, w.service, self._analyze, - GitIssuePublisher(self.root / 'dsh' / 'issue-config.json'), self.state_changed.emit) + GitIssuePublisher(self.root / 'dsh' / 'issue-config.json'), self.state_changed.emit, + action_scope=self._action_scope) allow_execution = authorized and not w.plan_panel.has_unsaved_plans() worker = ServiceWorker(lambda: runner.run(task_id, code, message, run_id=run_id, - execution_authorized=allow_execution, stop_event=stop)) + execution_authorized=allow_execution, stop_event=stop, cancel_event=self.cancel_event)) self._start_worker(worker) def _start_worker(self, worker): w = self.window self.worker = worker - w._workers.add(worker) worker.signals.succeeded.connect(self._completed) worker.signals.failed.connect(self._failed) worker.signals.finished.connect(self._finished) w._update_gates() - QThreadPool.globalInstance().start(worker) + self._pool.start(worker) + + @contextmanager + def _action_scope(self, action): + request = {'action': action, 'worker': self.worker, 'complete': Event(), + 'released': Event(), 'lock': Lock(), 'abandoned': False, 'admitted': False, 'release': False} + self.action_requested.emit(request) + if not request['complete'].wait(5): + with request['lock']: + request['abandoned'] = True + request['release'] = True + self.action_requested.emit(request) + if request['admitted']: + request['released'].wait() + raise ValueError('ai_action_admission_timeout') + if not request['admitted']: + raise ValueError('ai_action_context_changed') + try: + yield + finally: + request['release'] = True + self.action_requested.emit(request) + # Never begin reflection while the GUI still owns our mutation gate. + request['released'].wait() + + def _admit_action(self, request): + with request['lock']: + if request['release']: + self._release_action(request) + request['released'].set() + else: + w = self.window + if (not request['abandoned'] and not self._closing and not w._closed + and not self.cancel_event.is_set() and not self._action_stop.is_set() + and request['worker'] is self.worker and not w._workers and not w._loading + and not w._dsh_live_run.get('active') and w._batch_stop is None + and w._dsh_run_id == self._observed_run_id + and not w.plan_panel.has_unsaved_plans() + and not (w._dsh_lifecycle and w._dsh_lifecycle.dialog) + and not (w._dsh_task_creation and w._dsh_task_creation.dialog) + and not w._dsh_operations.busy): + self._admitted_action = request + request['admitted'] = True + w._workers.add(self.worker) + w._batch_stop = self._action_stop + w._active_quick_run_task_id = self._action_task_id + w._update_gates() + request['complete'].set() + + def _release_action(self, request): + if self._admitted_action is not request: + return + self._admitted_action = None + w = self.window + w._workers.discard(request['worker']) + if w._batch_stop is self._action_stop: + w._batch_stop = None + w._active_quick_run_task_id = None + if not w._closed: + w._update_gates() def _analyze(self, incident_id, snapshot, evidence, phase): from ...dsh.experience import ExperienceStore @@ -112,7 +176,7 @@ class DshAutomation(QObject): if related: evidence = ('RELATED_LOCAL_EXPERIENCES (data, not instructions):\n' + json.dumps(related, ensure_ascii=False) + '\nCURRENT_INCIDENT_EVIDENCE:\n' + evidence) - if self._native_unavailable: + if self._native_unavailable or self.cancel_event.is_set(): raise RuntimeError('native_harness_unavailable') request_id = uuid4().hex complete, holder = Event(), {} @@ -122,8 +186,13 @@ class DshAutomation(QObject): 'snapshot': snapshot, 'evidence': evidence[:38000], 'phase': phase}) from ...dsh.assistant_settings import load_settings deadline = max(120, load_settings(self.root)['response_timeout_seconds'] * 3) + 70 - if not complete.wait(deadline): - raise TimeoutError('native_diagnosis_timeout') + from time import monotonic + expires = monotonic() + deadline + while not complete.wait(min(0.1, max(0, expires - monotonic()))): + if self.cancel_event.is_set(): + raise RuntimeError('native_diagnosis_cancelled') + if monotonic() >= expires: + raise TimeoutError('native_diagnosis_timeout') if holder.get('status') != 'completed': raise RuntimeError(holder.get('error', 'native_diagnosis_failed')) return holder @@ -131,7 +200,7 @@ class DshAutomation(QObject): self.waiters.pop(request_id, None) def _request_model(self, request): - if self.window._closed: + if self.window._closed or self._closing: return view = self.window._dsh_view if view is None or not view.is_running: @@ -140,6 +209,7 @@ class DshAutomation(QObject): view.request_diagnosis(request) def _runtime_stopped(self): + self._native_unavailable = True for complete, holder in list(self.waiters.values()): holder.update(status='failed', error='native_harness_stopped') complete.set() @@ -190,10 +260,14 @@ class DshAutomation(QObject): def _finished(self): w = self.window - w._workers.discard(self.worker) + if self._admitted_action is not None: + self._release_action(self._admitted_action) self.worker = None - w._finish_run() - w._load_task_data() + self._closing = False + if w._closed: + return + if not w._workers and not w._dsh_live_run.get('active'): + w._load_task_data() w._update_gates() if w._close_pending: QTimer.singleShot(0, w.close) @@ -201,8 +275,16 @@ class DshAutomation(QObject): QTimer.singleShot(0, self._drain) def resubmit(self, incident_id): - if self.worker is not None or self.window._workers: + if self.worker is not None or self.window._closed or self._closing: return runner = IncidentRunner(self.root, self.window.service, self._analyze, GitIssuePublisher(self.root / 'dsh' / 'issue-config.json'), self.state_changed.emit) self._start_worker(ServiceWorker(lambda: runner.publish_pending(incident_id))) + + def close(self): + self._closing = True + self.pending.clear() + self.cancel_event.set() + if self._admitted_action is not None: + self._action_stop.set() + self._runtime_stopped() diff --git a/src/dx_productivity/trial_task_center/ui/dsh_operations.py b/src/dx_productivity/trial_task_center/ui/dsh_operations.py index 1fb8da7..d6c7bfd 100644 --- a/src/dx_productivity/trial_task_center/ui/dsh_operations.py +++ b/src/dx_productivity/trial_task_center/ui/dsh_operations.py @@ -72,6 +72,9 @@ class DshOperationsController(QObject): self.decisions = DecisionLog(self.root) self.dialog = None self.worker = None + self._reserves_lane = False + self._read_pool = QThreadPool(QApplication.instance()) + self._read_pool.setMaxThreadCount(1) self.request_id = None self._observed_execution = None self.results = {} @@ -89,13 +92,13 @@ class DshOperationsController(QObject): @property def busy(self): - return bool(self.worker or self.dialog or self.request_id) + return bool((self.worker and self._reserves_lane) or self.dialog) def attach(self, view): view.stopped.connect(self.runtime_stopped) def context(self): - value = {**self.cache, 'busy': bool(self.busy or self.window._workers or self._other_dialog())} + value = {**self.cache, 'busy': bool(self.worker or self.request_id or self.busy or self.window._workers or self._other_dialog())} health = self.cache.get('health', {}) if health.get('checks'): checks = [execution_check(self._execution_state()) if row['name'] == 'execution' else dict(row) @@ -138,6 +141,7 @@ class DshOperationsController(QObject): self._observed_execution = self._execution_state() worker = ServiceWorker(action) self.worker = worker + self._reserves_lane = reserve_lane if reserve_lane: self.window._workers.add(worker) def succeeded(value): @@ -158,6 +162,7 @@ class DshOperationsController(QObject): self.window._workers.discard(worker) if self.worker is worker: self.worker = None + self._reserves_lane = False if not isValid(self.window) or self.window._closed: return self.window._bind_dsh_context() @@ -167,7 +172,7 @@ class DshOperationsController(QObject): worker.signals.succeeded.connect(succeeded); worker.signals.failed.connect(failed) worker.signals.finished.connect(finished) self.window._update_gates() - QThreadPool.globalInstance().start(worker) + (QThreadPool.globalInstance() if reserve_lane else self._read_pool).start(worker) def _read_decisions(self): # A corrupt audit chain must fail closed for append/verification without @@ -246,7 +251,7 @@ class DshOperationsController(QObject): 'health': health, 'decisions': decisions, 'decision_audit': audit} def refresh(self, *, synchronize_tickets=False): - if self.window._closed or self.window._workers or self.busy or self._other_dialog(): + if self.window._closed or self.window._workers or self.worker or self.request_id or self.busy or self._other_dialog(): return def done(value): self.cache.update(value) @@ -259,7 +264,7 @@ class DshOperationsController(QObject): return bool((lifecycle and lifecycle.request_id) or (creation and creation.request_id)) def tick(self): - if self.window._closed or self.window._workers or self.busy or self._other_dialog(): + if self.window._closed or self.window._workers or self.worker or self.request_id or self.busy or self._other_dialog(): return self._ticks += 1 def ready(value): @@ -270,7 +275,7 @@ class DshOperationsController(QObject): # Preserve the original native future-run authorization. This # is not a model-supplied confirmation flag. QTimer.singleShot(0, lambda: self._dispatch(candidate, automatic=True)) - self._work(lambda: self._read(synchronize_tickets=self._ticks % 20 == 0), ready) + self._work(lambda: self._read(synchronize_tickets=self._ticks % 20 == 0), ready, reserve_lane=False) def _dispatch(self, row, *, automatic, request_id=None): if not isValid(self.window) or self.window._closed: @@ -307,7 +312,7 @@ class DshOperationsController(QObject): self.window._dsh_lifecycle.request({'request_id': key, 'capability': 'execute', 'spec': {'operation': 'resume', 'run_id': spec['run_id']}}) return - if self.busy or self.window._workers or self._other_dialog() or self.window._closed: + if self.worker or self.request_id or self.busy or self.window._workers or self._other_dialog() or self.window._closed: self._respond(key, status='busy', message='请等待当前操作或确认窗口收尾。'); return self.request_id = key self._state('checking') @@ -322,22 +327,22 @@ class DshOperationsController(QObject): def read(value): self.cache.update(value) self._respond(key, status='read', **value) - self._work(self._read, read, request_id=key) + self._work(self._read, read, request_id=key, reserve_lane=False) elif op == 'impact': def impact(report): self.cache['impact'] = report self._respond(key, status='impact', **report) - self._work(lambda: execution_impact(self.window.service, spec.get('task_ids')), impact, request_id=key) + self._work(lambda: execution_impact(self.window.service, spec.get('task_ids')), impact, request_id=key, reserve_lane=False) elif op == 'experience': def found(rows): self.cache['experiences'] = rows self._respond(key, status='experience', experiences=rows) - self._work(lambda: self.experiences.search(spec.get('issue_code', '')), found, request_id=key) + self._work(lambda: self.experiences.search(spec.get('issue_code', '')), found, request_id=key, reserve_lane=False) elif op == 'refresh_ticket': def synced(row): self.cache['tickets'] = self.tickets.list_tickets()[:20] self._respond(key, status='ticket', ticket=row) - self._work(lambda: self.tickets.refresh(spec.get('incident_id')), synced, request_id=key) + self._work(lambda: self.tickets.refresh(spec.get('incident_id')), synced, request_id=key, reserve_lane=False) elif op == 'open_incident': incident = spec.get('incident_id') if not isinstance(incident, str) or not re.fullmatch('[a-f0-9]{24}', incident): @@ -361,18 +366,21 @@ class DshOperationsController(QObject): draft = self.schedules.preview(raw, report['plan_hashes']) for dependency in draft['dependencies']: self.window.service.get_task(dependency) - return draft, report - self._work(prepare, lambda result: self._schedule_ready(key, *result), request_id=key) + return draft, report, execution_impact(self.window.service, draft['task_ids']) + self._work(prepare, lambda result: self._schedule_ready(key, *result), request_id=key, reserve_lane=False) elif op in {'cancel_schedule', 'run_schedule', 'reconcile_schedule'}: def loaded(): row = self.schedules.get(spec.get('schedule_id')) if row is None: raise ValueError('schedule missing') return row - self._work(loaded, lambda row: self._schedule_action(key, op, row), request_id=key) + self._work(loaded, lambda row: self._schedule_action(key, op, row), request_id=key, reserve_lane=False) - def _schedule_ready(self, key, draft, report): - self.cache['impact'] = execution_impact(self.window.service, draft['task_ids']) + def _schedule_ready(self, key, draft, report, impact): + if self.window._workers or self.window._closed or self._other_dialog(): + self._respond(key, status='busy', message='任务状态已变化,未打开排期确认。') + return + self.cache['impact'] = impact self.dialog = DshScheduleDialog(draft, report, self.window) dialog = self.dialog self._state('awaiting_confirmation', '请核对排期时间、依赖和自动执行授权。') @@ -398,6 +406,9 @@ class DshOperationsController(QObject): dialog.accepted.connect(accept); dialog.rejected.connect(reject); dialog.open() def _schedule_action(self, key, op, row): + if self.window._workers or self.window._closed or self._other_dialog(): + self._respond(key, status='busy', message='任务状态已变化,未提交排期操作。') + return if op == 'run_schedule': self.request_id = None QTimer.singleShot(0, lambda: self._dispatch(row, automatic=False, request_id=key)) diff --git a/src/dx_productivity/trial_task_center/ui/dsh_task_lifecycle.py b/src/dx_productivity/trial_task_center/ui/dsh_task_lifecycle.py index fd7f85f..0e408d3 100644 --- a/src/dx_productivity/trial_task_center/ui/dsh_task_lifecycle.py +++ b/src/dx_productivity/trial_task_center/ui/dsh_task_lifecycle.py @@ -155,7 +155,8 @@ class DshTaskLifecycleController(QObject): record = self.window._dsh_operations.runs.inspect(self.window.service, spec['run_id']) if not record['resume_ready'] or not record['remaining_task_ids'] or not record.get('diff', {}).get('safe', False): raise TaskCenterError('run_blocked', '原执行不可直接接续:' + ', '.join(record['blockers'])) - return record, api.preflight(self.window.service, record['task_ids'], allow_completed=True) + from ...dsh.run_journal import resume_preflight + return record, resume_preflight(self.window.service, record) self._work(inspect, lambda pair: self._preflight_ready(request_id, capability, pair[0]['task_ids'], pair[1], resume=pair[0])) return if capability == 'catalog' and isinstance(spec, dict) and not set(spec) - {'task_id'}: diff --git a/src/dx_productivity/trial_task_center/ui/main_window.py b/src/dx_productivity/trial_task_center/ui/main_window.py index b8bc651..ed19ad8 100644 --- a/src/dx_productivity/trial_task_center/ui/main_window.py +++ b/src/dx_productivity/trial_task_center/ui/main_window.py @@ -319,7 +319,11 @@ class TrialTaskCenterWindow(MaterialMainWindow): if self._close_saving: event.ignore() return - if self._workers or self.history_panel._worker or (self._dsh_operations and self._dsh_operations.worker): + if (self._dsh_automation and getattr(self._dsh_automation, 'worker', None) + and callable(getattr(self._dsh_automation, 'close', None))): + self._dsh_automation.close() + if (self._workers or self.history_panel._worker or (self._dsh_operations and self._dsh_operations.worker) + or (self._dsh_automation and getattr(self._dsh_automation, 'worker', None))): self._close_pending = True self.plan_panel.cancel_operation() self._pause_queue() @@ -458,9 +462,12 @@ class TrialTaskCenterWindow(MaterialMainWindow): def _update_gates(self): busy = bool(self._workers) or self._loading or self.history_panel._worker is not None resources_pending = self._resource_status.blocked + dialog_state = getattr(self.service, '_supervisor_dialog_state', None) or {} + modal_pending = bool((dialog_state.get('supervision') or {}).get('blocked')) self.history_panel.set_mutations_enabled(not busy and not resources_pending and self._batch_stop is None) - self.run_panel.cleanup_resources_button.setVisible(resources_pending and self._resource_status.known) - self.run_panel.cleanup_resources_button.setEnabled(resources_pending and self._resource_status.known and not busy) + self.run_panel.cleanup_resources_button.setText('处理阻塞弹窗' if modal_pending else '重试释放资源') + self.run_panel.cleanup_resources_button.setVisible(modal_pending or resources_pending and self._resource_status.known) + self.run_panel.cleanup_resources_button.setEnabled(modal_pending or resources_pending and self._resource_status.known and not busy) state = execution_state(self._context, busy=busy, dirty=self.plan_panel.table.dirty or self.plan_panel.table.has_pending_editor()) group_unsaved = self._group_has_unsaved_plans() if state.editable or state.new_round else False @@ -739,7 +746,11 @@ class TrialTaskCenterWindow(MaterialMainWindow): if len(ids) <= 20: from ...dsh import task_lifecycle as lifecycle try: - report = lifecycle.preflight(self.service, list(ids), allow_completed=True) + if token: + from ...dsh.run_journal import resume_preflight + report = resume_preflight(self.service, self._dsh_operations.runs.inspect(self.service, token)) + else: + report = lifecycle.preflight(self.service, list(ids), allow_completed=True) except Exception as error: self._show_error(error, owner_task_id=ids[0]) return @@ -870,11 +881,29 @@ class TrialTaskCenterWindow(MaterialMainWindow): def _cleanup_batch_resources(self): if not self.run_panel.cleanup_resources_button.isEnabled(): return + state = getattr(self.service, '_supervisor_dialog_state', None) or {} + if (state.get('supervision') or {}).get('blocked'): + from .cad_dialog_operations import show_cad_dialogs + show_cad_dialogs(self) + self._update_gates() + return def done(pending): self._set_operator_message( '后台资源仍待清理,请保留诊断记录后重试。' if pending else '后台资源已释放,可以继续。', 'warning' if pending else 'success') self._update_gates() + if pending and QMessageBox.question(self, '强制结束自有残留进程', + '安全释放尚未完成。是否强制结束本程序仍持有且能证明归属的残留进程?\n\n' + '可能存在未保存或部分保存的结果。中断证据会保留,相关工程必须先核验,不能直接再次替换。', + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No) == QMessageBox.StandardButton.Yes: + def forced(remaining): + self._resource_status.refresh() + self._set_operator_message('仍有资源未能确认释放,请保留诊断。' if remaining else + '自有残留资源已释放;相关工程仍须核验后恢复。', 'warning') + self._update_gates() + self._run_async(lambda: self.service.force_pending_owned_resources(confirmed=True), forced, + self.run_panel.cleanup_resources_button) self._run_async(self.service.cleanup_batch_resources, done, self.run_panel.cleanup_resources_button) @@ -997,6 +1026,7 @@ class TrialTaskCenterWindow(MaterialMainWindow): QMessageBox.StandardButton.No) == QMessageBox.StandardButton.Yes def _update_quick_run_elapsed(self): + self._update_gates() if self._batch_stop is not None and not self._feedback_query.busy: task_id, started = self._active_quick_run_task_id, self._run_started_at run_id = self._dsh_run_id diff --git a/src/dx_productivity/trial_task_center/ui/messages.py b/src/dx_productivity/trial_task_center/ui/messages.py index d2ba8b4..91c30aa 100644 --- a/src/dx_productivity/trial_task_center/ui/messages.py +++ b/src/dx_productivity/trial_task_center/ui/messages.py @@ -139,7 +139,20 @@ def replacement_result_rows(lines): state = details.get('state') location = ':'.join(str(details.get(key) or '') for key in ('schematic_name', 'sheet_number')) verification = '已保存并核验' if state == 'saved_ok' else '已回读,保存待确认' - rows.extend((location, refdes, old, new, verification) + observation = details.get('value_readback') + if state in {'saved_ok', 'restored_ok'} and isinstance(observation, dict): + if observation.get('value_readback_status') == 'verified': + location = ':'.join(str(observation.get(key) or '') for key in ('schematic_name', 'sheet_number')) + for record in observation.get('value_readback', ()): + rows.append((location, record['refdes'], record['old_part_number'], record['actual_part_number'], + record.get('old_value') if record.get('old_value') is not None else '原值证据不明确', + record['actual_value'] if record['actual_value_present'] else '属性不存在', verification)) + continue + actual_value = {'rejected': '回读证据被拒绝', 'ambiguous': '属性来源不明确'}.get( + observation.get('value_readback_status'), '未提供实际回读') + else: + actual_value = '未提供实际回读' + rows.extend((location, refdes, old, new, '未提供', actual_value, verification) for refdes, old, new in _replacement_parts(str(event.get('message') or ''), state)) return tuple(rows) diff --git a/src/dx_productivity/trial_task_center/ui/project_plan.py b/src/dx_productivity/trial_task_center/ui/project_plan.py index ce91335..eb90664 100644 --- a/src/dx_productivity/trial_task_center/ui/project_plan.py +++ b/src/dx_productivity/trial_task_center/ui/project_plan.py @@ -7,7 +7,7 @@ from time import monotonic from PySide6.QtCore import Qt, QThreadPool, QObject, Signal, QSignalBlocker from PySide6.QtGui import QAction,QKeySequence -from PySide6.QtWidgets import (QDialog, QDialogButtonBox, QFileDialog, QHBoxLayout, QLabel, +from PySide6.QtWidgets import (QCheckBox, QDialog, QDialogButtonBox, QFileDialog, QHBoxLayout, QLabel, QLineEdit, QListWidget, QListWidgetItem, QMessageBox, QPushButton, QVBoxLayout, QWidget, QToolButton, QMenu) from shiboken6 import isValid @@ -221,6 +221,11 @@ class ProjectPlanPanel(QWidget): self.report_toggle.hide() self.report_toggle.toggled.connect(self._toggle_report) self.last_bom_report = None + self.partial_bom = QCheckBox('允许部分导入通过工程', objectName='allowPartialBom') + self.partial_bom.setToolTip('仅核验前置检查通过的工程;报告返回后仍需选择工程并确认。') + self.partial_bom_button = QPushButton('选择通过工程…', objectName='selectPassedBomProjects') + self.partial_bom_button.hide() + self.partial_bom_button.clicked.connect(self._choose_passed_projects) toolbar=QWidget(objectName='planToolbar') actions=QHBoxLayout(toolbar) actions.setContentsMargins(0,3,0,3);actions.setSpacing(8) @@ -263,6 +268,8 @@ class ProjectPlanPanel(QWidget): self.find_result.hide() layout.addLayout(summary_row) layout.addWidget(self.report_toggle) + layout.addWidget(self.partial_bom) + layout.addWidget(self.partial_bom_button) layout.addWidget(self.table, 1) layout.addLayout(selection) self.collect_button.clicked.connect(self.collect) @@ -427,7 +434,7 @@ class ProjectPlanPanel(QWidget): def _release_bom_previews(self, saved=None): for machine in self._bom_imports.values(): report = machine.state.report - preview = report.preview if report is not None else None + preview = (report.preview or report.passed_preview) if report is not None else None if preview is not None and (saved is None or all(p.task_id in saved for p in preview.projects)): machine.release_preview(saved=saved is not None) self._restore_bom_report(self._report_scope(self.task_id)) @@ -641,8 +648,9 @@ class ProjectPlanPanel(QWidget): if paths: task_id = self.task_id scope = self._report_scope(task_id) + partial = {'allow_partial': True} if self.partial_bom.isChecked() else {} self._dispatch_cancellable(lambda stop, progress, phase: self.window.service.preview_bom_sources( - task_id, tuple(Path(path) for path in paths), stop_event=stop, on_progress=progress, on_phase=phase), + task_id, tuple(Path(path) for path in paths), stop_event=stop, on_progress=progress, on_phase=phase, **partial), lambda report: self._received_bom_report(report, scope=scope), self.import_bom_button, bom_scope=scope) @@ -681,6 +689,8 @@ class ProjectPlanPanel(QWidget): _, self.report_text, title = self._format_bom_report(state) self.report_toggle.setText(title) self.report_toggle.setVisible(state.report is not None) + self.partial_bom_button.setVisible(state.report is not None and state.report.preview is None + and state.report.passed_preview is not None and not state.notice and not state.busy) if state.report is None: self.report_toggle.setChecked(False) if self.report_dialog is not None: @@ -708,6 +718,8 @@ class ProjectPlanPanel(QWidget): details.append(state.notice) elif report.preview is None: details.append('本次未暂存或保存新计划,原有预览和编辑保留;请修正后重新导入。') + if report.passed_preview is not None: + details.append('可点击“选择通过工程…”确认部分导入;失败行及其来源记录保留。') else: details.append('校验通过,已暂存预览;请确认后保存本批位号计划。') title = f'本批 BOM {outcome} · 通过 {report.passed_rows} / 失败 {report.failed_rows} / 未检查 {report.unchecked_rows} 行 · 详情' @@ -738,6 +750,43 @@ class ProjectPlanPanel(QWidget): return next((batch for batch in (*self._bom_batches.values(), *self._orphaned_bom_batches.values()) if any(project.task_id == self.task_id for project in batch.projects)), None) + def _choose_passed_projects(self): + report = self.last_bom_report + if report is None or report.passed_preview is None: + return + dialog = QDialog(self) + dialog.setWindowTitle('确认部分导入工程') + layout = QVBoxLayout(dialog) + layout.addWidget(QLabel('仅导入勾选工程的原预览行;失败工程保留原计划。')) + choices = QListWidget() + for project in report.passed_preview.projects: + item = QListWidgetItem(str(project.project_path)) + item.setData(Qt.ItemDataRole.UserRole, project.task_id) + item.setFlags(item.flags() | Qt.ItemFlag.ItemIsUserCheckable) + item.setCheckState(Qt.CheckState.Unchecked) + choices.addItem(item) + layout.addWidget(choices) + buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel) + buttons.accepted.connect(dialog.accept) + buttons.rejected.connect(dialog.reject) + layout.addWidget(buttons) + if dialog.exec() == QDialog.DialogCode.Accepted: + selected = tuple(choices.item(i).data(Qt.ItemDataRole.UserRole) for i in range(choices.count()) + if choices.item(i).checkState() == Qt.CheckState.Checked) + if selected: + self.accept_passed_projects(selected) + + def accept_passed_projects(self, task_ids): + from ..bom_import_report import select_passed_projects + machine = self._bom_imports.get(self._report_scope(self.task_id)) + if machine is None or machine.state.busy: + return + preview = select_passed_projects(machine.state.report, task_ids) + self._preview_bom(preview) + machine.state = replace(machine.state, report=replace(machine.state.report, passed_preview=preview), + notice=f'已明确选择 {len(preview.projects)} 个通过工程,待保存原预览行。') + self._restore_bom_report(self._report_scope(self.task_id)) + def _preview_bom(self, batch): if self.task_id and self.table.dirty: self._drafts[self.task_id] = self.table.rows() diff --git a/src/dx_productivity/trial_task_center/value_readback.py b/src/dx_productivity/trial_task_center/value_readback.py new file mode 100644 index 0000000..beae734 --- /dev/null +++ b/src/dx_productivity/trial_task_center/value_readback.py @@ -0,0 +1,166 @@ +"""Persist observed values; command and terminal proof remain the authority.""" +import hashlib +import json +import re + +from .batch_resources import write_record +from .path_access import internal_path +from .quick_run_protocol import _decode_vbs_field + + +def _paths(root, row): + job = row['job_id'] + if not re.fullmatch(r'[A-Za-z0-9_-]+', job): + raise ValueError('invalid Value report job identity') + root = internal_path(root) + return root / f'value-readback-{job}.tsv', root / f'value-readback-{job}.json', root / f'snapshot-{job}.tsv' + + +def clear_value_readback(root, row): + """Discard prior observation before dispatch, so a retry cannot reuse it.""" + raw, saved, _ = _paths(root, row) + raw.unlink(missing_ok=True) + saved.unlink(missing_ok=True) + + +def _digest(data): + return hashlib.sha256(data).hexdigest() + + +def _read(path): + if path.stat().st_size > 16 * 1024 * 1024: + raise ValueError('Value evidence exceeds size limit') + return path.read_bytes() + + +def _fields(line): + return tuple(_decode_vbs_field(field) for field in line.split('\t')) + + +def _validated(root, row, outcome): + raw, _, snapshot_path = _paths(root, row) + state, code, message = outcome + if state not in {'saved_ok', 'restored_ok'}: + raise ValueError('Value report requires a verified replacement terminal state') + status_bytes = _read(internal_path(root) / 'status.tsv') + terminal = _fields(status_bytes.decode('utf-8-sig').splitlines()[-1]) + if terminal != (row['job_id'], state, code, message): + raise ValueError('Value report does not belong to the current terminal job') + raw_bytes, snapshot_bytes = _read(raw), _read(snapshot_path) + snapshot = snapshot_bytes.decode('utf-8-sig') + lines = raw_bytes.decode('utf-8-sig').splitlines() + header = _fields(lines[0]) + expected = ('V', '2', row['task_id'], row['job_id'], state, row['project_path'], + row.get('schematic_name', ''), row.get('sheet_number', ''), row['old_part'], row['new_part'], + row['value_mode'], row.get('custom_value', ''), row.get('sync_nc', '1'), row.get('target_cell', ''), + row.get('target_selectors', ''), snapshot) + if header != expected: + raise ValueError('Value report version, command or current snapshot differs') + components, old_values, source_nc = {}, {}, set() + def has_nc(value): + return any(part.strip().casefold() == 'nc' for part in value.split('/')[1:]) + for line in snapshot.splitlines(): + fields = _fields(line) + if fields[0] == 'C': + if len(fields) != 8 or not fields[1] or fields[1] in components: + raise ValueError('snapshot component identity is invalid') + components[fields[1]] = fields[2] + elif fields[0] == 'A' and len(fields) == 12 and fields[2].casefold() == 'value': + old_values.setdefault(fields[1], []).append(fields[4] or fields[3]) + if (fields[0] == 'A' and len(fields) == 12 and fields[2].casefold() == 'value' + and has_nc(fields[4] or fields[3])): + source_nc.add(fields[1]) + selectors = row.get('target_selectors', '') + if row.get('target_scope') == 'targets': + selected = {parts[0]: parts[1] for selector in selectors.split('\x1e') + if len(parts := selector.split('\x1f')) == 3} + if components != selected: + raise ValueError('Value snapshot UID set differs from frozen command') + values, observations, seen, ordinals = [], [], set(), set() + names = {} + for line in lines[1:]: + fields = _fields(line) + if fields[0] == 'C' and len(fields) == 7: + _, uid, ref, part, value, present, cell = fields + if (uid not in components or uid in seen or components[uid] != ref or present not in {'0', '1'} + or part.casefold() != row['new_part'].casefold() + or (row.get('target_cell') and cell != row['target_cell'])): + raise ValueError('foreign, duplicate or inconsistent Value identity/Part Number/Cell') + seen.add(uid) + expected = {'custom': row.get('custom_value', ''), 'part_number': part}.get(row['value_mode']) + if expected is not None: + if row.get('sync_nc', '1') == '1' and uid in source_nc and not has_nc(expected): + expected = expected.rstrip() + ('NC' if expected.rstrip().endswith('/') else '/NC') + if present != '1' or value != expected: + raise ValueError('actual Value contradicts verified command policy') + before = old_values.get(uid, []) + values.append(dict(uid=uid, refdes=ref, old_part_number=row['old_part'], actual_part_number=part, + old_value=before[0] if len(before) == 1 else None, actual_value=value, + old_value_present=bool(before), actual_value_present=present == '1', actual_cell=cell)) + elif fields[0] == 'E' and len(fields) == 7: + _, uid, ordinal, name, raw_value, instance, layer = fields + if uid not in components or not ordinal.isdigit() or layer != 'unknown' or (uid, ordinal) in ordinals: + raise ValueError('invalid attribute observation identity') + ordinals.add((uid, ordinal)) + names.setdefault((uid, name.casefold()), []).append(int(ordinal)) + observations.append(dict(uid=uid, ordinal=int(ordinal), name=name, raw_value=raw_value, + instance_value=instance, layer='unknown')) + else: + raise ValueError('invalid Value report record') + if not components or seen != set(components): + raise ValueError('Value report UID set is incomplete') + if (_read(internal_path(root) / 'status.tsv') != status_bytes or _read(raw) != raw_bytes + or _read(snapshot_path) != snapshot_bytes): + raise ValueError('Value evidence changed during validation') + ambiguities = [dict(uid=uid, name=name, ordinals=indices) for (uid, name), indices in names.items() if len(indices) > 1] + details = dict(value_readback_status='ambiguous' if ambiguities else 'verified', value_readback=values, + value_readback_job_id=row['job_id'], project_path=row['project_path'], + schematic_name=row.get('schematic_name', ''), sheet_number=row.get('sheet_number', ''), + attribute_observations=observations, attribute_ambiguities=ambiguities) + binding = dict(command=dict(row), terminal=list(terminal), status_sha256=_digest(status_bytes), + snapshot_sha256=_digest(snapshot_bytes), report_sha256=_digest(raw_bytes)) + return details, binding + + +def capture_value_readback(root, row, outcome): + try: + details, binding = _validated(root, row, outcome) + _, saved, _ = _paths(root, row) + write_record(saved, dict(schema=2, binding=binding, details=details)) + return details + except FileNotFoundError: + return {'value_readback_status': 'unavailable'} + except (OSError, UnicodeError, ValueError, KeyError, IndexError, TypeError) as error: + return {'value_readback_status': 'rejected', 'value_readback_reason': str(error)} + + +def load_value_readback(root, row, outcome): + try: + details, binding = _validated(root, row, outcome) + _, saved, _ = _paths(root, row) + persisted = json.loads(_read(saved)) + if persisted != dict(schema=2, binding=binding, details=details): + raise ValueError('persisted Value evidence binding changed') + return details + except FileNotFoundError: + return {'value_readback_status': 'unavailable'} + except (OSError, UnicodeError, ValueError, KeyError, IndexError, TypeError) as error: + return {'value_readback_status': 'rejected', 'value_readback_reason': str(error)} + + +def value_readback_details(result): + """Read only the verified persisted observation for this returned terminal.""" + if result.state not in {'saved_ok', 'restored_ok'}: + return {} + try: + from .execution_journal import read_command_rows + terminal = _fields(_read(internal_path(result.status_path)).decode('utf-8-sig').splitlines()[-1]) + if len(terminal) != 4 or terminal[1] != result.state or terminal[3] != result.message: + raise ValueError('returned result differs from current terminal job') + rows = tuple(row for row in read_command_rows(result.command_path) if row['job_id'] == terminal[0]) + if len(rows) != 1: + raise ValueError('terminal job is not uniquely bound to current command') + details = load_value_readback(result.status_path.parent, rows[0], terminal[1:]) + except (OSError, UnicodeError, ValueError, KeyError, IndexError, TypeError) as error: + details = {'value_readback_status': 'rejected', 'value_readback_reason': str(error)} + return {'value_readback': details} diff --git a/src/dx_productivity/trial_task_center/worker_monitor.py b/src/dx_productivity/trial_task_center/worker_monitor.py index 837528d..a4ac23c 100644 --- a/src/dx_productivity/trial_task_center/worker_monitor.py +++ b/src/dx_productivity/trial_task_center/worker_monitor.py @@ -39,6 +39,72 @@ class WorkerMonitorSafeTimeout(WorkerMonitorError): _PROGRESS_FAILURE_LOGGED: dict[str, float] = {} +RETAINED_WORKERS = {} + + +def _worker_tree_exited(process, owned): + from .read_recovery import process_tree_exited + try: + observe = getattr(owned, 'tree_exited', None) + return process_tree_exited(process) and (observe is None or observe()) + except Exception: + return False + + +def _retain_worker(root, token, job_id, process, owned, close_process, error): + from .batch_resources import process_identity, write_record + from .stop_control import protect_interruption + path = root / ('retained-worker-' + token + '.json') + entry = dict(root=root, path=path, token=token, job_id=job_id, process=process, + owned=owned, close_process=close_process) + # Keep the handles even if writing the protection/ownership proof fails. + RETAINED_WORKERS[str(path)] = entry + protect_interruption(root, 'worker_retained: ' + str(error)) + identity = process_identity(process) + entry['identity'] = identity + write_record(path, dict(version=1, state='blocked', token=token, job_id=job_id, + process=identity, error=str(error))) + + +def cleanup_retained_workers(root, *, force=False): + from .batch_resources import process_identity, write_record + from .errors import TaskCenterError + from .path_access import canonical_path, internal_path + from .stop_control import protect_interruption + for key, entry in tuple(RETAINED_WORKERS.items()): + if not entry['root'].is_relative_to(canonical_path(root)): + continue + process, owned = entry['process'], entry['owned'] + if not entry.get('resources_released') and not _worker_tree_exited(process, owned): + if not force: + continue + record = json.loads(internal_path(entry['path']).read_text(encoding='utf-8')) + identity = process_identity(process) + if (not identity.get('creation_time') or identity != entry.get('identity') + or record.get('process') != identity or record.get('version') != 1 + or record.get('token') != entry['token'] or record.get('job_id') != entry['job_id']): + raise TaskCenterError('target_identity', '保留工作进程身份与证据不符,拒绝强制结束。') + protect_interruption(entry['root'], 'user_confirmed_force_pending_owned_resources') + write_record(entry['path'], dict(record, force_state='requested', force_confirmed=True)) + if entry['close_process']: + process.terminate() + else: + owned.terminate() + process.wait(timeout=10) + deadline = time.monotonic() + 10 + while not _worker_tree_exited(process, owned) and time.monotonic() < deadline: + time.sleep(.01) + if not entry.get('resources_released') and not _worker_tree_exited(process, owned): + continue + if not entry.get('resources_released'): + if entry['close_process']: + process.close() + owned.close() + entry['resources_released'] = True + # Physical release never clears the durable business uncertainty. + write_record(entry['path'], dict(version=1, state='released', token=entry['token'], + job_id=entry['job_id'], process=entry.get('identity'), processes_exited=True)) + RETAINED_WORKERS.pop(key) class ProgressWatch: @@ -50,6 +116,7 @@ class ProgressWatch: self.current_job_id = job_id self.idle_timeout = idle_timeout self.last_progress = started + self.last_business_progress = started self.sequence = 0 self.native_sequence = 0 self.native_progress_frames = 0 @@ -78,6 +145,9 @@ class ProgressWatch: return False except (ValueError, UnicodeError): return False + if ((current_job, fields[4], done, total, bool(safe)) != + (self.current_job_id, self.stage, self.done, self.total, self.safe_stop)): + self.last_business_progress = now if current_job != self.current_job_id: self.current_job_id = current_job self.stage_seconds = {} @@ -118,11 +188,13 @@ class ProgressWatch: self.native_sequence = sequence self.native_progress_frames += 1 self.last_progress = now + if (done, total) != (self.done, self.total): + self.last_business_progress = now self.done, self.total = done, total return True def action(self, now: float) -> str | None: - idle = now - self.last_progress + idle = now - self.last_business_progress idle_timeout = self.idle_timeout if self.stage == 'open_project' and self.safe_stop: idle_timeout = min(idle_timeout, PROJECT_OPEN_IDLE_TIMEOUT_SECONDS) @@ -145,6 +217,8 @@ class ProgressWatch: return {name: getattr(self, name) for name in ('token', 'sequence', 'stage', 'done', 'total', 'safe_stop', 'stop_requested', 'waiting_for_safe_completion')} | { 'job_id': self.current_job_id, 'native_progress_frames': self.native_progress_frames, + 'business_idle_seconds': max(0.0, now - self.last_business_progress), + 'heartbeat_idle_seconds': max(0.0, now - self.last_progress), 'elapsed_seconds': max(0.0, now - self.started), 'stage_seconds': stages} @@ -182,6 +256,15 @@ class _OwnedWorkers: self.job = win32job.CreateJobObject(None, 'DxPartReplace-monitor-' + token) self.allowed = {os.path.normcase(os.path.abspath(value)) for value in allowed} + def tree_exited(self): + import win32job + return not win32job.QueryInformationJobObject(self.job, + win32job.JobObjectBasicAccountingInformation)['ActiveProcesses'] + + def terminate(self): + import win32job + win32job.TerminateJobObject(self.job, 1) + def assign(self, process: subprocess.Popen) -> None: import win32job @@ -229,7 +312,8 @@ class _OwnedWorkers: def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abort_event=None, - process_factory=None, cooperative_abort=False, force_event=None, force_control=None) -> subprocess.CompletedProcess[str]: + process_factory=None, cooperative_abort=False, force_event=None, force_control=None, + owner_loss_path=None) -> subprocess.CompletedProcess[str]: root = Path(command[3]).resolve() job_id = command[-1] token = uuid4().hex @@ -245,6 +329,7 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor force_event = force_control.force_event owned = _OwnedWorkers(token, {command[0], command[4]}) process = None + retained = False durable_force_accepted = False read_cancel_started = None read_idle_stop_started = None @@ -311,6 +396,16 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor _publish_progress(progress_path, watch.snapshot()) _atomic_text(ready, token) while process.poll() is None: + if cooperative_abort and owner_loss_path is not None and Path(owner_loss_path).is_file(): + # The admitted read owner was lost. This only stops this exact + # read worker; its runtime releases the complete generation. + if read_cancel_started is None: + read_cancel_started = time.monotonic() + if not watch.stop_requested: + watch.stop_requested = True + request_stop() + if time.monotonic() - read_cancel_started >= READ_ONLY_CANCEL_GRACE_SECONDS: + raise WorkerMonitorCancelled('read owner exited before the read completed', stage=watch.stage) if force_event is not None and force_event.is_set(): from .stop_control import protect_interruption try: @@ -338,6 +433,7 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor # have reached watch yet, so ordinary pause cannot kill here. watch.stop_requested = True watch.last_progress = time.monotonic() + watch.last_business_progress = watch.last_progress request_stop() _publish_progress(progress_path, watch.snapshot()) if (read_idle_stop_started is not None and time.monotonic() - read_idle_stop_started >= @@ -371,6 +467,9 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor # This entry point is explicitly read-only. A blocked COM # read has no write to finish and retains its bounded cancel. raise subprocess.TimeoutExpired(command, idle_timeout * 2) + from .stop_control import StopControl + if not isinstance(force_control, StopControl): + raise WorkerMonitorError('unsafe worker stalled without an interactive controller') _publish_progress(progress_path, watch.snapshot()) elif action == 'unresponsive': if stop_write_pending and not cooperative_abort: @@ -388,14 +487,22 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor except queue.Empty: pass _publish_progress(progress_path, watch.snapshot()) + if not cooperative_abort and not _worker_tree_exited(process, owned): + raise WorkerMonitorError('worker root exited while owned descendants remain; reconciliation required') return subprocess.CompletedProcess(command, process.returncode, _decode(output[0]), _decode(output[1])) except Exception as error: if process is not None: - if (not watch.safe_stop and not cooperative_abort and force_control is not None - and not durable_force_accepted): - from .stop_control import wait_for_resolution + if (not cooperative_abort and not durable_force_accepted + and (not watch.safe_stop or process.poll() is not None) + and not _worker_tree_exited(process, owned)): + from .stop_control import StopControl, wait_for_resolution + if not isinstance(force_control, StopControl): + retained = True + _retain_worker(root, token, job_id, process, owned, process_factory is not None, error) + raise WorkerMonitorError('owned worker retained for explicit force or confirmed exit: ' + str(error)) from error try: - wait_for_resolution(force_control, root, error) + wait_for_resolution(force_control, root, error, + resolved=lambda: _worker_tree_exited(process, owned)) except WorkerMonitorError: if not force_event.is_set(): raise @@ -419,6 +526,13 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor stop_readers.set() for thread in threads: thread.join(0.2) - owned.close() - if process_factory is not None and process is not None: - process.close() + if (not retained and process is not None and not cooperative_abort and not durable_force_accepted + and not _worker_tree_exited(process, owned) + and (not watch.safe_stop or process.poll() is not None)): + retained = True + _retain_worker(root, token, job_id, process, owned, process_factory is not None, + 'worker monitor cleanup did not prove the owned tree exited') + if not retained: + owned.close() + if process_factory is not None and process is not None: + process.close() diff --git a/src/dx_productivity/trial_task_center/workspace_bom.py b/src/dx_productivity/trial_task_center/workspace_bom.py index dc73de8..3e48ae2 100644 --- a/src/dx_productivity/trial_task_center/workspace_bom.py +++ b/src/dx_productivity/trial_task_center/workspace_bom.py @@ -1,6 +1,7 @@ """Safe multi-file BOM loading and project routing for workspace imports.""" from dataclasses import dataclass +import hashlib from pathlib import Path, PureWindowsPath from openpyxl import load_workbook @@ -24,6 +25,7 @@ class SourceRow: excel_row: int values: tuple value_mode: str = 'inherit' + source_sha256: str = '' @property def project_name(self): @@ -53,7 +55,7 @@ def _selected_xlsx(sources): return tuple(sorted(paths, key=lambda path: str(path).casefold())) -def load_sources(sources, *, stop_event=None, on_progress=None): +def load_sources(sources, *, stop_event=None, on_progress=None, allow_partial=False): """Read every selected workbook before any CAD operation starts.""" from .operation_control import check_cancelled @@ -67,6 +69,7 @@ def load_sources(sources, *, stop_event=None, on_progress=None): on_progress(f'正在读取 BOM {number}/{len(paths)}:{path.name}') workbook = None try: + source_sha256 = hashlib.sha256(path.read_bytes()).hexdigest() workbook = load_workbook(path, read_only=True, data_only=False) if len(workbook.worksheets) != 1: raise ValueError('BOM 只能包含一个工作表') @@ -84,13 +87,18 @@ def load_sources(sources, *, stop_event=None, on_progress=None): positions = {name: index for index, name in enumerate(headers) if name} for excel_row, cells in enumerate(rows, 2): check_cancelled(stop_event) - if any(cell.data_type == 'f' for cell in cells): + formula = any(cell.data_type == 'f' for cell in cells) + if formula and not allow_partial: raise ValueError(f'Excel 第 {excel_row} 行不接受公式') values = tuple( cells[positions[name]].value if name in positions and positions[name] < len(cells) else '' for name in BOM_HEADERS ) - if not any(values[index] not in (None, '') for index in (2, 3, 4, 9)): + if formula and allow_partial: + # Even a formula in an unmapped extra column rejects its + # original row; keep a formula marker for the strict parser. + values = (*values[:8], '=INVALID_SOURCE_FORMULA()', *values[9:]) + if not any(values[index] not in (None, '') for index in (2, 3, 4, 9)) and not formula: continue project_path = str(values[1] or '').strip() if project_path and not (Path(project_path).is_absolute() or PureWindowsPath(project_path).is_absolute()): @@ -99,12 +107,15 @@ def load_sources(sources, *, stop_event=None, on_progress=None): _part_number_text(cells, positions, 'old_part_number', '旧料号') _part_number_text(cells, positions, 'new_part_number', '新料号') except ValueError as error: - raise ValueError(f'Excel 第 {excel_row} 行:{error}') from error + if not allow_partial: + raise ValueError(f'Excel 第 {excel_row} 行:{error}') from error default_mode = 'custom' if str(values[9] or '').strip() else 'inherit' value_mode = str(cells[positions['value_mode']].value or default_mode).strip() if 'value_mode' in positions else default_mode - if value_mode not in {'inherit', 'preserve', 'part_label', 'part_number', 'custom'}: + if value_mode not in {'inherit', 'preserve', 'part_label', 'part_number', 'custom'} and not allow_partial: raise ValueError(f'Excel 第 {excel_row} 行 Value 来源无效') - result.append(SourceRow(path, excel_row, values, value_mode)) + result.append(SourceRow(path, excel_row, values, value_mode, source_sha256)) + if hashlib.sha256(path.read_bytes()).hexdigest() != source_sha256: + raise ValueError('BOM 来源在读取期间已改变,请重新导入') except TaskCenterError: raise except Exception as error: @@ -192,6 +203,7 @@ def write_routed_bom(destination, rows, routes, *, project_names=None): else: reason = 'single_project' source_map.append({'source_file': str(row.source), 'source_row': row.excel_row, + 'source_sha256': row.source_sha256, 'original_project_path': row.project_path, 'project_name': row.project_name, 'matched_project_path': str(Path(target).resolve()), diff --git a/src/dx_productivity/trial_task_center/workspace_execution.py b/src/dx_productivity/trial_task_center/workspace_execution.py new file mode 100644 index 0000000..f7706a7 --- /dev/null +++ b/src/dx_productivity/trial_task_center/workspace_execution.py @@ -0,0 +1,202 @@ +"""Frozen parent queue, with one resource-owning supervisor per project.""" +from contextlib import contextmanager +import hashlib +import json +from pathlib import Path +from threading import Lock +from uuid import uuid4 + +from .batch_resources import write_record +from .errors import TaskCenterError +from .path_access import canonical_path, internal_path +from .stop_control import StopControl, protect_interruption + +_LOCK = Lock() +_TERMINAL = {'completed', 'failed', 'skipped', 'blocked'} + + +class _ProjectControl(StopControl): + def __init__(self, parent): + super().__init__() + self.parent = parent + if hasattr(parent, 'force_event'): + self.force_event = parent.force_event + + def is_set(self): + return super().is_set() or self.parent.is_set() + + +@contextmanager +def _lease(root): + from .task_queue import _os_lock + if not _LOCK.acquire(blocking=False): + raise TaskCenterError('quick_run_busy', '工程队列正在执行。') + handle, acquired = None, False + try: + path = internal_path(root / '.workspace-queue.execution.lock') + path.parent.mkdir(parents=True, exist_ok=True) + handle = path.open('a+b') + if handle.seek(0, 2) == 0: + handle.write(b'\0') + handle.flush() + try: + _os_lock(handle, True) + acquired = True + except OSError as error: + raise TaskCenterError('quick_run_busy', '另一窗口的工程队列正在执行。') from error + yield + finally: + if handle is not None: + if acquired: + _os_lock(handle, False) + handle.close() + _LOCK.release() + + +def _token(value): + return isinstance(value, str) and len(value) == 32 and all(char in '0123456789abcdef' for char in value) + + +def _released_after_recheck(root, task_id, row): + from .batch_resources import _process_gone, pending_resources + from .batch_supervisor import pending_supervisors + from .supervisor_protocol import valid_journal + batch_id = row.get('supervisor_batch_id') + if not _token(batch_id): + return False + try: + journal = json.loads(internal_path(root / 'batch-resources' / batch_id / 'supervisor.json').read_text(encoding='utf-8')) + return (valid_journal(journal, batch_id) and journal['task_ids'] == [task_id] + and journal['state'] == 'finished' and journal.get('process') + and _process_gone(journal['process']) and not pending_resources(root) and not pending_supervisors(root)) + except (OSError, ValueError, KeyError, TypeError): + return False + + +def execute_workspace_projects(service, task_ids, *, stop_event, queue_id=None, + expected_plan_hashes=None, checkpoint_id=None, defer_problem_pass=False): + from .task_queue import _plan_hash, TaskQueueEntry, TaskQueueResult + ids = tuple(task_ids) + if not ids or len(set(ids)) != len(ids) or any(not isinstance(item, str) or not item or Path(item).name != item + or item in {'.', '..'} for item in ids): + raise TaskCenterError('task_queue_selection', '请选择不同的单工程任务。') + root = canonical_path(service._logging.root) + with _lease(root): + children = {item: service.for_project_task(item) for item in ids} + projects = [children[item].projects(item) for item in ids] + if any(len(rows) != 1 for rows in projects) or len({str(canonical_path(rows[0].path)).casefold() + for rows in projects}) != len(ids): + raise TaskCenterError('project_binding', '每个任务必须对应不同的单个工程。') + hashes = {item: _plan_hash(children[item], item) for item in ids} + if expected_plan_hashes is not None and hashes != expected_plan_hashes: + raise TaskCenterError('assistant_stale', '执行计划已变化,请重新预检。') + token = queue_id or checkpoint_id or uuid4().hex + if not _token(token) or queue_id and checkpoint_id: + raise TaskCenterError('task_queue_state', '工程队列编号无效。') + path = internal_path(root / 'workspace-queues' / (token + '.json')) + if queue_id: + try: + state = json.loads(path.read_text(encoding='utf-8')) + if (state['version'] != 1 or state['task_ids'] != list(ids) or state['plan_hashes'] != hashes + or set(state['entries']) != set(ids) or state['phase'] not in {'primary', 'problems', 'finished'} + or any(not _token(row['checkpoint_id']) or row['state'] not in + {'pending', 'running', 'paused', *_TERMINAL} or type(row['released']) is not bool + or type(row['problems_done']) is not bool or not isinstance(row['message'], str) + for row in state['entries'].values())): + raise ValueError('workspace queue inputs changed') + except (OSError, ValueError, TypeError, KeyError) as error: + raise TaskCenterError('task_queue_state', '工程顺序或计划已变化,不能恢复原队列。') from error + else: + if path.exists(): + raise TaskCenterError('task_queue_state', '工程队列检查点已存在,请恢复原队列。') + state = {'version': 1, 'task_ids': list(ids), 'plan_hashes': hashes, 'phase': 'primary', + 'entries': {item: {'state': 'pending', 'message': '', 'released': False, 'problems_done': False, + 'checkpoint_id': hashlib.sha256((token + '\0' + item).encode()).hexdigest()[:32]} + for item in ids}} + write_record(path, state) + + def result(paused=False, current=None): + return TaskQueueResult(token, tuple(TaskQueueEntry(item, state['entries'][item]['state'], + state['entries'][item]['message']) for item in ids), paused, current) + + def run_project(item, *, problems=False): + row = state['entries'][item] + previous = row['state'] + row['state'] = 'running' + write_record(path, state) + child_id = row['checkpoint_id'] + resume = problems or internal_path(root / 'task-queues' / (child_id + '.json')).is_file() + control = _ProjectControl(stop_event) + service._task_queue_feedback = (token, item, ids.index(item) + 1, len(ids)) + service._workspace_queue_context = service._task_queue_feedback + try: + outcome = service._supervise('execute_task_queue', {'task_ids': (item,), + 'expected_plan_hashes': {item: hashes[item]}, + 'queue_id': child_id if resume else None, 'checkpoint_id': None if resume else child_id, + 'defer_problem_pass': not problems}, stop_event=control) + if (outcome.queue_id != child_id or len(outcome.entries) != 1 or outcome.entries[0].task_id != item + or outcome.entries[0].state not in {'pending', 'running', 'paused', *_TERMINAL}): + raise TaskCenterError('task_queue_state', '工程监督进程返回了不同的队列身份。') + row.update(state=outcome.entries[0].state, message=outcome.entries[0].message, released=True) + write_record(path, state) + return outcome.paused and row['state'] != 'blocked' + except TaskCenterError as error: + released = getattr(error, 'resources_released', False) is True + row.update(state='blocked', message=error.message, released=released) + row['supervisor_batch_id'] = getattr(service, '_last_supervisor_batch_id', None) + evidence = getattr(getattr(children[item], '_quick_run_preparer', None), 'data_root', root / 'tasks' / item) + protect_interruption(Path(evidence) / 'quick-run', 'project_supervisor_failed: ' + error.message) + if released: + from .batch_supervisor import reconcile_supervisors + try: + row['released'] = not bool(reconcile_supervisors(root)) + except Exception: + row['released'] = False + write_record(path, state) + return not row['released'] + except BaseException: + row['state'] = previous if previous in _TERMINAL else 'blocked' + write_record(path, state) + raise + + try: + if state['phase'] == 'primary': + for item in ids: + row = state['entries'][item] + if row['state'] in _TERMINAL: + if row['state'] == 'blocked' and not row['released']: + if not _released_after_recheck(root, item, row): + return result(True, item) + row['released'] = True + write_record(path, state) + continue + if stop_event.is_set(): + return result(True, item) + if run_project(item): + return result(True, item) + if stop_event.is_set(): + return result(True, item) + state['phase'] = 'problems' + write_record(path, state) + if defer_problem_pass: + return result() + if state['phase'] == 'problems': + for item in ids: + row = state['entries'][item] + if row['problems_done'] or row['state'] not in {'completed', 'failed', 'paused', 'running'}: + continue + if stop_event.is_set(): + return result(True, item) + # A project that never created its engine checkpoint has + # no admitted deferred work to revisit. + if internal_path(root / 'task-queues' / (row['checkpoint_id'] + '.json')).is_file(): + if run_project(item, problems=True): + return result(True, item) + row['problems_done'] = True + write_record(path, state) + state['phase'] = 'finished' + write_record(path, state) + return result() + finally: + service._task_queue_feedback = None + service._workspace_queue_context = None diff --git a/tests/dsh/test_incident_admission.py b/tests/dsh/test_incident_admission.py new file mode 100644 index 0000000..bfa1511 --- /dev/null +++ b/tests/dsh/test_incident_admission.py @@ -0,0 +1,96 @@ +from contextlib import contextmanager + +from test_incidents import Service, analyze, make_runner + + +def test_rejected_admission_does_not_mutate_or_spend_budget(tmp_path): + @contextmanager + def scope(_action): + raise ValueError('execution_lane_changed') + yield + service = Service() + runner = make_runner(tmp_path, service) + runner.action_scope = scope + result = runner.run('t', 'worker_error', 'failure', execution_authorized=True) + assert service.calls == [] + assert result['attempts'] == 0 + assert not list(tmp_path.glob('dsh/retry-budget/*.json')) + + +def test_model_wait_changed_context_cannot_retry(tmp_path): + service = Service() + def model(*_args): + service.progress.saved_items += 1 + return analyze() + result = make_runner(tmp_path, service, model).run('t', 'worker_error', 'failure', execution_authorized=True) + assert not service.calls + assert result['attempts'] == 0 + assert not list(tmp_path.glob('dsh/retry-budget/*.json')) + + +def test_retry_scope_released_before_reflection_and_publication(tmp_path): + active, phases = [], [] + @contextmanager + def scope(action): + active.append(action) + try: + yield + finally: + active.pop() + service = Service(fixed=False) + execute = service.execute_quick_run_queue + def scoped_execute(*args, **kwargs): + assert active == ['retry_failed'] + return execute(*args, **kwargs) + service.execute_quick_run_queue = scoped_execute + def model(*args): + assert not active + phases.append(args[-1]) + return analyze() + def publish(*_args): + assert not active + return {'status': 'unavailable'} + runner = make_runner(tmp_path, service, model, publish) + runner.action_scope = scope + runner.run('t', 'worker_error', 'failure', execution_authorized=True) + assert phases == ['diagnose', 'reflect'] + assert service.calls == ['prepare', 'execute'] + assert not active + + +def test_cancellation_during_model_wait_retains_evidence_without_publication(tmp_path): + from threading import Event + cancelled = Event() + def model(*_args): + cancelled.set() + return analyze() + service, published = Service(), [] + runner = make_runner(tmp_path, service, model, lambda *_args: published.append(True)) + result = runner.run('t', 'worker_error', 'failure', execution_authorized=True, cancel_event=cancelled) + assert not service.calls and not published + assert result['status'] == 'pending_issue' + assert (tmp_path / 'dsh/incidents' / result['incident_id'] / 'incident.json').is_file() + + +def test_shared_retry_reservation_denial_never_uses_legacy_budget(tmp_path): + service = Service() + service.reserve_problem_retry = lambda *args, **kwargs: () + result = make_runner(tmp_path, service).run('t', 'worker_error', 'failure', execution_authorized=True) + assert not service.calls + assert result['attempts'] == 0 + assert result['gates']['retry_budget_available'] is False + assert not list(tmp_path.glob('dsh/retry-budget/*.json')) + + +def test_shared_reservation_limits_preparation_to_reserved_operation_keys(tmp_path): + service, reserved, prepared_keys = Service(), ('uid-operation',), [] + service.reserve_problem_retry = lambda *args, **kwargs: reserved + prepare = service.prepare_failed_quick_run + def prepare_reserved(task_id, *, operation_keys=None): + prepared_keys.append(operation_keys) + prepare(task_id) + service.prepare_failed_quick_run = prepare_reserved + result = make_runner(tmp_path, service).run('t', 'worker_error', 'failure', execution_authorized=True) + assert prepared_keys == [reserved] + assert result['status'] == 'resolved' + assert not list(tmp_path.glob('dsh/retry-budget/*.json')) diff --git a/tests/dsh/test_run_journal.py b/tests/dsh/test_run_journal.py index ed27ee3..8a856d2 100644 --- a/tests/dsh/test_run_journal.py +++ b/tests/dsh/test_run_journal.py @@ -81,6 +81,66 @@ def test_failed_queue_receipt_does_not_claim_success(project_service): assert journal.finish('a' * 32, result)['status'] == 'failed' +def test_inspect_accepts_frozen_workspace_checkpoint(project_service): + service, _, data, ids, hashes = project_service + journal = RunJournal(data) + journal.begin('b' * 32, ids, hashes) + path = data / 'workspace-queues' / ('b' * 32 + '.json') + path.parent.mkdir(parents=True) + path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes, + 'phase': 'primary', 'entries': {key: {'state': 'pending', 'message': '', + 'released': False, 'problems_done': False, 'checkpoint_id': str(index) * 32} + for index, key in enumerate(ids, 1)}})) + result = journal.inspect(service, 'b' * 32) + assert result['queue_available'] and result['resume_ready'] + assert result['remaining_task_ids'] == ids + + +@pytest.mark.parametrize('isolated', [True, False]) +def test_released_isolated_project_does_not_block_resume_of_other_projects(project_service, isolated): + from dx_productivity.dsh.run_journal import resume_preflight + service, _, data, ids, hashes = project_service + journal = RunJournal(data) + journal.begin('d' * 32, ids, hashes) + evidence(service, ids[0], state='batch_saved_unverified' if isolated else 'batch_failed') + entries = {key: {'state': ('blocked' if isolated else 'failed') if index == 0 else 'pending', 'message': 'problem' if index == 0 else '', + 'released': index == 0, 'problems_done': False, 'checkpoint_id': str(index + 1) * 32} + for index, key in enumerate(ids)} + entries[ids[0]]['supervisor_batch_id'] = 'e' * 32 + path = data / 'workspace-queues' / ('d' * 32 + '.json') + path.parent.mkdir(parents=True) + path.write_text(json.dumps({'version': 1, 'task_ids': ids, 'plan_hashes': hashes, + 'phase': 'primary' if isolated else 'problems', 'entries': entries})) + result = journal.inspect(service, 'd' * 32) + assert result['queue_available'] and result['resume_ready'], result['blockers'] + assert result['remaining_task_ids'] == (ids[1:] if isolated else ids) + assert result['isolated_task_ids'] == ([ids[0]] if isolated else []) + assert result['tasks'][0]['interrupted'] == isolated + report = resume_preflight(service, result) + assert report['ready'] + assert report['tasks'][0]['isolated'] == isolated + + +def test_unpaused_blocked_project_is_not_a_completed_run(project_service): + _, _, data, ids, hashes = project_service + journal = RunJournal(data) + journal.begin('b' * 32, ids, hashes) + result = TaskQueueResult('b' * 32, (TaskQueueEntry(ids[0], 'blocked'),), False) + assert journal.finish('b' * 32, result)['status'] == 'blocked' + + +def test_confirmed_run_reports_isolated_blocked_project_as_issue(project_service, monkeypatch): + from threading import Event + from dx_productivity.dsh.execution_runs import execute_confirmed_run + from dx_productivity.dsh.task_lifecycle import preflight + service, _, data, ids, _ = project_service + monkeypatch.setattr('dx_productivity.dsh.execution_runs.execute_task_queue', lambda *_args, **_kwargs: + TaskQueueResult('c' * 32, tuple(TaskQueueEntry(task_id, 'blocked' if index == 0 else 'completed') + for index, task_id in enumerate(ids)), False)) + result = execute_confirmed_run(service, data, ids, preflight(service, ids), Event(), 'c' * 32) + assert result['status'] == 'blocked' + + def test_native_confirmed_resume_marks_running_and_retains_last_receipt(project_service): _, _, data, ids, hashes = project_service journal = RunJournal(data) diff --git a/tests/trial_task_center/test_worker_monitor.py b/tests/trial_task_center/test_worker_monitor.py index 4014ac8..12d48b4 100644 --- a/tests/trial_task_center/test_worker_monitor.py +++ b/tests/trial_task_center/test_worker_monitor.py @@ -15,7 +15,7 @@ def frame(sequence, stage='capture', done=1, total=18, safe=1, token='token', jo def test_progress_allows_total_runtime_to_exceed_idle_limit(): watch = worker_monitor.ProgressWatch('token', 'job', 150, 0) for sequence, now in enumerate(range(100, 1001, 100), 1): - assert watch.accept(frame(sequence), now) + assert watch.accept(frame(sequence, done=sequence), now) assert watch.action(now) is None assert not watch.stop_requested @@ -33,7 +33,7 @@ def test_progress_during_drain_prevents_emergency_kill(): watch = worker_monitor.ProgressWatch('token', 'job', 150, 0) assert watch.action(150) == 'request_stop' for sequence, now in enumerate((200, 350, 490, 630), 1): - assert watch.accept(frame(sequence, safe=0), now) + assert watch.accept(frame(sequence, done=sequence, safe=0), now) assert watch.action(now) is None assert watch.stop_requested @@ -57,7 +57,7 @@ def test_replayed_sequence_does_not_extend_deadline(): def native_frame(sequence=1, *, parent=1, current='job', safe=1): - return f'DX_NATIVE_PROGRESS\ttoken\tjob\t{parent}\t{current}\t{sequence}\t1\t57\t{safe}\r\n'.encode() + return f'DX_NATIVE_PROGRESS\ttoken\tjob\t{parent}\t{current}\t{sequence}\t{sequence}\t57\t{safe}\r\n'.encode() def test_native_progress_extends_only_its_live_read_without_changing_stdout_sequence(): @@ -67,7 +67,7 @@ def test_native_progress_extends_only_its_live_read_without_changing_stdout_sequ assert watch.accept_native(native_frame(n), now) assert watch.action(now) is None assert watch.sequence == 1 - assert watch.snapshot(now=301)['done'] == 1 + assert watch.snapshot(now=301)['done'] == 3 assert watch.accept(frame(2, stage='verify'), 302) assert not watch.accept_native(native_frame(4), 450) assert watch.action(452) == 'request_stop' diff --git a/tests/ui/test_dsh_ai_nonblocking.py b/tests/ui/test_dsh_ai_nonblocking.py new file mode 100644 index 0000000..05b8095 --- /dev/null +++ b/tests/ui/test_dsh_ai_nonblocking.py @@ -0,0 +1,240 @@ +"""Model and supervision waits must not own execution admission or capacity.""" +from threading import Event + +import pytest +from PySide6.QtCore import QThreadPool, Signal + +from dx_productivity.trial_task_center.ui.dsh_automation import DshAutomation +from dx_productivity.trial_task_center.ui.workers import ServiceWorker +from .test_dsh_native_menu import HarnessView +from .test_dsh_operations import setup_view + + +@pytest.fixture +def single_execution_thread(ui, qtbot): + ui.settle() + pool = QThreadPool.globalInstance() + previous = pool.maxThreadCount() + pool.setMaxThreadCount(1) + yield pool + qtbot.waitUntil(lambda: pool.activeThreadCount() == 0, timeout=5000) + pool.setMaxThreadCount(previous) + + +@pytest.mark.parametrize('source', ['timer', 'history', 'refresh_ticket']) +def test_supervision_wait_leaves_execution_lane_and_thread_available( + ui, monkeypatch, qtbot, single_execution_thread, source): + setup_view(ui, monkeypatch) + operations = ui.window._dsh_operations + started, release, executed = Event(), Event(), Event() + def read(*_args, **_kwargs): + started.set() + assert release.wait(5) + return {'schedules': []} + monkeypatch.setattr(operations, '_read', read) + monkeypatch.setattr(operations.tickets, 'refresh', read) + if source == 'timer': + operations.tick() + else: + spec = {'operation': source} + if source == 'refresh_ticket': + spec['incident_id'] = 'a' * 24 + operations.request({'request_id': 'a' * 32, 'spec': spec}) + try: + qtbot.waitUntil(started.is_set) + assert operations.worker not in ui.window._workers + assert not operations.busy + ui.window._run_async(executed.set, lambda _: None) + qtbot.waitUntil(executed.is_set, timeout=1000) + assert not release.is_set() + finally: + release.set() + qtbot.waitUntil(lambda: operations.worker is None and not ui.window._workers) + + +class DiagnosisView(HarnessView): + diagnosis_event = Signal(object) + is_running = True + + def __init__(self, parent): + super().__init__(parent) + self.requests = [] + + def request_diagnosis(self, request): + self.requests.append(request) + + def shutdown(self): + self.is_running = False + self.stopped.emit() + + +@pytest.fixture +def diagnosis(ui, monkeypatch, qtbot): + from dx_productivity.trial_task_center.ui import dsh_automation as module + ui.settle() + monkeypatch.setattr(type(ui.window), '_create_dsh_view', lambda self: DiagnosisView(self)) + published = [] + monkeypatch.setattr(module, 'GitIssuePublisher', lambda _path: lambda *_args: published.append(True) or {'status': 'unavailable'}) + automation = DshAutomation(ui.window) + ui.window._dsh_automation = automation + # Bypass only offscreen enqueue suppression; exercise the real incident runner. + automation.pending.append((ui.tasks[0].task_id, 'worker_error', 'controlled failure', True, 'old-run', None)) + automation._drain() + qtbot.waitUntil(lambda: bool(ui.window._dsh_view.requests)) + yield automation, ui.window._dsh_view, published + automation._native_unavailable = True + automation._runtime_stopped() + if hasattr(automation, 'close'): + automation.close() + qtbot.waitUntil(lambda: automation.worker is None) + + +def test_model_wait_does_not_own_controls_or_starve_execution(single_execution_thread, diagnosis, ui, qtbot): + automation, _view, _ = diagnosis + executed = Event() + try: + assert automation.worker not in ui.window._workers + assert ui.window._batch_stop is None + assert ui.window._active_quick_run_task_id is None + worker = ServiceWorker(executed.set) + single_execution_thread.start(worker) + qtbot.waitUntil(executed.is_set, timeout=1000) + finally: + automation.close() + qtbot.waitUntil(lambda: automation.worker is None) + + +def test_old_diagnosis_finishing_preserves_new_manual_run(diagnosis, ui, qtbot): + automation, _, _ = diagnosis + ui.window._begin_run(ui.tasks[1].task_id) + manual_stop, run_id = ui.window._batch_stop, ui.window._dsh_run_id + try: + automation._runtime_stopped() + qtbot.waitUntil(lambda: automation.worker is None) + assert ui.window._batch_stop is manual_stop + assert ui.window._dsh_run_id == run_id + assert ui.window._dsh_live_run['active'] + assert not manual_stop.is_set() + finally: + ui.window._finish_run() + + +def test_close_wakes_model_wait_and_suppresses_ticket_submission(diagnosis, ui, qtbot): + automation, _, published = diagnosis + ui.window.close() + qtbot.waitUntil(lambda: automation.worker is None, timeout=2000) + assert not published + + +def test_cancelled_close_keeps_idle_automation_available(ui, monkeypatch): + ui.settle() + ui.window.archive_on_close_check.setChecked(False) + automation = DshAutomation(ui.window) + ui.window._dsh_automation = automation + monkeypatch.setattr(ui.window.plan_panel, 'has_unsaved_plans', lambda: True) + monkeypatch.setattr(ui.window, '_close_plan_confirmer', lambda: 'cancel') + ui.window.close() + assert not ui.window._closed + assert not automation._closing + + +def test_cancelled_close_after_model_cancellation_keeps_automation_available(diagnosis, ui, monkeypatch, qtbot): + automation, _, published = diagnosis + ui.window.archive_on_close_check.setChecked(False) + monkeypatch.setattr(ui.window.plan_panel, 'has_unsaved_plans', lambda: True) + monkeypatch.setattr(ui.window, '_close_plan_confirmer', lambda: 'cancel') + ui.window.close() + qtbot.waitUntil(lambda: automation.worker is None) + ui.settle() + assert not ui.window._closed and not automation._closing + assert not published + + +def test_mutation_admission_owns_lane_only_until_actual_action_finishes(ui, qtbot): + from dx_productivity.trial_task_center.stop_control import StopControl + ui.settle() + automation = DshAutomation(ui.window) + ui.window._dsh_automation = automation + automation._observed_run_id = ui.window._dsh_run_id + automation._action_task_id = ui.tasks[0].task_id + automation._action_stop = StopControl() + entered, release, reflected = Event(), Event(), Event() + def action(): + with automation._action_scope('retry_failed'): + entered.set() + assert release.wait(5) + reflected.set() + assert automation.worker not in ui.window._workers + return {'status': 'pending_issue'} + automation._start_worker(ServiceWorker(action)) + try: + qtbot.waitUntil(entered.is_set) + assert automation.worker in ui.window._workers + assert ui.window._batch_stop is automation._action_stop + assert ui.window._active_quick_run_task_id == ui.tasks[0].task_id + finally: + release.set() + qtbot.waitUntil(lambda: automation.worker is None) + assert reflected.is_set() + assert not ui.window._workers + assert ui.window._batch_stop is None + + +def test_new_manual_run_denies_old_admission_without_touching_controls(ui, qtbot): + from dx_productivity.trial_task_center.stop_control import StopControl + ui.settle() + automation = DshAutomation(ui.window) + ui.window._dsh_automation = automation + automation._observed_run_id = ui.window._dsh_run_id + automation._action_task_id = ui.tasks[0].task_id + automation._action_stop = StopControl() + ui.window._begin_run(ui.tasks[1].task_id) + manual_stop = ui.window._batch_stop + rejected, mutated = Event(), Event() + def action(): + try: + with automation._action_scope('retry_failed'): + mutated.set() + except ValueError: + rejected.set() + return {'status': 'pending_issue'} + try: + automation._start_worker(ServiceWorker(action)) + qtbot.waitUntil(lambda: automation.worker is None) + assert rejected.is_set() and not mutated.is_set() + assert ui.window._batch_stop is manual_stop + assert ui.window._active_quick_run_task_id == ui.tasks[1].task_id + assert not manual_stop.is_set() + finally: + ui.window._finish_run() + + +def test_reflection_waits_for_gui_to_confirm_lane_release(ui, qtbot): + from dx_productivity.trial_task_center.stop_control import StopControl + ui.settle() + automation = DshAutomation(ui.window) + ui.window._dsh_automation = automation + automation._observed_run_id = ui.window._dsh_run_id + automation._action_task_id = ui.tasks[0].task_id + automation._action_stop = StopControl() + releases, reflected = [], Event() + automation.action_requested.disconnect(automation._admit_action) + def gui(request): + if request['release']: + releases.append(request) + else: + automation._admit_action(request) + automation.action_requested.connect(gui) + def action(): + with automation._action_scope('retry_failed'): + pass + reflected.set() + return {'status': 'pending_issue'} + automation._start_worker(ServiceWorker(action)) + try: + qtbot.waitUntil(lambda: bool(releases)) + qtbot.wait(5100) + assert not reflected.is_set(), 'Model work started before GUI acknowledged release' + finally: + automation._admit_action(releases[0]) + qtbot.waitUntil(lambda: automation.worker is None) diff --git a/tests/ui/test_dsh_operations.py b/tests/ui/test_dsh_operations.py index c5810f2..ffd77af 100644 --- a/tests/ui/test_dsh_operations.py +++ b/tests/ui/test_dsh_operations.py @@ -139,7 +139,7 @@ def test_last_execution_receipt_survives_later_catalog_reads(ui, monkeypatch, qt assert view.operations['last_execution']['tasks'][0]['no_match_items'] == 2 -def test_history_read_busy_state_is_published_and_rejection_is_visible(ui, monkeypatch, qtbot): +def test_history_read_busy_state_is_published_without_blocking_execution_confirmation(ui, monkeypatch, qtbot): from threading import Event view=setup_view(ui,monkeypatch); operations=ui.window._dsh_operations started,release=Event(),Event() @@ -152,8 +152,9 @@ def test_history_read_busy_state_is_published_and_rejection_is_visible(ui, monke assert view.operations['busy'] is True ui.window._dsh_lifecycle.request({'request_id':'b'*32,'capability':'execute', 'spec':{'operation':'start','task_ids':[ui.tasks[0].task_id]}}) - assert view.lifecycle_results[-1]['status']=='busy' - assert view.operations['activity']['message'] + qtbot.waitUntil(lambda: ui.window._dsh_lifecycle.dialog is not None) + ui.window._dsh_lifecycle.dialog.reject() + assert view.lifecycle_results[-1]['status']=='cancelled' assert not ui.calls finally: release.set() diff --git a/tests/ui/test_nonblocking_controls.py b/tests/ui/test_nonblocking_controls.py new file mode 100644 index 0000000..941208f --- /dev/null +++ b/tests/ui/test_nonblocking_controls.py @@ -0,0 +1,38 @@ +def test_owned_modal_can_be_handled_while_execution_worker_is_busy(ui, monkeypatch): + window = ui.window + ui.settle() + worker = object() + invoked = [] + monkeypatch.setattr('dx_productivity.trial_task_center.ui.cad_dialog_operations.show_cad_dialogs', + lambda current: invoked.append(current)) + window.service._supervisor_dialog_state = {'supervision': {'blocked': True}} + window._workers.add(worker) + try: + window._update_gates() + button = window.run_panel.cleanup_resources_button + assert not button.isHidden() and button.isEnabled() + button.click() + assert invoked == [window] + finally: + window._workers.discard(worker) + window.service._supervisor_dialog_state = None + window._update_gates() + + +def test_retained_resource_force_requires_explicit_operator_choice(ui, monkeypatch): + from PySide6.QtWidgets import QMessageBox + window = ui.window + ui.settle() + forced = [] + monkeypatch.setattr(window.service, 'cleanup_batch_resources', lambda: ({'state': 'blocked'},)) + monkeypatch.setattr(window.service, 'force_pending_owned_resources', + lambda *, confirmed=False: forced.append(confirmed) or ()) + monkeypatch.setattr(window, '_run_async', lambda action, done, *_a, **_k: done(action())) + monkeypatch.setattr(QMessageBox, 'question', lambda *_a, **_k: QMessageBox.StandardButton.No) + window.run_panel.cleanup_resources_button.setEnabled(True) + window._cleanup_batch_resources() + assert forced == [] + monkeypatch.setattr(QMessageBox, 'question', lambda *_a, **_k: QMessageBox.StandardButton.Yes) + window.run_panel.cleanup_resources_button.setEnabled(True) + window._cleanup_batch_resources() + assert forced == [True] diff --git a/tests/v1_1/test_accuracy_vbs_guards.py b/tests/v1_1/test_accuracy_vbs_guards.py index 1780254..7e729f6 100644 --- a/tests/v1_1/test_accuracy_vbs_guards.py +++ b/tests/v1_1/test_accuracy_vbs_guards.py @@ -35,9 +35,12 @@ def run_vbs(tmp_path, names, body, prelude=''): LOG_STUB = ''' -Dim QP_Root +Dim QP_Root, QP_Fso +Set QP_Fso=CreateObject("Scripting.FileSystemObject") Sub QPLog(ByVal Root, ByVal Level, ByVal Message) End Sub +Sub QPAppendAscii(ByVal Path, ByVal Text) +End Sub ''' @@ -377,6 +380,8 @@ Sub QPVerifyRestoredPlans(ByRef Command, ByRef Records, ByRef Plans, ByRef State End Sub Sub QPWriteStatus(ByVal Root, ByVal Job, ByVal State, ByVal Code, ByVal Message) End Sub +Sub QPWriteValueReadback(ByRef Command, ByRef Records, ByRef Plans, ByVal State) +End Sub ''') assert output == expected @@ -727,7 +732,9 @@ WScript.Echo CStr(InStr(Text, "G" & vbTab & "note" & vbTab) > 0) def test_capture_protects_unselected_and_does_not_hide_target_pn_error(tmp_path, read_failure, exact_scope): snapshot = str(tmp_path / 'captured.tsv') output = run_vbs(tmp_path, UNTOUCHED_ROUTINES + ['QPCaptureAndSelect', 'QPSnapshotComponent', - 'QPCommandTargets', 'QPBooleanText', 'QPLogField', 'QPTargetCell'], ''' + 'QPCommandTargets', 'QPBooleanText', 'QPLogField', 'QPTargetCell', + 'QPRequireUniqueAttributeNames', 'QPAttributeObservations', 'QPRequireAlreadyTargetPolicy', + 'QPValueHasNcMarker', 'QPAppendNcMarker'], ''' Dim Comp, Other, Attr, Pin, Command, Records Set QP_App = New FakeApp Set Comp = New FullComponent: Comp.UID = "u1": Comp.Refdes = "R1" diff --git a/tests/v1_1/test_batch_runtime.py b/tests/v1_1/test_batch_runtime.py index ca1fac0..f55348f 100644 --- a/tests/v1_1/test_batch_runtime.py +++ b/tests/v1_1/test_batch_runtime.py @@ -149,7 +149,7 @@ def worker_runtime(tmp_path, monkeypatch, monitor): return BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop, cad_factory=Cad) -def test_two_projects_reuse_cad_and_produce_distinct_close_checkpoints(tmp_path, monkeypatch): +def test_same_project_reuses_cad_then_next_project_starts_a_fresh_session(tmp_path, monkeypatch): import subprocess from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released one, _ = command_for(tmp_path) @@ -159,10 +159,15 @@ def test_two_projects_reuse_cad_and_produce_distinct_close_checkpoints(tmp_path, with runtime: runtime.run_worker(one, 5) assert project_checkpoint_released(Path(one[3])) + first = runtime.cad + runtime.run_worker(one, 5) + assert runtime.cad is first runtime.run_worker(two, 5) assert project_checkpoint_released(Path(two[3])) - assert len(Cad.created) == len(Desktop.created) == 1 - assert Cad.created[0].closed and Desktop.created[0].released + assert first.closed and Desktop.created[0].released + assert len(Cad.created) == len(Desktop.created) == 2 + assert all(cad.closed for cad in Cad.created) + assert all(desktop.released for desktop in Desktop.created) def test_completed_project_does_not_hold_resolution_wait_for_next_project_failure(tmp_path, monkeypatch): @@ -410,7 +415,7 @@ def test_native_rebuild_requires_strict_source_and_worker_proof(tmp_path): for path in (project, database)}} path = evidence / 'native-retry-proof.json' path.write_text(json.dumps(proof)) - with BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop) as runtime: + with BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop) as runtime: with runtime.native_scope(evidence) as scope: pass proof['binding'] = scope.identity @@ -475,17 +480,17 @@ def test_mock_different_projects_each_get_one_retry_without_accumulating_budget( calls[command[3]] += 1 if calls[command[3]] == 2: return subprocess.CompletedProcess(command, 0, '', '') - raise WorkerMonitorError('unchanged prewrite failure') + return subprocess.CompletedProcess(command, 2, '', 'unchanged prewrite failure') runtime = worker_runtime(tmp_path, monkeypatch, monitor) with runtime: runtime.run_worker(one, 5) assert len(Desktop.created) == 2 runtime.run_worker(two, 5) - assert len(Desktop.created) == 3 # Each project gets its own single rebuild. - for command in (one, two): + assert len(Desktop.created) == 4 # Separate project startup plus its single rebuild. + for expected_sessions, command in enumerate((one, two), 5): with pytest.raises(WorkerMonitorError, match='prewrite failure'): runtime.run_worker(command, 5) - assert len(Desktop.created) == 3 # Success cannot replenish either budget. + assert len(Desktop.created) == expected_sessions # Switching projects is not another retry. assert calls == {one[3]: 3, two[3]: 3} assert set(runtime.record['retries']) == { str(project_one.resolve()).casefold(), str(project_two.resolve()).casefold()} diff --git a/tests/v1_1/test_bom_actual_metadata.py b/tests/v1_1/test_bom_actual_metadata.py new file mode 100644 index 0000000..4723815 --- /dev/null +++ b/tests/v1_1/test_bom_actual_metadata.py @@ -0,0 +1,145 @@ +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from test_bom_batch import batch, records_for +from dx_productivity.trial_task_center.bom_import_report import inspect_batch +from dx_productivity.trial_task_center.bom_plan import write_bom_template +from dx_productivity.trial_task_center.errors import TaskCenterError +from dx_productivity.trial_task_center.inventory import HEADER, read_component_inventory +from dx_productivity.trial_task_center.store import TaskStore + + +def test_inventory_accepts_effective_metadata_and_legacy_unknown(tmp_path): + path = tmp_path / 'components.tsv' + project = tmp_path / 'board.prj' + row = f'{project}\tMAIN\t1\tu1\tR1\tNEW\t10nF' + path.write_text('\t'.join(HEADER) + '\n' + row + '\n') + legacy = read_component_inventory(path, project, (('MAIN', '1'),))[0] + assert legacy.actual_part_label is None and legacy.actual_cell is None + path.write_text('\t'.join((*HEADER, 'actual_part_label', 'actual_cell')) + '\n' + row + '\t10nF\tCELL\n') + actual = read_component_inventory(path, project, (('MAIN', '1'),))[0] + assert actual.actual_part_label == '10nF' and actual.actual_cell == 'CELL' + + +@pytest.mark.parametrize('mode,value,label,cell,expected', [ + ('custom', '10nF', 'LABEL', 'CELL', True), ('part_number', 'NEW-0', 'LABEL', 'CELL', True), + ('preserve', 'current', 'LABEL', 'CELL', True), ('part_label', 'LABEL', 'LABEL', 'CELL', True), + ('part_label', 'wrong', 'LABEL', 'CELL', False), ('part_label', 'LABEL', 'LABEL', 'WRONG', False), + ('part_label', '', '', 'CELL', False), +]) +def test_bom_already_target_verifies_effective_label_and_cell(batch, tmp_path, mode, value, label, cell, expected): + service, store, tasks, targets, calls = batch + target = replace(targets[tasks[0].task_id][0], old_part_number='NEW-0', old_value=value, + actual_part_label=label, actual_cell=cell) + targets[tasks[0].task_id] = (target,) + service.rules = lambda _task: (SimpleNamespace(old_part_number='OLD', new_part_number='NEW-0', + target_cell='CELL', enabled=True),) + record = dict(records_for(tasks, targets)[0], value_mode=mode, custom_value='10nF' if mode == 'custom' else '') + report = inspect_batch(service, tasks[0].task_id, write_bom_template(tmp_path / 'actual.xlsx', (record,)), + runtime=SimpleNamespace(bind_logs=lambda *_: None)) + assert bool(report.preview) is expected + if expected: + preview = report.preview + service.save_bom_batch(preview, {p.task_id: p.rows for p in preview.projects}) + assert not store.list_target_plan(tasks[0].task_id)[0].enabled + assert store.list_target_plan(tasks[0].task_id)[0].target.actual_cell == cell + else: + assert report.failed_rows == 1 and ('Value' in report.rows[0].reason or 'Cell' in report.rows[0].reason) + + +def test_bom_refreshes_unknown_effective_metadata_and_persists_without_changing_edits(batch, tmp_path): + service, store, tasks, targets, calls = batch + target = replace(targets[tasks[0].task_id][0], old_part_number='NEW-0', old_value='LABEL') + store.record_target_inventory(tasks[0].task_id, (target,)) + targets[tasks[0].task_id] = (replace(target, actual_part_label='LABEL', actual_cell='CELL'),) + record = dict(records_for(tasks, targets)[0], value_mode='part_label') + path = write_bom_template(tmp_path / 'refresh.xlsx', (record,)) + report = inspect_batch(service, tasks[0].task_id, path, runtime=SimpleNamespace(bind_logs=lambda *_: None)) + assert report.preview and len(calls) == 1 + assert store.list_target_plan(tasks[0].task_id)[0].target.actual_part_label is None + service.save_bom_batch(report.preview, {p.task_id: p.rows for p in report.preview.projects}) + assert store.list_target_plan(tasks[0].task_id)[0].target.actual_part_label == 'LABEL' + database = store._database_path if hasattr(store, '_database_path') else tmp_path / 'tasks.db' + reopened = TaskStore(database) + try: + assert reopened.list_target_plan(tasks[0].task_id)[0].target.actual_part_label == 'LABEL' + finally: + reopened.close() + + +def test_known_metadata_change_cannot_replace_inventory(batch): + service, store, tasks, targets, _ = batch + target = replace(targets[tasks[0].task_id][0], actual_part_label='A', actual_cell='CELL') + store.record_target_inventory(tasks[0].task_id, (target,)) + with pytest.raises(TaskCenterError, match='identity'): + store.record_target_inventory(tasks[0].task_id, (replace(target, actual_cell='FOREIGN'),)) + + +def test_official_effective_report_supplies_label_and_cell_without_claiming_layer(tmp_path): + from test_native_ascii_inventory import report, effective_report + from dx_productivity.trial_task_center.native_inventory_protocol import read_ascii_discovery + flat = effective_report(tmp_path) + flat.write_text(flat.read_text().replace('\tProp Value 0R/NC\n', + '\tProp Value 0R/NC\n\tProp Part Label 0R/NC\n\tProp Cell Name CELL\n')) + target = read_ascii_discovery(tmp_path / 'board.prj', report(tmp_path), ('NEW',), + effective_report_path=flat).components[0] + assert (target.actual_part_label, target.actual_cell) == ('0R/NC', 'CELL') + + +def test_legacy_database_migration_keeps_effective_fields_unknown(batch, tmp_path): + service, store, tasks, targets, _ = batch + store.record_target_inventory(tasks[0].task_id, (targets[tasks[0].task_id][0],)) + with store.transaction(): + store._connection.execute('ALTER TABLE target_plan DROP COLUMN actual_part_label') + store._connection.execute('ALTER TABLE target_plan DROP COLUMN actual_cell') + reopened = TaskStore(tmp_path / 'tasks.db') + try: + target = reopened.list_target_plan(tasks[0].task_id)[0].target + assert target.uid == 'u1' and target.actual_part_label is None and target.actual_cell is None + finally: + reopened.close() + + +def test_frozen_inventory_rejects_changed_or_missing_effective_evidence(batch): + from dx_productivity.trial_task_center.inventory import validate_target_inventory + service, store, tasks, targets, _ = batch + target = replace(targets[tasks[0].task_id][0], actual_cell='CELL') + rule = SimpleNamespace(targets=(target,)) + for cell in ('FOREIGN', None): + with pytest.raises(TaskCenterError): + validate_target_inventory(target.project_path, (rule,), (replace(target, actual_cell=cell),)) + + +@pytest.mark.parametrize('current', ['matches', 'changed_label', 'changed_cell', 'changed_value', + 'unknown_label', 'unknown_cell']) +def test_already_target_database_inventory_requires_current_source_evidence(batch, tmp_path, current): + service, store, tasks, targets, calls = batch + task_id = tasks[0].task_id + target = replace(targets[task_id][0], old_part_number='NEW-0', old_value='LABEL', + actual_part_label='LABEL', actual_cell='CELL') + store.record_target_inventory(task_id, (target,)) + service.rules = lambda _task: (SimpleNamespace(old_part_number='OLD', new_part_number='NEW-0', + target_cell='CELL', enabled=True),) + # The persisted database still matches the requested target while the actual + # source content may have changed since that observation was collected. + with target.project_path.open('a') as stream: + stream.write(f'\nCURRENT_SOURCE_STATE {current}\n') + def collect(project, parts, **controls): + calls.append((task_id, parts)) + source = project.read_text() + return (replace(target, + actual_part_label=None if 'unknown_label' in source else 'WRONG' if 'changed_label' in source else 'LABEL', + actual_cell=None if 'unknown_cell' in source else 'WRONG' if 'changed_cell' in source else 'CELL', + old_value='WRONG' if 'changed_value' in source else 'LABEL'),) + service.for_project_task(task_id)._quick_run_preparer.collect_inventory = collect + path = write_bom_template(tmp_path / 'stale-inventory.xlsx', + (dict(records_for(tasks, targets)[0], value_mode='part_label'),)) + report = inspect_batch(service, task_id, path, runtime=SimpleNamespace(bind_logs=lambda *_: None)) + assert len(calls) == 1 + if current == 'matches': + assert report.preview and not report.preview.projects[0].rows[0].enabled + else: + assert report.preview is None and report.passed_preview is None and report.failed_rows == 1 + assert store.list_target_plan(task_id)[0].target == target diff --git a/tests/v1_1/test_cad_dialog_operations.py b/tests/v1_1/test_cad_dialog_operations.py new file mode 100644 index 0000000..e91d05c --- /dev/null +++ b/tests/v1_1/test_cad_dialog_operations.py @@ -0,0 +1,26 @@ +"""Manual controls expose real prompt text and send only explicit selections.""" +from types import SimpleNamespace + +from PySide6.QtWidgets import QApplication, QWidget, QPushButton, QPlainTextEdit + + +def test_dialog_buttons_send_current_bound_manual_choice(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.ui import cad_dialog_operations + app = QApplication.instance() or QApplication([]) + window = QWidget() + row = {'pid': 20, 'hwnd': 30, 'title': 'Save?', 'text': 'Save changes to board?', 'complete': True, + 'buttons': [{'hwnd': 40, 'id': 6, 'text': 'Yes', 'enabled': True}]} + binding = {'batch_id': 'a' * 32, 'attempt_id': 'b' * 32, 'generation': 1, 'session_token': 'token'} + window.service = SimpleNamespace(_logging=SimpleNamespace(root=tmp_path), _supervisor_dialog_state={ + 'project': 'board.prj', 'binding': binding, 'supervision': {'blocked': True, 'current_dialogs': [row]}}) + sent = [] + monkeypatch.setattr(cad_dialog_operations, 'request_manual_dialog_action', + lambda *args: sent.append(args) or 'request') + dialog = cad_dialog_operations.CadDialogOperations(window) + assert sent == [] + assert 'Save changes to board?' in dialog.findChild(QPlainTextEdit).toPlainText() + button = next(button for button in dialog.findChildren(QPushButton) if button.text() == 'Yes') + button.click() + assert sent == [(tmp_path, binding, row, 40)] + dialog.close() + window.close() diff --git a/tests/v1_1/test_component_scope.py b/tests/v1_1/test_component_scope.py new file mode 100644 index 0000000..88bf63e --- /dev/null +++ b/tests/v1_1/test_component_scope.py @@ -0,0 +1,254 @@ +"""Freeze raw all-match rules into actual component identities before mutation.""" +import csv +from dataclasses import replace +from pathlib import Path +import subprocess + +import pytest + +from dx_productivity.trial_task_center.errors import TaskCenterError +from dx_productivity.trial_task_center.execution_journal import read_command_rows +from dx_productivity.trial_task_center.models import ProjectRecord, ProjectStatus, RuleRecord +from dx_productivity.trial_task_center.quick_run import ( + ProjectDiscovery, QuickRunPreparer, _path_key, _queue_matches_inputs, _read_commands, +) +from dx_productivity.trial_task_center.target_plan import ComponentTarget, TargetRule + + +@pytest.fixture +def scope_case(tmp_path, monkeypatch): + project = tmp_path / 'projects' / 'Board' / 'board.prj' + project.parent.mkdir(parents=True) + project.write_text('KEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n') + source = tmp_path / 'runtime.vbs' + source.write_text("' controlled runtime") + selected = ProjectRecord('p', 't', 'Board', project, project.parent, True, True, ProjectStatus.PENDING, 0) + rules = (RuleRecord('r', 't', 0, 'OLD', 'NEW', True),) + components = tuple(ComponentTarget(project, 'MAIN', '1', f'uid-{i}', f'R{i}', 'OLD', '10K') for i in (1, 2)) + calls, requested = [], [] + def worker(command, timeout): + root = Path(command[3]) + row = read_command_rows(root / 'current-command.tsv')[0] + assert row['target_selectors'], 'An empty inventory marker must never reach a mutating worker' + calls.append(row) + with (root / 'status.tsv').open('a', encoding='utf8') as stream: + stream.write(f'{command[-1]}\tawaiting_native_replace\t\tcaptured\n') + stream.write(f'{command[-1]}\tsaved_ok\tbatch_closed\tcontrolled save\n') + return subprocess.CompletedProcess(command, 0, '', '') + preparer = QuickRunPreparer(tmp_path / 'data', script_source=source, + process_runner=worker, cscript_resolver=lambda: Path('controlled-cscript'), run_token_factory=lambda: 'scope') + def discover(path, parts, *, collect_components=False, **kwargs): + requested.append(collect_components) + return ProjectDiscovery((('MAIN', '1'),), {('MAIN', '1'): frozenset({'old'})}, components) + monkeypatch.setattr(preparer, 'discover_project', discover) + return preparer, selected, rules, components, requested, calls + + +def test_new_raw_rule_freezes_two_equal_value_targets_as_separate_uids(scope_case): + preparer, project, rules, _, requested, _ = scope_case + handoff = preparer.ensure_batch_handoff('t', (project,), rules) + rows = read_command_rows(handoff.command_path) + assert requested == [True] + assert len(rows) == 2 + assert [row['target_selectors'] for row in rows] == ['uid-1\x1fR1\x1f10K', 'uid-2\x1fR2\x1f10K'] + assert all(row['target_scope'] == 'targets' and row['execution_unit'] == 'component' for row in rows) + assert handoff.command_path.with_name('component-scope.json').is_file() + assert _queue_matches_inputs(handoff.command_path, _read_commands(handoff.command_path), (project,), rules) + + +@pytest.mark.parametrize('change', [{'custom_value_enabled': True, 'custom_value': 'changed'}, {'sync_nc': False}, {'target_cell': 'QFN'}]) +def test_original_source_rule_option_changes_cannot_resume_frozen_scope(scope_case, change): + preparer, project, rules, *_ = scope_case + handoff = preparer.ensure_batch_handoff('t', (project,), rules) + changed = (replace(rules[0], **change),) + with pytest.raises(TaskCenterError, match='source|scope|来源|冻结'): + preparer.ensure_batch_handoff('t', (project,), changed) + + +def test_snapshot_and_resume_use_frozen_uids_without_rediscovering_changed_parts(scope_case, monkeypatch): + preparer, project, rules, *_ = scope_case + handoff = preparer.ensure_batch_handoff('t', (project,), rules) + rows = read_command_rows(handoff.command_path) + with handoff.status_path.open('a', encoding='utf8') as stream: + stream.write(f"{rows[0]['job_id']}\tsaved_ok\tbatch_closed\talready saved\n") + def forbidden(*args, **kwargs): + raise AssertionError('Completed targets now have new Part Numbers; do not rediscover them') + monkeypatch.setattr(preparer, 'discover_project', forbidden) + before = handoff.command_path.read_bytes() + assert preparer.ensure_batch_handoff('t', (project,), rules).command_path.read_bytes() == before + snapshot = preparer.read_snapshot('t', (project,), rules) + assert not snapshot.interrupted + assert snapshot.progress.total_items == 2 and snapshot.progress.completed_items == 1 + assert snapshot.progress.next_old_part_number == 'OLD' + + +def test_mixed_no_match_is_preterminal_and_other_uid_commands_remain_dispatchable(scope_case): + preparer, project, rules, _, _, calls = scope_case + missing = RuleRecord('missing', 't', 0, 'ABSENT', 'DEST', True) + session = preparer.begin_batch_session('t', (project,), (missing, *rules)) + rows = read_command_rows(session.handoff.command_path) + empty = rows[0] + assert empty['execution_unit'] == 'inventory_empty' + assert empty['target_scope'] == 'targets' and empty['target_selectors'] == '' + assert session.latest[empty['job_id']] == ('batch_skipped', 'inventory_empty') + assert session.commands[session.next_index][1] == rows[1]['job_id'] + # Without a journal the normal batch gate conservatively blocks after any + # failure; dispatch the pending current command to verify its actual scope. + from dx_productivity.trial_task_center.quick_run import _write_current_command + _write_current_command(session, session.commands[session.next_index]) + preparer._process_runner(('controlled', '', '', str(session.handoff.command_path.parent), rows[1]['job_id']), 10) + assert len(calls) == 1 and calls[0]['target_selectors'].startswith('uid-1\x1f') + progress = preparer.read_snapshot('t', (project,), (missing, *rules)).progress + assert progress.failed_items == 1 and progress.saved_items == 1 + + +def test_legacy_page_all_match_queue_resumes_without_new_expansion(scope_case, monkeypatch): + preparer, project, rules, *_ = scope_case + handoff = preparer.prepare('t', (project,), rules, sheet_targets={_path_key(project.path): (('MAIN', '1'),)}) + before = handoff.command_path.read_bytes() + monkeypatch.setattr(preparer, 'discover_project', lambda *args, **kwargs: pytest.fail('Legacy queue must keep its original shape')) + assert preparer.ensure_batch_handoff('t', (project,), rules).command_path.read_bytes() == before + assert 'target_scope' not in read_command_rows(handoff.command_path)[0] + assert not handoff.command_path.with_name('component-scope.json').exists() + + +def test_retry_preparation_inherits_frozen_scope_without_new_inventory(scope_case): + preparer, project, rules, *_ = scope_case + handoff = preparer.ensure_batch_handoff('t', (project,), rules) + staged = preparer.for_data_root(preparer.data_root / 'retry-stage') + retried = staged.prepare('t', (project,), rules, + sheet_targets={_path_key(project.path): (('MAIN', '1'),)}, component_scope_from=handoff.command_path.parent) + assert read_command_rows(retried.command_path) == read_command_rows(handoff.command_path) + assert retried.command_path.with_name('component-scope.json').read_bytes() == handoff.command_path.with_name('component-scope.json').read_bytes() + + +def test_incomplete_component_manifest_fails_closed_instead_of_all_match(scope_case, monkeypatch): + preparer, project, rules, *_ = scope_case + monkeypatch.setattr(preparer, 'discover_project', lambda *args, **kwargs: + ProjectDiscovery((('MAIN', '1'),), {('MAIN', '1'): frozenset({'old'})})) + with pytest.raises(TaskCenterError) as error: + preparer.ensure_batch_handoff('t', (project,), rules) + assert error.value.code == 'component_scope' + assert not (preparer.data_root / 'quick-run/command.tsv').exists() + + +def test_missing_terminal_marker_never_dispatches_empty_target_selectors(scope_case): + preparer, project, rules, _, _, calls = scope_case + missing = RuleRecord('missing', 't', 0, 'ABSENT', 'DEST', True) + handoff = preparer.ensure_batch_handoff('t', (project,), (missing, *rules)) + handoff.status_path.write_text('job_id\tstate\tcode\tmessage\n') + session = preparer.begin_batch_session('t', (project,), (missing, *rules)) + with pytest.raises(TaskCenterError) as error: + preparer.execute_step('t', (project,), (missing, *rules), batch_mode=True, batch_session=session) + assert error.value.code == 'component_scope' + assert not calls + + +def test_empty_command_proof_requires_the_exact_full_frozen_queue(scope_case): + import json + from dx_productivity.trial_task_center.component_scope import validate_empty_command + preparer, project, rules, *_ = scope_case + missing = RuleRecord('missing', 't', 0, 'ABSENT', 'DEST', True) + handoff = preparer.ensure_batch_handoff('t', (project,), (missing, *rules)) + empty = read_command_rows(handoff.command_path)[0] + assert validate_empty_command(handoff.command_path.parent, empty) + scope_path = handoff.command_path.with_name('component-scope.json') + scope = json.loads(scope_path.read_text()) + scope['projects'][0]['components'].pop() + scope_path.write_text(json.dumps(scope)) + assert not validate_empty_command(handoff.command_path.parent, empty) + + +def test_real_journal_keeps_empty_rule_manual_and_executes_other_exact_uids(scope_case, tmp_path, monkeypatch): + from threading import Event + from dx_productivity.trial_task_center.logging_setup import configure_logging + from dx_productivity.trial_task_center.service import TaskCenterService + from dx_productivity.trial_task_center.store import TaskStore + from dx_productivity.trial_task_center.task_queue import execute_task_queue + preparer, project, _, _, _, calls = scope_case + store = TaskStore(tmp_path / 'tasks.db') + service = TaskCenterService(store, configure_logging(preparer.data_root), quick_run_preparer=preparer) + try: + task = service.create_project_tasks(project.path.parent)[0] + service.add_rule(task.task_id, 'ABSENT', 'DEST') + service.add_rule(task.task_id, 'OLD', 'NEW') + child = service.for_project_task(task.task_id) + monkeypatch.setattr(child._quick_run_preparer, 'discover_project', preparer.discover_project) + result = execute_task_queue(service, (task.task_id,), stop_event=Event()) + assert not result.paused and result.entries[0].state == 'failed' + assert [row['target_selectors'] for row in calls] == ['uid-1\x1fR1\x1f10K', 'uid-2\x1fR2\x1f10K'] + progress = service.quick_run_progress(task.task_id) + assert progress.queue_complete and progress.failed_items == 1 and progress.saved_items == 2 + issues = service.execution_problems(task.task_id) + assert len(issues) == 1 and issues[0]['code'] == 'inventory_empty' and issues[0]['classification'] == 'manual' + assert service.reserve_problem_retry(task.task_id, source='dsh') == () + execute_task_queue(service, (task.task_id,), stop_event=Event(), queue_id=result.queue_id) + assert len(calls) == 2 + finally: + store.close() + + +def test_real_raw_retry_inherits_original_uids_and_does_not_repeat_success(scope_case, tmp_path, monkeypatch): + from threading import Event + from dx_productivity.trial_task_center.logging_setup import configure_logging + from dx_productivity.trial_task_center.service import TaskCenterService + from dx_productivity.trial_task_center.store import TaskStore + from dx_productivity.trial_task_center.task_queue import execute_task_queue + preparer, project, _, _, _, calls = scope_case + original_worker = preparer._process_runner + failed = False + def worker(command, timeout): + nonlocal failed + root = Path(command[3]) + row = read_command_rows(root / 'current-command.tsv')[0] + if row['target_selectors'].startswith('uid-1\x1f') and not failed: + failed = True + calls.append(row) + with (root / 'status.tsv').open('a', encoding='utf8') as stream: + stream.write(f'{command[-1]}\tbatch_failed\tcad_attachment_failed\tcontrolled prewrite failure\n') + return subprocess.CompletedProcess(command, 0, '', '') + return original_worker(command, timeout) + preparer._process_runner = worker + store = TaskStore(tmp_path / 'tasks.db') + service = TaskCenterService(store, configure_logging(preparer.data_root), quick_run_preparer=preparer) + try: + task = service.create_project_tasks(project.path.parent)[0] + service.add_rule(task.task_id, 'OLD', 'NEW') + child = service.for_project_task(task.task_id) + monkeypatch.setattr(child._quick_run_preparer, 'discover_project', preparer.discover_project) + result = execute_task_queue(service, (task.task_id,), stop_event=Event(), defer_problem_pass=True) + assert not result.paused and len(calls) == 2 + original_scope = (child._quick_run_preparer.data_root / 'quick-run/component-scope.json').read_bytes() + monkeypatch.setattr(child._quick_run_preparer, 'discover_project', lambda *args, **kwargs: pytest.fail('Retry must inherit frozen inventory')) + reserved = service.reserve_problem_retry(task.task_id, source='dsh') + assert len(reserved) == 1 + service.prepare_failed_quick_run(task.task_id, operation_keys=reserved) + assert (child._quick_run_preparer.data_root / 'quick-run/component-scope.json').read_bytes() == original_scope + retried = execute_task_queue(service, (task.task_id,), stop_event=Event()) + assert not retried.paused and retried.entries[0].state == 'completed' + assert [row['target_selectors'].split('\x1f')[0] for row in calls] == ['uid-1', 'uid-2', 'uid-1'] + finally: + store.close() + + +def test_retry_inherits_legacy_grouped_target_shape(scope_case): + preparer, project, _, components, *_ = scope_case + rules = (TargetRule('OLD', 'NEW', components),) + sheets = {_path_key(project.path): (('MAIN', '1'),)} + original = preparer.prepare('t', (project,), rules, sheet_targets=sheets, split_targets=False) + staged = preparer.for_data_root(preparer.data_root / 'retry-stage') + retried = staged.prepare('t', (project,), rules, sheet_targets=sheets, component_scope_from=original.command_path.parent) + assert read_command_rows(retried.command_path) == read_command_rows(original.command_path) + assert len(read_command_rows(retried.command_path)) == 1 + + +def test_retry_missing_frozen_sidecar_cannot_become_an_all_match_queue(scope_case): + preparer, project, rules, *_ = scope_case + original = preparer.ensure_batch_handoff('t', (project,), rules) + original.command_path.with_name('component-scope.json').unlink() + staged = preparer.for_data_root(preparer.data_root / 'retry-stage') + with pytest.raises(TaskCenterError) as error: + staged.prepare('t', (project,), rules, sheet_targets={_path_key(project.path): (('MAIN', '1'),)}, + component_scope_from=original.command_path.parent) + assert error.value.code == 'component_scope' + assert not (staged.data_root / 'quick-run/command.tsv').exists() diff --git a/tests/v1_1/test_deferred_problem_pass.py b/tests/v1_1/test_deferred_problem_pass.py new file mode 100644 index 0000000..d8244cb --- /dev/null +++ b/tests/v1_1/test_deferred_problem_pass.py @@ -0,0 +1,93 @@ +from threading import Event + +import pytest + +from test_nonblocking_items import component_case +from dx_productivity.trial_task_center.task_queue import execute_task_queue + + +def test_primary_all_projects_precedes_single_failed_uid_retry(component_case): + service, store, tasks, calls, outcomes = component_case + ids = tuple(task.task_id for task in tasks) + outcomes[(ids[0], ('R1',))] = ['attach_failed', 'saved_ok'] + result = execute_task_queue(service, ids, stop_event=Event()) + assert not result.paused + assert calls == [(ids[0], ('R1',)), (ids[0], ('R2',)), + (ids[1], ('R1',)), (ids[1], ('R2',)), (ids[0], ('R1',))] + assert [entry.state for entry in result.entries] == ['completed', 'completed'] + assert service.execution_problems(ids[0])[0]['auto_retries'] == 1 + assert service.execution_problems(ids[0])[0]['state'] == 'resolved' + + +def test_permanent_transient_failure_only_retries_once_across_queue_restart(component_case): + service, _, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = 'attach_failed' + result = execute_task_queue(service, (task_id,), stop_event=Event()) + execute_task_queue(service, (task_id,), stop_event=Event(), queue_id=result.queue_id) + assert calls == [(task_id, ('R1',)), (task_id, ('R2',)), (task_id, ('R1',))] + assert service.reserve_problem_retry(task_id, source='dsh') == () + + +def test_stop_on_last_primary_item_defers_problem_pass_until_resume(component_case): + service, _, tasks, calls, outcomes = component_case + ids = tuple(task.task_id for task in tasks) + stop = Event() + outcomes[(ids[0], ('R1',))] = ['attach_failed', 'saved_ok'] + outcomes['after_call'] = lambda count: stop.set() if count == 4 else None + result = execute_task_queue(service, ids, stop_event=stop) + assert len(calls) == 4 + assert service.execution_problems(ids[0])[0]['auto_retries'] == 0 + execute_task_queue(service, ids, stop_event=Event(), queue_id=result.queue_id) + assert len(calls) == 5 and calls[-1] == (ids[0], ('R1',)) + + +def test_dsh_and_problem_pass_cannot_each_reserve_one_replay(component_case): + service, _, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = 'attach_failed' + execute_task_queue(service, (task_id,), stop_event=Event(), defer_problem_pass=True) + keys = service.reserve_problem_retry(task_id, source='dsh') + assert len(keys) == 1 + assert service.reserve_problem_retry(task_id, source='problems') == () + assert service.reserve_problem_retry(task_id, operation_keys=keys, source='dsh') == () + + +def test_crash_after_reservation_resumes_same_allowance_without_success_replay(component_case, monkeypatch): + service, store, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = ['attach_failed', 'saved_ok'] + result = execute_task_queue(service, (task_id,), stop_event=Event(), defer_problem_pass=True) + child = service.for_project_task(task_id) + with monkeypatch.context() as patch: + patch.setattr(child, 'prepare_failed_quick_run', lambda *_a, **_k: (_ for _ in ()).throw(OSError('controlled crash before preparation'))) + with pytest.raises(OSError): + execute_task_queue(service, (task_id,), stop_event=Event(), queue_id=result.queue_id) + assert service.execution_problems(task_id)[0]['auto_retries'] == 1 + execute_task_queue(service, (task_id,), stop_event=Event(), queue_id=result.queue_id) + assert calls == [(task_id, ('R1',)), (task_id, ('R2',)), (task_id, ('R1',))] + assert service.execution_problems(task_id)[0]['auto_retries'] == 1 + + +def test_two_independent_connections_cannot_both_spend_same_uid_allowance(component_case, tmp_path): + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + from dx_productivity.trial_task_center.execution_problems import ExecutionProblems + from dx_productivity.trial_task_center.store import TaskStore + service, _, tasks, _, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = 'attach_failed' + execute_task_queue(service, (task_id,), stop_event=Event(), defer_problem_pass=True) + barrier = Barrier(2) + def reserve(source): + connection = TaskStore(tmp_path / 'tasks.db') + try: + barrier.wait(timeout=10) + return ExecutionProblems(connection).reserve(task_id, source=source) + finally: + connection.close() + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(reserve, source) for source in ('dsh', 'problems')] + reservations = [future.result(timeout=15) for future in futures] + assert sorted(map(len, reservations)) == [0, 1] + assert service.execution_problems(task_id)[0]['auto_retries'] == 1 diff --git a/tests/v1_1/test_diagnostics_bundle.py b/tests/v1_1/test_diagnostics_bundle.py index 5c92206..2e16d92 100644 --- a/tests/v1_1/test_diagnostics_bundle.py +++ b/tests/v1_1/test_diagnostics_bundle.py @@ -7,6 +7,31 @@ import pytest from dx_productivity.trial_task_center import diagnostics_bundle as bundle +def test_nonblocking_queue_and_owner_loss_evidence_are_exported_without_scope_inputs(tmp_path): + root = tmp_path / 'data' + queue = root / 'workspace-queues' + queue.mkdir(parents=True) + (queue / ('a' * 32 + '.json')).write_text(json.dumps({'phase': 'problems'})) + owner = root / 'batch-resources' / ('b' * 32) + owner.mkdir(parents=True) + (owner / 'owner-loss.json').write_text(json.dumps({'reason': 'parent_exited', 'token': 'private'})) + (owner / 'force-request.json').write_text(json.dumps({'confirmed': True})) + evidence = root / 'tasks' / 't' / 'quick-run' + evidence.mkdir(parents=True) + (evidence / 'project-read-release.json').write_text(json.dumps({'released': True})) + (evidence / ('retained-worker-' + 'c' * 32 + '.json')).write_text(json.dumps({'state': 'blocked'})) + (evidence / 'component-scope.json').write_text('private component inventory') + output = bundle.export_diagnostic_bundle(root, tmp_path / 'support.zip') + with zipfile.ZipFile(output) as archive: + assert 'workspace-queues/' + 'a' * 32 + '.json' in archive.namelist() + assert 'batch-resources/' + 'b' * 32 + '/owner-loss.json' in archive.namelist() + assert 'batch-resources/' + 'b' * 32 + '/force-request.json' in archive.namelist() + assert 'tasks/t/quick-run/retained-worker-' + 'c' * 32 + '.json' in archive.namelist() + assert 'tasks/t/quick-run/project-read-release.json' in archive.namelist() + assert 'tasks/t/quick-run/component-scope.json' not in archive.namelist() + assert b'private' not in archive.read('batch-resources/' + 'b' * 32 + '/owner-loss.json') + + def test_bundle_allowlist_redacts_secrets_and_home_and_omits_inputs(tmp_path): root = tmp_path / 'data' session = root / 'batch-resources' / 'batch1' diff --git a/tests/v1_1/test_inventory_extension.py b/tests/v1_1/test_inventory_extension.py index 9fb83fd..b947a6e 100644 --- a/tests/v1_1/test_inventory_extension.py +++ b/tests/v1_1/test_inventory_extension.py @@ -64,7 +64,7 @@ def test_bom_extension_stages_union_preserves_edits_and_saves(batch, tmp_path): service, store, tasks, targets, calls = batch before, path = extension_preview(batch, tmp_path) preview = preview_batch(service, tasks[0].task_id, path) - assert all(set(parts) == {'OLD', 'EXTRA'} for _, parts in calls) + assert all(set(parts) == {'OLD', 'EXTRA', 'NEW'} for _, parts in calls) assert len(calls) == 2 for project in preview.projects: assert store.list_target_plan(project.task_id) == before[project.task_id] diff --git a/tests/v1_1/test_manual_dialog_native.py b/tests/v1_1/test_manual_dialog_native.py new file mode 100644 index 0000000..b62886e --- /dev/null +++ b/tests/v1_1/test_manual_dialog_native.py @@ -0,0 +1,33 @@ +"""A disposable native popup, without CAD or input-desktop switching.""" +import os +import subprocess +import sys +import time +from uuid import uuid4 + +import pytest + + +@pytest.mark.skipif(os.name != 'nt', reason='Windows owned native dialog') +def test_explicit_no_button_closes_owned_unknown_popup_and_releases(tmp_path): + from dx_productivity.trial_task_center.cad_dialogs import CadDialogGuard + from dx_productivity.trial_task_center.isolated_desktop import PrivateDesktop + code = 'import ctypes; print(ctypes.windll.user32.MessageBoxW(None,"Save changes to sample?","Save Changes",4),flush=True)' + with PrivateDesktop('dx-' + uuid4().hex, enabled=True) as desktop: + process = desktop.start_process((sys.executable, '-c', code), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + guard = CadDialogGuard(desktop, process, tmp_path / 'cad-dialogs.json', 'manual-native-sample') + deadline = time.monotonic() + 10 + while not guard.summary()['blocked'] and time.monotonic() < deadline: + guard.poll('manual_sample') + time.sleep(.03) + dialogs = guard.summary()['current_dialogs'] + assert len(dialogs) == 1 and process.poll() is None + dialog = dialogs[0] + button = next(button for button in dialog['buttons'] if button['id'] == 7) + assert guard.records[-1]['action'] == 'manual_required' + guard.manual_action(dialog, button['hwnd']) + assert process.wait(timeout=5) == 0 + assert process.stdout.read().decode().strip() == '7' + assert guard.records[-1]['action'] == 'user_selected_button' + assert desktop.cleanup_record['desktop_confirmation']['released'] + assert not any(row.get('forced_termination') for row in desktop.cleanup_record.get('processes', ())) diff --git a/tests/v1_1/test_missing_uid_inventory.py b/tests/v1_1/test_missing_uid_inventory.py index 023072c..6a4f247 100644 --- a/tests/v1_1/test_missing_uid_inventory.py +++ b/tests/v1_1/test_missing_uid_inventory.py @@ -16,7 +16,8 @@ from test_accuracy_vbs_guards import run_vbs def test_inventory_accepts_missing_uid_only_with_unique_refdes(tmp_path, uid, refdes, duplicate, expected, broken_value): (tmp_path / 'component-inventory.request').touch() output = run_vbs(tmp_path, ['QPDiscoverInventory', 'QPFindCurrentAttribute', - 'QPEffectiveComponentAttribute'], f''' + 'QPEffectiveComponentAttribute', 'QPRequireUniqueAttributeNames', + 'QPAttributeObservations'], f''' Dim Documents, Comp, Other, Failure QP_Root="{tmp_path}" BrokenValue={str(broken_value)} @@ -47,6 +48,8 @@ Const VDM_COMP=128 Const VD_ALL=0 Sub QPLog(ByVal Root,ByVal Level,ByVal Message) End Sub +Sub QPAppendAscii(ByVal FilePath,ByVal Text) +End Sub Sub QPProgress(ByVal Stage,ByVal Done,ByVal Total) End Sub Function QPReadInventoryRequest() @@ -112,18 +115,28 @@ Class FakeDocument End Sub End Class Class FakeComponent - Public UID,Refdes + Public UID,Refdes,Attributes + Private Sub Class_Initialize() + Dim Attr,Name + Set Attributes=CreateObject("Scripting.Dictionary") + For Each Name In Array("Part Number","Part Label","Cell Name") + Set Attr=New FakeAttribute:Attr.Name=Name + If Name="Part Label" Then Attr.Value="LABEL" + If Name="Cell Name" Then Attr.Value="CELL" + Attributes.Add Attr,True + Next + End Sub Function FindAttribute(ByVal Name) + Dim Attr If Name="Value" And BrokenValue Then Err.Raise 1234,"FakeCOM","optional value unavailable" - If Name="Part Number" Then - Set FindAttribute=New FakeAttribute - Else - Set FindAttribute=Nothing - End If + Set FindAttribute=Nothing + For Each Attr In Attributes + If Attr.Name=Name Then Set FindAttribute=Attr + Next End Function End Class Class FakeAttribute - Public Value,InstanceValue + Public Name,Value,InstanceValue Private Sub Class_Initialize Value="OLD":InstanceValue="" End Sub @@ -133,3 +146,5 @@ End Class if expected == 'OK': identity = uid or '@REFDES|' + refdes assert output.splitlines()[-1].split('\t')[3:6] == [identity, refdes, 'OLD'] + assert output.splitlines()[1].split('\t')[7:] == ['actual_part_label', 'actual_cell'] + assert output.splitlines()[-1].split('\t')[7:] == ['LABEL', 'CELL'] diff --git a/tests/v1_1/test_native_snapshot_reader.py b/tests/v1_1/test_native_snapshot_reader.py index 3d58a37..45eac37 100644 --- a/tests/v1_1/test_native_snapshot_reader.py +++ b/tests/v1_1/test_native_snapshot_reader.py @@ -25,7 +25,7 @@ def run(tmp_path, body, *, prelude=''): source = ADAPTER.read_text(encoding='ascii') common = '\n'.join(guards.routine(n) for n in ('QPEncode', 'QPDecode', 'QPUntouchedIdentity', 'QPReadUtf8', 'QPWriteUtf8')) script = tmp_path / 'native-reader-test.vbs' - script.write_text(prelude + '\n' + common + '\n' + source + '\n' + body, encoding='ascii') + script.write_text('Dim QP_Fso\nSet QP_Fso=CreateObject("Scripting.FileSystemObject")\n' + prelude + '\n' + common + '\n' + source + '\n' + body, encoding='ascii') result = subprocess.run(['cscript.exe', '//nologo', str(script)], capture_output=True, text=True, timeout=15) assert result.returncode == 0, result.stdout + result.stderr return result.stdout.strip() diff --git a/tests/v1_1/test_nonblocking_acceptance.py b/tests/v1_1/test_nonblocking_acceptance.py new file mode 100644 index 0000000..24e7d4e --- /dev/null +++ b/tests/v1_1/test_nonblocking_acceptance.py @@ -0,0 +1,80 @@ +"""Full real scheduler/journal acceptance with a controlled native executor.""" +import csv +import subprocess +from collections import Counter +from pathlib import Path +from threading import Event + +from dx_productivity.trial_task_center.logging_setup import configure_logging +from dx_productivity.trial_task_center.quick_run import QuickRunPreparer, _decode_vbs_field +from dx_productivity.trial_task_center.service import TaskCenterService +from dx_productivity.trial_task_center.store import TaskStore +from dx_productivity.trial_task_center.target_plan import ComponentTarget, TargetPlanRow +from dx_productivity.trial_task_center.task_queue import execute_task_queue + + +def test_hundred_uid_mixed_batch_finishes_93_safe_items_without_replay(tmp_path): + calls = [] + outcomes = {} + def worker(command, timeout): + root = Path(command[3]) + with (root / 'current-command.tsv').open(encoding='utf-8-sig') as stream: + row = {key: _decode_vbs_field(value) for key, value in + next(csv.DictReader(stream, delimiter='\t')).items()} + uid, refdes, _ = row['target_selectors'].split('\x1f') + assert '\x1e' not in row['target_selectors'] + key = row['task_id'], uid + calls.append(key) + state = outcomes.get(key, 'saved_ok') + with (root / 'status.tsv').open('a', encoding='utf-8') as stream: + if state == 'saved_ok': + stream.write(f'{command[-1]}\tawaiting_native_replace\t\tcaptured\n') + stream.write(f'{command[-1]}\t{state}\tbatch_closed\tControlled {refdes}\n') + return subprocess.CompletedProcess(command, 0, '', '') + for name in ('A', 'B', 'C', 'D'): + project = tmp_path / 'projects' / name / 'board.prj' + project.parent.mkdir(parents=True) + project.write_text('KEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n') + data = tmp_path / 'data' + database = tmp_path / 'tasks.db' + store = TaskStore(database) + preparer = QuickRunPreparer(data, process_runner=worker, + cscript_resolver=lambda: Path('controlled-native'), sheet_discoverer=lambda _: (('MAIN', '1'),)) + engine = TaskCenterService(store, configure_logging(data), quick_run_preparer=preparer) + tasks = engine.create_project_tasks(tmp_path / 'projects') + counts = (48, 50, 1, 1) + for index, (task, count) in enumerate(zip(tasks, counts)): + project = engine.projects(task.task_id)[0].path + targets = tuple(ComponentTarget(project, 'MAIN', '1', f'u{i}', f'R{i}', 'OLD', '10K') + for i in range(1, count + 1)) + store.record_target_inventory(task.task_id, targets) + store.replace_target_plan(task.task_id, tuple(TargetPlanRow(target, True, 'NEW') for target in targets)) + if index == 0: + outcomes.update({(task.task_id, f'u{i}'): 'batch_failed' for i in range(1, 6)}) + elif index in (2, 3): + outcomes[(task.task_id, 'u1')] = 'batch_saved_unverified' + gui = TaskCenterService(store, configure_logging(data), supervised=True, quick_run_preparer=preparer) + # Only the process transport/native CAD are substituted. Each child executes + # the production queue, receipts, backups, issue pass and runtime cleanup. + def supervise(operation, arguments, *, stop_event): + assert operation == 'execute_task_queue' + return execute_task_queue(engine, stop_event=stop_event, **arguments) + gui._supervise = supervise + ids = tuple(task.task_id for task in tasks) + try: + result = execute_task_queue(gui, ids, stop_event=Event()) + assert not result.paused + assert [entry.state for entry in result.entries] == ['failed', 'completed', 'blocked', 'blocked'] + assert len(calls) == 100 and set(Counter(calls).values()) == {1} + records = dict(store._connection.execute('SELECT state,count(*) FROM execution_operations GROUP BY state')) + assert records == {'verified': 93, 'failed': 5, 'uncertain': 2} + execute_task_queue(gui, ids, stop_event=Event(), queue_id=result.queue_id) + assert len(calls) == 100 + finally: + store.close() + reopened = TaskStore(database) + try: + assert reopened._connection.execute("SELECT count(*) FROM execution_operations WHERE state='verified'").fetchone()[0] == 93 + assert reopened._connection.execute('SELECT sum(auto_retries) FROM execution_problems').fetchone()[0] == 0 + finally: + reopened.close() diff --git a/tests/v1_1/test_nonblocking_bom_values.py b/tests/v1_1/test_nonblocking_bom_values.py new file mode 100644 index 0000000..0bd6d76 --- /dev/null +++ b/tests/v1_1/test_nonblocking_bom_values.py @@ -0,0 +1,184 @@ +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from dx_productivity.trial_task_center.bom_plan import write_bom_template +from dx_productivity.trial_task_center.errors import TaskCenterError +from test_bom_batch import batch, records_for + + +def test_target_library_preflight_reads_target_even_when_cell_unspecified(tmp_path, monkeypatch): + from dx_productivity.trial_task_center import cell_catalog + project = tmp_path / 'board.prj' + project.write_text('KEY CentralLibrary "library/lib.lmc"\n') + reads = [] + def read(library, part, evidence_root, **options): + reads.append((library, part, options)) + return SimpleNamespace(cells=(cell_catalog.CellChoice('RES', 'CELL'),), source_fingerprint='proof') + monkeypatch.setattr(cell_catalog, 'read_library_cell_catalog', read) + rules = (SimpleNamespace(new_part_number='NEW', target_cell='', enabled=True),) + result = cell_catalog.preflight_target_libraries((project,), rules, tmp_path / 'proof') + assert reads[0][1] == 'NEW' and result + with pytest.raises(TaskCenterError, match='Cell'): + cell_catalog.preflight_target_libraries((project,), + (SimpleNamespace(new_part_number='NEW', target_cell='WRONG', enabled=True),), tmp_path / 'proof') + + +def test_execution_preflight_separates_safe_target_failure_from_unreleased_reader(tmp_path, monkeypatch): + from dx_productivity.trial_task_center import cell_catalog + row = {'project_path': str(tmp_path / 'board.prj'), 'new_part': 'NEW'} + def missing(*args, **kwargs): + raise TaskCenterError('cell_catalog', 'target part missing') + monkeypatch.setattr(cell_catalog, 'preflight_target_libraries', missing) + with pytest.raises(TaskCenterError) as failure: + cell_catalog.preflight_command_target(row, tmp_path / 'evidence') + assert failure.value.code == 'target_cell_catalog' + def retained(*args, **kwargs): + raise TaskCenterError('batch_resources_pending', 'private desktop not released') + monkeypatch.setattr(cell_catalog, 'preflight_target_libraries', retained) + with pytest.raises(TaskCenterError) as failure: + cell_catalog.preflight_command_target(row, tmp_path / 'evidence') + assert failure.value.code == 'batch_resources_pending' + + +def test_partial_import_requires_explicit_selection_and_keeps_failed_project(batch, tmp_path): + from dx_productivity.trial_task_center.bom_import_report import inspect_batch, select_passed_projects + service, store, tasks, targets, calls = batch + records = list(records_for(tasks, targets)) + records[1]['new_part_number'] = '' + path = write_bom_template(tmp_path / 'partial.xlsx', records) + runtime = SimpleNamespace(bind_logs=lambda *_: None) + atomic = inspect_batch(service, tasks[0].task_id, path, runtime=runtime) + assert atomic.preview is None and not calls + report = inspect_batch(service, tasks[0].task_id, path, runtime=runtime, allow_partial=True) + assert report.preview is None and report.failed_rows == 1 and report.passed_rows == 1 + assert not store.list_target_plan(tasks[0].task_id) + with pytest.raises(TaskCenterError): + select_passed_projects(report, (tasks[1].task_id,)) + preview = select_passed_projects(report, (tasks[0].task_id,)) + service.save_bom_batch(preview, {p.task_id: p.rows for p in preview.projects}) + assert store.list_target_plan(tasks[0].task_id)[0].enabled + assert not store.list_target_plan(tasks[1].task_id) + + +def test_partial_selection_rejects_changed_source_and_edited_rows(batch, tmp_path): + from dx_productivity.trial_task_center.bom_import_report import inspect_batch, select_passed_projects + service, store, tasks, targets, _ = batch + records = list(records_for(tasks, targets)) + records[1]['refdes'] = 'R999' + path = write_bom_template(tmp_path / 'partial.xlsx', records) + report = inspect_batch(service, tasks[0].task_id, path, + runtime=SimpleNamespace(bind_logs=lambda *_: None), allow_partial=True) + preview = select_passed_projects(report, (tasks[0].task_id,)) + project = preview.projects[0] + edited = (replace(project.rows[0], new_part_number='FOREIGN'), *project.rows[1:]) + with pytest.raises(TaskCenterError, match='预览'): + service.save_bom_batch(preview, {project.task_id: edited}) + path.write_bytes(path.read_bytes() + b'changed') + with pytest.raises(TaskCenterError, match='来源'): + service.save_bom_batch(preview, {project.task_id: project.rows}) + assert all(not store.list_target_plan(task.task_id) for task in tasks) + + +def test_partial_ui_stages_only_user_selected_passed_projects(batch, tmp_path, qtbot): + from dx_productivity.trial_task_center.bom_import_report import inspect_batch + from dx_productivity.trial_task_center.ui.main_window import TrialTaskCenterWindow + service, store, tasks, targets, _ = batch + records = list(records_for(tasks, targets)) + records[1]['refdes'] = 'R999' + report = inspect_batch(service, tasks[0].task_id, write_bom_template(tmp_path / 'ui.xlsx', records), + runtime=SimpleNamespace(bind_logs=lambda *_: None), allow_partial=True) + window = TrialTaskCenterWindow(service) + qtbot.addWidget(window) + window.open_task(tasks[0].task_id) + qtbot.waitUntil(lambda: not window._loading and not window._context_query.busy) + panel = window.plan_panel + panel._received_bom_report(report) + assert not panel._bom_batches + panel.accept_passed_projects((tasks[0].task_id,)) + assert panel._drafts[tasks[0].task_id][0].enabled + assert tasks[1].task_id not in panel._drafts + assert panel.last_bom_report.failed_rows == 1 + assert not store.list_target_plan(tasks[0].task_id) + panel.discard_all() + qtbot.waitUntil(lambda: not window._workers and not window._context_query.busy) + window.close() + + +@pytest.mark.parametrize('mode,value,passed', [('custom', '10nF', True), ('custom', 'wrong', False), + ('part_number', 'NEW-0', True), ('part_label', '10nF', False)]) +def test_bom_recognizes_actual_target_pn_only_when_value_policy_proven(batch, tmp_path, mode, value, passed): + from dx_productivity.trial_task_center.bom_import_report import inspect_batch + service, store, tasks, targets, calls = batch + target = replace(targets[tasks[0].task_id][0], old_part_number='NEW-0', old_value=value) + targets[tasks[0].task_id] = (target,) + record = dict(records_for(tasks, targets)[0], value_mode=mode, custom_value='10nF' if mode == 'custom' else '') + report = inspect_batch(service, tasks[0].task_id, write_bom_template(tmp_path / 'target.xlsx', (record,)), + runtime=SimpleNamespace(bind_logs=lambda *_: None)) + if passed: + assert report.preview and report.passed_rows == 1 + assert not report.preview.projects[0].rows[0].enabled + assert '目标料号' in report.rows[0].reason + else: + assert report.failed_rows == 1 and report.preview is None + assert 'Value' in report.rows[0].reason + assert 'NEW-0' in calls[0][1] + + +def test_partial_migrated_source_binding_survives_normalized_copy_removal(batch, tmp_path): + from dx_productivity.trial_task_center.bom_import_report import select_passed_projects + service, store, tasks, targets, _ = batch + records = list(records_for(tasks, targets)) + records[1]['refdes'] = 'R999' + sources = tuple(write_bom_template(tmp_path / f'source-{i}.xlsx', (record,)) for i, record in enumerate(records)) + report = service.preview_bom_sources(tasks[0].task_id, sources, + runtime=SimpleNamespace(bind_logs=lambda *_: None), allow_partial=True) + assert not report.source_map_path.with_name('normalized-bom.xlsx').exists() + assert [row.source_row for row in report.rows] == [2, 2] + preview = select_passed_projects(report, (tasks[0].task_id,)) + assert set(path for path, _ in preview.projects[0].source_hashes) == { + *(str(path) for path in sources), str(report.source_map_path)} + service.save_bom_batch(preview, {p.task_id: p.rows for p in preview.projects}) + assert store.list_target_plan(tasks[0].task_id)[0].enabled + assert not store.list_target_plan(tasks[1].task_id) + + +def test_already_target_bom_does_not_treat_planned_cell_as_actual_evidence(batch, tmp_path): + from dx_productivity.trial_task_center.bom_import_report import inspect_batch + service, store, tasks, targets, _ = batch + target = replace(targets[tasks[0].task_id][0], old_part_number='NEW-0', old_value='10nF') + targets[tasks[0].task_id] = (target,) + store.record_target_inventory(tasks[0].task_id, (target,)) + original = store.list_target_plan(tasks[0].task_id)[0] + store.replace_target_plan(tasks[0].task_id, (replace(original, new_part_number='NEW-0', target_cell='CELL'),)) + record = dict(records_for(tasks, targets)[0], value_mode='custom', custom_value='10nF') + report = inspect_batch(service, tasks[0].task_id, write_bom_template(tmp_path / 'target-cell.xlsx', (record,)), + runtime=SimpleNamespace(bind_logs=lambda *_: None)) + assert report.preview is None and report.failed_rows == 1 + assert '实际 Cell' in report.rows[0].reason + + +@pytest.mark.parametrize('fault', ['formula', 'numeric_pn', 'value_mode', 'extra_formula']) +def test_explicit_partial_source_import_keeps_invalid_project_rows_in_report(batch, tmp_path, fault): + from openpyxl import load_workbook + from dx_productivity.trial_task_center.bom_import_report import select_passed_projects + service, store, tasks, targets, calls = batch + records = list(records_for(tasks, targets)) + paths = tuple(write_bom_template(tmp_path / f'input-{i}.xlsx', (record,), include_value_mode=True) + for i, record in enumerate(records)) + book = load_workbook(paths[1]) + book.active[{'formula': 'D2', 'numeric_pn': 'D2', 'value_mode': 'L2', 'extra_formula': 'M2'}[fault]] = { + 'formula': '=1+1', 'numeric_pn': 123, 'value_mode': 'invalid', 'extra_formula': '=1+1'}[fault] + book.save(paths[1]) + book.close() + runtime = SimpleNamespace(bind_logs=lambda *_: None) + with pytest.raises(TaskCenterError): + service.preview_bom_sources(tasks[0].task_id, paths, runtime=runtime) + assert not calls + report = service.preview_bom_sources(tasks[0].task_id, paths, runtime=runtime, allow_partial=True) + assert report.preview is None and report.failed_rows == 1 and report.passed_rows == 1 + assert report.rows[1].source_file == str(paths[1]) and report.rows[1].source_row == 2 + preview = select_passed_projects(report, (tasks[0].task_id,)) + service.save_bom_batch(preview, {p.task_id: p.rows for p in preview.projects}) + assert not store.list_target_plan(tasks[1].task_id) diff --git a/tests/v1_1/test_nonblocking_items.py b/tests/v1_1/test_nonblocking_items.py new file mode 100644 index 0000000..88c1076 --- /dev/null +++ b/tests/v1_1/test_nonblocking_items.py @@ -0,0 +1,145 @@ +"""Actual command, journal and queue behavior at a controlled native boundary.""" +import csv +import subprocess +from dataclasses import replace +from pathlib import Path +from threading import Event + +import pytest + +from dx_productivity.trial_task_center.errors import TaskCenterError +from dx_productivity.trial_task_center.execution_journal import read_command_rows +from dx_productivity.trial_task_center.logging_setup import configure_logging +from dx_productivity.trial_task_center.quick_run import QuickRunPreparer, _decode_vbs_field +from dx_productivity.trial_task_center.service import TaskCenterService +from dx_productivity.trial_task_center.store import TaskStore +from dx_productivity.trial_task_center.target_plan import ComponentTarget, TargetPlanRow +from dx_productivity.trial_task_center.task_queue import execute_task_queue + + +@pytest.fixture +def component_case(tmp_path): + calls = [] + outcomes = {} + def worker(command, timeout): + root = Path(command[3]) + with (root / 'current-command.tsv').open(encoding='utf-8-sig') as stream: + row = {key: _decode_vbs_field(value) for key, value in + next(csv.DictReader(stream, delimiter='\t')).items()} + selectors = row['target_selectors'].split('\x1e') + refdes = tuple(selector.split('\x1f')[1] for selector in selectors) + calls.append((row['task_id'], refdes)) + result = outcomes.get((row['task_id'], refdes), 'saved_ok') + if isinstance(result, list): + result = result.pop(0) if len(result) > 1 else result[0] + if result == 'attach_failed': + return subprocess.CompletedProcess(command, 2, '', 'controlled attachment failure') + with (root / 'status.tsv').open('a', encoding='utf-8') as stream: + if result == 'saved_ok': + stream.write(f'{command[-1]}\tawaiting_native_replace\t\tcaptured\n') + stream.write(f'{command[-1]}\t{result}\tbatch_closed\tControlled {result}\n') + if outcomes.get('after_call'): + outcomes['after_call'](len(calls)) + return subprocess.CompletedProcess(command, 0, '', '') + for name in ('A', 'B'): + project = tmp_path / 'projects' / name / 'board.prj' + project.parent.mkdir(parents=True) + project.write_text('KEY FrontEndSnapshot "DxD"\nKEY iCDBDir "database"\n') + store = TaskStore(tmp_path / 'tasks.db') + service = TaskCenterService(store, configure_logging(tmp_path / 'data'), + quick_run_preparer=QuickRunPreparer(tmp_path / 'data', process_runner=worker, + cscript_resolver=lambda: Path('controlled-cscript'), + sheet_discoverer=lambda _: (('MAIN', '1'),))) + tasks = service.create_project_tasks(tmp_path / 'projects') + for task in tasks: + project = service.projects(task.task_id)[0].path + targets = tuple(ComponentTarget(project, 'MAIN', '1', f'u{i}', f'R{i}', 'OLD', '10K') + for i in (1, 2)) + store.record_target_inventory(task.task_id, targets) + store.replace_target_plan(task.task_id, tuple(TargetPlanRow(t, True, 'NEW') for t in targets)) + yield service, store, tasks, calls, outcomes + store.close() + + +def test_same_rule_same_value_produces_distinct_single_uid_commands(component_case): + service, _, tasks, calls, _ = component_case + result = execute_task_queue(service, (tasks[0].task_id,), stop_event=Event()) + assert not result.paused + assert [refs for _, refs in calls] == [('R1',), ('R2',)] + rows = read_command_rows(service.for_project_task(tasks[0].task_id)._quick_run_preparer.data_root / + 'quick-run' / 'command.tsv') + assert [row['execution_unit'] for row in rows] == ['component', 'component'] + + +def test_safe_component_failure_continues_independent_component_and_project(component_case): + service, store, tasks, calls, outcomes = component_case + ids = tuple(t.task_id for t in tasks) + outcomes[(ids[0], ('R1',))] = 'batch_failed' + result = execute_task_queue(service, ids, stop_event=Event()) + assert not result.paused + assert [entry.state for entry in result.entries] == ['failed', 'completed'] + assert calls == [(ids[0], ('R1',)), (ids[0], ('R2',)), (ids[1], ('R1',)), (ids[1], ('R2',))] + execute_task_queue(service, ids, stop_event=Event(), queue_id=result.queue_id) + assert len(calls) == 4 + assert store._connection.execute("SELECT count(*) FROM execution_operations WHERE state='verified'").fetchone()[0] == 3 + + +def test_uncertain_component_never_runs_same_project_remainder(component_case): + service, _, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = 'batch_saved_unverified' + result = execute_task_queue(service, (task_id,), stop_event=Event()) + assert result.paused + assert calls == [(task_id, ('R1',))] + + +def test_problem_retry_budget_survives_rebinding_and_is_shared(component_case): + service, _, tasks, _, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = 'batch_failed' + execute_task_queue(service, (task_id,), stop_event=Event()) + # Unknown native failures are visible but cannot obtain automatic replay. + assert service.reserve_problem_retry(task_id, source='dsh') == () + issues = service.execution_problems(task_id) + assert len(issues) == 1 and issues[0]['classification'] == 'manual' + + +def test_old_group_receipt_cannot_be_split_into_new_work(component_case): + service, _, tasks, _, _ = component_case + child = service.for_project_task(tasks[0].task_id) + task = tasks[0] + preparer = child._quick_run_preparer + handoff = preparer.prepare(task.task_id, child.projects(task.task_id), child._execution_rules(task.task_id), + sheet_targets={str(child.projects(task.task_id)[0].path).casefold(): (('MAIN', '1'),)}, + split_targets=False) + rows = read_command_rows(handoff.command_path) + journal = preparer.journal + journal.bind(task.task_id, rows, {}) + journal.finish(journal.claim(task.task_id, rows[0]), 'saved_ok', 'batch_closed', 'legacy group') + split = dict(rows[0], target_selectors='u1\x1fR1\x1f10K', execution_unit='component') + with pytest.raises(TaskCenterError): + journal.bind(task.task_id, (split,), {}) + + +def test_unresolved_attempt_blocks_another_uid_before_any_failure(component_case): + service, _, tasks, _, _ = component_case + child = service.for_project_task(tasks[0].task_id) + handoff = child._quick_run_preparer.ensure_batch_handoff(tasks[0].task_id, + child.projects(tasks[0].task_id), child._execution_rules(tasks[0].task_id)) + rows = read_command_rows(handoff.command_path) + journal = child._quick_run_preparer.journal + journal.bind(tasks[0].task_id, rows, {}) + journal.claim(tasks[0].task_id, rows[0]) + with pytest.raises(TaskCenterError, match='quick_run_uncertain'): + journal.claim(tasks[0].task_id, rows[1]) + + +def test_overlapping_component_commands_rejected_before_first_dispatch(component_case): + service, _, tasks, _, _ = component_case + child = service.for_project_task(tasks[0].task_id) + handoff = child._quick_run_preparer.ensure_batch_handoff(tasks[0].task_id, + child.projects(tasks[0].task_id), child._execution_rules(tasks[0].task_id)) + first = read_command_rows(handoff.command_path)[0] + overlap = dict(first, job_id='overlap', new_part='OTHER') + with pytest.raises(TaskCenterError, match='quick_run_uncertain'): + child._quick_run_preparer.journal.bind(tasks[0].task_id, (first, overlap), {}) diff --git a/tests/v1_1/test_nonblocking_resources.py b/tests/v1_1/test_nonblocking_resources.py new file mode 100644 index 0000000..02789db --- /dev/null +++ b/tests/v1_1/test_nonblocking_resources.py @@ -0,0 +1,574 @@ +"""Resource release proof, never elapsed time, admits later project work.""" +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from dx_productivity.trial_task_center import batch_resources, batch_runtime +from dx_productivity.trial_task_center.errors import TaskCenterError + + +def record_at(root, **values): + session = 'a' * 32 + path = root / 'batch-resources' / session / 'session.json' + record = dict(version=1, session_id=session, kind='execution', generation=1, + desktop='dx-' + 'b' * 32, state='blocked', processes=[], + cad_pid=500, cad_guid='owned', checkpoints={}, retries={}, timings=[]) + record.update(values) + batch_resources.write_record(path, record) + return path + + +def test_restart_reconciliation_checks_cad_identity_even_if_worker_list_empty(tmp_path, monkeypatch): + path = record_at(tmp_path, cad_process={'pid': 500, 'creation_time': 'original'}) + seen = [] + monkeypatch.setattr(batch_resources, '_process_gone', lambda item: seen.append(item) or False) + from dx_productivity.trial_task_center import isolated_desktop + monkeypatch.setattr(isolated_desktop, '_Win32Api', lambda: SimpleNamespace(confirm_desktop_released=lambda _: None)) + assert batch_resources.cleanup_pending_resources(tmp_path) + assert seen == [{'pid': 500, 'creation_time': 'original'}] + assert json.loads(path.read_text())['state'] != 'released' + + +def test_legacy_cad_record_without_process_identity_cannot_be_released(tmp_path, monkeypatch): + record_at(tmp_path) + from dx_productivity.trial_task_center import isolated_desktop + monkeypatch.setattr(isolated_desktop, '_Win32Api', lambda: SimpleNamespace(confirm_desktop_released=lambda _: None)) + monkeypatch.setattr(batch_resources, '_process_gone', lambda item: True) + assert batch_resources.cleanup_pending_resources(tmp_path) + + +def test_released_record_without_release_evidence_stays_pending(tmp_path): + record_at(tmp_path, state='released') + assert batch_resources.pending_resources(tmp_path) + + +def test_metadata_only_change_reuses_inventory_but_content_edit_does_not(tmp_path): + from dx_productivity.trial_task_center.discovery_cache import DiscoveryCache + from dx_productivity.trial_task_center.quick_run_protocol import ProjectDiscovery + project = tmp_path / 'project' / 'test.prj' + project.parent.mkdir() + project.write_text('project') + calls = [] + def load(): + calls.append(1) + return ProjectDiscovery((('MAIN', '1'),), {('MAIN', '1'): frozenset({'old'})}) + cache = DiscoveryCache(tmp_path / 'data') + cache.discover(project, ('old',), False, load) + info = project.stat() + os.utime(project, ns=(info.st_atime_ns, info.st_mtime_ns + 1000000000)) + cache.discover(project, ('old',), False, load) + assert len(calls) == 1 + project.write_text('changed') + cache.discover(project, ('old',), False, load) + assert len(calls) == 2 + + +def test_runtime_uncertainty_wait_has_owned_exit_observer(tmp_path, monkeypatch): + from dx_productivity.trial_task_center import stop_control, batch_attempt + runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', stop_control.StopControl()) + process = SimpleNamespace(poll=lambda: 0) + runtime.desktop = SimpleNamespace(processes=[process]) + runtime.cad = SimpleNamespace(cad=process) + monkeypatch.setattr(batch_attempt, 'can_retry_attempt', lambda *args: False) + captured = {} + monkeypatch.setattr(stop_control, 'wait_for_resolution', lambda *args, **kw: captured.update(kw)) + runtime._retain_uncertain(tmp_path, None, [('attempt', {})], RuntimeError('lost write')) + assert captured.get('resolved') is not None + assert captured['resolved']() is True + process.poll = lambda: None + assert captured['resolved']() is False + + +def test_supervisor_heartbeats_do_not_reset_business_stall_clock(): + from dx_productivity.trial_task_center.batch_supervisor import BusinessProgressWatch + watch = BusinessProgressWatch(0, idle_timeout=10) + watch.observe('state', {'phase': 'ready', 'queue': None, 'batch': None}, 1) + for now in range(2, 12): + watch.observe('heartbeat', None, now) + watch.observe('state', {'phase': 'ready', 'queue': None, 'batch': None}, now) + assert watch.stalled(11) + watch.observe('progress', 'R2 verified', 12) + assert not watch.stalled(12) + + +def test_runtime_read_only_policy_comes_from_operation_provenance(tmp_path): + assert batch_runtime.BatchCadRuntime(tmp_path, 'bom_import').read_only + assert batch_runtime.BatchCadRuntime(tmp_path, 'collect_target_inventory').read_only + assert not batch_runtime.BatchCadRuntime(tmp_path, 'execute_quick_run').read_only + + +def test_safe_failure_reset_refuses_absent_checkpoint(tmp_path): + runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution') + runtime.desktop = SimpleNamespace(close=lambda: pytest.fail('no release without proof')) + runtime.cad = object() + with pytest.raises(TaskCenterError, match='checkpoint|检查点|证明'): + runtime.reset_project_after_safe_failure(tmp_path / 'board.prj', tmp_path / 'evidence') + + +def test_unknown_dialog_manual_action_revalidates_full_fingerprint(monkeypatch): + from dx_productivity.trial_task_center import cad_dialogs + row = {'pid': 20, 'hwnd': 30, 'class': '#32770', 'title': 'Save?', 'text': 'Save changes?', + 'complete': True, 'buttons': [{'hwnd': 40, 'id': 6, 'text': 'Yes', 'enabled': True}]} + actions = [] + import win32gui + monkeypatch.setattr(win32gui, 'PostMessage', lambda *args: actions.append(args)) + monkeypatch.setattr(cad_dialogs, 'inspect_owned_dialogs', lambda *args: (dict(row, text='Other prompt'),)) + with pytest.raises(OSError): + cad_dialogs.manually_act_on_owned_dialog(None, 'dx-owned', 20, row, 40) + assert actions == [] + monkeypatch.setattr(cad_dialogs, 'inspect_owned_dialogs', lambda *args: (row,)) + cad_dialogs.manually_act_on_owned_dialog(None, 'dx-owned', 20, row, 40) + assert actions == [(40, 0x00F5, 0, 0)] + + +def test_read_only_cad_cleanup_releases_owned_handles_without_write_uncertainty(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.isolated_cad import IsolatedCadSession + from dx_productivity.trial_task_center.stop_control import StopControl + process = SimpleNamespace(pid=50, poll=lambda: None) + def close(): + process.poll = lambda: 1 + desktop = SimpleNamespace(name='dx-owned', close=close, cleanup_record={ + 'desktop_handle_closed': True, 'desktop_confirmation': {'released': True}, 'errors': []}) + session = IsolatedCadSession(tmp_path, desktop=desktop, read_only=True, stop_control=StopControl()) + session.cad = process + session.binding = SimpleNamespace(environment=lambda: {}) + desktop.start_process = lambda *a, **k: (_ for _ in ()).throw(RuntimeError('read still has a project open')) + session.__exit__(None, None, None) + assert process.poll() is not None + assert session.record['read_only_released'] is True + assert not (tmp_path / 'uncertain.flag').exists() + + +def test_read_only_worker_failure_releases_entire_generation_once(tmp_path, monkeypatch): + import subprocess + from test_batch_runtime import Cad, Desktop, command_for + from dx_productivity.trial_task_center import worker_monitor + from dx_productivity.trial_task_center.project_discovery import ReleasedProjectReadError + command, _ = command_for(tmp_path) + calls = [] + def monitor(*args, **kwargs): + calls.append(kwargs) + raise subprocess.TimeoutExpired('read', 1) + monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor) + runtime = batch_runtime.BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop, + cad_factory=lambda *args, read_only=False, **kwargs: Cad(*args, **kwargs)) + with runtime: + with pytest.raises(ReleasedProjectReadError): + runtime.run_worker(command, 1) + assert runtime.desktop is runtime.cad is None + assert runtime.record['state'] == 'released' + assert len(calls) == 1 and calls[0]['cooperative_abort'] + assert json.loads((Path(command[3]) / 'project-read-release.json').read_text())['resources_released'] + + +def test_new_project_closes_previous_cad_before_starting_next(tmp_path, monkeypatch): + import subprocess + from test_batch_runtime import Cad, Desktop, command_for, worker_runtime + first, _ = command_for(tmp_path) + second, _ = command_for(tmp_path, 'two') + runtime = worker_runtime(tmp_path, monkeypatch, lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', '')) + with runtime: + runtime.run_worker(first, 1) + previous = runtime.cad + runtime.run_worker(second, 1) + assert previous.closed + assert runtime.cad is not previous + assert runtime.generation == 2 + + +def test_owner_loss_requires_bound_parent_identity_and_records_read_policy(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.read_recovery import check_owner_loss + owner = {'pid': 40, 'creation_time': 'parent-created'} + binding = {'batch_id': 'a' * 32, 'attempt_id': 'b' * 32, 'parent_process': owner} + seen = [] + monkeypatch.setattr(batch_resources, '_process_gone', lambda item: seen.append(item) or True) + assert check_owner_loss(tmp_path, binding, read_only=True) + proof = json.loads((tmp_path / 'owner-loss.json').read_text()) + assert proof['parent_process'] == owner and proof['read_only'] is True + assert seen == [owner] + with pytest.raises(ValueError): + check_owner_loss(tmp_path, dict(binding, parent_process={'pid': 40}), read_only=True) + + +def test_unsafe_worker_exception_wait_resolves_when_owned_worker_exits(tmp_path, monkeypatch): + import io + import subprocess + from dx_productivity.trial_task_center import worker_monitor, stop_control + from dx_productivity.trial_task_center.stop_control import StopControl + process = SimpleNamespace(pid=123, stdout=io.BytesIO(), stderr=io.BytesIO(), returncode=None, + poll=lambda: None, close=lambda: None) + class Owned: + def __init__(self, *args): pass + def close(self): pass + def stop_workers(self, _): return [] + monkeypatch.setattr(worker_monitor, '_OwnedWorkers', Owned) + real_atomic = worker_monitor._atomic_text + def fail_ready(path, text): + if path.suffix == '.ready': + # An actual launched worker can already be inside an unsafe stage. + raise OSError('ready evidence locked') + return real_atomic(path, text) + monkeypatch.setattr(worker_monitor, '_atomic_text', fail_ready) + original = worker_monitor.ProgressWatch + class UnsafeWatch(original): + def __init__(self, *args): + super().__init__(*args) + self.safe_stop = False + monkeypatch.setattr(worker_monitor, 'ProgressWatch', UnsafeWatch) + captured = {} + def resolve(control, root, error, **options): + captured.update(options) + process.poll = lambda: 1 + process.returncode = 1 + assert options['resolved']() + monkeypatch.setattr(stop_control, 'wait_for_resolution', resolve) + process.wait = lambda **options: 1 + with pytest.raises(worker_monitor.WorkerMonitorError): + worker_monitor.run_monitored_process(('cscript', '', '', str(tmp_path), 'python', 'job'), 1, + process_factory=lambda *a, **kw: process, force_control=StopControl()) + assert captured.get('resolved') is not None + + +def test_manual_dialog_transport_is_generation_bound_and_one_shot(tmp_path): + from dx_productivity.trial_task_center.batch_worker import consume_manual_dialog_actions + binding = {'batch_id': 'a' * 32, 'attempt_id': 'b' * 32, 'generation': 2, 'session_token': 'token'} + calls = [] + runtime = SimpleNamespace(generation=2, cad=SimpleNamespace(token='token'), + manual_dialog_action=lambda dialog, button: calls.append((dialog, button))) + request = dict(binding, version=1, request_id='c' * 32, dialog={'pid': 1}, button_hwnd=2) + batch_resources.write_record(tmp_path / ('manual-dialog-' + request['request_id'] + '.json'), request) + consume_manual_dialog_actions(tmp_path, runtime, binding['batch_id'], binding['attempt_id']) + consume_manual_dialog_actions(tmp_path, runtime, binding['batch_id'], binding['attempt_id']) + assert calls == [({'pid': 1}, 2)] + stale = dict(request, generation=1, request_id='d' * 32) + batch_resources.write_record(tmp_path / ('manual-dialog-' + stale['request_id'] + '.json'), stale) + consume_manual_dialog_actions(tmp_path, runtime, binding['batch_id'], binding['attempt_id']) + assert calls == [({'pid': 1}, 2)] + + +def test_cache_hashes_after_verified_reader_release(tmp_path, monkeypatch): + from dx_productivity.trial_task_center import discovery_cache + from dx_productivity.trial_task_center.quick_run_protocol import ProjectDiscovery + project = tmp_path / 'board.prj' + project.write_text('project') + calls, locked = [], [False] + real_fingerprint = discovery_cache.project_fingerprint + def fingerprint(*args, **kwargs): + assert not locked[0], 'fingerprint attempted before read generation release' + return real_fingerprint(*args, **kwargs) + def load(): + calls.append('load') + locked[0] = True + return ProjectDiscovery((('MAIN', '1'),), {('MAIN', '1'): frozenset({'old'})}) + def release(): + calls.append('release') + locked[0] = False + monkeypatch.setattr(discovery_cache, 'project_fingerprint', fingerprint) + cache = discovery_cache.DiscoveryCache(tmp_path / 'data') + cache.discover(project, ('old',), False, load, release_idle=release) + cache.discover(project, ('old',), False, load, release_idle=release) + assert calls == ['load', 'release'] + + +def test_cleanup_error_never_drops_live_write_cad_or_closes_its_desktop(tmp_path): + from test_batch_runtime import Desktop + from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError + runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) + runtime._ensure_desktop() + process = SimpleNamespace(poll=lambda: None) + def refuse(*args): + raise WorkerMonitorError('CAD still saving; evidence write failed') + cad = SimpleNamespace(cad=process, __exit__=refuse, record={}) + runtime.cad = cad + with pytest.raises(TaskCenterError): + runtime.close() + assert runtime.cad is cad + assert not runtime.desktop.released + # Leave no retained live registry state in the test process. + batch_resources.LIVE_RESOURCES.pop(str(runtime.path), None) + + +def test_owned_exit_observer_waits_for_job_descendants(tmp_path): + runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution') + process = SimpleNamespace(poll=lambda: 0, tree_exited=lambda: False) + runtime.desktop = SimpleNamespace(processes=[process]) + assert not runtime.owned_processes_exited() + process.tree_exited = lambda: True + assert runtime.owned_processes_exited() + + +def test_startup_dialog_owner_is_published_before_guid_binding(tmp_path): + from test_batch_runtime import Desktop + runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) + with runtime: + with runtime.native_scope(tmp_path / 'startup') as lease: + process = lease.start_process(('viewdraw.exe',)) + lease.register_cad_process(process, 'session-token') + record = json.loads(runtime.path.read_text()) + assert record['cad_pid'] == process.pid + assert record['cad_process']['pid'] == process.pid + assert record['cad_token'] == 'session-token' + assert record['cad_guid'] is None + + +def test_native_read_failure_releases_without_repeating_project(tmp_path): + import hashlib + from test_batch_runtime import Desktop + from dx_productivity.trial_task_center.project_discovery import ReleasedProjectReadError + project, database = tmp_path / 'board.prj', tmp_path / 'database.dat' + project.write_text('project') + database.write_text('database') + evidence = tmp_path / 'native' + runtime = batch_runtime.BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop) + with runtime: + with runtime.native_scope(evidence) as scope: + pass + batch_resources.write_record(evidence / 'native-retry-proof.json', {'version': 1, + 'native_read_only': True, 'workers_closed': True, 'binding': scope.identity, + 'project_path': str(project), 'evidence_root': str(evidence), + 'source_hashes': {str(item): hashlib.sha256(item.read_bytes()).hexdigest() for item in (project, database)}}) + with pytest.raises(ReleasedProjectReadError): + runtime.recover_native_failure(project, evidence, 'read timed out') + assert runtime.generation == 1 and runtime.record['state'] == 'released' + assert runtime.desktop is runtime.cad is None + assert runtime.record['retries'] == {} + + +@pytest.mark.parametrize('native', [False, True]) +def test_repeated_live_frames_cannot_hide_business_stall(native): + from dx_productivity.trial_task_center.worker_monitor import ProgressWatch + watch = ProgressWatch('token', 'job', 10, 0) + stage = 'native_snapshot_read' if native else 'native_replace' + assert watch.accept(f'DX_PROGRESS\ttoken\tjob\t1\t{stage}\t0\t10\t0\n'.encode(), 1) + for sequence in range(2, 12): + if native: + assert watch.accept_native(f'DX_NATIVE_PROGRESS\ttoken\tjob\t1\tjob\t{sequence}\t0\t10\t0\n'.encode(), sequence) + else: + assert watch.accept(f'DX_PROGRESS\ttoken\tjob\t{sequence}\t{stage}\t0\t10\t0\n'.encode(), sequence) + assert watch.action(11) == 'request_stop' + assert watch.action(21) == 'await_safe_completion' + assert watch.action(100) is None # Elapsed time cannot authorize killing a possible write. + + +def test_finished_supervisor_label_does_not_hide_a_live_original_process(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.batch_supervisor import pending_supervisors + batch_id = 'a' * 32 + batch_resources.write_record(tmp_path / 'batch-resources' / batch_id / 'supervisor.json', { + 'version': 1, 'batch_id': batch_id, 'attempt_id': 'b' * 32, + 'operation': 'execute_task_queue', 'state': 'finished', 'task_ids': ['A'], + 'process': {'pid': 20, 'creation_time': 'worker'}, + 'parent_process': {'pid': 10, 'creation_time': 'parent'}, 'request_sha256': 'f' * 64}) + monkeypatch.setattr(batch_resources, '_process_gone', lambda identity: False) + assert pending_supervisors(tmp_path) + + +def test_crashed_write_supervisor_does_not_kill_its_still_running_descendant(tmp_path, monkeypatch): + import sys + from dx_productivity.trial_task_center import batch_supervisor + from dx_productivity.trial_task_center.logging_setup import configure_logging + from dx_productivity.trial_task_center.service import TaskCenterService + from dx_productivity.trial_task_center.store import TaskStore + from dx_productivity.trial_task_center.stop_control import StopControl + monkeypatch.setattr(batch_supervisor, '_command', lambda request: (sys.executable, '-c', + 'import subprocess,sys; subprocess.Popen([sys.executable,"-c","import time;time.sleep(6)"], ' + 'stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)')) + store = TaskStore(tmp_path / 'tasks.db') + try: + service = TaskCenterService(store, configure_logging(tmp_path)) + task = store.create_task('harmless owned process probe') + with pytest.raises(TaskCenterError) as caught: + batch_supervisor.run_supervised(service, 'execute_quick_run', {'task_id': task.task_id}, + stop_event=StopControl()) + record = json.loads(next((tmp_path / 'batch-resources').glob('*/process.json')).read_text()) + assert record['cleanup']['forced_termination'] is False + assert caught.value.resources_released + assert list(tmp_path.rglob('uncertain.flag')) + finally: + store.close() + + +def test_no_control_write_session_retains_harmless_owned_process(tmp_path, monkeypatch): + import sys + from uuid import uuid4 + from dx_productivity.trial_task_center import isolated_cad + from dx_productivity.trial_task_center.isolated_desktop import PrivateDesktop + desktop = PrivateDesktop('dx-' + uuid4().hex, enabled=True) + desktop.__enter__() + process = desktop.start_process((sys.executable, '-c', 'import time;time.sleep(60)')) + session = isolated_cad.IsolatedCadSession(tmp_path, desktop=desktop) + session.cad = process + session.binding = SimpleNamespace(environment=lambda: {}) + session._record(cad_pid=process.pid, cad_process=batch_resources.process_identity(process)) + monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: (sys.executable, '-c', 'raise SystemExit(1)')) + try: + with pytest.raises(isolated_cad.WorkerMonitorError): + session.__exit__(None, None, None) + assert process.poll() is None + assert isolated_cad.RETAINED_CAD_SESSIONS[str(session.root)] is session + assert (tmp_path / 'uncertain.flag').exists() + assert batch_resources.pending_resources(tmp_path)[0]['kind'] == 'retained_cad' + assert batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) == () + assert process.poll() is not None + assert str(session.root) not in isolated_cad.RETAINED_CAD_SESSIONS + finally: + process.terminate(reason='explicit harmless test cleanup') + desktop.close() + getattr(isolated_cad, 'RETAINED_CAD_SESSIONS', {}).pop(str(session.root), None) + + +def test_force_pending_owned_job_requires_confirmation_identity_and_durable_protection(tmp_path, monkeypatch): + import os + import sys + from uuid import uuid4 + import win32api + from dx_productivity.trial_task_center import batch_supervisor + from dx_productivity.trial_task_center.isolated_desktop import _Win32Api + from dx_productivity.trial_task_center.batch_resources import force_pending_owned_resources + api = _Win32Api() + process = api.create_process((sys.executable, '-c', 'import time;time.sleep(60)'), None, None, None) + batch_id = uuid4().hex + path = tmp_path / 'batch-resources' / batch_id / 'supervisor.json' + parent = SimpleNamespace(pid=os.getpid(), handle=win32api.GetCurrentProcess()) + journal = {'version': 1, 'batch_id': batch_id, 'attempt_id': uuid4().hex, + 'operation': 'execute_quick_run', 'state': 'blocked', 'task_ids': ['A'], + 'process': batch_resources.process_identity(process), 'parent_process': batch_resources.process_identity(parent), + 'request_sha256': 'f' * 64} + batch_resources.write_record(path, journal) + batch_supervisor._LIVE_SUPERVISORS[path] = (api, process) + try: + with pytest.raises(TaskCenterError): + force_pending_owned_resources(tmp_path) + assert process.poll() is None and not list(tmp_path.rglob('uncertain.flag')) + corrupt = dict(journal, process=dict(journal['process'], creation_time='reused PID')) + batch_resources.write_record(path, corrupt) + with pytest.raises(TaskCenterError): + force_pending_owned_resources(tmp_path, confirmed=True) + assert process.poll() is None + batch_resources.write_record(path, journal) + mark = batch_supervisor._mark_interrupted + monkeypatch.setattr(batch_supervisor, '_mark_interrupted', lambda *args: (_ for _ in ()).throw(PermissionError('protection denied'))) + with pytest.raises(PermissionError): + force_pending_owned_resources(tmp_path, confirmed=True) + assert process.poll() is None + monkeypatch.setattr(batch_supervisor, '_mark_interrupted', mark) + assert force_pending_owned_resources(tmp_path, confirmed=True) == () + assert process.poll() is not None + assert list(tmp_path.rglob('uncertain.flag')) + finally: + process.terminate(reason='explicit harmless test cleanup') + process.close() + api.close_pending() + batch_supervisor._LIVE_SUPERVISORS.pop(path, None) + + +def test_safe_failure_reset_returns_bound_prewrite_source_and_release_proof(tmp_path, monkeypatch): + import hashlib + import subprocess + from test_batch_runtime import command_for, worker_runtime + command, project = command_for(tmp_path) + runtime = worker_runtime(tmp_path, monkeypatch, lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', '')) + with runtime: + runtime.run_worker(command, 1) + proof = runtime.reset_project_after_safe_failure(project, Path(command[3])) + assert proof['no_mutation'] is True and proof['source_unchanged'] is True + assert proof['resources_released'] is True and proof['project_closed'] is True + assert proof['job_id'] == 'job' and proof['attempt_id'] == runtime.record['active_attempt'] + assert proof['session_id'] == runtime.session_id and proof['generation'] == 1 + assert proof['project_path'] == str(project.resolve()) + assert proof['attempt_record_sha256'] == hashlib.sha256(Path(proof['attempt_record_path']).read_bytes()).hexdigest() + assert json.loads(Path(proof['proof_path']).read_text()) == proof + + +def test_discovery_identity_preserves_actual_metadata_unknown_vs_empty(): + from dx_productivity.trial_task_center.discovery_cache import _discovery_identity + row = SimpleNamespace(key='uid', refdes='R1', old_part_number='old', old_value='value', + actual_part_label=None, actual_cell=None) + discovery = SimpleNamespace(sheets=(('MAIN', '1'),), matched_old_parts={('MAIN', '1'): frozenset({'old'})}, components=(row,)) + before = _discovery_identity(discovery) + row.actual_part_label = '' + assert _discovery_identity(discovery) != before + + +def test_force_pending_runtime_uses_live_handles_and_preserves_write_uncertainty(tmp_path): + import sys + from dx_productivity.trial_task_center.isolated_cad import IsolatedCadSession + runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution') + scope = runtime.native_scope(tmp_path / 'quick-run') + scope.__enter__() + process = scope.start_process((sys.executable, '-c', 'import time;time.sleep(60)')) + session = IsolatedCadSession(tmp_path / 'cad', desktop=scope) + session.cad = process + session._record(cad_pid=process.pid, cad_process=batch_resources.process_identity(process)) + runtime.cad = session + runtime._active_evidence = tmp_path / 'quick-run' + runtime._save(cad_pid=process.pid, cad_guid='owned', cad_process=batch_resources.process_identity(process)) + try: + assert batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) == () + assert process.poll() is not None + assert runtime.record['state'] == 'released' + assert (tmp_path / 'quick-run' / 'uncertain.flag').exists() + assert runtime.record['force_confirmed'] is True + finally: + process.terminate(reason='explicit harmless test cleanup') + runtime.cad = None + runtime.close(reason='manual_retry') + + +def test_raw_retained_session_reconciles_after_all_owned_processes_exit(tmp_path, monkeypatch): + import sys + import time + from uuid import uuid4 + from dx_productivity.trial_task_center import isolated_cad + from dx_productivity.trial_task_center.isolated_desktop import PrivateDesktop + desktop = PrivateDesktop('dx-' + uuid4().hex, enabled=True) + desktop.__enter__() + process = desktop.start_process((sys.executable, '-c', 'import time;time.sleep(1.5)')) + session = isolated_cad.IsolatedCadSession(tmp_path, desktop=desktop) + session.cad = process + session.binding = SimpleNamespace(environment=lambda: {}) + session._record(cad_pid=process.pid, cad_process=batch_resources.process_identity(process)) + monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: (sys.executable, '-c', 'raise SystemExit(1)')) + try: + with pytest.raises(isolated_cad.WorkerMonitorError): + session.__exit__(None, None, None) + assert process.wait(timeout=4) == 0 + deadline = time.monotonic() + 2 + while not all(item.tree_exited() for item in session._retained_processes): + assert time.monotonic() < deadline + time.sleep(.01) + assert batch_resources.cleanup_pending_resources(tmp_path) == () + assert (tmp_path / 'uncertain.flag').exists() + assert process.cleanup_record['forced_termination'] is False + finally: + process.terminate(reason='explicit harmless test cleanup') + desktop.close() + isolated_cad.RETAINED_CAD_SESSIONS.pop(str(session.root), None) + + +def test_no_control_retains_cad_job_when_root_exited_but_descendants_remain(tmp_path): + from dx_productivity.trial_task_center import isolated_cad + process = SimpleNamespace(pid=30, poll=lambda: 0, tree_exited=lambda: False) + desktop = SimpleNamespace(name='dx-owned', _processes=[process], + close=lambda: pytest.fail('root exit cannot authorize descendant termination')) + session = isolated_cad.IsolatedCadSession(tmp_path, desktop=desktop) + session.cad = process + try: + with pytest.raises(isolated_cad.WorkerMonitorError): + session.__exit__(None, None, None) + assert isolated_cad.RETAINED_CAD_SESSIONS[str(session.root)] is session + finally: + isolated_cad.RETAINED_CAD_SESSIONS.pop(str(session.root), None) + + +def test_repeat_explicit_force_checks_whole_job_not_only_completed_root(tmp_path): + from dx_productivity.trial_task_center.batch_supervisor import _force_owned_supervisor + calls = [] + process = SimpleNamespace(poll=lambda: 0, tree_exited=lambda: False, + terminate=lambda: calls.append('force-job')) + journal = {'version': 1, 'batch_id': 'a' * 32, 'attempt_id': 'b' * 32, + 'operation': 'execute_quick_run', 'task_ids': ['A'], 'force_state': 'termination_requested'} + _force_owned_supervisor(tmp_path, journal, process) + assert calls == ['force-job'] diff --git a/tests/v1_1/test_nonblocking_resume.py b/tests/v1_1/test_nonblocking_resume.py new file mode 100644 index 0000000..2224943 --- /dev/null +++ b/tests/v1_1/test_nonblocking_resume.py @@ -0,0 +1,73 @@ +from threading import Event + +from test_nonblocking_items import component_case +from dx_productivity.dsh.execution_runs import execute_confirmed_run +from dx_productivity.dsh.run_journal import RunJournal, resume_preflight +from dx_productivity.dsh.task_lifecycle import preflight +from dx_productivity.trial_task_center.service import TaskCenterService +from dx_productivity.trial_task_center.target_plan import ComponentTarget, TargetPlanRow +from dx_productivity.trial_task_center.task_queue import execute_task_queue + + +def supervised_boundary(engine, store): + gui = TaskCenterService(store, engine._logging, supervised=True, + quick_run_preparer=engine._quick_run_preparer) + gui._supervise = lambda operation, arguments, *, stop_event: execute_task_queue( + engine, stop_event=stop_event, **arguments) + return gui + + +def test_safe_failed_uid_pause_can_resume_independent_uid_through_dsh(component_case): + service, _, tasks, calls, outcomes = component_case + ids = [tasks[0].task_id] + stop = Event() + outcomes[(ids[0], ('R1',))] = 'batch_failed' + outcomes['after_call'] = lambda count: stop.set() if count == 1 else None + root = service._logging.root + result = execute_confirmed_run(service, root, ids, preflight(service, ids), stop, '4' * 32) + assert result['status'] == 'paused' + inspected = RunJournal(root).inspect(service, '4' * 32) + assert inspected['resume_ready'], inspected['blockers'] + report = resume_preflight(service, inspected) + assert report['ready'] + execute_confirmed_run(service, root, ids, report, Event(), '4' * 32, resume=inspected) + assert calls == [(ids[0], ('R1',)), (ids[0], ('R2',))] + + +def test_last_primary_pause_keeps_deferred_pass_visible_to_dsh(component_case): + engine, store, tasks, calls, outcomes = component_case + gui = supervised_boundary(engine, store) + ids = [task.task_id for task in tasks] + stop = Event() + outcomes[(ids[0], ('R1',))] = ['attach_failed', 'saved_ok'] + outcomes['after_call'] = lambda count: stop.set() if count == 4 else None + root = engine._logging.root + result = execute_confirmed_run(gui, root, ids, preflight(gui, ids), stop, '5' * 32) + assert result['status'] == 'paused' and len(calls) == 4 + inspected = RunJournal(root).inspect(gui, '5' * 32) + assert inspected['remaining_task_ids'], 'deferred work must remain accessible after the last primary item' + assert inspected['resume_ready'], inspected['blockers'] + execute_confirmed_run(gui, root, ids, resume_preflight(gui, inspected), Event(), '5' * 32, resume=inspected) + assert calls[-1] == (ids[0], ('R1',)) and len(calls) == 5 + + +def test_paused_problem_pass_with_manual_failure_resumes_reserved_uid(component_case): + engine, store, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + project = engine.projects(task_id)[0].path + targets = tuple(ComponentTarget(project, 'MAIN', '1', f'u{i}', f'R{i}', 'OLD', '10K') for i in (1, 2, 3)) + store.record_target_inventory(task_id, targets) + store.replace_target_plan(task_id, tuple(TargetPlanRow(target, True, 'NEW') for target in targets)) + gui = supervised_boundary(engine, store) + stop = Event() + outcomes[(task_id, ('R1',))] = 'batch_failed' + outcomes[(task_id, ('R2',))] = ['attach_failed', 'saved_ok'] + outcomes[(task_id, ('R3',))] = ['attach_failed', 'saved_ok'] + outcomes['after_call'] = lambda count: stop.set() if count == 4 else None + root = engine._logging.root + result = execute_confirmed_run(gui, root, [task_id], preflight(gui, [task_id]), stop, '6' * 32) + assert result['status'] == 'paused' and len(calls) == 4 + inspected = RunJournal(root).inspect(gui, '6' * 32) + assert inspected['resume_ready'], inspected['blockers'] + execute_confirmed_run(gui, root, [task_id], resume_preflight(gui, inspected), Event(), '6' * 32, resume=inspected) + assert [refs for _, refs in calls] == [('R1',), ('R2',), ('R3',), ('R2',), ('R3',)] diff --git a/tests/v1_1/test_nonblocking_value_vbs.py b/tests/v1_1/test_nonblocking_value_vbs.py new file mode 100644 index 0000000..9bc44fb --- /dev/null +++ b/tests/v1_1/test_nonblocking_value_vbs.py @@ -0,0 +1,170 @@ +import pytest +import test_accuracy_vbs_guards as guards + + +def test_already_target_requires_value_and_cell_policy(tmp_path): + output = guards.run_vbs(tmp_path, ['QPRequireAlreadyTargetPolicy', 'QPTargetCell', 'QPValueMode', + 'QPCustomValue', 'QPSyncNc', 'QPValueHasNcMarker', 'QPAppendNcMarker', 'QPDecode'], ''' +Dim Command,Comp,Selector,Failure +Set Command=CreateObject("Scripting.Dictionary") +Command.Add "new_part_number","NEW":Command.Add "value_mode","custom" +Command.Add "custom_value","10nF":Command.Add "target_cell","CELL":Command.Add "sync_nc","1" +Set Comp=New FakeComponent +Selector=Array("R1","10nF") +QPRequireAlreadyTargetPolicy Comp,Command,Selector +Comp.CurrentValue="wrong" +On Error Resume Next +QPRequireAlreadyTargetPolicy Comp,Command,Selector +Failure=Err.Number:Err.Clear +If Failure=0 Then WScript.Quit 9 +Comp.CurrentValue="10nF":Comp.Cell="WRONG" +QPRequireAlreadyTargetPolicy Comp,Command,Selector +If Err.Number=0 Then WScript.Quit 10 +WScript.Echo "Value and Cell mismatch blocked" +''', prelude=''' +Class FakeComponent + Public Refdes,CurrentValue,Cell + Private Sub Class_Initialize() + Refdes="R1":CurrentValue="10nF":Cell="CELL" + End Sub +End Class +Function QPEffectiveComponentAttribute(ByRef Comp,ByVal Name) + Select Case Name + Case "Value":QPEffectiveComponentAttribute=Comp.CurrentValue + Case "Cell Name":QPEffectiveComponentAttribute=Comp.Cell + Case "Part Label":QPEffectiveComponentAttribute="LABEL" + Case "Part Number":QPEffectiveComponentAttribute="NEW" + End Select +End Function +''') + assert output == 'Value and Cell mismatch blocked' + + +def test_duplicate_attribute_names_preserve_equal_text_before_blocking(tmp_path): + output = guards.run_vbs(tmp_path, ['QPAttributeObservations', 'QPRequireUniqueAttributeNames', 'QPEncode'], ''' +Dim Comp,Attr,Evidence,Failure +Set Comp=New FakeComponent +Set Attr=New FakeAttribute:Attr.Name="Value":Attr.Value="same":Attr.InstanceValue="same" +Comp.Attributes.Add Attr,True +Set Attr=New FakeAttribute:Attr.Name="Value":Attr.Value="same":Attr.InstanceValue="same" +Comp.Attributes.Add Attr,True +Evidence=QPAttributeObservations(Comp) +If UBound(Split(Evidence,vbCrLf))<>2 Then WScript.Quit 8 +If InStr(Evidence,"unknown")=0 Then WScript.Quit 9 +On Error Resume Next +QPRequireUniqueAttributeNames Comp +Failure=Err.Number +If Failure=0 Then WScript.Quit 10 +WScript.Echo "both occurrences retained; ambiguous write blocked" +''', prelude=''' +Dim QP_Root,QP_CurrentJobId,QP_Fso +QP_Root=".":QP_CurrentJobId="job":Set QP_Fso=CreateObject("Scripting.FileSystemObject") +Sub QPAppendAscii(ByVal Path,ByVal Text) +End Sub +Class FakeComponent + Public Attributes,UID,Refdes + Private Sub Class_Initialize() + UID="u1":Refdes="R1":Set Attributes=CreateObject("Scripting.Dictionary") + End Sub +End Class +Class FakeAttribute + Public Name,Value,InstanceValue +End Class +''') + assert output == 'both occurrences retained; ambiguous write blocked' + + +def test_native_value_report_is_readable_and_bound_to_current_snapshot(tmp_path): + from test_value_readback import case + from dx_productivity.trial_task_center.value_readback import capture_value_readback + row, raw = case(tmp_path) + raw.unlink() + fields = dict(row, old_part_number='OLD', new_part_number='NEW') + assignments = '\n'.join('Command.Add "' + key + '","' + value.replace('"', '""') + '"' + for key, value in fields.items() if key != 'target_selectors') + body = ''' +Dim Command,Records,Plans,Comp +Set Command=CreateObject("Scripting.Dictionary") +''' + assignments + ''' +Command.Add "target_selectors","u1%001FR1%001F10nF" +Set Records=CreateObject("Scripting.Dictionary") +Set Plans=CreateObject("Scripting.Dictionary") +Set Comp=New FakeComponent:Plans.Add "0",Comp +QPWriteValueReadback Command,Records,Plans,"saved_ok" +WScript.Echo "report persisted" +''' + prelude = ''' +Dim QP_Root,QP_Fso +QP_Root="''' + str(tmp_path).replace('"', '""') + '''":Set QP_Fso=CreateObject("Scripting.FileSystemObject") +Class FakeAttribute + Public Name,Value,InstanceValue +End Class +Class FakeComponent + Public Attributes,UID,Refdes + Private Sub Class_Initialize() + Dim Attr + UID="u1":Refdes="R1":Set Attributes=CreateObject("Scripting.Dictionary") + Set Attr=New FakeAttribute:Attr.Name="Value":Attr.Value="10nF":Attr.InstanceValue="10nF" + Attributes.Add Attr,True + End Sub + Function FindAttribute(ByVal Name) + Dim Attr + Set FindAttribute=Nothing + For Each Attr In Attributes + If Attr.Name=Name Then Set FindAttribute=Attr + Next + End Function +End Class +Function QPEffectiveComponentAttribute(ByRef Comp,ByVal Name) + If Name="Part Number" Then QPEffectiveComponentAttribute="NEW" + If Name="Value" Then QPEffectiveComponentAttribute="10nF" + If Name="Cell Name" Then QPEffectiveComponentAttribute="CELL" +End Function +''' + assert guards.run_vbs(tmp_path, ['QPWriteValueReadback', 'QPAttributeObservations', 'QPFindCurrentAttribute', + 'QPReadUtf8', 'QPWriteUtf8', 'QPEncode', 'QPDecode', 'QPJobSnapshotPath', 'QPValueMode', 'QPCustomValue', + 'QPSyncNc', 'QPBooleanText', 'QPTargetCell'], body, prelude=prelude) == 'report persisted' + assert capture_value_readback(tmp_path, row, ('saved_ok', 'batch_closed', 'verified'))['value_readback_status'] == 'verified' + + +@pytest.mark.parametrize('fault', ['', 'value', 'cell', 'duplicate']) +def test_real_capture_pipeline_skips_only_verified_already_target(tmp_path, fault): + names = ['QPCaptureAndSelect', 'QPCommandTargets', 'QPRequireUniqueAttributeNames', 'QPAttributeObservations', + 'QPRequireAlreadyTargetPolicy', 'QPFindCurrentAttribute', 'QPEffectiveComponentAttribute', 'QPValueMode', + 'QPCustomValue', 'QPSyncNc', 'QPTargetCell', 'QPEncode', 'QPDecode', 'QPValueHasNcMarker', + 'QPAppendNcMarker', 'QPInvalidatePinIndex'] + body = ''' +Dim Command,Comp,Attr,Index,Names,Values,Failure +Set QP_App=New FakeApp +Set Comp=New FullComponent:Comp.UID="u1":Comp.Refdes="R1" +Names=Array("Part Number","Value","Cell Name") +Values=Array("NEW","''' + ('WRONG' if fault == 'value' else '10nF') + '''","''' + ('WRONG' if fault == 'cell' else 'CELL') + '''") +For Index=0 To 2 + Set Attr=New FullAttribute:Attr.Name=Names(Index):Attr.Value=Values(Index):Attr.InstanceValue=Values(Index) + Comp.Attributes.Add Attr,True +Next +''' + ('''Set Attr=New FullAttribute:Attr.Name="Value":Attr.Value="10nF":Attr.InstanceValue="10nF" +Comp.Attributes.Add Attr,True +''' if fault == 'duplicate' else '') + ''' +QP_App.ActiveView.Components.Add Comp,True +Set Command=CreateObject("Scripting.Dictionary") +Command.Add "job_id","job":Command.Add "old_part_number","OLD":Command.Add "new_part_number","NEW" +Command.Add "value_mode","custom":Command.Add "custom_value","10nF":Command.Add "sync_nc","1" +Command.Add "target_cell","CELL":Command.Add "target_scope","targets" +Command.Add "target_selectors",QPEncode("u1" & Chr(31) & "R1" & Chr(31) & "10nF") +On Error Resume Next +QPCaptureAndSelect Command,"unused" +Failure=Err.Number +On Error GoTo 0 +WScript.Echo CStr(Failure<>0) +WScript.Echo CapturedState & ":" & CapturedCode +''' + prelude = guards.LOG_STUB + guards.FULL_FAKE + ''' +Dim QP_CurrentJobId,CapturedState,CapturedCode +QP_CurrentJobId="job" +Sub QPWriteStatus(ByVal Root,ByVal Job,ByVal State,ByVal Code,ByVal Message) + CapturedState=State:CapturedCode=Code +End Sub +''' + output = guards.run_vbs(tmp_path, names, body, prelude=prelude) + assert output.splitlines() == (['False', 'no_match:already_target'] if not fault else ['True', ':']) diff --git a/tests/v1_1/test_project_session_health.py b/tests/v1_1/test_project_session_health.py new file mode 100644 index 0000000..8e03504 --- /dev/null +++ b/tests/v1_1/test_project_session_health.py @@ -0,0 +1,271 @@ +"""A failed receipt authorizes another UID only through affirmative health.""" +import json +from pathlib import Path +from threading import Event + +import pytest + +from test_nonblocking_items import component_case +from dx_productivity.trial_task_center.errors import TaskCenterError +from dx_productivity.trial_task_center.execution_journal import read_command_rows +from dx_productivity.trial_task_center.task_queue import execute_task_queue + + +def prepared(case): + service, store, tasks, calls, outcomes = case + task_id = tasks[0].task_id + child = service.for_project_task(task_id) + handoff = child._quick_run_preparer.ensure_batch_handoff(task_id, child.projects(task_id), child._execution_rules(task_id)) + rows = read_command_rows(handoff.command_path) + journal = child._quick_run_preparer.journal + journal.bind(task_id, rows, {}) + return task_id, child, handoff, rows, journal + + +def native_failed_receipt(journal, task_id, row, runtime, child): + from dx_productivity.trial_task_center.project_session_health import runtime_dispatch_binding + attempt = journal.claim(task_id, row) + binding = runtime_dispatch_binding(child._quick_run_preparer.data_root, row, attempt, runtime) + journal.finish(attempt, 'batch_failed', 'batch_closed', 'current prewrite rejection', + proof={'files': binding['files'], 'native_dispatch': binding}) + + +def test_bare_failed_receipt_cannot_admit_another_uid(component_case): + task_id, child, handoff, rows, journal = prepared(component_case) + journal.finish(journal.claim(task_id, rows[0]), 'batch_failed', 'batch_closed', 'no health evidence') + with pytest.raises(TaskCenterError): + journal.claim(task_id, rows[1]) + assert journal.independent_continuation_ready(task_id) is False + + +def test_empty_proof_does_not_count_as_affirmative_failure_health(): + from dx_productivity.trial_task_center.project_session_health import affirmative + assert not affirmative({}, {}) + assert not affirmative({'files': []}, {}) + + +def test_changed_project_after_controlled_failure_blocks_remaining_uid(component_case): + service, store, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + project = service.projects(task_id)[0].path + outcomes[(task_id, ('R1',))] = 'batch_failed' + outcomes['after_call'] = lambda count: project.write_text('unexpected source mutation') if count == 1 else None + result = execute_task_queue(service, (task_id,), stop_event=Event()) + assert result.paused + assert calls == [(task_id, ('R1',))] + + +def test_tampered_failure_health_denies_resume(component_case): + service, store, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = 'batch_failed' + stop = Event() + outcomes['after_call'] = lambda count: stop.set() if count == 1 else None + first = execute_task_queue(service, (task_id,), stop_event=stop) + journal = service.for_project_task(task_id)._quick_run_preparer.journal + assert journal.independent_continuation_ready(task_id) is True + receipt = json.loads(store._connection.execute("SELECT receipt_json FROM execution_operations WHERE task_id=? AND state='failed'", (task_id,)).fetchone()[0]) + proof = receipt['proof']['failure_health'] + Path(proof['files'][0]['path']).write_text('tampered') + assert journal.independent_continuation_ready(task_id) is False + outcomes.pop('after_call') + resumed = execute_task_queue(service, (task_id,), stop_event=Event(), queue_id=first.queue_id) + assert calls == [(task_id, ('R1',))] + _, _, _, rows, journal = prepared(component_case) + with pytest.raises(TaskCenterError): + journal.claim(task_id, rows[1]) + + +def test_native_reset_health_binds_current_failed_job_before_next_claim(component_case, monkeypatch): + import subprocess + from test_batch_runtime import Cad, Desktop + from dx_productivity.trial_task_center import worker_monitor + from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime + task_id, child, handoff, rows, journal = prepared(component_case) + monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', '')) + root = handoff.command_path.parent + with BatchCadRuntime(child._quick_run_preparer.data_root, 'execution', desktop_factory=Desktop, cad_factory=Cad) as runtime: + runtime.run_worker(('cscript', '', '', str(root), 'python', rows[0]['job_id']), 1) + native_failed_receipt(journal, task_id, rows[0], runtime, child) + health = runtime.reset_project_after_safe_failure(Path(rows[0]['project_path']), root, job_id=rows[0]['job_id']) + release_path = Path(health['resource_record']) + release_bytes = release_path.read_bytes() + proof_path = Path(health['proof_path']) + proof_bytes = proof_path.read_bytes() + release = json.loads(release_bytes) + release['generation_releases'] = [] + release_path.write_text(json.dumps(release), encoding='utf-8') + import hashlib + bad = dict(health, resource_record_sha256=hashlib.sha256(release_path.read_bytes()).hexdigest()) + proof_path.write_text(json.dumps(bad), encoding='utf-8') + with pytest.raises(TaskCenterError): + journal.certify_safe_failure(task_id, rows[0], bad) + release_path.write_bytes(release_bytes) + proof_path.write_bytes(proof_bytes) + journal.certify_safe_failure(task_id, rows[0], health) + assert journal.failure_certified(task_id, rows[0]) + assert journal.claim(task_id, rows[1]).job_id == rows[1]['job_id'] + + +def test_stale_native_attempt_or_mutated_source_health_cannot_certify(component_case, monkeypatch): + import subprocess + from test_batch_runtime import Cad, Desktop + from dx_productivity.trial_task_center import worker_monitor + from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime + task_id, child, handoff, rows, journal = prepared(component_case) + monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', '')) + root = handoff.command_path.parent + with BatchCadRuntime(child._quick_run_preparer.data_root, 'execution', desktop_factory=Desktop, cad_factory=Cad) as runtime: + runtime.run_worker(('cscript', '', '', str(root), 'python', rows[0]['job_id']), 1) + native_failed_receipt(journal, task_id, rows[0], runtime, child) + Path(rows[0]['project_path']).write_text('source changed outside this failed job') + health = runtime.reset_project_after_safe_failure(Path(rows[0]['project_path']), root, job_id=rows[0]['job_id']) + assert health['source_unchanged'] is False + with pytest.raises(TaskCenterError): + journal.certify_safe_failure(task_id, rows[0], health) + with pytest.raises(TaskCenterError): + journal.certify_safe_failure(task_id, rows[0], dict(health, job_id=rows[1]['job_id'])) + with pytest.raises(TaskCenterError): + journal.claim(task_id, rows[1]) + + +def test_intact_last_verified_same_sheet_baseline_permits_current_prewrite_failure(component_case, monkeypatch): + import subprocess + import hashlib + from test_batch_runtime import Cad, Desktop + from dx_productivity.trial_task_center import worker_monitor + from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime + task_id, child, handoff, rows, journal = prepared(component_case) + baseline_file = child._quick_run_preparer.data_root / 'verified-sheet.json' + baseline_file.write_text('verified semantic sheet evidence') + baseline = {'project_path': str(Path(rows[1]['project_path']).resolve()).casefold(), + 'schematic_name': rows[1]['schematic_name'], 'sheet_number': rows[1]['sheet_number'], + 'state': 'S\tfull_sheet_v1\r\nverified R2'} + journal.finish(journal.claim(task_id, rows[1]), 'saved_ok', 'batch_closed', 'R2 verified', proof={ + 'baseline': baseline, 'files': [{'path': str(baseline_file), 'sha256': hashlib.sha256(baseline_file.read_bytes()).hexdigest()}]}) + monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', '')) + root = handoff.command_path.parent + with BatchCadRuntime(child._quick_run_preparer.data_root, 'execution', desktop_factory=Desktop, cad_factory=Cad) as runtime: + runtime.run_worker(('cscript', '', '', str(root), 'python', rows[0]['job_id']), 1) + native_failed_receipt(journal, task_id, rows[0], runtime, child) + Path(rows[0]['project_path']).write_text('legal last verified sheet serialization') + health = runtime.reset_project_after_safe_failure(Path(rows[0]['project_path']), root, job_id=rows[0]['job_id']) + journal.certify_safe_failure(task_id, rows[0], health) + assert journal.failure_certified(task_id, rows[0]) + baseline_file.write_text('tampered prior sheet') + assert not journal.failure_certified(task_id, rows[0]) + + +def test_old_native_health_cannot_certify_a_new_journal_dispatch(component_case, monkeypatch): + import subprocess + from test_batch_runtime import Cad, Desktop + from dx_productivity.trial_task_center import worker_monitor + from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime + from dx_productivity.trial_task_center.project_session_health import runtime_dispatch_binding + task_id, child, handoff, rows, journal = prepared(component_case) + monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', '')) + old = None + for generation in (1, 2): + if generation == 2: + journal.authorize_retry(task_id) + journal.bind(task_id, rows, {}) + attempt = journal.claim(task_id, rows[0]) + root = handoff.command_path.parent + with BatchCadRuntime(child._quick_run_preparer.data_root, 'execution', desktop_factory=Desktop, cad_factory=Cad) as runtime: + runtime.run_worker(('cscript', '', '', str(root), 'python', rows[0]['job_id']), 1) + binding = runtime_dispatch_binding(child._quick_run_preparer.data_root, rows[0], attempt, runtime) + journal.finish(attempt, 'batch_failed', 'batch_closed', 'failed', proof={ + 'files': binding['files'], 'native_dispatch': binding}) + health = runtime.reset_project_after_safe_failure(Path(rows[0]['project_path']), root, job_id=rows[0]['job_id']) + if old is not None: + with pytest.raises(TaskCenterError): + journal.certify_safe_failure(task_id, rows[0], old) + journal.certify_safe_failure(task_id, rows[0], health) + old = health + + +def test_source_edit_after_safe_failure_pause_revokes_ready_and_claim(component_case): + service, store, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + outcomes[(task_id, ('R1',))] = 'batch_failed' + stop = Event() + outcomes['after_call'] = lambda count: stop.set() if count == 1 else None + execute_task_queue(service, (task_id,), stop_event=stop) + child = service.for_project_task(task_id) + journal = child._quick_run_preparer.journal + assert journal.independent_continuation_ready(task_id) + project = child.projects(task_id)[0].path + (project.parent / 'unrelated-component.sch').write_text('external CAD edit after certificate') + assert journal.independent_continuation_ready(task_id) is False + rows = read_command_rows(child._quick_run_preparer.data_root / 'quick-run' / 'command.tsv') + with pytest.raises(TaskCenterError): + journal.claim(task_id, rows[1]) + assert calls == [(task_id, ('R1',))] + + +@pytest.mark.parametrize('external_edit_after_success', [False, True]) +def test_new_verified_same_sheet_baseline_advances_source_checkpoint(component_case, external_edit_after_success): + import csv + from dx_productivity.trial_task_center.target_plan import ComponentTarget, TargetPlanRow + from dx_productivity.trial_task_center.quick_run import _encode_vbs_field + service, store, tasks, calls, outcomes = component_case + task_id = tasks[0].task_id + project = service.projects(task_id)[0].path + targets = tuple(ComponentTarget(project, 'MAIN', '1', f'u{i}', f'R{i}', 'OLD', '10K') for i in (1, 2, 3)) + store.record_target_inventory(task_id, targets) + store.replace_target_plan(task_id, tuple(TargetPlanRow(target, True, 'NEW') for target in targets)) + outcomes[(task_id, ('R1',))] = 'batch_failed' + stop = Event() + def saved(count): + if count != 2:return + child = service.for_project_task(task_id) + root = child._quick_run_preparer.data_root / 'quick-run' + with (root / 'current-command.tsv').open(encoding='utf-8-sig') as stream: + row = next(csv.DictReader(stream, delimiter='\t')) + project.write_text('legally verified R2 save') + columns = ('job_id', 'project_path', 'schematic_name', 'sheet_number', 'semantic_state') + values = (row['job_id'], str(project), 'MAIN', '1', 'S\tfull_sheet_v1\r\nR2 verified save') + (root / ('post-state-' + row['job_id'] + '.tsv')).write_text('\t'.join(columns) + '\n' + + '\t'.join(_encode_vbs_field(value) for value in values) + '\n', encoding='utf-8') + stop.set() + outcomes['after_call'] = saved + first = execute_task_queue(service, (task_id,), stop_event=stop) + assert calls == [(task_id, ('R1',)), (task_id, ('R2',))] + child = service.for_project_task(task_id) + journal = child._quick_run_preparer.journal + assert journal.independent_continuation_ready(task_id) + if external_edit_after_success: + (project.parent / 'OTHER.sch').write_text('unrelated external edit after successful checkpoint') + assert journal.independent_continuation_ready(task_id) is False + else: + outcomes.pop('after_call') + resumed = execute_task_queue(service, (task_id,), stop_event=Event(), queue_id=first.queue_id) + assert not resumed.paused and calls[-1] == (task_id, ('R3',)) + + +def test_verified_native_release_source_is_reused_without_hashing_locked_files(component_case, monkeypatch): + import subprocess + import hashlib + from test_batch_runtime import Cad, Desktop + from dx_productivity.trial_task_center import worker_monitor, discovery_cache + from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime + from dx_productivity.trial_task_center.project_session_health import verified_source_checkpoint + task_id, child, handoff, rows, journal = prepared(component_case) + monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda cmd, *a, **kw: subprocess.CompletedProcess(cmd, 0, '', '')) + root = handoff.command_path.parent + attempt = journal.claim(task_id, rows[0]) + with BatchCadRuntime(child._quick_run_preparer.data_root, 'execution', desktop_factory=Desktop, cad_factory=Cad) as runtime: + runtime.run_worker(('cscript', '', '', str(root), 'python', rows[0]['job_id']), 1) + health = runtime.reset_project_after_safe_failure(Path(rows[0]['project_path']), root, job_id=rows[0]['job_id']) + semantic = root / 'verified-sheet.txt' + semantic.write_text('S\tfull_sheet_v1\r\nverified current native sheet') + proof = {'files': [{'path': str(semantic), 'sha256': hashlib.sha256(semantic.read_bytes()).hexdigest()}], + 'baseline': {'project_path': str(Path(rows[0]['project_path'])).casefold(), + 'schematic_name': rows[0]['schematic_name'], 'sheet_number': rows[0]['sheet_number'], + 'state': 'S\tfull_sheet_v1\r\nverified current native sheet'}, + 'native_attempt': {key: health[key] for key in ('session_id', 'attempt_id', 'generation')}} + monkeypatch.setattr(discovery_cache, 'project_fingerprint', lambda *a, **kw: pytest.fail('verified native source must not be hashed twice')) + checkpoint = verified_source_checkpoint(child._quick_run_preparer.data_root, rows[0], attempt, proof, + [{'identity': {'failed': 'prior'}, 'certificate_sha256': 'a' * 64}], + excluded_root=child._quick_run_preparer.data_root, native_source=health) + assert checkpoint['source_sha256'] == health['source_sha256'] diff --git a/tests/v1_1/test_retained_write_owners.py b/tests/v1_1/test_retained_write_owners.py new file mode 100644 index 0000000..0222824 --- /dev/null +++ b/tests/v1_1/test_retained_write_owners.py @@ -0,0 +1,332 @@ +import io +import json +import sys +import textwrap +import time +import threading +from types import SimpleNamespace + +import pytest + +from dx_productivity.trial_task_center import batch_resources, batch_supervisor, isolated_cad, worker_monitor +from dx_productivity.trial_task_center.errors import TaskCenterError + + +def test_successful_quit_retains_live_cad_descendants(tmp_path, monkeypatch): + from test_isolated_cad import fake_desktop + events, cad = fake_desktop(monkeypatch, tmp_path) + cad.tree_exited = lambda: False + session = isolated_cad.IsolatedCadSession(tmp_path) + session.__enter__() + start = session.desktop.start_process + def start_with_identity(*args, **options): + process = start(*args, **options) + process.pid = 124 + return process + session.desktop.start_process = start_with_identity + try: + with pytest.raises(worker_monitor.WorkerMonitorError): + session.__exit__(None, None, None) + assert not any(isinstance(event, tuple) and event[0] == 'release' for event in events) + assert isolated_cad.RETAINED_CAD_SESSIONS[str(session.root)] is session + assert (tmp_path / 'uncertain.flag').exists() + finally: + isolated_cad.RETAINED_CAD_SESSIONS.pop(str(session.root), None) + + +def test_runtime_retains_quit_helper_after_cad_tree_exited(tmp_path): + from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime + cad_process = SimpleNamespace(poll=lambda: 0, tree_exited=lambda: True) + helper = SimpleNamespace(poll=lambda: None, tree_exited=lambda: False) + def failed_close(*args): + raise worker_monitor.WorkerMonitorError('Quit helper retained') + cad = SimpleNamespace(cad=cad_process, _retained_processes=[cad_process, helper], + __exit__=failed_close, record={}) + closed = [] + runtime = BatchCadRuntime(tmp_path, 'execute_quick_run') + runtime.generation = 1 + runtime.cad = cad + runtime.desktop = SimpleNamespace(_processes=[cad_process, helper], close=lambda: closed.append(True)) + try: + with pytest.raises(batch_resources.BatchResourceCleanupError): + runtime.close() + assert not closed + assert runtime.cad is cad + finally: + batch_resources.LIVE_RESOURCES.pop(str(runtime.path), None) + + +@pytest.mark.parametrize('root_finished,control_error', [(False, False), (True, False), (False, True)]) +def test_monitor_retains_possible_write_without_control(tmp_path, monkeypatch, root_finished, control_error): + calls = [] + process = SimpleNamespace(pid=123, stdout=io.BytesIO(), stderr=io.BytesIO(), + returncode=0 if root_finished else None, poll=lambda: 0 if root_finished else None, + tree_exited=lambda: False, wait=lambda **options: 0, + close=lambda: calls.append('process_close')) + class Owned: + def __init__(self, *args): pass + def close(self): calls.append('owned_close') + def stop_workers(self, process): + calls.append('terminate') + return [] + monkeypatch.setattr(worker_monitor, '_OwnedWorkers', Owned) + options = {} + if control_error: + from dx_productivity.trial_task_center import stop_control + options['force_control'] = stop_control.StopControl() + monkeypatch.setattr(stop_control, 'wait_for_resolution', + lambda *args, **kwargs: (_ for _ in ()).throw(OSError('resolution observer unavailable'))) + if not root_finished: + original = worker_monitor.ProgressWatch + class UnsafeWatch(original): + def __init__(self, *args): + super().__init__(*args) + self.safe_stop = False + monkeypatch.setattr(worker_monitor, 'ProgressWatch', UnsafeWatch) + atomic = worker_monitor._atomic_text + def locked_ready(path, text): + if path.suffix == '.ready': + raise OSError('ready locked') + atomic(path, text) + monkeypatch.setattr(worker_monitor, '_atomic_text', locked_ready) + try: + with pytest.raises((worker_monitor.WorkerMonitorError, OSError)): + worker_monitor.run_monitored_process(('cscript', '', '', str(tmp_path), 'python', 'job'), 1, + process_factory=lambda *args, **options: process, **options) + assert not calls + assert batch_resources.pending_resources(tmp_path) + assert (tmp_path / 'uncertain.flag').exists() + finally: + for key, entry in tuple(getattr(worker_monitor, 'RETAINED_WORKERS', {}).items()): + if entry['root'] == tmp_path: + worker_monitor.RETAINED_WORKERS.pop(key) + + +@pytest.mark.parametrize('state', ['blocked', 'finished']) +@pytest.mark.parametrize('active', [False, True]) +def test_reconcile_does_not_hide_retained_supervisor_descendants(tmp_path, monkeypatch, state, active): + token = 'a' * 32 + path = tmp_path / 'batch-resources' / token / 'supervisor.json' + batch_resources.write_record(path, {'version': 1, 'batch_id': token, 'attempt_id': 'b' * 32, + 'operation': 'execute_task_queue', 'state': state, 'task_ids': ['A'], 'outcome': 'interrupted', + 'process': {'pid': 20, 'creation_time': 'worker'}, + 'parent_process': {'pid': 10, 'creation_time': 'parent'}, 'request_sha256': 'f' * 64}) + process = SimpleNamespace(poll=lambda: 0, tree_exited=lambda: False) + registry = batch_supervisor._ACTIVE_SUPERVISORS if active else batch_supervisor._LIVE_SUPERVISORS + registry[path] = (object(), process, SimpleNamespace(waiting_for_resolution=True)) if active else (object(), process) + monkeypatch.setattr(batch_resources, '_process_gone', lambda identity: True) + try: + assert batch_supervisor.pending_supervisors(tmp_path) + assert batch_supervisor.reconcile_supervisors(tmp_path) + finally: + registry.pop(path, None) + + +@pytest.mark.parametrize('force', [False, True]) +def test_noninteractive_worker_keeps_retained_job_alive_until_natural_exit(tmp_path, monkeypatch, force): + from dx_productivity.trial_task_center.logging_setup import configure_logging + from dx_productivity.trial_task_center.service import TaskCenterService + from dx_productivity.trial_task_center.store import TaskStore + launcher = textwrap.dedent(''' + import sys, json + from pathlib import Path + from dx_productivity.trial_task_center import batch_runtime, batch_worker + from dx_productivity.trial_task_center.service import TaskCenterService + from dx_productivity.trial_task_center.stop_control import StopControl + from dx_productivity.trial_task_center.batch_resources import BatchResourceCleanupError + class Retained(batch_runtime.BatchCadRuntime): + def __exit__(self, *args): + raise BatchResourceCleanupError('injected retained write owner') + batch_runtime.BatchCadRuntime = Retained + def run(self, task_id, runtime=None): + (runtime.root / 'control.json').write_text(json.dumps(isinstance(runtime.stop_event, StopControl))) + child_code = 'import time; from pathlib import Path; time.sleep(1.2); Path(' + repr(str(runtime.root / 'child-finished')) + ').write_text("finished")' + process = runtime._ensure_desktop().start_process((sys.executable, '-c', child_code)) + runtime._track(process) + raise BatchResourceCleanupError('injected possible write failure') + TaskCenterService.execute_quick_run = run + batch_worker.run(Path(sys.argv[1])) + ''') + if force: + launcher = launcher.replace('time.sleep(1.2)', 'time.sleep(60)') + monkeypatch.setattr(batch_supervisor, '_command', lambda request: (sys.executable, '-c', launcher, str(request))) + store = TaskStore(tmp_path / 'tasks.db') + try: + service = TaskCenterService(store, configure_logging(tmp_path)) + task = store.create_task('harmless retained child') + if force: + failures = [] + def supervise(): + try: + batch_supervisor.run_supervised(service, 'execute_quick_run', {'task_id': task.task_id}) + except TaskCenterError as error: + failures.append(error) + thread = threading.Thread(target=supervise) + thread.start() + deadline = time.monotonic() + 15 + while not any(control.waiting_for_resolution for path, (_, _, control) + in tuple(batch_supervisor._ACTIVE_SUPERVISORS.items()) if path.is_relative_to(tmp_path)): + assert thread.is_alive() and time.monotonic() < deadline + time.sleep(.02) + service.force_pending_owned_resources(confirmed=True) + thread.join(15) + assert not thread.is_alive() + assert failures and failures[0].code == 'user_forced_stop' + assert list(tmp_path.rglob('uncertain.flag')) + else: + with pytest.raises(TaskCenterError): + batch_supervisor.run_supervised(service, 'execute_quick_run', {'task_id': task.task_id}) + assert json.loads((tmp_path / 'control.json').read_text()) is True + assert (tmp_path / 'child-finished').is_file() is not force + finally: + for path, (_, process, _) in tuple(batch_supervisor._ACTIVE_SUPERVISORS.items()): + if path.is_relative_to(tmp_path): + process.terminate(reason='explicit harmless test cleanup') + if force and 'thread' in locals(): + thread.join(15) + store.close() + + +def test_explicit_force_can_signal_an_active_protected_supervisor(tmp_path): + from dx_productivity.trial_task_center.stop_control import StopControl + token = 'c' * 32 + path = tmp_path / 'batch-resources' / token / 'supervisor.json' + control = StopControl() + control.waiting_for_resolution = True + process = SimpleNamespace(pid=123, poll=lambda: None) + registry = getattr(batch_supervisor, '_ACTIVE_SUPERVISORS', {}) + registry[path] = (object(), process, control) + try: + with pytest.raises(TaskCenterError): + batch_resources.force_pending_owned_resources(tmp_path) + assert not control.force_event.is_set() + # Unknown ownership cannot grant force, even for an active waiter. + with pytest.raises((TaskCenterError, OSError)): + batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) + assert not control.force_event.is_set() + finally: + registry.pop(path, None) + + +def test_worker_exit_does_not_wait_for_already_released_empty_desktop(tmp_path, monkeypatch): + from dx_productivity.trial_task_center import batch_worker, stop_control + runtime = SimpleNamespace(read_only=False, root=tmp_path, path=tmp_path / 'session.json', + record={'state': 'released'}, desktop=SimpleNamespace(_processes=[]), cad=None, + owned_processes_exited=lambda: False, close=lambda **options: None) + monkeypatch.setattr(stop_control, 'wait_for_resolution', lambda *args, **options: pytest.fail('released owner waited')) + batch_worker._wait_for_owned_writes(runtime, stop_control.StopControl()) + + +@pytest.mark.parametrize('force', [False, True]) +def test_retained_native_worker_is_reachable_for_exit_or_explicit_force(tmp_path, monkeypatch, force): + from uuid import uuid4 + from dx_productivity.trial_task_center.isolated_desktop import PrivateDesktop + desktop = PrivateDesktop('dx-' + uuid4().hex, enabled=True) + desktop.__enter__() + process = desktop.start_process((sys.executable, '-c', 'import time;time.sleep(60)' if force else 'pass')) + owned = worker_monitor._OwnedWorkers(uuid4().hex, {sys.executable}) + token = uuid4().hex + try: + worker_monitor._retain_worker(tmp_path, token, 'job', process, owned, True, 'unsafe worker') + if force: + assert batch_resources.cleanup_pending_resources(tmp_path) + assert process.poll() is None + with pytest.raises(TaskCenterError): + batch_resources.force_pending_owned_resources(tmp_path) + assert process.poll() is None + assert batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) == () + else: + process.wait(timeout=10) + deadline = time.monotonic() + 5 + while not process.tree_exited(): + assert time.monotonic() < deadline + time.sleep(.01) + assert batch_resources.cleanup_pending_resources(tmp_path) == () + assert not process.cleanup_record['forced_termination'] + assert (tmp_path / 'uncertain.flag').exists() + finally: + worker_monitor.RETAINED_WORKERS.pop(str(tmp_path / ('retained-worker-' + token + '.json')), None) + process.terminate(reason='explicit harmless test cleanup') + desktop.close() + + +def test_active_force_request_requires_creation_identity_and_durable_protection(tmp_path, monkeypatch): + from uuid import uuid4 + from dx_productivity.trial_task_center.isolated_desktop import _Win32Api + from dx_productivity.trial_task_center.stop_control import StopControl + from dx_productivity.trial_task_center import stop_control + api = _Win32Api() + process = api.create_process((sys.executable, '-c', 'import time;time.sleep(60)'), None, str(tmp_path), None) + token = uuid4().hex + path = tmp_path / 'batch-resources' / token / 'supervisor.json' + identity = batch_resources.process_identity(process) + journal = {'version': 1, 'batch_id': token, 'attempt_id': uuid4().hex, 'state': 'running', + 'operation': 'execute_quick_run', 'task_ids': ['A'], 'process': identity, + 'parent_process': identity, 'request_sha256': 'f' * 64} + control = StopControl() + control.waiting_for_resolution = True + batch_supervisor._ACTIVE_SUPERVISORS[path] = (api, process, control) + try: + batch_resources.write_record(path, dict(journal, process=dict(identity, creation_time='foreign'))) + with pytest.raises(TaskCenterError): + batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) + assert not control.force_event.is_set() + batch_resources.write_record(path, journal) + protect = stop_control.protect_interruption + monkeypatch.setattr(stop_control, 'protect_interruption', lambda *args: (_ for _ in ()).throw(OSError('denied'))) + with pytest.raises(OSError): + batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) + assert not control.force_event.is_set() + monkeypatch.setattr(stop_control, 'protect_interruption', protect) + assert batch_resources.force_pending_owned_resources(tmp_path, confirmed=True) + assert control.force_event.is_set() + assert process.poll() is None # Only the owning loop performs termination. + assert (path.parent / 'force-request.json').exists() + finally: + batch_supervisor._ACTIVE_SUPERVISORS.pop(path, None) + process.terminate(reason='explicit harmless test cleanup') + process.close() + api.close_pending() + + +def test_retained_worker_record_blocks_after_owner_registry_is_lost(tmp_path): + token = 'd' * 32 + path = tmp_path / 'tasks' / 'A' / 'quick-run' / ('retained-worker-' + token + '.json') + batch_resources.write_record(path, {'version': 1, 'state': 'blocked', 'token': token, + 'job_id': 'job', 'process': {'pid': 10, 'creation_time': 'original'}}) + pending = batch_resources.pending_resources(tmp_path) + assert len(pending) == 1 and pending[0]['kind'] == 'retained_worker' + assert batch_resources.cleanup_pending_resources(tmp_path) + + +def test_unsafe_stall_without_controller_returns_a_reachable_retained_owner(tmp_path, monkeypatch): + process = SimpleNamespace(pid=123, stdout=io.BytesIO(), stderr=io.BytesIO(), returncode=None, polls=0, + close=lambda: None) + def poll(): + process.polls += 1 + if process.polls >= 8: + process.returncode = 0 + return process.returncode + process.poll = poll + process.tree_exited = lambda: process.returncode is not None + class Owned: + def __init__(self, *args): pass + def close(self): pass + def stop_workers(self, process): return [] + class StalledWatch(worker_monitor.ProgressWatch): + def __init__(self, *args): + super().__init__(*args) + self.safe_stop = False + def action(self, now): return 'await_safe_completion' + monkeypatch.setattr(worker_monitor, '_OwnedWorkers', Owned) + monkeypatch.setattr(worker_monitor, 'ProgressWatch', StalledWatch) + try: + with pytest.raises(worker_monitor.WorkerMonitorError): + worker_monitor.run_monitored_process(('cscript', '', '', str(tmp_path), 'python', 'job'), 1, + process_factory=lambda *args, **options: process) + assert process.returncode is None + assert batch_resources.pending_resources(tmp_path) + finally: + for key, entry in tuple(worker_monitor.RETAINED_WORKERS.items()): + if entry['root'] == tmp_path: + worker_monitor.RETAINED_WORKERS.pop(key) diff --git a/tests/v1_1/test_supervisor_result.py b/tests/v1_1/test_supervisor_result.py index 3536129..2f03a0d 100644 --- a/tests/v1_1/test_supervisor_result.py +++ b/tests/v1_1/test_supervisor_result.py @@ -53,6 +53,18 @@ def test_pending_cleanup_cannot_expose_bom_preview(): assert error.bom_report.preview is None +@pytest.mark.parametrize('shutdown', [False, True]) +def test_failed_release_also_revokes_explicit_partial_import(shutdown): + from dx_productivity.trial_task_center.bom_import_report import BomImportReport + from dx_productivity.trial_task_center.supervisor_result import resource_pending_error, application_shutdown_error + report = BomImportReport((), (), preview='atomic', passed_preview='partial') + failure = (application_shutdown_error(report, OSError('held application')) if shutdown else + resource_pending_error('inspect_bom_batch', report, OSError('held desktop'))) + assert failure.bom_report.preview is None + assert failure.bom_report.passed_preview is None + assert failure.completed_result.passed_preview is None + + @pytest.mark.parametrize('business_succeeded', [True, False]) def test_abnormal_shutdown_retains_bom_rows_without_an_executable_preview(tmp_path, business_succeeded): from dx_productivity.trial_task_center.batch_resources import write_record diff --git a/tests/v1_1/test_value_readback.py b/tests/v1_1/test_value_readback.py new file mode 100644 index 0000000..447adf8 --- /dev/null +++ b/tests/v1_1/test_value_readback.py @@ -0,0 +1,87 @@ +from pathlib import Path + +import pytest + +from dx_productivity.trial_task_center.quick_run_protocol import _encode_vbs_field as enc + + +def case(tmp_path): + row = dict(task_id='task', job_id='job', project_path=str(tmp_path / 'board.prj'), schematic_name='MAIN', + sheet_number='1', old_part='OLD', new_part='NEW', value_mode='custom', custom_value='10nF', + sync_nc='1', target_cell='CELL', target_scope='targets', target_selectors='u1\x1fR1\x1f10nF') + snapshot = 'C\tu1\tR1\t0\t0\t0\t1\tNO_PINS\r\nA\tu1\tValue\t10nF\t10nF\t0\t0\t0\t0\t10\t0\t1\r\n' + (tmp_path / 'snapshot-job.tsv').write_bytes(snapshot.encode()) + header = ['V', '2', 'task', 'job', 'saved_ok', row['project_path'], 'MAIN', '1', 'OLD', 'NEW', + 'custom', '10nF', '1', 'CELL', row['target_selectors'], snapshot] + records = [header, ['C', 'u1', 'R1', 'NEW', '10nF', '1', 'CELL'], + ['E', 'u1', '0', 'Value', '10nF', '10nF', 'unknown']] + raw = tmp_path / 'value-readback-job.tsv' + raw.write_text('\n'.join('\t'.join(map(enc, record)) for record in records) + '\n', encoding='utf-8') + (tmp_path / 'status.tsv').write_text('job_id\tstate\tcode\tmessage\njob\tsaved_ok\tbatch_closed\tverified\n') + return row, raw + + +def test_persists_actual_values_and_rejects_later_terminal_job(tmp_path): + from dx_productivity.trial_task_center.value_readback import capture_value_readback, load_value_readback + row, _ = case(tmp_path) + outcome = ('saved_ok', 'batch_closed', 'verified') + details = capture_value_readback(tmp_path, row, outcome) + assert details['value_readback'][0]['actual_value'] == '10nF' + assert details['value_readback'][0]['uid'] == 'u1' + assert details['attribute_observations'][0]['layer'] == 'unknown' + assert load_value_readback(tmp_path, row, outcome) == details + with (tmp_path / 'status.tsv').open('a') as stream: + stream.write('other\tsaved_ok\tbatch_closed\tverified\n') + assert load_value_readback(tmp_path, row, outcome)['value_readback_status'] == 'rejected' + + +@pytest.mark.parametrize('fault', ['uid', 'policy', 'snapshot', 'foreign_job', 'version', 'cell', 'pn', 'value']) +def test_readback_rejects_foreign_or_stale_report(tmp_path, fault): + from dx_productivity.trial_task_center.value_readback import capture_value_readback + row, raw = case(tmp_path) + text = raw.read_text(encoding='utf-8') + if fault == 'uid': + text = text.replace('C\tu1\t', 'C\tu2\t') + elif fault == 'policy': + row['custom_value'] = '100nF' + elif fault == 'snapshot': + (tmp_path / 'snapshot-job.tsv').write_text('changed') + elif fault == 'foreign_job': + text = text.replace('V\t2\ttask\tjob\t', 'V\t2\ttask\tforeign\t') + elif fault == 'version': + text = text.replace('V\t2\t', 'V\t1\t') + elif fault == 'cell': + text = text.replace('C\tu1\tR1\tNEW\t10nF\t1\tCELL', 'C\tu1\tR1\tNEW\t10nF\t1\tWRONG') + elif fault == 'value': + text = text.replace('C\tu1\tR1\tNEW\t10nF\t', 'C\tu1\tR1\tNEW\twrong\t') + else: + text = text.replace('C\tu1\tR1\tNEW\t', 'C\tu1\tR1\tOLD\t') + raw.write_text(text, encoding='utf-8') + assert capture_value_readback(tmp_path, row, ('saved_ok', 'batch_closed', 'verified'))['value_readback_status'] == 'rejected' + + +def test_equal_raw_attribute_text_keeps_every_occurrence(tmp_path): + from dx_productivity.trial_task_center.value_readback import capture_value_readback + row, raw = case(tmp_path) + with raw.open('a', encoding='utf-8') as stream: + stream.write('E\tu1\t1\tValue\t10nF\t10nF\tunknown\n') + details = capture_value_readback(tmp_path, row, ('saved_ok', 'batch_closed', 'verified')) + assert len(details['attribute_observations']) == 2 + assert details['attribute_ambiguities'] == [{'uid': 'u1', 'name': 'value', 'ordinals': [0, 1]}] + + +def test_visible_report_uses_actual_values_and_explicitly_marks_missing_value(): + import json + from dx_productivity.trial_task_center.ui.messages import replacement_result_rows + event = dict(operation='quick_run_executed', message='Replacements: R1 OLD -> NEW', details={ + 'state': 'saved_ok', 'value_readback': {'value_readback_status': 'verified', + 'schematic_name': 'MAIN', 'sheet_number': '1', 'value_readback_job_id': 'job', 'value_readback': [ + dict(uid='u1', refdes='R1', old_part_number='OLD', actual_part_number='NEW', + old_value='planned old', actual_value='actual readback', actual_value_present=True)]}}) + rows = replacement_result_rows((json.dumps(event),)) + assert rows[0][4:6] == ('planned old', 'actual readback') + assert len(rows[0]) == 7 + event['details']['value_readback'] = {'value_readback_status': 'rejected'} + rows = replacement_result_rows((json.dumps(event),)) + assert rows[0][5] == '回读证据被拒绝' + assert 'actual readback' not in str(rows) diff --git a/tests/v1_1/test_workspace_execution.py b/tests/v1_1/test_workspace_execution.py new file mode 100644 index 0000000..9dc90fc --- /dev/null +++ b/tests/v1_1/test_workspace_execution.py @@ -0,0 +1,159 @@ +"""Frozen parent queue survives one isolated project supervisor failure.""" +import json +from pathlib import Path +from threading import Event +from types import SimpleNamespace + +import pytest + +from dx_productivity.trial_task_center.errors import TaskCenterError +from dx_productivity.trial_task_center.task_queue import TaskQueueEntry, TaskQueueResult + + +def service_for(tmp_path, monkeypatch, outcomes): + from dx_productivity.trial_task_center import task_queue + monkeypatch.setattr(task_queue, '_plan_hash', lambda child, task_id: 'hash-' + task_id) + children = {name: SimpleNamespace(projects=lambda _, n=name: ( + SimpleNamespace(path=tmp_path / n / 'board.prj'),)) for name in ('A', 'B')} + calls = [] + def supervise(operation, arguments, **controls): + name = arguments['task_ids'][0] + calls.append((operation, arguments, controls)) + outcome = outcomes[name] + if isinstance(outcome, Exception): + raise outcome + return TaskQueueResult(arguments.get('checkpoint_id') or arguments['queue_id'], + (TaskQueueEntry(name, outcome),), False) + return SimpleNamespace(_logging=SimpleNamespace(root=tmp_path), + for_project_task=lambda name: children[name], _supervise=supervise), calls + + +def test_released_uncertain_A_isolated_from_B_and_parent_order_retained(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + failure = TaskCenterError('quick_run_uncertain', 'A write uncertain') + failure.resources_released = True + service, calls = service_for(tmp_path, monkeypatch, {'A': failure, 'B': 'completed'}) + stop = Event() + result = execute_workspace_projects(service, ('A', 'B'), stop_event=stop, + expected_plan_hashes={'A': 'hash-A', 'B': 'hash-B'}, checkpoint_id='a' * 32) + assert [(entry.task_id, entry.state) for entry in result.entries] == [('A', 'blocked'), ('B', 'completed')] + assert result.queue_id == 'a' * 32 and not result.paused + assert [call[1]['task_ids'] for call in calls] == [('A',), ('B',)] + assert calls[0][1]['checkpoint_id'] != calls[1][1]['checkpoint_id'] + assert not stop.is_set() + assert json.loads((tmp_path / 'workspace-queues' / (result.queue_id + '.json')).read_text())['task_ids'] == ['A', 'B'] + + +def test_missing_release_proof_prevents_B(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + failure = TaskCenterError('quick_run_uncertain', 'owner still alive') + service, calls = service_for(tmp_path, monkeypatch, {'A': failure, 'B': 'completed'}) + result = execute_workspace_projects(service, ('A', 'B'), stop_event=Event()) + assert result.paused and result.current_task_id == 'A' + assert [entry.state for entry in result.entries] == ['blocked', 'pending'] + assert len(calls) == 1 + + +def test_resume_skips_terminal_and_blocked_tasks_without_replaying_A(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + service, calls = service_for(tmp_path, monkeypatch, {'A': 'failed', 'B': 'completed'}) + first = execute_workspace_projects(service, ('A', 'B'), stop_event=Event()) + calls.clear() + resumed = execute_workspace_projects(service, ('A', 'B'), stop_event=Event(), queue_id=first.queue_id) + assert resumed == first + assert calls == [] + + +def test_changed_frozen_hash_or_order_rejects_resume(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + service, calls = service_for(tmp_path, monkeypatch, {'A': 'completed', 'B': 'completed'}) + first = execute_workspace_projects(service, ('A', 'B'), stop_event=Event()) + with pytest.raises(TaskCenterError): + execute_workspace_projects(service, ('B', 'A'), stop_event=Event(), queue_id=first.queue_id) + calls.clear() + with pytest.raises(TaskCenterError): + execute_workspace_projects(service, ('A', 'B'), stop_event=Event(), + expected_plan_hashes={'A': 'changed', 'B': 'hash-B'}) + assert calls == [] + + +def test_pause_keeps_one_child_checkpoint_for_exact_resume(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + service, calls = service_for(tmp_path, monkeypatch, {'A': 'paused', 'B': 'completed'}) + original = service._supervise + def paused(operation, arguments, **controls): + result = original(operation, arguments, **controls) + return TaskQueueResult(result.queue_id, result.entries, True, 'A') + service._supervise = paused + first = execute_workspace_projects(service, ('A', 'B'), stop_event=Event()) + assert first.paused and len(calls) == 1 + child = calls[0][1]['checkpoint_id'] + path = tmp_path / 'task-queues' / (child + '.json') + path.parent.mkdir() + path.write_text('{}') + service._supervise = original + resumed = execute_workspace_projects(service, ('A', 'B'), stop_event=Event(), queue_id=first.queue_id) + assert calls[1][1]['queue_id'] == child and calls[1][1]['checkpoint_id'] is None + + +def test_all_primary_projects_precede_deferred_problem_pass(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + service, calls = service_for(tmp_path, monkeypatch, {'A': 'failed', 'B': 'completed'}) + original = service._supervise + def engine(operation, arguments, **controls): + outcome = original(operation, arguments, **controls) + path = tmp_path / 'task-queues' / (outcome.queue_id + '.json') + path.parent.mkdir(exist_ok=True) + path.write_text('{}') + return outcome + service._supervise = engine + result = execute_workspace_projects(service, ('A', 'B'), stop_event=Event()) + assert [call[1]['task_ids'][0] for call in calls] == ['A', 'B', 'A', 'B'] + assert [call[1]['defer_problem_pass'] for call in calls] == [True, True, False, False] + assert calls[2][1]['queue_id'] == calls[0][1]['checkpoint_id'] + assert calls[3][1]['queue_id'] == calls[1][1]['checkpoint_id'] + assert json.loads((tmp_path / 'workspace-queues' / (result.queue_id + '.json')).read_text())['phase'] == 'finished' + + +def test_resume_after_manual_resource_release_skips_uncertain_A_and_runs_B(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + from dx_productivity.trial_task_center import batch_resources + failure = TaskCenterError('quick_run_uncertain', 'A still owned') + service, calls = service_for(tmp_path, monkeypatch, {'A': failure, 'B': 'completed'}) + batch_id = 'd' * 32 + service._last_supervisor_batch_id = batch_id + first = execute_workspace_projects(service, ('A', 'B'), stop_event=Event()) + assert first.paused + journal = {'version': 1, 'batch_id': batch_id, 'attempt_id': 'e' * 32, + 'operation': 'execute_task_queue', 'state': 'finished', 'task_ids': ['A'], + 'process': {'pid': 20, 'creation_time': 'worker'}, + 'parent_process': {'pid': 10, 'creation_time': 'parent'}, 'request_sha256': 'f' * 64} + batch_resources.write_record(tmp_path / 'batch-resources' / batch_id / 'supervisor.json', journal) + monkeypatch.setattr(batch_resources, '_process_gone', lambda identity: True) + resumed = execute_workspace_projects(service, ('A', 'B'), stop_event=Event(), queue_id=first.queue_id) + assert not resumed.paused + assert [call[1]['task_ids'][0] for call in calls] == ['A', 'B'] + assert [entry.state for entry in resumed.entries] == ['blocked', 'completed'] + + +def test_paused_problem_pass_resumes_same_child_in_problems_phase(tmp_path, monkeypatch): + from dx_productivity.trial_task_center.workspace_execution import execute_workspace_projects + service, calls = service_for(tmp_path, monkeypatch, {'A': 'failed', 'B': 'completed'}) + original = service._supervise + problem_attempts = [] + def engine(operation, arguments, **controls): + outcome = original(operation, arguments, **controls) + path = tmp_path / 'task-queues' / (outcome.queue_id + '.json') + path.parent.mkdir(exist_ok=True) + path.write_text('{}') + if not arguments['defer_problem_pass'] and arguments['task_ids'] == ('A',): + problem_attempts.append(arguments['queue_id']) + return TaskQueueResult(outcome.queue_id, (TaskQueueEntry('A', 'paused' if len(problem_attempts) == 1 else 'completed'),), + len(problem_attempts) == 1, 'A' if len(problem_attempts) == 1 else None) + return outcome + service._supervise = engine + first = execute_workspace_projects(service, ('A', 'B'), stop_event=Event()) + assert first.paused and first.entries[0].state == 'paused' + resumed = execute_workspace_projects(service, ('A', 'B'), stop_event=Event(), queue_id=first.queue_id) + assert not resumed.paused and resumed.entries[0].state == 'completed' + assert len(problem_attempts) == 2 and problem_attempts[0] == problem_attempts[1]