diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 88721da..43551ab 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -42,6 +42,7 @@ jobs: tests/trial_task_center/test_bom_template_contract.py tests/trial_task_center/test_bom_import_review.py tests/v1_1/test_cad_dialogs.py + tests/v1_1/test_lxb_blocking_recovery.py --junitxml=build/ci-results.xml --tb=short - name: Windows release scope run: | diff --git a/docs/testing/2026-11-06-lxb-blocking-capabilities.md b/docs/testing/2026-11-06-lxb-blocking-capabilities.md new file mode 100644 index 0000000..b6e61e3 --- /dev/null +++ b/docs/testing/2026-11-06-lxb-blocking-capabilities.md @@ -0,0 +1,42 @@ +# lxb 非 UI 能力核对、运行验证与选择性吸收 + +对照:当前优化分支 `ce435dd`,lxb `4270952`(`origin/lxb/dxpartreplace-1.2.0-beta.31-windows-x64`)。两者分别在独立工作树检查;没有把 lxb 整个提交合并到当前分支。 + +## 能力与处置 + +| 能力 | lxb 实现 | 当前与本轮结果 | +| --- | --- | --- | +| 只读取消有独立期限 | `worker_monitor.py` 的 `READ_ONLY_CANCEL_GRACE_SECONDS=5`,有进度也不延长明确取消 | **本轮吸收 worker 层逻辑**;不代表整个 BOM 关闭/资源恢复链已完成迁移 | +| 只读读取与打开工程超时 | 只读空闲上限 45 秒;安全 `open_project` 45 秒请求停止、90 秒判定无响应 | **本轮吸收**;可能写入的阶段不使用该安全打开门禁 | +| 会话检查点等待期间响应暂停 | `project_session.py:run_step` 将 runtime 的停止事件送至本会话的 abort | **本轮吸收**;仍由 monitor 判定 safe/unsafe,普通暂停不变成强制结束 | +| GUI 失联和重启后只读残留回收 | 父进程 PID/创建时间、请求哈希、claim、操作类型关联;失联宽限及保护落盘 | 原分支相关模拟用例通过;**未移植整条所有权链**。需要同时适配 guardian 生存周期、当前新监督操作及已有证据格式 | +| 逐工程独立监督与超时后继续 | `workspace_execution.py` 每次执行单个 task;监督层不确定状态宽限 5 秒、无业务进度 360 秒后保护并退出,由外层尝试下一工程 | 原分支真实子进程模拟用例通过;**当前没有该外层调度**。不能只拷 wrapper,需一并核验释放证明、冻结范围、检查点、暂停/恢复和历史 | +| 只读失败释放整代资源再跳过工程 | `BatchCadRuntime`/`native_inventory.py` 的只读策略与 `project-read-release.json` | 原分支相关模拟通过;当前有 ReleasedProjectReadError 基础,但完整只读所有权策略**待适配** | +| 读缓存容忍元数据变化、释放后重验指纹 | `discovery_cache.py` 比内容哈希、运行数据库变化后二次等值读取、释放后重验 | 部分原用例通过,但发现一项异常样本结构不匹配而失败;**本轮不吸收**,不把该组标为全部验收通过 | +| 实际 Value 回读报告 | VBS 写 `value-readback.tsv`,解析器绑定 terminal job、UID、状态与版本,拒绝旧报告 | 原分支定向测试通过;**待独立吸收**,保留当前逐项策略和语义证明 | +| BOM 部分工程通过、已是新料号时跳过 | workspace 检查范围与 `allow_target`,不对已是目标的项再次替换 | 有实现;**本轮未改当前整批确认合同**,也不能据此宣称会修正已是目标但 Value 错误的项 | +| 中心库目标料号预检 | 有 `read_library_part_presence` 底层函数,但无 BOM 业务调用 | **lxb 也未启用**。测试明确禁止导入/保存时查询中心库,不能列为已完成能力 | +| 读链诊断与新近证据优先 | `read_trace.py`、读取状态、指纹差异、日志尾部和轮转 | 可独立吸收;诊断白名单必须与当前 CAD 弹窗、BOM source-map、监督证据取并集 | +| 中断资源 `cleanup=null` 与残留桌面复核 | null 转对象;核对 CAD PID、句柄证明和已退出 orphan owner | 原分支 null 用例通过;接受“空桌面”的策略涉及证据准入,**本轮未放宽当前释放条件** | + +当前已具备:逐项 Value、迁移多表 BOM、来源映射、防重复执行账本、安全失败继续后续工程、已知脚本弹窗处理与退出确认。不应再次以 lxb 旧实现覆盖。 + +仍未完成:同工程失败后的安全余项继续;适配当前账本的工程隔离;首轮结束后按原逻辑项持久额度集中重试。lxb 的外层工程跳过不等于这些项级能力。PR #19 的 AI 等待/恢复增强仍在 DSH 分支,未因本轮核对而合入。 + +## 实际运行记录 + +- lxb 独立工作树:`test_read_session_recovery.py` + `test_workspace_stop.py -k "not stop_button_requests"`:**50 passed、1 deselected**。覆盖旧父进程退出、只读操作/写操作区分、PID 复用、请求/claim 关联、只读取消、资源恢复、超时保护当前工程并继续下一工程。部分使用真实 Python 子进程;CAD 和任务结果为受控替身,不是实际换料验收。 +- lxb 额外能力组:指纹不可验证、null cleanup、Value 报告、库预检边界及三个缓存场景:**21 passed、1 failed**。失败为 `test_changing_readable_fingerprint_releases_session_and_skips_project`:测试构造二列指纹,`_fingerprint_difference` 访问真实指纹中的 size/hash 位置时出现 IndexError。已保留结论,不改原分支测试来制造通过,也不认定为已复现的真实 CAD 根因。 +- 当前分支新对照测试:取消期限缺失、读取沿用执行期限、安全打开期限缺失、会话暂停未送达均先失败;会话用例先修正替身参数签名,再确认失败为 `[False] != [True]`,不将夹具错误当作功能复现。 +- 选择性移植后:`test_lxb_blocking_recovery.py` + 既有 `test_worker_monitor.py`、`test_project_session.py`:**46 passed**。 +- 写入保护专项:慢保存不被空闲超时终止、保护落盘前不得强制结束、不确定写入保留直到明确强制:**3 passed**。 +- 独立复核进一步发现 lxb 暂停转发的旧安全状态竞态:worker 已发出 `safe=0` 并进入写入,但 monitor 尚未消费该帧时,普通暂停可能依据旧 `safe=True` 直接清理 worker。停止请求文件写入失败也可触发同一问题。两种情况均以真实 Python 子进程先复现失败,再修正:普通暂停只请求 worker 边界;写停止请求失败保留进程、显示控制错误并重试,不能凭旧遥测结束写入。 +- 最终受影响集合(新增 6 个用例、既有 monitor/session 42 个用例及写入保护 3 项):**51 passed**。其中两条竞态样本覆盖正常 stop 写入和首次 PermissionError,不与上面的阶段性通过数累加。 + +测试使用独立短路径。最初 lxb 测试因 basetemp 父目录不存在而未进入业务,修正测试输出位置后得到上面的 50 项结果。各测试组不与以前历史结果累加。 + +## 保留的保护与边界 + +没有移植 lxb 对当前源码的回退:不删除 `_active_attempts`、关闭失败持久阻断、原子恢复发布、未决 job 校验、Value/UID/语义证明;不删除正文公式与料号类型校验、逐行 value_mode 或 source-map;不把 Part Label 缺失静默降级为沿用旧 Value。 + +本轮只修改 worker 取消/打开期限及会话暂停转发,并修正上述暂停竞态;保持 CAD 弹窗监测、原有 `resolved` 回调、诊断失败可见性、写入不确定性保护。CI 增加对应文件;Windows 子进程样本在其他平台跳过。本轮未运行真实 CAD、PCB、真实 AI、外部工单、安装包构建或整库重复回归。 diff --git a/src/dx_productivity/trial_task_center/project_session.py b/src/dx_productivity/trial_task_center/project_session.py index c8358d1..b705537 100644 --- a/src/dx_productivity/trial_task_center/project_session.py +++ b/src/dx_productivity/trial_task_center/project_session.py @@ -91,6 +91,11 @@ class ProjectSessionWorker: raise WorkerMonitorError('project session exited before authorized closing') self._failure_delivered = True return self._result + stop_control = getattr(self.runtime, 'stop_event', None) + if stop_control is not None and stop_control.is_set() and not self._abort.is_set(): + # A step cannot return to the queue's stop check before its checkpoint. + # The owned monitor still decides whether the current stage can stop. + self._abort.set() try: fields = (self.root / 'project-session-result.tsv').read_text(encoding='utf-8-sig').strip().split('\t') except (FileNotFoundError, PermissionError): diff --git a/src/dx_productivity/trial_task_center/worker_monitor.py b/src/dx_productivity/trial_task_center/worker_monitor.py index ebf59a9..837528d 100644 --- a/src/dx_productivity/trial_task_center/worker_monitor.py +++ b/src/dx_productivity/trial_task_center/worker_monitor.py @@ -13,11 +13,31 @@ import time import threading from uuid import uuid4 +PROJECT_OPEN_IDLE_TIMEOUT_SECONDS = 45 +READ_ONLY_CANCEL_GRACE_SECONDS = 5 +READ_ONLY_IDLE_TIMEOUT_SECONDS = 45 + class WorkerMonitorError(subprocess.SubprocessError): pass +class WorkerMonitorCancelled(WorkerMonitorError): + """An owned worker was stopped at an admitted cancellation boundary.""" + def __init__(self, message, *, stage): + super().__init__(message) + self.stage = stage + self.cleanup_notes = () + + +class WorkerMonitorSafeTimeout(WorkerMonitorError): + """A project-open stage timed out before any possible mutation.""" + def __init__(self, message, *, stage): + super().__init__(message) + self.stage = stage + self.cleanup_notes = () + + _PROGRESS_FAILURE_LOGGED: dict[str, float] = {} @@ -103,10 +123,13 @@ class ProgressWatch: def action(self, now: float) -> str | None: idle = now - self.last_progress - if not self.stop_requested and idle >= self.idle_timeout: + idle_timeout = self.idle_timeout + if self.stage == 'open_project' and self.safe_stop: + idle_timeout = min(idle_timeout, PROJECT_OPEN_IDLE_TIMEOUT_SECONDS) + if not self.stop_requested and idle >= idle_timeout: self.stop_requested = True return 'request_stop' - if self.stop_requested and idle >= self.idle_timeout * 2: + if self.stop_requested and idle >= idle_timeout * 2: if not self.safe_stop: if not self.waiting_for_safe_completion: self.waiting_for_safe_completion = True @@ -215,12 +238,18 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor progress_path = root / 'worker-progress.json' native_progress_path = root / f'monitor-{token}.native-progress' root.mkdir(parents=True, exist_ok=True) - watch = ProgressWatch(token, job_id, idle_timeout, time.monotonic()) + watch = ProgressWatch(token, job_id, + min(idle_timeout, READ_ONLY_IDLE_TIMEOUT_SECONDS) if cooperative_abort else idle_timeout, + time.monotonic()) if force_control is not None: force_event = force_control.force_event owned = _OwnedWorkers(token, {command[0], command[4]}) process = None durable_force_accepted = False + read_cancel_started = None + read_idle_stop_started = None + stop_write_pending = False + stop_warning = '' threads = [] messages = queue.Queue(maxsize=1024) stop_readers = threading.Event() @@ -247,6 +276,25 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor elif len(output[channel]) < 1024 * 1024: output[channel].extend(line[:1024 * 1024 - len(output[channel])]) + def request_stop(): + nonlocal stop_write_pending, stop_warning + try: + _atomic_text(stop, token) + except OSError as error: + stop_write_pending = True + warning = '暂停请求尚未写入,保留进程并重试:' + str(error) + if warning != stop_warning: + import logging + logging.getLogger(__name__).warning('%s', warning) + stop_warning = warning + if force_control is not None: + force_control.control_error = warning + else: + stop_write_pending = False + if force_control is not None and force_control.control_error == stop_warning: + force_control.control_error = '' + stop_warning = '' + try: if process_factory is None: process = subprocess.Popen((*command, token), stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -276,20 +324,25 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor else: durable_force_accepted = True raise WorkerMonitorError('user forced stop; execution requires reconciliation') + if stop_write_pending: + request_stop() if abort_event is not None and abort_event.is_set(): - if not cooperative_abort and watch.safe_stop: - raise WorkerMonitorError('project session supervisor requested worker cleanup') + if cooperative_abort and read_cancel_started is None: + read_cancel_started = time.monotonic() + if (read_cancel_started is not None and + time.monotonic() - read_cancel_started >= READ_ONLY_CANCEL_GRACE_SECONDS): + raise WorkerMonitorCancelled( + 'read-only worker did not acknowledge cancellation within its grace period', stage=watch.stage) if not watch.stop_requested: - # Read-only callers request the VBS safe boundary first. - # The existing idle watchdog still bounds a stalled COM call. + # Request a worker boundary; queued unsafe progress may not + # have reached watch yet, so ordinary pause cannot kill here. watch.stop_requested = True watch.last_progress = time.monotonic() - try: - _atomic_text(stop, token) - except OSError: - if watch.safe_stop: - raise + request_stop() _publish_progress(progress_path, watch.snapshot()) + if (read_idle_stop_started is not None and time.monotonic() - read_idle_stop_started >= + min(READ_ONLY_CANCEL_GRACE_SECONDS, watch.idle_timeout)): + raise subprocess.TimeoutExpired(command, watch.idle_timeout) try: receive(*messages.get(timeout=min(0.1, idle_timeout / 10))) for pending in range(128): @@ -309,11 +362,9 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor break action = watch.action(time.monotonic()) if action == 'request_stop': - try: - _atomic_text(stop, token) - except OSError: - if watch.safe_stop: - raise + if cooperative_abort: + read_idle_stop_started = time.monotonic() + request_stop() _publish_progress(progress_path, watch.snapshot()) elif action == 'await_safe_completion': if cooperative_abort: @@ -322,6 +373,13 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor raise subprocess.TimeoutExpired(command, idle_timeout * 2) _publish_progress(progress_path, watch.snapshot()) elif action == 'unresponsive': + if stop_write_pending and not cooperative_abort: + watch.waiting_for_safe_completion = True + _publish_progress(progress_path, watch.snapshot()) + continue + if watch.safe_stop and watch.stage == 'open_project': + raise WorkerMonitorSafeTimeout('project open made no progress; safe stop was requested', + stage=watch.stage) raise subprocess.TimeoutExpired(command, idle_timeout * 2) drain_deadline = time.monotonic() + 5 while (any(thread.is_alive() for thread in threads) or not messages.empty()) and time.monotonic() < drain_deadline: @@ -351,6 +409,8 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor _atomic_text(progress_path, json.dumps(diagnostic, ensure_ascii=False)) except OSError: pass + if isinstance(error, (WorkerMonitorCancelled, WorkerMonitorSafeTimeout)) and not failures: + raise error if isinstance(error, subprocess.TimeoutExpired): raise raise WorkerMonitorError(f'Worker monitoring failed after launch; execution requires reconciliation: {error}') from error diff --git a/tests/v1_1/test_lxb_blocking_recovery.py b/tests/v1_1/test_lxb_blocking_recovery.py new file mode 100644 index 0000000..04772e1 --- /dev/null +++ b/tests/v1_1/test_lxb_blocking_recovery.py @@ -0,0 +1,140 @@ +"""Selected lxb nonblocking contracts; disposable workers only, no CAD.""" +import subprocess +import os +import sys +import threading +import time +from types import SimpleNamespace + +import pytest +from dx_productivity.trial_task_center import worker_monitor +from dx_productivity.trial_task_center.stop_control import StopControl + +@pytest.mark.skipif(os.name != 'nt', reason='Windows owned Job and read cancellation sample') +def test_cancel_blocked_read_has_deadline_even_when_progress_keeps_arriving(tmp_path, monkeypatch): + """A huge ordinary idle limit must not delay an explicit import cancel.""" + script = tmp_path / 'reader.py' + script.write_text('''import pathlib,sys,time +root=pathlib.Path(sys.argv[1]);job=sys.argv[4];token=sys.argv[5] +while not (root/f'monitor-{token}.ready').exists(): time.sleep(.01) +(root/'reading').touch() +for n in range(1,500): + print(f'DX_PROGRESS\\t{token}\\t{job}\\t{n}\\tnative_snapshot_read\\t0\\t1\\t0',flush=True) + time.sleep(.02) +''') + monkeypatch.setattr(worker_monitor, 'READ_ONLY_CANCEL_GRACE_SECONDS', .15, raising=False) + control = StopControl() + def cancel(): + while not (tmp_path / 'reading').exists(): + time.sleep(.01) + control.set() + thread = threading.Thread(target=cancel, daemon=True) + thread.start() + started = time.monotonic() + command = (sys._base_executable, '-u', str(script), str(tmp_path), sys._base_executable, '', 'job') + from dx_productivity.trial_task_center.isolated_desktop import _Win32Api + api = _Win32Api() + def start(launch, **options): + return api.create_process(launch, None, tmp_path, None, **options) + try: + with pytest.raises(worker_monitor.WorkerMonitorCancelled): + worker_monitor.run_monitored_process(command, 150, abort_event=control, + cooperative_abort=True, force_control=control, process_factory=start) + assert time.monotonic() - started < 3 + assert not control.waiting_for_resolution + assert not (tmp_path / 'uncertain.flag').exists() + finally: + control.set() + thread.join(1) + api.close_pending() + +@pytest.mark.skipif(os.name != 'nt', reason='Windows owned worker watchdog sample') +def test_frozen_read_uses_short_read_watchdog_instead_of_execution_idle_limit(tmp_path, monkeypatch): + script = tmp_path / 'frozen.py' + script.write_text('import time; time.sleep(1.5)') + monkeypatch.setattr(worker_monitor, 'READ_ONLY_IDLE_TIMEOUT_SECONDS', .15, raising=False) + command = (sys._base_executable, '-u', str(script), str(tmp_path), sys._base_executable, '', 'job') + started = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired): + worker_monitor.run_monitored_process(command, 150, cooperative_abort=True) + assert time.monotonic() - started < 1.3 + + +def test_project_open_has_short_watchdog_only_at_a_safe_boundary(): + watch = worker_monitor.ProgressWatch('token', 'job', 150, 0) + assert watch.accept(b'DX_PROGRESS\ttoken\tjob\t1\topen_project\t0\t1\t1\n', 0) + assert watch.action(45) == 'request_stop' + assert watch.action(90) == 'unresponsive' + unsafe = worker_monitor.ProgressWatch('token', 'job', 150, 0) + assert unsafe.accept(b'DX_PROGRESS\ttoken\tjob\t1\topen_project\t0\t1\t0\n', 0) + assert unsafe.action(90) is None + + +def test_project_step_forwards_pause_to_its_worker_before_checkpoint(tmp_path): + from dx_productivity.trial_task_center.project_session import ProjectSessionWorker + control = StopControl() + control.set() + requested = [] + def worker(command, timeout, *, abort_event, **_controls): + requested.append(abort_event.wait(.5)) + return subprocess.CompletedProcess(command, 2, '', 'stopped at safe boundary') + runtime = SimpleNamespace(stop_event=control, run_worker=worker) + session = ProjectSessionWorker(tmp_path / 'board.prj', tmp_path, runtime=runtime) + command = (sys.executable, '-u', 'sample', str(tmp_path), sys.executable, '', 'job') + try: + assert session.run_step(command, 150).returncode == 2 + assert requested == [True] + assert not control.force_event.is_set() + finally: + session.close() + + +@pytest.mark.skipif(os.name != 'nt', reason='Windows Job with a real unsafe worker') +@pytest.mark.parametrize('deny_stop', [False, True]) +def test_ordinary_pause_cannot_kill_write_before_queued_unsafe_progress_is_read(tmp_path, monkeypatch, deny_stop): + script = tmp_path / 'writing.py' + script.write_text('''import pathlib,sys,time +root=pathlib.Path(sys.argv[1]);job=sys.argv[4];token=sys.argv[5] +while not (root/f'monitor-{token}.ready').exists(): time.sleep(.005) +print(f'DX_PROGRESS\\t{token}\\t{job}\\t1\\tnative_replace\\t0\\t1\\t0',flush=True) +(root/'writing').touch() +while not (root/f'monitor-{token}.stop').exists(): time.sleep(.005) +(root/'pause-seen').touch() +while not (root/'finish-write').exists(): time.sleep(.005) +(root/'saved').touch() +''', encoding='utf-8') + control = StopControl() + atomic = worker_monitor._atomic_text + stop_attempts = [] + def publish(path, value): + if path.suffix == '.stop': + stop_attempts.append(path) + if deny_stop and len(stop_attempts) == 1: + raise PermissionError('controlled stop token sharing violation') + atomic(path, value) + if path.suffix == '.ready': + deadline = time.monotonic() + 3 + while not (tmp_path / 'writing').exists() and time.monotonic() < deadline: + time.sleep(.005) + assert (tmp_path / 'writing').exists() + control.set() # unsafe progress is queued, but watch still says starting/safe. + monkeypatch.setattr(worker_monitor, '_atomic_text', publish) + observed = [] + def complete_write(): + deadline = time.monotonic() + 3 + while not (tmp_path / 'pause-seen').exists() and time.monotonic() < deadline: + time.sleep(.005) + observed.append((tmp_path / 'pause-seen').exists()) + (tmp_path / 'finish-write').touch() + driver = threading.Thread(target=complete_write, daemon=True) + driver.start() + command = (sys._base_executable, '-u', str(script), str(tmp_path), sys._base_executable, '', 'job') + try: + result = worker_monitor.run_monitored_process(command, 10, abort_event=control, force_control=control) + assert result.returncode == 0 and (tmp_path / 'saved').exists() + assert not control.force_event.is_set() and not (tmp_path / 'uncertain.flag').exists() + finally: + (tmp_path / 'finish-write').touch() + driver.join(4) + assert observed == [True] + assert len(stop_attempts) >= (2 if deny_stop else 1)