fix: await natural worker tree exit before reconciling completed work
Core regression / Python core and release configuration (push) Successful in 39s
Core regression / JavaScript syntax (push) Successful in 2s
Core regression / JavaScript syntax (pull_request) Successful in 2s
Core regression / Python core and release configuration (pull_request) Failing after 40s

This commit is contained in:
阿白叔 committed 2026-11-09 09:47:35 +08:00
1 parent 189ddc238b
commit f12ddaddae
4 files changed
+97 -6

No files matched your search

+10 -1
View File
@@ -1,6 +1,6 @@
# PR #24 审查修复与验收契约
审查起点:`e78cdd3`;对应评论 306。此轮更新原 PR,不合并。最终全量结果、JUnit 和受测提交 SHA 在 PR 描述中发布,不能用 Linux 便携白名单通过替代 Windows 全量验收。
审查起点:`e78cdd3`;对应评论 306。11 月 8 日更新原 PR;11 月 9 日用户授权复审修复、验收后合并。最终全量结果、JUnit 和受测提交 SHA 在 PR 描述中发布,不能用 Linux 便携白名单通过替代 Windows 全量验收。
## 有意改变的既有断言
@@ -24,6 +24,15 @@
- 安全打开超时清理时,根进程退出和 Job 子进程计数可能错开;在现有 10 秒清理预算内等待明确的整树退出证明。没有证明继续保留句柄,不扩大超时、不强制结束未知子进程。
- Job 枚举后 `OpenProcess` 返回 87 表示该 PID 已退出;仅在尚未取得句柄时忽略。取得句柄后的查询或结束错误仍保留为清理失败,并有反例测试。
- 清理线程完成回调释放执行通道后,才接续已明确授权的强制释放;模态确认框嵌套事件循环提前收到完成信号时也不会丢失授权或重复执行。
- 正常工作进程退出后,在包含输出收尾的 10 秒预算内等待整树自然退出;正常或非零退出码均原样保留,观测期间不杀进程、不重放。未能证明退出时继续保留归属和未决保护。错误 87 仅接受真实 Win32 异常的 `winerror`,不从普通异常参数推测。
## 问题轮次操作约定
主轮次的单 UID 安全失败仍允许后续独立项继续。所有主轮次完成后,问题重试轮次若遇到某工程写入未决或范围不符,会持久化该工程的阻塞原因并暂停同队列后续工程的问题重试;已完成结果与已预留额度保留。先核验阻塞工程,再恢复原队列,不跳过未决工程、不扩大选择范围。
## 2026-11-09 复审
`189ddc2` 的原验收记录为 1302 项通过;复审者随后两次全量分别报告 1 与 2 个原生进程用例失败。旧通过记录只代表当次运行,不能作为不存在时序问题的证明。本轮补充正常退出的确定性反例及未退出保护,修复 Job 退出观测窗口;新的受测提交、完整结果与 JUnit 在 PR 描述中单独发布,不以隔离、跳过或 serial 标记代替修复。
## 原 PR 的模块意图与风险
@@ -1260,6 +1260,11 @@ class TrialTaskCenterWindow(MaterialMainWindow):
def _run_async(self, operation, callback, button=None, *, expected_task_id=None,
retain_result=False, operation_area='', on_finished=None, **_unused):
"""Call on_finished after releasing the worker lane, including failures.
A modal result callback can receive finished before that callback returns.
Closing windows suppress the continuation.
"""
if (self._workers or self._closed or (self._dsh_task_creation is not None and self._dsh_task_creation.dialog is not None)
or (self._dsh_lifecycle is not None and self._dsh_lifecycle.dialog is not None)
or (self._dsh_operations is not None and self._dsh_operations.dialog is not None)):
@@ -52,6 +52,12 @@ def _worker_tree_exited(process, owned):
return False
def _wait_worker_tree_exit(process, owned, deadline):
while not _worker_tree_exited(process, owned) and time.monotonic() < deadline:
time.sleep(.01)
return _worker_tree_exited(process, owned)
def _retain_worker(root, token, job_id, process, owned, close_process, error):
from .batch_resources import process_identity, write_record
from .stop_control import protect_interruption
@@ -276,6 +282,7 @@ class _OwnedWorkers:
win32job.AssignProcessToJobObject(self.job, process._handle)
def stop_workers(self, process: subprocess.Popen) -> list[str]:
import pywintypes
import win32api
import win32job
@@ -301,8 +308,9 @@ class _OwnedWorkers:
else:
failures.append(f'preserved_non_worker pid={process_id}')
except Exception as error:
code = getattr(error, 'winerror', error.args[0] if error.args else None)
if handle is not None or code != 87: # OpenProcess may observe an already exited PID.
vanished = (handle is None and isinstance(error, (OSError, pywintypes.error))
and getattr(error, 'winerror', None) == 87)
if not vanished: # OpenProcess may observe an already exited PID.
failures.append(f'child_cleanup pid={process_id} error={error}')
finally:
if handle is not None:
@@ -487,6 +495,7 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor
raise WorkerMonitorSafeTimeout('project open made no progress; safe stop was requested',
stage=watch.stage)
raise subprocess.TimeoutExpired(command, idle_timeout * 2)
exit_deadline = time.monotonic() + 10
drain_deadline = time.monotonic() + 5
while (any(thread.is_alive() for thread in threads) or not messages.empty()) and time.monotonic() < drain_deadline:
try:
@@ -494,7 +503,8 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor
except queue.Empty:
pass
_publish_progress(progress_path, watch.snapshot())
if not cooperative_abort and not _worker_tree_exited(process, owned):
# A root can exit before Job accounting confirms its descendants exited.
if not cooperative_abort and not _wait_worker_tree_exit(process, owned, exit_deadline):
raise WorkerMonitorError('worker root exited while owned descendants remain; reconciliation required')
return subprocess.CompletedProcess(command, process.returncode, _decode(output[0]), _decode(output[1]))
except Exception as error:
@@ -524,8 +534,7 @@ def run_monitored_process(command: tuple[str, ...], idle_timeout: float, *, abor
# Root exit and Job accounting need not become observable in
# the same tick. Confirm naturally exiting descendants within
# the existing worker-cleanup budget; never force unknown ones.
while not _worker_tree_exited(process, owned) and time.monotonic() < cleanup_deadline:
time.sleep(.01)
_wait_worker_tree_exit(process, owned, cleanup_deadline)
diagnostic = watch.snapshot() | {'monitor_error': str(error), 'worker_cleanup_notes': failures}
try:
_atomic_text(progress_path, json.dumps(diagnostic, ensure_ascii=False))
@@ -8,6 +8,58 @@ import pytest
from dx_productivity.trial_task_center import worker_monitor
@pytest.mark.parametrize('exit_after', [0, .03, None])
@pytest.mark.parametrize('returncode', [0, 7])
@pytest.mark.parametrize('tree_witness', ['monitor', 'native_adapter'])
def test_normal_completion_waits_for_tree_exit_without_killing_or_replaying(tmp_path, monkeypatch, exit_after, returncode, tree_witness):
clock = SimpleNamespace(now=0.0)
monkeypatch.setattr(worker_monitor, 'time', SimpleNamespace(monotonic=lambda: clock.now,
sleep=lambda delay: setattr(clock, 'now', clock.now + delay)))
process = SimpleNamespace(pid=123, returncode=returncode, stdout=io.BytesIO(b'real receipt\n'),
stderr=io.BytesIO(), closed=False)
process.poll = lambda: process.returncode
process.close = lambda: setattr(process, 'closed', True)
def tree_confirmed():
return owners[0].allow_exit or exit_after is not None and clock.now >= exit_after
process.tree_exited = tree_confirmed if tree_witness == 'native_adapter' else lambda: True
class Owned:
def __init__(self, *_args):
self.closed = self.allow_exit = False
owners.append(self)
def tree_exited(self):
return tree_confirmed() if tree_witness == 'monitor' else True
def stop_workers(self, _process):
pytest.fail('observing a normal exit must not terminate a worker')
def close(self):
self.closed = True
owners, launches = [], []
def launch(*_args, **_kwargs):
launches.append(1)
return process
monkeypatch.setattr(worker_monitor, '_OwnedWorkers', Owned)
try:
if exit_after is None:
with pytest.raises(worker_monitor.WorkerMonitorError, match='owned worker retained'):
worker_monitor.run_monitored_process(('worker', '', '', str(tmp_path), 'python', 'job'), 1,
process_factory=launch)
assert 10 <= clock.now <= 10.02
assert not owners[0].closed and not process.closed
assert (tmp_path / 'uncertain.flag').exists()
assert any(entry['process'] is process for entry in worker_monitor.RETAINED_WORKERS.values())
else:
result = worker_monitor.run_monitored_process(('worker', '', '', str(tmp_path), 'python', 'job'), 1,
process_factory=launch)
assert result.returncode == returncode and result.stdout == 'real receipt\n'
assert exit_after <= clock.now <= exit_after + .02
assert owners[0].closed and process.closed
assert not (tmp_path / 'uncertain.flag').exists()
assert launches == [1]
finally:
owners[0].allow_exit = True
worker_monitor.cleanup_retained_workers(tmp_path)
assert not any(entry['process'] is process for entry in worker_monitor.RETAINED_WORKERS.values())
@pytest.mark.parametrize('descendants_exit', [True, False])
def test_safe_timeout_waits_for_positive_tree_exit_without_extending_cleanup_budget(tmp_path, monkeypatch, descendants_exit):
clock = SimpleNamespace(now=0.0)
@@ -75,6 +127,22 @@ def test_child_that_exits_during_job_enumeration_is_not_a_cleanup_failure(monkey
assert owned.stop_workers(process) == []
@pytest.mark.skipif(os.name != 'nt', reason='Win32 pre-handle exception classification')
@pytest.mark.parametrize('error', [ValueError(87), OSError(87, 'not a Windows error'),
OSError(5, 'denied')])
def test_non_windows_error_87_is_not_mistaken_for_a_vanished_pid(monkeypatch, error):
import win32api
import win32job
owned = worker_monitor._OwnedWorkers.__new__(worker_monitor._OwnedWorkers)
owned.job, owned.allowed = object(), set()
monkeypatch.setattr(win32job, 'QueryInformationJobObject', lambda *_: (1, 2))
def failed(*_args):
raise error
monkeypatch.setattr(win32api, 'OpenProcess', failed)
notes = owned.stop_workers(SimpleNamespace(pid=1, poll=lambda: 0))
assert len(notes) == 1 and notes[0].startswith('child_cleanup pid=2')
@pytest.mark.skipif(os.name != 'nt', reason='Win32 post-open cleanup error')
def test_error_87_after_opening_child_remains_a_cleanup_failure(monkeypatch):
import ctypes