feat(runtime): reuse private desktop and CAD per batch with safe rebuilds

This commit is contained in:
阿白叔 committed 2026-10-21 15:00:17 +08:00
1 parent 681403a4d2
commit 89c7c603f6
6 files changed
+1015 -8

No files matched your search

@@ -0,0 +1,22 @@
# 批次私有桌面实施计划
用户已在会话中明确批准完整设计及实施;自动重试策略为仅修改前重试一次。基线 e15ee25,不发布安装包。
## 实施任务
- [x] T1:批次运行时。显式传递资源,惰性共用桌面/CAD;项目关闭检查点独立于批末桌面释放;持久化生命周期,资源失败阻止新批次,支持重新清理。正常退出优先,无法确认旧资源释放不重建。
- [x] T2:写前持久化。每次尝试有新的身份与代次,首个可能修改调用前 fsync 写入标记;失败禁止写入。仅当前尝试明确尚未写入且身份与只读指纹安全时允许一次重建重试。
- [x] T3:入口及界面。BOM/任务队列分别拥有一次运行时;单独操作兼容;跨任务复用 CAD,同工程多步骤继续复用。清理状态独立于已核验任务结果,提供清理入口;原模板和队列可读。
- [ ] T4:验证与审查。故障注入与取消竞争、旧结果隔离、共享数量和缓存全命中;工单回放及两份真实工程副本替换/保存回读/图连接校验;独立审查并修复,分阶段 Git 提交后推送。
## 接口约定
`BatchCadRuntime(root, kind, stop_event=None)` 为上下文管理器,提供 `native_scope(evidence)`、`run_worker(command, timeout, abort_event=None, cooperative_abort=False)`;`root` 是应用数据根。原生 scope 提供桌面式 start_process/close 接口,关闭只回收该次 CLI 子进程。CAD run_worker 仅在需要时启动并保存项目关闭检查点。
监督端不保存 COM 对象,运行时通过可选 `runtime` 参数显式传递;工作进程中继续核验桌面、CAD PID/GUID 和执行标识。可调用 `guard_pending_resources(root)` 与 `cleanup_pending_resources(root)`,后者仅清理资源,不执行替换。
写前协议模块独立于 Qt/COM。每个进程尝试写入 `DXPART_ATTEMPT_FILE`,由监督端先落盘准备记录,原生写入前验证并置为 mutation_possible。标记和身份缺失不能证明安全。
## 验收约束
所有真实 CAD 写入仅对原仓库 build 下的隔离副本进行,不触碰源工程与中心库。不删除原 feedback 或历史证据,不放宽保护锁。性能按同一副本和相同输入比较,不承诺实机残留根因已治愈。
@@ -0,0 +1,169 @@
"""Persistent resource ownership, independent of project replacement results."""
from __future__ import annotations
import json
import os
from pathlib import Path
import re
import time
from uuid import uuid4
from .errors import TaskCenterError
# Recovery retains owned native handles after failed cleanup. This registry is
# never an implicit execution context and never contains COM application objects.
LIVE_RESOURCES = {}
class BatchResourceCleanupError(TaskCenterError):
def __init__(self, message, *, completed_result=None):
super().__init__('batch_resources_pending', message)
self.completed_result = completed_result
def write_record(path, record):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + '.' + uuid4().hex + '.tmp')
try:
with temporary.open('w', encoding='utf-8', newline='\n') as stream:
json.dump(record, stream, ensure_ascii=False, indent=2)
stream.flush()
os.fsync(stream.fileno())
deadline = time.monotonic() + 1.0
while True:
try:
os.replace(temporary, path)
break
except PermissionError:
if time.monotonic() >= deadline:
raise
time.sleep(.02)
finally:
temporary.unlink(missing_ok=True)
def _valid_record(record, path):
if not isinstance(record, dict):
return False
session = record.get('session_id')
processes = record.get('processes')
return (record.get('version') == 1 and isinstance(session, str)
and re.fullmatch('[0-9a-f]{32}', session) is not None and path.parent.name == session
and isinstance(record.get('kind'), str) and bool(record['kind'])
and type(record.get('generation')) is int and record['generation'] >= 1
and isinstance(record.get('desktop'), str)
and re.fullmatch('dx-[0-9a-f]{32}', record['desktop']) is not None
and record.get('state') in {'starting', 'ready', 'recycling', 'closing', 'blocked', 'released'}
and isinstance(processes, list) and all(isinstance(item, dict)
and type(item.get('pid')) is int and item['pid'] > 0
and 'creation_time' in item and (item['creation_time'] is None
or isinstance(item['creation_time'], str)) for item in processes)
and 'cad_pid' in record and (record['cad_pid'] is None
or type(record['cad_pid']) is int and record['cad_pid'] > 0)
and 'cad_guid' in record and (record['cad_guid'] is None
or isinstance(record['cad_guid'], str) and bool(record['cad_guid']))
and all(isinstance(record.get(key), dict) for key in ('checkpoints', 'retries'))
and isinstance(record.get('timings'), list))
def pending_resources(root):
pending = []
for path in sorted((Path(root).resolve() / 'batch-resources').glob('*/session.json')):
try:
record = json.loads(path.read_text(encoding='utf-8'))
if not _valid_record(record, path):
raise ValueError('invalid resource ownership record')
if record.get('state') != 'released':
pending.append(dict(record, record_path=str(path)))
except (OSError, ValueError, TypeError) as error:
pending.append({'state': 'blocked', 'record_path': str(path), 'error': str(error)})
return tuple(pending)
def guard_pending_resources(root):
if pending_resources(root):
raise BatchResourceCleanupError('上一批资源尚未确认释放;请先重试释放资源,再开始新批次。')
def process_identity(process):
result = {'pid': process.pid, 'creation_time': None}
handle = getattr(process, 'handle', None)
if handle is not None:
import win32process
result['creation_time'] = str(win32process.GetProcessTimes(handle)['CreationTime'])
return result
def _process_gone(identity):
import win32api
import win32event
import win32process
try:
handle = win32api.OpenProcess(0x101000, False, identity['pid'])
except Exception as error:
code = getattr(error, 'winerror', error.args[0] if error.args else None)
if code == 87:
return True
raise
try:
if win32event.WaitForSingleObject(handle, 0) == 0:
return True
created = identity.get('creation_time')
# A reused PID is an unrelated process and must never be terminated.
return created is not None and str(win32process.GetProcessTimes(handle)['CreationTime']) != created
finally:
handle.Close()
def cleanup_pending_resources(root):
from .isolated_desktop import _Win32Api
for record in pending_resources(root):
path = Path(record['record_path'])
live = LIVE_RESOURCES.get(str(path))
try:
if live is not None:
live.close()
continue
if not record.get('session_id'):
continue # Corrupt evidence cannot authorize resource actions.
desktop = record.get('desktop')
if desktop:
_Win32Api().confirm_desktop_released(desktop)
if not all(_process_gone(item) for item in record.get('processes', ())):
continue
# After restart only confirm disappearance. Never attach to COM,
# close a project, or terminate a PID whose handles we no longer own.
record.pop('record_path', None)
write_record(path, dict(record, state='released', reconciled=True, released_at=time.time()))
except Exception as error:
if record.get('session_id'):
record.pop('record_path', None)
write_record(path, dict(record, state='blocked', error=str(error)))
return pending_resources(root)
def project_checkpoint_released(root):
root = Path(root).resolve()
try:
checkpoint = json.loads((root / 'project-checkpoint.json').read_text(encoding='utf-8'))
path = Path(checkpoint['resource_record']).resolve()
record = json.loads(path.read_text(encoding='utf-8'))
key = str(root).casefold()
return (_valid_record(record, path) and checkpoint.get('version') == 1
and type(checkpoint.get('generation')) is int
and isinstance(checkpoint.get('attempt_id'), str)
and re.fullmatch('[0-9a-f]{32}', checkpoint['attempt_id']) is not None
and type(checkpoint.get('cad_pid')) is int and checkpoint['cad_pid'] > 0
and isinstance(checkpoint.get('cad_guid'), str) and bool(checkpoint['cad_guid'])
and isinstance(checkpoint.get('project_path'), str)
and Path(checkpoint['project_path']).is_absolute()
and checkpoint['project_closed'] is True and checkpoint['worker_closed'] is True
and checkpoint['evidence_root'] == str(root)
and record['state'] in {'ready', 'closing', 'released'}
and (checkpoint['session_id'], checkpoint['generation'], checkpoint['cad_pid'], checkpoint['cad_guid'])
== (record['session_id'], record['generation'], record['cad_pid'], record['cad_guid'])
and record['checkpoints'][key] == checkpoint['attempt_id'])
except (OSError, ValueError, TypeError, KeyError):
return False
@@ -0,0 +1,338 @@
"""One lazy private desktop/CAD owner per import or execution batch."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import subprocess
import time
from uuid import uuid4
from .batch_resources import (BatchResourceCleanupError, LIVE_RESOURCES, cleanup_pending_resources,
guard_pending_resources, pending_resources, process_identity, project_checkpoint_released, write_record)
from .errors import TaskCenterError
from .isolated_desktop import PrivateDesktop
from .operation_control import check_cancelled
from .worker_monitor import WorkerMonitorError
class _DesktopLease:
"""Close only this operation's processes; the batch owns the desktop itself."""
def __init__(self, owner, evidence, *, native=False):
self.owner, self.evidence = owner, Path(evidence)
self.native, self.attempt_id = native, uuid4().hex
self.generation, self.desktop = None, None
self.processes = []
self.cleanup_record = None
@property
def name(self):
return (self.desktop if self.desktop is not None else self.owner._ensure_desktop()).name
def __enter__(self):
self.desktop = self.owner._ensure_desktop()
self.generation = self.owner.generation
if self.native:
self.owner.record['native_operations'][str(self.evidence.resolve())] = {
'binding': self.identity, 'workers_closed': False}
self.owner._save()
return self
@property
def identity(self):
return {'session_id': self.owner.session_id, 'generation': self.generation,
'attempt_id': self.attempt_id}
def __exit__(self, *_exc):
self.close()
def start_process(self, command, **options):
available = self.owner.record['state'] == 'ready' or (
not self.native and self.owner.record['state'] == 'closing')
if self.desktop is None or self.generation != self.owner.generation or not available:
raise WorkerMonitorError('stale or unavailable batch resource lease')
process = self.desktop.start_process(command, **options)
self.processes.append(process)
self.owner._track(process)
return process
def close(self):
errors, remaining, records = [], [], []
for process in reversed(self.processes):
try:
self.owner._release_process(process, desktop=self.desktop, generation=self.generation)
records.append(getattr(process, 'cleanup_record', {}))
except Exception as error:
errors.append(str(error))
remaining.append(process)
self.processes = list(reversed(remaining))
self.cleanup_record = {'scope': 'operation', 'processes': records,
'workers_closed': not errors, 'desktop_retained_by_batch': True, 'errors': errors}
if errors:
self.owner._save(state='blocked', error='; '.join(errors))
raise BatchResourceCleanupError('工作进程未能确认退出;批次已停止。' + '; '.join(errors))
if self.native and self.generation == self.owner.generation:
self.owner.record['native_operations'][str(self.evidence.resolve())] = {
'binding': self.identity, 'workers_closed': True}
self.owner._save()
class BatchCadRuntime:
def __init__(self, root, kind, stop_event=None, *, desktop_factory=None, cad_factory=None):
self.root = Path(root).resolve()
self.kind, self.stop_event = kind, stop_event
self.session_id = uuid4().hex
self.path = self.root / 'batch-resources' / self.session_id / 'session.json'
self.desktop_factory = desktop_factory or PrivateDesktop
self.cad_factory = cad_factory
self.desktop = self.cad = None
self.generation = 0
self.result = None
self.record = {'version': 1, 'session_id': self.session_id, 'kind': kind, 'state': 'new',
'generation': 0, 'desktop': None, 'cad_pid': None, 'cad_guid': None,
'processes': [], 'checkpoints': {}, 'retries': {}, 'timings': [], 'native_operations': {}}
self._retry_counts = {}
self._project_attempts = {}
def __enter__(self):
guard_pending_resources(self.root)
return self
def __exit__(self, error_type, error, traceback):
try:
self.close()
except Exception as cleanup:
raise BatchResourceCleanupError('批次资源释放待处理;已核验任务结果保留。' + str(cleanup),
completed_result=self.result) from cleanup
def _save(self, **values):
record = dict(self.record, **values)
write_record(self.path, record)
self.record = record
def _timing(self, phase, started, **details):
self.record['timings'].append({'phase': phase, 'seconds': time.monotonic() - started,
'generation': self.generation, **details})
def _ensure_desktop(self, abort_event=None):
if self.record['state'] == 'blocked':
raise BatchResourceCleanupError('当前批次资源已阻断,不能启动新的工作进程。')
if self.desktop is None:
check_cancelled(self.stop_event if abort_event is None else abort_event)
self.generation += 1
name = 'dx-' + uuid4().hex
self._save(state='starting', generation=self.generation, desktop=name,
processes=[], cad_pid=None, cad_guid=None, checkpoints={}, native_operations={})
LIVE_RESOURCES[str(self.path)] = self
self.desktop = self.desktop_factory(name, enabled=True)
started = time.monotonic()
try:
self.desktop.__enter__()
except Exception:
self._save(state='blocked', error='private desktop startup failed')
raise
self._timing('desktop_start', started)
self._save(state='ready')
return self.desktop
def _track(self, process):
identity = process_identity(process)
if identity not in self.record['processes']:
self.record['processes'].append(identity)
self._save()
def _release_process(self, process, *, desktop=None, generation=None):
process.close()
release = getattr(desktop if desktop is not None else self.desktop, 'forget_closed_process', None)
if release:
release(process)
if generation is None or generation == self.generation:
self.record['processes'] = [item for item in self.record['processes'] if item['pid'] != process.pid]
def native_scope(self, evidence):
return _DesktopLease(self, evidence, native=True)
def _ensure_cad(self, abort_event=None):
if self.cad is None:
from .isolated_cad import IsolatedCadSession
factory = self.cad_factory or IsolatedCadSession
lease = _DesktopLease(self, self.path.parent)
started = time.monotonic()
cad = factory(self.path.parent / f'generation-{self.generation + (self.desktop is None)}',
abort_event=abort_event, desktop=lease)
self.cad = cad # Retain cleanup ownership even if startup fails.
try:
cad.__enter__()
except Exception as error:
from .isolated_cad import PrivateCadStartupError
if isinstance(error, PrivateCadStartupError):
# __enter__ certified its local processes/handles closed.
# A never-ready instance must not receive a second Quit.
self.cad = None
raise
self._timing('cad_start', started)
self._save(state='ready', cad_pid=cad.binding.pid, cad_guid=cad.binding.guid)
return self.cad
def _prepare_attempts(self, root, command, attempt_id):
from .batch_attempt import prepare_attempt
from .quick_run import _read_commands
rows = _read_commands(root / 'command.tsv')
selected_rows = [row for row in rows if row[1] == command[-1]]
if len(selected_rows) != 1 or len({row[1] for row in rows}) != len(rows):
raise TaskCenterError('target_identity', '批次工作命令身份缺失或重复,已停止。')
selected = selected_rows[0]
project = Path(selected[2]).resolve()
self._save(active_project=str(project), active_task=selected[0], active_job=selected[1])
directory = root / 'batch-attempts' / (self.session_id + f'-g{self.generation}-' + attempt_id)
expected = []
for row in rows:
if Path(row[2]).resolve() == project:
path = directory / (row[1] + '.json')
record = prepare_attempt(path, session_id=self.session_id, generation=self.generation,
task_id=row[0], job_id=row[1], project_path=str(project), attempt_id=attempt_id)
expected.append((path, record))
self._project_attempts.setdefault(str(project).casefold(), []).extend(expected)
return project, directory, expected
def _can_retry(self, project, expected):
from .batch_attempt import can_retry_attempt
key = str(project).casefold()
return (not self._retry_counts.get(key) and expected
and all(can_retry_attempt(path, record) for path, record in self._project_attempts.get(key, ())))
def _checkpoint(self, root, project, attempt_id):
self.cad.checkpoint(root / ('cad-closed-' + attempt_id + '.json'))
key = str(root).casefold()
self.record['checkpoints'][key] = attempt_id
self._save(state='ready')
write_record(root / 'project-checkpoint.json', {'version': 1, 'resource_record': str(self.path),
'session_id': self.session_id, 'generation': self.generation, 'attempt_id': attempt_id,
'evidence_root': str(root), 'project_path': str(project), 'project_closed': True,
'worker_closed': True, 'cad_pid': self.record['cad_pid'], 'cad_guid': self.record['cad_guid']})
def run_worker(self, command, timeout, *, abort_event=None, cooperative_abort=False):
from .discovery_cache import project_fingerprint
from .isolated_cad import PrivateCadStartupError
from .worker_monitor import run_monitored_process
root = Path(command[3]).resolve()
self._ensure_desktop(abort_event)
while True:
check_cancelled(abort_event or self.stop_event)
attempt_id = uuid4().hex
# No previous empty-project certificate may authorize continuation
# once another operation could attach or open a project.
self.record['checkpoints'].pop(str(root).casefold(), None)
self._save(active_attempt=attempt_id)
project, directory, expected = self._prepare_attempts(root, command, attempt_id)
before = project_fingerprint(project, self.root)
status = root / 'status.tsv'
status_before = status.read_bytes() if status.exists() else None
started = time.monotonic()
failure = None
try:
cad = self._ensure_cad(abort_event)
cad.checkpoint(root / ('cad-ready-' + attempt_id + '.json'))
environment = {'DXPART_ATTEMPT_FILE': str(directory), 'DXPART_ATTEMPT_ID': attempt_id,
'DXPART_BATCH_SESSION_ID': self.session_id, 'DXPART_BATCH_GENERATION': str(self.generation)}
def start_worker(launch, **options):
return cad.start_worker(launch, extra_environment=environment, **options)
result = run_monitored_process(command, timeout, abort_event=abort_event,
cooperative_abort=cooperative_abort, process_factory=start_worker)
self._checkpoint(root, project, attempt_id)
if result.returncode not in (2, 3):
return result
failure = WorkerMonitorError(result.stderr or 'pre-mutation worker startup failed')
except (PrivateCadStartupError, WorkerMonitorError, subprocess.TimeoutExpired, OSError) as error:
failure = error
finally:
self._timing('project_worker', started, project=str(project), attempt_id=attempt_id,
task_id=self.record['active_task'], job_id=self.record['active_job'])
# Missing proof, changed inputs/status, cancellation or a previous
# write anywhere in this project can never authorize replay.
after = project_fingerprint(project, self.root)
status_after = status.read_bytes() if status.exists() else None
# Runtime database changes require separate identity evidence. Until
# available for a failed read, use the stricter full fingerprint.
safe = (before is not None and before == after
and status_before == status_after and self._can_retry(project, expected)
and not (root / 'recovery.request').exists()
and not (abort_event is not None and abort_event.is_set())
and not (self.stop_event is not None and self.stop_event.is_set()))
if not safe:
raise failure
key = str(project).casefold()
self._retry_counts[key] = 1
self.record['retries'][key] = {'count': 1, 'reason': str(failure), 'attempt_id': attempt_id}
self._save(state='recycling')
self.close(rebuilding=True) # Prove disappearance before creating another.
self.desktop = self.cad = None
self._ensure_desktop(abort_event)
def recover_native_failure(self, project, evidence, reason):
"""One read-only recovery, based on durable source and worker proof."""
from .batch_attempt import can_retry_attempt
project, evidence = Path(project).resolve(), Path(evidence).resolve()
key = str(project).casefold()
if (self._retry_counts.get(key) or self.record['state'] == 'blocked'
or self.stop_event is not None and self.stop_event.is_set()):
return False
try:
proof = json.loads((evidence / 'native-retry-proof.json').read_text(encoding='utf-8'))
sources = proof['source_hashes']
operation = self.record['native_operations'][str(evidence)]
if (proof['version'] != 1 or proof['native_read_only'] is not True
or operation['workers_closed'] is not True or proof['binding'] != operation['binding']
or proof['binding']['session_id'] != self.session_id
or proof['binding']['generation'] != self.generation
or proof['workers_closed'] is not True or proof['project_path'] != str(project)
or proof['evidence_root'] != str(evidence) or str(project) not in sources
or len(sources) != 2
or not all(Path(name).resolve().is_relative_to(project.parent) for name in sources)
or not all(hashlib.sha256(Path(name).read_bytes()).hexdigest() == digest
for name, digest in sources.items())
or not all(can_retry_attempt(path, record)
for path, record in self._project_attempts.get(key, ()))):
return False
except (OSError, ValueError, KeyError, TypeError):
return False
self._retry_counts[key] = 1
self.record['retries'][key] = {'count': 1, 'reason': str(reason), 'backend': 'native_cli'}
self._save(state='recycling')
self.close(rebuilding=True)
self.desktop = self.cad = None
self._ensure_desktop()
return True
def close(self, *, rebuilding=False):
if self.desktop is None and not self.generation:
return # A cache-only batch never acquired resources.
if self.record['state'] == 'released':
return
started = time.monotonic()
errors = []
try:
self._save(state='closing')
except Exception as error:
errors.append('resource journal: ' + str(error))
if self.cad is not None:
cad, self.cad = self.cad, None
try:
cad.__exit__(None, None, None)
except Exception as error:
if rebuilding:
self.record['shutdown_warning'] = str(error)
else:
errors.append(str(error))
if self.desktop is not None:
try:
self.desktop.close()
except Exception as error:
errors.append(str(error))
self._timing('batch_release', started)
self.record['cleanup'] = getattr(self.desktop, 'cleanup_record', None)
if errors:
self._save(state='blocked', error='; '.join(errors))
raise BatchResourceCleanupError('; '.join(errors), completed_result=self.result)
self._save(state='released', released_at=time.time(), processes=[])
LIVE_RESOURCES.pop(str(self.path), None)
@@ -65,10 +65,11 @@ def _designer_executable():
class IsolatedCadSession:
"""The supervisor never attaches to COM on the user's input desktop."""
def __init__(self, root, *, abort_event=None):
def __init__(self, root, *, abort_event=None, desktop=None):
self.root = Path(root).resolve()
self.token = uuid4().hex
self.desktop = PrivateDesktop('dx-' + self.token, enabled=True)
self.borrowed_desktop = desktop is not None
self.desktop = desktop if desktop is not None else PrivateDesktop('dx-' + self.token, enabled=True)
self.cad = None
self.binding = None
self.started = time.monotonic()
@@ -132,10 +133,36 @@ class IsolatedCadSession:
if self.abort_event is not None and self.abort_event.is_set():
raise WorkerMonitorError('private CAD startup was cancelled before execution')
def start_worker(self, command, **options):
def start_worker(self, command, *, extra_environment=None, **options):
if self.binding is None or self.cad.poll() is not None:
raise WorkerMonitorError('owned private CAD is unavailable')
return self.desktop.start_process(command, env=_child_environment(self.binding.environment()), **options)
environment = self.binding.environment()
environment.update(extra_environment or {})
# The verified CAD binding cannot be overridden by attempt metadata.
binding = self.binding.environment()
for key in ('DXPART_DESKTOP', 'DXPART_CAD_PID', 'DXPART_CAD_GUID', 'DXPART_SESSION_TOKEN'):
environment[key] = binding[key]
return self.desktop.start_process(command, env=_child_environment(environment), **options)
def checkpoint(self, evidence):
"""Certify an idle, identity-verified CAD before crossing project boundaries."""
if self.binding is None or self.cad.poll() is not None:
raise WorkerMonitorError('owned CAD unavailable at project checkpoint')
evidence = Path(evidence)
evidence.parent.mkdir(parents=True, exist_ok=True)
process = self.start_worker(_helper_command('checkpoint', evidence, self.cad.pid),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
if process.wait(timeout=20):
raise WorkerMonitorError('project close checkpoint failed: ' +
process.stderr.read().decode('utf-8', errors='replace'))
result = json.loads(evidence.read_text(encoding='utf-8'))
if (result.get('binding') != asdict(self.binding) or result.get('project_path') != ''
or result.get('project_closed') is not True or self.cad.poll() is not None):
raise WorkerMonitorError('project close checkpoint identity mismatch')
finally:
process.close()
return result
def __exit__(self, error_type, error, traceback):
closing_started = time.monotonic()
@@ -174,7 +201,9 @@ class IsolatedCadSession:
finally:
try:
self.desktop.close()
self._record(state='released', clean=error is None and cleanup_error is None,
self._record(state='session_closed' if self.borrowed_desktop else 'released',
desktop_retained_by_batch=self.borrowed_desktop,
clean=error is None and cleanup_error is None,
shutdown_seconds=time.monotonic() - closing_started,
desktop_cleanup=getattr(self.desktop, 'cleanup_record', None), **exit_codes)
except Exception as failure:
@@ -190,8 +219,11 @@ class IsolatedCadSession:
self._record(state='uncertain', error=message, **details)
def run_isolated_process(command, timeout, *, abort_event=None, cooperative_abort=False):
def run_isolated_process(command, timeout, *, abort_event=None, cooperative_abort=False, runtime=None):
from .worker_monitor import run_monitored_process
if runtime is not None:
return runtime.run_worker(command, timeout, abort_event=abort_event,
cooperative_abort=cooperative_abort)
try:
with IsolatedCadSession(Path(command[3]), abort_event=abort_event) as session:
return run_monitored_process(command, timeout, abort_event=abort_event, process_factory=session.start_worker,
@@ -252,14 +284,18 @@ def _native_action(action, evidence, expected_pid):
binding = CadBinding(desktop, expected_pid, str(app.Guid()), token)
verify_application(app, binding, require_view=False)
Path(evidence).write_text(json.dumps(asdict(binding)), encoding='utf-8')
elif action == 'quit':
elif action in {'quit', 'checkpoint'}:
binding = CadBinding.from_environment()
# ActiveViewHandle can retain a destroyed/reused HWND after closing
# the last project. The private ROT/GUID binds this empty instance.
verify_application(app, binding, require_view=False)
if project_file_path(app):
raise ValueError('owned CAD still has a project open; no implicit save or discard')
app.Quit()
if action == 'quit':
app.Quit()
else:
from .batch_resources import write_record
write_record(evidence, {'binding': asdict(binding), 'project_path': '', 'project_closed': True})
else:
raise ValueError('unknown private CAD action')
except BaseException as failure:
@@ -473,6 +473,16 @@ class PrivateDesktop:
self._processes.append(process)
return process
def forget_closed_process(self, process):
"""Drop a worker only after its owning wrapper confirmed all cleanup."""
record = getattr(process, 'cleanup_record', {})
if (record.get('process_handle_closed') is not True
or record.get('job_handle_closed') is not True
or record.get('unconfirmed_descendant_handles', 0)
or record.get('errors') or process.poll() is None):
raise OSError('cannot forget a worker with unconfirmed owned resources')
self._processes = [item for item in self._processes if item is not process]
def close(self) -> None:
errors, remaining = [], []
self.cleanup_record = {'desktop': self.name, 'processes': []}
+432
View File
@@ -0,0 +1,432 @@
"""Batch resources are lazy, shared, and independently blocked after failed release."""
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from dx_productivity.trial_task_center.errors import TaskCenterError
class Desktop:
created = []
fail_release = False
def __init__(self, name, **_options):
self.name = name
self.processes = []
self.released = False
self.cleanup_record = None
def __enter__(self):
self.created.append(self)
return self
def start_process(self, command, **_options):
process = SimpleNamespace(pid=100 + len(self.processes), closed=False,
cleanup_record={}, returncode=0)
def close():
process.closed = True
process.cleanup_record = {'pid': process.pid, 'process_handle_closed': True,
'job_handle_closed': True, 'errors': []}
process.close = close
process.poll = lambda: process.returncode
self.processes.append(process)
return process
def close(self):
for process in self.processes:
process.close()
self.cleanup_record = {'desktop_handle_closed': True,
'desktop_confirmation': {'released': not self.fail_release}, 'errors': []}
if self.fail_release:
raise OSError('outside owner retains desktop')
self.released = True
@pytest.fixture(autouse=True)
def reset_desktop():
Desktop.created = []
Desktop.fail_release = False
def test_cache_only_batch_does_not_create_desktop_or_cad(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop):
pass
assert not Desktop.created
def test_native_operations_share_one_desktop_but_release_each_worker(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
names = []
for name in ('first', 'second'):
with runtime.native_scope(tmp_path / name) as scope:
names.append(scope.name)
process = scope.start_process(('inventory.exe',))
assert not process.closed
assert process.closed and not Desktop.created[0].released
assert names[0] == names[1]
assert len(Desktop.created) == 1 and Desktop.created[0].released
def test_failed_final_release_blocks_next_batch_and_resource_cleanup_never_replays(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import (
BatchCadRuntime, cleanup_pending_resources, guard_pending_resources, pending_resources)
Desktop.fail_release = True
with pytest.raises(TaskCenterError) as caught:
with BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) as runtime:
with runtime.native_scope(tmp_path / 'operation'):
pass
runtime.result = 'verified completed task'
assert caught.value.code == 'batch_resources_pending'
assert caught.value.completed_result == 'verified completed task'
assert pending_resources(tmp_path)
with pytest.raises(TaskCenterError):
guard_pending_resources(tmp_path)
assert not list(tmp_path.rglob('uncertain.flag'))
Desktop.fail_release = False
assert cleanup_pending_resources(tmp_path) == ()
guard_pending_resources(tmp_path)
assert len(Desktop.created) == 1
def test_cleanup_failure_takes_priority_over_cancel(tmp_path):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
Desktop.fail_release = True
with pytest.raises(TaskCenterError) as caught:
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
with runtime.native_scope(tmp_path / 'operation'):
pass
raise TaskCenterError('operation_cancelled', 'cancel requested')
assert caught.value.code == 'batch_resources_pending'
Desktop.fail_release = False
from dx_productivity.trial_task_center.batch_runtime import cleanup_pending_resources
cleanup_pending_resources(tmp_path)
class Cad:
created = []
def __init__(self, root, *, abort_event, desktop):
self.root, self.desktop, self.abort_event = root, desktop, abort_event
self.binding = SimpleNamespace(pid=500, guid='owned')
self.closed = False
def __enter__(self):
self.desktop.__enter__()
self.created.append(self)
return self
def checkpoint(self, evidence):
assert not self.closed
def start_worker(self, *args, **kwargs):
raise AssertionError('monitor is injected')
def __exit__(self, *_exc):
self.closed = True
self.desktop.close()
def command_for(tmp_path, name='one'):
root = tmp_path / 'data' / name
root.mkdir(parents=True)
project = tmp_path / ('project-' + name) / 'test.prj'
project.parent.mkdir()
project.write_text('project identity')
(root / 'command.tsv').write_text('task_id\tjob_id\tproject_path\told_part\tnew_part\n'
f'task\tjob\t{project}\told\tnew\n')
return ('cscript', '//nologo', 'worker.vbs', str(root), 'python', 'job'), project
def worker_runtime(tmp_path, monkeypatch, monitor):
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
from dx_productivity.trial_task_center import worker_monitor
monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor)
Cad.created = []
return BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop, cad_factory=Cad)
def test_two_projects_reuse_cad_and_produce_distinct_close_checkpoints(tmp_path, monkeypatch):
import subprocess
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
one, _ = command_for(tmp_path)
two, _ = command_for(tmp_path, 'two')
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
with runtime:
runtime.run_worker(one, 5)
assert project_checkpoint_released(Path(one[3]))
runtime.run_worker(two, 5)
assert project_checkpoint_released(Path(two[3]))
assert len(Cad.created) == len(Desktop.created) == 1
assert Cad.created[0].closed and Desktop.created[0].released
@pytest.mark.parametrize('second_fails', [False, True])
def test_prewrite_failure_releases_old_desktop_and_retries_only_once(tmp_path, monkeypatch, second_fails):
import subprocess
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
command, _ = command_for(tmp_path)
calls = []
def monitor(command, *args, **kwargs):
calls.append(len(calls))
if len(calls) == 2:
assert Desktop.created[0].released
return subprocess.CompletedProcess(command, 2 if len(calls) == 1 or second_fails else 0, '', 'startup')
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
with runtime:
if second_fails:
with pytest.raises(WorkerMonitorError):
runtime.run_worker(command, 5)
else:
runtime.run_worker(command, 5)
assert len(calls) == len(Desktop.created) == len(Cad.created) == 2
assert runtime.generation == 2
def test_possible_mutation_prevents_automatic_replay(tmp_path, monkeypatch):
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
command, _ = command_for(tmp_path)
calls = []
def monitor(*args, **kwargs):
calls.append(1)
path = next((Path(command[3]) / 'batch-attempts').rglob('job.json'))
record = json.loads(path.read_text())
record['state'] = 'mutation_possible'
path.write_text(json.dumps(record))
raise WorkerMonitorError('lost connection after possible write')
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
with runtime, pytest.raises(WorkerMonitorError):
runtime.run_worker(command, 5)
assert len(calls) == len(Desktop.created) == 1
def test_previous_checkpoint_invalidated_before_new_worker(tmp_path, monkeypatch):
import subprocess
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
command, _ = command_for(tmp_path)
calls = []
def monitor(*args, **kwargs):
assert not project_checkpoint_released(Path(command[3]))
calls.append(1)
return subprocess.CompletedProcess(command, 0, '', '')
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
with runtime:
runtime.run_worker(command, 5)
assert project_checkpoint_released(Path(command[3]))
def fail_checkpoint(*args):
raise WorkerMonitorError('cannot certify empty project')
runtime.cad.checkpoint = fail_checkpoint
# No retry after evidence has been marked as possibly modified.
path = next((Path(command[3]) / 'batch-attempts').rglob('job.json'))
record = json.loads(path.read_text())
record['state'] = 'mutation_possible'
path.write_text(json.dumps(record))
with pytest.raises(WorkerMonitorError):
runtime.run_worker(command, 5)
assert not project_checkpoint_released(Path(command[3]))
def test_attempt_publication_failure_never_launches_worker(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_attempt
command, _ = command_for(tmp_path)
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k: pytest.fail('must not launch'))
def fail(*args, **kwargs):
raise OSError('disk unavailable')
monkeypatch.setattr(batch_attempt, 'prepare_attempt', fail)
with runtime, pytest.raises(OSError):
runtime.run_worker(command, 5)
assert not Cad.created
def test_boundary_pause_does_not_abort_current_cad_startup(tmp_path, monkeypatch):
import subprocess
from threading import Event
command, _ = command_for(tmp_path)
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
pause, worker_abort = Event(), Event()
runtime.stop_event = pause
original_enter = Cad.__enter__
def enter(cad):
pause.set() # User requests pause after the current task has begun.
assert cad.abort_event is worker_abort
assert not cad.abort_event.is_set()
return original_enter(cad)
monkeypatch.setattr(Cad, '__enter__', enter)
with runtime:
runtime.run_worker(command, 5, abort_event=worker_abort)
assert not list(tmp_path.rglob('uncertain.flag'))
def test_cleanup_still_runs_when_resource_journal_write_fails(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_runtime
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
with pytest.raises(TaskCenterError) as caught:
with runtime:
with runtime.native_scope(tmp_path / 'read'):
pass
real_write = batch_runtime.write_record
def failing_write(path, record):
if record['state'] == 'closing':
raise OSError('journal unavailable')
return real_write(path, record)
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
assert Desktop.created[0].released
assert caught.value.code == 'batch_resources_pending'
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
runtime.close()
def test_real_startup_failure_with_released_lease_can_retry(tmp_path, monkeypatch):
import io
import subprocess
from dx_productivity.trial_task_center import isolated_cad, worker_monitor
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
command, _ = command_for(tmp_path)
starts = []
def executable():
starts.append(1)
if len(starts) == 1:
raise worker_monitor.WorkerMonitorError('registry temporarily unavailable')
return 'cad.exe'
class BootstrapDesktop(Desktop):
def start_process(self, command, **options):
process = super().start_process(command, **options)
process.stderr = io.BytesIO()
process.wait = lambda timeout: process.returncode
if command[0] == 'cad.exe':
self.cad = process
process.returncode = None
else:
action, evidence, pid = command
environment = options['env']
binding = {'desktop': self.name, 'pid': pid, 'guid': 'owned',
'token': environment['DXPART_SESSION_TOKEN']}
if action == 'probe':
Path(evidence).write_text(json.dumps(binding))
elif action == 'checkpoint':
Path(evidence).write_text(json.dumps({'binding': binding, 'project_path': '', 'project_closed': True}))
elif action == 'quit':
self.cad.returncode = 0
return process
monkeypatch.setattr(isolated_cad, '_designer_executable', executable)
monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: args)
monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
with BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=BootstrapDesktop) as runtime:
runtime.run_worker(command, 5)
assert len(starts) == len(Desktop.created) == 2
assert all(desktop.released for desktop in Desktop.created)
def test_native_rebuild_requires_strict_source_and_worker_proof(tmp_path):
import hashlib
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
command, project = command_for(tmp_path)
database = project.parent / 'database.dat'
database.write_bytes(b'database')
evidence = Path(command[3]) / 'native'
evidence.mkdir()
proof = {'version': 1, 'native_read_only': True, 'workers_closed': True,
'project_path': str(project.resolve()), 'evidence_root': str(evidence.resolve()),
'source_hashes': {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest()
for path in (project, database)}}
path = evidence / 'native-retry-proof.json'
path.write_text(json.dumps(proof))
with BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop) as runtime:
with runtime.native_scope(evidence) as scope:
pass
proof['binding'] = scope.identity
path.write_text(json.dumps(proof))
database.write_bytes(b'changed externally')
assert not runtime.recover_native_failure(project, evidence, 'timeout')
database.write_bytes(b'database')
assert runtime.recover_native_failure(project, evidence, 'timeout')
assert Desktop.created[0].released
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
with pytest.raises(WorkerMonitorError, match='stale'):
scope.start_process(('late-old-worker.exe',))
assert scope.identity['generation'] == 1 and runtime.generation == 2
scope.close() # A late close cannot grant the old operation a new identity.
assert str(evidence.resolve()) not in runtime.record['native_operations']
assert not runtime.recover_native_failure(project, evidence, 'another timeout')
assert len(Desktop.created) == 2
def test_release_publication_failure_can_be_reconciled_without_replaying(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_runtime
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
real_write = batch_runtime.write_record
def failing_write(path, record):
if record['state'] == 'released':
raise OSError('cannot publish release')
return real_write(path, record)
with pytest.raises(TaskCenterError):
with runtime:
with runtime.native_scope(tmp_path / 'read'):
pass
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
assert Desktop.created[0].released and batch_runtime.pending_resources(tmp_path)
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
assert batch_runtime.cleanup_pending_resources(tmp_path) == ()
assert len(Desktop.created) == 1
@pytest.mark.parametrize('record', [
{'version': 1, 'session_id': 'old', 'state': 'blocked'},
{'version': 1, 'session_id': 'old', 'state': 'released'},
[],
])
def test_unknown_resource_evidence_never_authorizes_cleanup(tmp_path, record):
from dx_productivity.trial_task_center.batch_resources import cleanup_pending_resources, pending_resources
path = tmp_path / 'batch-resources' / 'old' / 'session.json'
path.parent.mkdir(parents=True)
path.write_text(json.dumps(record))
assert pending_resources(tmp_path)
assert cleanup_pending_resources(tmp_path)
assert json.loads(path.read_text()) == record
def test_stale_generation_and_unknown_checkpoint_schema_are_rejected(tmp_path, monkeypatch):
import subprocess
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
command, _ = command_for(tmp_path)
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k:
subprocess.CompletedProcess(command, 0, '', ''))
with runtime:
runtime.run_worker(command, 5)
path = Path(command[3]) / 'project-checkpoint.json'
checkpoint = json.loads(path.read_text())
assert project_checkpoint_released(Path(command[3]))
for changes in ({'version': 999}, {'generation': 999}, {'attempt_id': 'old'}, {'cad_guid': 'foreign'}):
path.write_text(json.dumps(dict(checkpoint, **changes)))
assert not project_checkpoint_released(Path(command[3]))
def test_restart_reconciles_only_confirmed_disappearance_without_replacement(tmp_path, monkeypatch):
from dx_productivity.trial_task_center import batch_resources, isolated_desktop
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
Desktop.fail_release = True
runtime = BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
with pytest.raises(TaskCenterError):
with runtime:
with runtime.native_scope(tmp_path / 'read'):
pass
# Simulate restarting the UI: there are no retained owned handles to act on.
batch_resources.LIVE_RESOURCES.pop(str(runtime.path))
present = [True]
class Api:
def confirm_desktop_released(self, name):
assert name == runtime.record['desktop']
if present[0]:
raise OSError('desktop still held by external thread')
monkeypatch.setattr(isolated_desktop, '_Win32Api', Api)
assert batch_resources.cleanup_pending_resources(tmp_path)
present[0] = False
assert batch_resources.cleanup_pending_resources(tmp_path) == ()
assert len(Desktop.created) == 1