feat(runtime): reuse private desktop and CAD per batch with safe rebuilds
This commit is contained in:
1 parent
681403a4d2
commit
89c7c603f6
6 files changed
+1015
-8
No files matched your search
@@ -0,0 +1,22 @@
|
||||
# 批次私有桌面实施计划
|
||||
|
||||
用户已在会话中明确批准完整设计及实施;自动重试策略为仅修改前重试一次。基线 e15ee25,不发布安装包。
|
||||
|
||||
## 实施任务
|
||||
|
||||
- [x] T1:批次运行时。显式传递资源,惰性共用桌面/CAD;项目关闭检查点独立于批末桌面释放;持久化生命周期,资源失败阻止新批次,支持重新清理。正常退出优先,无法确认旧资源释放不重建。
|
||||
- [x] T2:写前持久化。每次尝试有新的身份与代次,首个可能修改调用前 fsync 写入标记;失败禁止写入。仅当前尝试明确尚未写入且身份与只读指纹安全时允许一次重建重试。
|
||||
- [x] T3:入口及界面。BOM/任务队列分别拥有一次运行时;单独操作兼容;跨任务复用 CAD,同工程多步骤继续复用。清理状态独立于已核验任务结果,提供清理入口;原模板和队列可读。
|
||||
- [ ] T4:验证与审查。故障注入与取消竞争、旧结果隔离、共享数量和缓存全命中;工单回放及两份真实工程副本替换/保存回读/图连接校验;独立审查并修复,分阶段 Git 提交后推送。
|
||||
|
||||
## 接口约定
|
||||
|
||||
`BatchCadRuntime(root, kind, stop_event=None)` 为上下文管理器,提供 `native_scope(evidence)`、`run_worker(command, timeout, abort_event=None, cooperative_abort=False)`;`root` 是应用数据根。原生 scope 提供桌面式 start_process/close 接口,关闭只回收该次 CLI 子进程。CAD run_worker 仅在需要时启动并保存项目关闭检查点。
|
||||
|
||||
监督端不保存 COM 对象,运行时通过可选 `runtime` 参数显式传递;工作进程中继续核验桌面、CAD PID/GUID 和执行标识。可调用 `guard_pending_resources(root)` 与 `cleanup_pending_resources(root)`,后者仅清理资源,不执行替换。
|
||||
|
||||
写前协议模块独立于 Qt/COM。每个进程尝试写入 `DXPART_ATTEMPT_FILE`,由监督端先落盘准备记录,原生写入前验证并置为 mutation_possible。标记和身份缺失不能证明安全。
|
||||
|
||||
## 验收约束
|
||||
|
||||
所有真实 CAD 写入仅对原仓库 build 下的隔离副本进行,不触碰源工程与中心库。不删除原 feedback 或历史证据,不放宽保护锁。性能按同一副本和相同输入比较,不承诺实机残留根因已治愈。
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Persistent resource ownership, independent of project replacement results."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from .errors import TaskCenterError
|
||||
|
||||
|
||||
# Recovery retains owned native handles after failed cleanup. This registry is
|
||||
# never an implicit execution context and never contains COM application objects.
|
||||
LIVE_RESOURCES = {}
|
||||
|
||||
|
||||
class BatchResourceCleanupError(TaskCenterError):
|
||||
def __init__(self, message, *, completed_result=None):
|
||||
super().__init__('batch_resources_pending', message)
|
||||
self.completed_result = completed_result
|
||||
|
||||
|
||||
def write_record(path, record):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(path.name + '.' + uuid4().hex + '.tmp')
|
||||
try:
|
||||
with temporary.open('w', encoding='utf-8', newline='\n') as stream:
|
||||
json.dump(record, stream, ensure_ascii=False, indent=2)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
deadline = time.monotonic() + 1.0
|
||||
while True:
|
||||
try:
|
||||
os.replace(temporary, path)
|
||||
break
|
||||
except PermissionError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(.02)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _valid_record(record, path):
|
||||
if not isinstance(record, dict):
|
||||
return False
|
||||
session = record.get('session_id')
|
||||
processes = record.get('processes')
|
||||
return (record.get('version') == 1 and isinstance(session, str)
|
||||
and re.fullmatch('[0-9a-f]{32}', session) is not None and path.parent.name == session
|
||||
and isinstance(record.get('kind'), str) and bool(record['kind'])
|
||||
and type(record.get('generation')) is int and record['generation'] >= 1
|
||||
and isinstance(record.get('desktop'), str)
|
||||
and re.fullmatch('dx-[0-9a-f]{32}', record['desktop']) is not None
|
||||
and record.get('state') in {'starting', 'ready', 'recycling', 'closing', 'blocked', 'released'}
|
||||
and isinstance(processes, list) and all(isinstance(item, dict)
|
||||
and type(item.get('pid')) is int and item['pid'] > 0
|
||||
and 'creation_time' in item and (item['creation_time'] is None
|
||||
or isinstance(item['creation_time'], str)) for item in processes)
|
||||
and 'cad_pid' in record and (record['cad_pid'] is None
|
||||
or type(record['cad_pid']) is int and record['cad_pid'] > 0)
|
||||
and 'cad_guid' in record and (record['cad_guid'] is None
|
||||
or isinstance(record['cad_guid'], str) and bool(record['cad_guid']))
|
||||
and all(isinstance(record.get(key), dict) for key in ('checkpoints', 'retries'))
|
||||
and isinstance(record.get('timings'), list))
|
||||
|
||||
|
||||
def pending_resources(root):
|
||||
pending = []
|
||||
for path in sorted((Path(root).resolve() / 'batch-resources').glob('*/session.json')):
|
||||
try:
|
||||
record = json.loads(path.read_text(encoding='utf-8'))
|
||||
if not _valid_record(record, path):
|
||||
raise ValueError('invalid resource ownership record')
|
||||
if record.get('state') != 'released':
|
||||
pending.append(dict(record, record_path=str(path)))
|
||||
except (OSError, ValueError, TypeError) as error:
|
||||
pending.append({'state': 'blocked', 'record_path': str(path), 'error': str(error)})
|
||||
return tuple(pending)
|
||||
|
||||
|
||||
def guard_pending_resources(root):
|
||||
if pending_resources(root):
|
||||
raise BatchResourceCleanupError('上一批资源尚未确认释放;请先重试释放资源,再开始新批次。')
|
||||
|
||||
|
||||
def process_identity(process):
|
||||
result = {'pid': process.pid, 'creation_time': None}
|
||||
handle = getattr(process, 'handle', None)
|
||||
if handle is not None:
|
||||
import win32process
|
||||
result['creation_time'] = str(win32process.GetProcessTimes(handle)['CreationTime'])
|
||||
return result
|
||||
|
||||
|
||||
def _process_gone(identity):
|
||||
import win32api
|
||||
import win32event
|
||||
import win32process
|
||||
try:
|
||||
handle = win32api.OpenProcess(0x101000, False, identity['pid'])
|
||||
except Exception as error:
|
||||
code = getattr(error, 'winerror', error.args[0] if error.args else None)
|
||||
if code == 87:
|
||||
return True
|
||||
raise
|
||||
try:
|
||||
if win32event.WaitForSingleObject(handle, 0) == 0:
|
||||
return True
|
||||
created = identity.get('creation_time')
|
||||
# A reused PID is an unrelated process and must never be terminated.
|
||||
return created is not None and str(win32process.GetProcessTimes(handle)['CreationTime']) != created
|
||||
finally:
|
||||
handle.Close()
|
||||
|
||||
|
||||
def cleanup_pending_resources(root):
|
||||
from .isolated_desktop import _Win32Api
|
||||
for record in pending_resources(root):
|
||||
path = Path(record['record_path'])
|
||||
live = LIVE_RESOURCES.get(str(path))
|
||||
try:
|
||||
if live is not None:
|
||||
live.close()
|
||||
continue
|
||||
if not record.get('session_id'):
|
||||
continue # Corrupt evidence cannot authorize resource actions.
|
||||
desktop = record.get('desktop')
|
||||
if desktop:
|
||||
_Win32Api().confirm_desktop_released(desktop)
|
||||
if not all(_process_gone(item) for item in record.get('processes', ())):
|
||||
continue
|
||||
# After restart only confirm disappearance. Never attach to COM,
|
||||
# close a project, or terminate a PID whose handles we no longer own.
|
||||
record.pop('record_path', None)
|
||||
write_record(path, dict(record, state='released', reconciled=True, released_at=time.time()))
|
||||
except Exception as error:
|
||||
if record.get('session_id'):
|
||||
record.pop('record_path', None)
|
||||
write_record(path, dict(record, state='blocked', error=str(error)))
|
||||
return pending_resources(root)
|
||||
|
||||
|
||||
def project_checkpoint_released(root):
|
||||
root = Path(root).resolve()
|
||||
try:
|
||||
checkpoint = json.loads((root / 'project-checkpoint.json').read_text(encoding='utf-8'))
|
||||
path = Path(checkpoint['resource_record']).resolve()
|
||||
record = json.loads(path.read_text(encoding='utf-8'))
|
||||
key = str(root).casefold()
|
||||
return (_valid_record(record, path) and checkpoint.get('version') == 1
|
||||
and type(checkpoint.get('generation')) is int
|
||||
and isinstance(checkpoint.get('attempt_id'), str)
|
||||
and re.fullmatch('[0-9a-f]{32}', checkpoint['attempt_id']) is not None
|
||||
and type(checkpoint.get('cad_pid')) is int and checkpoint['cad_pid'] > 0
|
||||
and isinstance(checkpoint.get('cad_guid'), str) and bool(checkpoint['cad_guid'])
|
||||
and isinstance(checkpoint.get('project_path'), str)
|
||||
and Path(checkpoint['project_path']).is_absolute()
|
||||
and checkpoint['project_closed'] is True and checkpoint['worker_closed'] is True
|
||||
and checkpoint['evidence_root'] == str(root)
|
||||
and record['state'] in {'ready', 'closing', 'released'}
|
||||
and (checkpoint['session_id'], checkpoint['generation'], checkpoint['cad_pid'], checkpoint['cad_guid'])
|
||||
== (record['session_id'], record['generation'], record['cad_pid'], record['cad_guid'])
|
||||
and record['checkpoints'][key] == checkpoint['attempt_id'])
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
return False
|
||||
@@ -0,0 +1,338 @@
|
||||
"""One lazy private desktop/CAD owner per import or execution batch."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from .batch_resources import (BatchResourceCleanupError, LIVE_RESOURCES, cleanup_pending_resources,
|
||||
guard_pending_resources, pending_resources, process_identity, project_checkpoint_released, write_record)
|
||||
from .errors import TaskCenterError
|
||||
from .isolated_desktop import PrivateDesktop
|
||||
from .operation_control import check_cancelled
|
||||
from .worker_monitor import WorkerMonitorError
|
||||
|
||||
|
||||
class _DesktopLease:
|
||||
"""Close only this operation's processes; the batch owns the desktop itself."""
|
||||
def __init__(self, owner, evidence, *, native=False):
|
||||
self.owner, self.evidence = owner, Path(evidence)
|
||||
self.native, self.attempt_id = native, uuid4().hex
|
||||
self.generation, self.desktop = None, None
|
||||
self.processes = []
|
||||
self.cleanup_record = None
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return (self.desktop if self.desktop is not None else self.owner._ensure_desktop()).name
|
||||
|
||||
def __enter__(self):
|
||||
self.desktop = self.owner._ensure_desktop()
|
||||
self.generation = self.owner.generation
|
||||
if self.native:
|
||||
self.owner.record['native_operations'][str(self.evidence.resolve())] = {
|
||||
'binding': self.identity, 'workers_closed': False}
|
||||
self.owner._save()
|
||||
return self
|
||||
|
||||
@property
|
||||
def identity(self):
|
||||
return {'session_id': self.owner.session_id, 'generation': self.generation,
|
||||
'attempt_id': self.attempt_id}
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
self.close()
|
||||
|
||||
def start_process(self, command, **options):
|
||||
available = self.owner.record['state'] == 'ready' or (
|
||||
not self.native and self.owner.record['state'] == 'closing')
|
||||
if self.desktop is None or self.generation != self.owner.generation or not available:
|
||||
raise WorkerMonitorError('stale or unavailable batch resource lease')
|
||||
process = self.desktop.start_process(command, **options)
|
||||
self.processes.append(process)
|
||||
self.owner._track(process)
|
||||
return process
|
||||
|
||||
def close(self):
|
||||
errors, remaining, records = [], [], []
|
||||
for process in reversed(self.processes):
|
||||
try:
|
||||
self.owner._release_process(process, desktop=self.desktop, generation=self.generation)
|
||||
records.append(getattr(process, 'cleanup_record', {}))
|
||||
except Exception as error:
|
||||
errors.append(str(error))
|
||||
remaining.append(process)
|
||||
self.processes = list(reversed(remaining))
|
||||
self.cleanup_record = {'scope': 'operation', 'processes': records,
|
||||
'workers_closed': not errors, 'desktop_retained_by_batch': True, 'errors': errors}
|
||||
if errors:
|
||||
self.owner._save(state='blocked', error='; '.join(errors))
|
||||
raise BatchResourceCleanupError('工作进程未能确认退出;批次已停止。' + '; '.join(errors))
|
||||
if self.native and self.generation == self.owner.generation:
|
||||
self.owner.record['native_operations'][str(self.evidence.resolve())] = {
|
||||
'binding': self.identity, 'workers_closed': True}
|
||||
self.owner._save()
|
||||
|
||||
|
||||
class BatchCadRuntime:
|
||||
def __init__(self, root, kind, stop_event=None, *, desktop_factory=None, cad_factory=None):
|
||||
self.root = Path(root).resolve()
|
||||
self.kind, self.stop_event = kind, stop_event
|
||||
self.session_id = uuid4().hex
|
||||
self.path = self.root / 'batch-resources' / self.session_id / 'session.json'
|
||||
self.desktop_factory = desktop_factory or PrivateDesktop
|
||||
self.cad_factory = cad_factory
|
||||
self.desktop = self.cad = None
|
||||
self.generation = 0
|
||||
self.result = None
|
||||
self.record = {'version': 1, 'session_id': self.session_id, 'kind': kind, 'state': 'new',
|
||||
'generation': 0, 'desktop': None, 'cad_pid': None, 'cad_guid': None,
|
||||
'processes': [], 'checkpoints': {}, 'retries': {}, 'timings': [], 'native_operations': {}}
|
||||
self._retry_counts = {}
|
||||
self._project_attempts = {}
|
||||
|
||||
def __enter__(self):
|
||||
guard_pending_resources(self.root)
|
||||
return self
|
||||
|
||||
def __exit__(self, error_type, error, traceback):
|
||||
try:
|
||||
self.close()
|
||||
except Exception as cleanup:
|
||||
raise BatchResourceCleanupError('批次资源释放待处理;已核验任务结果保留。' + str(cleanup),
|
||||
completed_result=self.result) from cleanup
|
||||
|
||||
def _save(self, **values):
|
||||
record = dict(self.record, **values)
|
||||
write_record(self.path, record)
|
||||
self.record = record
|
||||
|
||||
def _timing(self, phase, started, **details):
|
||||
self.record['timings'].append({'phase': phase, 'seconds': time.monotonic() - started,
|
||||
'generation': self.generation, **details})
|
||||
|
||||
def _ensure_desktop(self, abort_event=None):
|
||||
if self.record['state'] == 'blocked':
|
||||
raise BatchResourceCleanupError('当前批次资源已阻断,不能启动新的工作进程。')
|
||||
if self.desktop is None:
|
||||
check_cancelled(self.stop_event if abort_event is None else abort_event)
|
||||
self.generation += 1
|
||||
name = 'dx-' + uuid4().hex
|
||||
self._save(state='starting', generation=self.generation, desktop=name,
|
||||
processes=[], cad_pid=None, cad_guid=None, checkpoints={}, native_operations={})
|
||||
LIVE_RESOURCES[str(self.path)] = self
|
||||
self.desktop = self.desktop_factory(name, enabled=True)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
self.desktop.__enter__()
|
||||
except Exception:
|
||||
self._save(state='blocked', error='private desktop startup failed')
|
||||
raise
|
||||
self._timing('desktop_start', started)
|
||||
self._save(state='ready')
|
||||
return self.desktop
|
||||
|
||||
def _track(self, process):
|
||||
identity = process_identity(process)
|
||||
if identity not in self.record['processes']:
|
||||
self.record['processes'].append(identity)
|
||||
self._save()
|
||||
|
||||
def _release_process(self, process, *, desktop=None, generation=None):
|
||||
process.close()
|
||||
release = getattr(desktop if desktop is not None else self.desktop, 'forget_closed_process', None)
|
||||
if release:
|
||||
release(process)
|
||||
if generation is None or generation == self.generation:
|
||||
self.record['processes'] = [item for item in self.record['processes'] if item['pid'] != process.pid]
|
||||
|
||||
def native_scope(self, evidence):
|
||||
return _DesktopLease(self, evidence, native=True)
|
||||
|
||||
def _ensure_cad(self, abort_event=None):
|
||||
if self.cad is None:
|
||||
from .isolated_cad import IsolatedCadSession
|
||||
factory = self.cad_factory or IsolatedCadSession
|
||||
lease = _DesktopLease(self, self.path.parent)
|
||||
started = time.monotonic()
|
||||
cad = factory(self.path.parent / f'generation-{self.generation + (self.desktop is None)}',
|
||||
abort_event=abort_event, desktop=lease)
|
||||
self.cad = cad # Retain cleanup ownership even if startup fails.
|
||||
try:
|
||||
cad.__enter__()
|
||||
except Exception as error:
|
||||
from .isolated_cad import PrivateCadStartupError
|
||||
if isinstance(error, PrivateCadStartupError):
|
||||
# __enter__ certified its local processes/handles closed.
|
||||
# A never-ready instance must not receive a second Quit.
|
||||
self.cad = None
|
||||
raise
|
||||
self._timing('cad_start', started)
|
||||
self._save(state='ready', cad_pid=cad.binding.pid, cad_guid=cad.binding.guid)
|
||||
return self.cad
|
||||
|
||||
def _prepare_attempts(self, root, command, attempt_id):
|
||||
from .batch_attempt import prepare_attempt
|
||||
from .quick_run import _read_commands
|
||||
rows = _read_commands(root / 'command.tsv')
|
||||
selected_rows = [row for row in rows if row[1] == command[-1]]
|
||||
if len(selected_rows) != 1 or len({row[1] for row in rows}) != len(rows):
|
||||
raise TaskCenterError('target_identity', '批次工作命令身份缺失或重复,已停止。')
|
||||
selected = selected_rows[0]
|
||||
project = Path(selected[2]).resolve()
|
||||
self._save(active_project=str(project), active_task=selected[0], active_job=selected[1])
|
||||
directory = root / 'batch-attempts' / (self.session_id + f'-g{self.generation}-' + attempt_id)
|
||||
expected = []
|
||||
for row in rows:
|
||||
if Path(row[2]).resolve() == project:
|
||||
path = directory / (row[1] + '.json')
|
||||
record = prepare_attempt(path, session_id=self.session_id, generation=self.generation,
|
||||
task_id=row[0], job_id=row[1], project_path=str(project), attempt_id=attempt_id)
|
||||
expected.append((path, record))
|
||||
self._project_attempts.setdefault(str(project).casefold(), []).extend(expected)
|
||||
return project, directory, expected
|
||||
|
||||
def _can_retry(self, project, expected):
|
||||
from .batch_attempt import can_retry_attempt
|
||||
key = str(project).casefold()
|
||||
return (not self._retry_counts.get(key) and expected
|
||||
and all(can_retry_attempt(path, record) for path, record in self._project_attempts.get(key, ())))
|
||||
|
||||
def _checkpoint(self, root, project, attempt_id):
|
||||
self.cad.checkpoint(root / ('cad-closed-' + attempt_id + '.json'))
|
||||
key = str(root).casefold()
|
||||
self.record['checkpoints'][key] = attempt_id
|
||||
self._save(state='ready')
|
||||
write_record(root / 'project-checkpoint.json', {'version': 1, 'resource_record': str(self.path),
|
||||
'session_id': self.session_id, 'generation': self.generation, 'attempt_id': attempt_id,
|
||||
'evidence_root': str(root), 'project_path': str(project), 'project_closed': True,
|
||||
'worker_closed': True, 'cad_pid': self.record['cad_pid'], 'cad_guid': self.record['cad_guid']})
|
||||
|
||||
def run_worker(self, command, timeout, *, abort_event=None, cooperative_abort=False):
|
||||
from .discovery_cache import project_fingerprint
|
||||
from .isolated_cad import PrivateCadStartupError
|
||||
from .worker_monitor import run_monitored_process
|
||||
root = Path(command[3]).resolve()
|
||||
self._ensure_desktop(abort_event)
|
||||
while True:
|
||||
check_cancelled(abort_event or self.stop_event)
|
||||
attempt_id = uuid4().hex
|
||||
# No previous empty-project certificate may authorize continuation
|
||||
# once another operation could attach or open a project.
|
||||
self.record['checkpoints'].pop(str(root).casefold(), None)
|
||||
self._save(active_attempt=attempt_id)
|
||||
project, directory, expected = self._prepare_attempts(root, command, attempt_id)
|
||||
before = project_fingerprint(project, self.root)
|
||||
status = root / 'status.tsv'
|
||||
status_before = status.read_bytes() if status.exists() else None
|
||||
started = time.monotonic()
|
||||
failure = None
|
||||
try:
|
||||
cad = self._ensure_cad(abort_event)
|
||||
cad.checkpoint(root / ('cad-ready-' + attempt_id + '.json'))
|
||||
environment = {'DXPART_ATTEMPT_FILE': str(directory), 'DXPART_ATTEMPT_ID': attempt_id,
|
||||
'DXPART_BATCH_SESSION_ID': self.session_id, 'DXPART_BATCH_GENERATION': str(self.generation)}
|
||||
def start_worker(launch, **options):
|
||||
return cad.start_worker(launch, extra_environment=environment, **options)
|
||||
result = run_monitored_process(command, timeout, abort_event=abort_event,
|
||||
cooperative_abort=cooperative_abort, process_factory=start_worker)
|
||||
self._checkpoint(root, project, attempt_id)
|
||||
if result.returncode not in (2, 3):
|
||||
return result
|
||||
failure = WorkerMonitorError(result.stderr or 'pre-mutation worker startup failed')
|
||||
except (PrivateCadStartupError, WorkerMonitorError, subprocess.TimeoutExpired, OSError) as error:
|
||||
failure = error
|
||||
finally:
|
||||
self._timing('project_worker', started, project=str(project), attempt_id=attempt_id,
|
||||
task_id=self.record['active_task'], job_id=self.record['active_job'])
|
||||
# Missing proof, changed inputs/status, cancellation or a previous
|
||||
# write anywhere in this project can never authorize replay.
|
||||
after = project_fingerprint(project, self.root)
|
||||
status_after = status.read_bytes() if status.exists() else None
|
||||
# Runtime database changes require separate identity evidence. Until
|
||||
# available for a failed read, use the stricter full fingerprint.
|
||||
safe = (before is not None and before == after
|
||||
and status_before == status_after and self._can_retry(project, expected)
|
||||
and not (root / 'recovery.request').exists()
|
||||
and not (abort_event is not None and abort_event.is_set())
|
||||
and not (self.stop_event is not None and self.stop_event.is_set()))
|
||||
if not safe:
|
||||
raise failure
|
||||
key = str(project).casefold()
|
||||
self._retry_counts[key] = 1
|
||||
self.record['retries'][key] = {'count': 1, 'reason': str(failure), 'attempt_id': attempt_id}
|
||||
self._save(state='recycling')
|
||||
self.close(rebuilding=True) # Prove disappearance before creating another.
|
||||
self.desktop = self.cad = None
|
||||
self._ensure_desktop(abort_event)
|
||||
|
||||
def recover_native_failure(self, project, evidence, reason):
|
||||
"""One read-only recovery, based on durable source and worker proof."""
|
||||
from .batch_attempt import can_retry_attempt
|
||||
project, evidence = Path(project).resolve(), Path(evidence).resolve()
|
||||
key = str(project).casefold()
|
||||
if (self._retry_counts.get(key) or self.record['state'] == 'blocked'
|
||||
or self.stop_event is not None and self.stop_event.is_set()):
|
||||
return False
|
||||
try:
|
||||
proof = json.loads((evidence / 'native-retry-proof.json').read_text(encoding='utf-8'))
|
||||
sources = proof['source_hashes']
|
||||
operation = self.record['native_operations'][str(evidence)]
|
||||
if (proof['version'] != 1 or proof['native_read_only'] is not True
|
||||
or operation['workers_closed'] is not True or proof['binding'] != operation['binding']
|
||||
or proof['binding']['session_id'] != self.session_id
|
||||
or proof['binding']['generation'] != self.generation
|
||||
or proof['workers_closed'] is not True or proof['project_path'] != str(project)
|
||||
or proof['evidence_root'] != str(evidence) or str(project) not in sources
|
||||
or len(sources) != 2
|
||||
or not all(Path(name).resolve().is_relative_to(project.parent) for name in sources)
|
||||
or not all(hashlib.sha256(Path(name).read_bytes()).hexdigest() == digest
|
||||
for name, digest in sources.items())
|
||||
or not all(can_retry_attempt(path, record)
|
||||
for path, record in self._project_attempts.get(key, ()))):
|
||||
return False
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
return False
|
||||
self._retry_counts[key] = 1
|
||||
self.record['retries'][key] = {'count': 1, 'reason': str(reason), 'backend': 'native_cli'}
|
||||
self._save(state='recycling')
|
||||
self.close(rebuilding=True)
|
||||
self.desktop = self.cad = None
|
||||
self._ensure_desktop()
|
||||
return True
|
||||
|
||||
def close(self, *, rebuilding=False):
|
||||
if self.desktop is None and not self.generation:
|
||||
return # A cache-only batch never acquired resources.
|
||||
if self.record['state'] == 'released':
|
||||
return
|
||||
started = time.monotonic()
|
||||
errors = []
|
||||
try:
|
||||
self._save(state='closing')
|
||||
except Exception as error:
|
||||
errors.append('resource journal: ' + str(error))
|
||||
if self.cad is not None:
|
||||
cad, self.cad = self.cad, None
|
||||
try:
|
||||
cad.__exit__(None, None, None)
|
||||
except Exception as error:
|
||||
if rebuilding:
|
||||
self.record['shutdown_warning'] = str(error)
|
||||
else:
|
||||
errors.append(str(error))
|
||||
if self.desktop is not None:
|
||||
try:
|
||||
self.desktop.close()
|
||||
except Exception as error:
|
||||
errors.append(str(error))
|
||||
self._timing('batch_release', started)
|
||||
self.record['cleanup'] = getattr(self.desktop, 'cleanup_record', None)
|
||||
if errors:
|
||||
self._save(state='blocked', error='; '.join(errors))
|
||||
raise BatchResourceCleanupError('; '.join(errors), completed_result=self.result)
|
||||
self._save(state='released', released_at=time.time(), processes=[])
|
||||
LIVE_RESOURCES.pop(str(self.path), None)
|
||||
@@ -65,10 +65,11 @@ def _designer_executable():
|
||||
|
||||
class IsolatedCadSession:
|
||||
"""The supervisor never attaches to COM on the user's input desktop."""
|
||||
def __init__(self, root, *, abort_event=None):
|
||||
def __init__(self, root, *, abort_event=None, desktop=None):
|
||||
self.root = Path(root).resolve()
|
||||
self.token = uuid4().hex
|
||||
self.desktop = PrivateDesktop('dx-' + self.token, enabled=True)
|
||||
self.borrowed_desktop = desktop is not None
|
||||
self.desktop = desktop if desktop is not None else PrivateDesktop('dx-' + self.token, enabled=True)
|
||||
self.cad = None
|
||||
self.binding = None
|
||||
self.started = time.monotonic()
|
||||
@@ -132,10 +133,36 @@ class IsolatedCadSession:
|
||||
if self.abort_event is not None and self.abort_event.is_set():
|
||||
raise WorkerMonitorError('private CAD startup was cancelled before execution')
|
||||
|
||||
def start_worker(self, command, **options):
|
||||
def start_worker(self, command, *, extra_environment=None, **options):
|
||||
if self.binding is None or self.cad.poll() is not None:
|
||||
raise WorkerMonitorError('owned private CAD is unavailable')
|
||||
return self.desktop.start_process(command, env=_child_environment(self.binding.environment()), **options)
|
||||
environment = self.binding.environment()
|
||||
environment.update(extra_environment or {})
|
||||
# The verified CAD binding cannot be overridden by attempt metadata.
|
||||
binding = self.binding.environment()
|
||||
for key in ('DXPART_DESKTOP', 'DXPART_CAD_PID', 'DXPART_CAD_GUID', 'DXPART_SESSION_TOKEN'):
|
||||
environment[key] = binding[key]
|
||||
return self.desktop.start_process(command, env=_child_environment(environment), **options)
|
||||
|
||||
def checkpoint(self, evidence):
|
||||
"""Certify an idle, identity-verified CAD before crossing project boundaries."""
|
||||
if self.binding is None or self.cad.poll() is not None:
|
||||
raise WorkerMonitorError('owned CAD unavailable at project checkpoint')
|
||||
evidence = Path(evidence)
|
||||
evidence.parent.mkdir(parents=True, exist_ok=True)
|
||||
process = self.start_worker(_helper_command('checkpoint', evidence, self.cad.pid),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
try:
|
||||
if process.wait(timeout=20):
|
||||
raise WorkerMonitorError('project close checkpoint failed: ' +
|
||||
process.stderr.read().decode('utf-8', errors='replace'))
|
||||
result = json.loads(evidence.read_text(encoding='utf-8'))
|
||||
if (result.get('binding') != asdict(self.binding) or result.get('project_path') != ''
|
||||
or result.get('project_closed') is not True or self.cad.poll() is not None):
|
||||
raise WorkerMonitorError('project close checkpoint identity mismatch')
|
||||
finally:
|
||||
process.close()
|
||||
return result
|
||||
|
||||
def __exit__(self, error_type, error, traceback):
|
||||
closing_started = time.monotonic()
|
||||
@@ -174,7 +201,9 @@ class IsolatedCadSession:
|
||||
finally:
|
||||
try:
|
||||
self.desktop.close()
|
||||
self._record(state='released', clean=error is None and cleanup_error is None,
|
||||
self._record(state='session_closed' if self.borrowed_desktop else 'released',
|
||||
desktop_retained_by_batch=self.borrowed_desktop,
|
||||
clean=error is None and cleanup_error is None,
|
||||
shutdown_seconds=time.monotonic() - closing_started,
|
||||
desktop_cleanup=getattr(self.desktop, 'cleanup_record', None), **exit_codes)
|
||||
except Exception as failure:
|
||||
@@ -190,8 +219,11 @@ class IsolatedCadSession:
|
||||
self._record(state='uncertain', error=message, **details)
|
||||
|
||||
|
||||
def run_isolated_process(command, timeout, *, abort_event=None, cooperative_abort=False):
|
||||
def run_isolated_process(command, timeout, *, abort_event=None, cooperative_abort=False, runtime=None):
|
||||
from .worker_monitor import run_monitored_process
|
||||
if runtime is not None:
|
||||
return runtime.run_worker(command, timeout, abort_event=abort_event,
|
||||
cooperative_abort=cooperative_abort)
|
||||
try:
|
||||
with IsolatedCadSession(Path(command[3]), abort_event=abort_event) as session:
|
||||
return run_monitored_process(command, timeout, abort_event=abort_event, process_factory=session.start_worker,
|
||||
@@ -252,14 +284,18 @@ def _native_action(action, evidence, expected_pid):
|
||||
binding = CadBinding(desktop, expected_pid, str(app.Guid()), token)
|
||||
verify_application(app, binding, require_view=False)
|
||||
Path(evidence).write_text(json.dumps(asdict(binding)), encoding='utf-8')
|
||||
elif action == 'quit':
|
||||
elif action in {'quit', 'checkpoint'}:
|
||||
binding = CadBinding.from_environment()
|
||||
# ActiveViewHandle can retain a destroyed/reused HWND after closing
|
||||
# the last project. The private ROT/GUID binds this empty instance.
|
||||
verify_application(app, binding, require_view=False)
|
||||
if project_file_path(app):
|
||||
raise ValueError('owned CAD still has a project open; no implicit save or discard')
|
||||
app.Quit()
|
||||
if action == 'quit':
|
||||
app.Quit()
|
||||
else:
|
||||
from .batch_resources import write_record
|
||||
write_record(evidence, {'binding': asdict(binding), 'project_path': '', 'project_closed': True})
|
||||
else:
|
||||
raise ValueError('unknown private CAD action')
|
||||
except BaseException as failure:
|
||||
|
||||
@@ -473,6 +473,16 @@ class PrivateDesktop:
|
||||
self._processes.append(process)
|
||||
return process
|
||||
|
||||
def forget_closed_process(self, process):
|
||||
"""Drop a worker only after its owning wrapper confirmed all cleanup."""
|
||||
record = getattr(process, 'cleanup_record', {})
|
||||
if (record.get('process_handle_closed') is not True
|
||||
or record.get('job_handle_closed') is not True
|
||||
or record.get('unconfirmed_descendant_handles', 0)
|
||||
or record.get('errors') or process.poll() is None):
|
||||
raise OSError('cannot forget a worker with unconfirmed owned resources')
|
||||
self._processes = [item for item in self._processes if item is not process]
|
||||
|
||||
def close(self) -> None:
|
||||
errors, remaining = [], []
|
||||
self.cleanup_record = {'desktop': self.name, 'processes': []}
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
"""Batch resources are lazy, shared, and independently blocked after failed release."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from dx_productivity.trial_task_center.errors import TaskCenterError
|
||||
|
||||
|
||||
class Desktop:
|
||||
created = []
|
||||
fail_release = False
|
||||
|
||||
def __init__(self, name, **_options):
|
||||
self.name = name
|
||||
self.processes = []
|
||||
self.released = False
|
||||
self.cleanup_record = None
|
||||
|
||||
def __enter__(self):
|
||||
self.created.append(self)
|
||||
return self
|
||||
|
||||
def start_process(self, command, **_options):
|
||||
process = SimpleNamespace(pid=100 + len(self.processes), closed=False,
|
||||
cleanup_record={}, returncode=0)
|
||||
def close():
|
||||
process.closed = True
|
||||
process.cleanup_record = {'pid': process.pid, 'process_handle_closed': True,
|
||||
'job_handle_closed': True, 'errors': []}
|
||||
process.close = close
|
||||
process.poll = lambda: process.returncode
|
||||
self.processes.append(process)
|
||||
return process
|
||||
|
||||
def close(self):
|
||||
for process in self.processes:
|
||||
process.close()
|
||||
self.cleanup_record = {'desktop_handle_closed': True,
|
||||
'desktop_confirmation': {'released': not self.fail_release}, 'errors': []}
|
||||
if self.fail_release:
|
||||
raise OSError('outside owner retains desktop')
|
||||
self.released = True
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_desktop():
|
||||
Desktop.created = []
|
||||
Desktop.fail_release = False
|
||||
|
||||
|
||||
def test_cache_only_batch_does_not_create_desktop_or_cad(tmp_path):
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
||||
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop):
|
||||
pass
|
||||
assert not Desktop.created
|
||||
|
||||
|
||||
def test_native_operations_share_one_desktop_but_release_each_worker(tmp_path):
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
||||
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
|
||||
names = []
|
||||
for name in ('first', 'second'):
|
||||
with runtime.native_scope(tmp_path / name) as scope:
|
||||
names.append(scope.name)
|
||||
process = scope.start_process(('inventory.exe',))
|
||||
assert not process.closed
|
||||
assert process.closed and not Desktop.created[0].released
|
||||
assert names[0] == names[1]
|
||||
assert len(Desktop.created) == 1 and Desktop.created[0].released
|
||||
|
||||
|
||||
def test_failed_final_release_blocks_next_batch_and_resource_cleanup_never_replays(tmp_path):
|
||||
from dx_productivity.trial_task_center.batch_runtime import (
|
||||
BatchCadRuntime, cleanup_pending_resources, guard_pending_resources, pending_resources)
|
||||
Desktop.fail_release = True
|
||||
with pytest.raises(TaskCenterError) as caught:
|
||||
with BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop) as runtime:
|
||||
with runtime.native_scope(tmp_path / 'operation'):
|
||||
pass
|
||||
runtime.result = 'verified completed task'
|
||||
assert caught.value.code == 'batch_resources_pending'
|
||||
assert caught.value.completed_result == 'verified completed task'
|
||||
assert pending_resources(tmp_path)
|
||||
with pytest.raises(TaskCenterError):
|
||||
guard_pending_resources(tmp_path)
|
||||
assert not list(tmp_path.rglob('uncertain.flag'))
|
||||
Desktop.fail_release = False
|
||||
assert cleanup_pending_resources(tmp_path) == ()
|
||||
guard_pending_resources(tmp_path)
|
||||
assert len(Desktop.created) == 1
|
||||
|
||||
|
||||
def test_cleanup_failure_takes_priority_over_cancel(tmp_path):
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
||||
Desktop.fail_release = True
|
||||
with pytest.raises(TaskCenterError) as caught:
|
||||
with BatchCadRuntime(tmp_path, 'bom_import', desktop_factory=Desktop) as runtime:
|
||||
with runtime.native_scope(tmp_path / 'operation'):
|
||||
pass
|
||||
raise TaskCenterError('operation_cancelled', 'cancel requested')
|
||||
assert caught.value.code == 'batch_resources_pending'
|
||||
Desktop.fail_release = False
|
||||
from dx_productivity.trial_task_center.batch_runtime import cleanup_pending_resources
|
||||
cleanup_pending_resources(tmp_path)
|
||||
|
||||
|
||||
class Cad:
|
||||
created = []
|
||||
|
||||
def __init__(self, root, *, abort_event, desktop):
|
||||
self.root, self.desktop, self.abort_event = root, desktop, abort_event
|
||||
self.binding = SimpleNamespace(pid=500, guid='owned')
|
||||
self.closed = False
|
||||
|
||||
def __enter__(self):
|
||||
self.desktop.__enter__()
|
||||
self.created.append(self)
|
||||
return self
|
||||
|
||||
def checkpoint(self, evidence):
|
||||
assert not self.closed
|
||||
|
||||
def start_worker(self, *args, **kwargs):
|
||||
raise AssertionError('monitor is injected')
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
self.closed = True
|
||||
self.desktop.close()
|
||||
|
||||
|
||||
def command_for(tmp_path, name='one'):
|
||||
root = tmp_path / 'data' / name
|
||||
root.mkdir(parents=True)
|
||||
project = tmp_path / ('project-' + name) / 'test.prj'
|
||||
project.parent.mkdir()
|
||||
project.write_text('project identity')
|
||||
(root / 'command.tsv').write_text('task_id\tjob_id\tproject_path\told_part\tnew_part\n'
|
||||
f'task\tjob\t{project}\told\tnew\n')
|
||||
return ('cscript', '//nologo', 'worker.vbs', str(root), 'python', 'job'), project
|
||||
|
||||
|
||||
def worker_runtime(tmp_path, monkeypatch, monitor):
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
||||
from dx_productivity.trial_task_center import worker_monitor
|
||||
monkeypatch.setattr(worker_monitor, 'run_monitored_process', monitor)
|
||||
Cad.created = []
|
||||
return BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=Desktop, cad_factory=Cad)
|
||||
|
||||
|
||||
def test_two_projects_reuse_cad_and_produce_distinct_close_checkpoints(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
|
||||
one, _ = command_for(tmp_path)
|
||||
two, _ = command_for(tmp_path, 'two')
|
||||
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
|
||||
subprocess.CompletedProcess(command, 0, '', ''))
|
||||
with runtime:
|
||||
runtime.run_worker(one, 5)
|
||||
assert project_checkpoint_released(Path(one[3]))
|
||||
runtime.run_worker(two, 5)
|
||||
assert project_checkpoint_released(Path(two[3]))
|
||||
assert len(Cad.created) == len(Desktop.created) == 1
|
||||
assert Cad.created[0].closed and Desktop.created[0].released
|
||||
|
||||
|
||||
@pytest.mark.parametrize('second_fails', [False, True])
|
||||
def test_prewrite_failure_releases_old_desktop_and_retries_only_once(tmp_path, monkeypatch, second_fails):
|
||||
import subprocess
|
||||
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
||||
command, _ = command_for(tmp_path)
|
||||
calls = []
|
||||
def monitor(command, *args, **kwargs):
|
||||
calls.append(len(calls))
|
||||
if len(calls) == 2:
|
||||
assert Desktop.created[0].released
|
||||
return subprocess.CompletedProcess(command, 2 if len(calls) == 1 or second_fails else 0, '', 'startup')
|
||||
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
||||
with runtime:
|
||||
if second_fails:
|
||||
with pytest.raises(WorkerMonitorError):
|
||||
runtime.run_worker(command, 5)
|
||||
else:
|
||||
runtime.run_worker(command, 5)
|
||||
assert len(calls) == len(Desktop.created) == len(Cad.created) == 2
|
||||
assert runtime.generation == 2
|
||||
|
||||
|
||||
def test_possible_mutation_prevents_automatic_replay(tmp_path, monkeypatch):
|
||||
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
||||
command, _ = command_for(tmp_path)
|
||||
calls = []
|
||||
def monitor(*args, **kwargs):
|
||||
calls.append(1)
|
||||
path = next((Path(command[3]) / 'batch-attempts').rglob('job.json'))
|
||||
record = json.loads(path.read_text())
|
||||
record['state'] = 'mutation_possible'
|
||||
path.write_text(json.dumps(record))
|
||||
raise WorkerMonitorError('lost connection after possible write')
|
||||
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
||||
with runtime, pytest.raises(WorkerMonitorError):
|
||||
runtime.run_worker(command, 5)
|
||||
assert len(calls) == len(Desktop.created) == 1
|
||||
|
||||
|
||||
def test_previous_checkpoint_invalidated_before_new_worker(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
|
||||
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
||||
command, _ = command_for(tmp_path)
|
||||
calls = []
|
||||
def monitor(*args, **kwargs):
|
||||
assert not project_checkpoint_released(Path(command[3]))
|
||||
calls.append(1)
|
||||
return subprocess.CompletedProcess(command, 0, '', '')
|
||||
runtime = worker_runtime(tmp_path, monkeypatch, monitor)
|
||||
with runtime:
|
||||
runtime.run_worker(command, 5)
|
||||
assert project_checkpoint_released(Path(command[3]))
|
||||
def fail_checkpoint(*args):
|
||||
raise WorkerMonitorError('cannot certify empty project')
|
||||
runtime.cad.checkpoint = fail_checkpoint
|
||||
# No retry after evidence has been marked as possibly modified.
|
||||
path = next((Path(command[3]) / 'batch-attempts').rglob('job.json'))
|
||||
record = json.loads(path.read_text())
|
||||
record['state'] = 'mutation_possible'
|
||||
path.write_text(json.dumps(record))
|
||||
with pytest.raises(WorkerMonitorError):
|
||||
runtime.run_worker(command, 5)
|
||||
assert not project_checkpoint_released(Path(command[3]))
|
||||
|
||||
|
||||
def test_attempt_publication_failure_never_launches_worker(tmp_path, monkeypatch):
|
||||
from dx_productivity.trial_task_center import batch_attempt
|
||||
command, _ = command_for(tmp_path)
|
||||
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k: pytest.fail('must not launch'))
|
||||
def fail(*args, **kwargs):
|
||||
raise OSError('disk unavailable')
|
||||
monkeypatch.setattr(batch_attempt, 'prepare_attempt', fail)
|
||||
with runtime, pytest.raises(OSError):
|
||||
runtime.run_worker(command, 5)
|
||||
assert not Cad.created
|
||||
|
||||
|
||||
def test_boundary_pause_does_not_abort_current_cad_startup(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
from threading import Event
|
||||
command, _ = command_for(tmp_path)
|
||||
runtime = worker_runtime(tmp_path, monkeypatch, lambda command, *a, **k:
|
||||
subprocess.CompletedProcess(command, 0, '', ''))
|
||||
pause, worker_abort = Event(), Event()
|
||||
runtime.stop_event = pause
|
||||
original_enter = Cad.__enter__
|
||||
def enter(cad):
|
||||
pause.set() # User requests pause after the current task has begun.
|
||||
assert cad.abort_event is worker_abort
|
||||
assert not cad.abort_event.is_set()
|
||||
return original_enter(cad)
|
||||
monkeypatch.setattr(Cad, '__enter__', enter)
|
||||
with runtime:
|
||||
runtime.run_worker(command, 5, abort_event=worker_abort)
|
||||
assert not list(tmp_path.rglob('uncertain.flag'))
|
||||
|
||||
|
||||
def test_cleanup_still_runs_when_resource_journal_write_fails(tmp_path, monkeypatch):
|
||||
from dx_productivity.trial_task_center import batch_runtime
|
||||
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
||||
with pytest.raises(TaskCenterError) as caught:
|
||||
with runtime:
|
||||
with runtime.native_scope(tmp_path / 'read'):
|
||||
pass
|
||||
real_write = batch_runtime.write_record
|
||||
def failing_write(path, record):
|
||||
if record['state'] == 'closing':
|
||||
raise OSError('journal unavailable')
|
||||
return real_write(path, record)
|
||||
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
|
||||
assert Desktop.created[0].released
|
||||
assert caught.value.code == 'batch_resources_pending'
|
||||
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_real_startup_failure_with_released_lease_can_retry(tmp_path, monkeypatch):
|
||||
import io
|
||||
import subprocess
|
||||
from dx_productivity.trial_task_center import isolated_cad, worker_monitor
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
||||
command, _ = command_for(tmp_path)
|
||||
starts = []
|
||||
def executable():
|
||||
starts.append(1)
|
||||
if len(starts) == 1:
|
||||
raise worker_monitor.WorkerMonitorError('registry temporarily unavailable')
|
||||
return 'cad.exe'
|
||||
class BootstrapDesktop(Desktop):
|
||||
def start_process(self, command, **options):
|
||||
process = super().start_process(command, **options)
|
||||
process.stderr = io.BytesIO()
|
||||
process.wait = lambda timeout: process.returncode
|
||||
if command[0] == 'cad.exe':
|
||||
self.cad = process
|
||||
process.returncode = None
|
||||
else:
|
||||
action, evidence, pid = command
|
||||
environment = options['env']
|
||||
binding = {'desktop': self.name, 'pid': pid, 'guid': 'owned',
|
||||
'token': environment['DXPART_SESSION_TOKEN']}
|
||||
if action == 'probe':
|
||||
Path(evidence).write_text(json.dumps(binding))
|
||||
elif action == 'checkpoint':
|
||||
Path(evidence).write_text(json.dumps({'binding': binding, 'project_path': '', 'project_closed': True}))
|
||||
elif action == 'quit':
|
||||
self.cad.returncode = 0
|
||||
return process
|
||||
monkeypatch.setattr(isolated_cad, '_designer_executable', executable)
|
||||
monkeypatch.setattr(isolated_cad, '_helper_command', lambda *args: args)
|
||||
monkeypatch.setattr(worker_monitor, 'run_monitored_process', lambda *a, **k:
|
||||
subprocess.CompletedProcess(command, 0, '', ''))
|
||||
with BatchCadRuntime(tmp_path / 'data', 'execution', desktop_factory=BootstrapDesktop) as runtime:
|
||||
runtime.run_worker(command, 5)
|
||||
assert len(starts) == len(Desktop.created) == 2
|
||||
assert all(desktop.released for desktop in Desktop.created)
|
||||
|
||||
|
||||
def test_native_rebuild_requires_strict_source_and_worker_proof(tmp_path):
|
||||
import hashlib
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
||||
command, project = command_for(tmp_path)
|
||||
database = project.parent / 'database.dat'
|
||||
database.write_bytes(b'database')
|
||||
evidence = Path(command[3]) / 'native'
|
||||
evidence.mkdir()
|
||||
proof = {'version': 1, 'native_read_only': True, 'workers_closed': True,
|
||||
'project_path': str(project.resolve()), 'evidence_root': str(evidence.resolve()),
|
||||
'source_hashes': {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in (project, database)}}
|
||||
path = evidence / 'native-retry-proof.json'
|
||||
path.write_text(json.dumps(proof))
|
||||
with BatchCadRuntime(tmp_path / 'data', 'bom_import', desktop_factory=Desktop) as runtime:
|
||||
with runtime.native_scope(evidence) as scope:
|
||||
pass
|
||||
proof['binding'] = scope.identity
|
||||
path.write_text(json.dumps(proof))
|
||||
database.write_bytes(b'changed externally')
|
||||
assert not runtime.recover_native_failure(project, evidence, 'timeout')
|
||||
database.write_bytes(b'database')
|
||||
assert runtime.recover_native_failure(project, evidence, 'timeout')
|
||||
assert Desktop.created[0].released
|
||||
from dx_productivity.trial_task_center.worker_monitor import WorkerMonitorError
|
||||
with pytest.raises(WorkerMonitorError, match='stale'):
|
||||
scope.start_process(('late-old-worker.exe',))
|
||||
assert scope.identity['generation'] == 1 and runtime.generation == 2
|
||||
scope.close() # A late close cannot grant the old operation a new identity.
|
||||
assert str(evidence.resolve()) not in runtime.record['native_operations']
|
||||
assert not runtime.recover_native_failure(project, evidence, 'another timeout')
|
||||
assert len(Desktop.created) == 2
|
||||
|
||||
|
||||
def test_release_publication_failure_can_be_reconciled_without_replaying(tmp_path, monkeypatch):
|
||||
from dx_productivity.trial_task_center import batch_runtime
|
||||
runtime = batch_runtime.BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
||||
real_write = batch_runtime.write_record
|
||||
def failing_write(path, record):
|
||||
if record['state'] == 'released':
|
||||
raise OSError('cannot publish release')
|
||||
return real_write(path, record)
|
||||
with pytest.raises(TaskCenterError):
|
||||
with runtime:
|
||||
with runtime.native_scope(tmp_path / 'read'):
|
||||
pass
|
||||
monkeypatch.setattr(batch_runtime, 'write_record', failing_write)
|
||||
assert Desktop.created[0].released and batch_runtime.pending_resources(tmp_path)
|
||||
monkeypatch.setattr(batch_runtime, 'write_record', real_write)
|
||||
assert batch_runtime.cleanup_pending_resources(tmp_path) == ()
|
||||
assert len(Desktop.created) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize('record', [
|
||||
{'version': 1, 'session_id': 'old', 'state': 'blocked'},
|
||||
{'version': 1, 'session_id': 'old', 'state': 'released'},
|
||||
[],
|
||||
])
|
||||
def test_unknown_resource_evidence_never_authorizes_cleanup(tmp_path, record):
|
||||
from dx_productivity.trial_task_center.batch_resources import cleanup_pending_resources, pending_resources
|
||||
path = tmp_path / 'batch-resources' / 'old' / 'session.json'
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(json.dumps(record))
|
||||
assert pending_resources(tmp_path)
|
||||
assert cleanup_pending_resources(tmp_path)
|
||||
assert json.loads(path.read_text()) == record
|
||||
|
||||
|
||||
def test_stale_generation_and_unknown_checkpoint_schema_are_rejected(tmp_path, monkeypatch):
|
||||
import subprocess
|
||||
from dx_productivity.trial_task_center.batch_resources import project_checkpoint_released
|
||||
command, _ = command_for(tmp_path)
|
||||
runtime = worker_runtime(tmp_path, monkeypatch, lambda *a, **k:
|
||||
subprocess.CompletedProcess(command, 0, '', ''))
|
||||
with runtime:
|
||||
runtime.run_worker(command, 5)
|
||||
path = Path(command[3]) / 'project-checkpoint.json'
|
||||
checkpoint = json.loads(path.read_text())
|
||||
assert project_checkpoint_released(Path(command[3]))
|
||||
for changes in ({'version': 999}, {'generation': 999}, {'attempt_id': 'old'}, {'cad_guid': 'foreign'}):
|
||||
path.write_text(json.dumps(dict(checkpoint, **changes)))
|
||||
assert not project_checkpoint_released(Path(command[3]))
|
||||
|
||||
|
||||
def test_restart_reconciles_only_confirmed_disappearance_without_replacement(tmp_path, monkeypatch):
|
||||
from dx_productivity.trial_task_center import batch_resources, isolated_desktop
|
||||
from dx_productivity.trial_task_center.batch_runtime import BatchCadRuntime
|
||||
Desktop.fail_release = True
|
||||
runtime = BatchCadRuntime(tmp_path, 'execution', desktop_factory=Desktop)
|
||||
with pytest.raises(TaskCenterError):
|
||||
with runtime:
|
||||
with runtime.native_scope(tmp_path / 'read'):
|
||||
pass
|
||||
# Simulate restarting the UI: there are no retained owned handles to act on.
|
||||
batch_resources.LIVE_RESOURCES.pop(str(runtime.path))
|
||||
present = [True]
|
||||
class Api:
|
||||
def confirm_desktop_released(self, name):
|
||||
assert name == runtime.record['desktop']
|
||||
if present[0]:
|
||||
raise OSError('desktop still held by external thread')
|
||||
monkeypatch.setattr(isolated_desktop, '_Win32Api', Api)
|
||||
assert batch_resources.cleanup_pending_resources(tmp_path)
|
||||
present[0] = False
|
||||
assert batch_resources.cleanup_pending_resources(tmp_path) == ()
|
||||
assert len(Desktop.created) == 1
|
||||
Reference in new issue
Block a user