fix: bind manual recovery entries and current native diagnostics
This commit is contained in:
1 parent
8374cabe5c
commit
601dd7b6b2
14 files changed
+694
-69
No files matched your search
@@ -97,9 +97,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
from .owner_probe import owner_probe_main
|
||||
return owner_probe_main(args.owner_probe)
|
||||
if args.native_backup_worker:
|
||||
from .native_backup import native_backup_worker_main
|
||||
from .desktop_worker import run_native_worker
|
||||
return run_native_worker(native_backup_worker_main, args.native_backup_worker)
|
||||
from .native_backup import run_native_backup_worker
|
||||
return run_native_backup_worker(args.native_backup_worker)
|
||||
if args.management_worker:
|
||||
from .native_management import management_worker_main
|
||||
from .desktop_worker import run_native_worker
|
||||
|
||||
@@ -8,7 +8,7 @@ import re
|
||||
ERROR_CODE = 'ICDB_REPAIR_REQUIRED'
|
||||
LOG_LIMIT = 128 * 1024
|
||||
RECOVERY_ACTION = (
|
||||
'停止批次并确认原服务器及其他客户端已正常关闭;保留完整一致副本后,'
|
||||
'停止受影响工程的写入并确认原服务器及其他客户端已正常关闭;保留完整一致副本后,'
|
||||
"使用同版本 iCDB Project Backup 的 Repair project 检查并修复正确目标。"
|
||||
'共享符号库问题需核对中心库,不能只修复单个 CBB 的 Work 工程。'
|
||||
'不要删除 sAddress.adr/锁文件;修复后重新打开、校验并确认权威 READY。'
|
||||
@@ -16,6 +16,9 @@ RECOVERY_ACTION = (
|
||||
_SESSION_LOSS = re.compile(
|
||||
r'\bthe\s+last\s+server\s+session\s+on\s+different\s+server\s+machine\s+'
|
||||
r'lost\s+the\s+connection\s+to\s+the\s+database\b', re.IGNORECASE)
|
||||
_INCONSISTENT = re.compile(
|
||||
r"Project database \[([^\]\r\n]+)\] is inconsistent\. It has been manually copied while the iCDB Server was running\.\s*"
|
||||
r"Please use iCDB Project Backup 'Repair project' functionality to fix this issue\.", re.I)
|
||||
|
||||
_ICDB_MESSAGE = re.compile(r'(?<![A-Za-z0-9_])icdb(?![A-Za-z0-9_.\\/])', re.I)
|
||||
_FAILURE_MESSAGE = re.compile(
|
||||
@@ -78,11 +81,25 @@ def parse_session_loss(text):
|
||||
return record
|
||||
|
||||
|
||||
def parse_recovery_request(text):
|
||||
record = parse_session_loss(text)
|
||||
if record is not None:
|
||||
return record
|
||||
match = _INCONSISTENT.search(str(text or ''))
|
||||
if match is not None:
|
||||
shared = 'symbollibs' in ntpath.normpath(match[1]).casefold().split('\\')
|
||||
return {'error_code': ERROR_CODE, 'kind': 'database_inconsistent',
|
||||
'native_message': match[0], 'database_path': match[1], 'scope': 'central_library' if shared else 'project',
|
||||
'machine': '', 'session': ''}
|
||||
return None
|
||||
|
||||
|
||||
def recovery_detail(record):
|
||||
target = ('中心库共享符号数据库' if record['scope'] == 'central_library' else
|
||||
'工程数据库' if record['scope'] == 'project' else '报错中指定的数据库')
|
||||
reason = '数据库副本不一致' if record.get('kind') == 'database_inconsistent' else '上次服务器会话失联'
|
||||
return (
|
||||
f"{ERROR_CODE}: {target}的上次服务器会话失联。\n"
|
||||
f"{ERROR_CODE}: {target}{reason};发现问题,未修复。\n"
|
||||
f"{record['native_message']}\n"
|
||||
f'{RECOVERY_ACTION}'
|
||||
)
|
||||
@@ -100,7 +117,8 @@ def snapshot_log(path):
|
||||
return {'size': size, 'tail_sha256': hashlib.sha256(tail).hexdigest(),
|
||||
'truncated': size > LOG_LIMIT,
|
||||
'icdb_errors': [' '.join(message.split()) for message in _icdb_error_paragraphs(decode_log(tail))],
|
||||
'session_losses': losses}
|
||||
'session_losses': losses,
|
||||
'inconsistent_databases': [match[1] for match in _INCONSISTENT.finditer(decode_log(tail))]}
|
||||
except FileNotFoundError:
|
||||
return {'size': 0, 'tail_sha256': hashlib.sha256(b'').hexdigest(), 'icdb_errors': []}
|
||||
except OSError as error:
|
||||
@@ -161,6 +179,12 @@ def read_log_failure(path, baseline):
|
||||
historical = baseline.get('session_losses', []) if baseline else []
|
||||
recovery = next((record for record in _session_losses(text)
|
||||
if appended or _session_identity(record) not in historical), None)
|
||||
if recovery is None:
|
||||
old = baseline.get('inconsistent_databases') if baseline else None
|
||||
match = next((match for match in _INCONSISTENT.finditer(text)
|
||||
if appended or old is not None and match[1] not in old), None)
|
||||
if match is not None:
|
||||
recovery = parse_recovery_request(match[0])
|
||||
return text, recovery
|
||||
|
||||
|
||||
|
||||
@@ -19,22 +19,25 @@ def assert_native_idle(target=None):
|
||||
from .native_export_guard import assert_retained_exports_idle
|
||||
from .native_engineering_identity import _OWNER_NAMES
|
||||
from .library_lock import _key, _owner_matches_scope
|
||||
library = None
|
||||
library = Path(target.library) if target is not None and getattr(target, 'library', None) else None
|
||||
roots = ()
|
||||
if target is not None:
|
||||
from .backup_recovery import _project_text
|
||||
roots = (Path(target.project).parent, Path(target.database))
|
||||
try:
|
||||
lines = re.findall(r'^KEY CentralLibrary\b[^\r\n]*', _project_text(target.project), re.M)
|
||||
references = [re.fullmatch(r'KEY CentralLibrary "([^"\r\n]+)"\s*', line) for line in lines]
|
||||
if not references or any(match is None for match in references):
|
||||
raise ValueError('CentralLibrary references are missing or invalid.')
|
||||
libraries = {(Path(target.project).parent/match[1]).resolve(strict=True) for match in references}
|
||||
if len(libraries) == 1 and all(path.suffix.casefold() == '.lmc' and path.is_file() for path in libraries):
|
||||
library = libraries.pop()
|
||||
roots += (library.parent/'SymbolLibs',)
|
||||
except (OSError, ValueError):
|
||||
pass # Missing/ambiguous original references cannot narrow retained ownership.
|
||||
if library is not None:
|
||||
roots += (library.parent/'SymbolLibs',)
|
||||
else:
|
||||
try:
|
||||
lines = re.findall(r'^KEY CentralLibrary\b[^\r\n]*', _project_text(target.project), re.M)
|
||||
references = [re.fullmatch(r'KEY CentralLibrary "([^"\r\n]+)"\s*', line) for line in lines]
|
||||
if not references or any(match is None for match in references):
|
||||
raise ValueError('CentralLibrary references are missing or invalid.')
|
||||
libraries = {(Path(target.project).parent/match[1]).resolve(strict=True) for match in references}
|
||||
if len(libraries) == 1 and all(path.suffix.casefold() == '.lmc' and path.is_file() for path in libraries):
|
||||
library = libraries.pop()
|
||||
roots += (library.parent/'SymbolLibs',)
|
||||
except (OSError, ValueError):
|
||||
pass # Missing/ambiguous original references cannot narrow retained ownership.
|
||||
assert_retained_exports_idle(library)
|
||||
if any(worker.poll() is None and (library is None or not getattr(worker, 'cbb_library', None)
|
||||
or _key(worker.cbb_library) == _key(library)) for worker in _retained_workers):
|
||||
|
||||
@@ -68,6 +68,10 @@ def pcb_startup_timeout_seconds(timeout_seconds):
|
||||
return max(60.0, float(timeout_seconds))
|
||||
|
||||
|
||||
def lm_dispatch_timeout_seconds(timeout_seconds):
|
||||
return max(.1, min(45.0, float(timeout_seconds)))
|
||||
|
||||
|
||||
def same_path(first, second):
|
||||
return os.path.normcase(os.path.abspath(str(first))) == os.path.normcase(os.path.abspath(str(second)))
|
||||
|
||||
@@ -698,12 +702,10 @@ def execute_job(job, emit):
|
||||
from .diagnostic_open import snapshot_diagnostic_logs
|
||||
job['diagnostic_log_baseline'] = snapshot_diagnostic_logs(job)
|
||||
else:
|
||||
ensure_pcb_central_library(
|
||||
job['pcb'],
|
||||
job['lmc_file'],
|
||||
evidence_dir=Path(root) / 'diagnostics',
|
||||
emit=emit,
|
||||
)
|
||||
from .maintenance_entry import maintenance_project_files
|
||||
for index, config in enumerate(maintenance_project_files(job['project'], job['pcb'])):
|
||||
ensure_project_central_library(config, job['lmc_file'],
|
||||
evidence_dir=Path(root)/'diagnostics'/f'project-{index}', emit=emit)
|
||||
emit(
|
||||
stage='verify_mode',
|
||||
mode=('library_manager_native' if bool(job.get('native_lm_verify', False))
|
||||
@@ -716,12 +718,13 @@ def execute_job(job, emit):
|
||||
pcb_started_ns = time.time_ns()
|
||||
if bool(job.get('native_lm_verify', False)):
|
||||
from .library_manager_verify import launch_library_verify
|
||||
dispatch_deadline = time.monotonic()+lm_dispatch_timeout_seconds(job['timeout'])
|
||||
emit(stage='lm_launch', library=job['lmc_file'], block=job['block_name'])
|
||||
lm_session = launch_library_verify(
|
||||
job['lmc_file'],
|
||||
job['block_name'],
|
||||
emit,
|
||||
time.monotonic() + pcb_startup_timeout_seconds(job['timeout']),
|
||||
dispatch_deadline,
|
||||
mentor_bin=job.get('mentor_bin'),
|
||||
interactive_input=bool(job.get('interactive_input', False)),
|
||||
)
|
||||
@@ -737,8 +740,8 @@ def execute_job(job, emit):
|
||||
worker_parent_pid = int(launcher_process.pid)
|
||||
emit(stage='pcb_launcher_started', launcher_pid=worker_parent_pid,
|
||||
command=list(command), strategy='reusable_circuit_direct')
|
||||
pcb_deadline = reset_verify_activity_deadline(
|
||||
0.0, time.monotonic(), timeout_seconds=pcb_startup_timeout_seconds(job['timeout']))
|
||||
pcb_deadline = (dispatch_deadline if job.get('native_lm_verify') else
|
||||
time.monotonic()+pcb_startup_timeout_seconds(job['timeout']))
|
||||
while time.monotonic() < pcb_deadline:
|
||||
if job.get('cancel_signal') and Path(job['cancel_signal']).exists():
|
||||
raise RuntimeError('VERIFY_CANCELLED: PCB 启动期间收到停止请求')
|
||||
@@ -771,6 +774,8 @@ def execute_job(job, emit):
|
||||
break
|
||||
time.sleep(.2)
|
||||
if worker_pid is None:
|
||||
if job.get('native_lm_verify'):
|
||||
raise RuntimeError('VERIFY_DISPATCH_FAILED: LM 菜单返回但未观察到目标 PCB,未执行校验')
|
||||
raise RuntimeError('ReusableCircuit common 启动后未观察到 wg PCB 子进程')
|
||||
if job.get('diagnostic_only'):
|
||||
# Global PCB COM can wait on another design's modal window.
|
||||
@@ -926,6 +931,8 @@ def execute_job(job, emit):
|
||||
)
|
||||
if completed and lm_session is not None:
|
||||
lm_session.close()
|
||||
elif worker_pid is None and lm_session is not None:
|
||||
lm_session.close() # No PCB was dispatched; close only this owned LM.
|
||||
|
||||
|
||||
def cleanup_verify_processes(lm_pid, worker_pid, worker_parent_pid=None, *,
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
"""Open the official backup UI; version selection and writes stay native."""
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from pathlib import PureWindowsPath
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -39,6 +43,36 @@ def native_backup_project(library, block):
|
||||
return project
|
||||
|
||||
|
||||
def native_backup_target(request, *, mode='versions', target_database=None):
|
||||
"""Bind the official tool without requiring an openable/healthy PCB."""
|
||||
from .backup_recovery import _project_text
|
||||
from .maintenance_entry import maintenance_project_files
|
||||
if mode not in ('versions', 'repair'):
|
||||
raise ValueError('NATIVE_TARGET_MODE_INVALID')
|
||||
project = native_backup_project(request.lmc_file, request.block_name)
|
||||
candidates = (project,)
|
||||
if target_database is not None:
|
||||
pcbs = list((project.parent/'PCB').glob('*.pcb'))
|
||||
if len(pcbs) != 1:
|
||||
raise ValueError('NATIVE_TARGET_UNCONFIRMED: 无法唯一绑定 PCB 工程')
|
||||
candidates = maintenance_project_files(project, pcbs[0])
|
||||
matches = []
|
||||
for config in candidates:
|
||||
values = re.findall(r'^KEY iCDBDir "([^"\r\n]+)"\s*$', _project_text(config), re.M)
|
||||
if len(values) != 1 or not values[0] or PureWindowsPath(values[0]).root or PureWindowsPath(values[0]).drive:
|
||||
if config == project or target_database is None:
|
||||
raise ValueError('NATIVE_TARGET_UNCONFIRMED: 主 PRJ 的 iCDBDir 未唯一绑定')
|
||||
continue
|
||||
database = (config.parent/values[0]).resolve()
|
||||
if not database.is_relative_to(project.parent.resolve()):
|
||||
raise ValueError('NATIVE_TARGET_UNCONFIRMED: 数据库不在当前工程内')
|
||||
if target_database is None or database == Path(target_database).resolve():
|
||||
matches.append((config, database))
|
||||
if len(matches) != 1:
|
||||
raise ValueError('NATIVE_TARGET_UNCONFIRMED: 诊断数据库未绑定唯一工程 PRJ')
|
||||
return matches[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NativeBackupSession:
|
||||
process: object
|
||||
@@ -46,10 +80,18 @@ class NativeBackupSession:
|
||||
library: Path
|
||||
block: str
|
||||
path_lease: object = None
|
||||
mode: str = 'versions'
|
||||
project: Path | None = None
|
||||
database: Path | None = None
|
||||
session_record_path: Path | None = None
|
||||
operation_id: str | None = None
|
||||
|
||||
|
||||
def launch_native_backup(request):
|
||||
project = native_backup_project(request.lmc_file, request.block_name)
|
||||
def launch_native_backup(request, *, mode='versions', target_database=None):
|
||||
from .library_lock import recheck_previous_release
|
||||
from .session_record import register_worker_session
|
||||
from .operation_queue import check_active, native_channel, task_stop
|
||||
project, database = native_backup_target(request, mode=mode, target_database=target_database)
|
||||
native_backup_executable(request.mentor_bin)
|
||||
root = Path(request.evidence_dir).resolve()/('native-backup-'+uuid4().hex[:12])
|
||||
library_root = Path(request.lmc_file).resolve().parent
|
||||
@@ -57,17 +99,54 @@ def launch_native_backup(request):
|
||||
raise ValueError('运行记录目录需位于中心库之外。')
|
||||
root.mkdir(parents=True)
|
||||
job = root/'request.json'
|
||||
job.write_text(json.dumps({'library': str(request.lmc_file), 'block': request.block_name,
|
||||
'project': str(project), 'mentor_bin': request.mentor_bin}, ensure_ascii=False), 'utf-8')
|
||||
recheck_previous_release(request.lmc_file, root/'previous-release',
|
||||
block_name=request.block_name, stop=task_stop(None))
|
||||
record = {'library': str(request.lmc_file), 'block': request.block_name,
|
||||
'project': str(project), 'database': str(database), 'mode': mode,
|
||||
'project_sha256': hashlib.sha256(project.read_bytes()).hexdigest(),
|
||||
'target_database': str(target_database) if target_database is not None else None,
|
||||
'mentor_bin': request.mentor_bin, 'automatic_repair': False,
|
||||
'operation': 'native_backup', 'operation_id': root.name,
|
||||
'session_record_path': str(root/'session.json'), 'start_gate': str(root/'start-worker.signal'),
|
||||
'cancel_signal': str(root/'cancel-worker.signal'), 'supervisor_pid': os.getpid(), 'desktop': None}
|
||||
job.write_text(json.dumps(record, ensure_ascii=False), 'utf-8')
|
||||
command = ([sys.executable] if getattr(sys, 'frozen', False) else
|
||||
[sys.executable, '-m', 'cbb_refresh_verify'])+['--native-backup-worker', str(job)]
|
||||
process = subprocess.Popen(command, creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
||||
return NativeBackupSession(process, job, Path(request.lmc_file), request.block_name)
|
||||
try:
|
||||
check_active()
|
||||
process = subprocess.Popen(command, creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
||||
register_worker_session(process, record)
|
||||
with native_channel():
|
||||
check_active()
|
||||
Path(record['start_gate']).touch()
|
||||
except BaseException:
|
||||
Path(record['cancel_signal']).touch() # The unadmitted child exits without native launch.
|
||||
raise
|
||||
return NativeBackupSession(process, job, Path(request.lmc_file), request.block_name,
|
||||
mode=mode, project=project, database=database, session_record_path=Path(record['session_record_path']),
|
||||
operation_id=record['operation_id'])
|
||||
|
||||
|
||||
class _BackupJobWriter:
|
||||
"""Independent Job handle keeps future descendants inside the writer boundary."""
|
||||
def __init__(self, job):
|
||||
import win32api, win32con
|
||||
current = win32api.GetCurrentProcess()
|
||||
self.job = win32api.DuplicateHandle(current, job, current, 0, False, win32con.DUPLICATE_SAME_ACCESS)
|
||||
|
||||
def poll(self):
|
||||
import win32job
|
||||
if self.job is None:
|
||||
return 0
|
||||
if win32job.QueryInformationJobObject(self.job, win32job.JobObjectBasicProcessIdList):
|
||||
return None
|
||||
self.job.Close(); self.job = None
|
||||
return 0
|
||||
|
||||
|
||||
class _NativeBackupProcess:
|
||||
"""Track this visible tool and its descendants without killing on close."""
|
||||
def __init__(self, command, cwd):
|
||||
def __init__(self, command, cwd, *, before_resume=None):
|
||||
import win32con, win32job, win32process
|
||||
self.job = win32job.CreateJobObject(None, '')
|
||||
startup = win32process.STARTUPINFO()
|
||||
@@ -80,6 +159,8 @@ class _NativeBackupProcess:
|
||||
raise
|
||||
try:
|
||||
win32job.AssignProcessToJobObject(self.job, self.handle)
|
||||
if before_resume is not None:
|
||||
before_resume(self)
|
||||
win32process.ResumeThread(thread)
|
||||
except Exception:
|
||||
# No native code has run while suspended; there can be no DB write.
|
||||
@@ -102,11 +183,54 @@ class _NativeBackupProcess:
|
||||
self.handle.Close(); self.job.Close()
|
||||
|
||||
|
||||
def run_native_backup_worker(job):
|
||||
"""Publish admission refusal only when this exact worker never entered native work."""
|
||||
from .desktop_worker import run_native_worker
|
||||
from .management import _write_json
|
||||
from .session_record import IDENTITY_KEYS, process_identity, read_session_record
|
||||
job = Path(job)
|
||||
entered = False
|
||||
request = json.loads(job.read_text('utf-8'))
|
||||
identity = process_identity(os.getpid())
|
||||
def registered():
|
||||
try:
|
||||
path = Path(request['session_record_path'])
|
||||
session = read_session_record(path, evidence_root=job.parent)
|
||||
return (session['operation_id'] == request['operation_id']
|
||||
and session.get('operation') == 'native_backup'
|
||||
and session.get('library') == request['library'] and session.get('block') == request['block']
|
||||
and request.get('project') in session.get('resource_scope', {}).get('projects', ())
|
||||
and any(all(process.get(key) == identity[key] for key in IDENTITY_KEYS)
|
||||
for process in session.get('processes', ()) if process.get('role') == 'worker'))
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
return False # Unknown registration never establishes a safe terminal result.
|
||||
def execute(path):
|
||||
nonlocal entered
|
||||
if not registered():
|
||||
raise ValueError('NATIVE_BACKUP_ADMISSION_UNCONFIRMED: 当前工程与工作进程未登记')
|
||||
entered = True
|
||||
return native_backup_worker_main(path)
|
||||
try:
|
||||
return run_native_worker(execute, job)
|
||||
except Exception as error:
|
||||
bound = registered()
|
||||
_write_json(job.with_name('result.json'), {'status': 'FAILED', 'detail': str(error),
|
||||
'operation_id': request.get('operation_id'), 'worker_identity': identity,
|
||||
'library': request.get('library'), 'block': request.get('block'),
|
||||
'mode': request.get('mode', 'versions'), 'project': request.get('project'),
|
||||
'database': request.get('database'), 'worker_entered': entered,
|
||||
'native_started': None if entered else False,
|
||||
'safe_to_continue': bound and not entered, 'restore_confirmed': False,
|
||||
'repair_confirmed': False, 'validation_passed': False, 'automatic_repair': False})
|
||||
return 1
|
||||
|
||||
|
||||
def native_backup_worker_main(job, *, process_factory=None):
|
||||
from .library_lock import LibraryLock
|
||||
from .management import _write_json
|
||||
job = Path(job)
|
||||
result = {'status': 'FAILED', 'safe_to_continue': True, 'restore_confirmed': False}
|
||||
result = {'status': 'FAILED', 'safe_to_continue': True, 'restore_confirmed': False,
|
||||
'repair_confirmed': False, 'validation_passed': False, 'automatic_repair': False}
|
||||
native = None
|
||||
def write_evidence(name, value):
|
||||
try:
|
||||
@@ -117,14 +241,34 @@ def native_backup_worker_main(job, *, process_factory=None):
|
||||
try:
|
||||
record = json.loads(job.read_text('utf-8'))
|
||||
library, block = Path(record['library']), record['block']
|
||||
result.update(library=str(library), block=block)
|
||||
project = native_backup_project(library, block)
|
||||
result.update(library=str(library), block=block, operation_id=record.get('operation_id'))
|
||||
from types import SimpleNamespace
|
||||
mode = record.get('mode', 'versions')
|
||||
project, database = native_backup_target(SimpleNamespace(lmc_file=library, block_name=block),
|
||||
mode=mode, target_database=record.get('target_database'))
|
||||
if project != Path(record['project']):
|
||||
raise ValueError('工程身份已变化,未打开原生工具。')
|
||||
if record.get('project_sha256') and hashlib.sha256(project.read_bytes()).hexdigest() != record['project_sha256']:
|
||||
raise ValueError('工程内容身份已变化,未打开原生工具。')
|
||||
if record.get('database') and database != Path(record['database']):
|
||||
raise ValueError('数据库身份已变化,未打开原生工具。')
|
||||
result.update(mode=mode, project=str(project), database=str(database),
|
||||
session_record_path=record.get('session_record_path'))
|
||||
executable = native_backup_executable(record.get('mentor_bin', ''))
|
||||
released = True
|
||||
with LibraryLock(library, checkpoint=False, safe_to_release=lambda: released):
|
||||
native = (process_factory or _NativeBackupProcess)([str(executable), '-p', str(project)], project.parent)
|
||||
with LibraryLock(library, checkpoint=bool(record.get('session_record_path')),
|
||||
block_name=block, safe_to_release=lambda: released):
|
||||
from .maintenance_entry import _assert_native_idle_for
|
||||
_assert_native_idle_for(SimpleNamespace(project=project, database=database, library=library))
|
||||
def admitted(process):
|
||||
from .session_record import update_worker_state
|
||||
from .desktop_worker import register_native_writer
|
||||
if record.get('session_record_path'):
|
||||
update_worker_state(record, 'NATIVE_CALL', identities=(('native', process.pid),), required=True)
|
||||
register_native_writer(_BackupJobWriter(process.job))
|
||||
command = [str(executable), '-p', str(project)] # User alone invokes Repair in the official UI.
|
||||
native = (process_factory(command, project.parent) if process_factory else
|
||||
_NativeBackupProcess(command, project.parent, before_resume=admitted))
|
||||
released = False
|
||||
result.update(status='OPEN', native_pid=native.pid, project=str(project), safe_to_continue=False)
|
||||
write_evidence('state.json', result)
|
||||
@@ -145,6 +289,7 @@ def native_backup_worker_main(job, *, process_factory=None):
|
||||
time.sleep(.25)
|
||||
released = True
|
||||
result.update(status='CLOSED', native_exit_code=exit_code, safe_to_continue=True)
|
||||
result['release'] = {'native_exited': True, 'remaining_pids': [], 'safe_to_continue': True}
|
||||
return 0 if result['native_exit_code'] == 0 else 1
|
||||
except Exception as error:
|
||||
result.update(status='FAILED', detail=str(error), safe_to_continue=native is None)
|
||||
|
||||
@@ -192,6 +192,10 @@ def _inspect_project(job, process, emit):
|
||||
result = {'status': 'UNCONFIRMED', 'project': str(project), 'database': str(database),
|
||||
'native_pid': process.pid, 'inspection_only': True, 'repair_requested': False,
|
||||
'repair_required': False, 'output': ''}
|
||||
if getattr(process, '_handle', None) is not None:
|
||||
from .residual_sessions import _handle_identity, _force_ownership
|
||||
result['native_identity'] = dict(pid=process.pid,
|
||||
**_handle_identity(process._handle), **_force_ownership(process._handle, process.pid))
|
||||
deadline = time.monotonic() + max(1, float(job['timeout']))
|
||||
closing = False
|
||||
last_state = None
|
||||
|
||||
@@ -48,8 +48,14 @@ def validate_owner(job, target_pid, launcher_pid, worker_pid):
|
||||
|
||||
@contextmanager
|
||||
def resource_guard(job, allowed_pids):
|
||||
import win32con, win32file
|
||||
from .icdb_ownership import database_directories, unreleased_addresses
|
||||
import hashlib
|
||||
from .file_probe import hold_release_files
|
||||
from .maintenance_entry import maintenance_project_files
|
||||
from .native_repair import RepairTarget, run_native_repair
|
||||
from .native_occupancy import confirmed_inspection
|
||||
from .session_record import IDENTITY_KEYS
|
||||
from .operation_queue import task_stop
|
||||
from .management import _write_json
|
||||
from .owner_probe import query_process_snapshot
|
||||
from .library_lock import _owner_is_blocking
|
||||
from .verify_runner import preflight, VerifyRequest
|
||||
@@ -58,21 +64,48 @@ def resource_guard(job, allowed_pids):
|
||||
if project.resolve() != Path(job['project']).resolve() or pcb.resolve() != Path(job['pcb']).resolve():
|
||||
raise ValueError('PCB_OPEN_PROJECT_CHANGED')
|
||||
roots = (project.parent, library.parent/'SymbolLibs')
|
||||
configs = maintenance_project_files(project, pcb)
|
||||
def binding():
|
||||
return {str(path.resolve()): hashlib.sha256(path.read_bytes()).hexdigest() for path in (pcb, *configs)}
|
||||
original_binding = binding()
|
||||
def check():
|
||||
if unreleased_addresses(library, project, pcb):
|
||||
raise ValueError('目标仍有数据库地址记录,暂不能确认读写打开。')
|
||||
for row in query_process_snapshot():
|
||||
if row['pid'] not in allowed_pids and _owner_is_blocking(row, library, roots):
|
||||
raise ValueError(f'目标还有其他会话,PID={row["pid"]};未确认读写打开。')
|
||||
check()
|
||||
paths = [library, pcb.parent/'Layout/LayoutDB.lyt',
|
||||
*(d/'icdb.dat' for d in database_directories(library, project, pcb) if (d/'icdb.dat').is_file())]
|
||||
stop = task_stop(None)
|
||||
with ExitStack() as stack:
|
||||
for path in dict.fromkeys(paths):
|
||||
handle = win32file.CreateFile(str(path), win32con.GENERIC_READ, 0, None,
|
||||
win32con.OPEN_EXISTING, win32con.FILE_ATTRIBUTE_NORMAL, None)
|
||||
stack.callback(handle.Close)
|
||||
files = stack.enter_context(hold_release_files(library, roots, stop=stop))
|
||||
inspections = []
|
||||
if files['addresses']:
|
||||
targets = files['inspection_targets']
|
||||
if any(Path(item['project']).resolve() not in {p.resolve() for p in configs} for item in targets):
|
||||
raise ValueError('PCB_OPEN_DATABASE_UNCONFIRMED: 地址未绑定实际工程配置,需人工核对')
|
||||
stack.close() # Native inspection must access the DB; repeat exact proof afterwards.
|
||||
for index, item in enumerate(targets):
|
||||
target = RepairTarget(Path(item['project']), Path(item['database']))
|
||||
evidence = Path(job['root'])/('open-inspection-'+str(index))
|
||||
result = run_native_repair(target, evidence, mentor_bin=job.get('mentor_bin'),
|
||||
timeout_seconds=min(45, float(job.get('timeout', 45))), inspection_only=True, stop=stop)
|
||||
identity = result.get('native_identity') or {}
|
||||
if (not confirmed_inspection(target, result) or result['status'] != 'NOT_IN_USE'
|
||||
or identity.get('pid') != result.get('native_pid')
|
||||
or any(identity.get(key) is None for key in IDENTITY_KEYS)
|
||||
or not identity.get('image') or not identity.get('owner_sid')):
|
||||
raise ValueError('PCB_OPEN_DATABASE_UNCONFIRMED: 原生检查未确认当前数据库未占用')
|
||||
inspections.append(result)
|
||||
current = stack.enter_context(hold_release_files(library, roots, stop=stop))
|
||||
if any(current[key] != files[key] for key in (
|
||||
'exclusive_files', 'binding_files', 'addresses', 'signatures', 'inspection_targets', 'resource_scope')):
|
||||
raise ValueError('PCB_OPEN_RESOURCE_CHANGED: 原生检查期间资源或地址身份变化')
|
||||
if binding() != original_binding:
|
||||
raise ValueError('PCB_OPEN_PROJECT_CHANGED: 原生检查期间工程引用变化')
|
||||
check()
|
||||
_write_json(Path(job['root'])/'pcb-open-resource-proof.json', {
|
||||
'operation_id': job.get('operation_id'), 'library': str(library),
|
||||
'project': str(project), 'pcb': str(pcb), 'binding': original_binding,
|
||||
'exclusive_files': files['exclusive_files'], 'addresses': files['addresses'],
|
||||
'native_inspections': inspections, 'repair_confirmed': False})
|
||||
yield
|
||||
|
||||
|
||||
|
||||
@@ -548,6 +548,8 @@ def run_verify(request, *, progress=None, flag_reader=None, process_runner=None,
|
||||
dialog_supervisor_factory=None, version_prompt=None, stop=None, context=None,
|
||||
admission_timeout_seconds=10):
|
||||
from .operation_queue import BatchTaskContext, TaskHandoff, current_context, task_stop
|
||||
if stop is not None and stop.is_set() or context is not None and context.stop.is_set():
|
||||
return VerifyResult('CANCELLED', stderr='已取消,未启动原生操作。', safe_to_continue=True)
|
||||
options = dict(progress=progress, flag_reader=flag_reader, process_runner=process_runner,
|
||||
port_check=port_check, dialog_supervisor_factory=dialog_supervisor_factory,
|
||||
version_prompt=version_prompt)
|
||||
@@ -776,13 +778,13 @@ def _run_verify_serialized(request, *, progress=None, flag_reader=None, process_
|
||||
finally:
|
||||
timing.end('failure_diagnostics')
|
||||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}:
|
||||
from .icdb_recovery import ERROR_CODE, RECOVERY_ACTION, parse_session_loss, recovery_detail
|
||||
recovery = parse_session_loss(result.stderr)
|
||||
from .icdb_recovery import ERROR_CODE, RECOVERY_ACTION, parse_recovery_request, recovery_detail
|
||||
recovery = parse_recovery_request(result.stderr)
|
||||
if recovery is None and result.evidence_path:
|
||||
try:
|
||||
saved = json.loads(result.evidence_path.with_name('diagnostics').joinpath(
|
||||
'icdb-recovery.json').read_text('utf-8'))
|
||||
recovery = parse_session_loss(saved.get('native_message', ''))
|
||||
recovery = parse_recovery_request(saved.get('native_message', ''))
|
||||
except (OSError, ValueError, AttributeError):
|
||||
pass
|
||||
if recovery or ERROR_CODE.casefold() in result.stderr.casefold():
|
||||
@@ -952,6 +954,7 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
launcher_pid = None
|
||||
lm_pid = None
|
||||
verify_invoked = False
|
||||
lm_dispatch_deadline = None
|
||||
invoke_returned = False
|
||||
pcb_baseline = set()
|
||||
last_observation = 0.0
|
||||
@@ -1045,6 +1048,9 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
if event.get('launcher_pid'): launcher_pid = event['launcher_pid']
|
||||
if event.get('lm_pid'): lm_pid = event['lm_pid']
|
||||
stage = event.get('stage')
|
||||
if stage == 'lm_launch' and lm_dispatch_deadline is None:
|
||||
from .mentor_adapter import lm_dispatch_timeout_seconds
|
||||
lm_dispatch_deadline = (event.get('monotonic_seconds') or time.monotonic()) + lm_dispatch_timeout_seconds(request.timeout_seconds)
|
||||
if stage == 'verify_invoke': verify_invoked = True
|
||||
if stage == 'verify_invoke_returned': invoke_returned = True
|
||||
if stage == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
|
||||
@@ -1162,7 +1168,8 @@ def _run_verify(request, *, timing, progress=None, flag_reader=None, process_run
|
||||
error_stage=phase.value,
|
||||
safe_to_continue=False,
|
||||
))
|
||||
if deadline.expired():
|
||||
if deadline.expired() or (lm_dispatch_deadline is not None and target_pid is None
|
||||
and time.monotonic() >= lm_dispatch_deadline):
|
||||
_write_failure_windows(root, latest_snapshots)
|
||||
# Stop automation without killing PCB/iCDB during a write.
|
||||
_write_process_tree(root, process, target_pid)
|
||||
|
||||
@@ -54,7 +54,7 @@ def test_preflight_rejects_pcb_junction_outside_current_block(tmp_path):
|
||||
preflight(VerifyRequest(library, 'CBB', tmp_path/'evidence'))
|
||||
|
||||
|
||||
def test_serial_guard_covers_preflight_and_content_check(tmp_path, monkeypatch):
|
||||
def test_nonnative_preparation_leaves_real_host_channel_available(tmp_path, monkeypatch):
|
||||
from cbb_refresh_verify import verify_runner as runner
|
||||
entered, release = Event(), Event()
|
||||
calls, results = [], []
|
||||
@@ -70,14 +70,14 @@ def test_serial_guard_covers_preflight_and_content_check(tmp_path, monkeypatch):
|
||||
thread.start()
|
||||
assert entered.wait(3)
|
||||
try:
|
||||
second = runner.run_verify(VerifyRequest(tmp_path/'lib.lmc', 'second', tmp_path/'evidence'))
|
||||
assert second.status == 'BLOCKED'
|
||||
assert second.safe_to_continue
|
||||
assert second.error_category == 'NATIVE_CHANNEL_BUSY'
|
||||
from cbb_refresh_verify.operation_queue import native_channel
|
||||
with native_channel():
|
||||
pass
|
||||
assert calls == ['first']
|
||||
finally:
|
||||
release.set()
|
||||
thread.join(6)
|
||||
assert results[0].status == 'PASS'
|
||||
|
||||
|
||||
def test_recovery_retains_temporary_network_mapping():
|
||||
|
||||
@@ -277,7 +277,8 @@ def test_contents_session_loss_records_each_item_without_launching_pcb_or_repair
|
||||
assert results[1].error_category == 'ICDB_REPAIR_REQUIRED'
|
||||
|
||||
|
||||
def test_supervisor_enriches_generic_failure_from_fresh_native_log(tmp_path, monkeypatch):
|
||||
@pytest.mark.parametrize('fault', ['session_loss', 'inconsistent'])
|
||||
def test_supervisor_enriches_generic_failure_from_fresh_native_log(tmp_path, monkeypatch, fault):
|
||||
from test_verify_runner import fixture, request
|
||||
from cbb_refresh_verify.icdb_recovery import snapshot_log
|
||||
from cbb_refresh_verify.verify_diagnostics import failure_logs_main
|
||||
@@ -292,7 +293,8 @@ def test_supervisor_enriches_generic_failure_from_fresh_native_log(tmp_path, mon
|
||||
(root/'diagnostics').mkdir()
|
||||
(root/'diagnostics/startup-log-baseline.json').write_text(json.dumps({
|
||||
'source': str(log), 'baseline': snapshot_log(log)}), encoding='utf-8')
|
||||
log.write_text(ERROR, encoding='utf-8')
|
||||
from test_issue15_native_entry import inconsistency
|
||||
log.write_text(ERROR if fault == 'session_loss' else inconsistency(work/'database'), encoding='utf-8')
|
||||
(root/'outcome.json').write_text(json.dumps({
|
||||
'status': 'FAIL', 'stderr': 'MENTOR_DATABASE_LOAD_FAILED: opening failed',
|
||||
'safe_to_continue': False}), encoding='utf-8')
|
||||
@@ -305,7 +307,11 @@ def test_supervisor_enriches_generic_failure_from_fresh_native_log(tmp_path, mon
|
||||
assert result.error_category == 'ICDB_REPAIR_REQUIRED'
|
||||
assert result.safe_to_continue is True
|
||||
assert 'opening failed' in result.stderr
|
||||
assert '200091-CBB' in result.stderr
|
||||
assert '__ReusableBlocksSymb' in result.stderr
|
||||
assert '2026-10-24 08.22.29-10308' in result.stderr
|
||||
if fault == 'session_loss':
|
||||
assert '200091-CBB' in result.stderr
|
||||
assert '__ReusableBlocksSymb' in result.stderr
|
||||
assert '2026-10-24 08.22.29-10308' in result.stderr
|
||||
else:
|
||||
assert str(work/'database') in result.stderr
|
||||
assert '未修复' in result.stderr
|
||||
assert json.loads(result.evidence_path.read_text('utf-8'))['error_category'] == 'ICDB_REPAIR_REQUIRED'
|
||||
@@ -0,0 +1,380 @@
|
||||
"""Issue 15 recovery entry regressions. Native boundaries never launch Mentor."""
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_real_owner_probe(monkeypatch):
|
||||
monkeypatch.setattr('cbb_refresh_verify.owner_probe.query_owners', lambda names: [])
|
||||
|
||||
|
||||
def test_ordinary_worker_aligns_both_bound_projects_and_retains_each_original(tmp_path, monkeypatch):
|
||||
from test_verify_worker_stability import exercise_worker
|
||||
from cbb_refresh_verify import mentor_adapter as adapter
|
||||
job, events, execute = exercise_worker(tmp_path, monkeypatch)
|
||||
main = Path(job['project']); aux = Path(job['pcb']).parent/'pcb.prj'
|
||||
original_main = b'KEY CentralLibrary "old-main.lmc"\r\n#\x81\x40\r\n'
|
||||
original_aux = b'\xff\xfe'+ 'KEY CentralLibrary "old-pcb.lmc"\r\n'.encode('utf-16-le')
|
||||
main.write_bytes(original_main); aux.write_bytes(original_aux)
|
||||
assert execute()['status'] == 'PASS'
|
||||
from cbb_refresh_verify.backup_recovery import _project_text
|
||||
for path in (main, aux):
|
||||
import re
|
||||
reference = re.search(r'KEY CentralLibrary "([^"]+)"', _project_text(path))[1]
|
||||
assert (path.parent/reference).resolve() == Path(job['lmc_file']).resolve()
|
||||
saved = [p.read_bytes() for p in (Path(job['root'])/'diagnostics').rglob('*.before-central-library')]
|
||||
assert original_main in saved and original_aux in saved
|
||||
|
||||
|
||||
def test_ordinary_legacy_binding_never_rewrites_template(tmp_path, monkeypatch):
|
||||
from test_verify_worker_stability import exercise_worker
|
||||
job, _, execute = exercise_worker(tmp_path, monkeypatch)
|
||||
main = Path(job['project']); pcb = Path(job['pcb'])
|
||||
main.write_text('KEY CentralLibrary "old.lmc"\n')
|
||||
pcb.write_text('KEY ProjectFile "../CBB.prj"\n')
|
||||
template = pcb.parent/'template.prj'; template.write_text('KEY CentralLibrary "do-not-touch.lmc"\n')
|
||||
original = template.read_bytes()
|
||||
assert execute()['status'] == 'PASS'
|
||||
assert template.read_bytes() == original
|
||||
assert 'old.lmc' not in main.read_text()
|
||||
|
||||
|
||||
def inconsistency(database):
|
||||
return (f'Project database [{database}] is inconsistent. It has been manually copied while the iCDB Server was running. '
|
||||
"Please use iCDB Project Backup 'Repair project' functionality to fix this issue.")
|
||||
|
||||
|
||||
def test_current_inconsistency_has_structured_database_and_old_error_is_not_reused(tmp_path):
|
||||
from cbb_refresh_verify import icdb_recovery as recovery, mentor_adapter as adapter
|
||||
source = tmp_path/'LogFiles/PCBLogFile.txt'; source.parent.mkdir()
|
||||
db = tmp_path/'database'; message = inconsistency(db)
|
||||
source.write_text(message+'\n')
|
||||
baseline = recovery.snapshot_log(source)
|
||||
assert recovery.read_log_failure(source, baseline)[1] is None
|
||||
source.write_text('new header\n'+message+'\n')
|
||||
assert recovery.read_log_failure(source, baseline)[1] is None
|
||||
baseline = recovery.snapshot_log(source)
|
||||
with source.open('a') as stream: stream.write(message+'\n')
|
||||
text, record = recovery.read_log_failure(source, baseline)
|
||||
assert record['database_path'] == str(db) and record['scope'] == 'project'
|
||||
assert record['kind'] == 'database_inconsistent'
|
||||
detail = adapter.pcb_log_reports_missing_server(tmp_path/'CBB.pcb', offset=baseline)
|
||||
assert 'ICDB_REPAIR_REQUIRED' in detail and str(db) in detail and '未修复' in detail
|
||||
|
||||
|
||||
def test_manual_repair_selects_diagnosed_bound_database_without_layout_or_healthy_header(tmp_path):
|
||||
from test_native_backup import project_fixture
|
||||
from cbb_refresh_verify import native_backup as native
|
||||
request, main = project_fixture(tmp_path)
|
||||
pcb = main.parent/'PCB/Block.pcb'; pcb.parent.mkdir()
|
||||
pcb.write_text('KEY ProjectFile "../Block.prj"\n')
|
||||
aux = pcb.parent/'pcb.prj'; aux.write_text('KEY iCDBDir "database"\n')
|
||||
for prj in (main, aux):
|
||||
(prj.parent/'database').mkdir(); (prj.parent/'database/icdb.dat').write_bytes(b'not-a-healthy-header')
|
||||
assert native.native_backup_target(request, mode='repair')[0] == main
|
||||
assert native.native_backup_target(request, mode='repair', target_database=aux.parent/'database')[0] == aux
|
||||
with pytest.raises(ValueError, match='TARGET'):
|
||||
native.native_backup_target(request, mode='repair', target_database=tmp_path/'foreign')
|
||||
|
||||
|
||||
def test_versions_launcher_rechecks_parent_retained_lease_and_persists_before_gate(tmp_path, monkeypatch):
|
||||
from test_native_backup import project_fixture
|
||||
from cbb_refresh_verify import native_backup as native, library_lock, session_record
|
||||
request, project = project_fixture(tmp_path)
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: tmp_path/'native.exe')
|
||||
calls = []
|
||||
monkeypatch.setattr(library_lock, 'recheck_previous_release', lambda *a, **k: calls.append('recheck'))
|
||||
monkeypatch.setattr(session_record, 'process_identity', lambda pid: dict(pid=pid, created='now', executable='python', owner='test'))
|
||||
def spawn(command, **kw):
|
||||
job = json.loads(Path(command[-1]).read_text())
|
||||
assert calls == ['recheck'] and not Path(job['start_gate']).exists()
|
||||
calls.append('spawn')
|
||||
return SimpleNamespace(pid=123, poll=lambda: None)
|
||||
monkeypatch.setattr(native.subprocess, 'Popen', spawn)
|
||||
session = native.launch_native_backup(request, mode='repair')
|
||||
record = json.loads(session.job.read_text())
|
||||
assert Path(record['start_gate']).is_file()
|
||||
assert Path(record['session_record_path']).is_file()
|
||||
assert session.mode == 'repair' and session.database == project.parent/'database'
|
||||
assert session.operation_id == record['operation_id']
|
||||
assert record['automatic_repair'] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['spawn', 'registration'])
|
||||
def test_backup_failure_leaves_start_gate_closed(tmp_path, monkeypatch, failure):
|
||||
from test_native_backup import project_fixture
|
||||
from cbb_refresh_verify import native_backup as native, session_record
|
||||
request, _ = project_fixture(tmp_path)
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: tmp_path/'native.exe')
|
||||
def spawn(*a, **k):
|
||||
if failure == 'spawn': raise OSError('start failed')
|
||||
return SimpleNamespace(pid=123, poll=lambda: None)
|
||||
monkeypatch.setattr(native.subprocess, 'Popen', spawn)
|
||||
monkeypatch.setattr(session_record, 'register_worker_session', lambda *a: (_ for _ in ()).throw(OSError('register failed')))
|
||||
with pytest.raises(OSError): native.launch_native_backup(request)
|
||||
job = next(request.evidence_dir.glob('native-backup-*/request.json'))
|
||||
record = json.loads(job.read_text())
|
||||
assert not Path(record['start_gate']).exists()
|
||||
assert Path(record['cancel_signal']).exists()
|
||||
|
||||
|
||||
def test_changed_prj_bytes_refused_before_recovery_tool_dispatch(tmp_path, monkeypatch):
|
||||
import hashlib
|
||||
from test_native_backup import project_fixture
|
||||
from cbb_refresh_verify import native_backup as native
|
||||
request, project = project_fixture(tmp_path)
|
||||
job = tmp_path/'request.json'
|
||||
record = dict(library=str(request.lmc_file), block=request.block_name, project=str(project),
|
||||
project_sha256=hashlib.sha256(project.read_bytes()).hexdigest())
|
||||
job.write_text(json.dumps(record))
|
||||
project.write_text(project.read_text()+'# changed identity\n')
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: tmp_path/'native.exe')
|
||||
assert native.native_backup_worker_main(job, process_factory=lambda *a: pytest.fail('changed PRJ dispatched')) == 1
|
||||
assert '身份' in json.loads(job.with_name('result.json').read_text('utf-8'))['detail']
|
||||
|
||||
|
||||
def test_real_parent_retained_sidecar_is_reconciled_before_child_acquires_it_and_reopens(tmp_path, monkeypatch):
|
||||
import subprocess, sys
|
||||
from test_native_backup import project_fixture
|
||||
from cbb_refresh_verify import native_backup as native, library_lock as locks
|
||||
from cbb_refresh_verify.file_probe import hold_release_files
|
||||
request, _ = project_fixture(tmp_path)
|
||||
with locks.LibraryLock(request.lmc_file, safe_to_release=lambda: False, block_name='Block') as retained:
|
||||
pass
|
||||
@contextmanager
|
||||
def proof(library, **kwargs):
|
||||
with hold_release_files(library, (library.parent/'ReusableBlocksLP/Block/Work',)) as files:
|
||||
yield {**files, 'native_owners': []}
|
||||
monkeypatch.setattr(locks, '_prove_library_released', proof)
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: tmp_path/'not-launched.exe')
|
||||
popen = subprocess.Popen
|
||||
script = (
|
||||
'import json,sys\nfrom pathlib import Path\n'
|
||||
'from cbb_refresh_verify.desktop_worker import run_native_worker\n'
|
||||
'from cbb_refresh_verify.library_lock import LibraryLock\n'
|
||||
'def action(path):\n'
|
||||
' j=json.loads(Path(path).read_text())\n'
|
||||
' with LibraryLock(j["library"],checkpoint=False):\n'
|
||||
' Path(path).with_name("child-acquired").write_text("owned")\n'
|
||||
' return 0\n'
|
||||
'raise SystemExit(run_native_worker(action,sys.argv[1]))\n')
|
||||
def spawn(command, **kwargs):
|
||||
if '--native-backup-worker' in command:
|
||||
assert not locks.has_retained_library(request.lmc_file)
|
||||
return popen([sys.executable, '-c', script, command[-1]], **kwargs)
|
||||
return popen(command, **kwargs)
|
||||
monkeypatch.setattr(native.subprocess, 'Popen', spawn)
|
||||
try:
|
||||
for mode in ('versions', 'repair'):
|
||||
session = native.launch_native_backup(request, mode=mode)
|
||||
assert session.process.wait(10) == 0
|
||||
assert session.job.with_name('child-acquired').read_text() == 'owned'
|
||||
assert retained.path.is_file()
|
||||
finally:
|
||||
owner = locks._owners.pop(retained.key, None)
|
||||
if owner: owner['handle'].Close()
|
||||
|
||||
|
||||
def test_native_registration_failure_never_resumes_suspended_child(tmp_path):
|
||||
import sys
|
||||
from cbb_refresh_verify.native_backup import _NativeBackupProcess
|
||||
marker = tmp_path/'must-not-run'
|
||||
def refuse(process): raise OSError('required identity journal failed')
|
||||
with pytest.raises(OSError, match='journal'):
|
||||
_NativeBackupProcess([sys.executable, '-c', f'from pathlib import Path;Path({str(marker)!r}).touch()'],
|
||||
tmp_path, before_resume=refuse)
|
||||
assert not marker.exists()
|
||||
|
||||
|
||||
def test_independent_backup_job_registration_retains_descendants_after_caller_handles_close(tmp_path):
|
||||
import sys, time
|
||||
from cbb_refresh_verify.native_backup import _NativeBackupProcess, _BackupJobWriter
|
||||
script = tmp_path/'parent.py'; marker = tmp_path/'child.pid'
|
||||
script.write_text('import subprocess,sys,time\nfrom pathlib import Path\n'
|
||||
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(1)"])\n'
|
||||
f'Path({str(marker)!r}).write_text(str(p.pid))\ntime.sleep(.1)\n')
|
||||
process = _NativeBackupProcess([sys.executable, str(script)], tmp_path)
|
||||
registered = _BackupJobWriter(process.job)
|
||||
try:
|
||||
deadline = time.monotonic()+5
|
||||
while process.poll() is None and time.monotonic()<deadline: time.sleep(.02)
|
||||
assert marker.exists() and process.poll() == 0
|
||||
process.close()
|
||||
assert registered.poll() is None
|
||||
while registered.poll() is None and time.monotonic()<deadline: time.sleep(.02)
|
||||
assert registered.poll() == 0
|
||||
finally:
|
||||
if registered.job is not None: registered.job.Close()
|
||||
|
||||
|
||||
def test_common_admission_refusal_publishes_bound_receipt_and_ui_can_reopen(qtbot, tmp_path, monkeypatch):
|
||||
from threading import Event, Thread
|
||||
from test_native_backup import project_fixture
|
||||
from test_ui_filters import load_window
|
||||
from cbb_refresh_verify import native_backup as native, desktop_worker
|
||||
from cbb_refresh_verify.application import main
|
||||
request, _ = project_fixture(tmp_path)
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: tmp_path/'native.exe')
|
||||
monkeypatch.setattr(native.subprocess, 'Popen', lambda *a, **k: SimpleNamespace(pid=os.getpid(), poll=lambda: None))
|
||||
session = native.launch_native_backup(request)
|
||||
entered, release = Event(), Event()
|
||||
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
|
||||
def held_writer():
|
||||
with desktop_worker.worker_native_channel():
|
||||
entered.set(); release.wait(5)
|
||||
holder = Thread(target=held_writer); holder.start()
|
||||
assert entered.wait(2)
|
||||
monkeypatch.setattr(native, 'native_backup_worker_main', lambda *a: pytest.fail('refused action entered'))
|
||||
try:
|
||||
assert main(['--native-backup-worker', str(session.job)]) == 1
|
||||
finally:
|
||||
release.set(); holder.join(3)
|
||||
record = json.loads(session.job.with_name('result.json').read_text('utf-8'))
|
||||
assert record['safe_to_continue'] is True and record['native_started'] is False
|
||||
assert record['operation_id'] == json.loads(session.job.read_text('utf-8'))['operation_id']
|
||||
assert record['worker_identity']['pid'] == os.getpid()
|
||||
session.process.poll = lambda: 1
|
||||
launched = []
|
||||
window = load_window(qtbot, tmp_path, monkeypatch,
|
||||
native_backup_launcher=lambda req: (launched.append(req) or session))
|
||||
session.block = 'Beta'; record['block'] = 'Beta'
|
||||
session.job.with_name('result.json').write_text(json.dumps(record), encoding='utf-8')
|
||||
window._native_backup_session = session
|
||||
window._poll_native_backup(); qtbot.waitUntil(lambda: not window.busy)
|
||||
assert window._native_backup_session is None
|
||||
assert 'NATIVE_WRITER_ACTIVE' in window.status_label.text()
|
||||
window.search_edit.setText('Beta'); window.select_visible_button.click()
|
||||
window.open_native_backup(); qtbot.waitUntil(lambda: not window.busy)
|
||||
assert len(launched) == 1
|
||||
window._native_backup_session = None # The test adapter has no live native client.
|
||||
|
||||
|
||||
def test_unbound_admission_refusal_cannot_claim_release(tmp_path, monkeypatch):
|
||||
from cbb_refresh_verify import native_backup as native, desktop_worker
|
||||
job = tmp_path/'request.json'
|
||||
job.write_text(json.dumps(dict(library=str(tmp_path/'lib.lmc'), block='A', operation_id='foreign',
|
||||
session_record_path=str(tmp_path/'session.json'), project=str(tmp_path/'A.prj'))))
|
||||
monkeypatch.setattr(desktop_worker, 'run_native_worker', lambda *a: (_ for _ in ()).throw(RuntimeError('refused')))
|
||||
assert native.run_native_backup_worker(job) == 1
|
||||
result = json.loads(job.with_name('result.json').read_text('utf-8'))
|
||||
assert result['safe_to_continue'] is False
|
||||
|
||||
|
||||
def test_legacy_unregistered_backup_job_cannot_enter_native_action(tmp_path, monkeypatch):
|
||||
from cbb_refresh_verify import native_backup as native, desktop_worker
|
||||
job = tmp_path/'request.json'
|
||||
job.write_text(json.dumps(dict(library=str(tmp_path/'lib.lmc'), block='A', project=str(tmp_path/'A.prj'))))
|
||||
monkeypatch.setattr(desktop_worker, '_automation_clients', [])
|
||||
monkeypatch.setattr(native, 'native_backup_worker_main', lambda *a: pytest.fail('unregistered action entered'))
|
||||
assert native.run_native_backup_worker(job) == 1
|
||||
assert json.loads(job.with_name('result.json').read_text('utf-8'))['safe_to_continue'] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize('owner_scope', ['target', 'unrelated'])
|
||||
def test_recovery_entry_scopes_external_ownership_before_launch(tmp_path, monkeypatch, owner_scope):
|
||||
from test_native_backup import project_fixture
|
||||
from cbb_refresh_verify import native_backup as native, owner_probe
|
||||
request, project = project_fixture(tmp_path)
|
||||
job = tmp_path/'request.json'
|
||||
job.write_text(json.dumps(dict(library=str(request.lmc_file), block=request.block_name,
|
||||
project=str(project), mode='repair')))
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: tmp_path/'native.exe')
|
||||
external = project if owner_scope == 'target' else tmp_path/'other/Work/other.prj'
|
||||
monkeypatch.setattr(owner_probe, 'query_owners', lambda names: [dict(pid=99999,
|
||||
name='ExpeditionPCB.exe', command=f'ExpeditionPCB.exe "{external}"', executable='ExpeditionPCB.exe')])
|
||||
def process(*args):
|
||||
if owner_scope == 'target': pytest.fail('external target owner bypassed')
|
||||
return SimpleNamespace(pid=456, poll=lambda: 0, session_pids=lambda: [], close=lambda: None)
|
||||
assert native.native_backup_worker_main(job, process_factory=process) == (1 if owner_scope == 'target' else 0)
|
||||
result = json.loads(job.with_name('result.json').read_text('utf-8'))
|
||||
assert result['safe_to_continue'] is True
|
||||
if owner_scope == 'target': assert 'OWNER_ACTIVE' in result['detail']
|
||||
else: assert result['status'] == 'CLOSED'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('user_timeout', [5, 600])
|
||||
def test_lm_menu_return_without_pcb_uses_dispatch_budget_and_closes_lm(tmp_path, monkeypatch, user_timeout):
|
||||
from test_verify_worker_stability import exercise_worker
|
||||
from cbb_refresh_verify import mentor_adapter as adapter, library_manager_verify as lm
|
||||
job, events, execute = exercise_worker(tmp_path, monkeypatch)
|
||||
job['native_lm_verify'] = True; job['timeout'] = user_timeout
|
||||
(Path(job['pcb']).parent/'pcb.prj').write_text('KEY CentralLibrary "old.lmc"\n')
|
||||
clock = [0.0]; closed = []
|
||||
monkeypatch.setattr(adapter.time, 'monotonic', lambda: clock[0])
|
||||
monkeypatch.setattr(adapter.time, 'sleep', lambda seconds: clock.__setitem__(0, clock[0]+seconds))
|
||||
monkeypatch.setattr(adapter, 'select_reusable_circuit_worker', lambda *a, **k: None)
|
||||
def launch(*args, **kw):
|
||||
assert args[3] <= min(45, user_timeout)
|
||||
return SimpleNamespace(process=SimpleNamespace(pid=123), close=lambda: closed.append(True))
|
||||
monkeypatch.setattr(lm, 'launch_library_verify', launch)
|
||||
with pytest.raises(RuntimeError, match='VERIFY_DISPATCH_FAILED'): execute()
|
||||
assert clock[0] <= min(45, user_timeout)+1 and closed == [True]
|
||||
|
||||
|
||||
def test_inconsistency_preserves_shared_database_scope():
|
||||
from cbb_refresh_verify.icdb_recovery import parse_recovery_request
|
||||
database = r'C:\\library\\SymbolLibs\\__ReusableBlocksSymb'
|
||||
assert parse_recovery_request(inconsistency(database))['scope'] == 'central_library'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('case', ['not_in_use', 'repair_required', 'in_use', 'unconfirmed', 'wrong_target',
|
||||
'no_identity', 'changed_address', 'changed_project', 'shared_address', 'exclusive'])
|
||||
def test_explicit_continue_requires_exact_fresh_native_and_exclusive_proof(tmp_path, monkeypatch, case):
|
||||
from test_native_repair import target_fixture
|
||||
from test_native_occupancy import output
|
||||
from cbb_refresh_verify import pcb_open_consent as consent, native_repair, owner_probe, session_record
|
||||
from cbb_refresh_verify.verify_runner import preflight
|
||||
request, _, target = target_fixture(tmp_path)
|
||||
request.evidence_dir.mkdir()
|
||||
project, pcb = preflight(request)
|
||||
project.write_text('KEY PCBDesignPath "PCB/actual.pcb"\nKEY iCDBDir "database"\n')
|
||||
pcb.write_text('KEY ProjectFile "../'+project.name+'"\n')
|
||||
address = target.database/'cdbsvr/sAddress.adr'; address.parent.mkdir(); address.write_bytes(b'unchanged address')
|
||||
if case == 'shared_address':
|
||||
shared = request.lmc_file.parent/'SymbolLibs/__ReusableBlocksSymb/cdbsvr/sAddress.adr'
|
||||
shared.parent.mkdir(parents=True); shared.write_bytes(b'unknown shared owner')
|
||||
monkeypatch.setattr(owner_probe, 'query_process_snapshot', lambda: [])
|
||||
calls = []
|
||||
def inspect(inspected, root, **kwargs):
|
||||
assert kwargs['inspection_only'] is True
|
||||
calls.append(inspected)
|
||||
record = dict(status='NOT_IN_USE', project=str(target.project), database=str(target.database),
|
||||
native_pid=os.getpid(), native_identity=session_record.process_identity(os.getpid()),
|
||||
inspection_only=True, repair_requested=False, repair_required=case == 'repair_required',
|
||||
native_exit_code=0, release={'native_exited': True},
|
||||
output=output(target.project, target.database, fault=case == 'repair_required'))
|
||||
if case == 'in_use': record.update(status='IN_USE', output=output(target.project, target.database, busy=True))
|
||||
if case == 'unconfirmed': record.update(status='UNCONFIRMED', output='not known')
|
||||
if case == 'wrong_target': record['database'] = str(tmp_path/'foreign')
|
||||
if case == 'no_identity': record.pop('native_identity')
|
||||
if case == 'changed_address': address.write_bytes(b'new address')
|
||||
if case == 'changed_project': project.write_text('KEY iCDBDir "another-db"\n')
|
||||
return record
|
||||
monkeypatch.setattr(native_repair, 'run_native_repair', inspect)
|
||||
job = dict(root=str(request.evidence_dir), lmc_file=str(request.lmc_file),
|
||||
block_name=request.block_name, project=str(project), pcb=str(pcb), operation_id='current-open')
|
||||
held = None
|
||||
if case == 'exclusive':
|
||||
import win32con, win32file
|
||||
held = win32file.CreateFile(str(target.database/'icdb.dat'), win32con.GENERIC_READ,
|
||||
0, None, win32con.OPEN_EXISTING, 0, None)
|
||||
try:
|
||||
if case in ('not_in_use', 'repair_required'):
|
||||
with consent.resource_guard(job, {os.getpid()}): pass
|
||||
assert calls == [target]
|
||||
evidence = json.loads((request.evidence_dir/'pcb-open-resource-proof.json').read_text())
|
||||
assert evidence['operation_id'] == 'current-open'
|
||||
assert evidence['repair_confirmed'] is False
|
||||
else:
|
||||
with pytest.raises((ValueError, OSError)):
|
||||
with consent.resource_guard(job, {os.getpid()}): pass
|
||||
finally:
|
||||
if held is not None: held.Close()
|
||||
if case not in ('shared_address', 'exclusive'):
|
||||
assert calls == [target]
|
||||
if case != 'changed_address': assert address.read_bytes() == b'unchanged address'
|
||||
@@ -559,20 +559,26 @@ def test_queue_close_has_one_deadline_and_reports_pending_scope(tmp_path):
|
||||
thread.join(2)
|
||||
|
||||
|
||||
def test_real_management_transaction_does_not_handoff_between_remove_and_add(management_setup):
|
||||
def test_real_management_transaction_does_not_handoff_between_remove_and_add(management_setup, monkeypatch):
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from test_management import execute
|
||||
from cbb_refresh_verify import operation_queue
|
||||
from cbb_refresh_verify.operation_queue import BatchTaskContext, NativeQueue
|
||||
request, adapter, source = management_setup
|
||||
offset = [0.0]
|
||||
real_clock = time.monotonic
|
||||
monkeypatch.setattr(operation_queue, 'time', SimpleNamespace(monotonic=lambda: real_clock()+offset[0]))
|
||||
original = adapter.remove
|
||||
def slow_remove(*args):
|
||||
result = original(*args)
|
||||
time.sleep(.4)
|
||||
offset[0] += 100 # Native time exceeds admission budget without timing filesystem preparation.
|
||||
time.sleep(.08) # Let the real admission monitor observe the active native phase.
|
||||
return result
|
||||
adapter.remove = slow_remove # Native external boundary; transaction stays real.
|
||||
context = BatchTaskContext(library=str(request.library), targets=('Block',))
|
||||
result = NativeQueue().run(lambda: execute(replace(request, operation='update',
|
||||
source=source, auto_verify=False), adapter), context=context, timeout_seconds=.2)
|
||||
source=source, auto_verify=False), adapter), context=context, timeout_seconds=10)
|
||||
assert result.status == 'UPLOADED'
|
||||
assert adapter.calls[0] == 'remove' and adapter.calls[1][0] == 'add'
|
||||
|
||||
|
||||
@@ -8,6 +8,11 @@ import win32con
|
||||
import win32file
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_real_owner_probe(monkeypatch):
|
||||
monkeypatch.setattr('cbb_refresh_verify.owner_probe.query_owners', lambda names: [])
|
||||
|
||||
|
||||
def project_fixture(tmp_path):
|
||||
from cbb_refresh_verify.verify_runner import VerifyRequest
|
||||
library = tmp_path/'library/library.lmc'; library.parent.mkdir(); library.touch()
|
||||
@@ -21,6 +26,9 @@ def test_backup_entry_can_open_project_without_layout_data(tmp_path, monkeypatch
|
||||
request, project = project_fixture(tmp_path)
|
||||
executable = tmp_path/'iCDBProjectBackup.exe'; executable.touch()
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: executable)
|
||||
from cbb_refresh_verify import session_record
|
||||
monkeypatch.setattr(session_record, 'process_identity', lambda pid:
|
||||
dict(pid=pid, created='fixture', executable='python.exe', owner='fixture'))
|
||||
calls = []
|
||||
monkeypatch.setattr(native.subprocess, 'Popen', lambda command, **kw:
|
||||
(calls.append(command) or SimpleNamespace(pid=123, poll=lambda: None)))
|
||||
@@ -32,7 +40,8 @@ def test_backup_entry_can_open_project_without_layout_data(tmp_path, monkeypatch
|
||||
assert session.library == request.lmc_file and session.block == 'Block'
|
||||
|
||||
|
||||
def test_native_session_closure_does_not_claim_restore_success(tmp_path, monkeypatch):
|
||||
@pytest.mark.parametrize('mode', ['versions', 'repair'])
|
||||
def test_native_session_closure_does_not_claim_restore_success(tmp_path, monkeypatch, mode):
|
||||
from cbb_refresh_verify import native_backup as native
|
||||
from cbb_refresh_verify.library_lock import LibraryLock
|
||||
request, project = project_fixture(tmp_path)
|
||||
@@ -40,7 +49,7 @@ def test_native_session_closure_does_not_claim_restore_success(tmp_path, monkeyp
|
||||
monkeypatch.setattr(native, 'native_backup_executable', lambda *_: executable)
|
||||
job = request.evidence_dir/'request.json'; job.parent.mkdir()
|
||||
job.write_text(json.dumps({'library': str(request.lmc_file), 'block': 'Block',
|
||||
'project': str(project), 'mentor_bin': ''}))
|
||||
'project': str(project), 'mentor_bin': '', 'mode': mode}))
|
||||
calls = []
|
||||
class NativeProcess:
|
||||
pid = 456
|
||||
@@ -58,6 +67,8 @@ def test_native_session_closure_does_not_claim_restore_success(tmp_path, monkeyp
|
||||
result = json.loads(job.with_name('result.json').read_text('utf-8'))
|
||||
assert result['status'] == 'CLOSED' and result['restore_confirmed'] is False
|
||||
assert result['native_exit_code'] == 0 and result['safe_to_continue'] is True
|
||||
assert result['mode'] == mode and result['repair_confirmed'] is False
|
||||
assert result['validation_passed'] is False
|
||||
with LibraryLock(request.lmc_file): pass
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ def exercise_worker(tmp_path, monkeypatch, *, integration_error=None):
|
||||
work = tmp_path/'ReusableBlocksLP/CBB/Work'
|
||||
pcb = work/'PCB/CBB.pcb'
|
||||
pcb.parent.mkdir(parents=True)
|
||||
pcb.touch()
|
||||
pcb.write_text('KEY ProjectFile "../CBB.prj"\n')
|
||||
project = work/'CBB.prj'
|
||||
project.touch()
|
||||
evidence = tmp_path/'evidence'
|
||||
|
||||
Reference in new issue
Block a user