Preserve tested Verify source, authoritative READY checks, bounded bridge retries, diagnostics and failure-only reporting. Include 336-test suite and packaging assets; exclude live Mentor samples, field logs, backups and distributions.
971 lines
48 KiB
Python
971 lines
48 KiB
Python
"""Serial Verify supervisor.
|
||
|
||
Each CBB gets one isolated, bounded automation worker on a private Windows
|
||
desktop. The worker reproduces the observed refresh flow: invoke the CBB's
|
||
native Library Manager Verify action, bind the resulting ReusableCircuit PCB,
|
||
verify the document identity, run Project Integration, save the Reusable Block
|
||
cell and close. Optionally the official
|
||
``ReuseBlockImporter.exe -chkContents`` check runs first when a LibSpec and a
|
||
DMS login configuration are supplied.
|
||
|
||
Nothing here injects mouse or keyboard input or edits the central library. The
|
||
native menu invocation happens on the worker's private desktop.
|
||
"""
|
||
from dataclasses import asdict, dataclass, field, replace
|
||
from pathlib import Path
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
import struct
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from uuid import uuid4
|
||
|
||
from .network_path import absolute_preserving_drive
|
||
from .runtime_timing import RunTimings
|
||
from .defaults import DEFAULT_TIMEOUT_SECONDS
|
||
|
||
SUCCESS_TEXT = re.compile(r'content checking completed successfully', re.IGNORECASE)
|
||
ERROR_TEXT = re.compile(r'failed|error|cannot connect|bad data', re.IGNORECASE)
|
||
INVALID_NAME_CHARS = '\\/:*?"<>|'
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class VerifyRequest:
|
||
lmc_file: Path
|
||
block_name: str
|
||
evidence_dir: Path
|
||
libspec: str = ''
|
||
dms_login_config: str = ''
|
||
mentor_bin: str = ''
|
||
# PCB startup on EE7.9 includes iCDB recovery and COM registration. The
|
||
# previous 30-second UI default was shorter than the observed startup
|
||
# window on this machine and caused a false timeout before document attach.
|
||
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
|
||
logging_context: object | None = None
|
||
# The production default stays on an isolated desktop. The explicit
|
||
# interactive mode is reserved for a real-input compatibility test on
|
||
# machines where EE7.9 cannot dispatch the tree menu on a private desktop.
|
||
interactive_input: bool = False
|
||
# The native Library Manager menu route is opt-in until a machine's menu
|
||
# dispatch has been proven. The flag is serialized to the worker so a
|
||
# Win10 compatibility run can select it explicitly.
|
||
native_lm_verify: bool = False
|
||
# A reserved iCDB range is a host-wide prerequisite failure. Do not
|
||
# launch a PCB that is guaranteed to time out; batch orchestration will
|
||
# record this item and continue with the next one.
|
||
strict_port_check: bool = True
|
||
# A mapped drive can be invisible to an elevated/private worker on Win10.
|
||
# Preserve the parent session's mapping identity so the worker can restore
|
||
# it before launching Library Manager/PCB.
|
||
mapped_drive: str = ''
|
||
mapped_remote_root: str = ''
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class VerifyResult:
|
||
status: str
|
||
stderr: str = ''
|
||
stdout: str = ''
|
||
evidence_path: Path | None = None
|
||
safe_to_continue: bool = True
|
||
library_flag: str = 'UNKNOWN'
|
||
library_flag_detail: str = ''
|
||
report_path: Path | None = None
|
||
content_check_exit_code: int | None = None
|
||
error_category: str = ""
|
||
error_stage: str = ""
|
||
bridge_attempts: tuple[dict, ...] = ()
|
||
timings_seconds: dict[str, float] = field(default_factory=dict)
|
||
incomplete_timing_phases: tuple[str, ...] = ()
|
||
|
||
|
||
def resolve_importer(mentor_bin: str | None = None) -> Path:
|
||
"""Configured bin first, then the EE7.9.5 environment defaults."""
|
||
candidates = []
|
||
if mentor_bin:
|
||
configured = Path(mentor_bin)
|
||
if configured.is_file() and configured.name.casefold() == 'reuseblockimporter.exe':
|
||
candidates.append(configured.parent)
|
||
else:
|
||
candidates.append(configured)
|
||
if configured.name.casefold() == 'sdd_home':
|
||
candidates.append(configured / 'lm' / 'win32' / 'bin')
|
||
from .mentor_paths import discover_sdd_home
|
||
discovered = discover_sdd_home(mentor_bin)
|
||
if discovered is not None:
|
||
candidates.append(discovered / 'lm' / 'win32' / 'bin')
|
||
for variable in ('SDD_LM_BIN', 'SDD_HOME'):
|
||
value = os.environ.get(variable)
|
||
if not value:
|
||
continue
|
||
root = Path(value)
|
||
candidates.append(root if root.name.lower() == 'bin' else root/'lm'/'win32'/'bin')
|
||
candidates.append(Path(r'D:\Mentor\7.9.5EE\SDD_HOME\lm\win32\bin'))
|
||
for candidate in candidates:
|
||
executable = candidate/'ReuseBlockImporter.exe'
|
||
if executable.is_file():
|
||
return executable
|
||
raise ValueError('ReuseBlockImporter.exe 未找到;请设置 SDD_LM_BIN')
|
||
|
||
|
||
def build_content_check_command(request, executable) -> tuple[str, ...]:
|
||
"""The documented Logical-Physical contents check invocation."""
|
||
return (str(executable), '-lmcFile', str(request.lmc_file), '-blockName', request.block_name,
|
||
'-logicalType', 'LP', '-chkContents', '-libSpec', str(request.libspec),
|
||
'-dmsLoginConfig', str(request.dms_login_config))
|
||
|
||
|
||
def contents_report_path(lmc_file, block_name) -> Path:
|
||
safe_name = re.sub(f'[{re.escape(INVALID_NAME_CHARS)}]', '_', str(block_name))
|
||
return Path(lmc_file).parent/'LogFiles'/f'restorerb_RULP_{safe_name}_contents.log'
|
||
|
||
|
||
def classify_output(exit_code: int | None, output: str, *, timed_out: bool = False) -> str:
|
||
"""Never invent success: PASS needs the success text and no error marker."""
|
||
if timed_out:
|
||
return 'TIMEOUT'
|
||
if SUCCESS_TEXT.search(output) and not ERROR_TEXT.search(output):
|
||
return 'PASS'
|
||
if ERROR_TEXT.search(output):
|
||
return 'FAIL'
|
||
if exit_code not in (None, 0):
|
||
return 'FAIL'
|
||
return 'UNKNOWN'
|
||
|
||
|
||
def preflight(request):
|
||
"""Validate the library entry and its ReusableCircuit sources before launch."""
|
||
library = absolute_preserving_drive(request.lmc_file)
|
||
name = request.block_name
|
||
if not name or name in ('.', '..') or any(character in name for character in INVALID_NAME_CHARS):
|
||
raise ValueError('CBB 名称非法')
|
||
if not library.is_file():
|
||
raise ValueError(f'中心库不存在: {library}')
|
||
work = library.parent/'ReusableBlocksLP'/name/'Work'
|
||
projects = list(work.glob('*.prj'))
|
||
if len(projects) != 1:
|
||
raise ValueError(f'{name}: 缺少或无法唯一确定 Work 下的工程 PRJ')
|
||
project = absolute_preserving_drive(projects[0])
|
||
content = project.read_text(encoding='utf-8', errors='replace')
|
||
refs = re.findall(r'^KEY PCBDesignPath "([^"]+)"', content, re.M)
|
||
if not refs:
|
||
raise ValueError(f'{name}: 工程没有 PCBDesignPath')
|
||
pcb = absolute_preserving_drive(project.parent/refs[0])
|
||
if not pcb.is_relative_to(absolute_preserving_drive(work)):
|
||
raise ValueError(f'{name}: PCB 引用越出当前 CBB,停止操作: {pcb}')
|
||
if not pcb.is_file() or not (pcb.parent/'Layout'/'LayoutDB.lyt').is_file():
|
||
raise ValueError(f'{name}: 缺少 PCB 或布局数据库: {pcb}')
|
||
return project, pcb
|
||
|
||
|
||
def preflight_skip_reason(lmc_file, block_name):
|
||
"""Return a user-facing skip reason for an import-incomplete CBB.
|
||
|
||
This is intentionally narrower than ``preflight``: only local source
|
||
structure problems are excluded before a batch starts. Runtime failures
|
||
such as ports, DMS, COM, or Project Integration remain normal per-item
|
||
failures and are still attempted when the user queues them.
|
||
"""
|
||
library = absolute_preserving_drive(lmc_file)
|
||
if not library.is_file():
|
||
return ''
|
||
work = library.parent / 'ReusableBlocksLP' / str(block_name) / 'Work'
|
||
if not work.is_dir():
|
||
return ''
|
||
request = VerifyRequest(library, str(block_name), library.parent / '.cbb-preflight-evidence')
|
||
try:
|
||
preflight(request)
|
||
except Exception as error:
|
||
detail = str(error)
|
||
if any(marker in detail for marker in ('缺少或无法唯一确定 Work 下的工程 PRJ',
|
||
'工程没有 PCBDesignPath',
|
||
'缺少 PCB 或布局数据库')):
|
||
return detail
|
||
return ''
|
||
|
||
|
||
def _run_process(command, timeout_seconds, cwd):
|
||
"""Run the importer, draining both pipes while it works (no pipe deadlock)."""
|
||
process = subprocess.Popen(command, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
||
try:
|
||
stdout, stderr = process.communicate(timeout=timeout_seconds)
|
||
return process.returncode, stdout.decode('utf-8', errors='replace'), stderr.decode('utf-8', errors='replace'), False
|
||
except subprocess.TimeoutExpired:
|
||
# A timed-out importer may have spawned a child that still owns the
|
||
# pipes. Bound both tree termination and the final drain so a hung
|
||
# helper cannot turn a 30-second item timeout into an unbounded wait.
|
||
try:
|
||
subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'],
|
||
capture_output=True, timeout=5,
|
||
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
||
except (OSError, subprocess.TimeoutExpired):
|
||
pass
|
||
try:
|
||
process.kill()
|
||
except OSError:
|
||
pass
|
||
try:
|
||
stdout, stderr = process.communicate(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
stdout, stderr = b'', b''
|
||
return None, stdout.decode('utf-8', errors='replace'), stderr.decode('utf-8', errors='replace'), True
|
||
|
||
|
||
def _stop_worker(process, *, timeout=5):
|
||
"""Best-effort worker tree termination that never masks the item result."""
|
||
try:
|
||
process.kill()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
process.wait(timeout=timeout)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _normalize_bridge_result(probe):
|
||
from .status_bridge import BridgeResult
|
||
if not isinstance(probe, BridgeResult):
|
||
detail = str(probe[1]) if isinstance(probe, tuple) and len(probe) > 1 else str(probe)
|
||
return BridgeResult('MALFORMED', '', None,
|
||
'BRIDGE_RESULT_INVALID: 非权威桥接器结果;' + detail)
|
||
if probe.status in {'READY', 'PENDING'} and (
|
||
probe.source != 'IsVerifiedReuseBlock'
|
||
or probe.raw_status != (1 if probe.status == 'READY' else 0)):
|
||
return BridgeResult('MALFORMED', probe.source, probe.raw_status,
|
||
'BRIDGE_RESULT_INVALID: status/source/raw_status 不一致')
|
||
return probe
|
||
|
||
|
||
def read_library_flag_with_retry(reader, library, block_name, *, attempts=3,
|
||
delays=(0.0, 3.0, 5.0), sleep=None, on_attempt=None):
|
||
"""Bounded read-only bridge retries; never repeat Verify or promote PENDING."""
|
||
from .status_bridge import BridgeResult
|
||
sleep = sleep or time.sleep
|
||
last = BridgeResult('UNAVAILABLE', '', None, '状态桥没有返回结果')
|
||
for index in range(max(1, int(attempts))):
|
||
if index > 0:
|
||
delay = delays[min(index, len(delays)-1)] if delays else 0
|
||
sleep(max(0.0, float(delay)))
|
||
started = time.monotonic()
|
||
try:
|
||
last = _normalize_bridge_result(reader(library, block_name))
|
||
except Exception as error:
|
||
last = BridgeResult('UNAVAILABLE', '', None, 'BRIDGE_UNAVAILABLE: ' + str(error))
|
||
if on_attempt:
|
||
on_attempt({'attempt': index + 1, 'status': last.status,
|
||
'source': last.source, 'raw_status': last.raw_status,
|
||
'detail': last.detail,
|
||
'query_seconds': round(time.monotonic()-started, 6)})
|
||
if last.status in {'READY', 'MALFORMED'}:
|
||
return last
|
||
return last
|
||
|
||
|
||
def classify_failure_category(detail):
|
||
text = str(detail or '')
|
||
lower = text.casefold()
|
||
if 'tf_floatinglangbar_wndtitle' in lower or 'cicerouiwndframe' in lower:
|
||
return 'TOOL_FALSE_DIALOG'
|
||
if 'could not load job preferences db' in lower or 'unable to create local cell library' in lower:
|
||
return 'MENTOR_LOCAL_CELL_LIBRARY'
|
||
if ('database_load_failed' in lower
|
||
or 'problem encountered while opening one of the database files' in lower
|
||
or 'database load is being terminated' in lower):
|
||
return 'MENTOR_DATABASE_LOAD_FAILED'
|
||
if 'project integration error' in lower or 'project_integration_failed' in lower:
|
||
return 'MENTOR_PROJECT_INTEGRATION'
|
||
if 'save_reusable_block_failed' in lower or 'error saving the reusable block' in lower:
|
||
return 'MENTOR_SAVE_FAILED'
|
||
if 'design_server_failed' in lower or 'could not run server' in lower or 'could not bring it alive' in lower:
|
||
return 'ENVIRONMENT_DESIGN_SERVER'
|
||
if 'bridge_unavailable' in lower or '权威状态读取失败' in text:
|
||
return 'BRIDGE_UNAVAILABLE'
|
||
if 'bridge_result_invalid' in lower:
|
||
return 'BRIDGE_RESULT_INVALID'
|
||
if 'pending' in lower and 'isverifiedreuseblock' in lower:
|
||
return 'MENTOR_VERIFY_NOT_COMMITTED'
|
||
if 'cert' in lower and ('incomplete' in lower or '成果' in text):
|
||
return 'TOOL_CERT_DIAGNOSTIC'
|
||
if 'unknown_dialog' in lower or 'dialog' in lower:
|
||
return 'TOOL_DIALOG_SUPERVISOR'
|
||
if 'timeout' in lower or '超时' in text:
|
||
return 'TOOL_TIMEOUT'
|
||
return 'UNCLASSIFIED'
|
||
|
||
|
||
def read_library_flag_if_available(library, block_name):
|
||
"""Read the authoritative Mentor Verify state through the x86 bridge.
|
||
|
||
Unit tests replace this external boundary; production never accepts Cert
|
||
files or legacy tree readers as a substitute for this gold standard.
|
||
"""
|
||
from .status_bridge import StatusBridgeClient
|
||
return StatusBridgeClient(
|
||
env={'CBB_STATUS_BRIDGE_ALLOW_PRIVATE': '1'}
|
||
).query(Path(library), block_name)
|
||
|
||
|
||
def apply_post_verify_status(result, bridge_result):
|
||
"""Only the authoritative bridge can confirm READY, never a Cert/tuple/icon."""
|
||
if result.status != 'PASS':
|
||
return result # a pre-existing READY cannot erase a real execution error
|
||
bridge_result = _normalize_bridge_result(bridge_result)
|
||
if bridge_result.status == 'READY':
|
||
return replace(result, library_flag='GREEN', library_flag_detail=(
|
||
f'READY ({bridge_result.source}, raw={bridge_result.raw_status})'))
|
||
detail = bridge_result.detail or (
|
||
f'执行已完成,但权威状态仍为 {bridge_result.status} '
|
||
f'({bridge_result.source}, raw={bridge_result.raw_status})')
|
||
pending = bridge_result.status == 'PENDING'
|
||
return replace(result, status='FAIL',
|
||
stderr='\n'.join(part for part in (result.stderr, detail) if part),
|
||
library_flag='NOT_GREEN' if pending else 'UNKNOWN', library_flag_detail=detail,
|
||
error_category=('MENTOR_VERIFY_NOT_COMMITTED' if pending else
|
||
'BRIDGE_RESULT_INVALID' if bridge_result.status == 'MALFORMED' else 'BRIDGE_UNAVAILABLE'),
|
||
error_stage='library_flag', safe_to_continue=True)
|
||
|
||
|
||
def worker_command(request_path):
|
||
if getattr(sys, 'frozen', False):
|
||
return [sys.executable, '--verify-worker', str(request_path)]
|
||
return [sys.executable, '-m', 'cbb_refresh_verify', '--verify-worker', str(request_path)]
|
||
|
||
|
||
def phase_for_event(event, current=None):
|
||
from .dialog_models import DialogPhase
|
||
|
||
stage = str(event.get('stage', ''))
|
||
mapping = {
|
||
'lm_launch': DialogPhase.PCB_STARTING,
|
||
'lm_started': DialogPhase.PCB_STARTING,
|
||
'lm_ready': DialogPhase.PCB_STARTING,
|
||
'lm_session_ready': DialogPhase.PCB_STARTING,
|
||
'verify_invoke': DialogPhase.PCB_STARTING,
|
||
'verify_invoke_returned': DialogPhase.PCB_STARTING,
|
||
'pcb_launch': DialogPhase.PCB_STARTING,
|
||
'pcb_started': DialogPhase.PCB_STARTING,
|
||
'com_wait': DialogPhase.DOCUMENT_ATTACH,
|
||
'identity_wait': DialogPhase.DOCUMENT_ATTACH,
|
||
'identity_verified': DialogPhase.IDENTITY_CHECK,
|
||
'license_key_requested': DialogPhase.LICENSE_ACQUISITION,
|
||
'license_token_received': DialogPhase.LICENSE_ACQUISITION,
|
||
'document_validated': DialogPhase.VALIDATE,
|
||
'integration_status': DialogPhase.PROJECT_INTEGRATION,
|
||
'integration_action': DialogPhase.PROJECT_INTEGRATION,
|
||
'integration_error': DialogPhase.PROJECT_INTEGRATION,
|
||
'save_start': DialogPhase.SAVE,
|
||
'save_helper': DialogPhase.SAVE,
|
||
'save_progress': DialogPhase.SAVE,
|
||
'save_quit_recovered': DialogPhase.CLOSE,
|
||
'saved_and_closed': DialogPhase.CLOSE,
|
||
'pcb_exit_recovered': DialogPhase.CLOSE,
|
||
'complete': DialogPhase.CLOSE,
|
||
}
|
||
return mapping.get(stage, current or DialogPhase.PCB_STARTING)
|
||
|
||
|
||
def classify_verify_dispatch(*, verify_invoked, invoke_returned, target_pid, observation):
|
||
"""Classify a returned Verify command that never started its PCB.
|
||
|
||
UI Automation can report a successful ``InvokePattern`` call even when
|
||
Library Manager only closed the popup and did not dispatch its native MFC
|
||
command. A returned menu call is therefore not a success signal. Keep
|
||
this predicate small and evidence based so the supervisor can distinguish
|
||
a dispatch failure from a generic inactivity timeout without changing the
|
||
underlying automation route.
|
||
"""
|
||
if not (verify_invoked and invoke_returned) or target_pid is not None:
|
||
return None
|
||
return (
|
||
'FAIL',
|
||
'VERIFY_DISPATCH_FAILED: LMC Verify 菜单调用已返回,但未启动目标 ReusableCircuit PCB',
|
||
)
|
||
|
||
|
||
def _write_failure_windows(root, snapshots):
|
||
if not snapshots:
|
||
return
|
||
records = [dict(time=time.strftime('%Y-%m-%d %H:%M:%S'), **snapshot.to_record())
|
||
for snapshot in snapshots]
|
||
(root/'window_snapshots.jsonl').write_text(
|
||
''.join(json.dumps(record, ensure_ascii=False) + '\n' for record in records), encoding='utf-8')
|
||
|
||
|
||
def _write_process_tree(root, process, target_pid):
|
||
record = {
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
'worker_pid': getattr(process, 'pid', None),
|
||
'companion_pids': list(getattr(process, 'companion_pids', ())),
|
||
'pcb_pid': target_pid,
|
||
}
|
||
temporary = root/'process_tree.json.tmp'
|
||
temporary.write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
temporary.replace(root/'process_tree.json')
|
||
|
||
|
||
def runtime_fingerprint(mentor_bin=''):
|
||
"""Return host/runtime facts needed to compare Win10 and Win11 runs."""
|
||
result = {
|
||
'platform': platform.platform(),
|
||
'system': platform.system(),
|
||
'release': platform.release(),
|
||
'version': platform.version(),
|
||
'machine': platform.machine(),
|
||
'python_version': platform.python_version(),
|
||
'python_bitness': struct.calcsize('P') * 8,
|
||
'mentor_bin': str(mentor_bin or ''),
|
||
'uia_backend': 'pywinauto.win32',
|
||
}
|
||
if os.name == 'nt':
|
||
try:
|
||
import winreg
|
||
with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
|
||
r'SOFTWARE\Microsoft\Windows NT\CurrentVersion') as key:
|
||
for name, output_name in (
|
||
('ProductName', 'windows_product_name'),
|
||
('CurrentBuild', 'windows_current_build'),
|
||
('UBR', 'windows_ubr'),
|
||
):
|
||
try:
|
||
result[output_name] = winreg.QueryValueEx(key, name)[0]
|
||
except OSError:
|
||
pass
|
||
except (ImportError, OSError):
|
||
pass
|
||
result['windows_build_family'] = windows_build_family(
|
||
result.get('windows_current_build'))
|
||
from .mentor_paths import describe_installation, discover_sdd_home
|
||
result['mentor_installation'] = describe_installation(
|
||
discover_sdd_home(mentor_bin))
|
||
return result
|
||
|
||
|
||
def windows_build_family(build):
|
||
"""Classify Windows from the build number, not ProductName.
|
||
|
||
Windows 11 commonly keeps ``ProductName=Windows 10 Pro`` for
|
||
compatibility. The first public Windows 11 build is 22000, so the build
|
||
number is the stable discriminator for compatibility reports.
|
||
"""
|
||
try:
|
||
number = int(str(build or '').split('.', 1)[0])
|
||
except (TypeError, ValueError):
|
||
return 'unknown'
|
||
if number >= 22000:
|
||
return 'Windows 11'
|
||
if number >= 10240:
|
||
return 'Windows 10'
|
||
return 'unknown'
|
||
|
||
|
||
def native_lm_verify_enabled():
|
||
"""Read the explicit compatibility switch for the native LM route."""
|
||
return os.environ.get('CBB_NATIVE_LM_VERIFY', '').strip().casefold() in {
|
||
'1', 'true', 'yes', 'on',
|
||
}
|
||
|
||
|
||
def _cleanup_session_processes(root, *, lm_pid, target_pid, pcb, pcb_baseline,
|
||
progress=None):
|
||
"""Reap Mentor processes after the worker can no longer run its ``finally``.
|
||
|
||
The outer supervisor owns the worker lifetime. A timeout or an unknown
|
||
dialog may therefore terminate the worker while its child LMC/PCB processes
|
||
remain alive. Delegate the same target-filtered cleanup used by the worker
|
||
and persist its events before the batch advances.
|
||
"""
|
||
from .mentor_adapter import cleanup_verify_processes
|
||
|
||
event_path = Path(root) / 'events.jsonl'
|
||
|
||
def emit(**event):
|
||
event = {
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
'source': 'supervisor_cleanup',
|
||
**event,
|
||
}
|
||
try:
|
||
with event_path.open('a', encoding='utf-8') as stream:
|
||
stream.write(json.dumps(event, ensure_ascii=False) + '\n')
|
||
except OSError:
|
||
pass
|
||
if progress:
|
||
progress(event)
|
||
|
||
cleanup_verify_processes(
|
||
lm_pid,
|
||
target_pid,
|
||
None,
|
||
expected_pcb=pcb,
|
||
baseline_pids=pcb_baseline,
|
||
emit=emit,
|
||
)
|
||
|
||
|
||
def run_verify(request, *, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||
dialog_supervisor_factory=None):
|
||
timing = RunTimings()
|
||
result = _run_verify(request, timing=timing, progress=progress, flag_reader=flag_reader,
|
||
process_runner=process_runner, port_check=port_check,
|
||
dialog_supervisor_factory=dialog_supervisor_factory)
|
||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'} and result.evidence_path:
|
||
job_path = result.evidence_path.with_name('request.json')
|
||
if job_path.is_file():
|
||
timing.begin('failure_diagnostics')
|
||
try:
|
||
from .verify_diagnostics import capture_failure_logs_with_timeout
|
||
capture_failure_logs_with_timeout(job_path)
|
||
except Exception:
|
||
# Never replace the underlying Mentor/bridge/timeout failure.
|
||
pass
|
||
finally:
|
||
timing.end('failure_diagnostics')
|
||
durations, unfinished = timing.snapshot()
|
||
result = replace(result, timings_seconds=durations, incomplete_timing_phases=unfinished,
|
||
error_category=(result.error_category or classify_failure_category(result.stderr)
|
||
if result.status not in {"PASS", "SKIPPED"} else ""),
|
||
error_stage=(result.error_stage or (unfinished[-1] if unfinished else 'execution')
|
||
if result.status not in {"PASS", "SKIPPED"} else ""))
|
||
if result.evidence_path:
|
||
result.evidence_path.write_text(json.dumps(
|
||
{'block_name': request.block_name, **asdict(result)},
|
||
ensure_ascii=False, default=str, indent=2), encoding='utf-8')
|
||
result.evidence_path.with_name('performance.json').write_text(json.dumps({
|
||
'block_name': request.block_name, 'status': result.status,
|
||
'unit': 'seconds', 'timings_seconds': durations,
|
||
'incomplete_timing_phases': unfinished,
|
||
'clock': 'monotonic; save helper subphases: VBScript Timer (midnight adjusted)',
|
||
'notes': 'phase totals overlap action/call subphases; skipped phases are absent',
|
||
}, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
if progress:
|
||
progress({'stage': 'performance', 'block': request.block_name,
|
||
'timings_seconds': durations, 'incomplete_timing_phases': unfinished,
|
||
'detail': ' '.join(f'{key}={value:.3f}s' for key, value in durations.items())})
|
||
if request.logging_context:
|
||
request.logging_context.logger.info('operation=cbb_verify block=%s status=%s flag=%s evidence=%s error=%s timings_seconds=%s',
|
||
request.block_name, result.status, result.library_flag, result.evidence_path, result.stderr, durations)
|
||
return result
|
||
|
||
|
||
def _run_verify(request, *, timing, progress=None, flag_reader=None, process_runner=None, port_check=None,
|
||
dialog_supervisor_factory=None):
|
||
"""Refresh one CBB and record its evidence.
|
||
|
||
``flag_reader`` is a read-only probe returning an authoritative ``BridgeResult``.
|
||
After successful execution, only IsVerifiedReuseBlock READY/raw=1 confirms
|
||
PASS/GREEN; Cert or icon observations never substitute for it. ``port_check`` is injectable so tests do
|
||
not depend on the host's port reservations.
|
||
"""
|
||
root = Path(request.evidence_dir).resolve()/('verify-' + uuid4().hex[:12])
|
||
root.mkdir(parents=True, exist_ok=False)
|
||
fingerprint = runtime_fingerprint(request.mentor_bin)
|
||
(root/'runtime.json').write_text(
|
||
json.dumps(fingerprint, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
if progress:
|
||
progress({'stage': 'runtime_fingerprint', 'block': request.block_name,
|
||
**fingerprint})
|
||
result_path = root/'result.json'
|
||
runner = process_runner or _run_process
|
||
|
||
def finish(result):
|
||
# Backups are transient protection for the active worker. Keep the
|
||
# small, actionable evidence files but reclaim the copied CBB tree so
|
||
# large libraries do not accumulate tens of megabytes per item.
|
||
try:
|
||
from .mentor_adapter import cleanup_evidence_backup
|
||
cleanup_evidence_backup(root)
|
||
except Exception:
|
||
# Cleanup must never hide or change the verification result.
|
||
pass
|
||
return replace(result, evidence_path=result_path)
|
||
|
||
timing.begin('preflight')
|
||
try:
|
||
project, pcb = preflight(request)
|
||
except Exception as error:
|
||
return finish(VerifyResult('BLOCKED', stderr=str(error)))
|
||
finally:
|
||
timing.end('preflight')
|
||
|
||
content_check_enabled = bool(str(request.libspec).strip() and str(request.dms_login_config).strip())
|
||
if content_check_enabled:
|
||
try:
|
||
executable = resolve_importer(request.mentor_bin)
|
||
except Exception as error:
|
||
return finish(VerifyResult('BLOCKED', stderr=str(error)))
|
||
command = build_content_check_command(request, executable)
|
||
if progress:
|
||
progress({'stage': 'content_check_start', 'block': request.block_name,
|
||
'detail': ' '.join(command)})
|
||
timing.begin('content_check')
|
||
try:
|
||
exit_code, stdout, stderr, timed_out = runner(command, request.timeout_seconds, Path(request.lmc_file).parent)
|
||
finally:
|
||
timing.end('content_check')
|
||
combined = '\n'.join(part for part in (stdout, stderr) if part)
|
||
report = contents_report_path(request.lmc_file, request.block_name)
|
||
try:
|
||
report.parent.mkdir(parents=True, exist_ok=True)
|
||
report.write_text(combined, encoding='utf-8')
|
||
except Exception as error:
|
||
stderr = f'{stderr}\n无法写入内容检查报告 {report}: {error}'
|
||
status = classify_output(exit_code, combined, timed_out=timed_out)
|
||
if status != 'PASS':
|
||
# A contents failure or an unusable DMS must block before touching the PCB.
|
||
return finish(VerifyResult(status if status in ('FAIL', 'TIMEOUT') else 'BLOCKED',
|
||
stderr=combined.strip() or f'内容检查结果未知(退出码 {exit_code})',
|
||
stdout=stdout, report_path=report, content_check_exit_code=exit_code))
|
||
if progress:
|
||
progress({'stage': 'content_check_done', 'block': request.block_name, 'status': 'PASS'})
|
||
|
||
# The design server needs a port in 10000-10100; a system reservation there
|
||
# blocks every item, so fail fast with the fix instead of timing out.
|
||
from .mentor_adapter import icdb_ports_bindable
|
||
check_ports = port_check or icdb_ports_bindable
|
||
ports_ok, ports_detail = check_ports()
|
||
if not ports_ok:
|
||
if progress:
|
||
progress({'stage': 'port_warning', 'block': request.block_name, 'detail': ports_detail})
|
||
if request.strict_port_check:
|
||
return finish(VerifyResult('BLOCKED', stderr=ports_detail))
|
||
|
||
job = {'lmc_file': str(absolute_preserving_drive(request.lmc_file)), 'block_name': request.block_name,
|
||
'project': str(project), 'pcb': str(pcb), 'root': str(root), 'timeout': request.timeout_seconds,
|
||
'mentor_bin': str(request.mentor_bin or ''),
|
||
'mapped_drive': str(request.mapped_drive or ''),
|
||
'mapped_remote_root': str(request.mapped_remote_root or ''),
|
||
'interactive_input': bool(request.interactive_input),
|
||
'native_lm_verify': bool(request.native_lm_verify),
|
||
'desktop': 'CbbVerify_' + uuid4().hex[:12],
|
||
'start_gate': str(root/'start-worker.signal')}
|
||
job_path = root/'request.json'
|
||
job_path.write_text(json.dumps(job, ensure_ascii=False), encoding='utf-8')
|
||
events = root/'events.jsonl'
|
||
offset = 0
|
||
target_pid = None
|
||
lm_pid = None
|
||
verify_invoked = False
|
||
invoke_returned = False
|
||
pcb_baseline = set()
|
||
last_observation = 0.0
|
||
last_post_verify_observation = {}
|
||
from .mentor_adapter import ActivityDeadline, pcb_startup_timeout_seconds
|
||
from .desktop_worker import DesktopWorker
|
||
from .dialog_supervisor import AgentDialogBackend, DialogSupervisor, Win32DialogBackend
|
||
from .dialog_models import DialogPhase
|
||
import win32event, win32con
|
||
mutex = win32event.CreateMutex(None, False, r'Local\CbbRefreshVerify795')
|
||
lock_status = win32event.WaitForSingleObject(mutex, 0)
|
||
if lock_status not in (win32con.WAIT_OBJECT_0, win32con.WAIT_ABANDONED):
|
||
mutex.Close()
|
||
return finish(VerifyResult('BLOCKED', stderr='已有 CBB 校验任务运行,请等待结束。', safe_to_continue=False))
|
||
try:
|
||
timing.begin('worker_setup')
|
||
with (root/'worker.log').open('w', encoding='utf-8') as log:
|
||
if request.interactive_input:
|
||
process = DesktopWorker(
|
||
worker_command(job_path), None, interactive_input=True)
|
||
else:
|
||
process = DesktopWorker(worker_command(job_path), job['desktop'])
|
||
deadline = ActivityDeadline(request.timeout_seconds)
|
||
last_activity_key = None
|
||
phase = DialogPhase.PCB_STARTING
|
||
dialog_agent = None
|
||
latest_snapshots = ()
|
||
if dialog_supervisor_factory is None:
|
||
if hasattr(process, 'start_companion'):
|
||
from .dialog_agent import DialogAgentClient
|
||
from .dialog_rules import default_rules_path, load_rules
|
||
dialog_agent = DialogAgentClient.start(process, root/'dialog-agent')
|
||
dialog_supervisor = DialogSupervisor(
|
||
AgentDialogBackend(dialog_agent),
|
||
rules=load_rules(default_rules_path()),
|
||
)
|
||
else:
|
||
dialog_supervisor = DialogSupervisor(Win32DialogBackend(
|
||
lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
|
||
desktop=process.desktop,
|
||
))
|
||
else:
|
||
dialog_supervisor = dialog_supervisor_factory(
|
||
desktop=process.desktop,
|
||
pids=lambda: ({value for value in (target_pid, lm_pid) if value is not None}),
|
||
)
|
||
Path(job['start_gate']).touch()
|
||
while process.poll() is None:
|
||
if events.exists():
|
||
with events.open(encoding='utf-8') as stream:
|
||
stream.seek(offset)
|
||
while line := stream.readline():
|
||
try:
|
||
event = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
break
|
||
timing.record(event)
|
||
if event.get('pcb_pid'): target_pid = event['pcb_pid']
|
||
if event.get('lm_pid'): lm_pid = event['lm_pid']
|
||
stage = event.get('stage')
|
||
if stage == 'verify_invoke': verify_invoked = True
|
||
if stage == 'verify_invoke_returned': invoke_returned = True
|
||
if stage == 'pcb_baseline': pcb_baseline = set(event.get('pids', ()))
|
||
phase = phase_for_event(event, phase)
|
||
activity_key = (stage, json.dumps(event.get('statuses'), sort_keys=True),
|
||
json.dumps(event.get('markers'), sort_keys=True),
|
||
event.get('action'), event.get('detail'))
|
||
# Repeated integration_status snapshots are normal
|
||
# polling, not progress. Only a changed state or
|
||
# a new stage/marker extends the inactivity window.
|
||
if activity_key != last_activity_key:
|
||
if stage in ('lm_started', 'lm_ready', 'verify_invoke',
|
||
'verify_invoke_returned', 'lm_session_ready',
|
||
'pcb_started', 'pcb_worker_started'):
|
||
deadline.timeout_seconds = pcb_startup_timeout_seconds(
|
||
request.timeout_seconds)
|
||
elif stage == 'identity_verified':
|
||
deadline.timeout_seconds = max(
|
||
0.1, float(request.timeout_seconds))
|
||
deadline.touch()
|
||
last_activity_key = activity_key
|
||
if progress: progress(event)
|
||
offset = stream.tell()
|
||
if verify_invoked and target_pid is None and time.monotonic() - last_observation >= .5:
|
||
from .post_verify_diagnostics import observe_post_verify
|
||
last_observation = time.monotonic()
|
||
try:
|
||
observation = observe_post_verify(process.desktop, pcb, pcb_baseline, lm_pid)
|
||
except Exception as error:
|
||
observation = {'error': str(error)}
|
||
last_post_verify_observation = observation
|
||
if dialog_agent is not None and target_pid is None:
|
||
# The Library Manager Verify route starts its own window
|
||
# before it launches Expedition PCB. Do not feed that
|
||
# large MFC main window to the PCB dialog probe; it can
|
||
# exhaust the UIA observation deadline before Verify has
|
||
# even produced a target process.
|
||
from .dialog_supervisor import DialogPollResult
|
||
dialog_result = DialogPollResult(activity=False)
|
||
elif dialog_agent is not None:
|
||
dialog_result = dialog_supervisor.poll(phase)
|
||
elif dialog_supervisor_factory is not None:
|
||
try:
|
||
dialog_result = dialog_supervisor.poll(phase)
|
||
except TypeError:
|
||
# Compatibility for injected legacy test supervisors only.
|
||
dialog_result = dialog_supervisor.poll()
|
||
else:
|
||
dialog_result = dialog_supervisor.poll()
|
||
# Normal progress stays in memory. Only actions/errors and the
|
||
# last failure snapshots are persisted, not every elapsed tick.
|
||
latest_snapshots = dialog_result.observed_snapshots
|
||
if dialog_result.snapshot is not None and (
|
||
dialog_result.action_sent or dialog_result.error_code):
|
||
snapshot_record = (dialog_result.snapshot.to_record()
|
||
if hasattr(dialog_result.snapshot, 'to_record')
|
||
else dialog_result.snapshot.as_log_record())
|
||
record = {
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
**snapshot_record,
|
||
'action': dialog_result.action,
|
||
'rule_id': dialog_result.rule_id,
|
||
'channel': dialog_result.channel,
|
||
'accepted_control': list(dialog_result.accepted_controls),
|
||
'error_code': dialog_result.error_code,
|
||
'detail': dialog_result.detail,
|
||
}
|
||
with (root/'dialogs.jsonl').open('a', encoding='utf-8') as stream:
|
||
stream.write(json.dumps(record, ensure_ascii=False) + '\n')
|
||
if dialog_result.activity:
|
||
deadline.touch()
|
||
if dialog_result.error_code:
|
||
_write_failure_windows(root, latest_snapshots)
|
||
_write_process_tree(root, process, target_pid)
|
||
_stop_worker(process)
|
||
return finish(VerifyResult(
|
||
'FAIL',
|
||
stderr=f'{dialog_result.error_code}: {dialog_result.detail}',
|
||
error_stage=phase.value,
|
||
safe_to_continue=True,
|
||
))
|
||
if deadline.expired():
|
||
_write_failure_windows(root, latest_snapshots)
|
||
# Kills the worker's whole job: worker, PCB and cscript helper.
|
||
_write_process_tree(root, process, target_pid)
|
||
if last_post_verify_observation:
|
||
(root/'post_verify_observations.jsonl').write_text(json.dumps({
|
||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||
'invoke_returned': invoke_returned,
|
||
**last_post_verify_observation,
|
||
}, ensure_ascii=False) + '\n', encoding='utf-8')
|
||
dispatch_failure = classify_verify_dispatch(
|
||
verify_invoked=verify_invoked,
|
||
invoke_returned=invoke_returned,
|
||
target_pid=target_pid,
|
||
observation=last_post_verify_observation,
|
||
)
|
||
_stop_worker(process)
|
||
if dispatch_failure is not None:
|
||
status, detail = dispatch_failure
|
||
if progress:
|
||
progress({
|
||
'stage': 'verify_dispatch_failed',
|
||
'block': request.block_name,
|
||
'detail': detail,
|
||
'observation': last_post_verify_observation,
|
||
})
|
||
return finish(VerifyResult(status, stderr=detail, safe_to_continue=True))
|
||
return finish(VerifyResult('TIMEOUT', stderr='操作超时;已记录现场,跳过当前 CBB。', error_stage=phase.value, safe_to_continue=True))
|
||
time.sleep(.2)
|
||
worker_result = root/'outcome.json'
|
||
if not worker_result.exists():
|
||
return finish(VerifyResult('FAIL', stderr=f'执行器异常退出 ({process.returncode}),见 worker.log', safe_to_continue=False))
|
||
outcome = json.loads(worker_result.read_text(encoding='utf-8'))
|
||
if outcome.get('status') == 'PASS' and process.returncode != 0:
|
||
return finish(VerifyResult('FAIL', stderr='执行器异常退出,不能确认成功', safe_to_continue=False))
|
||
result = VerifyResult(**outcome)
|
||
except Exception as error:
|
||
if 'process' in locals() and process.poll() is None:
|
||
_stop_worker(process)
|
||
return finish(VerifyResult('FAIL', stderr=str(error), safe_to_continue=False))
|
||
finally:
|
||
if 'dialog_agent' in locals() and dialog_agent is not None:
|
||
dialog_agent.close()
|
||
if 'process' in locals(): process.close()
|
||
if 'process' in locals():
|
||
try:
|
||
_cleanup_session_processes(
|
||
root,
|
||
lm_pid=lm_pid,
|
||
target_pid=target_pid,
|
||
pcb=pcb,
|
||
pcb_baseline=pcb_baseline,
|
||
progress=progress,
|
||
)
|
||
except Exception:
|
||
# Cleanup is best effort; the original Verify result remains
|
||
# authoritative and the evidence directory records everything
|
||
# that was successfully reaped.
|
||
pass
|
||
# Drain the timing tail even when the worker exited before a poll.
|
||
# This is optional telemetry: an unreadable log must not mask failure
|
||
# or prevent releasing the serial-operation mutex.
|
||
try:
|
||
if events.is_file():
|
||
with events.open(encoding='utf-8') as stream:
|
||
stream.seek(offset)
|
||
for line in stream:
|
||
try:
|
||
timing.record(json.loads(line))
|
||
except (json.JSONDecodeError, ValueError):
|
||
continue
|
||
except OSError:
|
||
pass
|
||
finally:
|
||
win32event.ReleaseMutex(mutex)
|
||
mutex.Close()
|
||
|
||
if result.status == 'PASS':
|
||
timing.begin('library_flag')
|
||
bridge_records = []
|
||
def record_attempt(record):
|
||
bridge_records.append(record)
|
||
if progress:
|
||
progress({'stage': 'library_flag_probe', 'block': request.block_name, **record})
|
||
try:
|
||
probe = read_library_flag_with_retry(
|
||
flag_reader or read_library_flag_if_available, Path(request.lmc_file),
|
||
request.block_name, on_attempt=record_attempt)
|
||
result = apply_post_verify_status(result, probe)
|
||
result = replace(result, bridge_attempts=tuple(bridge_records))
|
||
(root/'bridge-status.json').write_text(json.dumps(bridge_records, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
if progress:
|
||
progress({'stage': 'library_flag', 'block': request.block_name,
|
||
'library_flag': result.library_flag, 'detail': result.library_flag_detail})
|
||
finally:
|
||
timing.end('library_flag')
|
||
return finish(result)
|
||
|
||
|
||
def run_batch(requests, max_workers=1, verifier=None, *, stop=None, progress=None):
|
||
"""Run CBBs serially; one item's failure never aborts the queue."""
|
||
if max_workers != 1:
|
||
raise ValueError('Only serial Verify is supported for the shared Library Manager session')
|
||
if verifier is None:
|
||
verifier = lambda request: run_verify(request, flag_reader=read_library_flag_if_available) # noqa: E731
|
||
results = []
|
||
queue_stop = False
|
||
for request in requests:
|
||
if queue_stop or (stop is not None and stop.is_set()):
|
||
result = VerifyResult('SKIPPED', stderr='队列已停止,未操作此 CBB')
|
||
else:
|
||
try:
|
||
result = verifier(request)
|
||
except Exception as error:
|
||
# An unexpected worker exception is a failed item, not a batch abort.
|
||
result = VerifyResult('FAIL', stderr=str(error), safe_to_continue=False)
|
||
results.append(result)
|
||
# Failure status and safe_to_continue describe this item, not the batch;
|
||
# the queue is skipped only after an explicit stop signal.
|
||
queue_stop = queue_stop or result.status == 'QUEUE_STOP'
|
||
if result.status == 'QUEUE_STOP' and stop is not None:
|
||
stop.set()
|
||
if progress: progress(request.block_name, result)
|
||
return results
|
||
|
||
|
||
def failure_report_path(batch_report):
|
||
batch_report = Path(batch_report)
|
||
return batch_report.with_name(batch_report.name.replace('batch-', 'failures-', 1))
|
||
|
||
|
||
def _report_field(value):
|
||
# One CBB per row; full native multi-line text remains in evidence files.
|
||
return str(value).replace('\r', r'\r').replace('\n', r'\n').replace('\t', r'\t')
|
||
|
||
|
||
def write_failure_report(requests, results, root, *, path=None):
|
||
requests, results = tuple(requests), tuple(results)
|
||
if len(requests) != len(results):
|
||
raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告')
|
||
root = Path(root); root.mkdir(parents=True, exist_ok=True)
|
||
path = Path(path) if path else root/('failures-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
|
||
failed = [(req, result) for req, result in zip(requests, results)
|
||
if result.status not in {'PASS', 'SKIPPED', 'QUEUE_STOP'}]
|
||
passed = sum(result.status == 'PASS' for result in results)
|
||
skipped = len(results)-passed-len(failed)
|
||
lines = ['CBB Verify 失败清单', f'总数: {len(results)} 成功: {passed} 失败: {len(failed)} 未执行: {skipped}',
|
||
'CBB\t状态\t错误分类\t失败阶段\t桥状态\t原始错误\t证据\t实际耗时(秒)']
|
||
for req, result in failed:
|
||
timings = ' '.join(f'{key}={value:.3f}s' +
|
||
('(unfinished)' if key in result.incomplete_timing_phases else '')
|
||
for key, value in result.timings_seconds.items())
|
||
lines.append(f'{req.block_name}\t{result.status}\t{result.error_category or classify_failure_category(result.stderr)}\t'
|
||
f'{result.error_stage or "unknown"}\t{result.library_flag}\t{_report_field(result.stderr or result.stdout.strip()[:200])}\t{result.evidence_path or ""}\t{timings}')
|
||
if not failed:
|
||
lines.append('本批次没有失败项。')
|
||
path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig')
|
||
return path
|
||
|
||
|
||
def write_batch_report(requests, results, root):
|
||
requests, results = tuple(requests), tuple(results)
|
||
if len(requests) != len(results):
|
||
raise ValueError('requests/results 长度不一致,不能生成遗漏条目的报告')
|
||
root = Path(root); root.mkdir(parents=True, exist_ok=True)
|
||
path = root/('batch-' + time.strftime('%Y%m%d-%H%M%S') + '-' + uuid4().hex[:6] + '.log')
|
||
lines = ['CBB Verify 批量结果',
|
||
'流程: LMC 原生 Verify / ReusableCircuit PCB -> Project Integration -> Save/Close -> IsVerifiedReuseBlock READY(金标准)']
|
||
for req, result in zip(requests, results):
|
||
detail = _report_field(result.stderr or result.stdout.strip()[:200])
|
||
seconds = ' '.join(f'{key}={value:.3f}s' +
|
||
('(unfinished)' if key in result.incomplete_timing_phases else '')
|
||
for key, value in result.timings_seconds.items())
|
||
lines.append(f'{req.block_name}\t{result.status}\t{result.library_flag}\t{detail}\t{result.evidence_path or ""}\t{seconds}')
|
||
path.write_text('\n'.join(lines) + '\n', encoding='utf-8-sig')
|
||
write_failure_report(requests, results, root, path=failure_report_path(path))
|
||
return path
|