From 8029d146cb6448d5a0c269111028aaf3c61e5d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E7=99=BD=E5=8F=94?= <11794527+absfan@user.noreply.gitee.com> Date: Mon, 9 Nov 2026 14:35:48 +0800 Subject: [PATCH] Run isolated export tasks concurrently and overlap cleanup --- README.md | 6 + docs/releases/v0.1.2.md | 15 +++ docs/validation-2026-11-09-parallel.md | 45 +++++++ mentorexport/__init__.py | 2 +- mentorexport/cli.py | 18 ++- mentorexport/exports.py | 119 +++++++++++++----- tests/test_cli.py | 18 +++ tests/test_exports.py | 159 ++++++++++++++++++++++++- 8 files changed, 341 insertions(+), 41 deletions(-) create mode 100644 docs/releases/v0.1.2.md create mode 100644 docs/validation-2026-11-09-parallel.md diff --git a/README.md b/README.md index 388e4a4..5b041e0 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,15 @@ 导出.cmd --input "F:\工程目录" --output "F:\导出结果" --yes 导出.cmd --mentor-home "D:\Mentor\7.9.5EE\SDD_HOME" --input "F:\工程目录" --yes 导出.cmd --retry "F:\导出结果\批次_..." --yes + 导出.cmd --input "F:\工程目录" --output "F:\导出结果" --jobs 3 --yes + 导出.cmd --input "F:\工程目录" --jobs 1 --yes 不指定参数进入问答。--yes 适合使用固定参数重复运行;发现多套 Mentor 时需要 --mentor-home 指定。--timeout 可以调整单工具等待上限,默认 60 秒。 +默认同时运行 3 个独立导出任务,可用 --jobs 1 切换串行,--jobs 2 到 8 调整并发上限。各工具使用同一已保存输入的独立副本;工程按顺序处理,库件读取在 ASCII 平铺属性就绪后与其他导出重叠,同库缓存继续共享。快照名称探测也参与同一并发上限,不额外占用任务槽。 + +工具退出通过进程通知立即唤醒,不额外睡眠;收尾与其他导出重叠执行。后台进程自然退出的宽限期最多 2 秒,已退出则立即结束等待。若仍有自有后台进程需要终止,会保留文件并记录提示,不标为已验证成功,也不阻塞其他任务。运行记录分别保存工具执行与收尾耗时,工程记录保存实际并发数与总耗时。 + 按 Ctrl+C 请求停止,等待工具收尾;不要直接关闭窗口。下次使用 --retry 指定批次,可恢复未完成项。成功产物只有在输入、工具和输出摘要仍匹配时才复用;源工程变化会完整重新导出该工程。 默认尽力导出并持续完成批次。单项异常、缺工具/缺库、超时、格式检查提示或进程收尾提示,都记录下来并继续后续工具和工程;报告文件被占用也不阻塞导出。只有用户主动按 Ctrl+C 才停止批次。 diff --git a/docs/releases/v0.1.2.md b/docs/releases/v0.1.2.md new file mode 100644 index 0000000..5c66427 --- /dev/null +++ b/docs/releases/v0.1.2.md @@ -0,0 +1,15 @@ +# MentorExport 0.1.2 + +2026-11-09。加速原理图批量导出,保留 0.1.1 的单项问题不阻塞批次策略。 + +- 默认 3 个导出任务并发,支持 `--jobs 1` 串行和 `--jobs 2` 到 `--jobs 8`。问答模式直接采用默认 3,无需额外配置。 +- 每工具独立数据库副本、输出、日志、私有桌面和进程 Job;仅协调线程更新工程记录,输出分类和逐项顺序稳定。 +- ASCII 平铺属性完成后即可开始库件读取,与其他格式导出重叠;同库缓存继续在顺序处理的工程间共享。 +- 快照探测与正文导出重叠,同样计入并发上限。 +- 工具退出改为进程通知唤醒,消除固定轮询睡眠;收尾等待与其他导出重叠。保留后台进程最多 2 秒的自然退出宽限期,避免过早终止导致下次重试需要重导。 +- 若后台进程仍需强制结束,文件照常保留、显示提示,不作为已验证成功结果复用;库件记录也不进入已验证缓存。进程退出确认和工作副本保留机制继续生效。 +- 移除一次重复的库件记录写入,新增工具执行/进程收尾耗时及工程并发信息。 + +下载完整 ZIP 解压,双击 `导出.cmd`。本机需已有 Mentor 与可用许可,Python 和依赖已预置。实际兼容环境仍为 EE7.9.5;其他安装的原生库读取沿用已验证指纹准入。 + +实测同一对真实工程:v0.1.1 串行 156.015 秒,新版默认并发 74.594 秒,耗时减少 52.2%。22 项均取得且通过检查,EDF/KYN 连接和器件属性、关联库记录与串行结果相同;重试校验后复用耗时 13.609 秒。64 项自动测试通过。完整范围见 [并发验收](../validation-2026-11-09-parallel.md),实际速度取决于工程和机器环境。 diff --git a/docs/validation-2026-11-09-parallel.md b/docs/validation-2026-11-09-parallel.md new file mode 100644 index 0000000..6a178f9 --- /dev/null +++ b/docs/validation-2026-11-09-parallel.md @@ -0,0 +1,45 @@ +# 0.1.2 并发导出验收 + +2026-11-09。本机 Windows x64、Mentor EE7.9.5。默认每工程 3 个导出任务并发,工程顺序处理,同库缓存继续共享。`--jobs 1` 保留串行入口,最大并发 8。 + +## 自动测试及审查 + +`python -X utf8 -m pytest -q`:**64 passed in 34.73s**。 + +新增回归先失败后通过,覆盖独立副本上的真实调度重叠、并发上限、稳定输出顺序、快照与正文导出重叠、库件在 ASCII 就绪后提前运行、并发取消后收尾、CLI 参数传递、无效并发数、进程退出通知唤醒,以及强制收尾产物不能作为已验证结果复用。 + +独立审查确认协调线程独占工程记录写入,worker 使用独立状态与副本,同库分区缓存仍为单写入者。审查发现的强制结束后误入成功缓存已修正,普通导出和原生库读取都保留产物但标为提示;末轮针对进程等待与库缓存复查无待处理发现。 + +## 同输入实测 + +从之前已验收的两个真实工程复制固定输入,分别使用不可修改的 v0.1.1 发行包和最终 v0.1.2 发行包。每次是独立新进程和新批次,未用 `--retry` 代替首次导出。 + +| 项目 | v0.1.1 串行 | v0.1.2 默认 3 并发 | +| --- | ---: | ---: | +| 两工程完整导出 | 156.015 秒 | 74.594 秒 | +| 取得的导出项 | 22 | 22 | +| 带提示项 | 0 | 0 | +| 批次退出码 | 0 | 0 | + +本机样例耗时降低 **52.2%**,约为原速度的 **2.09 倍**。不同工程、磁盘负载及许可环境下不保证同一比例。 + +两工程逐一解析 EDIF 网表与 KYN:器件、字符串属性、物理引脚、网络及连接集合与串行结果完全一致。关联库件原生记录逐项完全一致。14 个工程输入文件摘要未变,工程内源数据库和源库检查均为 source_unchanged=true。 + +最终包复制到中文及空格目录,从 `C:\Windows\Temp` 通过真实 CMD 无参数问答启动,PATH 仅保留 System32,PYTHONHOME/PYTHONPATH 为无效目录。自动发现环境并使用预置 Python 成功。 + +随后使用 `--retry --jobs 1 --yes` 验证串行入口及重复使用:13.609 秒,退出 0,22 项成功产物校验后复用。664 个发行文件与 manifest、ZIP 内容均核对一致。 + +## 等待策略 + +固定 50 ms 轮询睡眠改为进程句柄等待,工具退出即唤醒;仍以最多 50 ms 检查用户取消和超时。正常后台进程保留最多 2 秒自然退出宽限期,等待与其他任务重叠。 + +曾以 0.25 秒宽限期测试,出现 14 项强制收尾提示,串行重试需重新导出并耗时 68.188 秒。最终保留 2 秒正常宽限期,解决重复运行退化;不靠过早终止后台进程制造加速。 + +## 证据 + +- 串行基线及初轮对照:`build/parallel-benchmark-20261109-141823/benchmark.json`。 +- 最终实际问答、串行重试及逐项比较:`build/parallel-release-acceptance-20261109-143025/acceptance.json`。 +- 原始逐工具记录新增 started_at、tool_seconds、cleanup_seconds;工程记录新增 execution.jobs 与 execution.seconds。 +- 最终 ZIP SHA256:`28616ced54ea893058d6dcde00025a9a72122814ae4f0021ee790ff9ed7bf22c`。 + +本次未增加跨工程并发或原生库分区并发;主要加速来自独立导出任务并行、库件提前开始、快照探测重叠、进程退出通知及减少重复记录写入。 diff --git a/mentorexport/__init__.py b/mentorexport/__init__.py index 8a2342c..8a8afe4 100644 --- a/mentorexport/__init__.py +++ b/mentorexport/__init__.py @@ -1,2 +1,2 @@ """Portable Mentor saved-schematic exports.""" -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/mentorexport/cli.py b/mentorexport/cli.py index d11079f..5ddc039 100644 --- a/mentorexport/cli.py +++ b/mentorexport/cli.py @@ -95,7 +95,9 @@ def previous_results(record): def run_batch(projects, problems, installation, output_root, scan_root, *, previous=None, - stop_event=None, timeout=60, exporter=None, progress=print): + stop_event=None, timeout=60, exporter=None, progress=print, jobs=3): + if not 1 <= jobs <= 8: + raise ValueError("并发数必须为 1 到 8") if exporter is None: from .exports import export_project exporter = export_project @@ -113,7 +115,7 @@ def run_batch(projects, problems, installation, output_root, scan_root, *, previ "scan_root": str(Path(scan_root).resolve()), "output_root": str(output_root), "created": datetime.now().astimezone().isoformat(), "installation": installation, "owner": process_identity(os.getpid()), - "problems": problems, "projects": [], "warnings": []} + "problems": problems, "projects": [], "warnings": [], "execution": {"jobs": jobs}} def warn(message): if message not in record["warnings"]: record["warnings"].append(message) @@ -156,7 +158,7 @@ def run_batch(projects, problems, installation, output_root, scan_root, *, previ try: result = exporter(project, installation, Path(row["folder"]), shared, previous=prior.get(project["path"]), stop_event=stop_event, - timeout=timeout, progress=progress) + timeout=timeout, progress=progress, jobs=jobs) row["result"] = result row["status"] = result["status"] if result.get("cleanup_uncertain"): @@ -241,6 +243,7 @@ def main(argv=None): parser.add_argument("--mentor-home", help="指定 Mentor SDD_HOME") parser.add_argument("--retry", help="重试已有批次,指定批次目录或 batch.json") parser.add_argument("--timeout", type=float, default=60, help="单工具超时秒数,默认 60") + parser.add_argument("--jobs", "-j", type=int, default=3, help="同时运行的导出任务数,默认 3;1 为串行,最多 8") parser.add_argument("--yes", "-y", action="store_true", help="采用已给定参数运行,不显示交互菜单") parser.add_argument("--check", action="store_true", help="只检查预置环境与 Mentor") parser.add_argument("--json", action="store_true", help="环境检查以 JSON 输出") @@ -249,6 +252,8 @@ def main(argv=None): try: if args.timeout <= 0: raise ValueError("超时必须大于 0") + if not 1 <= args.jobs <= 8: + raise ValueError("并发数必须为 1 到 8") errors = runtime_check() if errors: raise ValueError("预置运行环境不完整,请重新解压发行包:\n" + "\n".join(errors)) @@ -269,6 +274,7 @@ def main(argv=None): return 0 if installation["preflight"]["status"] == "success" else 2 print(f"\nMentorExport {__version__} · 原理图详细导出") print(f"Mentor:{installation['version']} {installation['root']}") + print(f"并发任务:{args.jobs}(--jobs 1 可切换串行)") print("读取已保存的原理图。请先保存目标工程;工具仅在副本上运行。") if not args.retry and settings.get("output"): candidates = sorted(Path(settings["output"]).glob("批次_*/batch.json"), reverse=True)[:30] @@ -277,7 +283,7 @@ def main(argv=None): print("发现未完成批次:" + str(unfinished[0].parent)) if not args.yes and _ask("输入 1 重试该批次;回车开始新批次") == "1": return main(["--retry", str(unfinished[0]), "--mentor-home", installation["root"], - "--timeout", str(args.timeout)]) + "--timeout", str(args.timeout), "--jobs", str(args.jobs)]) previous = None if args.retry: previous_path = Path(args.retry) @@ -329,7 +335,7 @@ def main(argv=None): signal.signal(signal.SIGINT, cancel) try: batch, code = run_batch(projects, problems, installation, output_root, scan_root, - previous=previous, stop_event=stop, timeout=args.timeout) + previous=previous, stop_event=stop, timeout=args.timeout, jobs=args.jobs) finally: signal.signal(signal.SIGINT, old_handler) record = load_json(batch / "batch.json", {"projects": []}) @@ -347,7 +353,7 @@ def main(argv=None): if action in ("1", "2"): os.startfile(str(batch if action == "1" else batch / "导出汇总.html")) elif action == "3": - return main(["--retry", str(batch), "--mentor-home", installation["root"], "--timeout", str(args.timeout)]) + return main(["--retry", str(batch), "--mentor-home", installation["root"], "--timeout", str(args.timeout), "--jobs", str(args.jobs)]) return code except KeyboardInterrupt: print("\n已取消。") diff --git a/mentorexport/exports.py b/mentorexport/exports.py index 2ac97ee..a8e8a4c 100644 --- a/mentorexport/exports.py +++ b/mentorexport/exports.py @@ -1,6 +1,7 @@ """Saved-project exports through short lived, owned, isolated tool processes.""" from copy import deepcopy from contextlib import contextmanager +from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED import ctypes from datetime import datetime, timezone import hashlib @@ -9,6 +10,7 @@ import os from pathlib import Path import re import shutil +import subprocess import tempfile import time import uuid @@ -94,7 +96,7 @@ def _run(command, cwd, environment, log_dir, *, stop_event, timeout): log_dir.mkdir(parents=True, exist_ok=True) desktop = PrivateDesktop("mexp-" + uuid.uuid4().hex, enabled=True) result = {"command": list(map(str, command)), "cwd": str(cwd), "exit_code": None, - "timed_out": False, "cancelled": False} + "timed_out": False, "cancelled": False, "started_at": datetime.now(timezone.utc).isoformat()} started = time.monotonic() try: with desktop: @@ -113,12 +115,18 @@ def _run(command, cwd, environment, log_dir, *, stop_event, timeout): except Exception as error: result["dialog_inspection_error"] = str(error) break - time.sleep(0.05) + try: + process.wait(timeout=min(0.05, max(0, timeout - (time.monotonic() - started)))) + except subprocess.TimeoutExpired: + pass result["exit_code"] = process.poll() + result["tool_seconds"] = round(time.monotonic() - started, 3) except Exception as error: result["error"] = str(error) finally: result["seconds"] = round(time.monotonic() - started, 3) + if "tool_seconds" in result: + result["cleanup_seconds"] = round(result["seconds"] - result["tool_seconds"], 3) result["cleanup"] = desktop.cleanup_record cleanup = result["cleanup"] or {} if cleanup.get("errors") or any(p.get("job_active_processes", 0) for p in cleanup.get("processes", [])): @@ -303,12 +311,16 @@ def _status(items): return "exported" if any(i.get("outputs") for i in items) else "skipped" -def export_project(project, installation, project_dir, shared_dir, *, previous=None, stop_event=None, timeout=45, progress=print): +def export_project(project, installation, project_dir, shared_dir, *, previous=None, stop_event=None, timeout=45, progress=print, jobs=3): + if not 1 <= jobs <= 8: + raise ValueError("并发数必须为 1 到 8") + project_started = time.monotonic() project_dir = Path(project_dir).resolve() project_dir.mkdir(parents=True, exist_ok=True) shared_dir = Path(shared_dir).resolve() result = {"status": "running", "project": deepcopy(project), "captured_at": datetime.now(timezone.utc).isoformat(), "source_fingerprint": None, "source_hashes": {}, "items": [], "source_unchanged": None, + "execution": {"jobs": jobs}, "coverage_limits": ["Exports preserve each tool/mode separately; no ERC or manufacturing certification.", "CMS requests read mode but may report ReadOnly=False; only isolated copies are opened.", "Cell coordinate units and rotation remain native integers; multi-slot/swappable mappings unverified."]} @@ -336,25 +348,22 @@ def export_project(project, installation, project_dir, shared_dir, *, previous=N return result environment = _environment(installation) old_items = {i["key"]: i for i in (previous or {}).get("items", [])} - discovery = {"status": "failed", "names": []} - result["snapshot_discovery"] = discovery - try: - snapshot_work = temp / "snapshot-discovery" - shutil.copytree(captured, snapshot_work) - discovery.update(_snapshot_metadata(project, installation, snapshot_work, - project_dir / "90_运行日志" / "snapshots", stop_event=stop_event, - timeout=min(timeout, 20))) - if discovery.get("cleanup_uncertain"): - result.update(cleanup_uncertain=True, workspace_preserved=str(temp)) - except Exception as error: - discovery.update(status="failed", reason=str(error)) - if discovery["status"] != "success": - result["coverage_limits"].append("Other snapshot names were not obtained: " + discovery.get("reason", "")) - _checkpoint(state, result) - for key, tool, folder in ITEMS: + def discover(): + discovery = {"status": "failed", "names": []} + try: + snapshot_work = temp / "snapshot-discovery" + shutil.copytree(captured, snapshot_work) + discovery.update(_snapshot_metadata(project, installation, snapshot_work, + project_dir / "90_运行日志" / "snapshots", stop_event=stop_event, + timeout=min(timeout, 20))) + except Exception as error: + discovery.update(status="failed", reason=str(error)) + return discovery + + def execute(spec, task_state): + key, tool, folder = spec output = None item = {"key": key, "status": "failed", "reason": "", "outputs": [], "provenance": {}} - result["items"].append(item) try: progress(project["name"] + ": " + key) identity = _tool_identity(key, tool, installation) @@ -363,19 +372,19 @@ def export_project(project, installation, project_dir, shared_dir, *, previous=N "input_fingerprint": fingerprint, "reused": False} if stop_event is not None and stop_event.is_set(): item.update(status="cancelled", reason="Cancelled before launch") - continue + return item, task_state if _reusable(old_items.get(key), fingerprint, identity): item.update(deepcopy(old_items[key])) item["provenance"]["reused"] = True item["reason"] = "Verified previous successful output reused" - continue + return item, task_state if key == "library": - _export_library(item, project, installation, result, project_dir / folder, shared_dir, + _export_library(item, project, installation, task_state, project_dir / folder, shared_dir, temp, environment, stop_event, timeout) - continue + return item, task_state if not identity.get("sha256") or (tool == "cms" and not identity.get("matches")): item.update(status="unavailable", reason="Tool missing or COM registration does not match selected installation") - continue + return item, task_state work = temp / key shutil.copytree(captured, work) output = work / "output" @@ -386,7 +395,7 @@ def export_project(project, installation, project_dir, shared_dir, *, previous=N item["provenance"].update(run) item["provenance"]["log_dir"] = str(log_dir) if run.get("cleanup_uncertain"): - result.update(cleanup_uncertain=True, workspace_preserved=str(temp)) + task_state.update(cleanup_uncertain=True, workspace_preserved=str(temp)) item["reason"] = "Owned process cleanup is uncertain; workspace preserved; remaining exports continue" elif run.get("cancelled"): item.update(status="cancelled", reason="Cancelled while tool was running") @@ -407,6 +416,8 @@ def export_project(project, installation, project_dir, shared_dir, *, previous=N item.update(status="success", outputs=[str(p) for p in published]) item["validation"] = {"status": "passed"} item["provenance"]["output_files"] = [_file_info(p) for p in published] + if any(process.get("forced_termination") for process in run.get("cleanup", {}).get("processes", [])): + item.update(status="exported", reason="Files retained; owned background processes exceeded shutdown grace and were terminated; background completion unverified") except Exception as error: item.update(status="failed", reason=str(error)) finally: @@ -425,7 +436,7 @@ def export_project(project, installation, project_dir, shared_dir, *, previous=N saved.append(_file_info(target)) except OSError as error: publication_errors.append(str(error)) - result.setdefault("workspace_preserved", str(temp)) + task_state.setdefault("workspace_preserved", str(temp)) try: if source.is_file() and source.stat().st_size: saved.append(_file_info(source)) @@ -443,8 +454,45 @@ def export_project(project, installation, project_dir, shared_dir, *, previous=N item["status"] = "exported" if item["status"] == "exported": item["validation"] = {"status": "warning", "reason": item["reason"]} - result["status"] = "running" - _checkpoint(state, result) + return item, task_state + + result["items"] = [{"key": key, "status": "pending", "outputs": [], "reason": "", "provenance": {}} + for key, tool, folder in ITEMS] + specs = {spec[0]: spec for spec in ITEMS} + positions = {item["key"]: index for index, item in enumerate(result["items"])} + ready = ["ascii_flat", "snapshots"] + [key for key, tool, folder in ITEMS if key not in ("ascii_flat", "library")] + # Only the coordinator publishes receipts; workers own distinct clones, items and logs. + with ThreadPoolExecutor(max_workers=jobs, thread_name_prefix="mentor-export") as pool: + active = {} + while ready or active: + while ready and len(active) < jobs: + key = ready.pop(0) + if key == "snapshots": + future = pool.submit(discover) + else: + task_state = {"items": list(result["items"]), + "library_source_hashes": result["library_source_hashes"]} + future = pool.submit(execute, specs[key], task_state) + active[future] = key + completed, _ = wait(active, return_when=FIRST_COMPLETED) + for future in completed: + key = active.pop(future) + if key == "snapshots": + metadata = future.result() + result["snapshot_discovery"] = metadata + task_state = metadata + if metadata["status"] != "success": + result["coverage_limits"].append("Other snapshot names were not obtained: " + metadata.get("reason", "")) + else: + item, task_state = future.result() + result["items"][positions[key]] = item + if key == "ascii_flat": + ready.insert(0, "library") + if task_state.get("cleanup_uncertain"): + result["cleanup_uncertain"] = True + if task_state.get("workspace_preserved"): + result["workspace_preserved"] = str(temp) + _checkpoint(state, result) try: after = _hashes(_inputs(project)) result["source_hashes_after"] = after @@ -459,6 +507,7 @@ def export_project(project, installation, project_dir, shared_dir, *, previous=N if item["status"] == "success": item.update(status="exported", reason="Source changed during export; captured output retained; re-export to verify current source", validation={"status": "warning", "reason": "Source changed during export"}) + result["execution"]["seconds"] = round(time.monotonic() - project_started, 3) result["status"] = _status(result["items"]) _checkpoint(state, result) return result @@ -619,6 +668,9 @@ def _export_library(item, project, installation, result, destination, shared_dir data["errors"].append({"partition": relative, "reason": run.get("error") or "Native read failed/timeout/cleanup error"}) persist() continue + trusted_run = not any(process.get("forced_termination") for process in run.get("cleanup", {}).get("processes", [])) + if not trusted_run: + data["errors"].append({"partition": relative, "reason": "Owned background process terminated during shutdown; records retained without verified reuse"}) try: native = json.loads((log / "stdout.txt").read_text(encoding="utf-8")) _validate_native(native, mode) @@ -634,15 +686,16 @@ def _export_library(item, project, installation, result, destination, shared_dir if known: old_names = {row["name"] for row in selected} selected.extend(row for row in known["selected"] if row["name"] not in old_names) - partitions[relative] = {"names": {row["name"] for row in native["records"]}, - "selected": selected, "run": deepcopy(run)} + if trusted_run: + partitions[relative] = {"names": {row["name"] for row in native["records"]}, + "selected": selected, "run": deepcopy(run)} # Remove unrelated full-library stdout only after selected records are durable. persist() (log / "stdout.txt").unlink() run["stdout_retained"] = False _checkpoint(log / "run.json", run) - partitions[relative]["run"] = deepcopy(run) - persist() + if trusted_run: + partitions[relative]["run"] = deepcopy(run) except (OSError, ValueError) as error: data["errors"].append({"partition": relative, "reason": str(error)}) persist() diff --git a/tests/test_cli.py b/tests/test_cli.py index be6b63e..3114519 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -131,3 +131,21 @@ def test_preflight_exception_is_a_warning_not_a_batch_blocker(tmp_path, monkeypa monkeypatch.setattr(cli, "_save_settings", lambda value: None) assert cli.main(["--input", str(tmp_path), "--output", str(tmp_path / "out"), "--yes"]) == 0 assert list((tmp_path / "out").glob("批次_*/batch.json")) + + +def test_batch_passes_worker_limit_and_records_execution_mode(tmp_path): + from mentorexport.cli import run_batch + limits = [] + def exporter(project, installation, folder, shared, **options): + limits.append(options["jobs"]) + return {"status": "success", "items": []} + batch, code = run_batch([{"name": "工程", "path": str(tmp_path / "a.prj")}], [], {}, + tmp_path / "out", tmp_path, exporter=exporter, jobs=2) + record = json.loads((batch / "batch.json").read_text(encoding="utf-8")) + assert code == 0 and limits == [2] and record["execution"]["jobs"] == 2 + + +def test_invalid_worker_limit_does_not_start_export(tmp_path): + from mentorexport.cli import main + for jobs in ("0", "9"): + assert main(["--input", str(tmp_path), "--jobs", jobs, "--yes"]) == 2 diff --git a/tests/test_exports.py b/tests/test_exports.py index df22e92..235a2e8 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -25,6 +25,163 @@ def backend(): return exports +def test_parallel_exports_overlap_on_distinct_clones_and_keep_receipt_order(sample, monkeypatch): + from threading import Barrier, Lock + exports = backend() + project, install, root = sample + barrier, lock = Barrier(2), Lock() + calls, workspaces = [], [] + run_ascii = ascii_runner(calls) + active = maximum = 0 + def runner(command, cwd, environment, log_dir, **kwargs): + nonlocal active, maximum + with lock: + active += 1 + maximum = max(maximum, active) + workspaces.append(Path(cwd)) + try: + barrier.wait(timeout=5) + return run_ascii(command, cwd, environment, log_dir, **kwargs) + finally: + with lock: + active -= 1 + monkeypatch.setattr(exports, "_run", runner) + result = exports.export_project(project, install, root / "parallel", root / "shared", jobs=2, progress=lambda _: None) + assert maximum == 2 and len(set(workspaces)) == 2 + assert [item["key"] for item in result["items"]] == [spec[0] for spec in exports.ITEMS] + assert [i["key"] for i in result["items"] if i["status"] == "success"] == ["ascii_flat", "ascii_hier"] + assert result["source_unchanged"] and result["execution"]["jobs"] == 2 + + +def test_library_starts_after_flat_without_waiting_for_other_exports(sample, monkeypatch): + from threading import Event + exports = backend() + project, install, root = sample + library_started = Event() + runner = ascii_runner([]) + def run(command, *args, **kwargs): + if "-h" in command: + assert library_started.wait(5), "library waited for unrelated exports" + return runner(command, *args, **kwargs) + def library(item, project, installation, result, *args): + flat = next(i for i in result["items"] if i["key"] == "ascii_flat") + assert flat["status"] == "success" and Path(flat["outputs"][0]).is_file() + library_started.set() + item.update(status="unavailable", reason="fixture without library") + monkeypatch.setattr(exports, "_run", run) + monkeypatch.setattr(exports, "_export_library", library) + result = exports.export_project(project, install, root / "parallel", root / "shared", jobs=2, progress=lambda _: None) + assert library_started.is_set() and result["items"][1]["status"] == "success" + + +def test_snapshot_listing_overlaps_with_exports(sample, monkeypatch): + from threading import Barrier + exports = backend() + project, install, root = sample + install["tools"]["constraints_xml"] = install["tools"]["ascii"] + barrier = Barrier(2) + runner = ascii_runner([]) + def run(command, cwd, environment, log_dir, **kwargs): + if "-list" in command or "-f" in command: + barrier.wait(timeout=5) + if "-list" in command: + Path(log_dir).mkdir(parents=True) + (Path(log_dir) / "stdout.txt").write_text(" -> Saved\n") + return {"exit_code": 0} + return runner(command, cwd, environment, log_dir, **kwargs) + monkeypatch.setattr(exports, "_run", run) + result = exports.export_project(project, install, root / "parallel", root / "shared", jobs=2, progress=lambda _: None) + assert result["snapshot_discovery"]["status"] == "success" + assert result["items"][0]["status"] == "success" + + +def test_forced_descendant_shutdown_keeps_files_but_never_reuses_as_verified(sample, monkeypatch): + exports = backend() + project, install, root = sample + calls = [] + ascii_run = ascii_runner(calls) + def runner(*args, **kwargs): + run = ascii_run(*args, **kwargs) + run["cleanup"] = {"errors": [], "processes": [{"forced_termination": True, + "forced_root_termination": False, "forced_descendant_pids": [123], + "termination_reason": "descendant_grace_expired", "job_active_processes": 0}]} + return run + monkeypatch.setattr(exports, "_run", runner) + first = exports.export_project(project, install, root / "first", root / "shared", jobs=2, progress=lambda _: None) + assert first["items"][0]["status"] == "exported" and first["items"][0]["outputs"] + assert first["items"][0]["validation"]["status"] == "warning" + exports.export_project(project, install, root / "retry", root / "shared", jobs=2, previous=first, progress=lambda _: None) + assert len(calls) == 4 + + +def test_forced_library_shutdown_retains_records_without_trusted_cache(sample, monkeypatch): + exports = backend() + project, install, root = sample + library = root / "library/PartsDBLibs" + library.mkdir(parents=True) + (library / "a.pdb").write_bytes(b"parts") + project["library"] = str(library.parent) + monkeypatch.setattr(exports, "VALIDATED_DLLS", {}) + monkeypatch.setattr(exports, "_LIBRARY_CACHE", {}) + calls = [] + def runner(command, cwd, environment, log_dir, **kwargs): + if "-f" in command or "-h" in command: + output = Path(next(arg.split("=", 1)[1] for arg in command if arg.startswith("-file="))) + output.write_text("iCDB Ascii Report\n" + ("Block Top\n" if "-h" in command else "Config default\n") + "\tProp Part Number USED\n") + return dict(exit_code=0, cleanup={"errors": []}) + calls.append(command) + Path(log_dir).mkdir(parents=True, exist_ok=True) + (Path(log_dir) / "stdout.txt").write_text(json.dumps({"mode": "parts", "count": 1, "records": [{"name": "USED"}]}), encoding="utf-8") + return dict(exit_code=0, cleanup={"errors": [], "processes": [{"forced_termination": True}]}) + monkeypatch.setattr(exports, "_run", runner) + first = exports.export_project(project, install, root / "first", root / "shared", jobs=2, progress=lambda _: None) + item = next(i for i in first["items"] if i["key"] == "library") + assert item["status"] == "exported" and item["validation"]["status"] == "warning" + data = next(json.loads(Path(path).read_text(encoding="utf-8")) for path in item["outputs"] if Path(path).name.startswith("used-")) + assert data["records"]["parts"][0]["name"] == "USED" and data["complete"] is False + exports.export_project(project, install, root / "retry", root / "shared", jobs=2, previous=first, progress=lambda _: None) + assert len(calls) == 2 + + +def test_parallel_cancel_drains_active_tasks_without_launching_queued_vendor_work(sample, monkeypatch): + from threading import Barrier + exports = backend() + project, install, root = sample + stop, barrier = Event(), Barrier(2) + calls = [] + def runner(command, cwd, *args, **kwargs): + calls.append(command) + barrier.wait(timeout=5) + stop.set() + return dict(exit_code=None, cancelled=True, cleanup={"errors": []}) + monkeypatch.setattr(exports, "_run", runner) + result = exports.export_project(project, install, root / "cancel", root / "shared", jobs=2, + stop_event=stop, progress=lambda _: None) + assert len(calls) == 2 and result["status"] == "cancelled" + assert all(item["status"] == "cancelled" for item in result["items"]) + + +def test_process_exit_notification_advances_without_polling_delay(tmp_path, monkeypatch): + exports = backend() + class Process: + pid = 123 + finished = False + def poll(self): return 0 if self.finished else None + def wait(self, timeout=None): + self.finished = True + return 0 + class Desktop: + cleanup_record = {"errors": [], "processes": []} + def __init__(self, *args, **kwargs): pass + def __enter__(self): return self + def __exit__(self, *args): pass + def start_process(self, *args, **kwargs): return Process() + def inspect_dialogs(self, pid): return [] + monkeypatch.setattr(exports, "PrivateDesktop", Desktop) + result = exports._run(["fixture"], tmp_path, {}, tmp_path / "logs", stop_event=None, timeout=0.15) + assert result["exit_code"] == 0 and not result["timed_out"] + + def ascii_runner(calls): def run(command, cwd, environment, log_dir, *, stop_event, timeout): calls.append(command) @@ -216,7 +373,7 @@ def test_uncertain_snapshot_cleanup_continues_exports_and_preserves_copy(sample, return dict(exit_code=0, cleanup_uncertain=True, cleanup={"errors": ["exit unconfirmed"]}) monkeypatch.setattr(exports, "_run", runner) result = exports.export_project(project, install, root / "attempt", root / "shared", progress=lambda _: None) - assert len(calls) == 3 and "-list" in calls[0] + assert len(calls) == 3 and any("-list" in call for call in calls) assert result["cleanup_uncertain"] is True assert result["status"] == "skipped" assert not any(i["status"] == "cancelled" for i in result["items"])