"""Exercise the real BOM import UI and native readers on disposable CBB copies.""" from __future__ import annotations import argparse from collections import defaultdict from dataclasses import asdict import hashlib import json import os from pathlib import Path import re import shutil import sys import time os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen') sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) from openpyxl import load_workbook from PySide6.QtCore import QThreadPool, QTimer from PySide6.QtGui import QFont, QFontDatabase from PySide6.QtWidgets import QApplication, QFileDialog from dx_productivity.trial_task_center.bom_plan import read_bom_records, write_bom_template from dx_productivity.trial_task_center.logging_setup import configure_logging from dx_productivity.trial_task_center.service import TaskCenterService from dx_productivity.trial_task_center.store import TaskStore from dx_productivity.trial_task_center.target_plan import ComponentTarget from dx_productivity.trial_task_center.ui.main_window import TrialTaskCenterWindow from private_desktop_observer import PrivateDesktopObserver IGNORED = {'PCB', 'ProjectBackup', 'CES', 'Log Files', 'HighSpeed', 'Integration', '__pycache__'} def save_json(path, value): path.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str), encoding='utf-8') def source_hashes(projects): result = {} for project in projects: for directory, folders, files in os.walk(project.parent): folders[:] = [name for name in folders if name not in IGNORED] for name in files: path = Path(directory) / name result[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest() return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--raw', type=Path, required=True) parser.add_argument('--converted-dir', type=Path, required=True) parser.add_argument('--library', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--project-copies', type=Path, help='Reuse already isolated project copies') parser.add_argument('--native-inventory', type=Path, help='Replay UI matching using captured real native inventories') args = parser.parse_args() output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) conversion = json.loads((args.converted_dir / 'conversion-report.json').read_text(encoding='utf-8')) mapped_source = Path(conversion['mapped_file']) source_projects = sorted({Path(row.project_path) for row in read_bom_records(mapped_source)}) before = source_hashes(source_projects) save_json(output / 'source-before.json', before) path_map = {} project_root = args.project_copies.resolve() if args.project_copies else output / 'projects' for source in source_projects: target = project_root / source.parent.name / source.name if not args.project_copies: shutil.copytree(source.parent, target.parent, ignore=shutil.ignore_patterns(*IGNORED)) text, count = re.subn(r'(?m)^KEY CentralLibrary .*$', lambda _: f'KEY CentralLibrary "{args.library.resolve().as_posix()}"', target.read_text(encoding='utf-8-sig')) assert count == 1 target.write_text(text, encoding='utf-8') assert target.is_file() and target.resolve().is_relative_to(project_root) assert not target.resolve().is_relative_to(source.parent) path_map[str(source).casefold()] = str(target) def remap(source, target): workbook = load_workbook(source) sheet = workbook.active for row in sheet.iter_rows(min_row=2): if row[1].value: row[1].value = path_map.get(str(row[1].value).casefold(), row[1].value) workbook.save(target) workbook.close() return target mapped = remap(mapped_source, output / 'mapped-test.xlsx') full = remap(Path(conversion['full_file']), output / 'full-test.xlsx') app = QApplication.instance() or QApplication([]) if app.platformName().casefold() == 'offscreen': font = Path(os.environ.get('WINDIR', 'C:/Windows')) / 'Fonts/msyh.ttc' if font.is_file(): font_id = QFontDatabase.addApplicationFont(str(font)) families = QFontDatabase.applicationFontFamilies(font_id) if families: app.setFont(QFont(families[0], 10)) store = TaskStore(output / 'tasks.db') service = TaskCenterService(store, configure_logging(output / 'data')) tasks = service.create_project_tasks(project_root, selected_paths=tuple(Path(path) for path in path_map.values())) if args.native_inventory: captured = json.loads(args.native_inventory.read_text(encoding='utf-8')) for task in tasks: project = service.projects(task.task_id)[0].path values = captured[str(project)] assert values, 'Replay requires an actual nonempty native inventory for every task' targets = tuple(ComponentTarget(Path(row['project_path']), row['schematic_name'], row['sheet_number'], row['uid'], row['refdes'], row['old_part_number'], row['old_value']) for row in values) store.record_target_inventory(task.task_id, targets) initial_plans = {task.task_id: tuple(store.list_target_plan(task.task_id)) for task in tasks} report = {'source': str(args.raw), 'mapped_input': str(mapped_source), 'task_count': len(tasks), 'stages': [], 'collections': [], 'errors': [], 'native_inventory_replay': bool(args.native_inventory)} window = TrialTaskCenterWindow(service) errors = [] original_error = window._show_error def show_error(error, **kwargs): errors.append({'code': getattr(error, 'code', type(error).__name__), 'message': str(error)}) original_error(error, **kwargs) window._show_error = show_error ticks, last_tick, max_gap = [0], [time.perf_counter()], [0.0] timer = QTimer() timer.setInterval(50) def tick(): now = time.perf_counter() ticks[0] += 1 max_gap[0] = max(max_gap[0], now - last_tick[0]) last_tick[0] = now timer.timeout.connect(tick) timer.start() def wait_idle(timeout=1800): deadline = time.monotonic() + timeout while (window._workers or window._loading or window._task_query.busy or window._context_query.busy or window._log_presenter.query.busy or window._feedback_query.busy): if time.monotonic() > deadline: raise TimeoutError('UI operation did not finish') app.processEvents() time.sleep(0.01) app.processEvents() inventory_by_task = {} for task in tasks: preparer = service.for_project_task(task.task_id)._quick_run_preparer original_collect = preparer.collect_inventory def collect(project, parts, task=task, original=original_collect, **controls): if args.native_inventory: raise RuntimeError('Captured-inventory UI replay must not start another native reader') if (output / 'stop.request').exists(): raise RuntimeError('Test stopped before starting the next native reader') started = time.perf_counter() print('collect_start', project.parent.name, len(parts), flush=True) try: rows = original(project, parts, **controls) inventory_by_task[task.task_id] = rows report['collections'].append({'task_id': task.task_id, 'project': str(project), 'parts': parts, 'components': len(rows), 'seconds': time.perf_counter() - started}) save_json(output / 'native-inventory.json', {key: [asdict(row) for row in values] for key, values in inventory_by_task.items()}) print('collect_done', project.parent.name, len(rows), round(time.perf_counter()-started, 3), flush=True) return rows except Exception as error: report['collections'].append({'task_id': task.task_id, 'project': str(project), 'seconds': time.perf_counter() - started, 'error': str(error)}) print('collect_failed', project.parent.name, str(error), flush=True) raise preparer.collect_inventory = collect selected_file = [str(args.raw)] original_dialog = QFileDialog.getOpenFileName QFileDialog.getOpenFileName = lambda *a, **k: (selected_file[0], 'Excel (*.xlsx)') def import_file(label, path): selected_file[0] = str(path) initial = len(errors) started = time.perf_counter() window.plan_panel.import_bom_button.click() click_seconds = time.perf_counter() - started wait_idle() batch = window.plan_panel._active_bom_batch() stage = {'stage': label, 'seconds': time.perf_counter() - started, 'click_seconds': click_seconds, 'errors': errors[initial:], 'preview_projects': len(batch.projects) if batch else 0, 'persisted_rows': sum(len(store.list_target_plan(t.task_id)) for t in tasks)} report['stages'].append(stage) save_json(output / 'report.json', report) print('stage', json.dumps(stage, ensure_ascii=False), flush=True) return batch observer = PrivateDesktopObserver(output) try: window.show() wait_idle() window.open_task(tasks[0].task_id) wait_idle() window.mapping_tabs.setCurrentIndex(1) observer.__enter__() if not args.native_inventory: assert import_file('raw_workbook', args.raw) is None assert import_file('full_conversion', full) is None assert not report['collections'] batch = import_file('mapped_conversion', mapped) if batch is None: if errors[-1]['code'] not in {'bom_import', 'target_inventory'}: raise RuntimeError('Native reader failed; diagnose this before repeating other projects') # Diagnose every project against real native results; never fabricate identities. records = read_bom_records(mapped) grouped = defaultdict(list) for record in records: grouped[record.project_path.casefold()].append(record) for task in tasks: if (output / 'stop.request').exists(): raise RuntimeError('Test stopped after native cleanup') if task.task_id in inventory_by_task: continue project = service.projects(task.task_id)[0].path parts = tuple(dict.fromkeys(r.old for r in grouped[str(project).casefold()])) window._run_async(lambda task=task, project=project, parts=parts: service.for_project_task(task.task_id)._quick_run_preparer.collect_inventory(project, parts), lambda _: None) wait_idle() problems, accepted = [], [] for number, record in enumerate(records, 2): inventory = next((inventory_by_task[t.task_id] for t in tasks if str(service.projects(t.task_id)[0].path).casefold() == record.project_path.casefold() and t.task_id in inventory_by_task), ()) matches = [sum(target.old_part_number.casefold() == record.old.casefold() and target.refdes.casefold() == ref.casefold() for target in inventory) for ref in record.refs] if any(count != 1 for count in matches): problems.append({'excel_row': number, 'project': record.project_path, 'old': record.old, 'new': record.new, 'enabled': record.enabled, 'refs': [{'refdes': ref, 'matches': count} for ref, count in zip(record.refs, matches) if count != 1]}) else: accepted.append({'project_path': record.project_path, 'old_part_number': record.old, 'new_part_number': record.new, 'refdes': ','.join(record.refs), 'enabled': record.enabled, 'sync_nc': record.sync_nc, 'custom_value': record.custom}) report['matching_problems'] = problems report['matched_candidate_rows'] = len(accepted) save_json(output / 'matching-problems.json', problems) if not accepted: raise RuntimeError('No BOM rows matched native inventories') subset = write_bom_template(output / 'native-matched-test.xlsx', accepted) batch = import_file('native_matched_subset', subset) assert batch is not None, errors assert all(tuple(store.list_target_plan(task.task_id)) == initial_plans[task.task_id] for task in tasks) report['preview_did_not_change_saved_plans'] = True for project in batch.projects: window.open_task(project.task_id) wait_idle() assert window.plan_panel.table.rows() == project.rows assert window.plan_panel.table.dirty window.grab().save(str(output / 'import-preview.png')) expected = {project.task_id: project.rows for project in batch.projects} started = time.perf_counter() window.plan_panel.save_button.click() wait_idle() assert all(tuple(store.list_target_plan(key)) == rows for key, rows in expected.items()) assert not window.plan_panel.has_unsaved_plan(expected) report['stages'].append({'stage': 'save_all_projects', 'seconds': time.perf_counter() - started, 'projects': len(expected), 'inventory_rows': sum(map(len, expected.values())), 'enabled_targets': sum(row.enabled for rows in expected.values() for row in rows)}) window.grab().save(str(output / 'saved-plan.png')) report['saved_verified'] = True report['expected_plan'] = {key: [asdict(row) for row in rows] for key, rows in expected.items()} except Exception as error: report['error'] = str(error) import traceback traceback.print_exc() finally: wait_idle() if hasattr(observer, 'thread'): observer.__exit__(None, None, None) timer.stop() report['errors'] = errors report['ui_timer_ticks'] = ticks[0] report['max_ui_tick_gap_seconds'] = max_gap[0] report['source_projects_unchanged'] = source_hashes(source_projects) == before report['replacement_executed'] = False QFileDialog.getOpenFileName = original_dialog window.close() QThreadPool.globalInstance().waitForDone() store.close() if report.get('saved_verified'): reopened = TaskStore(output / 'tasks.db') report['reopened_plan_verified'] = all( [asdict(row) for row in reopened.list_target_plan(key)] == rows for key, rows in report['expected_plan'].items()) reopened.close() report.pop('expected_plan', None) save_json(output / 'report.json', report) print(json.dumps(report, ensure_ascii=False, indent=2), flush=True) return 0 if report.get('saved_verified') and report.get('reopened_plan_verified') and report['source_projects_unchanged'] else 1 if __name__ == '__main__': raise SystemExit(main())