Files

293 lines
15 KiB
Python

"""Exercise the real BOM import UI and native readers on disposable CBB copies."""
from __future__ import annotations
import argparse
from collections import defaultdict
from dataclasses import asdict
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import sys
import time
os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen')
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
from openpyxl import load_workbook
from PySide6.QtCore import QThreadPool, QTimer
from PySide6.QtGui import QFont, QFontDatabase
from PySide6.QtWidgets import QApplication, QFileDialog
from dx_productivity.trial_task_center.bom_plan import read_bom_records, write_bom_template
from dx_productivity.trial_task_center.logging_setup import configure_logging
from dx_productivity.trial_task_center.service import TaskCenterService
from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.target_plan import ComponentTarget
from dx_productivity.trial_task_center.ui.main_window import TrialTaskCenterWindow
from private_desktop_observer import PrivateDesktopObserver
IGNORED = {'PCB', 'ProjectBackup', 'CES', 'Log Files', 'HighSpeed', 'Integration', '__pycache__'}
def save_json(path, value):
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str), encoding='utf-8')
def source_hashes(projects):
result = {}
for project in projects:
for directory, folders, files in os.walk(project.parent):
folders[:] = [name for name in folders if name not in IGNORED]
for name in files:
path = Path(directory) / name
result[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--raw', type=Path, required=True)
parser.add_argument('--converted-dir', type=Path, required=True)
parser.add_argument('--library', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--project-copies', type=Path, help='Reuse already isolated project copies')
parser.add_argument('--native-inventory', type=Path, help='Replay UI matching using captured real native inventories')
args = parser.parse_args()
output = args.output.resolve()
output.mkdir(parents=True, exist_ok=False)
conversion = json.loads((args.converted_dir / 'conversion-report.json').read_text(encoding='utf-8'))
mapped_source = Path(conversion['mapped_file'])
source_projects = sorted({Path(row.project_path) for row in read_bom_records(mapped_source)})
before = source_hashes(source_projects)
save_json(output / 'source-before.json', before)
path_map = {}
project_root = args.project_copies.resolve() if args.project_copies else output / 'projects'
for source in source_projects:
target = project_root / source.parent.name / source.name
if not args.project_copies:
shutil.copytree(source.parent, target.parent, ignore=shutil.ignore_patterns(*IGNORED))
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$',
lambda _: f'KEY CentralLibrary "{args.library.resolve().as_posix()}"',
target.read_text(encoding='utf-8-sig'))
assert count == 1
target.write_text(text, encoding='utf-8')
assert target.is_file() and target.resolve().is_relative_to(project_root)
assert not target.resolve().is_relative_to(source.parent)
path_map[str(source).casefold()] = str(target)
def remap(source, target):
workbook = load_workbook(source)
sheet = workbook.active
for row in sheet.iter_rows(min_row=2):
if row[1].value:
row[1].value = path_map.get(str(row[1].value).casefold(), row[1].value)
workbook.save(target)
workbook.close()
return target
mapped = remap(mapped_source, output / 'mapped-test.xlsx')
full = remap(Path(conversion['full_file']), output / 'full-test.xlsx')
app = QApplication.instance() or QApplication([])
if app.platformName().casefold() == 'offscreen':
font = Path(os.environ.get('WINDIR', 'C:/Windows')) / 'Fonts/msyh.ttc'
if font.is_file():
font_id = QFontDatabase.addApplicationFont(str(font))
families = QFontDatabase.applicationFontFamilies(font_id)
if families:
app.setFont(QFont(families[0], 10))
store = TaskStore(output / 'tasks.db')
service = TaskCenterService(store, configure_logging(output / 'data'))
tasks = service.create_project_tasks(project_root, selected_paths=tuple(Path(path) for path in path_map.values()))
if args.native_inventory:
captured = json.loads(args.native_inventory.read_text(encoding='utf-8'))
for task in tasks:
project = service.projects(task.task_id)[0].path
values = captured[str(project)]
assert values, 'Replay requires an actual nonempty native inventory for every task'
targets = tuple(ComponentTarget(Path(row['project_path']), row['schematic_name'], row['sheet_number'],
row['uid'], row['refdes'], row['old_part_number'], row['old_value']) for row in values)
store.record_target_inventory(task.task_id, targets)
initial_plans = {task.task_id: tuple(store.list_target_plan(task.task_id)) for task in tasks}
report = {'source': str(args.raw), 'mapped_input': str(mapped_source),
'task_count': len(tasks), 'stages': [], 'collections': [], 'errors': [],
'native_inventory_replay': bool(args.native_inventory)}
window = TrialTaskCenterWindow(service)
errors = []
original_error = window._show_error
def show_error(error, **kwargs):
errors.append({'code': getattr(error, 'code', type(error).__name__), 'message': str(error)})
original_error(error, **kwargs)
window._show_error = show_error
ticks, last_tick, max_gap = [0], [time.perf_counter()], [0.0]
timer = QTimer()
timer.setInterval(50)
def tick():
now = time.perf_counter()
ticks[0] += 1
max_gap[0] = max(max_gap[0], now - last_tick[0])
last_tick[0] = now
timer.timeout.connect(tick)
timer.start()
def wait_idle(timeout=1800):
deadline = time.monotonic() + timeout
while (window._workers or window._loading or window._task_query.busy or window._context_query.busy
or window._log_presenter.query.busy or window._feedback_query.busy):
if time.monotonic() > deadline:
raise TimeoutError('UI operation did not finish')
app.processEvents()
time.sleep(0.01)
app.processEvents()
inventory_by_task = {}
for task in tasks:
preparer = service.for_project_task(task.task_id)._quick_run_preparer
original_collect = preparer.collect_inventory
def collect(project, parts, task=task, original=original_collect, **controls):
if args.native_inventory:
raise RuntimeError('Captured-inventory UI replay must not start another native reader')
if (output / 'stop.request').exists():
raise RuntimeError('Test stopped before starting the next native reader')
started = time.perf_counter()
print('collect_start', project.parent.name, len(parts), flush=True)
try:
rows = original(project, parts, **controls)
inventory_by_task[task.task_id] = rows
report['collections'].append({'task_id': task.task_id, 'project': str(project),
'parts': parts, 'components': len(rows), 'seconds': time.perf_counter() - started})
save_json(output / 'native-inventory.json', {key: [asdict(row) for row in values]
for key, values in inventory_by_task.items()})
print('collect_done', project.parent.name, len(rows), round(time.perf_counter()-started, 3), flush=True)
return rows
except Exception as error:
report['collections'].append({'task_id': task.task_id, 'project': str(project),
'seconds': time.perf_counter() - started, 'error': str(error)})
print('collect_failed', project.parent.name, str(error), flush=True)
raise
preparer.collect_inventory = collect
selected_file = [str(args.raw)]
original_dialog = QFileDialog.getOpenFileName
QFileDialog.getOpenFileName = lambda *a, **k: (selected_file[0], 'Excel (*.xlsx)')
def import_file(label, path):
selected_file[0] = str(path)
initial = len(errors)
started = time.perf_counter()
window.plan_panel.import_bom_button.click()
click_seconds = time.perf_counter() - started
wait_idle()
batch = window.plan_panel._active_bom_batch()
stage = {'stage': label, 'seconds': time.perf_counter() - started, 'click_seconds': click_seconds,
'errors': errors[initial:], 'preview_projects': len(batch.projects) if batch else 0,
'persisted_rows': sum(len(store.list_target_plan(t.task_id)) for t in tasks)}
report['stages'].append(stage)
save_json(output / 'report.json', report)
print('stage', json.dumps(stage, ensure_ascii=False), flush=True)
return batch
observer = PrivateDesktopObserver(output)
try:
window.show()
wait_idle()
window.open_task(tasks[0].task_id)
wait_idle()
window.mapping_tabs.setCurrentIndex(1)
observer.__enter__()
if not args.native_inventory:
assert import_file('raw_workbook', args.raw) is None
assert import_file('full_conversion', full) is None
assert not report['collections']
batch = import_file('mapped_conversion', mapped)
if batch is None:
if errors[-1]['code'] not in {'bom_import', 'target_inventory'}:
raise RuntimeError('Native reader failed; diagnose this before repeating other projects')
# Diagnose every project against real native results; never fabricate identities.
records = read_bom_records(mapped)
grouped = defaultdict(list)
for record in records:
grouped[record.project_path.casefold()].append(record)
for task in tasks:
if (output / 'stop.request').exists():
raise RuntimeError('Test stopped after native cleanup')
if task.task_id in inventory_by_task:
continue
project = service.projects(task.task_id)[0].path
parts = tuple(dict.fromkeys(r.old for r in grouped[str(project).casefold()]))
window._run_async(lambda task=task, project=project, parts=parts:
service.for_project_task(task.task_id)._quick_run_preparer.collect_inventory(project, parts),
lambda _: None)
wait_idle()
problems, accepted = [], []
for number, record in enumerate(records, 2):
inventory = next((inventory_by_task[t.task_id] for t in tasks
if str(service.projects(t.task_id)[0].path).casefold() == record.project_path.casefold()
and t.task_id in inventory_by_task), ())
matches = [sum(target.old_part_number.casefold() == record.old.casefold()
and target.refdes.casefold() == ref.casefold() for target in inventory) for ref in record.refs]
if any(count != 1 for count in matches):
problems.append({'excel_row': number, 'project': record.project_path, 'old': record.old,
'new': record.new, 'enabled': record.enabled,
'refs': [{'refdes': ref, 'matches': count} for ref, count in zip(record.refs, matches) if count != 1]})
else:
accepted.append({'project_path': record.project_path, 'old_part_number': record.old,
'new_part_number': record.new, 'refdes': ','.join(record.refs),
'enabled': record.enabled, 'sync_nc': record.sync_nc, 'custom_value': record.custom})
report['matching_problems'] = problems
report['matched_candidate_rows'] = len(accepted)
save_json(output / 'matching-problems.json', problems)
if not accepted:
raise RuntimeError('No BOM rows matched native inventories')
subset = write_bom_template(output / 'native-matched-test.xlsx', accepted)
batch = import_file('native_matched_subset', subset)
assert batch is not None, errors
assert all(tuple(store.list_target_plan(task.task_id)) == initial_plans[task.task_id] for task in tasks)
report['preview_did_not_change_saved_plans'] = True
for project in batch.projects:
window.open_task(project.task_id)
wait_idle()
assert window.plan_panel.table.rows() == project.rows
assert window.plan_panel.table.dirty
window.grab().save(str(output / 'import-preview.png'))
expected = {project.task_id: project.rows for project in batch.projects}
started = time.perf_counter()
window.plan_panel.save_button.click()
wait_idle()
assert all(tuple(store.list_target_plan(key)) == rows for key, rows in expected.items())
assert not window.plan_panel.has_unsaved_plan(expected)
report['stages'].append({'stage': 'save_all_projects', 'seconds': time.perf_counter() - started,
'projects': len(expected), 'inventory_rows': sum(map(len, expected.values())),
'enabled_targets': sum(row.enabled for rows in expected.values() for row in rows)})
window.grab().save(str(output / 'saved-plan.png'))
report['saved_verified'] = True
report['expected_plan'] = {key: [asdict(row) for row in rows] for key, rows in expected.items()}
except Exception as error:
report['error'] = str(error)
import traceback
traceback.print_exc()
finally:
wait_idle()
if hasattr(observer, 'thread'):
observer.__exit__(None, None, None)
timer.stop()
report['errors'] = errors
report['ui_timer_ticks'] = ticks[0]
report['max_ui_tick_gap_seconds'] = max_gap[0]
report['source_projects_unchanged'] = source_hashes(source_projects) == before
report['replacement_executed'] = False
QFileDialog.getOpenFileName = original_dialog
window.close()
QThreadPool.globalInstance().waitForDone()
store.close()
if report.get('saved_verified'):
reopened = TaskStore(output / 'tasks.db')
report['reopened_plan_verified'] = all(
[asdict(row) for row in reopened.list_target_plan(key)] == rows
for key, rows in report['expected_plan'].items())
reopened.close()
report.pop('expected_plan', None)
save_json(output / 'report.json', report)
print(json.dumps(report, ensure_ascii=False, indent=2), flush=True)
return 0 if report.get('saved_verified') and report.get('reopened_plan_verified') and report['source_projects_unchanged'] else 1
if __name__ == '__main__':
raise SystemExit(main())