293 lines
15 KiB
Python
293 lines
15 KiB
Python
"""Exercise the real BOM import UI and native readers on disposable CBB copies."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import defaultdict
|
|
from dataclasses import asdict
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import time
|
|
|
|
os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen')
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
|
|
|
|
from openpyxl import load_workbook
|
|
from PySide6.QtCore import QThreadPool, QTimer
|
|
from PySide6.QtGui import QFont, QFontDatabase
|
|
from PySide6.QtWidgets import QApplication, QFileDialog
|
|
|
|
from dx_productivity.trial_task_center.bom_plan import read_bom_records, write_bom_template
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
from dx_productivity.trial_task_center.target_plan import ComponentTarget
|
|
from dx_productivity.trial_task_center.ui.main_window import TrialTaskCenterWindow
|
|
from private_desktop_observer import PrivateDesktopObserver
|
|
|
|
|
|
IGNORED = {'PCB', 'ProjectBackup', 'CES', 'Log Files', 'HighSpeed', 'Integration', '__pycache__'}
|
|
|
|
|
|
def save_json(path, value):
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str), encoding='utf-8')
|
|
|
|
|
|
def source_hashes(projects):
|
|
result = {}
|
|
for project in projects:
|
|
for directory, folders, files in os.walk(project.parent):
|
|
folders[:] = [name for name in folders if name not in IGNORED]
|
|
for name in files:
|
|
path = Path(directory) / name
|
|
result[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--raw', type=Path, required=True)
|
|
parser.add_argument('--converted-dir', type=Path, required=True)
|
|
parser.add_argument('--library', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--project-copies', type=Path, help='Reuse already isolated project copies')
|
|
parser.add_argument('--native-inventory', type=Path, help='Replay UI matching using captured real native inventories')
|
|
args = parser.parse_args()
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
conversion = json.loads((args.converted_dir / 'conversion-report.json').read_text(encoding='utf-8'))
|
|
mapped_source = Path(conversion['mapped_file'])
|
|
source_projects = sorted({Path(row.project_path) for row in read_bom_records(mapped_source)})
|
|
before = source_hashes(source_projects)
|
|
save_json(output / 'source-before.json', before)
|
|
path_map = {}
|
|
project_root = args.project_copies.resolve() if args.project_copies else output / 'projects'
|
|
for source in source_projects:
|
|
target = project_root / source.parent.name / source.name
|
|
if not args.project_copies:
|
|
shutil.copytree(source.parent, target.parent, ignore=shutil.ignore_patterns(*IGNORED))
|
|
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$',
|
|
lambda _: f'KEY CentralLibrary "{args.library.resolve().as_posix()}"',
|
|
target.read_text(encoding='utf-8-sig'))
|
|
assert count == 1
|
|
target.write_text(text, encoding='utf-8')
|
|
assert target.is_file() and target.resolve().is_relative_to(project_root)
|
|
assert not target.resolve().is_relative_to(source.parent)
|
|
path_map[str(source).casefold()] = str(target)
|
|
def remap(source, target):
|
|
workbook = load_workbook(source)
|
|
sheet = workbook.active
|
|
for row in sheet.iter_rows(min_row=2):
|
|
if row[1].value:
|
|
row[1].value = path_map.get(str(row[1].value).casefold(), row[1].value)
|
|
workbook.save(target)
|
|
workbook.close()
|
|
return target
|
|
mapped = remap(mapped_source, output / 'mapped-test.xlsx')
|
|
full = remap(Path(conversion['full_file']), output / 'full-test.xlsx')
|
|
app = QApplication.instance() or QApplication([])
|
|
if app.platformName().casefold() == 'offscreen':
|
|
font = Path(os.environ.get('WINDIR', 'C:/Windows')) / 'Fonts/msyh.ttc'
|
|
if font.is_file():
|
|
font_id = QFontDatabase.addApplicationFont(str(font))
|
|
families = QFontDatabase.applicationFontFamilies(font_id)
|
|
if families:
|
|
app.setFont(QFont(families[0], 10))
|
|
store = TaskStore(output / 'tasks.db')
|
|
service = TaskCenterService(store, configure_logging(output / 'data'))
|
|
tasks = service.create_project_tasks(project_root, selected_paths=tuple(Path(path) for path in path_map.values()))
|
|
if args.native_inventory:
|
|
captured = json.loads(args.native_inventory.read_text(encoding='utf-8'))
|
|
for task in tasks:
|
|
project = service.projects(task.task_id)[0].path
|
|
values = captured[str(project)]
|
|
assert values, 'Replay requires an actual nonempty native inventory for every task'
|
|
targets = tuple(ComponentTarget(Path(row['project_path']), row['schematic_name'], row['sheet_number'],
|
|
row['uid'], row['refdes'], row['old_part_number'], row['old_value']) for row in values)
|
|
store.record_target_inventory(task.task_id, targets)
|
|
initial_plans = {task.task_id: tuple(store.list_target_plan(task.task_id)) for task in tasks}
|
|
report = {'source': str(args.raw), 'mapped_input': str(mapped_source),
|
|
'task_count': len(tasks), 'stages': [], 'collections': [], 'errors': [],
|
|
'native_inventory_replay': bool(args.native_inventory)}
|
|
window = TrialTaskCenterWindow(service)
|
|
errors = []
|
|
original_error = window._show_error
|
|
def show_error(error, **kwargs):
|
|
errors.append({'code': getattr(error, 'code', type(error).__name__), 'message': str(error)})
|
|
original_error(error, **kwargs)
|
|
window._show_error = show_error
|
|
ticks, last_tick, max_gap = [0], [time.perf_counter()], [0.0]
|
|
timer = QTimer()
|
|
timer.setInterval(50)
|
|
def tick():
|
|
now = time.perf_counter()
|
|
ticks[0] += 1
|
|
max_gap[0] = max(max_gap[0], now - last_tick[0])
|
|
last_tick[0] = now
|
|
timer.timeout.connect(tick)
|
|
timer.start()
|
|
def wait_idle(timeout=1800):
|
|
deadline = time.monotonic() + timeout
|
|
while (window._workers or window._loading or window._task_query.busy or window._context_query.busy
|
|
or window._log_presenter.query.busy or window._feedback_query.busy):
|
|
if time.monotonic() > deadline:
|
|
raise TimeoutError('UI operation did not finish')
|
|
app.processEvents()
|
|
time.sleep(0.01)
|
|
app.processEvents()
|
|
inventory_by_task = {}
|
|
for task in tasks:
|
|
preparer = service.for_project_task(task.task_id)._quick_run_preparer
|
|
original_collect = preparer.collect_inventory
|
|
def collect(project, parts, task=task, original=original_collect, **controls):
|
|
if args.native_inventory:
|
|
raise RuntimeError('Captured-inventory UI replay must not start another native reader')
|
|
if (output / 'stop.request').exists():
|
|
raise RuntimeError('Test stopped before starting the next native reader')
|
|
started = time.perf_counter()
|
|
print('collect_start', project.parent.name, len(parts), flush=True)
|
|
try:
|
|
rows = original(project, parts, **controls)
|
|
inventory_by_task[task.task_id] = rows
|
|
report['collections'].append({'task_id': task.task_id, 'project': str(project),
|
|
'parts': parts, 'components': len(rows), 'seconds': time.perf_counter() - started})
|
|
save_json(output / 'native-inventory.json', {key: [asdict(row) for row in values]
|
|
for key, values in inventory_by_task.items()})
|
|
print('collect_done', project.parent.name, len(rows), round(time.perf_counter()-started, 3), flush=True)
|
|
return rows
|
|
except Exception as error:
|
|
report['collections'].append({'task_id': task.task_id, 'project': str(project),
|
|
'seconds': time.perf_counter() - started, 'error': str(error)})
|
|
print('collect_failed', project.parent.name, str(error), flush=True)
|
|
raise
|
|
preparer.collect_inventory = collect
|
|
selected_file = [str(args.raw)]
|
|
original_dialog = QFileDialog.getOpenFileName
|
|
QFileDialog.getOpenFileName = lambda *a, **k: (selected_file[0], 'Excel (*.xlsx)')
|
|
def import_file(label, path):
|
|
selected_file[0] = str(path)
|
|
initial = len(errors)
|
|
started = time.perf_counter()
|
|
window.plan_panel.import_bom_button.click()
|
|
click_seconds = time.perf_counter() - started
|
|
wait_idle()
|
|
batch = window.plan_panel._active_bom_batch()
|
|
stage = {'stage': label, 'seconds': time.perf_counter() - started, 'click_seconds': click_seconds,
|
|
'errors': errors[initial:], 'preview_projects': len(batch.projects) if batch else 0,
|
|
'persisted_rows': sum(len(store.list_target_plan(t.task_id)) for t in tasks)}
|
|
report['stages'].append(stage)
|
|
save_json(output / 'report.json', report)
|
|
print('stage', json.dumps(stage, ensure_ascii=False), flush=True)
|
|
return batch
|
|
observer = PrivateDesktopObserver(output)
|
|
try:
|
|
window.show()
|
|
wait_idle()
|
|
window.open_task(tasks[0].task_id)
|
|
wait_idle()
|
|
window.mapping_tabs.setCurrentIndex(1)
|
|
observer.__enter__()
|
|
if not args.native_inventory:
|
|
assert import_file('raw_workbook', args.raw) is None
|
|
assert import_file('full_conversion', full) is None
|
|
assert not report['collections']
|
|
batch = import_file('mapped_conversion', mapped)
|
|
if batch is None:
|
|
if errors[-1]['code'] not in {'bom_import', 'target_inventory'}:
|
|
raise RuntimeError('Native reader failed; diagnose this before repeating other projects')
|
|
# Diagnose every project against real native results; never fabricate identities.
|
|
records = read_bom_records(mapped)
|
|
grouped = defaultdict(list)
|
|
for record in records:
|
|
grouped[record.project_path.casefold()].append(record)
|
|
for task in tasks:
|
|
if (output / 'stop.request').exists():
|
|
raise RuntimeError('Test stopped after native cleanup')
|
|
if task.task_id in inventory_by_task:
|
|
continue
|
|
project = service.projects(task.task_id)[0].path
|
|
parts = tuple(dict.fromkeys(r.old for r in grouped[str(project).casefold()]))
|
|
window._run_async(lambda task=task, project=project, parts=parts:
|
|
service.for_project_task(task.task_id)._quick_run_preparer.collect_inventory(project, parts),
|
|
lambda _: None)
|
|
wait_idle()
|
|
problems, accepted = [], []
|
|
for number, record in enumerate(records, 2):
|
|
inventory = next((inventory_by_task[t.task_id] for t in tasks
|
|
if str(service.projects(t.task_id)[0].path).casefold() == record.project_path.casefold()
|
|
and t.task_id in inventory_by_task), ())
|
|
matches = [sum(target.old_part_number.casefold() == record.old.casefold()
|
|
and target.refdes.casefold() == ref.casefold() for target in inventory) for ref in record.refs]
|
|
if any(count != 1 for count in matches):
|
|
problems.append({'excel_row': number, 'project': record.project_path, 'old': record.old,
|
|
'new': record.new, 'enabled': record.enabled,
|
|
'refs': [{'refdes': ref, 'matches': count} for ref, count in zip(record.refs, matches) if count != 1]})
|
|
else:
|
|
accepted.append({'project_path': record.project_path, 'old_part_number': record.old,
|
|
'new_part_number': record.new, 'refdes': ','.join(record.refs),
|
|
'enabled': record.enabled, 'sync_nc': record.sync_nc, 'custom_value': record.custom})
|
|
report['matching_problems'] = problems
|
|
report['matched_candidate_rows'] = len(accepted)
|
|
save_json(output / 'matching-problems.json', problems)
|
|
if not accepted:
|
|
raise RuntimeError('No BOM rows matched native inventories')
|
|
subset = write_bom_template(output / 'native-matched-test.xlsx', accepted)
|
|
batch = import_file('native_matched_subset', subset)
|
|
assert batch is not None, errors
|
|
assert all(tuple(store.list_target_plan(task.task_id)) == initial_plans[task.task_id] for task in tasks)
|
|
report['preview_did_not_change_saved_plans'] = True
|
|
for project in batch.projects:
|
|
window.open_task(project.task_id)
|
|
wait_idle()
|
|
assert window.plan_panel.table.rows() == project.rows
|
|
assert window.plan_panel.table.dirty
|
|
window.grab().save(str(output / 'import-preview.png'))
|
|
expected = {project.task_id: project.rows for project in batch.projects}
|
|
started = time.perf_counter()
|
|
window.plan_panel.save_button.click()
|
|
wait_idle()
|
|
assert all(tuple(store.list_target_plan(key)) == rows for key, rows in expected.items())
|
|
assert not window.plan_panel.has_unsaved_plan(expected)
|
|
report['stages'].append({'stage': 'save_all_projects', 'seconds': time.perf_counter() - started,
|
|
'projects': len(expected), 'inventory_rows': sum(map(len, expected.values())),
|
|
'enabled_targets': sum(row.enabled for rows in expected.values() for row in rows)})
|
|
window.grab().save(str(output / 'saved-plan.png'))
|
|
report['saved_verified'] = True
|
|
report['expected_plan'] = {key: [asdict(row) for row in rows] for key, rows in expected.items()}
|
|
except Exception as error:
|
|
report['error'] = str(error)
|
|
import traceback
|
|
traceback.print_exc()
|
|
finally:
|
|
wait_idle()
|
|
if hasattr(observer, 'thread'):
|
|
observer.__exit__(None, None, None)
|
|
timer.stop()
|
|
report['errors'] = errors
|
|
report['ui_timer_ticks'] = ticks[0]
|
|
report['max_ui_tick_gap_seconds'] = max_gap[0]
|
|
report['source_projects_unchanged'] = source_hashes(source_projects) == before
|
|
report['replacement_executed'] = False
|
|
QFileDialog.getOpenFileName = original_dialog
|
|
window.close()
|
|
QThreadPool.globalInstance().waitForDone()
|
|
store.close()
|
|
if report.get('saved_verified'):
|
|
reopened = TaskStore(output / 'tasks.db')
|
|
report['reopened_plan_verified'] = all(
|
|
[asdict(row) for row in reopened.list_target_plan(key)] == rows
|
|
for key, rows in report['expected_plan'].items())
|
|
reopened.close()
|
|
report.pop('expected_plan', None)
|
|
save_json(output / 'report.json', report)
|
|
print(json.dumps(report, ensure_ascii=False, indent=2), flush=True)
|
|
return 0 if report.get('saved_verified') and report.get('reopened_plan_verified') and report['source_projects_unchanged'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|