Files
dxdesigner-part-replacement/tools/probe_bom_live_roundtrip.py

378 lines
20 KiB
Python

"""Drive the Qt BOM workflow and native replacement twice on disposable copies."""
from __future__ import annotations
import argparse
import csv
from dataclasses import asdict
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import sys
import time
import traceback
os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen')
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
from openpyxl import load_workbook
from PySide6.QtCore import QThreadPool, QTimer, Qt
from PySide6.QtGui import QFont, QFontDatabase
from PySide6.QtTest import QTest
from PySide6.QtWidgets import QApplication, QDialogButtonBox, QFileDialog, QMessageBox
from dx_productivity.trial_task_center import legacy_paths
from dx_productivity.trial_task_center.logging_setup import configure_logging
from dx_productivity.trial_task_center.models import TaskStatus
from dx_productivity.trial_task_center.quick_run import QuickRunPreparer
from dx_productivity.trial_task_center.service import TaskCenterService
from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.target_plan import ComponentTarget, TargetPlanRow
from dx_productivity.trial_task_center.ui.main_window import TrialTaskCenterWindow
from dx_productivity.trial_task_center.ui.project_plan import DirectoryTaskDialog
from private_desktop_observer import PrivateDesktopObserver
OLD, NEW = '4100-CA0000-T7', '4100-CA0000-T0'
def hashes(project):
return {str(p.relative_to(project.parent)): hashlib.sha256(p.read_bytes()).hexdigest()
for p in project.parent.rglob('*') if p.is_file()}
def export_measurements(output):
"""Retain failed attempts and native substage measurements separately."""
rows = []
for task in sorted((output / 'data/tasks').iterdir()):
events = task / 'events.jsonl'
if not events.is_file():
continue
for line in events.read_text(encoding='utf-8').splitlines():
event = json.loads(line)
details = event.get('details', {})
metric = details if event['operation'] == 'execution_timing' else details.get('timing')
if not metric:
continue
base = dict(task_id=task.name, timestamp_utc=event['timestamp_utc'], sequence=event['sequence'],
job_id=metric.get('job_id', ''), state=metric.get('state', ''),
project_path=metric.get('project_path', ''), phase=metric['phase'],
seconds=metric['elapsed_seconds'], unit=metric.get('unit', ''),
completed_units=metric.get('completed_units', ''), units_per_minute=metric.get('units_per_minute', ''))
rows.append(base | {'layer': 'controller'})
rows.extend(base | {'layer': 'native_worker', 'phase': phase, 'seconds': seconds,
'unit': '', 'completed_units': '', 'units_per_minute': ''}
for phase, seconds in metric.get('worker_stage_seconds', {}).items())
target = output / 'execution-measurements.csv'
if rows:
with target.open('w', encoding='utf-8-sig', newline='') as stream:
writer = csv.DictWriter(stream, fieldnames=tuple(rows[0]))
writer.writeheader()
writer.writerows(rows)
return target
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--project-root', type=Path, required=True)
parser.add_argument('--library', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--run-native', action='store_true', required=True)
parser.add_argument('--resume', action='store_true', help='Recover interrupted tasks or retry safe failures in this disposable run')
parser.add_argument('--verify-only', action='store_true', help='With --resume, only independently read completed round-two projects')
args = parser.parse_args()
if args.verify_only and not args.resume:
parser.error('--verify-only requires --resume')
output = args.output.resolve()
output.mkdir(parents=True, exist_ok=args.resume)
previous = json.loads((output / 'report.json').read_text(encoding='utf-8')) if args.resume else None
if previous and any(step['stage'] == '07-native-second-round' for step in previous['steps']):
args.verify_only = True # A readback retry must never repeat completed replacements.
library = args.library.resolve(strict=True)
originals, original_hashes, projects = [], {}, []
project_root = output / 'projects'
for index, name in enumerate(('Audio_D2128', 'Audio_T605A')):
source = (args.project_root / name / (name + '.prj')).resolve(strict=True)
originals.append(source)
original_hashes[str(source)] = previous['source_hashes'][str(source)] if previous else hashes(source)
folder = project_root / name
if index == 0:
folder = folder / ('long-project-' + 'x' * 70) / ('revision-' + 'y' * 70)
assert not folder.resolve().is_relative_to(source.parent)
if not args.resume:
shutil.copytree(source.parent, folder)
project = folder / source.name
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$',
lambda _: f'KEY CentralLibrary "{library.as_posix()}"', project.read_text(encoding='utf-8-sig'))
assert count == 1
if not args.resume:
project.write_text(text, encoding='utf-8')
projects.append(project)
assert len(str(projects[0])) > 260
excel_root = output / ('bom-files-' + 'a' * 70) / ('workbook-' + 'b' * 70)
excel_root.mkdir(parents=True, exist_ok=args.resume)
template = excel_root / 'default-template.xlsx'
assert len(str(template)) > 260
app = QApplication.instance() or QApplication([])
font = Path(os.environ.get('WINDIR', 'C:/Windows')) / 'Fonts/msyh.ttc'
if app.platformName() == 'offscreen' and font.is_file():
font_id = QFontDatabase.addApplicationFont(str(font))
families = QFontDatabase.applicationFontFamilies(font_id)
if families:
app.setFont(QFont(families[0], 10))
store = TaskStore(output / 'tasks.db')
service = TaskCenterService(store, configure_logging(output / 'data'))
window = TrialTaskCenterWindow(service, quick_run_confirmer=lambda *_: True,
close_plan_confirmer=lambda: 'discard')
report = previous or {'project_path_lengths': [len(str(p)) for p in projects], 'bom_path_length': len(str(template)),
'steps': [], 'errors': [], 'source_hashes': original_hashes, 'started': time.time()}
report.pop('error', None)
report.pop('verified', None)
def record(stage, started=None, **details):
entry = {'stage': stage, **details}
if started is not None:
entry['seconds'] = time.perf_counter() - started
report['steps'].append(entry)
(output / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding='utf-8')
brief = {k: v for k, v in entry.items() if k != 'report'}
if 'report' in entry:
brief['result'] = {key: entry['report'].get(key) for key in ('total_rows', 'passed_rows', 'failed_rows', 'unchecked_rows')}
print(json.dumps(brief, ensure_ascii=False, default=str), flush=True)
original_show_error = window._show_error
def show_error(error, **kwargs):
report['errors'].append(str(error))
original_show_error(error, **kwargs)
window._show_error = show_error
def idle(timeout=1800):
deadline = time.monotonic() + timeout
while (window._workers or window._loading or window._task_query.busy or window._context_query.busy
or window._log_presenter.query.busy or window._feedback_query.busy):
if time.monotonic() > deadline:
raise TimeoutError('UI operation exceeded native probe deadline')
app.processEvents()
time.sleep(.02)
app.processEvents()
def click(button):
assert button.isEnabled(), button.objectName()
QTest.mouseClick(button, Qt.MouseButton.LeftButton)
def screenshot(name):
window.grab().save(str(output / (name + '.png')))
original_open, original_save = QFileDialog.getOpenFileName, QFileDialog.getSaveFileName
original_question = QMessageBox.question
short_path = legacy_paths._short_path
legacy_paths._short_path = lambda _: None # Exercise junction fallback without OS setting changes.
selection = [str(template)]
QFileDialog.getOpenFileName = lambda *_a, **_k: (selection[0], 'Excel (*.xlsx)')
QFileDialog.getSaveFileName = lambda *_a, **_k: (str(template), 'Excel (*.xlsx)')
QMessageBox.question = lambda *_a, **_k: QMessageBox.StandardButton.Yes
dialog_errors = []
def drive_creation():
try:
dialog = app.activeModalWidget()
assert isinstance(dialog, DirectoryTaskDialog)
dialog.directory.setText(str(project_root))
click(dialog.scan)
def finish_scan():
if dialog._worker is not None:
QTimer.singleShot(30, finish_scan)
return
selected = {str(p.resolve()).casefold() for p in projects}
for i in range(dialog.projects.count()):
item = dialog.projects.item(i)
path = Path(item.data(Qt.ItemDataRole.UserRole)).resolve()
item.setCheckState(Qt.CheckState.Checked if str(path).casefold() in selected else Qt.CheckState.Unchecked)
try:
click(dialog.buttons.button(QDialogButtonBox.StandardButton.Ok))
except Exception as error:
dialog_errors.append(str(error))
dialog.reject()
finish_scan()
except Exception as error:
dialog_errors.append(str(error))
if app.activeModalWidget():
app.activeModalWidget().reject()
def import_file(path, label):
selection[0] = str(path)
start = time.perf_counter()
click(window.plan_panel.import_bom_button)
idle()
result = window.plan_panel.last_bom_report
assert result is not None, report['errors']
record(label, start, report=asdict(result) | {'total_rows': result.total_rows,
'passed_rows': result.passed_rows, 'failed_rows': result.failed_rows,
'unchecked_rows': result.unchecked_rows})
screenshot(label)
return result
def fill(round_number, initial=None, bad=False):
book = load_workbook(template)
sheet = book.active
headers = [c.value for c in sheet[1]]
originals_by_path = {str(p.resolve()).casefold(): p for p in projects}
scaffolds = [dict(zip(headers, values)) for values in sheet.iter_rows(min_row=2, values_only=True)]
sheet.delete_rows(2, sheet.max_row)
for row in scaffolds:
path = str(Path(row['project_path']).resolve()).casefold()
assert path in originals_by_path
row.update(old_part_number=OLD if round_number == 1 else NEW,
new_part_number=NEW if round_number == 1 else OLD, refdes='RV05', enabled=True, sync_nc=True)
sheet.append([row.get(h, '') for h in headers])
if initial:
kept = [t.target.refdes for t in initial[path] if t.target.refdes != 'RV05']
if kept:
unchanged = row | {'old_part_number': OLD, 'new_part_number': NEW,
'refdes': ','.join(kept), 'enabled': False}
sheet.append([unchanged.get(h, '') for h in headers])
if bad:
row = scaffolds[0] | {'old_part_number': OLD, 'new_part_number': NEW,
'refdes': 'R_DOES_NOT_EXIST', 'enabled': True}
sheet.append([row.get(h, '') for h in headers])
target = excel_root / f'round-{round_number}-{"invalid" if bad else "filled"}.xlsx'
book.save(target)
book.close()
return target
try:
window.show()
idle()
started = time.perf_counter()
if not args.resume:
QTimer.singleShot(80, drive_creation)
click(window.task_list.new_button)
idle()
assert not dialog_errors, dialog_errors
tasks = service.tasks()
assert len(tasks) == 2
if args.resume:
for task in tasks:
service.quick_run_progress(task.task_id)
window.open_task(tasks[0].task_id)
idle()
window.mapping_tabs.setCurrentIndex(1)
if not args.resume:
record('create_tasks_through_dialog', started, task_ids=[t.task_id for t in tasks])
click(window.plan_panel.export_bom_button)
idle()
assert template.is_file()
if not args.resume:
workbook = load_workbook(template, read_only=True)
record('export_default_template', columns=workbook.active.max_column, path=str(template))
workbook.close()
screenshot('01-template-exported')
with PrivateDesktopObserver(output):
if args.verify_only:
from dx_productivity.trial_task_center.new_round import latest_archive
initial = {}
for task in tasks:
assert service.get_task(task.task_id).status is TaskStatus.COMPLETED
facade = service.for_project_task(task.task_id)
archive = latest_archive(facade._quick_run_preparer)
assert archive is not None
plan = json.loads((archive / 'plan.json').read_text(encoding='utf-8'))
rows = tuple(TargetPlanRow(**(row | {'target': ComponentTarget(**(row['target'] |
{'project_path': Path(row['target']['project_path'])}))})) for row in plan['targets'])
initial[str(service.projects(task.task_id)[0].path.resolve()).casefold()] = rows
else:
if not args.resume:
failed = import_file(fill(1, bad=True), '02-import-with-failed-row')
assert failed.passed_rows == 2 and failed.failed_rows == 1 and failed.preview is None
assert not window.plan_panel._bom_batches
assert all(not service.target_plan(t.task_id) for t in tasks)
success = import_file(fill(1), '03-import-success')
assert success.preview and not success.failed_rows and success.passed_rows == 2
click(window.plan_panel.save_button)
idle()
initial = {str(service.projects(t.task_id)[0].path.resolve()).casefold(): service.target_plan(t.task_id) for t in tasks}
assert all(sum(r.enabled for r in rows) == 1 for rows in initial.values())
started = time.perf_counter()
if args.resume:
for task in tasks:
if service.get_task(task.task_id).status is TaskStatus.COMPLETED:
continue
window.open_task(task.task_id)
idle()
if service.get_task(task.task_id).status is TaskStatus.COMPLETED:
continue
if service.has_interrupted_quick_run(task.task_id):
click(window.run_panel.recover_button)
idle()
action = 'recover'
else:
click(window.run_panel.retry_button)
idle()
click(window.run_panel.quick_run_button)
idle()
action = 'retry'
record(f'04-{action}-native-task', started, task_id=task.task_id,
status=service.get_task(task.task_id).status)
else:
click(window.task_list.queue_button)
idle()
assert all(service.get_task(t.task_id).status is TaskStatus.COMPLETED for t in tasks), {
'statuses': {t.task_id: str(service.get_task(t.task_id).status) for t in tasks}, 'errors': report['errors']}
record('04-native-first-round', started, statuses={t.task_id: service.get_task(t.task_id).status for t in tasks})
screenshot('04-first-round-completed')
started = time.perf_counter()
click(window.run_panel.new_round_button)
idle()
assert all(service.get_task(t.task_id).status is TaskStatus.DRAFT for t in tasks)
assert all(not service.target_plan(t.task_id) for t in tasks)
record('05-start-new-round', started)
second = import_file(fill(2, initial=initial), '06-import-current-engineering-state')
assert second.preview and not second.failed_rows
for preview in second.preview.projects:
for row in preview.rows:
assert row.target.old_part_number == (NEW if row.target.refdes == 'RV05' else OLD)
click(window.plan_panel.save_button)
idle()
started = time.perf_counter()
click(window.task_list.queue_button)
idle()
assert all(service.get_task(t.task_id).status is TaskStatus.COMPLETED for t in tasks), report['errors']
record('07-native-second-round', started)
screenshot('07-second-round-completed')
for task in tasks:
project = service.projects(task.task_id)[0].path
started = time.perf_counter()
result = QuickRunPreparer(output / 'independent-read')._discover_project_sheets(
project, (OLD, NEW), collect_components=True)
before = {r.target.uid: r.target for r in initial[str(project.resolve()).casefold()]}
after = {r.uid: r for r in result.components}
assert set(before) == set(after)
assert all(after[uid].refdes == row.refdes and after[uid].old_part_number == OLD
and (row.refdes == 'RV05' or after[uid].old_value == row.old_value)
for uid, row in before.items()), {
uid: {'before': asdict(row), 'after': asdict(after[uid])} for uid, row in before.items()}
record('08-independent-native-readback', started, project=str(project),
components=[asdict(r) for r in result.components])
observation = json.loads((output / 'desktop-observation.json').read_text(encoding='utf-8'))
assert not observation['observer_errors']
assert not observation['owned_cad_became_foreground'] and not observation['owned_cad_on_input_desktop']
assert all(observation['cad_exit_confirmed'].values()) and all(observation['desktop_absent'].values())
report['verified'] = True
except Exception as error:
report['error'] = str(error)
traceback.print_exc()
finally:
idle()
report['source_projects_unchanged'] = all(hashes(p) == original_hashes[str(p)] for p in originals)
QFileDialog.getOpenFileName, QFileDialog.getSaveFileName = original_open, original_save
QMessageBox.question = original_question
window.close()
QThreadPool.globalInstance().waitForDone()
store.close()
legacy_paths._short_path = short_path
links = tuple(legacy_paths._aliases.links)
report['path_aliases'] = [str(p) for p in links]
report['alias_cleanup_errors'] = [str(e) for e in legacy_paths.cleanup_worker_aliases()]
report['aliases_removed'] = all(not p.exists() for p in links)
report['elapsed_seconds'] = time.time() - report['started']
report['measurements'] = str(export_measurements(output))
(output / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding='utf-8')
print(json.dumps({k: report[k] for k in ('verified', 'error', 'source_projects_unchanged', 'aliases_removed') if k in report}), flush=True)
return 0 if report.get('verified') and report['source_projects_unchanged'] and report['aliases_removed'] else 1
if __name__ == '__main__':
raise SystemExit(main())