378 lines
20 KiB
Python
378 lines
20 KiB
Python
"""Drive the Qt BOM workflow and native replacement twice on disposable copies."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
from dataclasses import asdict
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import traceback
|
|
|
|
os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen')
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
|
|
|
|
from openpyxl import load_workbook
|
|
from PySide6.QtCore import QThreadPool, QTimer, Qt
|
|
from PySide6.QtGui import QFont, QFontDatabase
|
|
from PySide6.QtTest import QTest
|
|
from PySide6.QtWidgets import QApplication, QDialogButtonBox, QFileDialog, QMessageBox
|
|
|
|
from dx_productivity.trial_task_center import legacy_paths
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.models import TaskStatus
|
|
from dx_productivity.trial_task_center.quick_run import QuickRunPreparer
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
from dx_productivity.trial_task_center.target_plan import ComponentTarget, TargetPlanRow
|
|
from dx_productivity.trial_task_center.ui.main_window import TrialTaskCenterWindow
|
|
from dx_productivity.trial_task_center.ui.project_plan import DirectoryTaskDialog
|
|
from private_desktop_observer import PrivateDesktopObserver
|
|
|
|
|
|
OLD, NEW = '4100-CA0000-T7', '4100-CA0000-T0'
|
|
|
|
|
|
def hashes(project):
|
|
return {str(p.relative_to(project.parent)): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in project.parent.rglob('*') if p.is_file()}
|
|
|
|
|
|
def export_measurements(output):
|
|
"""Retain failed attempts and native substage measurements separately."""
|
|
rows = []
|
|
for task in sorted((output / 'data/tasks').iterdir()):
|
|
events = task / 'events.jsonl'
|
|
if not events.is_file():
|
|
continue
|
|
for line in events.read_text(encoding='utf-8').splitlines():
|
|
event = json.loads(line)
|
|
details = event.get('details', {})
|
|
metric = details if event['operation'] == 'execution_timing' else details.get('timing')
|
|
if not metric:
|
|
continue
|
|
base = dict(task_id=task.name, timestamp_utc=event['timestamp_utc'], sequence=event['sequence'],
|
|
job_id=metric.get('job_id', ''), state=metric.get('state', ''),
|
|
project_path=metric.get('project_path', ''), phase=metric['phase'],
|
|
seconds=metric['elapsed_seconds'], unit=metric.get('unit', ''),
|
|
completed_units=metric.get('completed_units', ''), units_per_minute=metric.get('units_per_minute', ''))
|
|
rows.append(base | {'layer': 'controller'})
|
|
rows.extend(base | {'layer': 'native_worker', 'phase': phase, 'seconds': seconds,
|
|
'unit': '', 'completed_units': '', 'units_per_minute': ''}
|
|
for phase, seconds in metric.get('worker_stage_seconds', {}).items())
|
|
target = output / 'execution-measurements.csv'
|
|
if rows:
|
|
with target.open('w', encoding='utf-8-sig', newline='') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=tuple(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
return target
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--project-root', type=Path, required=True)
|
|
parser.add_argument('--library', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--run-native', action='store_true', required=True)
|
|
parser.add_argument('--resume', action='store_true', help='Recover interrupted tasks or retry safe failures in this disposable run')
|
|
parser.add_argument('--verify-only', action='store_true', help='With --resume, only independently read completed round-two projects')
|
|
args = parser.parse_args()
|
|
if args.verify_only and not args.resume:
|
|
parser.error('--verify-only requires --resume')
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=args.resume)
|
|
previous = json.loads((output / 'report.json').read_text(encoding='utf-8')) if args.resume else None
|
|
if previous and any(step['stage'] == '07-native-second-round' for step in previous['steps']):
|
|
args.verify_only = True # A readback retry must never repeat completed replacements.
|
|
library = args.library.resolve(strict=True)
|
|
originals, original_hashes, projects = [], {}, []
|
|
project_root = output / 'projects'
|
|
for index, name in enumerate(('Audio_D2128', 'Audio_T605A')):
|
|
source = (args.project_root / name / (name + '.prj')).resolve(strict=True)
|
|
originals.append(source)
|
|
original_hashes[str(source)] = previous['source_hashes'][str(source)] if previous else hashes(source)
|
|
folder = project_root / name
|
|
if index == 0:
|
|
folder = folder / ('long-project-' + 'x' * 70) / ('revision-' + 'y' * 70)
|
|
assert not folder.resolve().is_relative_to(source.parent)
|
|
if not args.resume:
|
|
shutil.copytree(source.parent, folder)
|
|
project = folder / source.name
|
|
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$',
|
|
lambda _: f'KEY CentralLibrary "{library.as_posix()}"', project.read_text(encoding='utf-8-sig'))
|
|
assert count == 1
|
|
if not args.resume:
|
|
project.write_text(text, encoding='utf-8')
|
|
projects.append(project)
|
|
assert len(str(projects[0])) > 260
|
|
excel_root = output / ('bom-files-' + 'a' * 70) / ('workbook-' + 'b' * 70)
|
|
excel_root.mkdir(parents=True, exist_ok=args.resume)
|
|
template = excel_root / 'default-template.xlsx'
|
|
assert len(str(template)) > 260
|
|
app = QApplication.instance() or QApplication([])
|
|
font = Path(os.environ.get('WINDIR', 'C:/Windows')) / 'Fonts/msyh.ttc'
|
|
if app.platformName() == 'offscreen' and font.is_file():
|
|
font_id = QFontDatabase.addApplicationFont(str(font))
|
|
families = QFontDatabase.applicationFontFamilies(font_id)
|
|
if families:
|
|
app.setFont(QFont(families[0], 10))
|
|
store = TaskStore(output / 'tasks.db')
|
|
service = TaskCenterService(store, configure_logging(output / 'data'))
|
|
window = TrialTaskCenterWindow(service, quick_run_confirmer=lambda *_: True,
|
|
close_plan_confirmer=lambda: 'discard')
|
|
report = previous or {'project_path_lengths': [len(str(p)) for p in projects], 'bom_path_length': len(str(template)),
|
|
'steps': [], 'errors': [], 'source_hashes': original_hashes, 'started': time.time()}
|
|
report.pop('error', None)
|
|
report.pop('verified', None)
|
|
def record(stage, started=None, **details):
|
|
entry = {'stage': stage, **details}
|
|
if started is not None:
|
|
entry['seconds'] = time.perf_counter() - started
|
|
report['steps'].append(entry)
|
|
(output / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding='utf-8')
|
|
brief = {k: v for k, v in entry.items() if k != 'report'}
|
|
if 'report' in entry:
|
|
brief['result'] = {key: entry['report'].get(key) for key in ('total_rows', 'passed_rows', 'failed_rows', 'unchecked_rows')}
|
|
print(json.dumps(brief, ensure_ascii=False, default=str), flush=True)
|
|
original_show_error = window._show_error
|
|
def show_error(error, **kwargs):
|
|
report['errors'].append(str(error))
|
|
original_show_error(error, **kwargs)
|
|
window._show_error = show_error
|
|
def idle(timeout=1800):
|
|
deadline = time.monotonic() + timeout
|
|
while (window._workers or window._loading or window._task_query.busy or window._context_query.busy
|
|
or window._log_presenter.query.busy or window._feedback_query.busy):
|
|
if time.monotonic() > deadline:
|
|
raise TimeoutError('UI operation exceeded native probe deadline')
|
|
app.processEvents()
|
|
time.sleep(.02)
|
|
app.processEvents()
|
|
def click(button):
|
|
assert button.isEnabled(), button.objectName()
|
|
QTest.mouseClick(button, Qt.MouseButton.LeftButton)
|
|
def screenshot(name):
|
|
window.grab().save(str(output / (name + '.png')))
|
|
original_open, original_save = QFileDialog.getOpenFileName, QFileDialog.getSaveFileName
|
|
original_question = QMessageBox.question
|
|
short_path = legacy_paths._short_path
|
|
legacy_paths._short_path = lambda _: None # Exercise junction fallback without OS setting changes.
|
|
selection = [str(template)]
|
|
QFileDialog.getOpenFileName = lambda *_a, **_k: (selection[0], 'Excel (*.xlsx)')
|
|
QFileDialog.getSaveFileName = lambda *_a, **_k: (str(template), 'Excel (*.xlsx)')
|
|
QMessageBox.question = lambda *_a, **_k: QMessageBox.StandardButton.Yes
|
|
dialog_errors = []
|
|
def drive_creation():
|
|
try:
|
|
dialog = app.activeModalWidget()
|
|
assert isinstance(dialog, DirectoryTaskDialog)
|
|
dialog.directory.setText(str(project_root))
|
|
click(dialog.scan)
|
|
def finish_scan():
|
|
if dialog._worker is not None:
|
|
QTimer.singleShot(30, finish_scan)
|
|
return
|
|
selected = {str(p.resolve()).casefold() for p in projects}
|
|
for i in range(dialog.projects.count()):
|
|
item = dialog.projects.item(i)
|
|
path = Path(item.data(Qt.ItemDataRole.UserRole)).resolve()
|
|
item.setCheckState(Qt.CheckState.Checked if str(path).casefold() in selected else Qt.CheckState.Unchecked)
|
|
try:
|
|
click(dialog.buttons.button(QDialogButtonBox.StandardButton.Ok))
|
|
except Exception as error:
|
|
dialog_errors.append(str(error))
|
|
dialog.reject()
|
|
finish_scan()
|
|
except Exception as error:
|
|
dialog_errors.append(str(error))
|
|
if app.activeModalWidget():
|
|
app.activeModalWidget().reject()
|
|
def import_file(path, label):
|
|
selection[0] = str(path)
|
|
start = time.perf_counter()
|
|
click(window.plan_panel.import_bom_button)
|
|
idle()
|
|
result = window.plan_panel.last_bom_report
|
|
assert result is not None, report['errors']
|
|
record(label, start, report=asdict(result) | {'total_rows': result.total_rows,
|
|
'passed_rows': result.passed_rows, 'failed_rows': result.failed_rows,
|
|
'unchecked_rows': result.unchecked_rows})
|
|
screenshot(label)
|
|
return result
|
|
def fill(round_number, initial=None, bad=False):
|
|
book = load_workbook(template)
|
|
sheet = book.active
|
|
headers = [c.value for c in sheet[1]]
|
|
originals_by_path = {str(p.resolve()).casefold(): p for p in projects}
|
|
scaffolds = [dict(zip(headers, values)) for values in sheet.iter_rows(min_row=2, values_only=True)]
|
|
sheet.delete_rows(2, sheet.max_row)
|
|
for row in scaffolds:
|
|
path = str(Path(row['project_path']).resolve()).casefold()
|
|
assert path in originals_by_path
|
|
row.update(old_part_number=OLD if round_number == 1 else NEW,
|
|
new_part_number=NEW if round_number == 1 else OLD, refdes='RV05', enabled=True, sync_nc=True)
|
|
sheet.append([row.get(h, '') for h in headers])
|
|
if initial:
|
|
kept = [t.target.refdes for t in initial[path] if t.target.refdes != 'RV05']
|
|
if kept:
|
|
unchanged = row | {'old_part_number': OLD, 'new_part_number': NEW,
|
|
'refdes': ','.join(kept), 'enabled': False}
|
|
sheet.append([unchanged.get(h, '') for h in headers])
|
|
if bad:
|
|
row = scaffolds[0] | {'old_part_number': OLD, 'new_part_number': NEW,
|
|
'refdes': 'R_DOES_NOT_EXIST', 'enabled': True}
|
|
sheet.append([row.get(h, '') for h in headers])
|
|
target = excel_root / f'round-{round_number}-{"invalid" if bad else "filled"}.xlsx'
|
|
book.save(target)
|
|
book.close()
|
|
return target
|
|
try:
|
|
window.show()
|
|
idle()
|
|
started = time.perf_counter()
|
|
if not args.resume:
|
|
QTimer.singleShot(80, drive_creation)
|
|
click(window.task_list.new_button)
|
|
idle()
|
|
assert not dialog_errors, dialog_errors
|
|
tasks = service.tasks()
|
|
assert len(tasks) == 2
|
|
if args.resume:
|
|
for task in tasks:
|
|
service.quick_run_progress(task.task_id)
|
|
window.open_task(tasks[0].task_id)
|
|
idle()
|
|
window.mapping_tabs.setCurrentIndex(1)
|
|
if not args.resume:
|
|
record('create_tasks_through_dialog', started, task_ids=[t.task_id for t in tasks])
|
|
click(window.plan_panel.export_bom_button)
|
|
idle()
|
|
assert template.is_file()
|
|
if not args.resume:
|
|
workbook = load_workbook(template, read_only=True)
|
|
record('export_default_template', columns=workbook.active.max_column, path=str(template))
|
|
workbook.close()
|
|
screenshot('01-template-exported')
|
|
with PrivateDesktopObserver(output):
|
|
if args.verify_only:
|
|
from dx_productivity.trial_task_center.new_round import latest_archive
|
|
initial = {}
|
|
for task in tasks:
|
|
assert service.get_task(task.task_id).status is TaskStatus.COMPLETED
|
|
facade = service.for_project_task(task.task_id)
|
|
archive = latest_archive(facade._quick_run_preparer)
|
|
assert archive is not None
|
|
plan = json.loads((archive / 'plan.json').read_text(encoding='utf-8'))
|
|
rows = tuple(TargetPlanRow(**(row | {'target': ComponentTarget(**(row['target'] |
|
|
{'project_path': Path(row['target']['project_path'])}))})) for row in plan['targets'])
|
|
initial[str(service.projects(task.task_id)[0].path.resolve()).casefold()] = rows
|
|
else:
|
|
if not args.resume:
|
|
failed = import_file(fill(1, bad=True), '02-import-with-failed-row')
|
|
assert failed.passed_rows == 2 and failed.failed_rows == 1 and failed.preview is None
|
|
assert not window.plan_panel._bom_batches
|
|
assert all(not service.target_plan(t.task_id) for t in tasks)
|
|
success = import_file(fill(1), '03-import-success')
|
|
assert success.preview and not success.failed_rows and success.passed_rows == 2
|
|
click(window.plan_panel.save_button)
|
|
idle()
|
|
initial = {str(service.projects(t.task_id)[0].path.resolve()).casefold(): service.target_plan(t.task_id) for t in tasks}
|
|
assert all(sum(r.enabled for r in rows) == 1 for rows in initial.values())
|
|
started = time.perf_counter()
|
|
if args.resume:
|
|
for task in tasks:
|
|
if service.get_task(task.task_id).status is TaskStatus.COMPLETED:
|
|
continue
|
|
window.open_task(task.task_id)
|
|
idle()
|
|
if service.get_task(task.task_id).status is TaskStatus.COMPLETED:
|
|
continue
|
|
if service.has_interrupted_quick_run(task.task_id):
|
|
click(window.run_panel.recover_button)
|
|
idle()
|
|
action = 'recover'
|
|
else:
|
|
click(window.run_panel.retry_button)
|
|
idle()
|
|
click(window.run_panel.quick_run_button)
|
|
idle()
|
|
action = 'retry'
|
|
record(f'04-{action}-native-task', started, task_id=task.task_id,
|
|
status=service.get_task(task.task_id).status)
|
|
else:
|
|
click(window.task_list.queue_button)
|
|
idle()
|
|
assert all(service.get_task(t.task_id).status is TaskStatus.COMPLETED for t in tasks), {
|
|
'statuses': {t.task_id: str(service.get_task(t.task_id).status) for t in tasks}, 'errors': report['errors']}
|
|
record('04-native-first-round', started, statuses={t.task_id: service.get_task(t.task_id).status for t in tasks})
|
|
screenshot('04-first-round-completed')
|
|
started = time.perf_counter()
|
|
click(window.run_panel.new_round_button)
|
|
idle()
|
|
assert all(service.get_task(t.task_id).status is TaskStatus.DRAFT for t in tasks)
|
|
assert all(not service.target_plan(t.task_id) for t in tasks)
|
|
record('05-start-new-round', started)
|
|
second = import_file(fill(2, initial=initial), '06-import-current-engineering-state')
|
|
assert second.preview and not second.failed_rows
|
|
for preview in second.preview.projects:
|
|
for row in preview.rows:
|
|
assert row.target.old_part_number == (NEW if row.target.refdes == 'RV05' else OLD)
|
|
click(window.plan_panel.save_button)
|
|
idle()
|
|
started = time.perf_counter()
|
|
click(window.task_list.queue_button)
|
|
idle()
|
|
assert all(service.get_task(t.task_id).status is TaskStatus.COMPLETED for t in tasks), report['errors']
|
|
record('07-native-second-round', started)
|
|
screenshot('07-second-round-completed')
|
|
for task in tasks:
|
|
project = service.projects(task.task_id)[0].path
|
|
started = time.perf_counter()
|
|
result = QuickRunPreparer(output / 'independent-read')._discover_project_sheets(
|
|
project, (OLD, NEW), collect_components=True)
|
|
before = {r.target.uid: r.target for r in initial[str(project.resolve()).casefold()]}
|
|
after = {r.uid: r for r in result.components}
|
|
assert set(before) == set(after)
|
|
assert all(after[uid].refdes == row.refdes and after[uid].old_part_number == OLD
|
|
and (row.refdes == 'RV05' or after[uid].old_value == row.old_value)
|
|
for uid, row in before.items()), {
|
|
uid: {'before': asdict(row), 'after': asdict(after[uid])} for uid, row in before.items()}
|
|
record('08-independent-native-readback', started, project=str(project),
|
|
components=[asdict(r) for r in result.components])
|
|
observation = json.loads((output / 'desktop-observation.json').read_text(encoding='utf-8'))
|
|
assert not observation['observer_errors']
|
|
assert not observation['owned_cad_became_foreground'] and not observation['owned_cad_on_input_desktop']
|
|
assert all(observation['cad_exit_confirmed'].values()) and all(observation['desktop_absent'].values())
|
|
report['verified'] = True
|
|
except Exception as error:
|
|
report['error'] = str(error)
|
|
traceback.print_exc()
|
|
finally:
|
|
idle()
|
|
report['source_projects_unchanged'] = all(hashes(p) == original_hashes[str(p)] for p in originals)
|
|
QFileDialog.getOpenFileName, QFileDialog.getSaveFileName = original_open, original_save
|
|
QMessageBox.question = original_question
|
|
window.close()
|
|
QThreadPool.globalInstance().waitForDone()
|
|
store.close()
|
|
legacy_paths._short_path = short_path
|
|
links = tuple(legacy_paths._aliases.links)
|
|
report['path_aliases'] = [str(p) for p in links]
|
|
report['alias_cleanup_errors'] = [str(e) for e in legacy_paths.cleanup_worker_aliases()]
|
|
report['aliases_removed'] = all(not p.exists() for p in links)
|
|
report['elapsed_seconds'] = time.time() - report['started']
|
|
report['measurements'] = str(export_measurements(output))
|
|
(output / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding='utf-8')
|
|
print(json.dumps({k: report[k] for k in ('verified', 'error', 'source_projects_unchanged', 'aliases_removed') if k in report}), flush=True)
|
|
return 0 if report.get('verified') and report['source_projects_unchanged'] and report['aliases_removed'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|