272 lines
15 KiB
Python
272 lines
15 KiB
Python
"""Hybrid source parent / frozen supervisor startup-failure CAD acceptance.
|
|
|
|
Prepare without CAD with --help. Run only after obtaining the shared CAD test
|
|
window. Production source/VBS are unchanged; parent _command alone is replaced.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from dataclasses import asdict
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from threading import Event, Thread
|
|
import time
|
|
import traceback
|
|
from uuid import uuid4
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO / 'src'))
|
|
sys.path.insert(0, str(REPO / 'tools'))
|
|
from probe_batch_runtime_live import hashes, compare_saved, save
|
|
|
|
|
|
def read_json(path):
|
|
try:
|
|
return json.loads(path.read_text(encoding='utf-8'))
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
class StartupFault:
|
|
"""Stop only certified own viewdraw PIDs while their session is starting."""
|
|
def __init__(self, root, output, failures, cancel):
|
|
self.root, self.output, self.failures, self.cancel = root, output, failures, cancel
|
|
self.stop = Event()
|
|
self.faults, self.observations, self.errors = [], [], []
|
|
self.seen = set()
|
|
|
|
def scan(self):
|
|
import win32api
|
|
import win32event
|
|
import win32process
|
|
from dx_productivity.trial_task_center.isolated_desktop import _Win32Api
|
|
from dx_productivity.trial_task_center.path_access import internal_path
|
|
for request in (self.root / 'batch-resources').glob('*/request.json'):
|
|
value = read_json(request)
|
|
if not value or value['operation'] != 'execute_task_queue':
|
|
continue
|
|
directory = request.parent
|
|
session = read_json(directory / 'session.json')
|
|
journal = read_json(directory / 'supervisor.json')
|
|
if not session or not journal or not journal.get('process'):
|
|
continue
|
|
assert session['session_id'] == directory.name == value['batch_id']
|
|
for record_path in directory.glob('generation-*/private-desktop.json'):
|
|
record = read_json(record_path)
|
|
if not record or not record.get('cad_pid'):
|
|
continue
|
|
pid = record['cad_pid']
|
|
key = (directory.name, record['token'])
|
|
if key in self.seen:
|
|
continue
|
|
generation = int(record_path.parent.name.removeprefix('generation-'))
|
|
if generation != session['generation'] or record['desktop'] != session['desktop']:
|
|
continue
|
|
identities = [item for item in session['processes'] if item['pid'] == pid]
|
|
if not identities:
|
|
continue
|
|
assert len(identities) == 1 and identities[0]['creation_time']
|
|
identity = identities[0]
|
|
handle = win32api.OpenProcess(0x1000 | 0x100000 | 1, False, pid)
|
|
try:
|
|
actual_created = str(win32process.GetProcessTimes(handle)['CreationTime'])
|
|
assert actual_created == identity['creation_time'], 'PID creation time mismatch'
|
|
listed = _Win32Api().service.GetProcessWindowStation().EnumDesktops()
|
|
observation = {'generation': generation, 'pid': pid,
|
|
'creation_time': actual_created, 'desktop': record['desktop'],
|
|
'listed_desktops': [name for name in listed if name.startswith('dx-')],
|
|
'state': record['state'], 'time_ns': time.time_ns()}
|
|
self.observations.append(observation)
|
|
if generation <= self.failures:
|
|
# Recheck the same record immediately before the one
|
|
# authorized termination. No COM attachment is needed.
|
|
current = read_json(record_path)
|
|
live_session = read_json(directory / 'session.json')
|
|
assert current and live_session
|
|
assert current['state'] == 'starting' and 'guid' not in current
|
|
assert current['cad_pid'] == pid and current['token'] == record['token']
|
|
assert current['desktop'] == live_session['desktop']
|
|
assert live_session['generation'] == generation
|
|
assert not list(record_path.parent.glob('cad-binding-*.json'))
|
|
assert not list(directory.parent.parent.rglob('cad-ready-*.json'))
|
|
prepared = list(directory.glob(f'attempts/g{generation}/**/*.json'))
|
|
assert len(prepared) == 1
|
|
marker = read_json(prepared[0])
|
|
assert marker and marker['state'] == 'prepared'
|
|
# The initial record is written immediately after
|
|
# launch, before bootstrap/project worker authorization.
|
|
assert win32event.WaitForSingleObject(handle, 0) == 258
|
|
win32process.TerminateProcess(handle, 71)
|
|
assert win32event.WaitForSingleObject(handle, 5000) == 0
|
|
self.faults.append({**observation, 'terminated_owned_startup_only': True,
|
|
'marker': str(prepared[0]), 'marker_before': marker,
|
|
'marker_sha256_before': hashlib.sha256(prepared[0].read_bytes()).hexdigest()})
|
|
self.seen.add(key)
|
|
save(self.output / 'fault-observations.json', {'faults': self.faults,
|
|
'observations': self.observations, 'errors': self.errors})
|
|
finally:
|
|
handle.Close()
|
|
|
|
def _watch(self):
|
|
while not self.stop.wait(.02):
|
|
try:
|
|
self.scan()
|
|
except Exception as error:
|
|
self.errors.append(repr(error))
|
|
self.cancel.set()
|
|
save(self.output / 'fault-observations.json', {'faults': self.faults,
|
|
'observations': self.observations, 'errors': self.errors})
|
|
return
|
|
|
|
def __enter__(self):
|
|
self.thread = Thread(target=self._watch, daemon=True)
|
|
self.thread.start()
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
self.stop.set()
|
|
self.thread.join(5)
|
|
assert not self.thread.is_alive()
|
|
|
|
|
|
def run(source, library, executable, base, failures):
|
|
from dx_productivity.trial_task_center import batch_supervisor
|
|
from dx_productivity.trial_task_center.batch_resources import pending_resources
|
|
from dx_productivity.trial_task_center.bom_plan import write_bom_template
|
|
from dx_productivity.trial_task_center.isolated_cad import IsolatedCadSession
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
from dx_productivity.trial_task_center.task_queue import execute_task_queue
|
|
from private_desktop_observer import PrivateDesktopObserver
|
|
|
|
source, library, executable = [path.resolve(strict=True) for path in (source, library, executable)]
|
|
assert executable.name == 'DxPartReplace.exe'
|
|
output = base.resolve() / ('run-' + uuid4().hex)
|
|
assert output.is_relative_to(REPO / 'build' / 'issue-3-frozen-startup-recovery')
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
source_hashes, library_hash = hashes(source), hashlib.sha256(library.read_bytes()).hexdigest()
|
|
folder = output / 'projects' / source.name
|
|
shutil.copytree(source, folder)
|
|
project = folder / (source.name + '.prj')
|
|
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$', lambda _: f'KEY CentralLibrary "{library.as_posix()}"',
|
|
project.read_text(encoding='utf-8-sig'))
|
|
assert count == 1
|
|
project.write_text(text, encoding='utf-8')
|
|
report = {'mode': 'source_supervised_parent_frozen_worker', 'gui_parent_covered': False,
|
|
'frozen_executable': str(executable), 'exe_sha256': hashlib.sha256(executable.read_bytes()).hexdigest(),
|
|
'project': str(project), 'requested_startup_failures': failures, 'launches': []}
|
|
original_command = batch_supervisor._command
|
|
def frozen_command(request):
|
|
command = (str(executable), '--batch-worker', str(request))
|
|
report['launches'].append({'command': command, 'request': read_json(request)})
|
|
save(output / 'report.json', report)
|
|
return command
|
|
batch_supervisor._command = frozen_command
|
|
store = TaskStore(output / 'tasks.db')
|
|
service = TaskCenterService(store, configure_logging(output / 'data'), supervised=True)
|
|
def independent(label):
|
|
evidence = output / 'independent-read' / 'quick-run-history' / ('sheet-discovery-' + label)
|
|
evidence.mkdir(parents=True)
|
|
destination, spec = output / (label + '.json'), evidence / 'spec.json'
|
|
save(spec, {'project': str(project), 'destination': str(destination)})
|
|
with IsolatedCadSession(evidence) as cad:
|
|
process = cad.start_worker((sys.executable, str(REPO / 'tools/probe_marker_write_failure_live.py'),
|
|
'--inspect-file', str(spec)), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
try:
|
|
code = process.wait(timeout=240)
|
|
(evidence / 'worker-output.txt').write_bytes(process.stdout.read() + b'\n' + process.stderr.read())
|
|
assert code == 0, (evidence / 'worker-output.txt').read_text(errors='replace')
|
|
finally:
|
|
process.close()
|
|
return read_json(destination)
|
|
started = time.monotonic()
|
|
try:
|
|
with PrivateDesktopObserver(output):
|
|
print('Independent BEFORE (source read-only inspector)', flush=True)
|
|
before = independent('before')
|
|
tasks = service.create_project_tasks(output / 'projects', selected_paths=(project,))
|
|
assert len(tasks) == 1
|
|
bom = write_bom_template(output / 'input.xlsx', ({'project_path': str(project),
|
|
'old_part_number': '4100-CA0000-T7', 'new_part_number': '4100-CA0000-T0',
|
|
'refdes': 'RV05', 'enabled': True, 'sync_nc': True},))
|
|
print('BOM import in frozen supervisor', flush=True)
|
|
imported = service.inspect_bom_batch(tasks[0].task_id, bom,
|
|
on_progress=lambda message: print(message, flush=True))
|
|
assert imported.passed_rows == 1 and imported.preview is not None
|
|
service.save_bom_batch(imported.preview, {item.task_id: item.rows for item in imported.preview.projects})
|
|
cancel = Event()
|
|
print('Frozen queue with owned startup CAD failure', flush=True)
|
|
with StartupFault(output / 'data', output, failures, cancel) as injector:
|
|
result = execute_task_queue(service, (tasks[0].task_id,), stop_event=cancel)
|
|
report['queue_result'] = asdict(result)
|
|
report['faults'], report['fault_errors'] = injector.faults, injector.errors
|
|
assert not injector.errors and len(injector.faults) == failures
|
|
execution = [item for item in report['launches'] if item['request']['operation'] == 'execute_task_queue']
|
|
assert len(execution) == 1
|
|
request = Path(execution[0]['command'][-1])
|
|
session = read_json(request.parent / 'session.json')
|
|
report['execution_session'] = session
|
|
assert session['state'] == 'released' and session['generation'] == 2
|
|
assert len(session['retries']) == 1 and next(iter(session['retries'].values()))['count'] == 1
|
|
for fault in injector.faults:
|
|
path = Path(fault['marker'])
|
|
assert read_json(path)['state'] == 'prepared'
|
|
assert hashlib.sha256(path.read_bytes()).hexdigest() == fault['marker_sha256_before']
|
|
assert not pending_resources(output / 'data')
|
|
assert not batch_supervisor.pending_supervisors(output / 'data')
|
|
first_release, last_release = session['generation_releases'][0], session['generation_releases'][-1]
|
|
assert first_release['cleanup']['desktop_confirmation']['released']
|
|
second_create = last_release['cleanup']['desktop_handle_history'][0]['time_ns']
|
|
assert first_release['released_at'] * 1e9 <= second_create
|
|
report['old_desktop_released_before_second_create'] = True
|
|
status_files = {str(path.relative_to(output)): path.read_text(encoding='utf-8-sig')
|
|
for path in (output / 'data').rglob('status.tsv')}
|
|
report['status_files'] = status_files
|
|
saved_count = sum(sum(line.split('\t')[1:2] == ['saved_ok'] for line in text.splitlines())
|
|
for text in status_files.values())
|
|
assert saved_count == (1 if failures == 1 else 0)
|
|
assert all(entry.state == ('completed' if failures == 1 else 'blocked') for entry in result.entries)
|
|
print('Independent AFTER (source read-only inspector)', flush=True)
|
|
after = independent('after')
|
|
report['graph_comparison'] = compare_saved(before, after) if failures == 1 else {'identical': before == after}
|
|
if failures == 2:
|
|
assert before == after
|
|
observation = read_json(output / 'desktop-observation.json')
|
|
assert not observation['observer_errors']
|
|
assert observation['input_desktop_before'] == observation['input_desktop_after']
|
|
assert not observation['owned_cad_became_foreground'] and not observation['owned_cad_on_input_desktop']
|
|
assert all(observation['cad_exit_confirmed'].values()) and all(observation['desktop_absent'].values())
|
|
report['verified'] = True
|
|
except BaseException as error:
|
|
report.update(verified=False, error=repr(error), traceback=traceback.format_exc())
|
|
traceback.print_exc()
|
|
finally:
|
|
batch_supervisor._command = original_command
|
|
store.close()
|
|
from dx_productivity.trial_task_center.legacy_paths import cleanup_worker_aliases
|
|
report['alias_cleanup_errors'] = list(map(str, cleanup_worker_aliases()))
|
|
report['source_unchanged'] = hashes(source) == source_hashes
|
|
report['library_unchanged'] = hashlib.sha256(library.read_bytes()).hexdigest() == library_hash
|
|
report['elapsed_seconds'] = time.monotonic() - started
|
|
save(output / 'report.json', report)
|
|
print(json.dumps({key: report.get(key) for key in ('verified', 'error', 'elapsed_seconds')}, ensure_ascii=False), flush=True)
|
|
return 0 if (report.get('verified') and report['source_unchanged'] and report['library_unchanged']
|
|
and not report['alias_cleanup_errors']) else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--source', type=Path, required=True)
|
|
parser.add_argument('--library', type=Path, required=True)
|
|
parser.add_argument('--executable', type=Path, default=REPO / 'dist/DxPartReplace/DxPartReplace.exe')
|
|
parser.add_argument('--output', type=Path, default=REPO / 'build/issue-3-frozen-startup-recovery')
|
|
parser.add_argument('--failures', type=int, choices=(1, 2), default=1)
|
|
args = parser.parse_args()
|
|
raise SystemExit(run(args.source, args.library, args.executable, args.output, args.failures))
|