Files
dxdesigner-part-replacement/tools/probe_frozen_startup_recovery.py

272 lines
15 KiB
Python

"""Hybrid source parent / frozen supervisor startup-failure CAD acceptance.
Prepare without CAD with --help. Run only after obtaining the shared CAD test
window. Production source/VBS are unchanged; parent _command alone is replaced.
"""
from __future__ import annotations
import argparse
from dataclasses import asdict
import hashlib
import json
from pathlib import Path
import re
import shutil
import subprocess
import sys
from threading import Event, Thread
import time
import traceback
from uuid import uuid4
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / 'src'))
sys.path.insert(0, str(REPO / 'tools'))
from probe_batch_runtime_live import hashes, compare_saved, save
def read_json(path):
try:
return json.loads(path.read_text(encoding='utf-8'))
except (OSError, ValueError):
return None
class StartupFault:
"""Stop only certified own viewdraw PIDs while their session is starting."""
def __init__(self, root, output, failures, cancel):
self.root, self.output, self.failures, self.cancel = root, output, failures, cancel
self.stop = Event()
self.faults, self.observations, self.errors = [], [], []
self.seen = set()
def scan(self):
import win32api
import win32event
import win32process
from dx_productivity.trial_task_center.isolated_desktop import _Win32Api
from dx_productivity.trial_task_center.path_access import internal_path
for request in (self.root / 'batch-resources').glob('*/request.json'):
value = read_json(request)
if not value or value['operation'] != 'execute_task_queue':
continue
directory = request.parent
session = read_json(directory / 'session.json')
journal = read_json(directory / 'supervisor.json')
if not session or not journal or not journal.get('process'):
continue
assert session['session_id'] == directory.name == value['batch_id']
for record_path in directory.glob('generation-*/private-desktop.json'):
record = read_json(record_path)
if not record or not record.get('cad_pid'):
continue
pid = record['cad_pid']
key = (directory.name, record['token'])
if key in self.seen:
continue
generation = int(record_path.parent.name.removeprefix('generation-'))
if generation != session['generation'] or record['desktop'] != session['desktop']:
continue
identities = [item for item in session['processes'] if item['pid'] == pid]
if not identities:
continue
assert len(identities) == 1 and identities[0]['creation_time']
identity = identities[0]
handle = win32api.OpenProcess(0x1000 | 0x100000 | 1, False, pid)
try:
actual_created = str(win32process.GetProcessTimes(handle)['CreationTime'])
assert actual_created == identity['creation_time'], 'PID creation time mismatch'
listed = _Win32Api().service.GetProcessWindowStation().EnumDesktops()
observation = {'generation': generation, 'pid': pid,
'creation_time': actual_created, 'desktop': record['desktop'],
'listed_desktops': [name for name in listed if name.startswith('dx-')],
'state': record['state'], 'time_ns': time.time_ns()}
self.observations.append(observation)
if generation <= self.failures:
# Recheck the same record immediately before the one
# authorized termination. No COM attachment is needed.
current = read_json(record_path)
live_session = read_json(directory / 'session.json')
assert current and live_session
assert current['state'] == 'starting' and 'guid' not in current
assert current['cad_pid'] == pid and current['token'] == record['token']
assert current['desktop'] == live_session['desktop']
assert live_session['generation'] == generation
assert not list(record_path.parent.glob('cad-binding-*.json'))
assert not list(directory.parent.parent.rglob('cad-ready-*.json'))
prepared = list(directory.glob(f'attempts/g{generation}/**/*.json'))
assert len(prepared) == 1
marker = read_json(prepared[0])
assert marker and marker['state'] == 'prepared'
# The initial record is written immediately after
# launch, before bootstrap/project worker authorization.
assert win32event.WaitForSingleObject(handle, 0) == 258
win32process.TerminateProcess(handle, 71)
assert win32event.WaitForSingleObject(handle, 5000) == 0
self.faults.append({**observation, 'terminated_owned_startup_only': True,
'marker': str(prepared[0]), 'marker_before': marker,
'marker_sha256_before': hashlib.sha256(prepared[0].read_bytes()).hexdigest()})
self.seen.add(key)
save(self.output / 'fault-observations.json', {'faults': self.faults,
'observations': self.observations, 'errors': self.errors})
finally:
handle.Close()
def _watch(self):
while not self.stop.wait(.02):
try:
self.scan()
except Exception as error:
self.errors.append(repr(error))
self.cancel.set()
save(self.output / 'fault-observations.json', {'faults': self.faults,
'observations': self.observations, 'errors': self.errors})
return
def __enter__(self):
self.thread = Thread(target=self._watch, daemon=True)
self.thread.start()
return self
def __exit__(self, *_):
self.stop.set()
self.thread.join(5)
assert not self.thread.is_alive()
def run(source, library, executable, base, failures):
from dx_productivity.trial_task_center import batch_supervisor
from dx_productivity.trial_task_center.batch_resources import pending_resources
from dx_productivity.trial_task_center.bom_plan import write_bom_template
from dx_productivity.trial_task_center.isolated_cad import IsolatedCadSession
from dx_productivity.trial_task_center.logging_setup import configure_logging
from dx_productivity.trial_task_center.service import TaskCenterService
from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.task_queue import execute_task_queue
from private_desktop_observer import PrivateDesktopObserver
source, library, executable = [path.resolve(strict=True) for path in (source, library, executable)]
assert executable.name == 'DxPartReplace.exe'
output = base.resolve() / ('run-' + uuid4().hex)
assert output.is_relative_to(REPO / 'build' / 'issue-3-frozen-startup-recovery')
output.mkdir(parents=True, exist_ok=False)
source_hashes, library_hash = hashes(source), hashlib.sha256(library.read_bytes()).hexdigest()
folder = output / 'projects' / source.name
shutil.copytree(source, folder)
project = folder / (source.name + '.prj')
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$', lambda _: f'KEY CentralLibrary "{library.as_posix()}"',
project.read_text(encoding='utf-8-sig'))
assert count == 1
project.write_text(text, encoding='utf-8')
report = {'mode': 'source_supervised_parent_frozen_worker', 'gui_parent_covered': False,
'frozen_executable': str(executable), 'exe_sha256': hashlib.sha256(executable.read_bytes()).hexdigest(),
'project': str(project), 'requested_startup_failures': failures, 'launches': []}
original_command = batch_supervisor._command
def frozen_command(request):
command = (str(executable), '--batch-worker', str(request))
report['launches'].append({'command': command, 'request': read_json(request)})
save(output / 'report.json', report)
return command
batch_supervisor._command = frozen_command
store = TaskStore(output / 'tasks.db')
service = TaskCenterService(store, configure_logging(output / 'data'), supervised=True)
def independent(label):
evidence = output / 'independent-read' / 'quick-run-history' / ('sheet-discovery-' + label)
evidence.mkdir(parents=True)
destination, spec = output / (label + '.json'), evidence / 'spec.json'
save(spec, {'project': str(project), 'destination': str(destination)})
with IsolatedCadSession(evidence) as cad:
process = cad.start_worker((sys.executable, str(REPO / 'tools/probe_marker_write_failure_live.py'),
'--inspect-file', str(spec)), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
code = process.wait(timeout=240)
(evidence / 'worker-output.txt').write_bytes(process.stdout.read() + b'\n' + process.stderr.read())
assert code == 0, (evidence / 'worker-output.txt').read_text(errors='replace')
finally:
process.close()
return read_json(destination)
started = time.monotonic()
try:
with PrivateDesktopObserver(output):
print('Independent BEFORE (source read-only inspector)', flush=True)
before = independent('before')
tasks = service.create_project_tasks(output / 'projects', selected_paths=(project,))
assert len(tasks) == 1
bom = write_bom_template(output / 'input.xlsx', ({'project_path': str(project),
'old_part_number': '4100-CA0000-T7', 'new_part_number': '4100-CA0000-T0',
'refdes': 'RV05', 'enabled': True, 'sync_nc': True},))
print('BOM import in frozen supervisor', flush=True)
imported = service.inspect_bom_batch(tasks[0].task_id, bom,
on_progress=lambda message: print(message, flush=True))
assert imported.passed_rows == 1 and imported.preview is not None
service.save_bom_batch(imported.preview, {item.task_id: item.rows for item in imported.preview.projects})
cancel = Event()
print('Frozen queue with owned startup CAD failure', flush=True)
with StartupFault(output / 'data', output, failures, cancel) as injector:
result = execute_task_queue(service, (tasks[0].task_id,), stop_event=cancel)
report['queue_result'] = asdict(result)
report['faults'], report['fault_errors'] = injector.faults, injector.errors
assert not injector.errors and len(injector.faults) == failures
execution = [item for item in report['launches'] if item['request']['operation'] == 'execute_task_queue']
assert len(execution) == 1
request = Path(execution[0]['command'][-1])
session = read_json(request.parent / 'session.json')
report['execution_session'] = session
assert session['state'] == 'released' and session['generation'] == 2
assert len(session['retries']) == 1 and next(iter(session['retries'].values()))['count'] == 1
for fault in injector.faults:
path = Path(fault['marker'])
assert read_json(path)['state'] == 'prepared'
assert hashlib.sha256(path.read_bytes()).hexdigest() == fault['marker_sha256_before']
assert not pending_resources(output / 'data')
assert not batch_supervisor.pending_supervisors(output / 'data')
first_release, last_release = session['generation_releases'][0], session['generation_releases'][-1]
assert first_release['cleanup']['desktop_confirmation']['released']
second_create = last_release['cleanup']['desktop_handle_history'][0]['time_ns']
assert first_release['released_at'] * 1e9 <= second_create
report['old_desktop_released_before_second_create'] = True
status_files = {str(path.relative_to(output)): path.read_text(encoding='utf-8-sig')
for path in (output / 'data').rglob('status.tsv')}
report['status_files'] = status_files
saved_count = sum(sum(line.split('\t')[1:2] == ['saved_ok'] for line in text.splitlines())
for text in status_files.values())
assert saved_count == (1 if failures == 1 else 0)
assert all(entry.state == ('completed' if failures == 1 else 'blocked') for entry in result.entries)
print('Independent AFTER (source read-only inspector)', flush=True)
after = independent('after')
report['graph_comparison'] = compare_saved(before, after) if failures == 1 else {'identical': before == after}
if failures == 2:
assert before == after
observation = read_json(output / 'desktop-observation.json')
assert not observation['observer_errors']
assert observation['input_desktop_before'] == observation['input_desktop_after']
assert not observation['owned_cad_became_foreground'] and not observation['owned_cad_on_input_desktop']
assert all(observation['cad_exit_confirmed'].values()) and all(observation['desktop_absent'].values())
report['verified'] = True
except BaseException as error:
report.update(verified=False, error=repr(error), traceback=traceback.format_exc())
traceback.print_exc()
finally:
batch_supervisor._command = original_command
store.close()
from dx_productivity.trial_task_center.legacy_paths import cleanup_worker_aliases
report['alias_cleanup_errors'] = list(map(str, cleanup_worker_aliases()))
report['source_unchanged'] = hashes(source) == source_hashes
report['library_unchanged'] = hashlib.sha256(library.read_bytes()).hexdigest() == library_hash
report['elapsed_seconds'] = time.monotonic() - started
save(output / 'report.json', report)
print(json.dumps({key: report.get(key) for key in ('verified', 'error', 'elapsed_seconds')}, ensure_ascii=False), flush=True)
return 0 if (report.get('verified') and report['source_unchanged'] and report['library_unchanged']
and not report['alias_cleanup_errors']) else 1
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path, required=True)
parser.add_argument('--library', type=Path, required=True)
parser.add_argument('--executable', type=Path, default=REPO / 'dist/DxPartReplace/DxPartReplace.exe')
parser.add_argument('--output', type=Path, default=REPO / 'build/issue-3-frozen-startup-recovery')
parser.add_argument('--failures', type=int, choices=(1, 2), default=1)
args = parser.parse_args()
raise SystemExit(run(args.source, args.library, args.executable, args.output, args.failures))