Files
dxdesigner-part-replacement/tools/probe_frozen_postwrite_crash.py

274 lines
15 KiB
Python

"""Crash one certified owned frozen supervisor after actual native replacement.
Hybrid source non-GUI parent / frozen worker/helper/VBS. No production injection
switches. New disposable two-project copies only; never resumes business.
"""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import re
import shutil
import subprocess
import sys
from threading import Event, Thread
import time
import traceback
from uuid import uuid4
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / 'src'))
sys.path.insert(0, str(REPO / 'tools'))
from probe_batch_runtime_live import hashes, save
from probe_frozen_startup_recovery import read_json
def protection(root):
return {str(path.relative_to(root)): path.read_bytes().hex()
for path in root.rglob('uncertain.flag')}
class PostwriteCrash:
def __init__(self, root, output, task_ids, cancel):
self.root, self.output, self.task_ids, self.cancel = root, output, task_ids, cancel
self.stop = Event()
self.crash = None
self.errors = []
def scan(self):
import win32api
import win32event
import win32process
for path in (self.root / 'tasks' / self.task_ids[0] / 'quick-run').glob('*.native.json'):
native = read_json(path)
if not native or native.get('stage') not in {'native_replace_applied', 'native_replace_verified'}:
continue
assert native.get('replaced_uids'), 'no observed replacement identity'
for request_path in (self.root / 'batch-resources').glob('*/request.json'):
request = read_json(request_path)
if not request or request['operation'] != 'execute_task_queue':
continue
directory = request_path.parent
journal, session = read_json(directory / 'supervisor.json'), read_json(directory / 'session.json')
assert journal and session and journal['state'] == 'running', 'supervisor not running'
assert journal['batch_id'] == request['batch_id'] == session['session_id'] == directory.name
assert journal['task_ids'] == list(self.task_ids)
assert session['generation'] == 1 and session['retries'] == {}, 'unexpected generation/retry'
markers = list(directory.glob('attempts/**/*.json'))
assert len(markers) == 1, 'unexpected marker count'
marker = read_json(markers[0])
assert marker and marker['state'] == 'mutation_possible'
assert marker['task_id'] == self.task_ids[0]
if (directory / 'result.json').exists() or (directory / 'business-result.json').exists():
return # Too late for the requested missing-final-result fault.
queues = [read_json(item) for item in (self.root / 'task-queues').glob('*.json')]
assert len(queues) == 1 and queues[0]['entries'][self.task_ids[1]]['state'] == 'pending'
assert not (self.root / 'tasks' / self.task_ids[1] / 'quick-run' / 'status.tsv').exists()
identity = journal['process']
assert identity and identity['creation_time']
handle = win32api.OpenProcess(0x1000 | 0x100000 | 1, False, identity['pid'])
try:
created = str(win32process.GetProcessTimes(handle)['CreationTime'])
assert created == identity['creation_time'], 'supervisor PID creation time mismatch'
assert win32event.WaitForSingleObject(handle, 0) == 258
latest = read_json(path)
# If the verified report advanced, no assumption is made
# from an earlier native_replace_started observation.
if not latest or latest.get('stage') not in {'native_replace_applied', 'native_replace_verified'}:
return # applied can advance to pin promotion between reads.
assert read_json(markers[0])['state'] == 'mutation_possible'
for status in (self.root / 'tasks').glob('*/quick-run/status.tsv'):
if '\tsaved_ok\t' in status.read_text(encoding='utf-8-sig'):
return # Keep the already completed fixture; never replay it.
self.crash = {'time_ns': time.time_ns(), 'supervisor_identity': identity,
'creation_time_observed': created, 'batch_id': directory.name,
'native_report': latest, 'marker': marker, 'marker_path': str(markers[0]),
'marker_sha256': hashlib.sha256(markers[0].read_bytes()).hexdigest(),
'queue_before_crash': queues[0], 'terminated_owned_supervisor_only': True}
save(self.output / 'postwrite-crash.json', self.crash)
win32process.TerminateProcess(handle, 81)
assert win32event.WaitForSingleObject(handle, 5000) == 0
finally:
handle.Close()
return
def _watch(self):
while not self.stop.wait(.005):
try:
self.scan()
if self.crash is not None:
return
except Exception as error:
self.errors.append({'error': repr(error), 'traceback': traceback.format_exc()})
self.cancel.set()
save(self.output / 'watcher-errors.json', self.errors)
return
def __enter__(self):
self.thread = Thread(target=self._watch, daemon=True)
self.thread.start()
return self
def __exit__(self, *_):
self.stop.set()
self.thread.join(5)
assert not self.thread.is_alive()
def recheck_existing(output):
"""Fresh parent process: resource-only frozen request, never resumes queue."""
from dx_productivity.trial_task_center import batch_supervisor
from dx_productivity.trial_task_center.logging_setup import configure_logging
from dx_productivity.trial_task_center.service import TaskCenterService
from dx_productivity.trial_task_center.store import TaskStore
output = output.resolve(strict=True)
assert output.is_relative_to(REPO / 'build/issue-3-frozen-postwrite-crash')
inputs = read_json(output / 'inputs.json')
root = output / 'data'
before = protection(root)
assert before
crash = read_json(output / 'postwrite-crash.json')
marker = Path(crash['marker_path'])
assert read_json(marker)['state'] == 'mutation_possible'
launch = []
def command(request):
launch.append(read_json(request)['operation'])
return (inputs['executable'], '--batch-worker', str(request))
original_command = batch_supervisor._command
batch_supervisor._command = command
store = TaskStore(output / 'tasks.db')
try:
service = TaskCenterService(store, configure_logging(root), supervised=True)
assert protection(root) == before, 'startup cleared protection'
pending = service.cleanup_batch_resources()
assert not pending
assert protection(root) == before, 'resource cleanup cleared business protection'
assert hashlib.sha256(marker.read_bytes()).hexdigest() == crash['marker_sha256']
assert launch == ['cleanup_batch_resources']
assert hashes(Path(inputs['projects'][1]).parent) == inputs['second_project_before_execution']
execution = read_json(root / 'batch-resources' / crash['batch_id'] / 'session.json')
assert execution['generation'] == 1 and execution['retries'] == {}
assert execution['state'] == 'released'
assert len(list((root / 'batch-resources' / crash['batch_id']).glob('attempts/**/*.json'))) == 1
save(output / 'restart-resource-recheck.json', {'verified': True, 'launch_operations': launch,
'pending_resources': pending, 'protection_preserved': True, 'marker_preserved': True,
'business_replayed': False, 'second_project_unchanged': True, 'execution_session': execution,
'supervisor': read_json(root / 'batch-resources' / crash['batch_id'] / 'supervisor.json')})
finally:
batch_supervisor._command = original_command
store.close()
def run(source_root, library, executable, base):
from dx_productivity.trial_task_center import batch_supervisor
from dx_productivity.trial_task_center.bom_plan import write_bom_template
from dx_productivity.trial_task_center.errors import TaskCenterError
from dx_productivity.trial_task_center.logging_setup import configure_logging
from dx_productivity.trial_task_center.service import TaskCenterService
from dx_productivity.trial_task_center.store import TaskStore
from dx_productivity.trial_task_center.task_queue import execute_task_queue
from private_desktop_observer import PrivateDesktopObserver
source_root, library, executable = [item.resolve(strict=True) for item in (source_root, library, executable)]
output = base.resolve() / ('run-' + uuid4().hex)
assert output.is_relative_to(REPO / 'build/issue-3-frozen-postwrite-crash')
output.mkdir(parents=True, exist_ok=False)
names, projects, source_hashes = ('Audio_D2128', 'Audio_T605A'), [], {}
for name in names:
source_hashes[name] = hashes(source_root / name)
target = output / 'projects' / name
shutil.copytree(source_root / name, target)
project = target / (name + '.prj')
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$', lambda _: f'KEY CentralLibrary "{library.as_posix()}"',
project.read_text(encoding='utf-8-sig'))
assert count == 1
project.write_text(text, encoding='utf-8')
projects.append(project)
library_hash = hashlib.sha256(library.read_bytes()).hexdigest()
report = {'mode': 'source_non_gui_parent_frozen_worker_postwrite_crash', 'gui_parent_covered': False,
'frozen_executable_sha256': hashlib.sha256(executable.read_bytes()).hexdigest(), 'launches': []}
original_command = batch_supervisor._command
def command(request):
report['launches'].append(read_json(request))
return (str(executable), '--batch-worker', str(request))
batch_supervisor._command = command
store = TaskStore(output / 'tasks.db')
service = TaskCenterService(store, configure_logging(output / 'data'), supervised=True)
started = time.monotonic()
try:
with PrivateDesktopObserver(output):
tasks = service.create_project_tasks(output / 'projects', selected_paths=tuple(projects))
by_project = {service.projects(task.task_id)[0].path.resolve(): task for task in tasks}
ordered = tuple(by_project[project.resolve()].task_id for project in projects)
bom = write_bom_template(output / 'input.xlsx', ({'project_path': str(project),
'old_part_number': '4100-CA0000-T7', 'new_part_number': '4100-CA0000-T0',
'refdes': 'RV05', 'enabled': True, 'sync_nc': True} for project in projects))
imported = service.inspect_bom_batch(ordered[0], bom, on_progress=lambda text: print(text, flush=True))
assert imported.passed_rows == 2 and imported.preview is not None
service.save_bom_batch(imported.preview, {item.task_id: item.rows for item in imported.preview.projects})
save(output / 'inputs.json', {'executable': str(executable), 'projects': list(map(str, projects)),
'task_ids': ordered, 'second_project_before_execution': hashes(projects[1].parent)})
cancel = Event()
print('Frozen two-project queue: wait for actual applied/verified then crash own supervisor', flush=True)
with PostwriteCrash(output / 'data', output, ordered, cancel) as injector:
try:
result = execute_task_queue(service, ordered, stop_event=cancel)
report['unexpected_result'] = repr(result)
except TaskCenterError as failure:
report['operation_error'] = {'code': failure.code, 'message': failure.message}
assert not injector.errors and injector.crash is not None
assert report.get('operation_error', {}).get('code') == 'supervisor_interrupted'
root = output / 'data'
assert protection(root)
execution = root / 'batch-resources' / injector.crash['batch_id']
session = read_json(execution / 'session.json')
assert session['generation'] == 1 and session['retries'] == {}
assert len(list(execution.glob('attempts/**/*.json'))) == 1
assert not (root / 'tasks' / ordered[1] / 'quick-run' / 'status.tsv').exists()
queues = [read_json(path) for path in (root / 'task-queues').glob('*.json')]
assert len(queues) == 1 and queues[0]['entries'][ordered[1]]['state'] == 'pending'
report['queue_after_crash'] = queues[0]
assert not (execution / 'business-result.json').exists()
store.close()
print('Fresh parent process resource-only recheck', flush=True)
completed = subprocess.run((sys.executable, str(Path(__file__).resolve()), '--recheck-existing', str(output)),
capture_output=True, timeout=90)
(output / 'restart-output.txt').write_bytes(completed.stdout + b'\n' + completed.stderr)
assert completed.returncode == 0, (output / 'restart-output.txt').read_text(errors='replace')
observation = read_json(output / 'desktop-observation.json')
assert not observation['observer_errors']
assert all(observation['cad_exit_confirmed'].values()) and all(observation['desktop_absent'].values())
assert not observation['owned_cad_became_foreground'] and not observation['owned_cad_on_input_desktop']
assert observation['input_desktop_before'] == observation['input_desktop_after']
report.update(verified=True, business_replayed=False, next_project_started=False,
restart_resource_recheck=read_json(output / 'restart-resource-recheck.json'))
except BaseException as error:
report.update(verified=False, error=repr(error), traceback=traceback.format_exc())
traceback.print_exc()
finally:
batch_supervisor._command = original_command
store.close()
report['source_unchanged'] = all(hashes(source_root / name) == source_hashes[name] for name in names)
report['library_unchanged'] = hashlib.sha256(library.read_bytes()).hexdigest() == library_hash
report['elapsed_seconds'] = time.monotonic() - started
save(output / 'report.json', report)
print(json.dumps({'output': str(output), **{key: report.get(key) for key in
('verified', 'error', 'elapsed_seconds')}}, ensure_ascii=False), flush=True)
return 0 if report.get('verified') and report['source_unchanged'] and report['library_unchanged'] else 1
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source-root', type=Path)
parser.add_argument('--library', type=Path)
parser.add_argument('--executable', type=Path, default=REPO / 'dist/DxPartReplace/DxPartReplace.exe')
parser.add_argument('--output', type=Path, default=REPO / 'build/issue-3-frozen-postwrite-crash')
parser.add_argument('--recheck-existing', type=Path)
args = parser.parse_args()
if args.recheck_existing:
recheck_existing(args.recheck_existing)
else:
assert args.source_root and args.library
raise SystemExit(run(args.source_root, args.library, args.executable, args.output))