274 lines
15 KiB
Python
274 lines
15 KiB
Python
"""Crash one certified owned frozen supervisor after actual native replacement.
|
|
|
|
Hybrid source non-GUI parent / frozen worker/helper/VBS. No production injection
|
|
switches. New disposable two-project copies only; never resumes business.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from threading import Event, Thread
|
|
import time
|
|
import traceback
|
|
from uuid import uuid4
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO / 'src'))
|
|
sys.path.insert(0, str(REPO / 'tools'))
|
|
from probe_batch_runtime_live import hashes, save
|
|
from probe_frozen_startup_recovery import read_json
|
|
|
|
|
|
def protection(root):
|
|
return {str(path.relative_to(root)): path.read_bytes().hex()
|
|
for path in root.rglob('uncertain.flag')}
|
|
|
|
|
|
class PostwriteCrash:
|
|
def __init__(self, root, output, task_ids, cancel):
|
|
self.root, self.output, self.task_ids, self.cancel = root, output, task_ids, cancel
|
|
self.stop = Event()
|
|
self.crash = None
|
|
self.errors = []
|
|
|
|
def scan(self):
|
|
import win32api
|
|
import win32event
|
|
import win32process
|
|
for path in (self.root / 'tasks' / self.task_ids[0] / 'quick-run').glob('*.native.json'):
|
|
native = read_json(path)
|
|
if not native or native.get('stage') not in {'native_replace_applied', 'native_replace_verified'}:
|
|
continue
|
|
assert native.get('replaced_uids'), 'no observed replacement identity'
|
|
for request_path in (self.root / 'batch-resources').glob('*/request.json'):
|
|
request = read_json(request_path)
|
|
if not request or request['operation'] != 'execute_task_queue':
|
|
continue
|
|
directory = request_path.parent
|
|
journal, session = read_json(directory / 'supervisor.json'), read_json(directory / 'session.json')
|
|
assert journal and session and journal['state'] == 'running', 'supervisor not running'
|
|
assert journal['batch_id'] == request['batch_id'] == session['session_id'] == directory.name
|
|
assert journal['task_ids'] == list(self.task_ids)
|
|
assert session['generation'] == 1 and session['retries'] == {}, 'unexpected generation/retry'
|
|
markers = list(directory.glob('attempts/**/*.json'))
|
|
assert len(markers) == 1, 'unexpected marker count'
|
|
marker = read_json(markers[0])
|
|
assert marker and marker['state'] == 'mutation_possible'
|
|
assert marker['task_id'] == self.task_ids[0]
|
|
if (directory / 'result.json').exists() or (directory / 'business-result.json').exists():
|
|
return # Too late for the requested missing-final-result fault.
|
|
queues = [read_json(item) for item in (self.root / 'task-queues').glob('*.json')]
|
|
assert len(queues) == 1 and queues[0]['entries'][self.task_ids[1]]['state'] == 'pending'
|
|
assert not (self.root / 'tasks' / self.task_ids[1] / 'quick-run' / 'status.tsv').exists()
|
|
identity = journal['process']
|
|
assert identity and identity['creation_time']
|
|
handle = win32api.OpenProcess(0x1000 | 0x100000 | 1, False, identity['pid'])
|
|
try:
|
|
created = str(win32process.GetProcessTimes(handle)['CreationTime'])
|
|
assert created == identity['creation_time'], 'supervisor PID creation time mismatch'
|
|
assert win32event.WaitForSingleObject(handle, 0) == 258
|
|
latest = read_json(path)
|
|
# If the verified report advanced, no assumption is made
|
|
# from an earlier native_replace_started observation.
|
|
if not latest or latest.get('stage') not in {'native_replace_applied', 'native_replace_verified'}:
|
|
return # applied can advance to pin promotion between reads.
|
|
assert read_json(markers[0])['state'] == 'mutation_possible'
|
|
for status in (self.root / 'tasks').glob('*/quick-run/status.tsv'):
|
|
if '\tsaved_ok\t' in status.read_text(encoding='utf-8-sig'):
|
|
return # Keep the already completed fixture; never replay it.
|
|
self.crash = {'time_ns': time.time_ns(), 'supervisor_identity': identity,
|
|
'creation_time_observed': created, 'batch_id': directory.name,
|
|
'native_report': latest, 'marker': marker, 'marker_path': str(markers[0]),
|
|
'marker_sha256': hashlib.sha256(markers[0].read_bytes()).hexdigest(),
|
|
'queue_before_crash': queues[0], 'terminated_owned_supervisor_only': True}
|
|
save(self.output / 'postwrite-crash.json', self.crash)
|
|
win32process.TerminateProcess(handle, 81)
|
|
assert win32event.WaitForSingleObject(handle, 5000) == 0
|
|
finally:
|
|
handle.Close()
|
|
return
|
|
|
|
def _watch(self):
|
|
while not self.stop.wait(.005):
|
|
try:
|
|
self.scan()
|
|
if self.crash is not None:
|
|
return
|
|
except Exception as error:
|
|
self.errors.append({'error': repr(error), 'traceback': traceback.format_exc()})
|
|
self.cancel.set()
|
|
save(self.output / 'watcher-errors.json', self.errors)
|
|
return
|
|
|
|
def __enter__(self):
|
|
self.thread = Thread(target=self._watch, daemon=True)
|
|
self.thread.start()
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
self.stop.set()
|
|
self.thread.join(5)
|
|
assert not self.thread.is_alive()
|
|
|
|
|
|
def recheck_existing(output):
|
|
"""Fresh parent process: resource-only frozen request, never resumes queue."""
|
|
from dx_productivity.trial_task_center import batch_supervisor
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
output = output.resolve(strict=True)
|
|
assert output.is_relative_to(REPO / 'build/issue-3-frozen-postwrite-crash')
|
|
inputs = read_json(output / 'inputs.json')
|
|
root = output / 'data'
|
|
before = protection(root)
|
|
assert before
|
|
crash = read_json(output / 'postwrite-crash.json')
|
|
marker = Path(crash['marker_path'])
|
|
assert read_json(marker)['state'] == 'mutation_possible'
|
|
launch = []
|
|
def command(request):
|
|
launch.append(read_json(request)['operation'])
|
|
return (inputs['executable'], '--batch-worker', str(request))
|
|
original_command = batch_supervisor._command
|
|
batch_supervisor._command = command
|
|
store = TaskStore(output / 'tasks.db')
|
|
try:
|
|
service = TaskCenterService(store, configure_logging(root), supervised=True)
|
|
assert protection(root) == before, 'startup cleared protection'
|
|
pending = service.cleanup_batch_resources()
|
|
assert not pending
|
|
assert protection(root) == before, 'resource cleanup cleared business protection'
|
|
assert hashlib.sha256(marker.read_bytes()).hexdigest() == crash['marker_sha256']
|
|
assert launch == ['cleanup_batch_resources']
|
|
assert hashes(Path(inputs['projects'][1]).parent) == inputs['second_project_before_execution']
|
|
execution = read_json(root / 'batch-resources' / crash['batch_id'] / 'session.json')
|
|
assert execution['generation'] == 1 and execution['retries'] == {}
|
|
assert execution['state'] == 'released'
|
|
assert len(list((root / 'batch-resources' / crash['batch_id']).glob('attempts/**/*.json'))) == 1
|
|
save(output / 'restart-resource-recheck.json', {'verified': True, 'launch_operations': launch,
|
|
'pending_resources': pending, 'protection_preserved': True, 'marker_preserved': True,
|
|
'business_replayed': False, 'second_project_unchanged': True, 'execution_session': execution,
|
|
'supervisor': read_json(root / 'batch-resources' / crash['batch_id'] / 'supervisor.json')})
|
|
finally:
|
|
batch_supervisor._command = original_command
|
|
store.close()
|
|
|
|
|
|
def run(source_root, library, executable, base):
|
|
from dx_productivity.trial_task_center import batch_supervisor
|
|
from dx_productivity.trial_task_center.bom_plan import write_bom_template
|
|
from dx_productivity.trial_task_center.errors import TaskCenterError
|
|
from dx_productivity.trial_task_center.logging_setup import configure_logging
|
|
from dx_productivity.trial_task_center.service import TaskCenterService
|
|
from dx_productivity.trial_task_center.store import TaskStore
|
|
from dx_productivity.trial_task_center.task_queue import execute_task_queue
|
|
from private_desktop_observer import PrivateDesktopObserver
|
|
source_root, library, executable = [item.resolve(strict=True) for item in (source_root, library, executable)]
|
|
output = base.resolve() / ('run-' + uuid4().hex)
|
|
assert output.is_relative_to(REPO / 'build/issue-3-frozen-postwrite-crash')
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
names, projects, source_hashes = ('Audio_D2128', 'Audio_T605A'), [], {}
|
|
for name in names:
|
|
source_hashes[name] = hashes(source_root / name)
|
|
target = output / 'projects' / name
|
|
shutil.copytree(source_root / name, target)
|
|
project = target / (name + '.prj')
|
|
text, count = re.subn(r'(?m)^KEY CentralLibrary .*$', lambda _: f'KEY CentralLibrary "{library.as_posix()}"',
|
|
project.read_text(encoding='utf-8-sig'))
|
|
assert count == 1
|
|
project.write_text(text, encoding='utf-8')
|
|
projects.append(project)
|
|
library_hash = hashlib.sha256(library.read_bytes()).hexdigest()
|
|
report = {'mode': 'source_non_gui_parent_frozen_worker_postwrite_crash', 'gui_parent_covered': False,
|
|
'frozen_executable_sha256': hashlib.sha256(executable.read_bytes()).hexdigest(), 'launches': []}
|
|
original_command = batch_supervisor._command
|
|
def command(request):
|
|
report['launches'].append(read_json(request))
|
|
return (str(executable), '--batch-worker', str(request))
|
|
batch_supervisor._command = command
|
|
store = TaskStore(output / 'tasks.db')
|
|
service = TaskCenterService(store, configure_logging(output / 'data'), supervised=True)
|
|
started = time.monotonic()
|
|
try:
|
|
with PrivateDesktopObserver(output):
|
|
tasks = service.create_project_tasks(output / 'projects', selected_paths=tuple(projects))
|
|
by_project = {service.projects(task.task_id)[0].path.resolve(): task for task in tasks}
|
|
ordered = tuple(by_project[project.resolve()].task_id for project in projects)
|
|
bom = write_bom_template(output / 'input.xlsx', ({'project_path': str(project),
|
|
'old_part_number': '4100-CA0000-T7', 'new_part_number': '4100-CA0000-T0',
|
|
'refdes': 'RV05', 'enabled': True, 'sync_nc': True} for project in projects))
|
|
imported = service.inspect_bom_batch(ordered[0], bom, on_progress=lambda text: print(text, flush=True))
|
|
assert imported.passed_rows == 2 and imported.preview is not None
|
|
service.save_bom_batch(imported.preview, {item.task_id: item.rows for item in imported.preview.projects})
|
|
save(output / 'inputs.json', {'executable': str(executable), 'projects': list(map(str, projects)),
|
|
'task_ids': ordered, 'second_project_before_execution': hashes(projects[1].parent)})
|
|
cancel = Event()
|
|
print('Frozen two-project queue: wait for actual applied/verified then crash own supervisor', flush=True)
|
|
with PostwriteCrash(output / 'data', output, ordered, cancel) as injector:
|
|
try:
|
|
result = execute_task_queue(service, ordered, stop_event=cancel)
|
|
report['unexpected_result'] = repr(result)
|
|
except TaskCenterError as failure:
|
|
report['operation_error'] = {'code': failure.code, 'message': failure.message}
|
|
assert not injector.errors and injector.crash is not None
|
|
assert report.get('operation_error', {}).get('code') == 'supervisor_interrupted'
|
|
root = output / 'data'
|
|
assert protection(root)
|
|
execution = root / 'batch-resources' / injector.crash['batch_id']
|
|
session = read_json(execution / 'session.json')
|
|
assert session['generation'] == 1 and session['retries'] == {}
|
|
assert len(list(execution.glob('attempts/**/*.json'))) == 1
|
|
assert not (root / 'tasks' / ordered[1] / 'quick-run' / 'status.tsv').exists()
|
|
queues = [read_json(path) for path in (root / 'task-queues').glob('*.json')]
|
|
assert len(queues) == 1 and queues[0]['entries'][ordered[1]]['state'] == 'pending'
|
|
report['queue_after_crash'] = queues[0]
|
|
assert not (execution / 'business-result.json').exists()
|
|
store.close()
|
|
print('Fresh parent process resource-only recheck', flush=True)
|
|
completed = subprocess.run((sys.executable, str(Path(__file__).resolve()), '--recheck-existing', str(output)),
|
|
capture_output=True, timeout=90)
|
|
(output / 'restart-output.txt').write_bytes(completed.stdout + b'\n' + completed.stderr)
|
|
assert completed.returncode == 0, (output / 'restart-output.txt').read_text(errors='replace')
|
|
observation = read_json(output / 'desktop-observation.json')
|
|
assert not observation['observer_errors']
|
|
assert all(observation['cad_exit_confirmed'].values()) and all(observation['desktop_absent'].values())
|
|
assert not observation['owned_cad_became_foreground'] and not observation['owned_cad_on_input_desktop']
|
|
assert observation['input_desktop_before'] == observation['input_desktop_after']
|
|
report.update(verified=True, business_replayed=False, next_project_started=False,
|
|
restart_resource_recheck=read_json(output / 'restart-resource-recheck.json'))
|
|
except BaseException as error:
|
|
report.update(verified=False, error=repr(error), traceback=traceback.format_exc())
|
|
traceback.print_exc()
|
|
finally:
|
|
batch_supervisor._command = original_command
|
|
store.close()
|
|
report['source_unchanged'] = all(hashes(source_root / name) == source_hashes[name] for name in names)
|
|
report['library_unchanged'] = hashlib.sha256(library.read_bytes()).hexdigest() == library_hash
|
|
report['elapsed_seconds'] = time.monotonic() - started
|
|
save(output / 'report.json', report)
|
|
print(json.dumps({'output': str(output), **{key: report.get(key) for key in
|
|
('verified', 'error', 'elapsed_seconds')}}, ensure_ascii=False), flush=True)
|
|
return 0 if report.get('verified') and report['source_unchanged'] and report['library_unchanged'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--source-root', type=Path)
|
|
parser.add_argument('--library', type=Path)
|
|
parser.add_argument('--executable', type=Path, default=REPO / 'dist/DxPartReplace/DxPartReplace.exe')
|
|
parser.add_argument('--output', type=Path, default=REPO / 'build/issue-3-frozen-postwrite-crash')
|
|
parser.add_argument('--recheck-existing', type=Path)
|
|
args = parser.parse_args()
|
|
if args.recheck_existing:
|
|
recheck_existing(args.recheck_existing)
|
|
else:
|
|
assert args.source_root and args.library
|
|
raise SystemExit(run(args.source_root, args.library, args.executable, args.output))
|