Files
dxdesigner-part-replacement/tools/accept_paired_samples.py
T

167 lines
11 KiB
Python

"""Paired isolated official/native and application-composed UI exploration."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
from acceptance_workspace import create_workspace_root
import time
from urllib.parse import urlsplit
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / 'src'))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--seed', type=int)
parser.add_argument('--count', type=int, default=20)
parser.add_argument('--arm', choices=('native', 'harness'))
parser.add_argument('--port', type=int, default=9251)
parser.add_argument('--prepare-only', action='store_true')
parser.add_argument('--purpose', choices=('exploration', 'holdout'), default='exploration')
parser.add_argument('--provider-config', type=Path, default=Path(os.environ['LOCALAPPDATA'])/'DxProductivity/trial-task-center/dsh')
args = parser.parse_args()
output = args.output.resolve(); output.mkdir(parents=True, exist_ok=True)
if args.arm:
return worker(args, output)
from random_acceptance_cases import generate
root = create_workspace_root(Path(os.environ['LOCALAPPDATA'])/'DxProductivity/acceptance-workspaces', 'paired-random-')
seed = args.seed
source_files = [REPO/'integration/dsh/start.mjs', REPO/'tools/random_acceptance_cases.py',
REPO/'tools/accept_native_runtime.mjs', REPO/'tools/accept_paired_samples.py',
REPO/'tools/verify_harness_samples_ui.mjs', REPO/'tools/harness_session_evidence.mjs',
REPO/'tools/accept_harness_runtime.mjs', REPO/'tools/acceptance_runtime_state.mjs',
REPO/'tools/acceptance_workspace.py',
REPO/'tools/dsh_skip_question.mjs', REPO/'tools/score_acceptance_semantics.py',
REPO/'tools/extract_acceptance_execution.mjs', REPO/'tools/summarize_paired_samples.mjs',
*sorted((REPO/'integration/dsh/task-plugin').rglob('*.mjs'))]
fingerprints = {str(p.relative_to(REPO)): hashlib.sha256(p.read_bytes()).hexdigest() for p in source_files}
for arm in ('native', 'harness'):
home = root/arm/'dsh/harness-home'
workspace = home/'workspace'
seed, cases = generate(workspace, seed, args.count)
destination = output/arm; destination.mkdir()
manifest = {'seed': str(seed), 'generation': 'stratified-random-v1', 'cases': cases,
'data': str(root/arm), 'workspace': str(workspace),
'runtime': arm, 'require_host_goal': False, 'case_budget_ms': 600000,
'driver_description':'Isolated Qt WebEngine + targeted CDP mouse/keyboard input',
'require_workspace_preflight': True,
'coverage': sorted({c['stratum'] for c in cases}), 'real_cad': False,
'git_head': subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip(),
'source_patch_sha256': hashlib.sha256(subprocess.check_output(['git','diff','--binary','HEAD'],cwd=REPO)).hexdigest(),
'source_files': fingerprints, 'command': [sys.executable,*sys.argv],
'real_model': not args.prepare_only, 'purpose': args.purpose}
(destination/'manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2),encoding='utf-8')
print(json.dumps({'seed': seed, 'output': str(output), 'root': str(root)}),flush=True)
if args.prepare_only: return 0
statuses = {}
for i,arm in enumerate(('native','harness')):
command = [sys.executable,'-X','utf8',str(Path(__file__).resolve()),'--output',str(output),
'--arm',arm,'--port',str(args.port+i),'--provider-config',str(args.provider_config)]
with (output/f'{arm}-worker.log').open('w',encoding='utf-8') as log:
process = subprocess.Popen(command,cwd=REPO,stdout=log,stderr=subprocess.STDOUT)
print(json.dumps({'arm':arm,'worker_pid':process.pid}),flush=True)
statuses[arm] = process.wait()
(output/'exit-status.json').write_text(json.dumps(statuses),encoding='utf-8')
for arm in ('native','harness'):
subprocess.run(['node',str(REPO/'tools/extract_acceptance_execution.mjs'),str(output/arm)],cwd=REPO,check=True)
subprocess.run([sys.executable,'-X','utf8',str(REPO/'tools/score_acceptance_semantics.py'),str(output/arm)],cwd=REPO,check=True)
subprocess.run(['node',str(REPO/'tools/summarize_paired_samples.mjs'),str(output)],cwd=REPO,check=True)
return int(any(statuses.values()))
def worker(args, output):
from dx_productivity.dsh.native_runtime import launch_environment
from PySide6.QtCore import QTimer, QUrl
from PySide6.QtWidgets import QApplication
from PySide6.QtWebEngineWidgets import QWebEngineView
destination = output/args.arm
manifest = json.loads((destination/'manifest.json').read_text(encoding='utf-8'))
existing_report = destination/'report.json'
if existing_report.exists():
previous = json.loads(existing_report.read_text(encoding='utf-8'))
if previous.get('status') in ('passed', 'failed') and not manifest.get('resume_completed'):
return int(previous['status'] == 'failed')
# Retain startup failures when the same exploration arm is resumed.
for name in ('runtime-error.log','runtime.log','driver.log','processes.json'):
previous_log = destination/name
if previous_log.exists() and previous_log.stat().st_size:
suffix = 1
while (destination/f'{name}.attempt-{suffix}').exists(): suffix += 1
shutil.copy2(previous_log,destination/f'{name}.attempt-{suffix}')
data = Path(manifest['data']); home = data/'dsh/harness-home'
source_home = args.provider_config/'harness-home'
env = launch_environment(args.provider_config)
env.update(DX_DSH_HOME=str(home), DX_DSH_CONTEXT_FILE=str(data/'context.json'),
DX_DSH_PARENT_PIPE='1', DX_DSH_WORKSPACE=manifest['workspace'],
DX_ACCEPTANCE_STATE_FILE=str(destination/'runtime-state.json'), DX_ACCEPTANCE_PYTHON=sys.executable)
(data/'context.json').write_text(json.dumps({'task_id':'','revision':'paired-exploration','events':[]}),encoding='utf-8')
profile = home/'profiles/dx-task'; profile.mkdir(parents=True,exist_ok=True)
# Private user model references and local secrets remain under isolated home.
source_patch = source_home/'profiles/dx-task/cordis.patch.yml'
if source_patch.exists(): shutil.copy2(source_patch,profile/'cordis.patch.yml')
credential = source_home/'.credentials.yaml'
if credential.exists(): shutil.copy2(credential,home/'.credentials.yaml')
patch = profile/'cordis.patch.yml'
patch_script = "const fs=require('fs'),yaml=require('./integration/dsh/node_modules/js-yaml');const p=process.argv[1];let rows=fs.existsSync(p)?yaml.load(fs.readFileSync(p,'utf8')):[];rows=rows.filter(r=>!['pwsh-sandbox','workspace-controller'].includes(r.id));rows.push({id:'pwsh-sandbox',config:{pwshPath:process.argv[2]}},{id:'workspace-controller',config:{documentsDirectory:process.argv[3]}});fs.writeFileSync(p,JSON.stringify(rows));"
subprocess.run(['node','-e',patch_script,str(patch),str(Path(os.environ['SystemRoot'])/'System32/WindowsPowerShell/v1.0/powershell.exe'),str(home/'documents')],cwd=REPO,check=True,stdout=subprocess.DEVNULL)
# Generate application profile privately so native reads precisely its effective model config.
prepare = "import {prepareProfileDirectory} from './integration/dsh/start.mjs'; await prepareProfileDirectory(process.argv[1]);"
subprocess.run(['node','--input-type=module','-e',prepare,str(profile)],cwd=REPO,env=env,check=True,stdout=subprocess.DEVNULL)
if args.arm == 'native':
command=['node',str(REPO/'tools/accept_native_runtime.mjs'),str(profile),str(home),manifest['workspace'],str(args.port+100),'run']
marker='NATIVE_READY '
else:
command=['node',str(REPO/'tools/accept_harness_runtime.mjs')]; marker='DX_DSH_READY '
runtime = subprocess.Popen(command,cwd=REPO,env=env,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=(destination/'runtime-error.log').open('w',encoding='utf-8'),text=True,encoding='utf-8')
(destination/'processes.json').write_text(json.dumps({'runtime_pid':runtime.pid,'worker_pid':os.getpid()}),encoding='utf-8')
# Startup reads occur on a thread, keeping the Qt event loop responsive.
import threading
state={'url':None,'driver':None,'started':time.monotonic()}
def read_ready():
with (destination/'runtime.log').open('w',encoding='utf-8') as log:
for line in runtime.stdout:
if line.startswith(marker): state['url']=json.loads(line[len(marker):])['url']; log.write(marker+'[local authenticated URL redacted]\n')
else: log.write(line)
log.flush()
threading.Thread(target=read_ready,daemon=True).start()
os.environ['QTWEBENGINE_REMOTE_DEBUGGING']=f'127.0.0.1:{args.port}'
os.environ.pop('QT_QPA_PLATFORM',None)
app=QApplication([]); view=QWebEngineView(); view.resize(1380,900); view.show()
def tick():
if state['url'] and not state.get('loaded'):
url=urlsplit(state['url']); manifest['runtime_origin']=f'{url.scheme}://{url.netloc}'
(destination/'manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2),encoding='utf-8')
view.load(QUrl(state['url'])); state['loaded']=True
log=(destination/'driver.log').open('w',encoding='utf-8')
state['driver']=subprocess.Popen(['node',str(REPO/'tools/verify_harness_samples_ui.mjs'),str(destination),str(args.port)],cwd=REPO,stdout=log,stderr=subprocess.STDOUT)
(destination/'processes.json').write_text(json.dumps({'runtime_pid':runtime.pid,'worker_pid':os.getpid(),'driver_pid':state['driver'].pid}),encoding='utf-8')
if state['driver'] and state['driver'].poll() is not None:
report_path=destination/'report.json'
if report_path.exists() and json.loads(report_path.read_text(encoding='utf-8')).get('status')=='observation_failed':
# Preserve the current live runtime/model after a permanent
# observer failure; never convert telemetry failure into abort.
if not state.get('observation_failed'):
print(json.dumps({'arm':args.arm,'status':'observation_failed','runtime_preserved':True,'runtime_pid':runtime.pid}),flush=True)
state['observation_failed']=True
return
timer.stop(); view.close(); app.quit()
elif runtime.poll() is not None or (not state['url'] and time.monotonic()-state['started']>180):
timer.stop(); view.close(); app.quit()
timer=QTimer(); timer.timeout.connect(tick); timer.start(250)
app.exec()
if runtime.poll() is None:
runtime.terminate()
try: runtime.wait(timeout=15)
except subprocess.TimeoutExpired: runtime.kill()
return state['driver'].returncode if state['driver'] else 1
if __name__=='__main__': raise SystemExit(main())