Files
dxdesigner-part-replacement/tools/packaged_window_driver.py
T

166 lines
7.7 KiB
Python

"""External Windows UI driver and PrintWindow-only instructional capture.
This utility cannot import application internals or fabricate task state. It
operates the executable through Windows accessibility and records only windows
owned by that executable. Workbook edits and acceptance assertions belong to
the calling scenario.
"""
from __future__ import annotations
import ctypes
from ctypes import wintypes
import json
from pathlib import Path
import time
from PIL import Image, ImageDraw, ImageFont
from pywinauto import Application
import win32con
import win32gui
import win32process
import win32ui
def window_image(hwnd):
"""Read this HWND's drawing surface, never the desktop framebuffer."""
left, top, right, bottom = win32gui.GetWindowRect(hwnd)
width, height = right - left, bottom - top
dc = win32gui.GetWindowDC(hwnd)
source = win32ui.CreateDCFromHandle(dc)
target = source.CreateCompatibleDC()
bitmap = win32ui.CreateBitmap()
bitmap.CreateCompatibleBitmap(source, width, height)
target.SelectObject(bitmap)
try:
print_window = ctypes.windll.user32.PrintWindow
print_window.argtypes = (wintypes.HWND, wintypes.HDC, wintypes.UINT)
print_window.restype = wintypes.BOOL
if not print_window(hwnd, target.GetSafeHdc(), 2):
raise RuntimeError(f'PrintWindow failed for {hwnd}')
return Image.frombuffer('RGB', (width, height), bitmap.GetBitmapBits(True), 'raw', 'BGRX', 0, 1).copy()
finally:
win32gui.DeleteObject(bitmap.GetHandle())
target.DeleteDC()
source.DeleteDC()
win32gui.ReleaseDC(hwnd, dc)
class WindowDriver:
def __init__(self, pid, output):
self.pid, self.output = pid, Path(output)
self.output.mkdir(parents=True, exist_ok=True)
self.app = Application(backend='uia').connect(process=pid, timeout=15)
self.frames, self.durations, self.steps = [], [], []
self.font = ImageFont.truetype('C:/Windows/Fonts/msyh.ttc', 22)
if (self.output / 'frames.json').exists():
self.steps = json.loads((self.output / 'frames.json').read_text(encoding='utf-8'))
self.frames = [Image.open(self.output / f'frame-{s["frame"]:03d}.png').convert('RGB') for s in self.steps]
self.durations = [s['duration_ms'] for s in self.steps]
def windows(self):
handles = []
def collect(handle, _):
if win32gui.IsWindowVisible(handle) and win32process.GetWindowThreadProcessId(handle)[1] == self.pid:
left, top, right, bottom = win32gui.GetWindowRect(handle)
if right - left > 100 and bottom - top > 80:
handles.append(handle)
win32gui.EnumWindows(collect, None)
return [self.app.window(handle=handle).wrapper_object() for handle in handles]
def main(self):
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
for window in self.windows():
if window.window_text().startswith('DxDesigner '):
return window
time.sleep(.2)
raise TimeoutError('Application main window not found')
def find(self, text=None, *, kind=None, automation_id=None, window=None, optional=False):
root = window or self.main()
# Do not recursively enumerate virtual Qt table cells through UIA.
def walk(parent, depth=0):
if depth > 12:
return
for child in parent.children():
yield child
if child.element_info.control_type not in {
'Table', 'DataGrid', 'List', 'ScrollBar', 'Header', 'Edit',
'Button', 'Text', 'CheckBox', 'ComboBox', 'TitleBar', 'MenuBar', 'MenuItem'}:
yield from walk(child, depth + 1)
controls = walk(root)
matches = [c for c in controls if (kind is None or c.element_info.control_type == kind)
and (text is None or c.window_text() == text)
and (automation_id is None or c.element_info.automation_id == automation_id
or c.element_info.automation_id.endswith('.' + automation_id))
and c.is_visible()]
if not matches and optional:
return None
if len(matches) != 1:
raise LookupError(f'Expected one {text!r}/{automation_id!r}, got {len(matches)}')
return matches[0]
def click(self, control):
# Posted window mouse events allow modal dialogs without blocking the driver.
self.point(control)
def point(self, control, *, dx=None, dy=None, double=False):
rect = control.rectangle()
if dx is None and control.element_info.control_type in ('ListItem', 'DataItem'):
dx = 20 # Qt may expose the full elided text width beyond its viewport.
root = control
while not root.handle or not win32gui.IsWindow(root.handle):
root = root.parent()
x = rect.left + (dx if dx is not None else rect.width() // 2)
y = rect.top + (dy if dy is not None else rect.height() // 2)
x, y = win32gui.ScreenToClient(root.handle, (x, y))
value = (y << 16) | (x & 0xFFFF)
for _ in range(2 if double else 1):
win32gui.PostMessage(root.handle, win32con.WM_LBUTTONDOWN, win32con.MK_LBUTTON, value)
win32gui.PostMessage(root.handle, win32con.WM_LBUTTONUP, 0, value)
time.sleep(.2)
def type_text(self, control, text):
control.set_edit_text(text)
time.sleep(.2)
def record(self, caption, *, highlight=None, duration=1600):
main = self.main()
image = window_image(main.handle)
left, top, _, _ = win32gui.GetWindowRect(main.handle)
# Owned modal/detail windows are composited from their own surfaces.
for other in reversed(self.windows()):
if other.handle == main.handle:
continue
other_left, other_top, _, _ = win32gui.GetWindowRect(other.handle)
image.paste(window_image(other.handle), (other_left - left, other_top - top))
draw = ImageDraw.Draw(image)
if highlight is not None:
rect = highlight.rectangle()
box = (rect.left - left - 3, rect.top - top - 3,
rect.right - left + 3, rect.bottom - top + 3)
draw.rounded_rectangle(box, radius=5, outline='#e35d30', width=4)
# Fixed canvas dimensions allow window resizing without desktop capture.
image.thumbnail((1280, 900), Image.Resampling.LANCZOS)
canvas = Image.new('RGB', (1280, 980), '#f3f5f8')
canvas.paste(image, ((1280 - image.width) // 2, 0))
draw = ImageDraw.Draw(canvas)
draw.rectangle((0, 900, 1280, 980), fill='#172b45')
draw.text((24, 914), caption, font=self.font, fill='white')
draw.text((24, 946), '窗口操作实录 · 等待过程已压缩 · 工程副本验收',
font=ImageFont.truetype('C:/Windows/Fonts/msyh.ttc', 16), fill='#bdcfdf')
index = len(self.frames)
canvas.save(self.output / f'frame-{index:03d}.png')
self.frames.append(canvas)
self.durations.append(duration)
self.steps.append(dict(frame=index, caption=caption, duration_ms=duration, wall_time=time.time()))
(self.output / 'frames.json').write_text(json.dumps(self.steps, ensure_ascii=False, indent=2), encoding='utf-8')
def save_gif(self, path):
if not self.frames:
raise RuntimeError('No window frames captured')
palette = self.frames[0].quantize(colors=160)
images = [frame.quantize(palette=palette, dither=Image.Dither.NONE) for frame in self.frames]
images[0].save(path, save_all=True, append_images=images[1:],
duration=self.durations, loop=0, optimize=True, disposal=2)