166 lines
7.7 KiB
Python
166 lines
7.7 KiB
Python
"""External Windows UI driver and PrintWindow-only instructional capture.
|
|
|
|
This utility cannot import application internals or fabricate task state. It
|
|
operates the executable through Windows accessibility and records only windows
|
|
owned by that executable. Workbook edits and acceptance assertions belong to
|
|
the calling scenario.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
|
|
from PIL import Image, ImageDraw, ImageFont
|
|
from pywinauto import Application
|
|
import win32con
|
|
import win32gui
|
|
import win32process
|
|
import win32ui
|
|
|
|
|
|
def window_image(hwnd):
|
|
"""Read this HWND's drawing surface, never the desktop framebuffer."""
|
|
left, top, right, bottom = win32gui.GetWindowRect(hwnd)
|
|
width, height = right - left, bottom - top
|
|
dc = win32gui.GetWindowDC(hwnd)
|
|
source = win32ui.CreateDCFromHandle(dc)
|
|
target = source.CreateCompatibleDC()
|
|
bitmap = win32ui.CreateBitmap()
|
|
bitmap.CreateCompatibleBitmap(source, width, height)
|
|
target.SelectObject(bitmap)
|
|
try:
|
|
print_window = ctypes.windll.user32.PrintWindow
|
|
print_window.argtypes = (wintypes.HWND, wintypes.HDC, wintypes.UINT)
|
|
print_window.restype = wintypes.BOOL
|
|
if not print_window(hwnd, target.GetSafeHdc(), 2):
|
|
raise RuntimeError(f'PrintWindow failed for {hwnd}')
|
|
return Image.frombuffer('RGB', (width, height), bitmap.GetBitmapBits(True), 'raw', 'BGRX', 0, 1).copy()
|
|
finally:
|
|
win32gui.DeleteObject(bitmap.GetHandle())
|
|
target.DeleteDC()
|
|
source.DeleteDC()
|
|
win32gui.ReleaseDC(hwnd, dc)
|
|
|
|
|
|
class WindowDriver:
|
|
def __init__(self, pid, output):
|
|
self.pid, self.output = pid, Path(output)
|
|
self.output.mkdir(parents=True, exist_ok=True)
|
|
self.app = Application(backend='uia').connect(process=pid, timeout=15)
|
|
self.frames, self.durations, self.steps = [], [], []
|
|
self.font = ImageFont.truetype('C:/Windows/Fonts/msyh.ttc', 22)
|
|
if (self.output / 'frames.json').exists():
|
|
self.steps = json.loads((self.output / 'frames.json').read_text(encoding='utf-8'))
|
|
self.frames = [Image.open(self.output / f'frame-{s["frame"]:03d}.png').convert('RGB') for s in self.steps]
|
|
self.durations = [s['duration_ms'] for s in self.steps]
|
|
|
|
def windows(self):
|
|
handles = []
|
|
def collect(handle, _):
|
|
if win32gui.IsWindowVisible(handle) and win32process.GetWindowThreadProcessId(handle)[1] == self.pid:
|
|
left, top, right, bottom = win32gui.GetWindowRect(handle)
|
|
if right - left > 100 and bottom - top > 80:
|
|
handles.append(handle)
|
|
win32gui.EnumWindows(collect, None)
|
|
return [self.app.window(handle=handle).wrapper_object() for handle in handles]
|
|
|
|
def main(self):
|
|
deadline = time.monotonic() + 15
|
|
while time.monotonic() < deadline:
|
|
for window in self.windows():
|
|
if window.window_text().startswith('DxDesigner '):
|
|
return window
|
|
time.sleep(.2)
|
|
raise TimeoutError('Application main window not found')
|
|
|
|
def find(self, text=None, *, kind=None, automation_id=None, window=None, optional=False):
|
|
root = window or self.main()
|
|
# Do not recursively enumerate virtual Qt table cells through UIA.
|
|
def walk(parent, depth=0):
|
|
if depth > 12:
|
|
return
|
|
for child in parent.children():
|
|
yield child
|
|
if child.element_info.control_type not in {
|
|
'Table', 'DataGrid', 'List', 'ScrollBar', 'Header', 'Edit',
|
|
'Button', 'Text', 'CheckBox', 'ComboBox', 'TitleBar', 'MenuBar', 'MenuItem'}:
|
|
yield from walk(child, depth + 1)
|
|
controls = walk(root)
|
|
matches = [c for c in controls if (kind is None or c.element_info.control_type == kind)
|
|
and (text is None or c.window_text() == text)
|
|
and (automation_id is None or c.element_info.automation_id == automation_id
|
|
or c.element_info.automation_id.endswith('.' + automation_id))
|
|
and c.is_visible()]
|
|
if not matches and optional:
|
|
return None
|
|
if len(matches) != 1:
|
|
raise LookupError(f'Expected one {text!r}/{automation_id!r}, got {len(matches)}')
|
|
return matches[0]
|
|
|
|
def click(self, control):
|
|
# Posted window mouse events allow modal dialogs without blocking the driver.
|
|
self.point(control)
|
|
|
|
def point(self, control, *, dx=None, dy=None, double=False):
|
|
rect = control.rectangle()
|
|
if dx is None and control.element_info.control_type in ('ListItem', 'DataItem'):
|
|
dx = 20 # Qt may expose the full elided text width beyond its viewport.
|
|
root = control
|
|
while not root.handle or not win32gui.IsWindow(root.handle):
|
|
root = root.parent()
|
|
x = rect.left + (dx if dx is not None else rect.width() // 2)
|
|
y = rect.top + (dy if dy is not None else rect.height() // 2)
|
|
x, y = win32gui.ScreenToClient(root.handle, (x, y))
|
|
value = (y << 16) | (x & 0xFFFF)
|
|
for _ in range(2 if double else 1):
|
|
win32gui.PostMessage(root.handle, win32con.WM_LBUTTONDOWN, win32con.MK_LBUTTON, value)
|
|
win32gui.PostMessage(root.handle, win32con.WM_LBUTTONUP, 0, value)
|
|
time.sleep(.2)
|
|
|
|
def type_text(self, control, text):
|
|
control.set_edit_text(text)
|
|
time.sleep(.2)
|
|
|
|
def record(self, caption, *, highlight=None, duration=1600):
|
|
main = self.main()
|
|
image = window_image(main.handle)
|
|
left, top, _, _ = win32gui.GetWindowRect(main.handle)
|
|
# Owned modal/detail windows are composited from their own surfaces.
|
|
for other in reversed(self.windows()):
|
|
if other.handle == main.handle:
|
|
continue
|
|
other_left, other_top, _, _ = win32gui.GetWindowRect(other.handle)
|
|
image.paste(window_image(other.handle), (other_left - left, other_top - top))
|
|
draw = ImageDraw.Draw(image)
|
|
if highlight is not None:
|
|
rect = highlight.rectangle()
|
|
box = (rect.left - left - 3, rect.top - top - 3,
|
|
rect.right - left + 3, rect.bottom - top + 3)
|
|
draw.rounded_rectangle(box, radius=5, outline='#e35d30', width=4)
|
|
# Fixed canvas dimensions allow window resizing without desktop capture.
|
|
image.thumbnail((1280, 900), Image.Resampling.LANCZOS)
|
|
canvas = Image.new('RGB', (1280, 980), '#f3f5f8')
|
|
canvas.paste(image, ((1280 - image.width) // 2, 0))
|
|
draw = ImageDraw.Draw(canvas)
|
|
draw.rectangle((0, 900, 1280, 980), fill='#172b45')
|
|
draw.text((24, 914), caption, font=self.font, fill='white')
|
|
draw.text((24, 946), '窗口操作实录 · 等待过程已压缩 · 工程副本验收',
|
|
font=ImageFont.truetype('C:/Windows/Fonts/msyh.ttc', 16), fill='#bdcfdf')
|
|
index = len(self.frames)
|
|
canvas.save(self.output / f'frame-{index:03d}.png')
|
|
self.frames.append(canvas)
|
|
self.durations.append(duration)
|
|
self.steps.append(dict(frame=index, caption=caption, duration_ms=duration, wall_time=time.time()))
|
|
(self.output / 'frames.json').write_text(json.dumps(self.steps, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
|
|
def save_gif(self, path):
|
|
if not self.frames:
|
|
raise RuntimeError('No window frames captured')
|
|
palette = self.frames[0].quantize(colors=160)
|
|
images = [frame.quantize(palette=palette, dither=Image.Dither.NONE) for frame in self.frames]
|
|
images[0].save(path, save_all=True, append_images=images[1:],
|
|
duration=self.durations, loop=0, optimize=True, disposal=2)
|