"""External Windows UI driver and PrintWindow-only instructional capture. This utility cannot import application internals or fabricate task state. It operates the executable through Windows accessibility and records only windows owned by that executable. Workbook edits and acceptance assertions belong to the calling scenario. """ from __future__ import annotations import ctypes from ctypes import wintypes import json from pathlib import Path import time from PIL import Image, ImageDraw, ImageFont from pywinauto import Application import win32con import win32gui import win32process import win32ui def window_image(hwnd): """Read this HWND's drawing surface, never the desktop framebuffer.""" left, top, right, bottom = win32gui.GetWindowRect(hwnd) width, height = right - left, bottom - top dc = win32gui.GetWindowDC(hwnd) source = win32ui.CreateDCFromHandle(dc) target = source.CreateCompatibleDC() bitmap = win32ui.CreateBitmap() bitmap.CreateCompatibleBitmap(source, width, height) target.SelectObject(bitmap) try: print_window = ctypes.windll.user32.PrintWindow print_window.argtypes = (wintypes.HWND, wintypes.HDC, wintypes.UINT) print_window.restype = wintypes.BOOL if not print_window(hwnd, target.GetSafeHdc(), 2): raise RuntimeError(f'PrintWindow failed for {hwnd}') return Image.frombuffer('RGB', (width, height), bitmap.GetBitmapBits(True), 'raw', 'BGRX', 0, 1).copy() finally: win32gui.DeleteObject(bitmap.GetHandle()) target.DeleteDC() source.DeleteDC() win32gui.ReleaseDC(hwnd, dc) class WindowDriver: def __init__(self, pid, output): self.pid, self.output = pid, Path(output) self.output.mkdir(parents=True, exist_ok=True) self.app = Application(backend='uia').connect(process=pid, timeout=15) self.frames, self.durations, self.steps = [], [], [] self.font = ImageFont.truetype('C:/Windows/Fonts/msyh.ttc', 22) if (self.output / 'frames.json').exists(): self.steps = json.loads((self.output / 'frames.json').read_text(encoding='utf-8')) self.frames = [Image.open(self.output / f'frame-{s["frame"]:03d}.png').convert('RGB') for s in self.steps] self.durations = [s['duration_ms'] for s in self.steps] def windows(self): handles = [] def collect(handle, _): if win32gui.IsWindowVisible(handle) and win32process.GetWindowThreadProcessId(handle)[1] == self.pid: left, top, right, bottom = win32gui.GetWindowRect(handle) if right - left > 100 and bottom - top > 80: handles.append(handle) win32gui.EnumWindows(collect, None) return [self.app.window(handle=handle).wrapper_object() for handle in handles] def main(self): deadline = time.monotonic() + 15 while time.monotonic() < deadline: for window in self.windows(): if window.window_text().startswith('DxDesigner '): return window time.sleep(.2) raise TimeoutError('Application main window not found') def find(self, text=None, *, kind=None, automation_id=None, window=None): root = window or self.main() # Do not recursively enumerate virtual Qt table cells through UIA. def walk(parent, depth=0): if depth > 12: return for child in parent.children(): yield child if child.element_info.control_type not in { 'Table', 'DataGrid', 'List', 'ScrollBar', 'Header', 'Edit', 'Button', 'Text', 'CheckBox', 'ComboBox', 'TitleBar', 'MenuBar', 'MenuItem'}: yield from walk(child, depth + 1) controls = walk(root) matches = [c for c in controls if (kind is None or c.element_info.control_type == kind) and (text is None or c.window_text() == text) and (automation_id is None or c.element_info.automation_id == automation_id or c.element_info.automation_id.endswith('.' + automation_id)) and c.is_visible()] if len(matches) != 1: raise LookupError(f'Expected one {text!r}/{automation_id!r}, got {len(matches)}') return matches[0] def click(self, control): # Posted window mouse events allow modal dialogs without blocking the driver. self.point(control) def point(self, control, *, dx=None, dy=None, double=False): rect = control.rectangle() if dx is None and control.element_info.control_type in ('ListItem', 'DataItem'): dx = 20 # Qt may expose the full elided text width beyond its viewport. root = control while not root.handle or not win32gui.IsWindow(root.handle): root = root.parent() x = rect.left + (dx if dx is not None else rect.width() // 2) y = rect.top + (dy if dy is not None else rect.height() // 2) x, y = win32gui.ScreenToClient(root.handle, (x, y)) value = (y << 16) | (x & 0xFFFF) for _ in range(2 if double else 1): win32gui.PostMessage(root.handle, win32con.WM_LBUTTONDOWN, win32con.MK_LBUTTON, value) win32gui.PostMessage(root.handle, win32con.WM_LBUTTONUP, 0, value) time.sleep(.2) def type_text(self, control, text): control.set_edit_text(text) time.sleep(.2) def record(self, caption, *, highlight=None, duration=1600): main = self.main() image = window_image(main.handle) left, top, _, _ = win32gui.GetWindowRect(main.handle) # Owned modal/detail windows are composited from their own surfaces. for other in reversed(self.windows()): if other.handle == main.handle: continue other_left, other_top, _, _ = win32gui.GetWindowRect(other.handle) image.paste(window_image(other.handle), (other_left - left, other_top - top)) draw = ImageDraw.Draw(image) if highlight is not None: rect = highlight.rectangle() box = (rect.left - left - 3, rect.top - top - 3, rect.right - left + 3, rect.bottom - top + 3) draw.rounded_rectangle(box, radius=5, outline='#e35d30', width=4) # Fixed canvas dimensions allow window resizing without desktop capture. image.thumbnail((1280, 900), Image.Resampling.LANCZOS) canvas = Image.new('RGB', (1280, 980), '#f3f5f8') canvas.paste(image, ((1280 - image.width) // 2, 0)) draw = ImageDraw.Draw(canvas) draw.rectangle((0, 900, 1280, 980), fill='#172b45') draw.text((24, 914), caption, font=self.font, fill='white') draw.text((24, 946), '窗口操作实录 · 等待过程已压缩 · 工程副本验收', font=ImageFont.truetype('C:/Windows/Fonts/msyh.ttc', 16), fill='#bdcfdf') index = len(self.frames) canvas.save(self.output / f'frame-{index:03d}.png') self.frames.append(canvas) self.durations.append(duration) self.steps.append(dict(frame=index, caption=caption, duration_ms=duration, wall_time=time.time())) (self.output / 'frames.json').write_text(json.dumps(self.steps, ensure_ascii=False, indent=2), encoding='utf-8') def save_gif(self, path): if not self.frames: raise RuntimeError('No window frames captured') palette = self.frames[0].quantize(colors=160) images = [frame.quantize(palette=palette, dither=Image.Dither.NONE) for frame in self.frames] images[0].save(path, save_all=True, append_images=images[1:], duration=self.durations, loop=0, optimize=True, disposal=2)