diff --git a/.claude/skills/qml-dev/SKILL.md b/.claude/skills/qml-dev/SKILL.md index d9eda2b49b..c333a24ebf 100644 --- a/.claude/skills/qml-dev/SKILL.md +++ b/.claude/skills/qml-dev/SKILL.md @@ -108,9 +108,20 @@ TextField, CheckBox, ComboBox…) automatically; plain `Item`/`Rectangle`/ and a restart via `scripts/storybook-agent.sh stop && … start `. - Values render locale-formatted (decimal comma on European locales): setting `0.2` reads back as `0,2`. -- `press` works on AbstractButton-derived controls. If a press has no - effect, check `read … | grep Enabled` first, then fall back to `click` - (needs the window unobstructed; synthesized events go to screen coords). +- **Interaction hierarchy — pure AX first.** `press`, `set`, `scroll`, + `read`, `tree`, `screenshot` are pure accessibility calls: they work on + background windows, never move the cursor, never steal focus, and cannot + conflict with the human using the machine. Use them for ~everything. +- The real-input commands (`click`, `rightclick`, `hover`, `key`, `type`, + `mousedown/up/move`) inject global HID events: they require the app + frontmost and are inherently disruptive. They self-protect — refusing when + the human used mouse/keyboard in the last 2s, when the app can't become + frontmost, or when another window covers the target — and restore the + cursor position afterwards. Treat a refusal as "wait and retry", never + pass --force on a machine a human is actively using. If a flow needs many + real-input events while the human works, pause and coordinate with them. +- `press` works on AbstractButton-derived controls; if it has no effect, + check `read … | grep Enabled` before reaching for `click`. - **Page knobs can pin state**: storybook pages often install `Binding` elements tying component state to the page's own controls (combos, text knobs). Interactions that imperatively write the same properties (e.g. @@ -119,6 +130,13 @@ TextField, CheckBox, ComboBox…) automatically; plain `Item`/`Rectangle`/ page knobs instead, or check the page QML before concluding a bug. - Reload errors and QML warnings land in `$TMPDIR/storybook-agent.log` — check it whenever the tree looks stale or empty. +- Hot reload can be unreliable for popup-heavy pages after production-QML + edits (stale component cache; popups relaunch from old components). If a + change doesn't show, restart the harness (`storybook-agent.sh stop` + + `start `) instead of debugging ghosts. +- An occluded window defers Qt layout polish, so screenshots can capture a + half-laid-out UI. Run `ax activate --pid P` (no cursor move) before + `ax screenshot` when pixel-accuracy matters. - The AX tree also contains Storybook's own chrome (sidebar, knobs pane). Filter with `--filter ` or match identifier substrings under your component's objectName. diff --git a/tools/ax/main.swift b/tools/ax/main.swift index 3a802bb812..d6d7f9af38 100644 --- a/tools/ax/main.swift +++ b/tools/ax/main.swift @@ -91,13 +91,98 @@ func collectMatches(_ el: AXUIElement, idSubstr: String, path: String, into: ino func findOne(app: AXUIElement, idSubstr: String) -> AXUIElement? { var matches: [(AXUIElement, String)] = [] collectMatches(app, idSubstr: idSubstr, path: "", into: &matches) - if matches.count > 1 { - FileHandle.standardError.write("warning: \(matches.count) matches for '\(idSubstr)', using first:\n".data(using: .utf8)!) - for (_, p) in matches { + guard !matches.isEmpty else { return nil } + // Identifiers are ancestor paths, so a container's name is a substring of + // every descendant's identifier. Prefer elements whose identifier *ends* + // with the query (the element itself), then the shortest identifier. + func rank(_ path: String) -> Int { + let id = path.split(separator: "/").last.map(String.init) ?? path + if id == idSubstr || id.hasSuffix("." + idSubstr) || id.hasSuffix(idSubstr) { return 0 } + return 1 + } + matches.sort { a, b in + let (ra, rb) = (rank(a.1), rank(b.1)) + if ra != rb { return ra < rb } + return a.1.count < b.1.count + } + let best = matches[0] + let ambiguous = matches.filter { rank($0.1) == rank(best.1) && $0.1.count == best.1.count } + if ambiguous.count > 1 { + FileHandle.standardError.write("warning: \(ambiguous.count) equally-ranked matches for '\(idSubstr)', using first:\n".data(using: .utf8)!) + for (_, p) in ambiguous.prefix(5) { FileHandle.standardError.write(" \(p)\n".data(using: .utf8)!) } } - return matches.first?.0 + return best.0 +} + +// MARK: - HID event helpers +// +// postToPid is unreliable for AppKit/Qt (events bypass window-server routing +// and get dropped), so real input goes through the HID tap. To keep this +// unobtrusive: the target app is activated first (required for routing), the +// user's cursor position is saved, and restored immediately after the events. + +func currentCursor() -> CGPoint { + CGEvent(source: nil)?.location ?? .zero +} + +func activateApp(_ pid: pid_t) { + let app = AXUIElementCreateApplication(pid) + AXUIElementSetAttributeValue(app, kAXFrontmostAttribute as CFString, kCFBooleanTrue) + usleep(150_000) +} + +// Refuses to fire real input while the human is using the machine, if the +// target app could not become frontmost, or if the target point is covered +// by another app's window. HID events are global — a misfire lands in the +// user's windows, so failing loudly beats guessing. --force skips the idle check. +func hidGuard(_ pid: pid_t, point: CGPoint?) { + if !args.contains("--force") { + let types: [CGEventType] = [.mouseMoved, .leftMouseDown, .keyDown, .scrollWheel] + let idle = types.map { + CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: $0) + }.min() ?? .infinity + if idle < 2.0 { + fail("user input detected \(String(format: "%.1f", idle))s ago — refusing to inject events while the human is active; retry when idle or pass --force") + } + } + activateApp(pid) + if let front = NSWorkspace.shared.frontmostApplication?.processIdentifier, front != pid { + fail("target app (pid \(pid)) could not become frontmost (frontmost is pid \(front)) — real input would land in the wrong app") + } + if let p = point { + let winList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] + for win in winList where ((win[kCGWindowLayer as String] as? Int) ?? 1) == 0 { + guard let b = win[kCGWindowBounds as String] as? [String: Any], + let rect = CGRect(dictionaryRepresentation: b as CFDictionary) else { continue } + if rect.contains(p) { + let owner = win[kCGWindowOwnerPID as String] as? pid_t ?? -1 + if owner != pid { + fail("target point is covered by another app's window (pid \(owner), \(win[kCGWindowOwnerName as String] ?? "?")) — refusing to click through it") + } + break + } + } + } +} + +func withCursorRestore(_ body: () -> Void) { + let saved = currentCursor() + body() + usleep(30_000) + CGWarpMouseCursorPosition(saved) + CGAssociateMouseAndMouseCursorPosition(1) +} + +func postMouse(_ type: CGEventType, _ button: CGMouseButton, at point: CGPoint, + buttonNumber: Int64? = nil, clickState: Int64 = 1) { + guard let ev = CGEvent(mouseEventSource: nil, mouseType: type, + mouseCursorPosition: point, mouseButton: button) else { fail("CGEvent failed") } + if type != .mouseMoved { ev.setIntegerValueField(.mouseEventClickState, value: clickState) } + if let n = buttonNumber { ev.setIntegerValueField(.mouseEventButtonNumber, value: n) } + ev.post(tap: .cghidEventTap) + usleep(30_000) } // MARK: - Output @@ -128,8 +213,19 @@ guard !args.isEmpty else { read --pid P --id S dump one element's attributes press --pid P --id S AXPress on first match set --pid P --id S --value V set AXValue (text fields) - click --pid P --id S synthesized mouse click at center - type --text V synthesized keystrokes (focused el) + activate --pid P bring app frontmost (no cursor move) + scroll --pid P --id S (--to 0..1 | --by D) pure-AX scrollbar drive + click --pid P --id S real click; activates app, cursor + warped back to user position after + rightclick --pid P --id S right click (context menus); same + hover --pid P --id S mouse-move over element (Qt hover) + mousedown|mouseup|mousemove --pid P (--id S | --x N --y N) + drag primitives (cursor restored on mouseup) + key --pid P --key K [--id S] esc|enter|tab|space|backspace|arrows| + home|end|pageup|pagedown|back|forward + (back/forward = mouse nav buttons 4/5) + type --pid P --text V real keystrokes into app's focused field + (prefer `set` for text fields) screenshot --pid P --out FILE capture app's front window """) exit(0) @@ -224,20 +320,126 @@ case "set": guard err == .success else { fail("set AXValue failed: \(err.rawValue)") } print("ok") -case "click": +case "activate": + // Bring the app frontmost. Does not move the cursor. + let err = AXUIElementSetAttributeValue(appElement(), kAXFrontmostAttribute as CFString, kCFBooleanTrue) + guard err == .success else { fail("activate failed: \(err.rawValue)") } + print("ok") + +case "click", "rightclick", "hover": + guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") } guard let el = findOne(app: appElement(), idSubstr: requireOpt("id")) else { fail("no match") } guard let f = frame(el) else { fail("element has no frame") } let center = CGPoint(x: f.midX, y: f.midY) - for type in [CGEventType.leftMouseDown, .leftMouseUp] { - guard let ev = CGEvent(mouseEventSource: nil, mouseType: type, - mouseCursorPosition: center, mouseButton: .left) else { fail("CGEvent failed") } - ev.post(tap: .cghidEventTap) - usleep(30_000) + hidGuard(pid, point: center) + withCursorRestore { + switch command { + case "click": + postMouse(.leftMouseDown, .left, at: center) + postMouse(.leftMouseUp, .left, at: center) + case "rightclick": + postMouse(.rightMouseDown, .right, at: center) + postMouse(.rightMouseUp, .right, at: center) + default: // hover + postMouse(.mouseMoved, .left, at: center) + } + } + print("ok") + +case "mousedown", "mouseup", "mousemove": + // Low-level primitives for drag / press-and-hold. Target by --id, or + // explicit --x/--y (screen coords), e.g. mousedown on A, mousemove to B, mouseup. + // NOTE: no cursor restore here — a drag sequence needs the cursor to stay + // where the caller put it; restore happens implicitly on the final mouseup. + guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") } + var point: CGPoint + if let xs = opt("x"), let ys = opt("y"), let x = Double(xs), let y = Double(ys) { + point = CGPoint(x: x, y: y) + } else if let idSub = opt("id"), let el = findOne(app: appElement(), idSubstr: idSub), let f = frame(el) { + point = CGPoint(x: f.midX, y: f.midY) + } else { + fail("need --id or --x/--y") + } + hidGuard(pid, point: point) + switch command { + case "mousedown": postMouse(.leftMouseDown, .left, at: point) + case "mouseup": + withCursorRestore { postMouse(.leftMouseUp, .left, at: point) } + default: postMouse(.leftMouseDragged, .left, at: point) + } + print("ok") + +case "scroll": + // Pure AX: drives the scrollbar's value — no cursor, no focus change. + // Matches an AXScrollBar directly, or finds one among the element's + // descendants. --to 0..1 sets absolute position; --by DELTA adjusts. + // Containers (Pane, ScrollView) often have no AX element of their own — + // search every identifier match and take the first that is, or contains, + // an AXScrollBar. + var matches: [(AXUIElement, String)] = [] + collectMatches(appElement(), idSubstr: requireOpt("id"), path: "", into: &matches) + var bar: AXUIElement? + outer: for (el, _) in matches { + if stringAttr(el, kAXRoleAttribute) == "AXScrollBar" { bar = el; break } + var queue = children(el) + while !queue.isEmpty { + let c = queue.removeFirst() + if stringAttr(c, kAXRoleAttribute) == "AXScrollBar" { bar = c; break outer } + queue.append(contentsOf: children(c)) + } + } + guard let scrollbar = bar else { fail("no AXScrollBar found among matches") } + let current = (attr(scrollbar, kAXValueAttribute) as? NSNumber)?.doubleValue ?? 0 + var target = current + if let to = opt("to"), let v = Double(to) { target = v } + else if let by = opt("by"), let v = Double(by) { target = current + v } + else { fail("need --to 0..1 or --by DELTA") } + target = min(max(target, 0), 1) + let err = AXUIElementSetAttributeValue(scrollbar, kAXValueAttribute as CFString, NSNumber(value: target)) + guard err == .success else { fail("set scrollbar value failed: \(err.rawValue)") } + print("ok \(current) -> \(target)") + +case "key": + // Named keys by virtual keycode; "back"/"forward" are the mouse + // navigation buttons (4/5), which Qt maps to back/forward navigation. + // Keyboard events route to the focused window, so the app is activated. + guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") } + let name = requireOpt("key").lowercased() + let keycodes: [String: CGKeyCode] = [ + "esc": 53, "escape": 53, "enter": 36, "return": 36, "tab": 48, + "space": 49, "backspace": 51, "delete": 51, + "left": 123, "right": 124, "down": 125, "up": 126, + "home": 115, "end": 119, "pageup": 116, "pagedown": 121, + ] + hidGuard(pid, point: nil) + if name == "back" || name == "forward" { + let buttonNumber: Int64 = name == "back" ? 3 : 4 + var point = currentCursor() + if let idSub = opt("id"), let el = findOne(app: appElement(), idSubstr: idSub), let f = frame(el) { + point = CGPoint(x: f.midX, y: f.midY) + } else if let win = attr(appElement(), kAXMainWindowAttribute).map({ $0 as! AXUIElement }), + let f = frame(win) { + point = CGPoint(x: f.midX, y: f.midY) + } + withCursorRestore { + postMouse(.otherMouseDown, .center, at: point, buttonNumber: buttonNumber) + postMouse(.otherMouseUp, .center, at: point, buttonNumber: buttonNumber) + } + } else { + guard let code = keycodes[name] else { fail("unknown key: \(name)") } + for keyDown in [true, false] { + guard let ev = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: keyDown) else { fail("CGEvent failed") } + ev.post(tap: .cghidEventTap) + } } print("ok") case "type": + // Real keystrokes via the HID tap; the app is activated first so the + // events land in its focused field. Prefer `set` (pure AX) when possible. + guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") } let text = requireOpt("text") + hidGuard(pid, point: nil) for scalar in text.unicodeScalars { var chars = [UniChar](String(scalar).utf16) for keyDown in [true, false] {