poc(qml-dev): AX-first interaction model + HID safety guards

- suffix-first identifier matching (container names no longer resolve to their first descendant) - scroll: pure-AX scrollbar drive (--to/--by), works on background windows - click/rightclick/hover/key/type/mouse*: real HID events (postToPid is dropped by AppKit/Qt) — auto-activate target, restore cursor after; guarded: refuse when the user was active <2s ago, when the app cannot become frontmost, or when another window covers the target point - key: esc/enter/tab/arrows/home/end/page + back/forward (mouse nav buttons) - skill: interaction hierarchy (pure AX first), popup hot-reload and occluded-window screenshot gotchas
2026-08-27 07:01:14 +00:00 · 2026-07-15 16:42:16 +03:00
parent e400143790
commit e2ca9c7905
2 changed files with 235 additions and 15 deletions
@@ -108,9 +108,20 @@ TextField, CheckBox, ComboBox…) automatically; plain `Item`/`Rectangle`/
  and a restart via `scripts/storybook-agent.sh stop && … start <Page>`.
 - Values render locale-formatted (decimal comma on European locales):
  setting `0.2` reads back as `0,2`.
- `press` works on AbstractButton-derived controls. If a press has no
-  effect, check `read … | grep Enabled` first, then fall back to `click`
-  (needs the window unobstructed; synthesized events go to screen coords).
+- **Interaction hierarchy — pure AX first.** `press`, `set`, `scroll`,
+  `read`, `tree`, `screenshot` are pure accessibility calls: they work on
+  background windows, never move the cursor, never steal focus, and cannot
+  conflict with the human using the machine. Use them for ~everything.
+- The real-input commands (`click`, `rightclick`, `hover`, `key`, `type`,
+  `mousedown/up/move`) inject global HID events: they require the app
+  frontmost and are inherently disruptive. They self-protect — refusing when
+  the human used mouse/keyboard in the last 2s, when the app can't become
+  frontmost, or when another window covers the target — and restore the
+  cursor position afterwards. Treat a refusal as "wait and retry", never
+  pass --force on a machine a human is actively using. If a flow needs many
+  real-input events while the human works, pause and coordinate with them.
+- `press` works on AbstractButton-derived controls; if it has no effect,
+  check `read … | grep Enabled` before reaching for `click`.
 - **Page knobs can pin state**: storybook pages often install `Binding`
  elements tying component state to the page's own controls (combos, text
  knobs). Interactions that imperatively write the same properties (e.g.
@@ -119,6 +130,13 @@ TextField, CheckBox, ComboBox…) automatically; plain `Item`/`Rectangle`/
  page knobs instead, or check the page QML before concluding a bug.
 - Reload errors and QML warnings land in `$TMPDIR/storybook-agent.log` —
  check it whenever the tree looks stale or empty.
+- Hot reload can be unreliable for popup-heavy pages after production-QML
+  edits (stale component cache; popups relaunch from old components). If a
+  change doesn't show, restart the harness (`storybook-agent.sh stop` +
+  `start <Page>`) instead of debugging ghosts.
+- An occluded window defers Qt layout polish, so screenshots can capture a
+  half-laid-out UI. Run `ax activate --pid P` (no cursor move) before
+  `ax screenshot` when pixel-accuracy matters.
 - The AX tree also contains Storybook's own chrome (sidebar, knobs pane).
  Filter with `--filter <Component>` or match identifier substrings under
  your component's objectName.
@@ -91,13 +91,98 @@ func collectMatches(_ el: AXUIElement, idSubstr: String, path: String, into: ino
 func findOne(app: AXUIElement, idSubstr: String) -> AXUIElement? {
    var matches: [(AXUIElement, String)] = []
    collectMatches(app, idSubstr: idSubstr, path: "", into: &matches)
-    if matches.count > 1 {
-        FileHandle.standardError.write("warning: \(matches.count) matches for '\(idSubstr)', using first:\n".data(using: .utf8)!)
-        for (_, p) in matches {
+    guard !matches.isEmpty else { return nil }
+    // Identifiers are ancestor paths, so a container's name is a substring of
+    // every descendant's identifier. Prefer elements whose identifier *ends*
+    // with the query (the element itself), then the shortest identifier.
+    func rank(_ path: String) -> Int {
+        let id = path.split(separator: "/").last.map(String.init) ?? path
+        if id == idSubstr || id.hasSuffix("." + idSubstr) || id.hasSuffix(idSubstr) { return 0 }
+        return 1
+    }
+    matches.sort { a, b in
+        let (ra, rb) = (rank(a.1), rank(b.1))
+        if ra != rb { return ra < rb }
+        return a.1.count < b.1.count
+    }
+    let best = matches[0]
+    let ambiguous = matches.filter { rank($0.1) == rank(best.1) && $0.1.count == best.1.count }
+    if ambiguous.count > 1 {
+        FileHandle.standardError.write("warning: \(ambiguous.count) equally-ranked matches for '\(idSubstr)', using first:\n".data(using: .utf8)!)
+        for (_, p) in ambiguous.prefix(5) {
            FileHandle.standardError.write("  \(p)\n".data(using: .utf8)!)
        }
    }
-    return matches.first?.0
+    return best.0
+}
+
+// MARK: - HID event helpers
+//
+// postToPid is unreliable for AppKit/Qt (events bypass window-server routing
+// and get dropped), so real input goes through the HID tap. To keep this
+// unobtrusive: the target app is activated first (required for routing), the
+// user's cursor position is saved, and restored immediately after the events.
+
+func currentCursor() -> CGPoint {
+    CGEvent(source: nil)?.location ?? .zero
+}
+
+func activateApp(_ pid: pid_t) {
+    let app = AXUIElementCreateApplication(pid)
+    AXUIElementSetAttributeValue(app, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
+    usleep(150_000)
+}
+
+// Refuses to fire real input while the human is using the machine, if the
+// target app could not become frontmost, or if the target point is covered
+// by another app's window. HID events are global — a misfire lands in the
+// user's windows, so failing loudly beats guessing. --force skips the idle check.
+func hidGuard(_ pid: pid_t, point: CGPoint?) {
+    if !args.contains("--force") {
+        let types: [CGEventType] = [.mouseMoved, .leftMouseDown, .keyDown, .scrollWheel]
+        let idle = types.map {
+            CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: $0)
+        }.min() ?? .infinity
+        if idle < 2.0 {
+            fail("user input detected \(String(format: "%.1f", idle))s ago — refusing to inject events while the human is active; retry when idle or pass --force")
+        }
+    }
+    activateApp(pid)
+    if let front = NSWorkspace.shared.frontmostApplication?.processIdentifier, front != pid {
+        fail("target app (pid \(pid)) could not become frontmost (frontmost is pid \(front)) — real input would land in the wrong app")
+    }
+    if let p = point {
+        let winList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
+        for win in winList where ((win[kCGWindowLayer as String] as? Int) ?? 1) == 0 {
+            guard let b = win[kCGWindowBounds as String] as? [String: Any],
+                  let rect = CGRect(dictionaryRepresentation: b as CFDictionary) else { continue }
+            if rect.contains(p) {
+                let owner = win[kCGWindowOwnerPID as String] as? pid_t ?? -1
+                if owner != pid {
+                    fail("target point is covered by another app's window (pid \(owner), \(win[kCGWindowOwnerName as String] ?? "?")) — refusing to click through it")
+                }
+                break
+            }
+        }
+    }
+}
+
+func withCursorRestore(_ body: () -> Void) {
+    let saved = currentCursor()
+    body()
+    usleep(30_000)
+    CGWarpMouseCursorPosition(saved)
+    CGAssociateMouseAndMouseCursorPosition(1)
+}
+
+func postMouse(_ type: CGEventType, _ button: CGMouseButton, at point: CGPoint,
+               buttonNumber: Int64? = nil, clickState: Int64 = 1) {
+    guard let ev = CGEvent(mouseEventSource: nil, mouseType: type,
+                           mouseCursorPosition: point, mouseButton: button) else { fail("CGEvent failed") }
+    if type != .mouseMoved { ev.setIntegerValueField(.mouseEventClickState, value: clickState) }
+    if let n = buttonNumber { ev.setIntegerValueField(.mouseEventButtonNumber, value: n) }
+    ev.post(tap: .cghidEventTap)
+    usleep(30_000)
 }

 // MARK: - Output
@@ -128,8 +213,19 @@ guard !args.isEmpty else {
      read      --pid P --id S             dump one element's attributes
      press     --pid P --id S             AXPress on first match
      set       --pid P --id S --value V   set AXValue (text fields)
-      click     --pid P --id S             synthesized mouse click at center
-      type      --text V                   synthesized keystrokes (focused el)
+      activate  --pid P                    bring app frontmost (no cursor move)
+      scroll    --pid P --id S (--to 0..1 | --by D)  pure-AX scrollbar drive
+      click     --pid P --id S             real click; activates app, cursor
+                                           warped back to user position after
+      rightclick --pid P --id S            right click (context menus); same
+      hover     --pid P --id S             mouse-move over element (Qt hover)
+      mousedown|mouseup|mousemove --pid P (--id S | --x N --y N)
+                                           drag primitives (cursor restored on mouseup)
+      key       --pid P --key K [--id S]   esc|enter|tab|space|backspace|arrows|
+                                           home|end|pageup|pagedown|back|forward
+                                           (back/forward = mouse nav buttons 4/5)
+      type      --pid P --text V           real keystrokes into app's focused field
+                                           (prefer `set` for text fields)
      screenshot --pid P --out FILE        capture app's front window
    """)
    exit(0)
@@ -224,20 +320,126 @@ case "set":
    guard err == .success else { fail("set AXValue failed: \(err.rawValue)") }
    print("ok")

-case "click":
+case "activate":
+    // Bring the app frontmost. Does not move the cursor.
+    let err = AXUIElementSetAttributeValue(appElement(), kAXFrontmostAttribute as CFString, kCFBooleanTrue)
+    guard err == .success else { fail("activate failed: \(err.rawValue)") }
+    print("ok")
+
+case "click", "rightclick", "hover":
+    guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") }
    guard let el = findOne(app: appElement(), idSubstr: requireOpt("id")) else { fail("no match") }
    guard let f = frame(el) else { fail("element has no frame") }
    let center = CGPoint(x: f.midX, y: f.midY)
-    for type in [CGEventType.leftMouseDown, .leftMouseUp] {
-        guard let ev = CGEvent(mouseEventSource: nil, mouseType: type,
-                               mouseCursorPosition: center, mouseButton: .left) else { fail("CGEvent failed") }
-        ev.post(tap: .cghidEventTap)
-        usleep(30_000)
+    hidGuard(pid, point: center)
+    withCursorRestore {
+        switch command {
+        case "click":
+            postMouse(.leftMouseDown, .left, at: center)
+            postMouse(.leftMouseUp, .left, at: center)
+        case "rightclick":
+            postMouse(.rightMouseDown, .right, at: center)
+            postMouse(.rightMouseUp, .right, at: center)
+        default: // hover
+            postMouse(.mouseMoved, .left, at: center)
+        }
+    }
+    print("ok")
+
+case "mousedown", "mouseup", "mousemove":
+    // Low-level primitives for drag / press-and-hold. Target by --id, or
+    // explicit --x/--y (screen coords), e.g. mousedown on A, mousemove to B, mouseup.
+    // NOTE: no cursor restore here — a drag sequence needs the cursor to stay
+    // where the caller put it; restore happens implicitly on the final mouseup.
+    guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") }
+    var point: CGPoint
+    if let xs = opt("x"), let ys = opt("y"), let x = Double(xs), let y = Double(ys) {
+        point = CGPoint(x: x, y: y)
+    } else if let idSub = opt("id"), let el = findOne(app: appElement(), idSubstr: idSub), let f = frame(el) {
+        point = CGPoint(x: f.midX, y: f.midY)
+    } else {
+        fail("need --id or --x/--y")
+    }
+    hidGuard(pid, point: point)
+    switch command {
+    case "mousedown": postMouse(.leftMouseDown, .left, at: point)
+    case "mouseup":
+        withCursorRestore { postMouse(.leftMouseUp, .left, at: point) }
+    default: postMouse(.leftMouseDragged, .left, at: point)
+    }
+    print("ok")
+
+case "scroll":
+    // Pure AX: drives the scrollbar's value — no cursor, no focus change.
+    // Matches an AXScrollBar directly, or finds one among the element's
+    // descendants. --to 0..1 sets absolute position; --by DELTA adjusts.
+    // Containers (Pane, ScrollView) often have no AX element of their own —
+    // search every identifier match and take the first that is, or contains,
+    // an AXScrollBar.
+    var matches: [(AXUIElement, String)] = []
+    collectMatches(appElement(), idSubstr: requireOpt("id"), path: "", into: &matches)
+    var bar: AXUIElement?
+    outer: for (el, _) in matches {
+        if stringAttr(el, kAXRoleAttribute) == "AXScrollBar" { bar = el; break }
+        var queue = children(el)
+        while !queue.isEmpty {
+            let c = queue.removeFirst()
+            if stringAttr(c, kAXRoleAttribute) == "AXScrollBar" { bar = c; break outer }
+            queue.append(contentsOf: children(c))
+        }
+    }
+    guard let scrollbar = bar else { fail("no AXScrollBar found among matches") }
+    let current = (attr(scrollbar, kAXValueAttribute) as? NSNumber)?.doubleValue ?? 0
+    var target = current
+    if let to = opt("to"), let v = Double(to) { target = v }
+    else if let by = opt("by"), let v = Double(by) { target = current + v }
+    else { fail("need --to 0..1 or --by DELTA") }
+    target = min(max(target, 0), 1)
+    let err = AXUIElementSetAttributeValue(scrollbar, kAXValueAttribute as CFString, NSNumber(value: target))
+    guard err == .success else { fail("set scrollbar value failed: \(err.rawValue)") }
+    print("ok \(current) -> \(target)")
+
+case "key":
+    // Named keys by virtual keycode; "back"/"forward" are the mouse
+    // navigation buttons (4/5), which Qt maps to back/forward navigation.
+    // Keyboard events route to the focused window, so the app is activated.
+    guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") }
+    let name = requireOpt("key").lowercased()
+    let keycodes: [String: CGKeyCode] = [
+        "esc": 53, "escape": 53, "enter": 36, "return": 36, "tab": 48,
+        "space": 49, "backspace": 51, "delete": 51,
+        "left": 123, "right": 124, "down": 125, "up": 126,
+        "home": 115, "end": 119, "pageup": 116, "pagedown": 121,
+    ]
+    hidGuard(pid, point: nil)
+    if name == "back" || name == "forward" {
+        let buttonNumber: Int64 = name == "back" ? 3 : 4
+        var point = currentCursor()
+        if let idSub = opt("id"), let el = findOne(app: appElement(), idSubstr: idSub), let f = frame(el) {
+            point = CGPoint(x: f.midX, y: f.midY)
+        } else if let win = attr(appElement(), kAXMainWindowAttribute).map({ $0 as! AXUIElement }),
+                  let f = frame(win) {
+            point = CGPoint(x: f.midX, y: f.midY)
+        }
+        withCursorRestore {
+            postMouse(.otherMouseDown, .center, at: point, buttonNumber: buttonNumber)
+            postMouse(.otherMouseUp, .center, at: point, buttonNumber: buttonNumber)
+        }
+    } else {
+        guard let code = keycodes[name] else { fail("unknown key: \(name)") }
+        for keyDown in [true, false] {
+            guard let ev = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: keyDown) else { fail("CGEvent failed") }
+            ev.post(tap: .cghidEventTap)
+        }
    }
    print("ok")

 case "type":
+    // Real keystrokes via the HID tap; the app is activated first so the
+    // events land in its focused field. Prefer `set` (pure AX) when possible.
+    guard let pidStr = opt("pid"), let pid = pid_t(pidStr) else { fail("missing/bad --pid") }
    let text = requireOpt("text")
+    hidGuard(pid, point: nil)
    for scalar in text.unicodeScalars {
        var chars = [UniChar](String(scalar).utf16)
        for keyDown in [true, false] {