From fe0dca43336f9779b0efcd754d7f493cc4590071 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:46:58 -0500 Subject: [PATCH 01/24] feat(overlay): exit annotate mode via a far-right X control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pencil no longer doubles as the exit affordance. Annotate mode now renders Copy / Export / Clear / X, with the X in the conventional far-right "close this mode" slot and drawn as a plain action (the expanded pill and the live catcher already signal the mode, so the old permanently-lit toggle glyph was redundant). Idle is unchanged: a lone pencil that enters the mode. Unlike the note actions the X is never gated on `hasNotes` — with zero notes everything else is inert, so it must stay live. Drops the now-dead `pencilOff` glyph and `PillButton.isActive` (no remaining call site) per the no-dead-code rule. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/Overlay/OverlayView.swift | 66 +++++++++++----------- Sources/AnnotKit/Overlay/PillStyle.swift | 23 +++----- Tests/AnnotKitTests/LucideIconTests.swift | 1 - 3 files changed, 41 insertions(+), 49 deletions(-) diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index acff07f..a7f73eb 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -426,20 +426,25 @@ private struct AnnotationCard: View { /// capability — no dead buttons (CLAUDE.md) — so Agentation's `settings`/`eye` /// (no settings model, no preview capability here) are deliberately omitted. /// -/// Left to right: annotate toggle (always), then — ONLY while annotating — two -/// DISTINCT persist actions (Copy to the clipboard as markdown, and Export to -/// `AGENTATION_NOTES.md`) and a destructive clear. Idle shows JUST the pencil; the -/// tools appear when annotate mode is on, and are disabled/dimmed while `pending` -/// is empty (acting on zero notes is a no-op). A count badge overlays the pill -/// whenever notes exist. The pencil toggle exits annotate mode, so there is no -/// separate close/exit control. Copy and Export never clear the retained set (only -/// Clear does), so the same notes can be both copied and exported. Copy/Export -/// flow through host callbacks (they need a sink); toggle needs the controller -/// (activation); clear reads/writes the session directly. +/// Idle is JUST the pencil, so the resting affordance is one unambiguous "start +/// annotating". Annotate mode replaces it outright with the working set, left to +/// right: two DISTINCT persist actions (Copy to the clipboard as markdown, and +/// Export to `AGENTATION_NOTES.md`), a destructive clear, and the X that leaves +/// the mode. The X sits FAR RIGHT because that is where a "close this mode" +/// control is looked for, and it is drawn as a plain action rather than a lit-up +/// toggle — the expanded pill and the live catcher already say annotate mode is +/// on, so a permanently bright glyph would only add noise. The three note actions +/// are disabled/dimmed while `pending` is empty (acting on zero notes is a +/// no-op); the X is never gated, because leaving must always work. A count badge +/// overlays the pill whenever notes exist. Copy and Export never clear the +/// retained set (only Clear does), so the same notes can be both copied and +/// exported. Copy/Export flow through host callbacks (they need a sink); the mode +/// control needs the controller (activation); clear reads/writes the session +/// directly. /// /// The pill itself is rendered unconditionally and is NEVER gated on an entrance -/// flag, so it stays visible across idle<->annotate; the note-action cluster -/// animates in and out as annotate mode toggles. +/// flag, so it stays visible across idle<->annotate; only its contents swap as +/// annotate mode toggles. private struct ToolbarView: View { @ObservedObject var session: AnnotationSession let onToggle: () -> Void @@ -454,16 +459,10 @@ private struct ToolbarView: View { var body: some View { HStack(spacing: 2) { - PillButton( - icon: annotating ? .pencilOff : .pencil, - isActive: annotating, - tooltip: annotating ? "Stop annotating" : "Annotate", - action: onToggle - ) - - // Copy/Export/Clear appear ONLY in annotate mode; idle shows just the - // pencil. While annotating they dim/disable when there are no notes - // (acting on zero notes is a no-op). + // The two modes share no controls, so they are two whole rows rather + // than one row with conditional members: idle is the pencil alone, + // annotate is the note actions (dimmed/disabled with no notes to act + // on) closed by the exit X. if annotating { PillButton( icon: justCopied ? .check : .copy, @@ -481,6 +480,12 @@ private struct ToolbarView: View { PillButton(icon: .trash, isDestructive: true, isDisabled: !hasNotes, tooltip: "Clear notes") { session.clear() } + // Deliberately NOT gated on `hasNotes`: with zero notes every other + // control is inert, so the X is the only live thing left and must + // still work. + PillButton(icon: .close, tooltip: "Stop annotating", action: onToggle) + } else { + PillButton(icon: .pencil, tooltip: "Annotate", action: onToggle) } } // Idle shows ONE 28pt button, so the inset must be EVEN (8pt all around -> @@ -504,9 +509,9 @@ private struct ToolbarView: View { } } .shadow(color: .black.opacity(0.4), radius: 12, y: 8) - // Animate the note-action cluster in/out as annotate mode toggles. The - // pill itself has no entrance gate: it is always visible (just the pencil - // when idle). + // Animate the row swap (and the pill's width with it) as annotate mode + // toggles. The pill itself has no entrance gate: it is always visible + // (just the pencil when idle). .animation(reduceMotion ? nil : .easeOut(duration: 0.15), value: annotating) } @@ -537,12 +542,10 @@ private struct ToolbarView: View { } /// A single 28pt circular icon button in the pill: transparent when idle, a faint -/// white wash on hover (red for destructive), and a bright white glyph (no circle -/// fill) when its toggle is active. Each carries a tooltip (`.help`) and a -/// matching accessibility label. +/// white wash on hover (red for destructive). Each carries a tooltip (`.help`) +/// and a matching accessibility label. private struct PillButton: View { let icon: LucideIcon - var isActive: Bool = false var isDestructive: Bool = false /// Dims the glyph and makes the button a true no-op (used by copy/export/clear /// while there are no notes to act on). @@ -560,12 +563,11 @@ private struct PillButton: View { if isDisabled { return PillStyle.iconIdle.opacity(0.4) } if let glyphTint { return glyphTint } if isDestructive && hovering { return .white } - if isActive { return .white } return hovering ? PillStyle.iconHover : PillStyle.iconIdle } - // Active state is carried by `glyphColor` (white when active) with NO circle - // fill, so the annotate toggle reads as a bright glyph, not a blue chip. + // Hover is the ONLY fill state: every control in the pill is a one-shot + // action, so no button ever wears a persistent chip. private var fillColor: Color { // No hover wash while disabled — the button must look inert. if hovering && !isDisabled { return isDestructive ? PillStyle.destructive : PillStyle.hoverBackground } diff --git a/Sources/AnnotKit/Overlay/PillStyle.swift b/Sources/AnnotKit/Overlay/PillStyle.swift index 765f464..f07b44d 100644 --- a/Sources/AnnotKit/Overlay/PillStyle.swift +++ b/Sources/AnnotKit/Overlay/PillStyle.swift @@ -62,23 +62,14 @@ enum IconPart { struct LucideIcon { let parts: [IconPart] - /// Lucide `pencil` — the annotate toggle's IDLE glyph. Uses the real Lucide - /// `d` strings; the parser approximates the small corner arcs (`a`) as a line - /// to the arc endpoint, which reads identically at 16pt. + /// Lucide `pencil` — the idle pill's ENTER-annotate-mode glyph. Uses the real + /// Lucide `d` strings; the parser approximates the small corner arcs (`a`) as + /// a line to the arc endpoint, which reads identically at 16pt. static let pencil = LucideIcon(parts: [ .path("M21.174 6.812a1 1 0 0 0-3.986-3.987L3.842 16.174a2 2 0 0 0-.5.83l-1.321 4.352a.5.5 0 0 0 .623.622l4.353-1.32a2 2 0 0 0 .83-.497z"), .path("m15 5 4 4"), ]) - /// Lucide `pencil-off` — the annotate toggle's ACTIVE (annotating) glyph: the - /// pencil with a diagonal slash through it. - static let pencilOff = LucideIcon(parts: [ - .path("m10 10-6.157 6.162a2 2 0 0 0-.5.833l-1.322 4.36a.5.5 0 0 0 .622.624l4.358-1.323a2 2 0 0 0 .83-.5L14 13.982"), - .path("m12.829 7.172 4.359-4.346a1 1 0 1 1 3.986 3.986l-4.353 4.353"), - .path("m15 5 4 4"), - .path("m2 2 20 20"), - ]) - static let check = LucideIcon(parts: [.path("M20 6 9 17l-5-5")]) /// Lucide `download` — export to a file. Drawn with straight strokes only @@ -210,10 +201,10 @@ struct LucideShape: Shape { path.addQuadCurve(to: scaled(end), control: scaled(ctrl)) case "A", "a": // Elliptical arc. The parser has no arc-to-bezier, so it draws a - // straight segment to the arc ENDPOINT — Lucide's pencil/pencil-off - // arcs are small corner rounds and the flat eraser diagonal, which - // read the same at 16pt. Consume all 7 params (rx ry rot large - // sweep x y). + // straight segment to the arc ENDPOINT — the only arcs in use are + // Lucide's pencil corner rounds and its flat eraser diagonal, + // which read the same at 16pt. Consume all 7 params (rx ry rot + // large sweep x y). guard nextNumber() != nil, nextNumber() != nil, nextNumber() != nil, nextNumber() != nil, nextNumber() != nil, let ax = nextNumber(), let ay = nextNumber() else { return } diff --git a/Tests/AnnotKitTests/LucideIconTests.swift b/Tests/AnnotKitTests/LucideIconTests.swift index b5007bb..d36adad 100644 --- a/Tests/AnnotKitTests/LucideIconTests.swift +++ b/Tests/AnnotKitTests/LucideIconTests.swift @@ -12,7 +12,6 @@ final class LucideIconTests: XCTestCase { private let icons: [(name: String, icon: LucideIcon)] = [ ("pencil", .pencil), - ("pencilOff", .pencilOff), ("check", .check), ("copy", .copy), ("download", .download), From 00fa0de99b64de6059c6e90596d2376ba2263635 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:48:07 -0500 Subject: [PATCH 02/24] feat(marquee): pure frame-selection resolution rule Adds MarqueeTargetRule, the rect counterpart to AnnotationTargetRule: a drawn frame binds to the LARGEST meaningful element it surrounds (>=85% of that element's own area), else to the TIGHTEST element enclosing the frame, else nil so the session can fall back to a region note. Largest-wins is deliberate: a click means "this exact spot" and descends, a box means "this whole thing" and must ascend past the labels it swallowed. Ties break on seeding first (the coextensive .axCardSurface leaf carries the identifier that locates code), then depth, then lowest index -- via an explicit strict-improvement fold, because max(by:)/min(by:) make no first-wins promise and a total tie must not resolve by iteration detail. Also declares the optional MarqueeTargetSource capability (declaration only; platform adapters land separately) and documents the rule in DECISIONS.md. Co-Authored-By: Claude Opus 5 (1M context) --- DECISIONS.md | 59 +++++ Sources/AnnotKit/ElementSource.swift | 19 ++ Sources/AnnotKit/MarqueeTargetRule.swift | 205 ++++++++++++++++ .../MarqueeTargetRuleTests.swift | 229 ++++++++++++++++++ 4 files changed, 512 insertions(+) create mode 100644 Sources/AnnotKit/MarqueeTargetRule.swift create mode 100644 Tests/AnnotKitTests/MarqueeTargetRuleTests.swift diff --git a/DECISIONS.md b/DECISIONS.md index 95b755d..52d527f 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -10,6 +10,7 @@ Resolves the open decisions from the plan (planning/annotkit in the cli repo). P | Versioning | SemVer, 0.x pre-1.0 | Breaking changes allowed while 0.x; 1.0 marks a stable public API. | | Default element source | Accessibility hierarchy | The only strategy that surfaces SwiftUI `accessibilityIdentifier` values. | | Annotation target rule | Deepest actionable, else deepest meaningful; anchor the selector to the nearest identifier | One rule on both platforms. Supersedes the earlier macOS "deepest meaningful" and iOS "nearest identified" split. See below. | +| Marquee target rule | Largest meaningful element ≥85% surrounded; else the tightest element enclosing the drawn frame | Rect selection, the deliberate inverse of the point rule's deepest-wins. See below. | | Opt-in element source | View tree (NSView/UIView) | Surfaces concrete view class names; richer for AppKit/UIKit hosts. Collapses to hosting views in pure SwiftUI. | | `pathname` mapping | Host-supplied route, inferred fallback | A native app has no URL routes; the host sets a route, else infer from the key window title or identifier. | | Overlay coverage | Primary screen (MVP) | The overlay covers the primary display; SwiftUI-local points map to AX screen coordinates there. Full multi-display placement is deferred (cli-a99qm.4.2). | @@ -70,6 +71,64 @@ Selector *anchoring* (`#Card >> …`) still requires a true ancestor because the selector may be positional or text-based while the `component` field still names the card, so the note locates the right code either way. +## Marquee target rule (VRT-cne0.5) + +The user press-drags a rectangle around what they mean. Given every element the +adapter can see (a flat array, not an ancestor chain — a marquee sweeps across +siblings and unrelated subtrees), the note binds by two passes over the +*standardized* rect, using the same eligibility as the point rule (never the +window, the application, chrome, or a window-spanning ghost group, and never a +zero-area frame): + +1. **Surrounded** — every eligible element the frame covers to ≥85% of that + element's OWN area; the **largest** wins. +2. **Enclosing** — nothing was surrounded, so the frame was drawn inside + something: every eligible element whose frame contains the whole rect; the + **smallest** wins. +3. Neither — nil, and the session falls back to a region note anchored near the + frame, exactly as a point that hit-tests to nothing does. + +Ties inside a pass: seeded beats unseeded, then shallower (surrounded) / deeper +(enclosing), then lowest index. Areas compare with exact `==`, no epsilon. + +**Why largest-wins**, when the point rule is deepest-wins? Because the gestures +mean opposite things. A click means "this exact spot", so it descends. Drawing a +box around a card means "I mean this *whole* thing", so it must ascend past the +labels and buttons the box also swallowed. Same tree, opposite intent — hence a +separate rule rather than a mode flag on `AnnotationTargetRule`. + +**Why 0.85 and not strict containment.** A hand-drawn rect clips edges. Users +drag roughly around a card and routinely shave a corner or slice through a +trailing chevron; at 1.0 that silently demotes to the enclosing fallback and +binds the note to the panel instead of the card — the exact failure marquee +exists to remove. 0.85 absorbs that sloppiness and still sits far above the +coverage a neighbouring card picks up when a drag merely overlaps its edge. + +**Why seeded-beats-unseeded exists at all.** It is not a general preference for +identified elements (that is the mistake the point rule documents above). It is +narrowly for the coextensive case: `.axCardSurface(id)` hangs the card's +identifier on a clear `Color.clear` background leaf that is *exactly* the same +frame as the card's content group. Both are surrounded identically, and only the +seeded one carries the identifier that locates code. `AXIntrospection.deepestChild` +already resolves this same pattern by exact equal-area comparison, which is why +no epsilon is used here — the two frames come from one layout computation, so the +arithmetic is bit-identical, and an epsilon would instead start collapsing +genuinely different elements into a seeding decision. + +**Why the enclosing fallback.** It is the rect generalization of the point-region +note (rule 5 above): a scribble over a card's padding surrounds nothing, and +dropping it would be the same lost-click bug `RegionAnchorSource` was added to +fix. Smallest-wins there because the tightest enclosure is the most specific — a +scribble inside a card must not resolve to the window-spanning panel that also +contains it. + +Depth and index are determinism-only tie-breaks; they exist so the same drag +always resolves to the same element. The pure decision lives in +`MarqueeTargetRule` and is unit-tested independent of AX; adapters expose it via +the optional `MarqueeTargetSource` capability, which returns a component-widening +ladder identical in contract to `ComponentLadderSource`, so widening and the +note's `component` field work unchanged. + ## IP hygiene (carried into the F7 legal gate) - Do not copy original Agentation source (PolyForm Shield 1.0.0, non-compete). Only the `AGENTATION_NOTES.md` file format is reused, reimplemented clean-room. diff --git a/Sources/AnnotKit/ElementSource.swift b/Sources/AnnotKit/ElementSource.swift index bda10eb..bc1e7e2 100644 --- a/Sources/AnnotKit/ElementSource.swift +++ b/Sources/AnnotKit/ElementSource.swift @@ -58,3 +58,22 @@ public protocol RegionAnchorSource { public protocol ComponentLadderSource { func componentLadder(at point: CGPoint) -> [Element] } + +/// Optional element-source capability: resolve a frame the user DREW (a marquee +/// press-drag) to an annotation target, per ``MarqueeTargetRule`` — the largest +/// meaningful element the frame surrounds, else the tightest element the frame +/// was drawn inside. Lets the user say "I mean this whole card" by circling it +/// rather than hunting for the one pixel that hit-tests to the card. Sources that +/// cannot offer it simply don't conform; the session then leaves marquee +/// selection off and only point selection is available. +@MainActor +public protocol MarqueeTargetSource { + /// The annotation target for a frame the user DREW (AX top-left screen + /// coordinates), with its component-widening ladder: the target first, then + /// each enclosing identified component, broadest last — the SAME contract as + /// ``ComponentLadderSource/componentLadder(at:)``, so the session's existing + /// ladder handling (widening, and the note's `component` field) works + /// unchanged. Empty when nothing in the frame is annotatable; the session + /// then falls back to a region note anchored near the frame. + func marqueeLadder(in rect: CGRect) -> [Element] +} diff --git a/Sources/AnnotKit/MarqueeTargetRule.swift b/Sources/AnnotKit/MarqueeTargetRule.swift new file mode 100644 index 0000000..cc17372 --- /dev/null +++ b/Sources/AnnotKit/MarqueeTargetRule.swift @@ -0,0 +1,205 @@ +import CoreGraphics +import Foundation + +/// One element considered for a marquee selection: its annotation-relevant +/// facts, its frame in AX screen coordinates (top-left origin), and its depth in +/// the source's tree (a tie-break only). +/// +/// The platform adapter reads these facts once — from the AX tree on macOS, from +/// the view tree on iOS — and hands the rule a flat array. Nothing here is a live +/// platform handle, so the rule stays pure and is unit-tested without a running +/// app. Unlike ``AnnotationTargetRule``, the array is NOT an ancestor chain: a +/// marquee sweeps across siblings and unrelated subtrees, so order carries no +/// meaning beyond being the final, stability-only tie-break. +public struct MarqueeCandidate: Sendable, Hashable { + public let element: TargetCandidate + /// AX screen coordinates, top-left origin — the same space the drawn frame + /// arrives in, so containment is a plain rect comparison with no conversion. + public let frame: CGRect + /// Distance from the source's root. Used ONLY to break ties between elements + /// that are geometrically indistinguishable; it never outranks area, because + /// tree depth is not a reliable proxy for "what the user drew around" (a + /// `.background` surface is a sibling leaf, not an ancestor — see + /// DECISIONS.md → "Component containment is GEOMETRIC"). + public let depth: Int + + public init(element: TargetCandidate, frame: CGRect, depth: Int) { + self.element = element + self.frame = frame + self.depth = depth + } + + /// Seeded = carries an `accessibilityIdentifier`. The identifier is the thing + /// that locates source code, so at equal geometry it is the tie-break that + /// decides whether the note is actionable for an agent. + fileprivate var isSeeded: Bool { !element.identifier.isEmpty } + + fileprivate var area: CGFloat { frame.width * frame.height } +} + +/// The marquee selection rule (see DECISIONS.md → "Marquee target rule"). The +/// user press-drags a rectangle around what they mean; given every element the +/// adapter could see, decide which one the note binds to. Pure and +/// platform-independent, the rect generalization of ``AnnotationTargetRule``: +/// adapters build `[MarqueeCandidate]` from their own node types and call +/// ``resolve(marquee:in:)``, so macOS and iOS resolve an identical drag +/// identically and the decision is unit-tested on its own. +public enum MarqueeTargetRule { + /// How the drawn frame related to the element it resolved to. The session + /// records it so the note can say whether the user framed a thing or framed a + /// spot inside a thing. + public enum Match: Sendable, Hashable { + /// The drawn frame SURROUNDS the element. + case surrounded + /// The frame was drawn INSIDE this element (nothing was surrounded). + case enclosing + } + + public struct Resolution: Sendable, Hashable { + /// Index into the `candidates` array passed to ``resolve(marquee:in:)``. + public let index: Int + public let match: Match + + public init(index: Int, match: Match) { + self.index = index + self.match = match + } + } + + /// Fraction of a candidate's own area that the drawn frame must cover for the + /// frame to count as surrounding it. + /// + /// Not 1.0: a hand-drawn rect clips edges. Users drag roughly around a card + /// and routinely shave a few points off a corner or slice through the + /// trailing chevron, and a strict-containment rule would silently demote that + /// to the enclosing fallback — the exact failure the feature exists to avoid. + /// 0.85 tolerates that sloppiness while still being far above the coverage a + /// neighbouring card picks up when a drag merely overlaps its edge. + public static let containmentThreshold: CGFloat = 0.85 + + /// The candidate a drawn frame binds to, or nil when the frame contains + /// nothing annotatable and does not sit inside anything annotatable — the + /// caller then falls back to a region note anchored near the frame, the same + /// way a point that hit-tests to nothing does. + /// + /// Two passes, in order: + /// + /// 1. **Surrounded.** Every eligible candidate the frame covers to at least + /// ``containmentThreshold`` of its own area; the LARGEST wins. Largest-wins + /// IS the feature: framing something means "I mean this whole thing", so + /// the card must beat the labels and buttons drawn inside it. (The point + /// rule does the opposite — deepest-wins — because a click means "I mean + /// this exact spot". Same tree, opposite intent, hence a separate rule.) + /// 2. **Enclosing.** Nothing was surrounded, so the user drew INSIDE + /// something: keep the eligible candidates whose frame contains the whole + /// drawn rect and take the SMALLEST, the tightest enclosure being the most + /// specific. This is the rect generalization of the point-region path — a + /// scribble over a card's padding binds to the card, not to the + /// window-spanning panel that also contains it. + /// + /// Ties inside each pass are broken by seeding, then depth, then index; see + /// ``isStrictlyBetterSurrounded(_:than:)``. + public static func resolve(marquee: CGRect, in candidates: [MarqueeCandidate]) -> Resolution? { + // A right-to-left or bottom-to-top drag arrives with negative width or + // height, where `contains`/`intersection` degenerate. Normalize once and + // use only the normalized rect below, so drag direction cannot change the + // answer. + let rect = marquee.standardized + // A click that never moved, or a drag along a single axis, is not a + // marquee. Rejecting it here keeps the caller from binding a note to + // whatever happens to enclose a zero-area rect (which is everything). + guard rect.width > 0, rect.height > 0 else { return nil } + + let eligible = candidates.indices.filter { i in + let candidate = candidates[i] + // The positive-area check is load-bearing twice over: it keeps a + // degenerate frame out of the coverage ratio's DENOMINATOR (0/0 is + // NaN, and every NaN comparison is false, so such a candidate would + // poison the fold in ways that depend on iteration order), and a + // zero-area element is nothing the user could have aimed at anyway. + return candidate.element.isEligibleMeaningful && candidate.frame.width > 0 && candidate.frame.height > 0 + } + + if let winner = fold(eligible, in: candidates, keeping: { candidate in + let overlap = candidate.frame.intersection(rect) + // Disjoint rects intersect to `.null`, whose size is zero — so the + // ratio is 0 and the candidate falls out, no special case needed. + return (overlap.width * overlap.height) / candidate.area >= containmentThreshold + }, preferring: isStrictlyBetterSurrounded) { + return Resolution(index: winner, match: .surrounded) + } + + if let winner = fold(eligible, in: candidates, keeping: { $0.frame.contains(rect) }, + preferring: isStrictlyBetterEnclosing) { + return Resolution(index: winner, match: .enclosing) + } + + return nil + } + + /// Ascending-index fold that replaces the incumbent ONLY on a strict + /// improvement — which is what makes "earlier index" the final tie-break. + /// `max(by:)`/`min(by:)` are deliberately not used: for elements the predicate + /// calls equal they make no first-wins guarantee, so a total tie between two + /// coextensive candidates would resolve by implementation detail and the same + /// drag could bind to different elements on different runs. + private static func fold( + _ indices: [Int], + in candidates: [MarqueeCandidate], + keeping isKept: (MarqueeCandidate) -> Bool, + preferring isStrictlyBetter: (MarqueeCandidate, MarqueeCandidate) -> Bool + ) -> Int? { + var best: Int? + for i in indices where isKept(candidates[i]) { + guard let incumbent = best else { + best = i + continue + } + if isStrictlyBetter(candidates[i], candidates[incumbent]) { best = i } + } + return best + } + + /// Strict weak ordering for pass 1 — true only when `lhs` is genuinely better, + /// never for equivalent candidates: + /// + /// 1. **Larger area.** See ``resolve(marquee:in:)``: framing means "this whole + /// thing". + /// 2. **Seeded beats unseeded**, at EXACTLY equal area. This exists for the + /// dominant VirgilHUD pattern: `.axCardSurface(id)` hangs the card's + /// `accessibilityIdentifier` on a clear `Color.clear` background leaf that + /// is exactly coextensive with the card's content group. A frame drawn + /// around the card surrounds both equally, and only the seeded one carries + /// the identifier that locates the code. + /// 3. **Shallower**, then 4. **earlier index** (the fold's replace-on-strict + /// -improvement rule): geometry and seeding have run out, and these exist + /// only so the same drag always resolves to the same element. + /// + /// Areas are compared with exact `==`, no epsilon. The equal-area case that + /// matters is a background surface and its content group computed from the + /// SAME layout frame, so the arithmetic is bit-identical; + /// `AXIntrospection.deepestChild` already relies on exact equal-area + /// comparison for precisely this pattern and shipped after dogfooding. An + /// epsilon would instead start collapsing genuinely different, merely + /// similar-sized elements into a tie decided by seeding. + private static func isStrictlyBetterSurrounded(_ lhs: MarqueeCandidate, than rhs: MarqueeCandidate) -> Bool { + if lhs.area != rhs.area { return lhs.area > rhs.area } + if lhs.isSeeded != rhs.isSeeded { return lhs.isSeeded } + if lhs.depth != rhs.depth { return lhs.depth < rhs.depth } + return false + } + + /// Strict weak ordering for pass 2, the mirror of + /// ``isStrictlyBetterSurrounded(_:than:)``: smaller wins (the tightest + /// enclosure is the most specific), then seeded, then DEEPER — deeper being + /// the tightest-enclosure tie-break, consistent with preferring the smaller + /// frame — then earlier index. Seeding still outranks depth for the same + /// coextensive-surface reason: the surface leaf and the content group enclose + /// the drawn rect identically, and the identifier is what the agent needs. + private static func isStrictlyBetterEnclosing(_ lhs: MarqueeCandidate, than rhs: MarqueeCandidate) -> Bool { + if lhs.area != rhs.area { return lhs.area < rhs.area } + if lhs.isSeeded != rhs.isSeeded { return lhs.isSeeded } + if lhs.depth != rhs.depth { return lhs.depth > rhs.depth } + return false + } +} diff --git a/Tests/AnnotKitTests/MarqueeTargetRuleTests.swift b/Tests/AnnotKitTests/MarqueeTargetRuleTests.swift new file mode 100644 index 0000000..2903b85 --- /dev/null +++ b/Tests/AnnotKitTests/MarqueeTargetRuleTests.swift @@ -0,0 +1,229 @@ +import CoreGraphics +import XCTest +@testable import AnnotKit + +/// The marquee rule, exercised on synthetic candidate arrays so the decision is +/// verified without a live AX tree. The macOS/iOS adapters build the same +/// `[MarqueeCandidate]` from their platform node types, so what these tests pin +/// down is what a real drag resolves to. +final class MarqueeTargetRuleTests: XCTestCase { + // A plausible window-sized stage: the panel below spans it, the cards sit + // inside it. + private let windowFrame = CGRect(x: 0, y: 0, width: 1000, height: 800) + + private func candidate( + _ frame: CGRect, + role: String = "AXGroup", + identifier: String = "", + label: String = "", + value: String = "", + isActionable: Bool = false, + isChrome: Bool = false, + isContainerRoot: Bool = false, + isWindowGhost: Bool = false, + depth: Int = 1 + ) -> MarqueeCandidate { + MarqueeCandidate( + element: TargetCandidate( + role: role, + identifier: identifier, + label: label, + value: value, + isActionable: isActionable, + isChrome: isChrome, + isContainerRoot: isContainerRoot, + isWindowGhost: isWindowGhost + ), + frame: frame, + depth: depth + ) + } + + /// The point of the feature: dragging a rectangle around a card means "I mean + /// this whole card", so the card must beat the title and the button drawn + /// inside it — the exact opposite of the deepest-wins point rule. + func testLargestSurroundedCandidateWinsOverItsOwnChildren() { + let candidates = [ + candidate(CGRect(x: 100, y: 100, width: 200, height: 24), value: "Models", depth: 3), + candidate(CGRect(x: 100, y: 60, width: 300, height: 200), identifier: "Settings.Models", depth: 2), + candidate(CGRect(x: 260, y: 200, width: 80, height: 28), label: "Edit", isActionable: true, depth: 3), + ] + let resolution = MarqueeTargetRule.resolve(marquee: CGRect(x: 90, y: 50, width: 320, height: 220), in: candidates) + XCTAssertEqual(resolution, MarqueeTargetRule.Resolution(index: 1, match: .surrounded), "the card, not its label") + } + + /// The 0.85 boundary is inclusive, with exact geometry on both sides of it: + /// the small card is covered to exactly 0.85 and counts as surrounded; the + /// larger card the drag merely clips at 0.84 does not, even though it would + /// win on area if it did. + func testContainmentThresholdBoundaryIsInclusive() { + let small = candidate(CGRect(x: 0, y: 0, width: 100, height: 100), label: "Small") // area 10_000 + let large = candidate(CGRect(x: 200, y: 0, width: 200, height: 100), label: "Large") // area 20_000 + + // x 15…368: small keeps 85×100 = 8_500 / 10_000 = 0.85 exactly (in); + // large keeps 168×100 = 16_800 / 20_000 = 0.84 (out). + let atBoundary = MarqueeTargetRule.resolve( + marquee: CGRect(x: 15, y: 0, width: 353, height: 100), in: [small, large]) + XCTAssertEqual(atBoundary, MarqueeTargetRule.Resolution(index: 0, match: .surrounded), + "0.85 counts, 0.84 does not") + + // Two points wider: large keeps 170×100 = 17_000 / 20_000 = 0.85, and now + // outranks the small card on area. + let justOver = MarqueeTargetRule.resolve( + marquee: CGRect(x: 15, y: 0, width: 355, height: 100), in: [small, large]) + XCTAssertEqual(justOver, MarqueeTargetRule.Resolution(index: 1, match: .surrounded)) + } + + /// AX regularly reports collapsed or not-yet-laid-out elements with an empty + /// frame. They must never win, and — the reason the positive-area check is + /// load-bearing — must not put a 0/0 NaN into the coverage ratio and corrupt + /// the comparison for everyone else. + func testZeroAreaCandidatesAreIgnored() { + let real = candidate(CGRect(x: 100, y: 100, width: 50, height: 50), label: "Real") + let flat = candidate(CGRect(x: 100, y: 100, width: 0, height: 50), identifier: "Collapsed") + let empty = candidate(.zero, identifier: "Unlaid") + let marquee = CGRect(x: 0, y: 0, width: 400, height: 400) + + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: marquee, in: [flat, empty, real]), + MarqueeTargetRule.Resolution(index: 2, match: .surrounded)) + XCTAssertNil(MarqueeTargetRule.resolve(marquee: marquee, in: [flat, empty]), + "nothing eligible remains, and no NaN slipped through") + } + + /// A drag across the whole window sweeps up the window itself, its + /// `NSHostingView` ghost group, and the traffic lights. None of them locates + /// app code, so none may be selected — the same exclusions the point rule + /// applies, reused via `isEligibleMeaningful`. + func testChromeContainerRootAndWindowGhostAreNeverSelected() { + let candidates = [ + candidate(windowFrame, role: "AXWindow", label: "Settings", isContainerRoot: true, depth: 0), + candidate(windowFrame, role: "AXGroup", isWindowGhost: true, depth: 1), + candidate(CGRect(x: 8, y: 8, width: 14, height: 14), role: "AXButton", + label: "close", isActionable: true, isChrome: true, depth: 2), + candidate(CGRect(x: 100, y: 100, width: 300, height: 200), identifier: "Settings.Models", depth: 3), + ] + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: windowFrame, in: candidates), + MarqueeTargetRule.Resolution(index: 3, match: .surrounded), + "the card, though the window and ghost are larger and fully inside") + + XCTAssertNil(MarqueeTargetRule.resolve(marquee: windowFrame, in: Array(candidates.prefix(3)))) + } + + /// The dominant VirgilHUD pattern: `.axCardSurface(id)` puts the card's + /// identifier on a clear `Color.clear` background leaf that is EXACTLY + /// coextensive with the card's content group. A frame around the card + /// surrounds both identically; the seeded one is the one whose identifier + /// locates the code. + func testSeededBeatsUnseededAtExactlyEqualArea() { + let cardFrame = CGRect(x: 100, y: 100, width: 300, height: 200) + let candidates = [ + candidate(cardFrame, role: "AXGroup", label: "Models", depth: 3), // content group + candidate(cardFrame, role: "AXUnknown", identifier: "Settings.Models", depth: 3), // .axCardSurface leaf + ] + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: CGRect(x: 90, y: 90, width: 320, height: 220), in: candidates), + MarqueeTargetRule.Resolution(index: 1, match: .surrounded), + "the seeded surface, not the coextensive content group") + } + + /// Geometry and seeding have run out, so the shallower node wins — the + /// container rather than the pass-through wrapper SwiftUI stacked inside it. + /// Determinism only; the index tie-break is deliberately not what decides it. + func testShallowerBeatsDeeperAtEqualAreaAndEqualSeeding() { + let frame = CGRect(x: 100, y: 100, width: 300, height: 200) + let candidates = [ + candidate(frame, label: "Wrapper", depth: 7), + candidate(frame, label: "Container", depth: 2), + ] + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: CGRect(x: 0, y: 0, width: 600, height: 600), in: candidates), + MarqueeTargetRule.Resolution(index: 1, match: .surrounded)) + } + + /// Two candidates indistinguishable on every ranked key must resolve to the + /// lowest index, always. This is why the fold replaces the incumbent only on a + /// STRICT improvement: `max(by:)` makes no first-wins promise, and the same + /// drag resolving to a different element run to run is the bug being + /// prevented. + func testTotalTieResolvesToLowestIndex() { + let frame = CGRect(x: 100, y: 100, width: 300, height: 200) + let candidates = [ + candidate(frame, identifier: "A", depth: 4), + candidate(frame, identifier: "B", depth: 4), + candidate(frame, identifier: "C", depth: 4), + ] + let marquee = CGRect(x: 90, y: 90, width: 320, height: 220) + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: marquee, in: candidates), + MarqueeTargetRule.Resolution(index: 0, match: .surrounded)) + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: marquee, in: Array(candidates.reversed())), + MarqueeTargetRule.Resolution(index: 0, match: .surrounded), + "position in the array decides, nothing else") + } + + /// A frame scribbled over a card's empty padding surrounds nothing, so it + /// binds to the TIGHTEST thing containing it — the card. The window-spanning + /// panel contains it too and must lose; this is the rect generalization of the + /// point-region path, which likewise refuses to resolve to the whole window. + func testEnclosingFallbackPicksTheTightestContainer() { + let candidates = [ + candidate(CGRect(x: 0, y: 0, width: 1000, height: 780), identifier: "Settings.Panel", depth: 2), + candidate(CGRect(x: 100, y: 100, width: 300, height: 200), identifier: "Settings.Models", depth: 3), + candidate(CGRect(x: 500, y: 100, width: 300, height: 200), identifier: "Settings.Keys", depth: 3), + ] + XCTAssertEqual( + MarqueeTargetRule.resolve(marquee: CGRect(x: 150, y: 250, width: 60, height: 30), in: candidates), + MarqueeTargetRule.Resolution(index: 1, match: .enclosing), + "the card, not the panel that also contains the scribble") + } + + /// When the drag both surrounds a control and sits inside a card, surrounding + /// wins: the user drew AROUND something, which is a statement of intent, while + /// being inside something is merely where the pointer happened to be. + func testSurroundedIsPreferredOverEnclosing() { + let candidates = [ + candidate(CGRect(x: 100, y: 100, width: 300, height: 200), identifier: "Settings.Models", depth: 2), + candidate(CGRect(x: 150, y: 150, width: 80, height: 28), label: "Edit", isActionable: true, depth: 3), + ] + XCTAssertEqual( + MarqueeTargetRule.resolve(marquee: CGRect(x: 140, y: 140, width: 100, height: 48), in: candidates), + MarqueeTargetRule.Resolution(index: 1, match: .surrounded), + "the surrounded button, though the enclosing card is also a valid answer") + } + + /// A frame drawn over empty chrome-free decoration matches nothing either way. + /// nil is the handoff: the session falls back to a region note anchored near + /// the frame rather than dropping the drag. + func testNilWhenNothingIsEligible() { + let candidates = [ + candidate(windowFrame, role: "AXWindow", isContainerRoot: true, depth: 0), + candidate(CGRect(x: 100, y: 100, width: 40, height: 2), role: "AXUnknown", depth: 4), // a divider + ] + XCTAssertNil(MarqueeTargetRule.resolve(marquee: CGRect(x: 600, y: 600, width: 50, height: 50), in: candidates)) + } + + /// A press-and-release with no movement, or a drag along a single axis, is a + /// click — not a marquee. Without the guard the degenerate rect is contained + /// by everything and the enclosing pass would bind a note to an arbitrary + /// container. + func testDegenerateMarqueeReturnsNil() { + let candidates = [candidate(CGRect(x: 0, y: 0, width: 400, height: 400), identifier: "Panel", depth: 1)] + XCTAssertNil(MarqueeTargetRule.resolve(marquee: CGRect(x: 200, y: 200, width: 0, height: 0), in: candidates)) + XCTAssertNil(MarqueeTargetRule.resolve(marquee: CGRect(x: 200, y: 200, width: 0, height: 80), in: candidates)) + XCTAssertNil(MarqueeTargetRule.resolve(marquee: CGRect(x: 200, y: 200, width: 80, height: 0), in: candidates)) + } + + /// Dragging up-and-left is as natural as down-and-right and arrives with + /// negative width/height, where `contains` and `intersection` degenerate. Both + /// directions must describe the same rectangle and resolve identically. + func testReversedDragResolvesIdenticallyToTheStandardizedOne() { + let candidates = [ + candidate(CGRect(x: 100, y: 100, width: 200, height: 24), value: "Models", depth: 3), + candidate(CGRect(x: 100, y: 60, width: 300, height: 200), identifier: "Settings.Models", depth: 2), + ] + let forward = CGRect(x: 90, y: 50, width: 320, height: 220) + // Same rectangle, dragged from its bottom-right corner back to its top-left. + let reversed = CGRect(x: 410, y: 270, width: -320, height: -220) + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: reversed, in: candidates), + MarqueeTargetRule.resolve(marquee: forward, in: candidates)) + XCTAssertEqual(MarqueeTargetRule.resolve(marquee: reversed, in: candidates), + MarqueeTargetRule.Resolution(index: 1, match: .surrounded)) + } +} From c40a4d2c50b71dc083fe6e214675219278d63b6f Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:51:26 -0500 Subject: [PATCH 03/24] chore: ignore agent worktree directories Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3fc6fd1..0baa786 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .build/ .swiftpm/ +.claude/worktrees/ *.xcodeproj *.xcworkspace DerivedData/ From 5e99ca61a54be05356ed3419f02366236d0b3908 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:57:32 -0500 Subject: [PATCH 04/24] feat(marquee): session entry point + persisted frame on notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AnnotationSession.select(inAXRect:)` is the marquee counterpart to `select(atAXPoint:)`. It delegates to `MarqueeTargetSource.marqueeLadder(in:)`, whose ladder contract is identical to `ComponentLadderSource`'s, so widening and the note's `component` field keep working untouched. A drag that resolves to nothing degrades to a REGION note anchored at the frame's centre, mirroring the no-hit click path but carrying the drawn frame instead of a point. The drawn frame is held ABSOLUTE (`selectedMarqueeRect`) and relativized only at `addNote`, because `widenSelection()` can rebind the note to an enclosing element AFTER the drag — a frame relativized at selection time would describe a box the note no longer names. Synthetic regions measure from `marqueeRegionOrigin` (the anchor) since their own frame IS the drawn rect and would trivially be (0, 0). Both new fields carry the exact stale-state hazard 7993a67 fixed for `selectedRegionOffset`: the `selected` didSet only clears on nil, never on replacement, so they are also cleared at the top of BOTH select entry points. Tests cover the regression in both directions (marquee -> click must not leak `regionRect`; region-click -> marquee must not leak `regionOffset`). `AnnotationNote.regionRect` follows the additive-optional pattern of `regionOffset` (doc comment, CodingKeys entry, trailing defaulted init param), so old JSON decodes to nil and click notes serialize unchanged. The formatter emits `**Region**: framed WxH at (x: X, y: Y) from the top-left of ` as an `else if` against the point-offset line — the two locators are mutually exclusive by construction and the chain documents that. 86 -> 99 tests, all green. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/AnnotationSink.swift | 14 +- .../AnnotKit/Overlay/AnnotationSession.swift | 133 +++++++++++- .../AnnotKit/Sinks/AnnotationFormatter.swift | 17 +- .../AnnotationSessionTests.swift | 190 +++++++++++++++++- Tests/AnnotKitTests/SinkTests.swift | 44 ++++ 5 files changed, 385 insertions(+), 13 deletions(-) diff --git a/Sources/AnnotKit/AnnotationSink.swift b/Sources/AnnotKit/AnnotationSink.swift index 4615a11..2ee8dd1 100644 --- a/Sources/AnnotKit/AnnotationSink.swift +++ b/Sources/AnnotKit/AnnotationSink.swift @@ -70,6 +70,14 @@ public struct AnnotationNote: Sendable, Hashable, Identifiable, Codable { /// the top-left of #Dashboard.Today"). Optional, so element notes serialize /// unchanged and old files decode (nil). public var regionOffset: CGPoint? + /// MARQUEE note: the frame the user DREW, with its origin relative to the + /// top-left of the element named by `selector` (its size is absolute). Like + /// `regionOffset` it is PERSISTED — it is the locator agents need when the + /// user framed a spot rather than clicked one ("a 320x48 band 40pt down inside + /// #Settings.Models"). Mutually exclusive with `regionOffset` by construction: + /// a note carries the point locator or the frame locator, never both. Optional, + /// so click notes serialize unchanged and old files decode (nil). + public var regionRect: CGRect? /// Explicit keys that OMIT `anchor`: `JSONFileSink` and the MCP /// `FileNotesStore` encode/decode `[AnnotationNote]` directly, so a naked @@ -79,7 +87,7 @@ public struct AnnotationNote: Sendable, Hashable, Identifiable, Codable { private enum CodingKeys: String, CodingKey { case id, route, selector, elementPath, selectedText, comment, screenshot, timestamp case component, elementRole, elementText, unseeded - case regionOffset + case regionOffset, regionRect } public init( @@ -96,7 +104,8 @@ public struct AnnotationNote: Sendable, Hashable, Identifiable, Codable { screenshot: CapturedImage? = nil, timestamp: String, anchor: CGPoint? = nil, - regionOffset: CGPoint? = nil + regionOffset: CGPoint? = nil, + regionRect: CGRect? = nil ) { self.id = id self.route = route @@ -112,5 +121,6 @@ public struct AnnotationNote: Sendable, Hashable, Identifiable, Codable { self.timestamp = timestamp self.anchor = anchor self.regionOffset = regionOffset + self.regionRect = regionRect } } diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index f765118..a795ade 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -23,12 +23,15 @@ public final class AnnotationSession: ObservableObject { @Published public private(set) var pending: [AnnotationNote] = [] @Published public private(set) var hovered: Element? @Published public private(set) var selected: Element? { - // The region offset and the widening ladder only make sense while their - // selection is alive; clearing the selection (capture, cancel, stop, pin - // editing) must never leave a stale offset or ladder for the NEXT note. + // The region offset, the drawn marquee frame, and the widening ladder only + // make sense while their selection is alive; clearing the selection + // (capture, cancel, stop, pin editing) must never leave a stale offset, + // frame, or ladder for the NEXT note. didSet { if selected == nil { selectedRegionOffset = nil + selectedMarqueeRect = nil + marqueeRegionOrigin = nil componentLadder = [] ladderIndex = 0 } @@ -38,6 +41,19 @@ public final class AnnotationSession: ObservableObject { /// (see ``select(atAXPoint:)``); nil for ordinary element selections. public private(set) var selectedRegionOffset: CGPoint? + /// The frame the user DREW for the current selection, in ABSOLUTE AX screen + /// coordinates; nil for click selections. Kept absolute (not element-relative) + /// so ``widenSelection()`` re-relativizes it for free: widening rebinds the + /// note to an enclosing element with a different origin, and a frame already + /// relativized at selection time would then silently describe the drag against + /// a box the note no longer names. + public private(set) var selectedMarqueeRect: CGRect? + /// Origin the drawn frame is measured from when the selection is a synthetic + /// REGION — whose own `frame` IS the drawn rect, so it cannot be its own + /// reference (it would relativize to 0,0 and locate nothing). nil for element + /// selections, which measure from the selected element's origin. + private var marqueeRegionOrigin: CGPoint? + /// The component-widening ladder for the current selection (target first, each /// enclosing identified component after), and the index of the currently /// selected rung. Populated on ``select(atAXPoint:)`` when the source offers a @@ -122,11 +138,15 @@ public final class AnnotationSession: ObservableObject { // Any catcher tap dismisses an open pin editor: a tap on empty space is a // click-away close, and a tap on an element hands the stage to the composer. editingNoteID = nil - // Every selection starts offset-free and ladder-free: a region -> element - // re-selection (the catcher stays active behind an open composer) must not - // leak the previous region's offset or ladder onto an ELEMENT note — the - // didSet only clears them when `selected` becomes nil, not on replacement. + // Every selection starts offset-free, frame-free and ladder-free: a + // region -> element or marquee -> click re-selection (the catcher stays + // active behind an open composer) must not leak the previous region's + // offset, the previous drag's drawn frame, or a stale ladder onto the next + // note — the didSet only clears them when `selected` becomes nil, not on + // replacement. selectedRegionOffset = nil + selectedMarqueeRect = nil + marqueeRegionOrigin = nil componentLadder = [] ladderIndex = 0 selected = source.hitTest(point) @@ -160,6 +180,84 @@ public final class AnnotationSession: ObservableObject { return selected } + /// Select the annotation target for a frame the user DREW (AX top-left screen + /// coordinates) — the marquee gesture: press-drag a rectangle AROUND what you + /// mean instead of hunting for the one pixel that hit-tests to it. + /// + /// The source resolves the frame (``MarqueeTargetSource``) and hands back the + /// same target-first, broadest-last ladder ``ComponentLadderSource`` produces, + /// so widening and the note's `component` field work unchanged. When nothing + /// in the frame is annotatable the drag is NOT dropped: it degrades to a + /// REGION note anchored at the frame's centre, exactly as a no-hit click does, + /// but carrying the drawn frame rather than a single point. + @discardableResult + public func select(inAXRect rect: CGRect) -> Element? { + guard mode == .annotating else { return nil } + // A drag on the catcher dismisses an open pin editor for the same reason a + // tap does: the composer is about to take the stage. + editingNoteID = nil + // Same stale-state hazard as the point path, and then some — a marquee -> + // click flow must not leak `selectedMarqueeRect` onto the click's note, and + // a region-click -> marquee flow must not leak `selectedRegionOffset` onto + // the framed one. The didSet only fires on nil, not on replacement. + selectedRegionOffset = nil + selectedMarqueeRect = nil + marqueeRegionOrigin = nil + componentLadder = [] + ladderIndex = 0 + + // Normalize once: a right-to-left or bottom-to-top drag arrives with + // negative extents. A zero-width or zero-height rect is a click that never + // moved, not a marquee — bail before the region fallback too, or a stray + // press would silently plant a degenerate framed note. + let normalized = rect.standardized + guard normalized.width > 0, normalized.height > 0 else { return nil } + + if let marqueeSource = source as? MarqueeTargetSource { + let ladder = marqueeSource.marqueeLadder(in: normalized) + if let target = ladder.first { + selected = target + componentLadder = ladder + ladderIndex = 0 + selectedMarqueeRect = normalized + return selected + } + } + + // Region fallback, mirroring the point path but anchored at the frame's + // CENTRE — the one point inside a drawn rect that is nearest everything it + // covers, so the anchor a user would name themselves. + if let anchorSource = source as? RegionAnchorSource, + let anchorElement = anchorSource.regionAnchor(at: CGPoint(x: normalized.midX, y: normalized.midY)) { + let offset = CGPoint( + x: (normalized.minX - anchorElement.frame.minX).rounded(), + y: (normalized.minY - anchorElement.frame.minY).rounded() + ) + let anchorName = anchorElement.label.isEmpty ? anchorElement.id : anchorElement.label + selected = Element( + id: anchorElement.id, + role: "AXRegion", + type: "Region", + label: "Region \(Int(normalized.width))x\(Int(normalized.height)) at (\(Int(offset.x)), \(Int(offset.y))) in \(anchorName)", + value: "", + // The DRAWN rect, not a marker at a point, so the overlay + // highlights the frame the user actually swept. + frame: normalized, + isVisible: true, + isActionable: false, + path: anchorElement.path + ) + selectedMarqueeRect = normalized + // The synthetic element's frame IS the drawn rect, so it cannot be its + // own measuring stick; record the anchor's origin instead. + marqueeRegionOrigin = anchorElement.frame.origin + // Deliberately NOT setting `selectedRegionOffset`: `regionOffset` stays + // the point-click locator so every note carries exactly one, and the + // formatter's framed/offset branches stay mutually exclusive. + } + return selected + } + /// Step the current selection UP to the next enclosing identified component /// (a coarser-grained note: the card instead of the label inside it). No-op /// when the selection is already at the broadest rung, is a region, or the @@ -171,6 +269,10 @@ public final class AnnotationSession: ObservableObject { ladderIndex += 1 // Widening always lands on a real element, so it is never a region note. selectedRegionOffset = nil + // `selectedMarqueeRect` is deliberately KEPT: it is absolute, so it is + // still the frame the user drew whichever rung is now bound, and `addNote` + // re-relativizes it against the widened element. `marqueeRegionOrigin` is + // moot here — regions get no ladder, so a widen can never start from one. // A non-nil assignment does not trip the didSet clear, so the ladder and // index survive for a further widen. selected = componentLadder[ladderIndex] @@ -206,6 +308,20 @@ public final class AnnotationSession: ObservableObject { ? ownIdentifier : (componentLadder.dropFirst().first?.id ?? element.path.last(where: { !($0.identifier ?? "").isEmpty })?.identifier) + // Relativize the drawn frame HERE rather than at selection — the asymmetry + // with `regionOffset` (computed at selection) is deliberate: + // ``widenSelection()`` can rebind the note to an enclosing element AFTER + // the frame was drawn, so the persisted rect must be measured against + // whatever element the note FINALLY names. A region never widens, so its + // offset's anchor is fixed the moment it is picked. + let regionRect: CGRect? = selectedMarqueeRect.map { drawn in + // A synthetic region's own frame IS the drawn rect, so it measures from + // its anchor (else it would trivially be 0,0); elements measure from + // themselves. + let base = marqueeRegionOrigin ?? element.frame.origin + return CGRect(x: (drawn.minX - base.x).rounded(), y: (drawn.minY - base.y).rounded(), + width: drawn.width.rounded(), height: drawn.height.rounded()) + } let note = AnnotationNote( id: makeID(), route: route(), @@ -220,7 +336,8 @@ public final class AnnotationSession: ObservableObject { screenshot: screenshot, timestamp: timestamp(), anchor: anchor, - regionOffset: selectedRegionOffset + regionOffset: selectedRegionOffset, + regionRect: regionRect ) pending.append(note) selected = nil diff --git a/Sources/AnnotKit/Sinks/AnnotationFormatter.swift b/Sources/AnnotKit/Sinks/AnnotationFormatter.swift index 2a7a154..898b24b 100644 --- a/Sources/AnnotKit/Sinks/AnnotationFormatter.swift +++ b/Sources/AnnotKit/Sinks/AnnotationFormatter.swift @@ -40,7 +40,14 @@ public enum AnnotationFormatter { if note.unseeded == true { lines.append("**Unseeded**: the clicked element has no accessibilityIdentifier — locate it via the Component above, then narrow by the Element role/text; consider seeding it") } - if let region = note.regionOffset { + // `else if`, not a second `if`: the two locators are mutually exclusive by + // construction (a note is framed or it is a point, never both), and the + // chain is what documents that — two independent lines would let a future + // leak of one into the other print a self-contradicting block instead of + // failing loudly. + if let rect = note.regionRect { + lines.append("**Region**: framed \(Int(rect.width))x\(Int(rect.height)) at (x: \(Int(rect.minX)), y: \(Int(rect.minY))) from the top-left of \(note.selector)") + } else if let region = note.regionOffset { lines.append("**Region**: (x: \(Int(region.x)), y: \(Int(region.y))) from the top-left of \(note.selector)") } if let selected = note.selectedText, !selected.isEmpty { @@ -74,6 +81,10 @@ public enum AnnotationFormatter { let timestamp: String let regionOffsetX: Int? let regionOffsetY: Int? + let regionRectX: Int? + let regionRectY: Int? + let regionRectWidth: Int? + let regionRectHeight: Int? let screenshotPixelWidth: Int? let screenshotPixelHeight: Int? @@ -91,6 +102,10 @@ public enum AnnotationFormatter { timestamp = note.timestamp regionOffsetX = note.regionOffset.map { Int($0.x) } regionOffsetY = note.regionOffset.map { Int($0.y) } + regionRectX = note.regionRect.map { Int($0.minX) } + regionRectY = note.regionRect.map { Int($0.minY) } + regionRectWidth = note.regionRect.map { Int($0.width) } + regionRectHeight = note.regionRect.map { Int($0.height) } screenshotPixelWidth = note.screenshot?.pixelWidth screenshotPixelHeight = note.screenshot?.pixelHeight } diff --git a/Tests/AnnotKitTests/AnnotationSessionTests.swift b/Tests/AnnotKitTests/AnnotationSessionTests.swift index 92e59bc..7ebe6e5 100644 --- a/Tests/AnnotKitTests/AnnotationSessionTests.swift +++ b/Tests/AnnotKitTests/AnnotationSessionTests.swift @@ -62,12 +62,43 @@ private final class LadderSource: ElementSource, ComponentLadderSource { } } +/// A source that serves a marquee ladder AND a point hit-test (and optionally a +/// region anchor), so one test can interleave drags and clicks — the mixed flow +/// the catcher actually produces, and the one that surfaces stale per-selection +/// state. +@MainActor +private final class MarqueeSource: ElementSource, MarqueeTargetSource, RegionAnchorSource { + var ladder: [Element] + var hit: Element? + var anchor: Element? + /// The last rect handed to ``marqueeLadder(in:)`` — lets a test assert the + /// session normalized before delegating. + private(set) var lastRect: CGRect? + + init(ladder: [Element], hit: Element? = nil, anchor: Element? = nil) { + self.ladder = ladder + self.hit = hit + self.anchor = anchor + } + func snapshot() -> [WindowSnapshot] { [] } + func hitTest(_ point: CGPoint) -> Element? { hit } + func marqueeLadder(in rect: CGRect) -> [Element] { + lastRect = rect + return ladder + } + func regionAnchor(at point: CGPoint) -> Element? { anchor } + func selector(for element: Element) -> String { "#\(element.id)" } + func screenshot(of element: Element?) async throws -> CapturedImage { + CapturedImage(pngData: Data(), pixelWidth: 1, pixelHeight: 1) + } +} + @MainActor final class AnnotationSessionTests: XCTestCase { - private func makeLadderElement(_ id: String) -> Element { + private func makeLadderElement(_ id: String, frame: CGRect = CGRect(x: 0, y: 0, width: 10, height: 10)) -> Element { Element( id: id, role: "AXGroup", type: "AXGroup", label: id, value: "", - frame: CGRect(x: 0, y: 0, width: 10, height: 10), isVisible: true, isActionable: false, + frame: frame, isVisible: true, isActionable: false, path: [PathComponent(role: "AXGroup", label: id, identifier: id, indexAmongRole: 0)] ) } @@ -199,6 +230,161 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertNil(note?.regionOffset, "element note must not inherit the stale region offset") } + // MARK: - Marquee selection + + func testMarqueeBindsToLadderTargetAndCapturesTheLadder() { + let ladder = [makeLadderElement("Card"), makeLadderElement("Settings.Models")] + let session = AnnotationSession(source: MarqueeSource(ladder: ladder), sink: NotesFileSink(path: "/dev/null")) + session.start() + let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) + XCTAssertEqual(session.select(inAXRect: drawn)?.id, "Card", "a marquee binds to the ladder's first rung") + XCTAssertEqual(session.selectedMarqueeRect, drawn, "the drawn frame is kept absolute") + XCTAssertNil(session.selectedRegionOffset, "a framed note carries no point locator") + XCTAssertTrue(session.canWidenSelection, "the marquee ladder drives widening like the point ladder") + } + + func testMarqueeNormalizesABackwardsDrag() { + // A bottom-right -> top-left drag arrives with negative extents. The + // session must normalize BEFORE delegating, or the source sees a rect whose + // `contains` degenerates and the persisted frame gets a negative size. + let source = MarqueeSource(ladder: [makeLadderElement("Card")]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(inAXRect: CGRect(x: 150, y: 140, width: -40, height: -20)) + XCTAssertEqual(source.lastRect, CGRect(x: 110, y: 120, width: 40, height: 20)) + XCTAssertEqual(session.selectedMarqueeRect, CGRect(x: 110, y: 120, width: 40, height: 20)) + } + + func testMarqueeThenClickDropsTheStaleFrame() { + // The 7993a67 hazard, marquee edition: the catcher stays live behind an + // open composer, so drag-then-click WITHOUT capturing is a supported flow. + // The click's note must not inherit the drag's frame — the `selected` + // didSet cannot help, because the value is replaced, never nilled. + let leaf = makeLadderElement("Card", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let source = MarqueeSource(ladder: [leaf]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(inAXRect: CGRect(x: 110, y: 120, width: 40, height: 20)) + XCTAssertNotNil(session.selectedMarqueeRect) + + source.hit = makeElement() + source.ladder = [] + session.select(atAXPoint: .zero) + XCTAssertEqual(session.selected?.id, "SaveButton", "the click re-selects a real element") + XCTAssertNil(session.selectedMarqueeRect, "the drawn frame must not survive re-selection") + XCTAssertNil(session.addNote(comment: "click after drag")?.regionRect, + "a click note must not inherit the stale marquee frame") + } + + func testRegionClickThenMarqueeDropsTheStaleOffset() { + // The other direction: a no-hit CLICK lands a point-region (setting + // `selectedRegionOffset`), then a drag lands a framed note. The framed note + // must carry exactly one locator — the rect — or the formatter would have + // two mutually exclusive Region lines to choose between. + let anchor = makeElement() + let source = MarqueeSource(ladder: [makeLadderElement("Card")], hit: nil, anchor: anchor) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: CGPoint(x: 22, y: 14)) + XCTAssertEqual(session.selectedRegionOffset, CGPoint(x: 22, y: 14), "the click lands a point-region") + + session.select(inAXRect: CGRect(x: 110, y: 120, width: 40, height: 20)) + XCTAssertEqual(session.selected?.id, "Card", "the drag re-selects a real element") + XCTAssertNil(session.selectedRegionOffset, "the region offset must not survive re-selection") + let note = session.addNote(comment: "drag after region click") + XCTAssertNil(note?.regionOffset, "a framed note must not inherit the stale point offset") + XCTAssertNotNil(note?.regionRect) + } + + func testWideningAfterAMarqueeReRelativizesTheFrame() { + // The frame is stored ABSOLUTE precisely so widening stays correct: the + // note rebinds to a bigger element with a different origin AFTER the drag, + // so the persisted rect must be measured against the widened element. A + // frame relativized at selection time would keep the leaf's numbers. + let leaf = makeLadderElement("Leaf", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let card = makeLadderElement("Card", frame: CGRect(x: 80, y: 60, width: 200, height: 150)) + let session = AnnotationSession( + source: MarqueeSource(ladder: [leaf, card]), sink: NotesFileSink(path: "/dev/null") + ) + session.start() + let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) + session.select(inAXRect: drawn) + XCTAssertEqual(session.widenSelection()?.id, "Card") + XCTAssertEqual(session.selectedMarqueeRect, drawn, "widening keeps the absolute drawn frame") + let note = session.addNote(comment: "framed then widened") + XCTAssertEqual(note?.selector, "#Card") + XCTAssertEqual(note?.regionRect, CGRect(x: 30, y: 60, width: 40, height: 20), + "origin re-measured from the widened element; size unchanged") + } + + func testCaptureClearsTheMarqueeFrame() { + // The didSet path (selection -> nil), distinct from the re-selection path + // above: after a capture the NEXT note must start frame-free. + let source = MarqueeSource(ladder: [makeLadderElement("Card", frame: CGRect(x: 100, y: 100, width: 60, height: 40))]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(inAXRect: CGRect(x: 110, y: 120, width: 40, height: 20)) + XCTAssertNotNil(session.addNote(comment: "framed")?.regionRect) + XCTAssertNil(session.selectedMarqueeRect, "capture clears the drawn frame") + + source.hit = makeElement() + session.select(atAXPoint: .zero) + XCTAssertNil(session.addNote(comment: "plain")?.regionRect, "the next note carries no frame") + } + + func testMarqueeWithoutMarqueeSourceFallsBackToAnAnchoredRegion() { + // No `MarqueeTargetSource` conformance: the drag degrades to a region note, + // but the FRAME survives — and it is measured from the ANCHOR, because the + // synthetic element's own frame IS the drawn rect and would trivially + // relativize to (0, 0), locating nothing. + let anchor = Element( + id: "SaveButton", role: "AXButton", type: "AXButton", label: "Save", value: "", + frame: CGRect(x: 10, y: 5, width: 100, height: 40), isVisible: true, isActionable: true, + path: [PathComponent(role: "AXButton", label: "Save", identifier: "SaveButton", indexAmongRole: 0)] + ) + let session = AnnotationSession( + source: EmptyWithAnchorSource(anchor: anchor), sink: NotesFileSink(path: "/dev/null") + ) + session.start() + let drawn = CGRect(x: 22, y: 14, width: 30, height: 12) + let selected = session.select(inAXRect: drawn) + XCTAssertEqual(selected?.role, "AXRegion") + XCTAssertEqual(selected?.frame, drawn, "the highlight shows the frame the user drew") + XCTAssertEqual(selected?.label, "Region 30x12 at (12, 9) in Save") + XCTAssertNil(session.selectedRegionOffset, "a framed region carries the rect, not a point offset") + + let note = session.addNote(comment: "gap looks off") + XCTAssertNil(note?.regionOffset) + XCTAssertEqual(note?.regionRect, CGRect(x: 12, y: 9, width: 30, height: 12)) + XCTAssertNotEqual(note?.regionRect?.origin, .zero, + "measured from the anchor, NOT from the synthetic element (which would be 0,0)") + } + + func testDegenerateMarqueeSelectsNothing() { + // A press that never moved (or moved on one axis only) is a click, not a + // marquee. It must not even reach the region fallback, or a stray press + // would plant a zero-area framed note. + let anchor = makeElement() + let source = MarqueeSource(ladder: [makeLadderElement("Card")], anchor: anchor) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + XCTAssertNil(session.select(inAXRect: CGRect(x: 10, y: 10, width: 0, height: 20))) + XCTAssertNil(session.select(inAXRect: CGRect(x: 10, y: 10, width: 20, height: 0))) + XCTAssertNil(session.select(inAXRect: .zero)) + XCTAssertNil(session.selected, "a degenerate drag selects nothing at all") + XCTAssertNil(source.lastRect, "the source is never consulted for a degenerate frame") + } + + func testSelectInRectIsGatedOnAnnotatingMode() { + let session = AnnotationSession( + source: MarqueeSource(ladder: [makeLadderElement("Card")]), sink: NotesFileSink(path: "/dev/null") + ) + let drawn = CGRect(x: 0, y: 0, width: 10, height: 10) + XCTAssertNil(session.select(inAXRect: drawn), "a drag before start must be nil") + session.start() + XCTAssertEqual(session.select(inAXRect: drawn)?.id, "Card") + } + func testClearHoverDropsHighlightButKeepsSelection() { let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) session.start() diff --git a/Tests/AnnotKitTests/SinkTests.swift b/Tests/AnnotKitTests/SinkTests.swift index 2f8644c..d91e7e1 100644 --- a/Tests/AnnotKitTests/SinkTests.swift +++ b/Tests/AnnotKitTests/SinkTests.swift @@ -43,6 +43,48 @@ final class SinkTests: XCTestCase { "element notes have no region line") } + func testMarkdownIncludesFramedRegionLineForMarqueeNotes() { + // The exact shape a consuming skill parses — do not reword. + var n = note(selector: "#Settings.Models") + n.regionRect = CGRect(x: 12, y: 40, width: 320, height: 48) + let md = AnnotationFormatter.markdown([n]) + XCTAssertTrue( + md.contains("**Region**: framed 320x48 at (x: 12, y: 40) from the top-left of #Settings.Models"), + md + ) + } + + func testMarkdownKeepsThePointRegionLineUnchangedWithoutARect() { + // The framed branch must not disturb the existing point-offset line: notes + // captured before marquee existed still render exactly as they did. + var n = note() + n.regionOffset = CGPoint(x: 22, y: 114) + XCTAssertTrue(AnnotationFormatter.markdown([n]) + .contains("**Region**: (x: 22, y: 114) from the top-left of #SaveButton")) + XCTAssertFalse(AnnotationFormatter.markdown([n]).contains("framed")) + } + + func testJSONCarriesTheFramedRectAsIntegers() throws { + var n = note() + n.regionRect = CGRect(x: 12, y: 40, width: 320, height: 48) + let json = try AnnotationFormatter.json([n]) + XCTAssertTrue(json.contains("\"regionRectX\" : 12")) + XCTAssertTrue(json.contains("\"regionRectY\" : 40")) + XCTAssertTrue(json.contains("\"regionRectWidth\" : 320")) + XCTAssertTrue(json.contains("\"regionRectHeight\" : 48")) + XCTAssertFalse(try AnnotationFormatter.json([note()]).contains("regionRect"), + "click notes gain no rect keys") + } + + func testRegionRectSurvivesAJSONRoundTrip() throws { + var n = note() + n.regionRect = CGRect(x: 12, y: 40, width: 320, height: 48) + let data = try JSONEncoder().encode([n]) + let decoded = try JSONDecoder().decode([AnnotationNote].self, from: data) + XCTAssertEqual(decoded[0].regionRect, CGRect(x: 12, y: 40, width: 320, height: 48), + "regionRect is PERSISTED, unlike `anchor`") + } + func testMarkdownIncludesComponentRoleAndUnseededHints() { var n = note() n.component = "Settings.Models" @@ -81,8 +123,10 @@ final class SinkTests: XCTestCase { """.utf8) let notes = try JSONDecoder().decode([AnnotationNote].self, from: old) XCTAssertNil(notes[0].regionOffset) + XCTAssertNil(notes[0].regionRect, "JSON predating the marquee field decodes with no rect") let reencoded = String(decoding: try JSONEncoder().encode(notes), as: UTF8.self) XCTAssertFalse(reencoded.contains("regionOffset")) + XCTAssertFalse(reencoded.contains("regionRect")) } func testJSONOmitsRawScreenshotButKeepsDimensions() throws { From 997577ae394607c6283018a0c4e97df36a4996d3 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:59:49 -0500 Subject: [PATCH 05/24] fix(marquee): clear the region measuring origin when widening Widening always rebinds to a real element, so a stale anchor origin would measure the persisted frame from the wrong box. Unreachable today (regions get no ladder) but it is the 7993a67 class of bug, so close it by construction rather than by invariant. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/Overlay/AnnotationSession.swift | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index a795ade..f8b9bad 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -269,10 +269,17 @@ public final class AnnotationSession: ObservableObject { ladderIndex += 1 // Widening always lands on a real element, so it is never a region note. selectedRegionOffset = nil + // ...and for the same reason it can never keep a region's measuring stick: + // every rung is a real element, so the drawn frame is measured from the + // element itself. Clearing is cheap insurance rather than a live fix — + // regions get no ladder today, so this state is currently unreachable — + // but leaving a stale anchor origin behind would silently measure the + // persisted rect from the wrong box, and that is exactly the class of bug + // `7993a67` was. + marqueeRegionOrigin = nil // `selectedMarqueeRect` is deliberately KEPT: it is absolute, so it is // still the frame the user drew whichever rung is now bound, and `addNote` - // re-relativizes it against the widened element. `marqueeRegionOrigin` is - // moot here — regions get no ladder, so a widen can never start from one. + // re-relativizes it against the widened element. // A non-nil assignment does not trip the didSet clear, so the ladder and // index survive for a further widen. selected = componentLadder[ladderIndex] From 7ffbba3f927bc901e4784946ab41b57ada872733 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:00:32 -0500 Subject: [PATCH 06/24] docs(marquee): state the click-routing caller contract on select(inAXRect:) A press that never moved returns nil here; the gesture recognizer owns routing it to the point path. Names the predicted symptom so 'clicking does nothing in annotate mode' is searchable from the API it traces to. Co-Authored-By: Claude Opus 5 (1M context) --- .../AnnotKit/Overlay/AnnotationSession.swift | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index f8b9bad..43c19b4 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -190,6 +190,15 @@ public final class AnnotationSession: ObservableObject { /// in the frame is annotatable the drag is NOT dropped: it degrades to a /// REGION note anchored at the frame's centre, exactly as a no-hit click does, /// but carrying the drawn frame rather than a single point. + /// + /// CALLER CONTRACT: a press that never moved is a CLICK, and this method + /// returns nil for it (see the degenerate guard below). The gesture recognizer + /// owns that routing — it must send a no-movement press, and any drag below its + /// own slop threshold, to ``select(atAXPoint:)`` instead. The session cannot + /// make that call for you: it sees only a rect, and cannot tell a deliberate + /// tiny drag from a jittery tap. Route it wrong and the symptom is "clicking + /// does nothing in annotate mode", which reads as a broken feature rather than + /// a missing threshold. @discardableResult public func select(inAXRect rect: CGRect) -> Element? { guard mode == .annotating else { return nil } @@ -207,9 +216,19 @@ public final class AnnotationSession: ObservableObject { ladderIndex = 0 // Normalize once: a right-to-left or bottom-to-top drag arrives with - // negative extents. A zero-width or zero-height rect is a click that never - // moved, not a marquee — bail before the region fallback too, or a stray - // press would silently plant a degenerate framed note. + // negative extents. + // + // A zero-width or zero-height rect is a click that never moved, not a + // marquee. Returning nil here is a DELIBERATE divergence from "the caller + // falls back to the region path when resolution yields nothing": that + // fallback is for a drag that framed nothing annotatable, whereas this is + // not a drag at all. Falling through would anchor a zero-area frame to + // whatever happens to sit near the press and plant a note the user never + // asked for — worse than nothing, because it looks deliberate. The gesture + // recognizer routes this case to ``select(atAXPoint:)`` instead (see the + // caller contract above). Note this also bails BEFORE consulting the + // source, so a degenerate rect never reaches `MarqueeTargetRule.resolve` + // or `regionAnchor(at:)`. let normalized = rect.standardized guard normalized.width > 0, normalized.height > 0 else { return nil } From 53dca4fcc7a2f37e1e68b2438e4aa47cd61f3c62 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:01:07 -0500 Subject: [PATCH 07/24] feat(marquee): macOS adapters resolve a drawn frame to an element Adds AXIntrospection.marqueeLadder(for:) with a single depth-consistent subtree walk, MarqueeTargetSource conformance on both macOS sources, and a live-AX probe phase. Extracts the geometric container scan shared with the point path, and excludes container roots from it: the chain climbs to AXApplication, whose children include our own overlay panel, so every widening ladder was topped by AnnotKit's own UI. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/macOS/AXIntrospection.swift | 137 ++++++++++++++- Sources/AnnotKit/macOS/MacElementSource.swift | 6 + .../macOS/MacViewTreeElementSource.swift | 122 ++++++++++++- Sources/AnnotKitOverlayProbe/main.swift | 163 +++++++++++++++++- 4 files changed, 420 insertions(+), 8 deletions(-) diff --git a/Sources/AnnotKit/macOS/AXIntrospection.swift b/Sources/AnnotKit/macOS/AXIntrospection.swift index 0b1ea15..fe59717 100644 --- a/Sources/AnnotKit/macOS/AXIntrospection.swift +++ b/Sources/AnnotKit/macOS/AXIntrospection.swift @@ -265,12 +265,45 @@ enum AXIntrospection { let candidates = chain.map { candidate(for: $0, windowFrame: windowFrame) } guard let targetIndex = AnnotationTargetRule.targetIndex(in: candidates) else { return [] } let target = chain[targetIndex] - let targetArea = area(of: target) + return [element(for: target, ancestorChain: chain)] + + enclosingComponents(of: target, containing: point, in: chain) + .map { element(for: $0, ancestorChain: ancestorChain(from: $0)) } + } + + /// The identified components that geometrically ENCLOSE `target` at `point`, + /// smallest-first — the widening rungs above a bound target. Shared verbatim + /// by the point path (``componentLadder(for:)``) and the frame path + /// (``marqueeLadder(for:)``) so a click and a drag onto the same element can + /// never widen through different components; duplicating this scan is exactly + /// how the two paths would silently drift apart. + /// + /// The scan is GEOMETRIC (DECISIONS.md): a `.axCardSurface` card hangs its + /// identifier on a clear background leaf that is a SIBLING of the card's + /// content, so it never appears in an ancestor chain. Scanning each ancestor + /// PLUS its direct children reaches those surfaces without a full-tree walk. + /// Deduped by identifier (the same surface is reachable from several + /// ancestors), and the `>= targetArea` floor keeps a smaller identified + /// sibling that merely happens to cover the point out of the widening ladder. + /// + /// Container ROOTS are excluded, matching ``AnnotationTargetRule/wideningLadder(in:)`` + /// stopping at the window. Load-bearing, not tidiness: the chain climbs to + /// `AXApplication`, whose direct children are the app's WINDOWS — including our + /// own overlay panel, which carries ``overlayWindowIdentifier`` and encloses + /// every point in the host. Without this the top rung of every ladder is + /// AnnotKit's own overlay, so widening would bind the user's note to our UI. + private static func enclosingComponents( + of target: AXUIElement, + containing point: CGPoint, + in rootFirstChain: [AXUIElement] + ) -> [AXUIElement] { + let targetArea = area(of: target) var containers: [(node: AXUIElement, area: CGFloat)] = [] var seen = Set() - for ancestor in chain { + for ancestor in rootFirstChain { for node in [ancestor] + elementArray(ancestor, kAXChildrenAttribute) { + let role = string(node, kAXRoleAttribute) ?? "" + guard role != "AXWindow", role != "AXApplication" else { continue } let id = string(node, kAXIdentifierAttribute) ?? "" guard !id.isEmpty, !seen.contains(id), !CFEqual(node, target) else { continue } let frame = frameScreen(of: node) @@ -282,9 +315,107 @@ enum AXIntrospection { } } containers.sort { $0.area < $1.area } + return containers.map(\.node) + } + + // MARK: - Marquee (drawn frame -> element) + + /// The component-widening ladder for a frame the user DREW: the element the + /// frame binds to per ``MarqueeTargetRule`` first, then each enclosing + /// identified component, broadest last. Same shape as + /// ``componentLadder(for:)``, because the session reuses its ladder machinery + /// verbatim — widening and the note's `component` field both assume + /// `ladder[0]` is the bound target. Empty when the frame resolves to nothing + /// (the session then captures a region note instead). + /// + /// Cost: this walks the whole window subtree ONCE, on drag RELEASE only — + /// never during the drag and never on hover. A full walk is affordable at that + /// rate; it would not be on the hover path, which is why the point path still + /// uses the ancestor-chain scan instead. + static func marqueeLadder(for rect: CGRect) -> [Element] { + // Standardize before anything geometric: a right-to-left / bottom-to-top + // drag arrives with negative extents, where `contains` degenerates and the + // window lookup below would silently find nothing. + let marquee = rect.standardized + let center = CGPoint(x: marquee.midX, y: marquee.midY) + let app = appElement() + // Same window pick as ``regionAnchor(for:)`` / ``hitBeneathOverlay(_:)``: + // `kAXWindows` is front-to-back, so the first non-overlay window containing + // the frame's center is the frontmost real target. + let windows = elementArray(app, kAXWindowsAttribute).filter { !isOverlayWindow($0) } + guard let window = windows.first(where: { frameScreen(of: $0).contains(center) }) else { return [] } + let windowFrame = frameScreen(of: window) + + // ONE recursive walk, so every candidate's depth is measured from the SAME + // root (window = 0). Depth is the rule's tie-break between geometrically + // indistinguishable candidates; assembling the array from several + // differently-rooted traversals would turn that tie-break into noise. + // + // The subtree is collected WHOLE — deliberately not pre-filtered to what + // intersects the drawn frame. The rule's second pass needs the candidates + // whose frames CONTAIN the frame (the user drew inside something), and an + // intersects-the-marquee filter is precisely what discards them. + var nodes: [AXUIElement] = [] + var candidates: [MarqueeCandidate] = [] + collectMarqueeCandidates( + window, windowFrame: windowFrame, depth: 0, nodes: &nodes, candidates: &candidates + ) + guard let resolution = MarqueeTargetRule.resolve(marquee: marquee, in: candidates) else { return [] } + let target = nodes[resolution.index] + let chain = ancestorChain(from: target) + + // The widening rungs are anchored at the TARGET's frame center, not the + // drawn frame's: a sloppy marquee can spill outside the element it bound + // to, and a container that does not contain the target is not a component + // the user could widen to. This is also the value the point path passes. + let targetFrame = frameScreen(of: target) + let targetCenter = CGPoint(x: targetFrame.midX, y: targetFrame.midY) return [element(for: target, ancestorChain: chain)] - + containers.map { element(for: $0.node, ancestorChain: ancestorChain(from: $0.node)) } + + enclosingComponents(of: target, containing: targetCenter, in: chain) + .map { element(for: $0, ancestorChain: ancestorChain(from: $0)) } + } + + /// Depth-first walk collecting a PARALLEL pair per node: the live + /// `AXUIElement` and its pure ``MarqueeCandidate``, so + /// ``MarqueeTargetRule/Resolution/index`` maps straight back to a live handle. + /// + /// The candidate is built with the same ``candidate(for:windowFrame:)`` the + /// point path uses, so chrome / container-root / window-ghost classification — + /// which is what the rule's eligibility filter reads — is identical for a click + /// and a drag by construction. + private static func collectMarqueeCandidates( + _ element: AXUIElement, + windowFrame: CGRect, + depth: Int, + nodes: inout [AXUIElement], + candidates: inout [MarqueeCandidate] + ) { + nodes.append(element) + candidates.append( + MarqueeCandidate( + element: candidate(for: element, windowFrame: windowFrame), + frame: frameScreen(of: element), + depth: depth + ) + ) + guard depth < maxDepth else { return } + for child in elementArray(element, kAXChildrenAttribute) { + // Starting from a non-overlay window should already put the overlay out + // of reach, but AppKit exposes an attached child PANEL through some + // parents' `kAXChildren`, so verify rather than assume: a marquee that + // swept the overlay's own hosting view would bind the note to our UI. + if isOverlayWindow(child) { continue } + // Chrome's whole subtree is skipped, not just the button: the traffic + // lights' inner glyph groups carry no chrome subrole of their own, so + // the rule's `isChrome` filter alone would let a marquee over the title + // bar bind to a glyph. The point path rejects chrome geometrically for + // the same reason. + if isChrome(child) { continue } + collectMarqueeCandidates( + child, windowFrame: windowFrame, depth: depth + 1, nodes: &nodes, candidates: &candidates + ) + } } /// Resolve `point` to a root-first ancestor chain of the deepest host element diff --git a/Sources/AnnotKit/macOS/MacElementSource.swift b/Sources/AnnotKit/macOS/MacElementSource.swift index 24e5248..03b0852 100644 --- a/Sources/AnnotKit/macOS/MacElementSource.swift +++ b/Sources/AnnotKit/macOS/MacElementSource.swift @@ -39,4 +39,10 @@ extension MacElementSource: ComponentLadderSource { AXIntrospection.componentLadder(for: point) } } + +extension MacElementSource: MarqueeTargetSource { + public func marqueeLadder(in rect: CGRect) -> [Element] { + AXIntrospection.marqueeLadder(for: rect) + } +} #endif diff --git a/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift b/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift index 7f79d2c..c77a99a 100644 --- a/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift +++ b/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift @@ -79,7 +79,7 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc public func hitTest(_ point: CGPoint) -> Element? { guard let chain = Self.resolvedChain(at: point) else { return nil } - let candidates = chain.map(Self.candidate(for:)) + let candidates = chain.map { Self.candidate(for: $0) } guard let index = AnnotationTargetRule.targetIndex(in: candidates) else { return nil } return Self.element(for: chain[index]) } @@ -88,7 +88,7 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc /// views), matching the AX and iOS sources so all three widen identically. public func componentLadder(at point: CGPoint) -> [Element] { guard let chain = Self.resolvedChain(at: point) else { return [] } - let candidates = chain.map(Self.candidate(for:)) + let candidates = chain.map { Self.candidate(for: $0) } return AnnotationTargetRule.wideningLadder(in: candidates).map { Self.element(for: chain[$0]) } } @@ -216,13 +216,21 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc /// Read one view's target-relevant facts into a pure ``TargetCandidate``. The /// NSView tree has no displayed value text, so `value` is always empty. - private static func candidate(for view: NSView) -> TargetCandidate { + /// + /// `isContainerRoot` defaults to false because on the POINT path the chain is + /// an ancestor chain rooted at the hit's window content view, and flagging it + /// there would change which element a click resolves to. The marquee walk + /// passes true for the content view: a whole-tree walk offers it as a real + /// candidate, and without the flag a large drag would surround it and bind the + /// note to the app's entire content view instead of the card inside it. + private static func candidate(for view: NSView, isContainerRoot: Bool = false) -> TargetCandidate { TargetCandidate( role: String(describing: Swift.type(of: view)), identifier: view.accessibilityIdentifier(), label: view.accessibilityLabel() ?? "", value: "", - isActionable: view is NSControl + isActionable: view is NSControl, + isContainerRoot: isContainerRoot ) } @@ -237,4 +245,110 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc NSScreen.screens.first?.frame.height ?? 0 } } + +// MARK: - Marquee (drawn frame -> view) + +extension MacViewTreeElementSource: MarqueeTargetSource { + /// The ladder for a frame the user DREW, over the `NSView` tree: the view the + /// frame binds to per ``MarqueeTargetRule`` first, then its enclosing + /// identified views, broadest last — the same contract as + /// ``componentLadder(at:)``, so the session's widening works from a framed + /// selection unchanged. + /// + /// Cost: one full walk of the hit window's view tree per drag RELEASE. Never + /// during the drag and never on hover, which is what makes a whole-tree walk + /// affordable here where the hover path must stay on the ancestor chain. + public func marqueeLadder(in rect: CGRect) -> [Element] { + // Standardize first: a right-to-left / bottom-to-top drag arrives with + // negative extents, where the window lookup's `contains` degenerates. + let marquee = rect.standardized + guard let content = Self.marqueeRoot(containing: CGPoint(x: marquee.midX, y: marquee.midY)) else { + return [] + } + + // ONE recursive walk from the content view, so every candidate's depth is + // measured from the SAME root (content view = 0) — depth is the rule's + // tie-break between geometrically indistinguishable candidates, and mixing + // differently-rooted numbering would make it noise. The subtree is + // collected WHOLE, deliberately not pre-filtered to what intersects the + // frame: the rule's enclosing pass needs the views that CONTAIN the frame, + // which such a filter is exactly what discards. + var views: [NSView] = [] + var candidates: [MarqueeCandidate] = [] + Self.collectMarqueeCandidates(content, isRoot: true, depth: 0, views: &views, candidates: &candidates) + + guard let resolution = MarqueeTargetRule.resolve(marquee: marquee, in: candidates) else { return [] } + let target = views[resolution.index] + return [Self.element(for: target)] + + Self.enclosingIdentifiedViews(of: target, upTo: content).map { Self.element(for: $0) } + } + + /// The content view of the frontmost visible non-overlay window under `point`. + /// `NSApp.orderedWindows` is front-to-back, mirroring how the AX source reads + /// `kAXWindows`, so both sources pick the same window for the same drag. The + /// overlay panel is excluded by the identifier ``OverlayController`` stamps on + /// it — the drawn frame always lies over the expanded overlay, so without this + /// every marquee would walk our own hosting view. + private static func marqueeRoot(containing point: CGPoint) -> NSView? { + for window in NSApp.orderedWindows { + guard window.isVisible, + window.accessibilityIdentifier() != AXIntrospection.overlayWindowIdentifier, + let content = window.contentView, + screenFrame(of: content).contains(point) + else { continue } + return content + } + return nil + } + + /// Depth-first walk collecting a PARALLEL pair per view — the live `NSView` + /// and its pure ``MarqueeCandidate`` — so + /// ``MarqueeTargetRule/Resolution/index`` maps straight back to a view. + /// + /// Hidden and fully transparent subtrees are skipped: they keep real frames, so + /// a marquee would happily "surround" a hidden view the user cannot even see, + /// and being the largest such frame it would win pass 1 outright. The point + /// path gets this for free from `NSView.hitTest`, which a whole-tree walk does + /// not go through. + private static func collectMarqueeCandidates( + _ view: NSView, + isRoot: Bool, + depth: Int, + views: inout [NSView], + candidates: inout [MarqueeCandidate] + ) { + views.append(view) + candidates.append( + MarqueeCandidate( + element: candidate(for: view, isContainerRoot: isRoot), + frame: screenFrame(of: view), + depth: depth + ) + ) + guard depth < maxDepth else { return } + for subview in view.subviews where !subview.isHidden && subview.alphaValue > 0.01 { + collectMarqueeCandidates( + subview, isRoot: false, depth: depth + 1, views: &views, candidates: &candidates + ) + } + } + + /// The identified superviews of `target`, nearest first (so broadest last), + /// stopping BEFORE `root` — the content view is the container root and is never + /// a widening rung, for the same reason it is never a target. Pure ancestry, + /// matching this source's point ladder; the AX source's extra geometric scan + /// exists for SwiftUI `.background` surfaces, which are AX-only artifacts with + /// no counterpart in the `NSView` tree. + private static func enclosingIdentifiedViews(of target: NSView, upTo root: NSView) -> [NSView] { + var out: [NSView] = [] + var current = target.superview + var depth = 0 + while let view = current, view !== root, depth < maxDepth { + if !view.accessibilityIdentifier().isEmpty { out.append(view) } + current = view.superview + depth += 1 + } + return out + } +} #endif diff --git a/Sources/AnnotKitOverlayProbe/main.swift b/Sources/AnnotKitOverlayProbe/main.swift index 2e02720..255a83c 100644 --- a/Sources/AnnotKitOverlayProbe/main.swift +++ b/Sources/AnnotKitOverlayProbe/main.swift @@ -1159,6 +1159,166 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { self.specController?.unmount() window.orderOut(nil) + self.phase7Marquee() + } + } + + // ---- Phase 7: marquee frame selection (F4) ------------------------------ + // The user press-drags a rectangle around what they mean. Unlike a click + // (deepest-wins) a FRAME means "this whole thing", so the LARGEST element the + // frame surrounds wins — the promise being that a sloppy rect around a card + // binds to the card and not to the label inside it. Reuses the Phase 6 + // fixture, whose seeded card / card text / section / button are exactly the + // nesting a marquee must disambiguate, and runs through the EXPANDED overlay + // because that is when a drag actually happens. + var marqueeController: OverlayController? + var marqueeSession: AnnotationSession? + var marqueeHost: NSWindow? + var passMarquee = true + func check7(_ cond: Bool, _ msg: String) { + passMarquee = passMarquee && cond + print(" " + (cond ? "ok " : "FAIL ") + msg) + } + + /// One-line rendering of a ladder for the log. + func describe(_ ladder: [Element]) -> String { + ladder.isEmpty ? "[] (region fallback)" : ladder.map { "#\($0.id) \(fmt($0.frame))" }.joined(separator: " -> ") + } + + /// The ladder contract the session depends on: `ladder[0]` is the bound + /// target, and every further rung is a DISTINCT, strictly enclosing component + /// (so widening from a framed selection only ever gets coarser). + func checkLadderShape(_ ladder: [Element], label: String) { + guard let target = ladder.first else { + check7(false, "\(label): ladder is non-empty") + return + } + var seen: Set = [target.id] + let targetArea = target.frame.width * target.frame.height + var wellFormed = true + for rung in ladder.dropFirst() { + let encloses = rung.frame.contains(target.frame) + let notSmaller = rung.frame.width * rung.frame.height >= targetArea + let distinct = seen.insert(rung.id).inserted + if !(encloses && notSmaller && distinct) { + wellFormed = false + print(" rung #\(rung.id) \(fmt(rung.frame)) encloses=\(encloses) " + + "notSmaller=\(notSmaller) distinct=\(distinct)") + } + } + check7(wellFormed, "\(label): every rung above ladder[0] encloses the target, is no smaller, and is distinct") + // Regression guard for a defect this probe found: the ancestor chain climbs + // to AXApplication, whose direct CHILDREN are the app's windows — our own + // overlay panel among them, identified and enclosing everything. It used to + // be the top rung of every ladder, so widening bound the note to AnnotKit's + // own UI. Never a host component, so never a rung. + check7(!ladder.contains { $0.id == overlayWindowIdentifier }, + "\(label): no rung is AnnotKit's own overlay panel") + } + + func phase7Marquee() { + print("\n--- Phase 7: marquee frame selection (drawn rect -> element) ---") + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 520, height: 480), + styleMask: [.titled, .closable], + backing: .buffered, + defer: false + ) + window.title = "AnnotKit Harness W7 (marquee)" + window.contentView = NSHostingView(rootView: ProbeSpecificityView()) + window.makeKeyAndOrderFront(nil) + marqueeHost = window + + let session = AnnotationSession( + source: MacElementSource(), + sink: NotesFileSink(path: NSTemporaryDirectory() + "annotkit-marquee.md") + ) + let controller = OverlayController(session: session) + controller.mount(on: window) + controller.start() + marqueeController = controller + marqueeSession = session + + DispatchQueue.main.asyncAfter(deadline: .now() + 0.9) { [weak self] in + guard let self else { return } + let source = MacElementSource() + let roots = source.snapshot().map(\.root) + guard let button = self.findElement(id: "Spec.Button", in: roots), + let card = self.findElement(id: "Spec.Card", in: roots), + let cardText = self.findElement(id: "Spec.CardText", in: roots), + let section = self.findElement(id: "Spec.Section", in: roots) else { + check7(false, "snapshot exposes Spec.Button/Card/CardText/Section") + self.marqueeController?.unmount() + window.orderOut(nil) + self.finish() + return + } + print(" fixture: card=\(fmt(card.frame)) cardText=\(fmt(cardText.frame)) " + + "section=\(fmt(section.frame)) button=\(fmt(button.frame))") + + // 7a — a SLOPPY frame around the card (drawn 8pt proud of it, the way a + // hand-drag overshoots) binds to the CARD. The frame also fully + // surrounds the card's text, so this is the check that largest-wins is + // in force: without it the drag would bind to the label inside. + let aroundCard = card.frame.insetBy(dx: -8, dy: -8) + let cardLadder = source.marqueeLadder(in: aroundCard) + print(" frame around card \(fmt(aroundCard)) -> \(self.describe(cardLadder))") + check7(aroundCard.contains(cardText.frame), "sanity: the drawn frame also surrounds the card's TEXT") + check7(cardLadder.first?.id == "Spec.Card", + "frame around the card -> #Spec.Card (got \(cardLadder.first.map { "#\($0.id)" } ?? "nil"))") + check7(cardLadder.first?.id != "Spec.CardText", + "frame around the card is NOT bound to the text inside it (the feature's core promise)") + self.checkLadderShape(cardLadder, label: "card ladder") + check7(cardLadder.dropFirst().contains { $0.id == "Spec.Section" }, + "the card ladder can still widen to #Spec.Section (widening works from a framed selection)") + + // 7b — a frame around the whole section binds to the SECTION, even + // though it surrounds the card, the button and the texts as well. + let aroundSection = section.frame.insetBy(dx: -6, dy: -6) + let sectionLadder = source.marqueeLadder(in: aroundSection) + print(" frame around section \(fmt(aroundSection)) -> \(self.describe(sectionLadder))") + check7(sectionLadder.first?.id == "Spec.Section", + "frame around the section -> #Spec.Section (got \(sectionLadder.first.map { "#\($0.id)" } ?? "nil"))") + self.checkLadderShape(sectionLadder, label: "section ladder") + + // 7c — a tight frame around just the button binds to the BUTTON: a + // small drag stays as specific as a click would have been. + let aroundButton = button.frame.insetBy(dx: -4, dy: -4) + let buttonLadder = source.marqueeLadder(in: aroundButton) + print(" frame around button \(fmt(aroundButton)) -> \(self.describe(buttonLadder))") + check7(buttonLadder.first?.id == "Spec.Button", + "frame around the button -> #Spec.Button (got \(buttonLadder.first.map { "#\($0.id)" } ?? "nil"))") + self.checkLadderShape(buttonLadder, label: "button ladder") + + // 7d — a frame drawn strictly INSIDE the card, in its padding, that + // surrounds NOTHING: the enclosing fallback binds it to the tightest + // thing it was drawn inside, the card — not the section that also + // contains it. This is the rect generalization of the point-region path. + let insideCard = CGRect(x: card.frame.minX + 6, y: card.frame.midY - 8, width: 16, height: 16) + let insideLadder = source.marqueeLadder(in: insideCard) + print(" frame inside card \(fmt(insideCard)) -> \(self.describe(insideLadder))") + check7(card.frame.contains(insideCard), "sanity: the inside frame is strictly within the card") + check7(!insideCard.intersects(cardText.frame), "sanity: the inside frame surrounds nothing (misses the text)") + check7(insideLadder.first?.id == "Spec.Card", + "frame inside the card -> #Spec.Card via the enclosing fallback (got \(insideLadder.first.map { "#\($0.id)" } ?? "nil"))") + self.checkLadderShape(insideLadder, label: "inside-card ladder") + + // 7e — a frame over empty space, outside every window: the adapter + // returns [] and hands the drag to the session's region fallback rather + // than binding a note to whatever happened to be frontmost. + let nowhere = CGRect(x: -20000, y: -20000, width: 120, height: 90) + let nowhereLadder = source.marqueeLadder(in: nowhere) + print(" frame over empty space \(fmt(nowhere)) -> \(self.describe(nowhereLadder))") + check7(nowhereLadder.isEmpty, "frame outside any window -> [] (region-fallback handoff)") + + // 7f — a press-release that never moved is a CLICK, not a marquee: a + // degenerate rect must not bind to everything that encloses it. + let degenerate = CGRect(origin: center(of: card.frame), size: .zero) + check7(source.marqueeLadder(in: degenerate).isEmpty, + "zero-area frame -> [] (a click is not a marquee)") + + self.marqueeController?.unmount() + window.orderOut(nil) self.finish() } } @@ -1200,8 +1360,9 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { print(" Phase 4 (window chrome excluded from the hit-test): \(passChrome ? "PASS" : "FAIL")") print(" Phase 5 (seeded container resolves on body hover): \(passCard ? "PASS" : "FAIL")") print(" Phase 6 (positional specificity by cursor position): \(passSpec ? "PASS" : "FAIL")") + print(" Phase 7 (marquee frame selection: drawn rect -> element): \(passMarquee ? "PASS" : "FAIL")") print("\n=== AnnotKitOverlayProbe complete ===") - exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec ? 0 : 1) + exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec && passMarquee ? 0 : 1) } func collectIDs(_ elements: [Element]) -> [String] { From 33d9822901d7644bc933f96fb3dfb69df63ba821 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:04:00 -0500 Subject: [PATCH 08/24] test(marquee): probe sub-phase 7g pins the overlay out of every ladder Collects every identifier in the overlay panel's subtree and intersects it against all resolved ladders, with a spanning check so the assertion cannot go vacuous. Guards a role-based check's blind spot: stray identified DESCENDANTS of the overlay rather than the panel node. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKitOverlayProbe/main.swift | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Sources/AnnotKitOverlayProbe/main.swift b/Sources/AnnotKitOverlayProbe/main.swift index 255a83c..9d3c7e2 100644 --- a/Sources/AnnotKitOverlayProbe/main.swift +++ b/Sources/AnnotKitOverlayProbe/main.swift @@ -1317,6 +1317,39 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { check7(source.marqueeLadder(in: degenerate).isEmpty, "zero-area frame -> [] (a click is not a marquee)") + // 7g — NOTHING from AnnotKit's own overlay can enter the candidate set. + // Sharper for a marquee than for a click: a drag rect by construction + // spans screen the overlay is drawn across, and overlay elements are + // genuinely identified and genuinely meaningful, so the rule cannot + // reject them — a large overlay surface would WIN pass 1 on area and + // bind the user's note to our own UI. The walk is rooted at the HOST + // window (the overlay is filtered out of `kAXWindows` by identifier + // BEFORE the root is picked, never by "key" or "frontmost"), so the + // overlay's descendants are out of reach by construction. This asserts + // that construction against the live tree instead of trusting it. + let app = AXUIElementCreateApplication(ProcessInfo.processInfo.processIdentifier) + AXUIElementSetAttributeValue(app, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + let rawWindows = AX.windows(app) + let overlayPanel = rawWindows.first { AX.string($0, kAXIdentifierAttribute) == overlayWindowIdentifier } + check7(overlayPanel != nil, "the overlay panel IS a live AX window during the drag (a real shadowing risk)") + if let overlayPanel { + let panelFrame = AX.frame(overlayPanel) + // Without this the whole sub-phase would be vacuous: an overlay that + // does not cover the drag region proves nothing about one that does. + check7(panelFrame.contains(card.frame), + "sanity: the expanded overlay SPANS the drag region (the card is drawn beneath it)") + let overlayIDs = Set(self.axCollectIDs(in: overlayPanel, depth: 0)) + let hostWindow = rawWindows.first { AX.string($0, kAXTitleAttribute) == "AnnotKit Harness W7 (marquee)" } + let panelIsAXChildOfHost = hostWindow.map { host in + AX.children(host).contains { CFEqual($0, overlayPanel) } + } ?? false + print(" overlay panel \(fmt(panelFrame)) carries \(overlayIDs.count) identified element(s); " + + "exposed as an AX CHILD of the host window: \(panelIsAXChildOfHost)") + let everyResult = cardLadder + sectionLadder + buttonLadder + insideLadder + check7(!everyResult.contains { overlayIDs.contains($0.id) }, + "no marquee result — target or rung — is an element from AnnotKit's own overlay subtree") + } + self.marqueeController?.unmount() window.orderOut(nil) self.finish() @@ -1332,6 +1365,19 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { return nil } + /// Every non-empty AX identifier in `element`'s subtree, including its own — + /// the set of ids a walk that strayed into the overlay would surface. + func axCollectIDs(in element: AXUIElement, depth: Int) -> [String] { + guard depth < 32 else { return [] } + var out: [String] = [] + let id = AX.string(element, kAXIdentifierAttribute) + if !id.isEmpty { out.append(id) } + for child in AX.children(element) { + out.append(contentsOf: axCollectIDs(in: child, depth: depth + 1)) + } + return out + } + /// Recursive raw-AX search for elements matching one of `subroles`. func axFindAll(in element: AXUIElement, subroles: Set, depth: Int) -> [AXUIElement] { guard depth < 12 else { return [] } From 2ea10f592882ec9aa64c945dc5fae4ccc84a871a Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:05:31 -0500 Subject: [PATCH 09/24] feat(overlay): marquee drag gesture with a tested click/frame threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the catcher's SpatialTapGesture with a single DragGesture (minimumDistance: 0) that branches at release: travel above the slop threshold routes to select(inAXRect:), anything at or below it routes to select(atAXPoint:) at the press's startLocation. The decision lives in a new pure MarqueeDrag type (ComposerPlacement pattern) rather than inside the gesture closure, because it is the part that silently breaks the mode when wrong: select(inAXRect:) returns nil for a zero-area rect, so a click routed into the marquee path makes clicking do nothing, and a jitter-drag is not zero-area at all — it resolves to whatever container the pointer sat in and plants a note the user never framed. One recognizer, not a composition: exclusive/simultaneous gestures make the recognizers negotiate, and the case that loses is the plain click. Also draws the in-progress band (dashed accent stroke, lighter fill) in place of the element highlight, and suppresses hover resolution while a band is live. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/Overlay/MarqueeDrag.swift | 67 +++++++++++++ Sources/AnnotKit/Overlay/OverlayView.swift | 73 ++++++++++++-- Tests/AnnotKitTests/MarqueeDragTests.swift | 110 +++++++++++++++++++++ 3 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 Sources/AnnotKit/Overlay/MarqueeDrag.swift create mode 100644 Tests/AnnotKitTests/MarqueeDragTests.swift diff --git a/Sources/AnnotKit/Overlay/MarqueeDrag.swift b/Sources/AnnotKit/Overlay/MarqueeDrag.swift new file mode 100644 index 0000000..7f2e181 --- /dev/null +++ b/Sources/AnnotKit/Overlay/MarqueeDrag.swift @@ -0,0 +1,67 @@ +import CoreGraphics + +/// The pure part of the marquee gesture: given where a press began and where the +/// pointer is now, decide whether it is a FRAME or a CLICK and produce the +/// geometry for whichever it is. +/// +/// This lives outside the SwiftUI closure for the same reason ``ComposerPlacement`` +/// does — a decision buried in a gesture callback can only be checked by a human +/// with a mouse, and this particular decision is the one that silently breaks the +/// whole mode when it is wrong. +/// +/// Why the threshold is the load-bearing part: +/// +/// * A press that never moved is a CLICK and must go to +/// ``AnnotationSession/select(atAXPoint:)``. ``AnnotationSession/select(inAXRect:)`` +/// returns nil for a zero-area rect (deliberately — see its CALLER CONTRACT), so +/// routing a click into the marquee path makes clicking do NOTHING in annotate +/// mode. That reads as a dead feature, not as a missing constant. +/// * A 3-point jitter-drag is the worse failure, because it is NOT zero-area: the +/// rule resolves it happily and lands on whatever container the pointer happened +/// to sit in, producing a plausible-looking note bound to something the user +/// never framed. A wrong note is more expensive than no note, because nobody +/// goes back to check it. +/// +/// Travel is measured as the hypotenuse, not per-axis: a 5-point-right, +/// 5-point-down drift is ~7 points of real movement and reads as intentional, while +/// a per-axis test would call it a click on both axes. +/// +/// Touch needs more slop than a cursor because a finger rolls several points on a +/// deliberate tap — a mouse-tuned threshold on iOS turns ordinary taps into +/// accidental marquees. +enum MarqueeDrag { + /// Travel a press must EXCEED before it counts as a frame rather than a click. + /// Exactly at the threshold is still a click: the boundary belongs to the + /// safer branch, since a click misread as a frame plants a wrong note whereas + /// a frame misread as a click just selects what is under the press. + #if os(iOS) + static let minimumTravel: CGFloat = 10 + #else + static let minimumTravel: CGFloat = 6 + #endif + + /// True when the press travelled far enough to be a deliberate frame. + static func isFrame(from start: CGPoint, to end: CGPoint) -> Bool { + let dx = end.x - start.x + let dy = end.y - start.y + return (dx * dx + dy * dy).squareRoot() > minimumTravel + } + + /// Window-local rect, normalized so any drag direction gives the same result. + /// Users drag up-left as readily as down-right; without `standardized` the + /// three "backwards" directions produce negative extents, which the session's + /// degenerate guard would treat as a non-drag. + static func localRect(from start: CGPoint, to end: CGPoint) -> CGRect { + CGRect(x: start.x, y: start.y, width: end.x - start.x, height: end.y - start.y) + .standardized + } + + /// AX screen rect: the window-local rect shifted by the surface's AX origin. + /// Gesture coordinates are window-local, and the AX queries behind + /// ``AnnotationSession/select(inAXRect:)`` are in AX screen space — the same + /// ADD-to-query / SUBTRACT-to-draw transform the click path and the highlight + /// share, which is why all of them agree on a secondary display. + static func axRect(from start: CGPoint, to end: CGPoint, axOrigin: CGPoint) -> CGRect { + localRect(from: start, to: end).offsetBy(dx: axOrigin.x, dy: axOrigin.y) + } +} diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index a7f73eb..ccea1b8 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -4,8 +4,9 @@ import SwiftUI /// /// Interaction is driven entirely through SwiftUI hit-testing, which fixes the /// two ways a global click monitor went wrong: a full-screen catcher (active -/// only in annotate mode, behind the chrome) receives hover and taps over the -/// app, while the toolbar and composer sit on top and consume their own clicks, +/// only in annotate mode, behind the chrome) receives hover, taps, and marquee +/// drags over the app (see ``MarqueeDrag`` for the click-vs-frame branch), while +/// the toolbar and composer sit on top and consume their own clicks, /// so tapping "Add note" can never re-select the element under the button. The /// whole overlay is `accessibilityHidden` so the AX point query sees through it /// to the app beneath. @@ -43,12 +44,34 @@ struct OverlayView: View { /// Draft text for the pin edit card, seeded from the note's comment when /// editing begins (see the `editingNoteID` seeding hook on the ZStack). @State private var editDraft: String = "" + /// The in-progress marquee band, WINDOW-LOCAL (gesture coordinates), non-nil + /// only between "this press has travelled far enough to be a frame" and the + /// release that resolves it. It is never the resolved selection — that comes + /// back as `session.selected` and is drawn by the highlight branch. + @State private var marqueeRect: CGRect? var body: some View { ZStack(alignment: .topLeading) { catcher - if let element = session.selected ?? session.hovered { + // The band REPLACES the element highlight while a frame is being drawn + // (same ZStack slot: above the catcher, below the chrome). Showing both + // would put a solid "this is what you get" highlight under a rectangle + // that has not resolved to anything yet. + if let marqueeRect { + // COORDINATES: `marqueeRect` is already window-local, so it is + // offset DIRECTLY — no `axOrigin` subtraction, unlike the element + // highlight one branch down, which arrives in AX screen space. + // Subtracting here "for symmetry" would slide the band off by the + // window's screen origin, invisible on the primary display at the + // global origin and badly wrong on every secondary display. + RoundedRectangle(cornerRadius: 3) + .stroke(Color.accentColor, style: StrokeStyle(lineWidth: 2, dash: [6, 4])) + .background(Color.accentColor.opacity(0.08)) + .frame(width: marqueeRect.width, height: marqueeRect.height) + .offset(x: marqueeRect.minX, y: marqueeRect.minY) + .allowsHitTesting(false) + } else if let element = session.selected ?? session.hovered { let originX = element.frame.minX - axOrigin.x let originY = element.frame.minY - axOrigin.y RoundedRectangle(cornerRadius: 3) @@ -126,6 +149,12 @@ struct OverlayView: View { .onContinuousHover { phase in switch phase { case .active(let point): + // While a band is live the user is FRAMING, not hovering: + // resolving an element under the moving pointer would draw a + // competing highlight underneath the band (and burn an AX + // query per motion event) for a selection the release is + // about to overwrite anyway. + guard marqueeRect == nil else { return } session.hover(atAXPoint: CGPoint(x: point.x + axOrigin.x, y: point.y + axOrigin.y)) case .ended: // The cursor left the catcher (it covers the host's full @@ -137,10 +166,42 @@ struct OverlayView: View { session.clearHover() } } + // ONE gesture handles both clicks and frames, branching on travel at + // release. It is deliberately NOT a DragGesture composed with the + // old SpatialTapGesture: `.exclusively`/`.simultaneously` make the + // recognizers negotiate, and the case that loses that negotiation is + // the plain click — which then does nothing at all in annotate mode. + // A single recognizer with a distance branch has no ambiguity to + // lose a click in. `minimumDistance: 0` is what lets it also see the + // press that never moved. .gesture( - SpatialTapGesture().onEnded { event in - session.select(atAXPoint: CGPoint(x: event.location.x + axOrigin.x, y: event.location.y + axOrigin.y)) - } + DragGesture(minimumDistance: 0) + .onChanged { value in + // Below the threshold nothing is drawn: a band that + // flickered up on every click would advertise a marquee + // the release is going to route as a click. + guard MarqueeDrag.isFrame(from: value.startLocation, to: value.location) else { return } + marqueeRect = MarqueeDrag.localRect(from: value.startLocation, to: value.location) + session.clearHover() + } + .onEnded { value in + marqueeRect = nil + if MarqueeDrag.isFrame(from: value.startLocation, to: value.location) { + session.select(inAXRect: MarqueeDrag.axRect( + from: value.startLocation, + to: value.location, + axOrigin: axOrigin + )) + } else { + // `startLocation`, not `location`: it is where the + // user AIMED, and it is stable for a press that + // drifted a point or two under the finger/cursor. + session.select(atAXPoint: CGPoint( + x: value.startLocation.x + axOrigin.x, + y: value.startLocation.y + axOrigin.y + )) + } + } ) } else { Color.clear.allowsHitTesting(false) diff --git a/Tests/AnnotKitTests/MarqueeDragTests.swift b/Tests/AnnotKitTests/MarqueeDragTests.swift new file mode 100644 index 0000000..c05dd83 --- /dev/null +++ b/Tests/AnnotKitTests/MarqueeDragTests.swift @@ -0,0 +1,110 @@ +import CoreGraphics +import XCTest +@testable import AnnotKit + +final class MarqueeDragTests: XCTestCase { + private let start = CGPoint(x: 100, y: 100) + + func testZeroTravelPressIsAClick() { + // The regression this threshold exists for: `select(inAXRect:)` returns nil + // for a zero-area rect, so a press routed into the marquee path makes + // clicking do nothing at all in annotate mode. + XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: start)) + XCTAssertEqual(MarqueeDrag.localRect(from: start, to: start), CGRect(x: 100, y: 100, width: 0, height: 0)) + } + + func testTravelJustBelowThresholdIsAClick() { + // A jitter-drag is NOT zero-area, so nothing downstream rejects it: it would + // resolve to whatever container the pointer sat in and plant a note the user + // never framed. + let below = CGPoint(x: start.x + MarqueeDrag.minimumTravel - 0.5, y: start.y) + XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: below)) + } + + func testTravelExactlyAtThresholdIsStillAClick() { + // The boundary belongs to the safer branch: a click misread as a frame + // plants a wrong note, a frame misread as a click just selects the press. + let exact = CGPoint(x: start.x + MarqueeDrag.minimumTravel, y: start.y) + XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: exact)) + } + + func testTravelJustAboveThresholdIsAFrame() { + let above = CGPoint(x: start.x + MarqueeDrag.minimumTravel + 0.5, y: start.y) + XCTAssertTrue(MarqueeDrag.isFrame(from: start, to: above)) + } + + func testDiagonalTravelIsMeasuredAsHypotenuseNotPerAxis() { + // Each axis alone is under the threshold; together they are over it. A + // per-axis test would call this deliberate diagonal drag a click. + let leg = MarqueeDrag.minimumTravel * 0.8 // hypotenuse = 1.131 * threshold + let diagonal = CGPoint(x: start.x + leg, y: start.y + leg) + XCTAssertLessThan(leg, MarqueeDrag.minimumTravel) + XCTAssertTrue(MarqueeDrag.isFrame(from: start, to: diagonal)) + + // And the converse: a diagonal whose hypotenuse is under the threshold is a + // click even though it moved on both axes. + let short = MarqueeDrag.minimumTravel * 0.5 // hypotenuse = 0.707 * threshold + XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: CGPoint(x: start.x + short, y: start.y + short))) + } + + func testAllFourDragDirectionsNormalizeToTheSameRect() { + // Users drag up-left as readily as down-right. Without normalization the + // three "backwards" directions arrive with negative extents, which the + // session's degenerate guard treats as a non-drag — the marquee would work + // in exactly one direction. + let expected = CGRect(x: 100, y: 100, width: 80, height: 60) + let a = CGPoint(x: 100, y: 100), b = CGPoint(x: 180, y: 160) + XCTAssertEqual(MarqueeDrag.localRect(from: a, to: b), expected) // down-right + XCTAssertEqual(MarqueeDrag.localRect(from: b, to: a), expected) // up-left + XCTAssertEqual(MarqueeDrag.localRect(from: CGPoint(x: 180, y: 100), to: CGPoint(x: 100, y: 160)), expected) // down-left + XCTAssertEqual(MarqueeDrag.localRect(from: CGPoint(x: 100, y: 160), to: CGPoint(x: 180, y: 100)), expected) // up-right + } + + func testAXRectIsWindowLocalRectAtOriginOnPrimaryDisplay() { + // `axOrigin == .zero` on the primary display at the global origin, so the + // AX rect and the drawn band coincide there — which is why an origin bug + // hides until someone uses a second display. + XCTAssertEqual( + MarqueeDrag.axRect(from: CGPoint(x: 100, y: 100), to: CGPoint(x: 180, y: 160), axOrigin: .zero), + CGRect(x: 100, y: 100, width: 80, height: 60) + ) + } + + func testAXRectAppliesANonZeroOriginExactly() { + // Secondary-display case: gesture coordinates are window-local, the AX + // queries are in screen space, so the origin is ADDED (the mirror of the + // highlight's subtraction). + XCTAssertEqual( + MarqueeDrag.axRect( + from: CGPoint(x: 100, y: 100), + to: CGPoint(x: 180, y: 160), + axOrigin: CGPoint(x: 1512, y: 30) + ), + CGRect(x: 1612, y: 130, width: 80, height: 60) + ) + } + + func testAXRectNormalizesBeforeShiftingSoABackwardsDragIsStillOnScreen() { + // A up-left drag shifted by the origin must land at the frame's top-left, + // not at the press point with negative extents (which would resolve to + // nothing at all). + XCTAssertEqual( + MarqueeDrag.axRect( + from: CGPoint(x: 180, y: 160), + to: CGPoint(x: 100, y: 100), + axOrigin: CGPoint(x: 1512, y: 30) + ), + CGRect(x: 1612, y: 130, width: 80, height: 60) + ) + } + + func testThresholdIsLargerOnTouchThanCursor() { + // A finger rolls several points on a deliberate tap, so a mouse-tuned + // threshold on iOS turns ordinary taps into accidental marquees. + #if os(iOS) + XCTAssertEqual(MarqueeDrag.minimumTravel, 10) + #else + XCTAssertEqual(MarqueeDrag.minimumTravel, 6) + #endif + } +} From 09ac045d8af4234ad998bc82943e2a34ed8b7682 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:05:56 -0500 Subject: [PATCH 10/24] docs: README covers frame selection Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 60a7da2..5c6b392 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # AnnotKit Native in-app annotation for AI coding agents. Click a UI element in your own -macOS or iOS app, attach a note, and emit an agent-readable, code-locating -annotation. The native analogue of the web Agentation tool. +macOS or iOS app — or drag a frame around it — attach a note, and emit an +agent-readable, code-locating annotation. The native analogue of the web +Agentation tool. -A click becomes a stable selector, an element path, a screenshot, and your +The gesture becomes a stable selector, an element path, a screenshot, and your comment, so an AI coding agent can locate the exact view instead of guessing from a verbal description. @@ -47,6 +48,12 @@ Annotation.install(sink: ClipboardSink(format: .json)) non-identified target to its nearest seeded `accessibilityIdentifier` (`#Settings.Models >> @Save`), so it round-trips a resolver and points an agent at the right component's code. See `DECISIONS.md`. +- **Drawing a frame** instead of clicking inverts that rule on purpose: a click + means "this exact spot" and descends, while a box drawn around a card means "I + mean this *whole* thing", so the **largest** element the frame surrounds wins + and the labels inside it do not. A frame drawn *inside* something binds to the + tightest element enclosing it, and the drawn rect rides along on the note. + Saves hunting for the one pixel that hit-tests to a composite component. - Notes are written in the `AGENTATION_NOTES.md` format that the `process-agentation-notes` skill consumes, or copied to the clipboard. From 5da04a42ab752e40c024b10e1d90fd9f9e4bade9 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:10:36 -0500 Subject: [PATCH 11/24] feat(ios): marquee frame selection + exclude AnnotKit's overlay window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port marquee selection to iOS: `IOSElementSource` now conforms to `MarqueeTargetSource`, walking the UIView tree once from a single window root to build `[MarqueeCandidate]` and resolving with the shared, pure `MarqueeTargetRule` — so an identical drag over an identical layout resolves identically on both platforms by construction, not by two implementations being kept in step by hand. Returns the same target-first, broadest-last ladder `componentLadder(at:)` produces, so the session's widening and the note's `component` field work unchanged. Also excludes AnnotKit's own overlay from `IOSElementSource.windows()` by `PassThroughWindow` TYPE identity. Unlike macOS (a separate NSPanel already unreachable via kAXWindows), the iOS overlay is a UIWindow in the HOST's scene sharing its pid, and its chrome is genuinely identified and meaningful — a marquee spans the area it draws across, so a large overlay surface could win the rule's first pass outright and bind the user's note to our own UI. Filtering in the shared helper makes snapshot, hitTest, keyWindow, componentLadder and the marquee path agree. Verified: `xcrun --sdk iphoneos swiftc -typecheck -target arm64-apple-ios17.0 -swift-version 6` clean (no warnings), and a Mac Catalyst build (the os(iOS) path) succeeds. macOS unaffected: 99 tests green, AnnotKitOverlayProbe all-PASS. Co-Authored-By: Claude Opus 5 (1M context) --- PARITY.md | 3 + Sources/AnnotKit/iOS/IOSElementSource.swift | 171 ++++++++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/PARITY.md b/PARITY.md index e7823fc..62ce317 100644 --- a/PARITY.md +++ b/PARITY.md @@ -13,6 +13,9 @@ row; each asymmetry is closed by code or has a tracked mitigation. | Hit test primitive | `AXUIElementCopyElementAtPosition` + NSView `hitTest` | `UIView.hitTest(_:with:)` | iOS has no global AX point query; uses view hitTest. Tracked: F5.2 | | Annotation target rule | shared `AnnotationTargetRule` over an AX candidate chain | shared `AnnotationTargetRule` over a UIView candidate chain | none — both build a `[TargetCandidate]` chain and apply the SAME rule (deepest actionable, else deepest meaningful). Closes the earlier split (macOS "deepest meaningful" vs iOS "nearest identified"), cli-got28.2 | | Component widening | `ComponentLadderSource` (AX chain) | `ComponentLadderSource` (UIView chain) | none — same ladder (target, then enclosing identified components) | +| Marquee selection (drawn frame → element) | `MarqueeTargetSource`: shared `MarqueeTargetRule` over `[MarqueeCandidate]` read from the AX tree | `MarqueeTargetSource`: shared `MarqueeTargetRule` over `[MarqueeCandidate]` read from the UIView tree | none — the DECISION is one pure rule (largest ≥85%-surrounded element, else the tightest enclosing one); the adapters differ only in how they read candidates out of their own tree. Both do ONE walk from a single root so depth (the rule's tie-break) is numbered comparably, both collect the subtree WHOLE (an intersects-the-frame filter would discard the enclosing-pass candidates), and both return the SAME target-first, broadest-last ladder as `ComponentLadderSource`, so the session's widening and `component` field work unchanged from a framed selection | +| Marquee drag threshold | cursor slop (a mouse does not move on a deliberate click) | larger touch slop | ASYMMETRIC BY DESIGN, owned by the drag UI, not the adapters: a finger rolls several points on a deliberate tap, so the macOS threshold on iOS would turn taps into marquees. Below the threshold both platforms route the gesture to the point path (`select(atAXPoint:)`), per the caller contract on `select(inAXRect:)` | +| Overlay excluded from element lookup | AX window identifier (`AXIntrospection.overlayWindowIdentifier`) filtered out of every `kAXWindows` read | `PassThroughWindow` TYPE identity filtered out of `IOSElementSource.windows()` | ASYMMETRIC BY NECESSITY — the hosts are different window kinds. macOS's overlay is a separate `NSPanel` matched by the identifier the controller stamps on it; iOS's is a `UIWindow` in the HOST's scene sharing its pid, so no pid/scene filter separates it and a type check (internal to the module) cannot drift the way an identifier convention can. Both filter in the shared window lookup, so snapshot / hit-test / region-anchor / marquee agree; leaving it in would let a marquee bind the user's note to AnnotKit's own UI | | Coordinate space | Cocoa bottom-left to AX top-left flip | UIKit top-left native | iOS needs no flip; shared `ScreenSpace` used only on macOS | | Screenshot | ScreenCaptureKit / `cacheDisplay` | `UIGraphicsImageRenderer` + `drawHierarchy` | both capture own hierarchy only; no cross-window or secure overlays | | Overlay host | resizing `NSPanel` (toolbar corner idle, full screen annotating) | pass-through `UIWindow` | both interactive; selection via the shared SwiftUI catcher, not a global monitor | diff --git a/Sources/AnnotKit/iOS/IOSElementSource.swift b/Sources/AnnotKit/iOS/IOSElementSource.swift index 41366ca..5a5f65c 100644 --- a/Sources/AnnotKit/iOS/IOSElementSource.swift +++ b/Sources/AnnotKit/iOS/IOSElementSource.swift @@ -260,10 +260,30 @@ public final class IOSElementSource: ElementSource, ComponentLadderSource { // MARK: - Window helpers + /// Every inspectable HOST window — AnnotKit's own overlay is never one. + /// + /// On macOS the overlay is a separate `NSPanel` filtered out of `kAXWindows` + /// by identifier; on iOS the risk is live rather than theoretical, because + /// ``PassThroughWindow`` sits in the SAME scene as the host and is returned by + /// `UIWindowScene.windows` like any other window. Its chrome is genuinely + /// identified and genuinely meaningful, so it passes + /// ``TargetCandidate/isEligibleMeaningful`` cleanly: leave it in and a marquee + /// (which by construction spans screen area the overlay draws across) can bind + /// the user's note to AnnotKit's own UI, and a snapshot/selector lists our + /// toolbar as if it were app content. + /// + /// Excluded by TYPE identity, not by name, identifier, or window level: the + /// type is internal to this module so the check cannot be defeated by a naming + /// convention drifting, and a pid filter is useless here because AnnotKit runs + /// in the host's process. Filtering in this ONE helper is what makes + /// ``snapshot()``, ``keyWindow``, `hitTest`, `componentLadder(at:)`, and the + /// marquee path agree — matching the macOS side, which excludes the overlay in + /// every window lookup rather than only in the one that motivated it. private static func windows() -> [UIWindow] { UIApplication.shared.connectedScenes .compactMap { $0 as? UIWindowScene } .flatMap(\.windows) + .filter { !($0 is PassThroughWindow) } } static var keyWindow: UIWindow? { @@ -276,4 +296,155 @@ public final class IOSElementSource: ElementSource, ComponentLadderSource { return window.convert(inWindow, to: nil) } } + +// MARK: - Marquee (drawn frame -> view) + +extension IOSElementSource: MarqueeTargetSource { + /// The ladder for a frame the user DREW, over the `UIView` tree: the view the + /// frame binds to per ``MarqueeTargetRule`` first, then each enclosing + /// identified component, broadest last — the SAME target-first contract as + /// ``componentLadder(at:)``, because the session assumes `ladder[0]` IS the + /// bound target for both widening and the note's `component` field. + /// + /// Structurally identical to the macOS path (`AXIntrospection.marqueeLadder`): + /// both platforms only differ in how they read candidates out of their tree, + /// and the decision itself is the one shared pure rule, so an identical drag + /// over an identical layout resolves identically by construction rather than + /// by two implementations being kept in step by hand. + /// + /// Cost: one full walk of the hit window's view tree per drag RELEASE — never + /// during the drag and never on touch-move. That rate is what makes a + /// whole-tree walk affordable here, where the point path must stay on the + /// ancestor chain. + public func marqueeLadder(in rect: CGRect) -> [Element] { + // Standardize before anything geometric: a right-to-left / bottom-to-top + // drag arrives with negative extents, where `contains` degenerates and the + // window lookup below would silently find nothing. + let marquee = rect.standardized + guard let root = Self.marqueeRoot(containing: CGPoint(x: marquee.midX, y: marquee.midY)) else { return [] } + + // ONE recursive walk from that single root, so every candidate's depth is + // measured from the SAME origin (window = 0, its children = 1, …). Depth is + // the rule's tie-break between geometrically indistinguishable candidates; + // numbering assembled from several differently-rooted traversals would turn + // that tie-break into noise. + // + // The subtree is collected WHOLE — deliberately not pre-filtered to views + // intersecting the drawn frame. The rule's second pass needs the candidates + // whose frames CONTAIN the frame (the user drew INSIDE something), and an + // intersects-the-marquee filter is exactly what discards them. + var views: [UIView] = [] + var candidates: [MarqueeCandidate] = [] + Self.collectMarqueeCandidates(root, depth: 0, views: &views, candidates: &candidates) + + guard let resolution = MarqueeTargetRule.resolve(marquee: marquee, in: candidates) else { return [] } + let target = views[resolution.index] + + // The widening rungs are anchored at the TARGET's frame centre, not the + // drawn frame's: a sloppy marquee can spill outside the element it bound + // to, and a container that does not contain the target is not a component + // the user could widen to. Same value the point path passes. + let targetFrame = Self.screenFrame(of: target) + let targetCentre = CGPoint(x: targetFrame.midX, y: targetFrame.midY) + return [Self.element(for: target)] + + Self.enclosingComponents(of: target, containing: targetCentre).map { Self.element(for: $0) } + } + + /// The frontmost visible non-overlay window containing `point`, or nil when the + /// drag happened over nothing of ours. + /// + /// `UIWindowScene.windows` has no documented front-to-back order, so the + /// frontmost is derived rather than assumed: highest `windowLevel` first, ties + /// broken by the LATER array position, which is UIKit's own within-level + /// ordering. Getting this wrong on a host that presents an alert or + /// share-sheet window would walk the window BEHIND the one the user is looking + /// at and bind the note to an element they cannot see. + /// + /// AnnotKit's own overlay cannot be picked here because ``windows()`` never + /// returns it — see that helper for why the exclusion lives there and not in + /// this method. + private static func marqueeRoot(containing point: CGPoint) -> UIWindow? { + windows() + .filter { !$0.isHidden && $0.alpha > 0.01 } + .enumerated() + .sorted { lhs, rhs in + lhs.element.windowLevel.rawValue == rhs.element.windowLevel.rawValue + ? lhs.offset > rhs.offset + : lhs.element.windowLevel.rawValue > rhs.element.windowLevel.rawValue + } + .first { screenFrame(of: $0.element).contains(point) }? + .element + } + + /// Depth-first walk collecting a PARALLEL pair per view — the live `UIView` and + /// its pure ``MarqueeCandidate`` — so ``MarqueeTargetRule/Resolution/index`` + /// maps straight back to a live view. + /// + /// The candidate is built with the same ``candidate(for:)`` the point path + /// uses, so container-root classification — which is what the rule's + /// eligibility filter reads — is identical for a tap and a drag by + /// construction, not by two call sites agreeing today. + /// + /// Hidden and effectively transparent subtrees are skipped WHOLE: they keep + /// real frames, so a marquee would happily "surround" a view the user cannot + /// see, and being a large such frame it could win pass 1 outright. The point + /// path gets this filtering free from `UIView.hitTest`, which a whole-tree walk + /// never goes through. + private static func collectMarqueeCandidates( + _ view: UIView, + depth: Int, + views: inout [UIView], + candidates: inout [MarqueeCandidate] + ) { + views.append(view) + candidates.append( + MarqueeCandidate(element: candidate(for: view), frame: screenFrame(of: view), depth: depth) + ) + guard depth < maxDepth else { return } + for subview in view.subviews where !subview.isHidden && subview.alpha > 0.01 { + collectMarqueeCandidates(subview, depth: depth + 1, views: &views, candidates: &candidates) + } + } + + /// The identified components that geometrically ENCLOSE `target` at `point`, + /// smallest-first — the widening rungs above a bound target, mirroring the + /// macOS `enclosingComponents(of:containing:in:)` so a drag widens through the + /// same kind of components on both platforms. + /// + /// The scan is GEOMETRIC, not pure ancestry (DECISIONS.md → "Component + /// containment is GEOMETRIC"): a card's identified background surface is + /// routinely a SIBLING of the card's content rather than its ancestor, so it + /// never appears in an ancestor chain. Scanning each ancestor PLUS its direct + /// subviews reaches those surfaces without a second whole-tree walk. Deduped by + /// identifier because the same surface is reachable from several ancestors, and + /// the `>= targetArea` floor keeps a SMALLER identified sibling that merely + /// happens to cover the target's centre out of a ladder that is supposed to + /// only ever widen. + /// + /// Container roots are excluded, matching + /// ``AnnotationTargetRule/wideningLadder(in:)`` stopping at the window: the + /// window encloses everything, so it would be the top rung of every ladder + /// while naming no component an agent could act on. + private static func enclosingComponents(of target: UIView, containing point: CGPoint) -> [UIView] { + let targetFrame = screenFrame(of: target) + let targetArea = targetFrame.width * targetFrame.height + var containers: [(view: UIView, area: CGFloat)] = [] + var seen = Set() + for ancestor in ancestorChain(from: target) { + for view in [ancestor] + ancestor.subviews { + guard !(view is UIWindow), view !== target else { continue } + guard !view.isHidden, view.alpha > 0.01 else { continue } + let identifier = view.accessibilityIdentifier ?? "" + guard !identifier.isEmpty, !seen.contains(identifier) else { continue } + let frame = screenFrame(of: view) + let area = frame.width * frame.height + guard frame.width > 0, frame.height > 0, frame.contains(point), area >= targetArea else { continue } + seen.insert(identifier) + containers.append((view, area)) + } + } + containers.sort { $0.area < $1.area } + return containers.map(\.view) + } +} #endif From 66425a216eef663b0835bfe4f89dded6aa1f6f87 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:36:44 -0500 Subject: [PATCH 12/24] feat(overlay): explicit point/frame selection modes with a tool selector Frame drawing becomes a MODE the user picks rather than a threshold the gesture infers. The pill's annotate row gains a two-button tool segment (point, frame) with the active one lit, split from the note actions by a hairline divider. Strict separation: in frame mode a plain click does nothing, and in point mode no press can produce a frame -- removing the implicit design's real hazard, where a jittery click crossed the travel threshold and silently planted a framed note. The threshold survives with a narrower job: telling a real drag from a wobble INSIDE frame mode. MarqueeDrag becomes SelectionGesture: it now owns the press->outcome decision for both tools, so the old name described half of it. Co-Authored-By: Claude Opus 5 (1M context) --- .../AnnotKit/Overlay/AnnotationSession.swift | 29 +++ Sources/AnnotKit/Overlay/MarqueeDrag.swift | 67 ------ Sources/AnnotKit/Overlay/OverlayView.swift | 154 +++++++++---- Sources/AnnotKit/Overlay/PillStyle.swift | 45 +++- .../AnnotKit/Overlay/SelectionGesture.swift | 109 +++++++++ .../AnnotationSessionTests.swift | 41 ++++ Tests/AnnotKitTests/LucideIconTests.swift | 2 + Tests/AnnotKitTests/MarqueeDragTests.swift | 110 ---------- .../AnnotKitTests/SelectionGestureTests.swift | 207 ++++++++++++++++++ 9 files changed, 540 insertions(+), 224 deletions(-) delete mode 100644 Sources/AnnotKit/Overlay/MarqueeDrag.swift create mode 100644 Sources/AnnotKit/Overlay/SelectionGesture.swift delete mode 100644 Tests/AnnotKitTests/MarqueeDragTests.swift create mode 100644 Tests/AnnotKitTests/SelectionGestureTests.swift diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index 43c19b4..17aa435 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -16,7 +16,28 @@ public final class AnnotationSession: ObservableObject { case annotating } + /// Which gesture the catcher interprets: click a point, or draw a frame around + /// what you mean. Explicit rather than inferred from how far a press travelled — + /// an implicit branch means the mode you get depends on how steady your hand was, + /// so a jittery click silently plants a FRAMED note bound to whatever container + /// the pointer sat in. Making it a chosen tool means the user always knows which + /// outcome a press can produce before they make it. + /// + /// Nested here (rather than beside ``SelectionGesture``) because it is session + /// state the UI binds to; `nonisolated` so the pure ``SelectionGesture/resolve`` + /// can switch on it without hopping to the main actor. + public nonisolated enum SelectionTool: Sendable, Hashable { + case point + case frame + } + @Published public private(set) var mode: Mode = .idle + /// The active selection tool. A user PREFERENCE, not per-session state: + /// ``stop()``, ``clear()`` and capturing a note deliberately leave it alone, so + /// a user who chose frame mode does not silently get dropped back to clicking + /// after every note. ``point`` is the default so existing muscle memory (click + /// the thing, type the note) is untouched for anyone who never opens the picker. + @Published public private(set) var tool: SelectionTool = .point /// The retained set of captured notes. Grows via ``addNote(comment:selectedText:screenshot:)`` /// and is emptied ONLY by ``clear()`` — ``export()`` and copy read it without /// mutating it, so the same set survives repeated copy/export. @@ -99,6 +120,14 @@ public final class AnnotationSession: ObservableObject { public func start() { mode = .annotating } + /// Switch which gesture the catcher interprets. Deliberately touches NOTHING + /// else: an open composer, the current selection and the retained notes all + /// survive, because changing how you will pick the NEXT target says nothing + /// about the note you are in the middle of writing. Discarding a half-typed + /// comment because the user reached for the other tool would be the kind of + /// data loss nobody reports — they just stop using the picker. + public func setTool(_ tool: SelectionTool) { self.tool = tool } + public func stop() { mode = .idle hovered = nil diff --git a/Sources/AnnotKit/Overlay/MarqueeDrag.swift b/Sources/AnnotKit/Overlay/MarqueeDrag.swift deleted file mode 100644 index 7f2e181..0000000 --- a/Sources/AnnotKit/Overlay/MarqueeDrag.swift +++ /dev/null @@ -1,67 +0,0 @@ -import CoreGraphics - -/// The pure part of the marquee gesture: given where a press began and where the -/// pointer is now, decide whether it is a FRAME or a CLICK and produce the -/// geometry for whichever it is. -/// -/// This lives outside the SwiftUI closure for the same reason ``ComposerPlacement`` -/// does — a decision buried in a gesture callback can only be checked by a human -/// with a mouse, and this particular decision is the one that silently breaks the -/// whole mode when it is wrong. -/// -/// Why the threshold is the load-bearing part: -/// -/// * A press that never moved is a CLICK and must go to -/// ``AnnotationSession/select(atAXPoint:)``. ``AnnotationSession/select(inAXRect:)`` -/// returns nil for a zero-area rect (deliberately — see its CALLER CONTRACT), so -/// routing a click into the marquee path makes clicking do NOTHING in annotate -/// mode. That reads as a dead feature, not as a missing constant. -/// * A 3-point jitter-drag is the worse failure, because it is NOT zero-area: the -/// rule resolves it happily and lands on whatever container the pointer happened -/// to sit in, producing a plausible-looking note bound to something the user -/// never framed. A wrong note is more expensive than no note, because nobody -/// goes back to check it. -/// -/// Travel is measured as the hypotenuse, not per-axis: a 5-point-right, -/// 5-point-down drift is ~7 points of real movement and reads as intentional, while -/// a per-axis test would call it a click on both axes. -/// -/// Touch needs more slop than a cursor because a finger rolls several points on a -/// deliberate tap — a mouse-tuned threshold on iOS turns ordinary taps into -/// accidental marquees. -enum MarqueeDrag { - /// Travel a press must EXCEED before it counts as a frame rather than a click. - /// Exactly at the threshold is still a click: the boundary belongs to the - /// safer branch, since a click misread as a frame plants a wrong note whereas - /// a frame misread as a click just selects what is under the press. - #if os(iOS) - static let minimumTravel: CGFloat = 10 - #else - static let minimumTravel: CGFloat = 6 - #endif - - /// True when the press travelled far enough to be a deliberate frame. - static func isFrame(from start: CGPoint, to end: CGPoint) -> Bool { - let dx = end.x - start.x - let dy = end.y - start.y - return (dx * dx + dy * dy).squareRoot() > minimumTravel - } - - /// Window-local rect, normalized so any drag direction gives the same result. - /// Users drag up-left as readily as down-right; without `standardized` the - /// three "backwards" directions produce negative extents, which the session's - /// degenerate guard would treat as a non-drag. - static func localRect(from start: CGPoint, to end: CGPoint) -> CGRect { - CGRect(x: start.x, y: start.y, width: end.x - start.x, height: end.y - start.y) - .standardized - } - - /// AX screen rect: the window-local rect shifted by the surface's AX origin. - /// Gesture coordinates are window-local, and the AX queries behind - /// ``AnnotationSession/select(inAXRect:)`` are in AX screen space — the same - /// ADD-to-query / SUBTRACT-to-draw transform the click path and the highlight - /// share, which is why all of them agree on a secondary display. - static func axRect(from start: CGPoint, to end: CGPoint, axOrigin: CGPoint) -> CGRect { - localRect(from: start, to: end).offsetBy(dx: axOrigin.x, dy: axOrigin.y) - } -} diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index ccea1b8..3bf398d 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -4,9 +4,10 @@ import SwiftUI /// /// Interaction is driven entirely through SwiftUI hit-testing, which fixes the /// two ways a global click monitor went wrong: a full-screen catcher (active -/// only in annotate mode, behind the chrome) receives hover, taps, and marquee -/// drags over the app (see ``MarqueeDrag`` for the click-vs-frame branch), while -/// the toolbar and composer sit on top and consume their own clicks, +/// only in annotate mode, behind the chrome) receives hover, taps, and frame +/// drags over the app (see ``SelectionGesture`` for the point-vs-frame routing, +/// which follows the toolbar's chosen tool rather than guessing from travel), +/// while the toolbar and composer sit on top and consume their own clicks, /// so tapping "Add note" can never re-select the element under the button. The /// whole overlay is `accessibilityHidden` so the AX point query sees through it /// to the app beneath. @@ -45,9 +46,10 @@ struct OverlayView: View { /// editing begins (see the `editingNoteID` seeding hook on the ZStack). @State private var editDraft: String = "" /// The in-progress marquee band, WINDOW-LOCAL (gesture coordinates), non-nil - /// only between "this press has travelled far enough to be a frame" and the - /// release that resolves it. It is never the resolved selection — that comes - /// back as `session.selected` and is drawn by the highlight branch. + /// only in FRAME mode, between "this press has travelled far enough to be a + /// real drag" and the release that resolves it. It is never the resolved + /// selection — that comes back as `session.selected` and is drawn by the + /// highlight branch. @State private var marqueeRect: CGRect? var body: some View { @@ -166,43 +168,61 @@ struct OverlayView: View { session.clearHover() } } - // ONE gesture handles both clicks and frames, branching on travel at - // release. It is deliberately NOT a DragGesture composed with the - // old SpatialTapGesture: `.exclusively`/`.simultaneously` make the + // ONE gesture serves BOTH tools, branching on `session.tool` at + // release. It is deliberately NOT a DragGesture composed with a + // SpatialTapGesture: `.exclusively`/`.simultaneously` make the // recognizers negotiate, and the case that loses that negotiation is // the plain click — which then does nothing at all in annotate mode. - // A single recognizer with a distance branch has no ambiguity to - // lose a click in. `minimumDistance: 0` is what lets it also see the - // press that never moved. + // A single recognizer that asks ``SelectionGesture`` what a press + // meant has no ambiguity to lose a click in. `minimumDistance: 0` is + // what lets it also see the press that never moved. .gesture( DragGesture(minimumDistance: 0) .onChanged { value in - // Below the threshold nothing is drawn: a band that - // flickered up on every click would advertise a marquee - // the release is going to route as a click. - guard MarqueeDrag.isFrame(from: value.startLocation, to: value.location) else { return } - marqueeRect = MarqueeDrag.localRect(from: value.startLocation, to: value.location) + // The band is FRAME-MODE-ONLY chrome. Drawing it in point + // mode would promise a rectangle the release is never + // going to honour — the user would let go expecting what + // they swept and get the element they first pressed on. + guard session.tool == .frame, + SelectionGesture.travelledFarEnough(from: value.startLocation, to: value.location) + else { return } + marqueeRect = SelectionGesture.localRect(from: value.startLocation, to: value.location) session.clearHover() } .onEnded { value in marqueeRect = nil - if MarqueeDrag.isFrame(from: value.startLocation, to: value.location) { - session.select(inAXRect: MarqueeDrag.axRect( - from: value.startLocation, - to: value.location, - axOrigin: axOrigin - )) - } else { - // `startLocation`, not `location`: it is where the - // user AIMED, and it is stable for a press that - // drifted a point or two under the finger/cursor. - session.select(atAXPoint: CGPoint( - x: value.startLocation.x + axOrigin.x, - y: value.startLocation.y + axOrigin.y - )) + switch SelectionGesture.resolve( + tool: session.tool, + from: value.startLocation, + to: value.location, + axOrigin: axOrigin + ) { + case .point(let point): + session.select(atAXPoint: point) + case .frame(let rect): + session.select(inAXRect: rect) + case .none: + // A too-short press in frame mode. Do NOTHING — + // explicitly not `cancelSelection()`: this fires for + // every stray click on the catcher, including the + // ones a user makes while a composer is open, and + // clearing there would discard a half-typed comment. + break } } ) + #if os(macOS) + // The pointer is what stops frame mode reading as broken. A click + // does nothing there by design, so the cursor has to say "drag here" + // BEFORE the press, not after it fails to do anything. + // + // `.rectSelection` rather than a generic crosshair: it is the system + // pointer for "drag out a rectangular selection", which is exactly + // this gesture, so the affordance is one the user has already learned + // elsewhere. (SwiftUI's `PointerStyle` has no `.crosshair` member — + // reaching for one does not compile.) + .pointerStyle(session.tool == .frame ? .rectSelection : nil) + #endif } else { Color.clear.allowsHitTesting(false) } @@ -488,10 +508,20 @@ private struct AnnotationCard: View { /// (no settings model, no preview capability here) are deliberately omitted. /// /// Idle is JUST the pencil, so the resting affordance is one unambiguous "start -/// annotating". Annotate mode replaces it outright with the working set, left to -/// right: two DISTINCT persist actions (Copy to the clipboard as markdown, and -/// Export to `AGENTATION_NOTES.md`), a destructive clear, and the X that leaves -/// the mode. The X sits FAR RIGHT because that is where a "close this mode" +/// annotating". Annotate mode replaces it outright with the working set, in two +/// groups split by a hairline divider. +/// +/// LEFT of the divider: the selection-tool segment — point (click the thing) and +/// frame (draw a rectangle around it) — with the active one lit. It leads the row +/// because it governs what every subsequent press on the catcher DOES, and because +/// frame mode's "a click does nothing" contract is only honest if the mode is +/// visible somewhere. These two are never disabled: they are how you get out of a +/// mode, so gating them could strand a user in one. +/// +/// RIGHT of the divider: what you then do with the notes, left to right — two +/// DISTINCT persist actions (Copy to the clipboard as markdown, and Export to +/// `AGENTATION_NOTES.md`), a destructive clear, and the X that leaves the mode. +/// The X sits FAR RIGHT because that is where a "close this mode" /// control is looked for, and it is drawn as a plain action rather than a lit-up /// toggle — the expanded pill and the live catcher already say annotate mode is /// on, so a permanently bright glyph would only add noise. The three note actions @@ -500,8 +530,8 @@ private struct AnnotationCard: View { /// overlays the pill whenever notes exist. Copy and Export never clear the /// retained set (only Clear does), so the same notes can be both copied and /// exported. Copy/Export flow through host callbacks (they need a sink); the mode -/// control needs the controller (activation); clear reads/writes the session -/// directly. +/// control needs the controller (activation); clear and the tool segment +/// read/write the session directly. /// /// The pill itself is rendered unconditionally and is NEVER gated on an entrance /// flag, so it stays visible across idle<->annotate; only its contents swap as @@ -522,9 +552,33 @@ private struct ToolbarView: View { HStack(spacing: 2) { // The two modes share no controls, so they are two whole rows rather // than one row with conditional members: idle is the pencil alone, - // annotate is the note actions (dimmed/disabled with no notes to act - // on) closed by the exit X. + // annotate is the tool segment, a divider, then the note actions + // (dimmed/disabled with no notes to act on) closed by the exit X. if annotating { + // Selection-tool segment. NOT gated on `hasNotes` — unlike every + // control to its right, these change how the NEXT press behaves, so + // they must stay live in an empty session, which is exactly when a + // user is choosing how to make their first selection. + PillButton( + icon: .mousePointer, + isActive: session.tool == .point, + tooltip: "Select by clicking", + action: { session.setTool(.point) } + ) + PillButton( + icon: .squareDashed, + isActive: session.tool == .frame, + tooltip: "Select by drawing a frame", + action: { session.setTool(.frame) } + ) + // Groups "how you select" apart from "what you do with the notes". + // Without it the six glyphs read as one undifferentiated strip and + // the two stateful buttons look like two more one-shot actions. + Rectangle() + .fill(PillStyle.divider) + .frame(width: 1, height: 16) + .padding(.horizontal, 3) + .allowsHitTesting(false) PillButton( icon: justCopied ? .check : .copy, isDisabled: !hasNotes, @@ -551,8 +605,10 @@ private struct ToolbarView: View { } // Idle shows ONE 28pt button, so the inset must be EVEN (8pt all around -> // a concentric 44x44 capsule hugging the hover wash). The 6pt horizontal - // inset is for the annotate-mode 4-button ROW only, where the buttons' - // own spacing makes the tighter ends read as balanced. + // inset is for the annotate-mode ROW only — now SIX buttons plus a divider, + // so the row is wide enough that trimming 2pt off each end reads as balanced + // rather than cramped, and the saved width keeps the pill off the window + // edge it is anchored 20pt from. .padding(.horizontal, annotating ? 6 : 8) .padding(.vertical, 8) // 28pt buttons + 8*2 -> 44pt pill height .background( @@ -611,6 +667,13 @@ private struct PillButton: View { /// Dims the glyph and makes the button a true no-op (used by copy/export/clear /// while there are no notes to act on). var isDisabled: Bool = false + /// Lights the glyph to full white for the SELECTED member of a segmented + /// control. Back after being removed as dead code: the selection-tool pair is + /// the pill's only control carrying PERSISTENT state — every other button is a + /// one-shot action, which is why hover was briefly the only state that existed. + /// A segment with no lit member is worse than no segment at all, because the + /// user cannot tell whether a click will select a point or do nothing. + var isActive: Bool = false var glyphTint: Color? = nil let tooltip: String let action: () -> Void @@ -623,12 +686,17 @@ private struct PillButton: View { // treatment reads as unavailable. if isDisabled { return PillStyle.iconIdle.opacity(0.4) } if let glyphTint { return glyphTint } + // Active outranks hover: hovering the ALREADY-active tool must not dim it + // toward the inactive treatment, which would read as "clicking this turns + // it off" for a segment that has no off. + if isActive { return PillStyle.iconActive } if isDestructive && hovering { return .white } return hovering ? PillStyle.iconHover : PillStyle.iconIdle } - // Hover is the ONLY fill state: every control in the pill is a one-shot - // action, so no button ever wears a persistent chip. + // Hover is the ONLY fill state, even for the active tool: the segment is + // distinguished by glyph BRIGHTNESS alone, so a persistent circle behind the + // active tool cannot be mistaken for the hover wash sitting on a neighbour. private var fillColor: Color { // No hover wash while disabled — the button must look inert. if hovering && !isDisabled { return isDestructive ? PillStyle.destructive : PillStyle.hoverBackground } diff --git a/Sources/AnnotKit/Overlay/PillStyle.swift b/Sources/AnnotKit/Overlay/PillStyle.swift index f07b44d..0b6fa0d 100644 --- a/Sources/AnnotKit/Overlay/PillStyle.swift +++ b/Sources/AnnotKit/Overlay/PillStyle.swift @@ -28,8 +28,9 @@ extension Color { /// sibling tools read the same: an opaque `#1A1A1A` container (the AnnotKit /// overlay is transparent, so material would show desktop through it), hairline /// white borders, low-opacity white glyphs that brighten on hover, and a red -/// destructive hover. The toggle-active and count-badge fills reuse -/// `Color.accentColor` to stay consistent with the highlight stroke. +/// destructive hover. The count-badge fill reuses `Color.accentColor` to stay +/// consistent with the highlight stroke; the active selection tool is marked with +/// a full-white glyph instead, so the accent keeps meaning "notes exist". enum PillStyle { static let background = Color(hex: "1A1A1A") static let border = Color.white.opacity(0.08) @@ -38,13 +39,22 @@ enum PillStyle { static let hoverBackground = Color.white.opacity(0.1) static let destructive = Color(hex: "EF4444") static let success = Color(hex: "22C55E") + /// The glyph of the ACTIVE tool in the selection-tool segment. Full white, not + /// an accent chip: the pill's only other lit state is the count badge, and a + /// second accent-colored thing in the row would read as another notification + /// rather than as "this tool is armed". + static let iconActive = Color.white + /// Hairline rule separating the tool segment from the note actions. The same + /// white-on-dark weight as ``border`` but a touch stronger, so it reads as a + /// deliberate group boundary at 1pt instead of disappearing into the capsule. + static let divider = Color.white.opacity(0.1) } // MARK: - Lucide icon model /// A primitive on Lucide's 24x24 design grid. Modeling each glyph as a small -/// union of primitives (instead of shipping a general SVG renderer for six static -/// icons) keeps them offline, dependency-free, and unit-testable — the +/// union of primitives (instead of shipping a general SVG renderer for a handful +/// of static icons) keeps them offline, dependency-free, and unit-testable — the /// no-speculative-abstraction rule from CLAUDE.md. `path` backs the few glyphs /// that need real curves, parsed from an SVG `d` string. enum IconPart { @@ -101,6 +111,33 @@ struct LucideIcon { .path("M18 6 6 18"), .path("M6 6l12 12"), ]) + + /// Lucide `mouse-pointer-2` — the POINT tool: select by clicking. The real + /// Lucide `d` string; its two tiny corner arcs (`a`) are approximated by the + /// parser as a line to the arc endpoint, which is invisible at 16pt on a shape + /// this angular. + static let mousePointer = LucideIcon(parts: [ + .path("M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"), + ]) + + /// Lucide `square-dashed` — the FRAME tool: select by drawing a frame. Twelve + /// short strokes rather than one outline, and that is the point: the gaps echo + /// the dashed rubber band the tool draws, so the button previews its own + /// gesture instead of reading as a generic square. + static let squareDashed = LucideIcon(parts: [ + .path("M5 3a2 2 0 0 0-2 2"), + .path("M19 3a2 2 0 0 1 2 2"), + .path("M21 19a2 2 0 0 1-2 2"), + .path("M5 21a2 2 0 0 1-2-2"), + .path("M9 3h1"), + .path("M9 21h1"), + .path("M14 3h1"), + .path("M14 21h1"), + .path("M3 9v1"), + .path("M21 9v1"), + .path("M3 14v1"), + .path("M21 14v1"), + ]) } // MARK: - Shape diff --git a/Sources/AnnotKit/Overlay/SelectionGesture.swift b/Sources/AnnotKit/Overlay/SelectionGesture.swift new file mode 100644 index 0000000..2f9021b --- /dev/null +++ b/Sources/AnnotKit/Overlay/SelectionGesture.swift @@ -0,0 +1,109 @@ +import CoreGraphics + +/// The pure part of the catcher's press→outcome decision: given the ACTIVE TOOL, +/// where a press began and where it ended, decide what the release resolves to and +/// produce the geometry for it. +/// +/// This lives outside the SwiftUI closure for the same reason ``ComposerPlacement`` +/// does — a decision buried in a gesture callback can only be checked by a human +/// with a mouse, and this particular decision is the one that silently breaks the +/// whole mode when it is wrong. +/// +/// The tool, NOT the travel distance, picks the branch. An earlier design inferred +/// it: a short press was a click and a long one a frame, from the same gesture. That +/// made the outcome depend on how steady the user's hand was, and the failure was +/// silent — a 12-point wobble on a click planted a framed note bound to whatever +/// container happened to be under the sweep. Travel now only answers a much smaller +/// question, asked ONLY inside frame mode: was this a real drag? +/// +/// The two modes are strictly separated: +/// +/// * **Point mode always yields a point**, no matter how far the press travelled. +/// A drag in point mode is a sloppy click, not a frame; there is no press that +/// does nothing in the default mode, because a dead click reads as a broken tool. +/// * **Frame mode below the threshold yields NOTHING.** That is the user's explicit +/// choice — the tool you picked is the outcome you get — and it also absorbs the +/// jitter case for free. A 3-point wobble is not zero-area, so nothing downstream +/// rejects it: ``AnnotationSession/select(inAXRect:)`` would resolve it happily +/// onto whatever container the pointer sat in and produce a plausible-looking note +/// bound to something the user never framed. A wrong note is more expensive than +/// no note, because nobody goes back to check it. In frame mode the crosshair +/// cursor is what tells the user a click alone will not do anything. +/// +/// Travel is measured as the hypotenuse, not per-axis: a 5-point-right, +/// 5-point-down drift is ~7 points of real movement and reads as intentional, while +/// a per-axis test would call it a click on both axes. +/// +/// Touch needs more slop than a cursor because a finger rolls several points on a +/// deliberate tap — a mouse-tuned threshold on iOS would make ordinary frame-mode +/// taps register as tiny accidental frames. +enum SelectionGesture { + /// What a completed press resolves to. All coordinates are AX SCREEN space + /// (window-local gesture coordinates already shifted by `axOrigin`), so the + /// caller hands the payload straight to the session with no further transform + /// — the transform living in one place is why the click path, the frame path + /// and the highlight agree on a secondary display. + enum Outcome: Equatable { + case point(CGPoint) + case frame(CGRect) + case none + } + + /// Travel a press must EXCEED, IN FRAME MODE ONLY, before it counts as a real + /// drag rather than a stray click. Exactly at the threshold is NOT a frame: the + /// boundary belongs to the branch that does nothing, since a doubtful frame + /// plants a note nobody asked for whereas a rejected one costs a second drag. + #if os(iOS) + static let minimumTravel: CGFloat = 10 + #else + static let minimumTravel: CGFloat = 6 + #endif + + /// Route a completed press. The one place the mode semantics live, so they are + /// pinned by tests rather than by a human dragging a mouse. + static func resolve( + tool: AnnotationSession.SelectionTool, + from start: CGPoint, + to end: CGPoint, + axOrigin: CGPoint + ) -> Outcome { + switch tool { + case .point: + // `startLocation`, not `location`: it is where the user AIMED, and it is + // deterministic for a press that drifted under the finger/cursor — the + // release point of a sloppy click can easily sit on the neighbouring + // control. + return .point(CGPoint(x: start.x + axOrigin.x, y: start.y + axOrigin.y)) + case .frame: + guard travelledFarEnough(from: start, to: end) else { return .none } + return .frame(axRect(from: start, to: end, axOrigin: axOrigin)) + } + } + + /// True when the press travelled far enough, in frame mode, to be a deliberate + /// drag. Also gates the drawn band, so the rubber band never appears for a press + /// the release is going to discard. + static func travelledFarEnough(from start: CGPoint, to end: CGPoint) -> Bool { + let dx = end.x - start.x + let dy = end.y - start.y + return (dx * dx + dy * dy).squareRoot() > minimumTravel + } + + /// Window-local rect, normalized so any drag direction gives the same result. + /// Users drag up-left as readily as down-right; without `standardized` the + /// three "backwards" directions produce negative extents, which the session's + /// degenerate guard would treat as a non-drag. + static func localRect(from start: CGPoint, to end: CGPoint) -> CGRect { + CGRect(x: start.x, y: start.y, width: end.x - start.x, height: end.y - start.y) + .standardized + } + + /// AX screen rect: the window-local rect shifted by the surface's AX origin. + /// Gesture coordinates are window-local, and the AX queries behind + /// ``AnnotationSession/select(inAXRect:)`` are in AX screen space — the same + /// ADD-to-query / SUBTRACT-to-draw transform the click path and the highlight + /// share, which is why all of them agree on a secondary display. + static func axRect(from start: CGPoint, to end: CGPoint, axOrigin: CGPoint) -> CGRect { + localRect(from: start, to: end).offsetBy(dx: axOrigin.x, dy: axOrigin.y) + } +} diff --git a/Tests/AnnotKitTests/AnnotationSessionTests.swift b/Tests/AnnotKitTests/AnnotationSessionTests.swift index 7ebe6e5..e269d20 100644 --- a/Tests/AnnotKitTests/AnnotationSessionTests.swift +++ b/Tests/AnnotKitTests/AnnotationSessionTests.swift @@ -385,6 +385,47 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertEqual(session.select(inAXRect: drawn)?.id, "Card") } + // MARK: - Selection tool + + /// Point selection is the default, so a host that never touches the tool keeps + /// the click-to-annotate behaviour that predates frame selection. + func testSelectionToolDefaultsToPoint() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + XCTAssertEqual(session.tool, .point) + } + + func testSetToolSwitchesBetweenPointAndFrame() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.setTool(.frame) + XCTAssertEqual(session.tool, .frame) + session.setTool(.point) + XCTAssertEqual(session.tool, .point) + } + + /// The tool is a PREFERENCE, not per-selection state: it must survive leaving + /// annotate mode, clearing the notes, and capturing. Resetting it on any of + /// these would silently drop the user back to point selection mid-session — + /// and the next drag would then do nothing at all, since a drag in point mode + /// is treated as a click. + func testSelectionToolSurvivesStopClearAndCapture() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + session.setTool(.frame) + + session.select(atAXPoint: .zero) + session.addNote(comment: "captured") + XCTAssertEqual(session.tool, .frame, "capturing a note must not reset the tool") + + session.clear() + XCTAssertEqual(session.tool, .frame, "clearing the notes must not reset the tool") + + session.stop() + XCTAssertEqual(session.tool, .frame, "leaving annotate mode must not reset the tool") + + session.start() + XCTAssertEqual(session.tool, .frame, "re-entering annotate mode keeps the chosen tool") + } + func testClearHoverDropsHighlightButKeepsSelection() { let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) session.start() diff --git a/Tests/AnnotKitTests/LucideIconTests.swift b/Tests/AnnotKitTests/LucideIconTests.swift index d36adad..0845cf0 100644 --- a/Tests/AnnotKitTests/LucideIconTests.swift +++ b/Tests/AnnotKitTests/LucideIconTests.swift @@ -17,6 +17,8 @@ final class LucideIconTests: XCTestCase { ("download", .download), ("trash", .trash), ("close", .close), + ("mousePointer", .mousePointer), + ("squareDashed", .squareDashed), ] func testEachIconRendersNonEmptyPath() { diff --git a/Tests/AnnotKitTests/MarqueeDragTests.swift b/Tests/AnnotKitTests/MarqueeDragTests.swift deleted file mode 100644 index c05dd83..0000000 --- a/Tests/AnnotKitTests/MarqueeDragTests.swift +++ /dev/null @@ -1,110 +0,0 @@ -import CoreGraphics -import XCTest -@testable import AnnotKit - -final class MarqueeDragTests: XCTestCase { - private let start = CGPoint(x: 100, y: 100) - - func testZeroTravelPressIsAClick() { - // The regression this threshold exists for: `select(inAXRect:)` returns nil - // for a zero-area rect, so a press routed into the marquee path makes - // clicking do nothing at all in annotate mode. - XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: start)) - XCTAssertEqual(MarqueeDrag.localRect(from: start, to: start), CGRect(x: 100, y: 100, width: 0, height: 0)) - } - - func testTravelJustBelowThresholdIsAClick() { - // A jitter-drag is NOT zero-area, so nothing downstream rejects it: it would - // resolve to whatever container the pointer sat in and plant a note the user - // never framed. - let below = CGPoint(x: start.x + MarqueeDrag.minimumTravel - 0.5, y: start.y) - XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: below)) - } - - func testTravelExactlyAtThresholdIsStillAClick() { - // The boundary belongs to the safer branch: a click misread as a frame - // plants a wrong note, a frame misread as a click just selects the press. - let exact = CGPoint(x: start.x + MarqueeDrag.minimumTravel, y: start.y) - XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: exact)) - } - - func testTravelJustAboveThresholdIsAFrame() { - let above = CGPoint(x: start.x + MarqueeDrag.minimumTravel + 0.5, y: start.y) - XCTAssertTrue(MarqueeDrag.isFrame(from: start, to: above)) - } - - func testDiagonalTravelIsMeasuredAsHypotenuseNotPerAxis() { - // Each axis alone is under the threshold; together they are over it. A - // per-axis test would call this deliberate diagonal drag a click. - let leg = MarqueeDrag.minimumTravel * 0.8 // hypotenuse = 1.131 * threshold - let diagonal = CGPoint(x: start.x + leg, y: start.y + leg) - XCTAssertLessThan(leg, MarqueeDrag.minimumTravel) - XCTAssertTrue(MarqueeDrag.isFrame(from: start, to: diagonal)) - - // And the converse: a diagonal whose hypotenuse is under the threshold is a - // click even though it moved on both axes. - let short = MarqueeDrag.minimumTravel * 0.5 // hypotenuse = 0.707 * threshold - XCTAssertFalse(MarqueeDrag.isFrame(from: start, to: CGPoint(x: start.x + short, y: start.y + short))) - } - - func testAllFourDragDirectionsNormalizeToTheSameRect() { - // Users drag up-left as readily as down-right. Without normalization the - // three "backwards" directions arrive with negative extents, which the - // session's degenerate guard treats as a non-drag — the marquee would work - // in exactly one direction. - let expected = CGRect(x: 100, y: 100, width: 80, height: 60) - let a = CGPoint(x: 100, y: 100), b = CGPoint(x: 180, y: 160) - XCTAssertEqual(MarqueeDrag.localRect(from: a, to: b), expected) // down-right - XCTAssertEqual(MarqueeDrag.localRect(from: b, to: a), expected) // up-left - XCTAssertEqual(MarqueeDrag.localRect(from: CGPoint(x: 180, y: 100), to: CGPoint(x: 100, y: 160)), expected) // down-left - XCTAssertEqual(MarqueeDrag.localRect(from: CGPoint(x: 100, y: 160), to: CGPoint(x: 180, y: 100)), expected) // up-right - } - - func testAXRectIsWindowLocalRectAtOriginOnPrimaryDisplay() { - // `axOrigin == .zero` on the primary display at the global origin, so the - // AX rect and the drawn band coincide there — which is why an origin bug - // hides until someone uses a second display. - XCTAssertEqual( - MarqueeDrag.axRect(from: CGPoint(x: 100, y: 100), to: CGPoint(x: 180, y: 160), axOrigin: .zero), - CGRect(x: 100, y: 100, width: 80, height: 60) - ) - } - - func testAXRectAppliesANonZeroOriginExactly() { - // Secondary-display case: gesture coordinates are window-local, the AX - // queries are in screen space, so the origin is ADDED (the mirror of the - // highlight's subtraction). - XCTAssertEqual( - MarqueeDrag.axRect( - from: CGPoint(x: 100, y: 100), - to: CGPoint(x: 180, y: 160), - axOrigin: CGPoint(x: 1512, y: 30) - ), - CGRect(x: 1612, y: 130, width: 80, height: 60) - ) - } - - func testAXRectNormalizesBeforeShiftingSoABackwardsDragIsStillOnScreen() { - // A up-left drag shifted by the origin must land at the frame's top-left, - // not at the press point with negative extents (which would resolve to - // nothing at all). - XCTAssertEqual( - MarqueeDrag.axRect( - from: CGPoint(x: 180, y: 160), - to: CGPoint(x: 100, y: 100), - axOrigin: CGPoint(x: 1512, y: 30) - ), - CGRect(x: 1612, y: 130, width: 80, height: 60) - ) - } - - func testThresholdIsLargerOnTouchThanCursor() { - // A finger rolls several points on a deliberate tap, so a mouse-tuned - // threshold on iOS turns ordinary taps into accidental marquees. - #if os(iOS) - XCTAssertEqual(MarqueeDrag.minimumTravel, 10) - #else - XCTAssertEqual(MarqueeDrag.minimumTravel, 6) - #endif - } -} diff --git a/Tests/AnnotKitTests/SelectionGestureTests.swift b/Tests/AnnotKitTests/SelectionGestureTests.swift new file mode 100644 index 0000000..ea60032 --- /dev/null +++ b/Tests/AnnotKitTests/SelectionGestureTests.swift @@ -0,0 +1,207 @@ +import CoreGraphics +import XCTest +@testable import AnnotKit + +final class SelectionGestureTests: XCTestCase { + private let start = CGPoint(x: 100, y: 100) + + // MARK: - Tool routing + + func testPointToolResolvesAZeroTravelPressToTheStartPoint() { + // The default tool's contract: a press always selects something. Routing it + // anywhere else makes clicking do nothing at all in annotate mode, which + // reads as a broken feature rather than a mode. + XCTAssertEqual( + SelectionGesture.resolve(tool: .point, from: start, to: start, axOrigin: .zero), + .point(CGPoint(x: 100, y: 100)) + ) + } + + func testPointToolResolvesEvenALargeDragToTheStartPointNotAFrame() { + // THE regression this whole feature exists to prevent: a long press-drag in + // point mode must never become a frame. The old implicit design branched on + // travel alone, so a user who dragged while clicking silently got a framed + // note bound to whatever the sweep covered. + let far = CGPoint(x: start.x + 400, y: start.y + 300) + XCTAssertEqual( + SelectionGesture.resolve(tool: .point, from: start, to: far, axOrigin: .zero), + .point(CGPoint(x: 100, y: 100)) // the PRESS point, not the release point + ) + } + + func testPointToolAppliesTheAXOriginToTheStartPoint() { + // Secondary-display case: the gesture is window-local, the AX query is in + // screen space, so the origin is ADDED — the same transform the frame path + // and the highlight use. + XCTAssertEqual( + SelectionGesture.resolve( + tool: .point, + from: start, + to: CGPoint(x: 180, y: 160), + axOrigin: CGPoint(x: 1512, y: 30) + ), + .point(CGPoint(x: 1612, y: 130)) + ) + } + + func testFrameToolResolvesAZeroTravelPressToNothing() { + // Strict separation: in frame mode a plain click does NOT fall back to point + // selection. It also could not usefully: `select(inAXRect:)` returns nil for + // a zero-area rect, so there is nothing to resolve either way. + XCTAssertEqual( + SelectionGesture.resolve(tool: .frame, from: start, to: start, axOrigin: .zero), + .none + ) + } + + func testFrameToolResolvesABelowThresholdJitterToNothing() { + // The dangerous case, and why the threshold survived the redesign: a 3-point + // wobble is NOT zero-area, so nothing downstream rejects it. It would resolve + // to whatever container the pointer sat in and plant a plausible-looking note + // the user never framed — worse than no note, because nobody re-checks it. + let below = CGPoint(x: start.x + SelectionGesture.minimumTravel - 0.5, y: start.y) + XCTAssertEqual(SelectionGesture.resolve(tool: .frame, from: start, to: below, axOrigin: .zero), .none) + } + + func testFrameToolResolvesExactlyTheThresholdToNothing() { + // The boundary belongs to the branch that does NOTHING. Between "a doubtful + // press plants a note" and "a doubtful press costs a second drag", the second + // is recoverable and the first is not — and the crosshair already tells the + // user this mode wants a real drag. + let exact = CGPoint(x: start.x + SelectionGesture.minimumTravel, y: start.y) + XCTAssertEqual(SelectionGesture.resolve(tool: .frame, from: start, to: exact, axOrigin: .zero), .none) + } + + func testFrameToolResolvesARealDragToTheAXRect() { + // Non-zero origin included deliberately: with `axOrigin == .zero` (the + // primary display at the global origin) a missing transform is invisible. + XCTAssertEqual( + SelectionGesture.resolve( + tool: .frame, + from: CGPoint(x: 100, y: 100), + to: CGPoint(x: 180, y: 160), + axOrigin: CGPoint(x: 1512, y: 30) + ), + .frame(CGRect(x: 1612, y: 130, width: 80, height: 60)) + ) + } + + func testFrameToolNormalizesABackwardsDrag() { + // Users drag up-left as readily as down-right; a negative-extent rect would + // hit the session's degenerate guard and resolve to nothing. + XCTAssertEqual( + SelectionGesture.resolve( + tool: .frame, + from: CGPoint(x: 180, y: 160), + to: CGPoint(x: 100, y: 100), + axOrigin: .zero + ), + .frame(CGRect(x: 100, y: 100, width: 80, height: 60)) + ) + } + + // MARK: - Travel threshold + + func testZeroTravelIsNotFarEnough() { + XCTAssertFalse(SelectionGesture.travelledFarEnough(from: start, to: start)) + XCTAssertEqual( + SelectionGesture.localRect(from: start, to: start), + CGRect(x: 100, y: 100, width: 0, height: 0) + ) + } + + func testTravelJustBelowThresholdIsNotFarEnough() { + let below = CGPoint(x: start.x + SelectionGesture.minimumTravel - 0.5, y: start.y) + XCTAssertFalse(SelectionGesture.travelledFarEnough(from: start, to: below)) + } + + func testTravelExactlyAtThresholdIsNotFarEnough() { + let exact = CGPoint(x: start.x + SelectionGesture.minimumTravel, y: start.y) + XCTAssertFalse(SelectionGesture.travelledFarEnough(from: start, to: exact)) + } + + func testTravelJustAboveThresholdIsFarEnough() { + let above = CGPoint(x: start.x + SelectionGesture.minimumTravel + 0.5, y: start.y) + XCTAssertTrue(SelectionGesture.travelledFarEnough(from: start, to: above)) + } + + func testDiagonalTravelIsMeasuredAsHypotenuseNotPerAxis() { + // Each axis alone is under the threshold; together they are over it. A + // per-axis test would call this deliberate diagonal drag too short. + let leg = SelectionGesture.minimumTravel * 0.8 // hypotenuse = 1.131 * threshold + let diagonal = CGPoint(x: start.x + leg, y: start.y + leg) + XCTAssertLessThan(leg, SelectionGesture.minimumTravel) + XCTAssertTrue(SelectionGesture.travelledFarEnough(from: start, to: diagonal)) + + // And the converse: a diagonal whose hypotenuse is under the threshold is + // still a wobble even though it moved on both axes. + let short = SelectionGesture.minimumTravel * 0.5 // hypotenuse = 0.707 * threshold + XCTAssertFalse( + SelectionGesture.travelledFarEnough(from: start, to: CGPoint(x: start.x + short, y: start.y + short)) + ) + } + + func testThresholdIsLargerOnTouchThanCursor() { + // A finger rolls several points on a deliberate tap, so a mouse-tuned + // threshold on iOS would turn ordinary frame-mode taps into tiny accidental + // frames. + #if os(iOS) + XCTAssertEqual(SelectionGesture.minimumTravel, 10) + #else + XCTAssertEqual(SelectionGesture.minimumTravel, 6) + #endif + } + + // MARK: - Geometry + + func testAllFourDragDirectionsNormalizeToTheSameRect() { + // Users drag up-left as readily as down-right. Without normalization the + // three "backwards" directions arrive with negative extents, which the + // session's degenerate guard treats as a non-drag — the marquee would work + // in exactly one direction. + let expected = CGRect(x: 100, y: 100, width: 80, height: 60) + let a = CGPoint(x: 100, y: 100), b = CGPoint(x: 180, y: 160) + XCTAssertEqual(SelectionGesture.localRect(from: a, to: b), expected) // down-right + XCTAssertEqual(SelectionGesture.localRect(from: b, to: a), expected) // up-left + XCTAssertEqual(SelectionGesture.localRect(from: CGPoint(x: 180, y: 100), to: CGPoint(x: 100, y: 160)), expected) // down-left + XCTAssertEqual(SelectionGesture.localRect(from: CGPoint(x: 100, y: 160), to: CGPoint(x: 180, y: 100)), expected) // up-right + } + + func testAXRectIsWindowLocalRectAtOriginOnPrimaryDisplay() { + // `axOrigin == .zero` on the primary display at the global origin, so the + // AX rect and the drawn band coincide there — which is why an origin bug + // hides until someone uses a second display. + XCTAssertEqual( + SelectionGesture.axRect(from: CGPoint(x: 100, y: 100), to: CGPoint(x: 180, y: 160), axOrigin: .zero), + CGRect(x: 100, y: 100, width: 80, height: 60) + ) + } + + func testAXRectAppliesANonZeroOriginExactly() { + // Secondary-display case: gesture coordinates are window-local, the AX + // queries are in screen space, so the origin is ADDED (the mirror of the + // highlight's subtraction). + XCTAssertEqual( + SelectionGesture.axRect( + from: CGPoint(x: 100, y: 100), + to: CGPoint(x: 180, y: 160), + axOrigin: CGPoint(x: 1512, y: 30) + ), + CGRect(x: 1612, y: 130, width: 80, height: 60) + ) + } + + func testAXRectNormalizesBeforeShiftingSoABackwardsDragIsStillOnScreen() { + // A up-left drag shifted by the origin must land at the frame's top-left, + // not at the press point with negative extents (which would resolve to + // nothing at all). + XCTAssertEqual( + SelectionGesture.axRect( + from: CGPoint(x: 180, y: 160), + to: CGPoint(x: 100, y: 100), + axOrigin: CGPoint(x: 1512, y: 30) + ), + CGRect(x: 1612, y: 130, width: 80, height: 60) + ) + } +} From 9f963e41df3ca523b2b698191d5fd04678f5f4f9 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 19:38:32 -0500 Subject: [PATCH 13/24] feat(overlay): Select Parent / Select Child replace the one-way Widen button Navigation is now bidirectional. Ascending is the old widenSelection(); descending prefers HISTORY (walk back down the path you climbed) and only queries the source for children when already at the deepest known rung, prepending the child so a re-ascent returns to the target and a re-descent replays to the same child rather than re-running the heuristic under a live UI. Fixes a silent corruption prepending would otherwise introduce: the note's component was path[1], which held only while the path was strictly upward (every upward rung is seeded). With a child at index 0, index 1 is the original target, which may be unseeded -- and an unseeded Element.id is a slash-joined path, so the note would have exported a grep target matching nothing while looking plausible. It now skips to the first SEEDED rung above the bound one. Pinned by a mutation-verified test. Adds ChildNavigationSource with a shared pure ordering rule, implemented on both macOS sources and iOS. canSelectChild reads a cache, never the source, so SwiftUI rendering cannot trigger an AX walk. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/ChildNavigationRule.swift | 85 ++++++ Sources/AnnotKit/ElementSource.swift | 27 +- .../AnnotKit/Overlay/AnnotationSession.swift | 243 +++++++++++++----- Sources/AnnotKit/Overlay/OverlayView.swift | 69 ++++- Sources/AnnotKit/iOS/IOSElementSource.swift | 78 ++++++ Sources/AnnotKit/macOS/AXIntrospection.swift | 117 +++++++++ Sources/AnnotKit/macOS/MacElementSource.swift | 6 + .../macOS/MacViewTreeElementSource.swift | 77 ++++++ .../AnnotationSessionTests.swift | 210 +++++++++++++-- 9 files changed, 812 insertions(+), 100 deletions(-) create mode 100644 Sources/AnnotKit/ChildNavigationRule.swift diff --git a/Sources/AnnotKit/ChildNavigationRule.swift b/Sources/AnnotKit/ChildNavigationRule.swift new file mode 100644 index 0000000..54292e5 --- /dev/null +++ b/Sources/AnnotKit/ChildNavigationRule.swift @@ -0,0 +1,85 @@ +import CoreGraphics +import Foundation + +/// One element considered as a CHILD of the currently-bound target: its +/// annotation-relevant facts and its frame in the source's screen space (AX +/// top-left on macOS, view-local on iOS — the same space the hint arrives in, so +/// containment is a plain rect test with no conversion). +/// +/// The platform adapter reads these facts once — from the AX tree on macOS, from +/// the view tree on either platform — and hands the rule a flat array, so nothing +/// here is a live platform handle and the ordering is unit-tested without a +/// running app. Deliberately NOT ``MarqueeCandidate``: depth is meaningless for a +/// single generation of siblings, and reusing a struct with a field the rule must +/// ignore invites a future tie-break reading it. +public struct ChildCandidate: Sendable, Hashable { + public let element: TargetCandidate + public let frame: CGRect + + public init(element: TargetCandidate, frame: CGRect) { + self.element = element + self.frame = frame + } + + /// Seeded = carries an `accessibilityIdentifier`. The identifier is what + /// locates source code, so at the same geometric standing it decides whether + /// descending produces a note an agent can act on. + fileprivate var isSeeded: Bool { !element.identifier.isEmpty } + + fileprivate var area: CGFloat { frame.width * frame.height } +} + +/// The child-navigation ordering rule: given every child the adapter could see +/// under the bound element, decide which one "Select Child" should descend into. +/// Pure and platform-independent, the downward counterpart of +/// ``AnnotationTargetRule`` — all three adapters (macOS AX, macOS view-tree, iOS) +/// build `[ChildCandidate]` from their own node types and call ``order(_:near:)``, +/// so a descent resolves identically everywhere and the decision is tested on its +/// own rather than by three hand-rolled sorts kept in step by hand. +public enum ChildNavigationRule { + /// Indices into `candidates`, most-likely-intended FIRST; ineligible entries + /// are dropped entirely, so an empty result means "leaf, nothing to descend + /// into". + /// + /// Ranking, in order: + /// + /// 1. **Contains the hint.** `hint` is the gesture's own anchor — the point + /// clicked, or the centre of the frame drawn. When one child sits under it + /// that child is the one the user was already pointing at, which beats any + /// guess made from geometry alone. + /// 2. **Seeded beats unseeded.** Descending into an unseeded child yields a + /// note whose `component` has to be inherited from an ancestor; descending + /// into a seeded one names a component directly. + /// 3. **Larger area.** Going DOWN one level should land on the substantial + /// thing inside (the card's content row), not the 8pt chevron that happens + /// to be its first sibling — the same "I mean this whole thing" instinct + /// ``MarqueeTargetRule``'s surrounded pass encodes, applied one level down. + /// 4. **Earlier index**, so the same selection always descends the same way. + /// `sorted(by:)` is not stable, so this is an explicit key rather than an + /// assumption: without it two coextensive children would swap places + /// between runs and repeating a descent could land somewhere new. + /// + /// Eligibility is ``TargetCandidate/isEligibleMeaningful``, the SAME predicate + /// the target rule uses, so window chrome, container roots, and window-ghost + /// groups can never be offered as children — descending into any of them binds + /// the note to something that names no app code. + public static func order(_ candidates: [ChildCandidate], near hint: CGPoint?) -> [Int] { + candidates.indices + .filter { i in + // Positive area twice over: a zero-area child is nothing the user + // could have meant, and it would otherwise sort last-but-present + // and be offered as a descent target on a leaf. + candidates[i].element.isEligibleMeaningful + && candidates[i].frame.width > 0 && candidates[i].frame.height > 0 + } + .sorted { lhs, rhs in + let (l, r) = (candidates[lhs], candidates[rhs]) + let lHit = hint.map(l.frame.contains) ?? false + let rHit = hint.map(r.frame.contains) ?? false + if lHit != rHit { return lHit } + if l.isSeeded != r.isSeeded { return l.isSeeded } + if l.area != r.area { return l.area > r.area } + return lhs < rhs + } + } +} diff --git a/Sources/AnnotKit/ElementSource.swift b/Sources/AnnotKit/ElementSource.swift index bc1e7e2..8e052ca 100644 --- a/Sources/AnnotKit/ElementSource.swift +++ b/Sources/AnnotKit/ElementSource.swift @@ -51,14 +51,35 @@ public protocol RegionAnchorSource { /// the annotation target first (same element ``ElementSource/hitTest(_:)`` /// returns), then each enclosing identified component, broadest last. Lets the /// session step a selection UP to a parent component for a coarser-grained note -/// (a click-again / widen affordance while the composer is open) without -/// re-hit-testing. Sources that cannot offer it simply don't conform; the session -/// then disables widening. +/// (the composer's Select Parent control) without re-hit-testing. Sources that +/// cannot offer it simply don't conform; the session then has a single-rung +/// selection path and Select Parent stays disabled. @MainActor public protocol ComponentLadderSource { func componentLadder(at point: CGPoint) -> [Element] } +/// Optional element-source capability: the meaningful children of an element — +/// the DOWNWARD counterpart of ``ComponentLadderSource``. Lets the session step a +/// selection INTO a component (the row inside the card, rather than the card) when +/// the user overshot, or when the target rule bound coarser than they meant. +/// Sources that cannot offer it simply don't conform; the session then leaves the +/// composer's Child control permanently disabled, and only upward navigation (and +/// downward navigation back through already-visited rungs) is available. +@MainActor +public protocol ChildNavigationSource { + /// Meaningful children of `element`, most-likely-intended FIRST. `hint` is the + /// gesture's anchor (the click point, or the drawn frame's centre) when there + /// is one. Empty when the element is a leaf. + /// + /// Ordering is the shared, pure ``ChildNavigationRule`` on every adapter, so + /// the same layout descends the same way on macOS and iOS. Implementations + /// MUST NOT snapshot the whole tree per call: this runs on every selection + /// change, and the bound element's frame centre gives a cheap descent path from + /// the containing window (the same trick the hit-test uses). + func children(of element: Element, near hint: CGPoint?) -> [Element] +} + /// Optional element-source capability: resolve a frame the user DREW (a marquee /// press-drag) to an annotation target, per ``MarqueeTargetRule`` — the largest /// meaningful element the frame surrounds, else the tightest element the frame diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index 17aa435..3b31bf1 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -44,17 +44,19 @@ public final class AnnotationSession: ObservableObject { @Published public private(set) var pending: [AnnotationNote] = [] @Published public private(set) var hovered: Element? @Published public private(set) var selected: Element? { - // The region offset, the drawn marquee frame, and the widening ladder only + // The region offset, the drawn marquee frame, and the navigation path only // make sense while their selection is alive; clearing the selection // (capture, cancel, stop, pin editing) must never leave a stale offset, - // frame, or ladder for the NEXT note. + // frame, or path for the NEXT note. didSet { if selected == nil { selectedRegionOffset = nil selectedMarqueeRect = nil marqueeRegionOrigin = nil - componentLadder = [] - ladderIndex = 0 + selectionPath = [] + pathIndex = 0 + cachedChildren = [] + selectionHint = nil } } } @@ -64,10 +66,10 @@ public final class AnnotationSession: ObservableObject { /// The frame the user DREW for the current selection, in ABSOLUTE AX screen /// coordinates; nil for click selections. Kept absolute (not element-relative) - /// so ``widenSelection()`` re-relativizes it for free: widening rebinds the - /// note to an enclosing element with a different origin, and a frame already - /// relativized at selection time would then silently describe the drag against - /// a box the note no longer names. + /// so navigation re-relativizes it for free: ``selectParent()``/``selectChild()`` + /// rebind the note to a different element with a different origin, and a frame + /// already relativized at selection time would then silently describe the drag + /// against a box the note no longer names. public private(set) var selectedMarqueeRect: CGRect? /// Origin the drawn frame is measured from when the selection is a synthetic /// REGION — whose own `frame` IS the drawn rect, so it cannot be its own @@ -75,17 +77,46 @@ public final class AnnotationSession: ObservableObject { /// selections, which measure from the selected element's origin. private var marqueeRegionOrigin: CGPoint? - /// The component-widening ladder for the current selection (target first, each - /// enclosing identified component after), and the index of the currently - /// selected rung. Populated on ``select(atAXPoint:)`` when the source offers a - /// ``ComponentLadderSource``; empty for region selections and unsupported - /// sources, which disables widening. - private var componentLadder: [Element] = [] - private var ladderIndex: Int = 0 - - /// Whether ``widenSelection()`` can step the current selection up to an - /// enclosing component. Drives the composer's widen affordance. - public var canWidenSelection: Bool { ladderIndex + 1 < componentLadder.count } + /// The BIDIRECTIONAL navigation path for the current selection, and the index + /// of the bound rung. Ordering is a CONVENTION the whole file depends on: + /// index 0 is the deepest rung known so far, ascending indices are + /// progressively broader, and `pathIndex` points at the rung the note is bound + /// to. Seeded on selection from the source's ``ComponentLadderSource`` / + /// ``MarqueeTargetSource`` ladder (which has exactly that shape), or from the + /// bare hit-test target when the source offers neither. EMPTY for region + /// selections, which is what makes both navigation directions inert for them. + /// + /// ``selectChild()`` PREPENDS a newly-discovered child, keeping index 0 "the + /// deepest known rung" — see that method for why re-querying instead would make + /// a round trip non-deterministic. + private var selectionPath: [Element] = [] + private var pathIndex: Int = 0 + /// Children of the CURRENTLY BOUND rung, cached because ``canSelectChild`` is + /// read on every SwiftUI render and a source-side child query is an AX/view + /// walk. Refreshed exactly where the bound element changes (the two `select` + /// paths, ``selectParent()``, ``selectChild()``); querying from the property + /// instead would put a tree walk in the render loop. + private var cachedChildren: [Element] = [] + /// The gesture's own anchor for the current selection — the point clicked, or + /// the centre of the frame drawn. Handed to the source as the child-ordering + /// hint, so descending prefers the child the user was already pointing at over + /// one picked from geometry alone. nil once the selection is gone. + private var selectionHint: CGPoint? + + /// Whether ``selectParent()`` can bind the note to an enclosing component. + /// O(1) — read during rendering. + public var canSelectParent: Bool { pathIndex + 1 < selectionPath.count } + /// Whether ``selectChild()`` can bind the note to a component inside the + /// current one: either back down through an already-visited rung (history), or + /// into a freshly-discovered child. O(1) by construction — it reads the cache + /// and NEVER calls into the source, because it runs on every render. + /// + /// The `!selectionPath.isEmpty` term is load-bearing beyond redundancy: a + /// REGION selection has no path, and this is what guarantees it never even + /// triggers a child query. + public var canSelectChild: Bool { + pathIndex > 0 || (!selectionPath.isEmpty && !cachedChildren.isEmpty) + } /// The id of the retained note whose in-overlay edit card is open, or nil when /// no editor is showing. UI-only: drives which pin's edit card the overlay /// renders. Mutually exclusive with ``selected`` (the composer) — opening one @@ -167,23 +198,28 @@ public final class AnnotationSession: ObservableObject { // Any catcher tap dismisses an open pin editor: a tap on empty space is a // click-away close, and a tap on an element hands the stage to the composer. editingNoteID = nil - // Every selection starts offset-free, frame-free and ladder-free: a + // Every selection starts offset-free, frame-free and path-free: a // region -> element or marquee -> click re-selection (the catcher stays // active behind an open composer) must not leak the previous region's - // offset, the previous drag's drawn frame, or a stale ladder onto the next - // note — the didSet only clears them when `selected` becomes nil, not on - // replacement. + // offset, the previous drag's drawn frame, or a stale navigation path onto + // the next note — the didSet only clears them when `selected` becomes nil, + // not on replacement. selectedRegionOffset = nil selectedMarqueeRect = nil marqueeRegionOrigin = nil - componentLadder = [] - ladderIndex = 0 + resetNavigation(hint: point) selected = source.hitTest(point) - // Capture the widening ladder for a real element selection (its first rung - // equals the hit-test target). Region selections get no ladder. - if selected != nil, let ladderSource = source as? ComponentLadderSource { - componentLadder = ladderSource.componentLadder(at: point) - ladderIndex = 0 + // Seed the navigation path for a real element selection. The ladder's first + // rung equals the hit-test target by contract, so it already satisfies the + // current-first convention. A source with no ladder capability (or one that + // returns nothing) still gets a ONE-rung path: that disables Select Parent + // exactly as before, but keeps Select Child live, since descending needs + // only a bound element and a `ChildNavigationSource`. Region selections + // fall through with an empty path and navigate nowhere. + if let target = selected { + let ladder = (source as? ComponentLadderSource)?.componentLadder(at: point) ?? [] + selectionPath = ladder.isEmpty ? [target] : ladder + refreshChildCache() } if selected == nil, let anchorSource = source as? RegionAnchorSource, @@ -241,8 +277,10 @@ public final class AnnotationSession: ObservableObject { selectedRegionOffset = nil selectedMarqueeRect = nil marqueeRegionOrigin = nil - componentLadder = [] - ladderIndex = 0 + // The hint is set below, once the rect is normalized — a backwards drag's + // raw `midX`/`midY` are still correct, but deriving it from the standardized + // rect keeps one definition of "the centre of what was drawn". + resetNavigation(hint: nil) // Normalize once: a right-to-left or bottom-to-top drag arrives with // negative extents. @@ -265,8 +303,13 @@ public final class AnnotationSession: ObservableObject { let ladder = marqueeSource.marqueeLadder(in: normalized) if let target = ladder.first { selected = target - componentLadder = ladder - ladderIndex = 0 + selectionPath = ladder + pathIndex = 0 + // The centre of the drawn frame is the anchor a user would name + // themselves, so it is the child-ordering hint — the same point the + // region fallback below anchors to. + selectionHint = CGPoint(x: normalized.midX, y: normalized.midY) + refreshChildCache() selectedMarqueeRect = normalized return selected } @@ -306,34 +349,99 @@ public final class AnnotationSession: ObservableObject { return selected } - /// Step the current selection UP to the next enclosing identified component - /// (a coarser-grained note: the card instead of the label inside it). No-op - /// when the selection is already at the broadest rung, is a region, or the - /// source offers no ``ComponentLadderSource``. The composer re-anchors to the - /// widened element's frame for free (it renders `selected`). + /// Bind the note to the ENCLOSING component (the card instead of the label + /// inside it). No-op at the broadest known rung, for a region selection, or + /// when the source offers no ``ComponentLadderSource``. The composer re-anchors + /// to the new element's frame for free (it renders `selected`). + @discardableResult + public func selectParent() -> Element? { + guard canSelectParent else { return nil } + pathIndex += 1 + return bindCurrentRung() + } + + /// Bind the note to a component INSIDE the current one. Two sources, and the + /// priority between them is the whole design: + /// + /// 1. **History** (`pathIndex > 0`) — step back down a rung already walked. + /// Deterministic, free, and it is the overwhelmingly common case: the user + /// pressed Parent once too often and wants to undo it. + /// 2. **Below the deepest known rung** (`pathIndex == 0`) — ask the source for + /// the bound element's children, take the most likely, and PREPEND it, so + /// index 0 still means "deepest known rung". + /// + /// Prepending (rather than re-querying on every descent) is what makes a round + /// trip hold in BOTH directions: after descending to child C, Parent returns to + /// the original target and Child returns to C ITSELF via case 1. Re-querying + /// would let a live UI — a hover state resolving, a list reflowing — hand back + /// a different "most likely" child, so pressing Parent then Child would land + /// somewhere the user never chose. No-op for a region selection (empty path, so + /// case 1 is false and ``canSelectChild``'s cache guard blocks case 2) and for + /// a leaf with no ``ChildNavigationSource`` children. @discardableResult - public func widenSelection() -> Element? { - guard ladderIndex + 1 < componentLadder.count else { return nil } - ladderIndex += 1 - // Widening always lands on a real element, so it is never a region note. + public func selectChild() -> Element? { + if pathIndex > 0 { + pathIndex -= 1 + return bindCurrentRung() + } + // `canSelectChild`'s `!selectionPath.isEmpty` guard, restated where it is + // enforced: a region must never mutate the path into existence. + guard !selectionPath.isEmpty, let child = cachedChildren.first else { return nil } + selectionPath.insert(child, at: 0) + // `pathIndex` deliberately STAYS 0 — the insert moved every existing rung up + // one, so 0 now addresses the child and the old target sits at 1. + return bindCurrentRung() + } + + /// Bind the note to `selectionPath[pathIndex]` and re-derive everything that + /// hangs off the bound element. Shared by both directions so they can never + /// drift into clearing different state. + private func bindCurrentRung() -> Element? { + // Navigation always lands on a real element, so it is never a region note. selectedRegionOffset = nil // ...and for the same reason it can never keep a region's measuring stick: // every rung is a real element, so the drawn frame is measured from the // element itself. Clearing is cheap insurance rather than a live fix — - // regions get no ladder today, so this state is currently unreachable — - // but leaving a stale anchor origin behind would silently measure the - // persisted rect from the wrong box, and that is exactly the class of bug - // `7993a67` was. + // regions get no path today, so this state is currently unreachable — but + // leaving a stale anchor origin behind would silently measure the persisted + // rect from the wrong box, and that is exactly the class of bug `7993a67` + // was. marqueeRegionOrigin = nil // `selectedMarqueeRect` is deliberately KEPT: it is absolute, so it is // still the frame the user drew whichever rung is now bound, and `addNote` - // re-relativizes it against the widened element. - // A non-nil assignment does not trip the didSet clear, so the ladder and - // index survive for a further widen. - selected = componentLadder[ladderIndex] + // re-relativizes it against that rung. + // A non-nil assignment does not trip the didSet clear, so the path and + // index survive for further navigation. + selected = selectionPath[pathIndex] + refreshChildCache() return selected } + /// Drop the whole navigation state for a selection that is being replaced. + /// Called at the top of both `select` paths because the `selected` didSet only + /// fires on nil, never on replacement. + private func resetNavigation(hint: CGPoint?) { + selectionPath = [] + pathIndex = 0 + cachedChildren = [] + selectionHint = hint + } + + /// Re-read the bound rung's children into the cache, so ``canSelectChild`` can + /// stay a pure property read. Costs one source-side query per SELECTION change + /// — not per render, and not per hover. + private func refreshChildCache() { + // An empty path means a region (or nothing selected): never query. + guard !selectionPath.isEmpty, + let element = selected, + let childSource = source as? ChildNavigationSource + else { + cachedChildren = [] + return + } + cachedChildren = childSource.children(of: element, near: selectionHint) + } + /// Capture a screenshot of the currently selected element, if any. public func screenshotSelected() async -> CapturedImage? { guard let selected else { return nil } @@ -353,22 +461,35 @@ public final class AnnotationSession: ObservableObject { // identifier of its own (the selector anchored to an ancestor or went // positional) — a miss worth turning into a seeding task rather than a // silent misattribution. `component` is the seeded component to grep: the - // target's own id when seeded, else the tightest enclosing seeded - // component from the GEOMETRIC widening ladder — which reaches a SwiftUI - // `.axCardSurface` card whose identifier sits on a background sibling, not - // an ancestor, so plain ancestry (the fallback for region/ladderless - // sources) would miss it. + // target's own identifier when seeded, else the first SEEDED rung strictly + // ABOVE the bound one on the GEOMETRIC navigation path — which reaches a + // SwiftUI `.axCardSurface` card whose identifier sits on a background + // sibling, not an ancestor, so plain ancestry (the fallback, for + // region/pathless sources) would miss it. + // + // Two details here are silent-corruption guards, not style: + // + // - `dropFirst(pathIndex + 1)`, not `dropFirst()`. The old form assumed the + // bound rung was index 0 and index 1 was seeded. ``selectChild()`` + // prepends, so index 1 can now be the ORIGINAL target — which may be + // unseeded — and the search must start above whatever rung is bound. + // - `path.last?.identifier`, not `.id`. An UNSEEDED element's `id` is a + // slash-joined path string, so taking `.id` would hand the agent a grep + // target that matches nothing while looking perfectly plausible in the + // exported note. let ownIdentifier = element.path.last?.identifier ?? "" let component = !ownIdentifier.isEmpty ? ownIdentifier - : (componentLadder.dropFirst().first?.id + : (selectionPath.dropFirst(pathIndex + 1) + .first(where: { !($0.path.last?.identifier ?? "").isEmpty })? + .path.last?.identifier ?? element.path.last(where: { !($0.identifier ?? "").isEmpty })?.identifier) // Relativize the drawn frame HERE rather than at selection — the asymmetry // with `regionOffset` (computed at selection) is deliberate: - // ``widenSelection()`` can rebind the note to an enclosing element AFTER - // the frame was drawn, so the persisted rect must be measured against - // whatever element the note FINALLY names. A region never widens, so its - // offset's anchor is fixed the moment it is picked. + // ``selectParent()``/``selectChild()`` can rebind the note to a different + // element AFTER the frame was drawn, so the persisted rect must be measured + // against whatever element the note FINALLY names. A region never navigates, + // so its offset's anchor is fixed the moment it is picked. let regionRect: CGRect? = selectedMarqueeRect.map { drawn in // A synthetic region's own frame IS the drawn rect, so it measures from // its anchor (else it would trivially be 0,0); elements measure from diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index 3bf398d..81fd8eb 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -243,22 +243,49 @@ struct OverlayView: View { comment = "" session.cancelSelection() }, - onFocusRequest: onFocusRequest + onFocusRequest: onFocusRequest, + // Tree navigation: rebind the note to the enclosing component, or to a + // component inside the current one. Its own row rather than a third + // footer button — the card is 260pt wide and Cancel/Add note already + // fill it, so a third control there would be cramped enough to misread. + // + // DISABLED, never hidden. The control this replaces (a lone "Widen" + // button) appeared and disappeared with availability, and that is a + // large part of why it was unlearnable: a control you have never seen + // is a control you cannot predict. Both buttons are always present, so + // "you can move the binding up and down the tree" is visible from the + // first note, and greying tells you where you are in the tree. + // + // No element name here: the card HEADER already shows the bound + // element (`session.selectionLabel`) and re-renders as you navigate, so + // it is the "where am I" indicator and repeating it would be noise. + navigation: { + HStack(spacing: 6) { + Button { session.selectParent() } label: { + Label("Parent", systemImage: "chevron.up") + } + .disabled(!session.canSelectParent) + // Tooltips name the EFFECT, not the mechanism. "Widen" described + // what the code did to the ladder and read as "make the + // highlight bigger"; what the user is choosing is which + // component the note is FILED AGAINST. + .help("Bind this note to the enclosing component") + Button { session.selectChild() } label: { + Label("Child", systemImage: "chevron.down") + } + .disabled(!session.canSelectChild) + .help("Bind this note to a component inside") + Spacer() + } + .controlSize(.small) + .frame(width: 260) + } ) { HStack { Button("Cancel") { comment = "" session.cancelSelection() } - // Step the selection up to the enclosing component (the card - // instead of the label inside it) for a coarser-grained note. - // Shown only when a broader identified component is available. - if session.canWidenSelection { - Button { session.widenSelection() } label: { - Label("Widen", systemImage: "arrow.up.backward.and.arrow.down.forward") - } - .help("Select the enclosing component") - } Spacer() Button("Add note") { addNote() } .buttonStyle(.borderedProminent) @@ -281,7 +308,12 @@ struct OverlayView: View { focusKey: note.id, onSubmit: { saveEdit(note) }, onCancel: { session.endEditing() }, - onFocusRequest: onFocusRequest + onFocusRequest: onFocusRequest, + // No navigation row: this card edits a note that has ALREADY been + // captured, whose selector, component and element path were frozen at + // capture. Offering Parent/Child here would move a highlight and change + // nothing about the record, which is worse than no control at all. + navigation: { EmptyView() } ) { HStack { Button(role: .destructive) { @@ -407,8 +439,9 @@ struct OverlayView: View { /// `@FocusState` + a next-tick re-assert to beat the insertion race), and the /// keyboard contract (Enter submits, Shift+Enter inserts a newline, Escape /// cancels on macOS). The two flows differ ONLY in the `header` text, the `text` -/// binding, the `placement` anchor, and the `footer` button row. -private struct AnnotationCard: View { +/// binding, the `placement` anchor, the optional `navigation` row, and the +/// `footer` button row. +private struct AnnotationCard: View { /// Header label: the composer shows the element's selection label; the editor /// shows the note's selector. let header: String @@ -428,6 +461,11 @@ private struct AnnotationCard: View { /// Make the host panel key so the field accepts keystrokes (`panel.makeKey()` /// on macOS; a no-op on iOS, where `@FocusState` alone raises the keyboard). let onFocusRequest: () -> Void + /// Row between the field and the footer: the composer's Parent/Child tree + /// navigation, `EmptyView` for the pin editor (whose binding is already fixed). + /// A separate slot rather than more footer buttons, so the 260pt footer keeps + /// exactly its two primary actions. + @ViewBuilder let navigation: () -> Navigation /// The differing two-button row (Cancel/Add note vs Delete/Save). @ViewBuilder let footer: () -> Footer @@ -459,6 +497,11 @@ private struct AnnotationCard: View { onSubmit() return .handled } + // No `.frame(width:)` here, unlike the rows around it: the editor + // passes `EmptyView`, and a frame modifier would give that nothing a + // 260pt-wide slot and an 8pt VStack gap on a card that has no nav row. + // The composer's row carries its own width instead. + navigation() footer() .frame(width: 260) } diff --git a/Sources/AnnotKit/iOS/IOSElementSource.swift b/Sources/AnnotKit/iOS/IOSElementSource.swift index 5a5f65c..31c2ccc 100644 --- a/Sources/AnnotKit/iOS/IOSElementSource.swift +++ b/Sources/AnnotKit/iOS/IOSElementSource.swift @@ -297,6 +297,84 @@ public final class IOSElementSource: ElementSource, ComponentLadderSource { } } +// MARK: - Child navigation (view -> the view inside it) + +extension IOSElementSource: ChildNavigationSource { + /// The meaningful subviews of `element`, most-likely-intended first, ordered by + /// the shared pure ``ChildNavigationRule`` — the same rule both macOS sources + /// use, so an identical layout descends identically on both platforms by + /// construction rather than by three sorts being kept in step by hand. + /// + /// Cost: no whole-tree walk. ``selector(for:)`` rebuilds every window's node + /// tree, which is affordable once per capture but not here — this runs on every + /// selection change. The element's frame CENTRE instead gives a containment + /// descent from its window to the live `UIView`, after which only that view's + /// own subtree is visited, each branch stopping at its first meaningful node. + public func children(of element: Element, near hint: CGPoint?) -> [Element] { + let centre = CGPoint(x: element.frame.midX, y: element.frame.midY) + guard let root = Self.marqueeRoot(containing: centre), + let view = Self.descend(root, matching: element, containing: centre, depth: 0) + else { return [] } + + var found: [UIView] = [] + Self.collectNearestMeaningful(under: view, depth: 0, into: &found) + let candidates = found.map { + ChildCandidate(element: Self.candidate(for: $0), frame: Self.screenFrame(of: $0)) + } + return ChildNavigationRule.order(candidates, near: hint).map { Self.element(for: found[$0]) } + } + + /// The live `UIView` behind a public ``Element``, found by descending the + /// containing window along frames that contain the element's own centre — + /// bounded by tree DEPTH, not tree size. `hitTest` is deliberately not reused: + /// it honours `isUserInteractionEnabled` and would refuse to re-find a + /// non-interactive view the user has already legitimately selected. + private static func descend( + _ view: UIView, matching element: Element, containing point: CGPoint, depth: Int + ) -> UIView? { + if matches(view, element) { return view } + guard depth < maxDepth else { return nil } + for subview in view.subviews where !subview.isHidden && subview.alpha > 0.01 { + let frame = screenFrame(of: subview) + guard frame.width > 0, frame.height > 0, frame.contains(point) else { continue } + if let found = descend(subview, matching: element, containing: point, depth: depth + 1) { return found } + } + return nil + } + + /// Identity test for re-finding a captured ``Element``: identifier, view type, + /// and frame together. See the macOS AX source's `matches` for why none of the + /// three is sufficient alone. + private static func matches(_ view: UIView, _ element: Element) -> Bool { + guard (view.accessibilityIdentifier ?? "") == (element.path.last?.identifier ?? "") else { return false } + guard String(describing: Swift.type(of: view)) == element.role else { return false } + let frame = screenFrame(of: view) + // Half a point of slack — a strict `==` would make re-finding fail on a + // fractional layout. + return abs(frame.minX - element.frame.minX) < 0.5 && abs(frame.minY - element.frame.minY) < 0.5 + && abs(frame.width - element.frame.width) < 0.5 && abs(frame.height - element.frame.height) < 0.5 + } + + /// The nearest MEANINGFUL descendants of `view`: each visible subview that is a + /// target in its own right, and for each that is not, the meaningful views + /// beneath it. Descending through unmeaningful wrappers is what keeps the Child + /// control alive under a SwiftUI host, whose layout containers carry no + /// identifier, label, or value of their own; each branch stops at its first + /// meaningful node, so this is not a subtree enumeration. + private static func collectNearestMeaningful(under view: UIView, depth: Int, into found: inout [UIView]) { + guard depth < maxDepth else { return } + for subview in view.subviews where !subview.isHidden && subview.alpha > 0.01 { + let frame = screenFrame(of: subview) + let isTarget = frame.width > 0 && frame.height > 0 && candidate(for: subview).isEligibleMeaningful + if isTarget { + found.append(subview) + } else { + collectNearestMeaningful(under: subview, depth: depth + 1, into: &found) + } + } + } +} + // MARK: - Marquee (drawn frame -> view) extension IOSElementSource: MarqueeTargetSource { diff --git a/Sources/AnnotKit/macOS/AXIntrospection.swift b/Sources/AnnotKit/macOS/AXIntrospection.swift index fe59717..6a752ac 100644 --- a/Sources/AnnotKit/macOS/AXIntrospection.swift +++ b/Sources/AnnotKit/macOS/AXIntrospection.swift @@ -418,6 +418,123 @@ enum AXIntrospection { } } + // MARK: - Child navigation (element -> the component inside it) + + /// The meaningful children of `element`, most-likely-intended first — the + /// DOWNWARD counterpart of ``componentLadder(for:)``. Ordering is the shared + /// pure ``ChildNavigationRule``, so macOS and iOS descend identically. + /// + /// Cost: deliberately NOT a snapshot. `selector(for:)` walks the entire app via + /// ``snapshotNodes()``, which is affordable once per CAPTURE but not here — + /// this runs on every selection change, including each step of a rapid + /// Parent/Child exploration. Instead the bound element's frame CENTRE gives a + /// containment descent from its window (the same trick ``hitBeneathOverlay(_:)`` + /// uses) to find the live handle, then only that element's own subtree is + /// visited, and each branch of it stops at the first meaningful node. + static func children(of target: Element, near hint: CGPoint?) -> [Element] { + let centre = CGPoint(x: target.frame.midX, y: target.frame.midY) + guard let node = locate(target, at: centre) else { return [] } + let windowFrame = ancestorChain(from: node) + .first { string($0, kAXRoleAttribute) == "AXWindow" } + .map(frameScreen(of:)) + + var found: [AXUIElement] = [] + collectNearestMeaningful(under: node, windowFrame: windowFrame, depth: 0, into: &found) + let candidates = found.map { + ChildCandidate(element: candidate(for: $0, windowFrame: windowFrame), frame: frameScreen(of: $0)) + } + return ChildNavigationRule.order(candidates, near: hint).map { + element(for: found[$0], ancestorChain: ancestorChain(from: found[$0])) + } + } + + /// The live AX handle behind a public ``Element``, found by descending the + /// containing window along frames that contain the element's own centre. + /// Bounded by tree DEPTH rather than tree size, which is the whole point: an + /// identity map would have to be rebuilt from a full snapshot every time the + /// tree changed. + /// + /// Returns nil when the element has left the tree, or when a clipping ancestor + /// does not contain the element's centre. Both degrade the same benign way — + /// no children, so the composer's Child control stays disabled — rather than + /// offering a descent into something that is not the bound element. + private static func locate(_ element: Element, at centre: CGPoint) -> AXUIElement? { + let app = appElement() + let windows = elementArray(app, kAXWindowsAttribute).filter { !isOverlayWindow($0) } + guard let window = windows.first(where: { frameScreen(of: $0).contains(centre) }) else { return nil } + return descend(window, matching: element, containing: centre, depth: 0) + } + + private static func descend( + _ node: AXUIElement, + matching element: Element, + containing point: CGPoint, + depth: Int + ) -> AXUIElement? { + if matches(node, element) { return node } + guard depth < maxDepth else { return nil } + for child in elementArray(node, kAXChildrenAttribute) { + if isOverlayWindow(child) || isChrome(child) { continue } + let frame = frameScreen(of: child) + guard frame.width > 0, frame.height > 0, frame.contains(point) else { continue } + if let found = descend(child, matching: element, containing: point, depth: depth + 1) { return found } + } + return nil + } + + /// Identity test for re-finding a captured ``Element``. Identifier AND role AND + /// frame, because none alone is enough: identifiers are absent on the unseeded + /// elements this feature exists to navigate around, roles repeat everywhere, + /// and a coextensive background surface shares a frame with the content group + /// in front of it (the `.axCardSurface` pattern) — matching on frame alone would + /// silently list the wrong node's children. + private static func matches(_ node: AXUIElement, _ element: Element) -> Bool { + guard (string(node, kAXIdentifierAttribute) ?? "") == (element.path.last?.identifier ?? "") else { + return false + } + guard (string(node, kAXRoleAttribute) ?? "") == element.role else { return false } + let frame = frameScreen(of: node) + // Half a point of slack: AX geometry round-trips through CGFloat conversions + // and a strict `==` would make re-finding fail on a fractional layout. + return abs(frame.minX - element.frame.minX) < 0.5 && abs(frame.minY - element.frame.minY) < 0.5 + && abs(frame.width - element.frame.width) < 0.5 && abs(frame.height - element.frame.height) < 0.5 + } + + /// The nearest MEANINGFUL descendants of `node`: each direct child that is a + /// target in its own right, and for each child that is not, the meaningful + /// nodes beneath it. + /// + /// Descending through unmeaningful wrappers is load-bearing, not thoroughness: + /// a SwiftUI `VStack` materializes as an unidentified, label-less `AXGroup`, so + /// a strict direct-children rule would find one ineligible group under most + /// cards, filter it out, and report every card as a leaf — the Child control + /// would be permanently disabled on exactly the UI it was built for. Each + /// branch stops at its first meaningful node, so this is not a subtree + /// enumeration. + private static func collectNearestMeaningful( + under node: AXUIElement, + windowFrame: CGRect?, + depth: Int, + into found: inout [AXUIElement] + ) { + guard depth < maxDepth else { return } + for child in elementArray(node, kAXChildrenAttribute) { + // Chrome and our own overlay are rejected here as well as by + // ``TargetCandidate/isEligibleMeaningful``: skipping the whole SUBTREE + // matters, because a traffic light's inner glyph groups carry no chrome + // subrole of their own and would otherwise be collected as children. + if isOverlayWindow(child) || isChrome(child) { continue } + let frame = frameScreen(of: child) + let isTarget = frame.width > 0 && frame.height > 0 + && candidate(for: child, windowFrame: windowFrame).isEligibleMeaningful + if isTarget { + found.append(child) + } else { + collectNearestMeaningful(under: child, windowFrame: windowFrame, depth: depth + 1, into: &found) + } + } + } + /// Resolve `point` to a root-first ancestor chain of the deepest host element /// beneath the overlay, or nil when nothing is annotatable (no hit, or a hit /// through window chrome). Shared by ``hitTest(_:)`` and ``componentLadder(for:)``. diff --git a/Sources/AnnotKit/macOS/MacElementSource.swift b/Sources/AnnotKit/macOS/MacElementSource.swift index 03b0852..60eea3b 100644 --- a/Sources/AnnotKit/macOS/MacElementSource.swift +++ b/Sources/AnnotKit/macOS/MacElementSource.swift @@ -40,6 +40,12 @@ extension MacElementSource: ComponentLadderSource { } } +extension MacElementSource: ChildNavigationSource { + public func children(of element: Element, near hint: CGPoint?) -> [Element] { + AXIntrospection.children(of: element, near: hint) + } +} + extension MacElementSource: MarqueeTargetSource { public func marqueeLadder(in rect: CGRect) -> [Element] { AXIntrospection.marqueeLadder(for: rect) diff --git a/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift b/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift index c77a99a..3e55c8d 100644 --- a/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift +++ b/Sources/AnnotKit/macOS/MacViewTreeElementSource.swift @@ -246,6 +246,83 @@ public final class MacViewTreeElementSource: ElementSource, ComponentLadderSourc } } +// MARK: - Child navigation (view -> the view inside it) + +extension MacViewTreeElementSource: ChildNavigationSource { + /// The meaningful subviews of `element`, most-likely-intended first, ordered by + /// the shared pure ``ChildNavigationRule`` so this source descends the same way + /// the AX and iOS sources do. + /// + /// Cost: no whole-tree walk. ``selector(for:)`` rebuilds every window's node + /// tree, which is fine once per capture but not here — this runs on every + /// selection change. The element's frame CENTRE instead gives a containment + /// descent from its window to the live `NSView`, after which only that view's + /// own subtree is visited, each branch stopping at its first meaningful node. + public func children(of element: Element, near hint: CGPoint?) -> [Element] { + let centre = CGPoint(x: element.frame.midX, y: element.frame.midY) + guard let root = Self.marqueeRoot(containing: centre), + let view = Self.descend(root, matching: element, containing: centre, depth: 0) + else { return [] } + + var found: [NSView] = [] + Self.collectNearestMeaningful(under: view, depth: 0, into: &found) + let candidates = found.map { + ChildCandidate(element: Self.candidate(for: $0), frame: Self.screenFrame(of: $0)) + } + return ChildNavigationRule.order(candidates, near: hint).map { Self.element(for: found[$0]) } + } + + /// The live `NSView` behind a public ``Element``, found by descending the + /// containing window along frames that contain the element's own centre — + /// bounded by tree DEPTH, not tree size. `hitTest` is deliberately not reused: + /// it honours `isHidden`/`hitTest` overrides and would refuse to re-find a + /// non-interactive view the user has already legitimately selected. + private static func descend( + _ view: NSView, matching element: Element, containing point: CGPoint, depth: Int + ) -> NSView? { + if matches(view, element) { return view } + guard depth < maxDepth else { return nil } + for subview in view.subviews where !subview.isHidden && subview.alphaValue > 0.01 { + let frame = screenFrame(of: subview) + guard frame.width > 0, frame.height > 0, frame.contains(point) else { continue } + if let found = descend(subview, matching: element, containing: point, depth: depth + 1) { return found } + } + return nil + } + + /// Identity test for re-finding a captured ``Element``: identifier, view type, + /// and frame together. See the AX source's `matches` for why none of the three + /// is sufficient alone. + private static func matches(_ view: NSView, _ element: Element) -> Bool { + guard view.accessibilityIdentifier() == (element.path.last?.identifier ?? "") else { return false } + guard String(describing: Swift.type(of: view)) == element.role else { return false } + let frame = screenFrame(of: view) + // Half a point of slack — frames round-trip through a screen-space flip, and + // a strict `==` would make re-finding fail on a fractional layout. + return abs(frame.minX - element.frame.minX) < 0.5 && abs(frame.minY - element.frame.minY) < 0.5 + && abs(frame.width - element.frame.width) < 0.5 && abs(frame.height - element.frame.height) < 0.5 + } + + /// The nearest MEANINGFUL descendants of `view`: each visible subview that is a + /// target in its own right, and for each that is not, the meaningful views + /// beneath it. Descending through unmeaningful wrappers is what keeps the Child + /// control alive under a SwiftUI/AppKit host, whose layout containers carry no + /// identifier or label of their own; each branch stops at its first meaningful + /// node, so this is not a subtree enumeration. + private static func collectNearestMeaningful(under view: NSView, depth: Int, into found: inout [NSView]) { + guard depth < maxDepth else { return } + for subview in view.subviews where !subview.isHidden && subview.alphaValue > 0.01 { + let frame = screenFrame(of: subview) + let isTarget = frame.width > 0 && frame.height > 0 && candidate(for: subview).isEligibleMeaningful + if isTarget { + found.append(subview) + } else { + collectNearestMeaningful(under: subview, depth: depth + 1, into: &found) + } + } + } +} + // MARK: - Marquee (drawn frame -> view) extension MacViewTreeElementSource: MarqueeTargetSource { diff --git a/Tests/AnnotKitTests/AnnotationSessionTests.swift b/Tests/AnnotKitTests/AnnotationSessionTests.swift index e269d20..585dcb6 100644 --- a/Tests/AnnotKitTests/AnnotationSessionTests.swift +++ b/Tests/AnnotKitTests/AnnotationSessionTests.swift @@ -47,8 +47,8 @@ private final class EmptyWithAnchorSource: ElementSource, RegionAnchorSource { } } -/// A source that hit-tests to a leaf and exposes a widening ladder (leaf, then -/// enclosing components), driving selection-widening tests. +/// A source that hit-tests to a leaf and exposes a component ladder (leaf, then +/// enclosing components), driving upward-navigation tests. @MainActor private final class LadderSource: ElementSource, ComponentLadderSource { let ladder: [Element] @@ -62,6 +62,39 @@ private final class LadderSource: ElementSource, ComponentLadderSource { } } +/// A ladder source that ALSO answers child queries, so one test can drive a +/// selection both up and down the tree. Records every query so a test can prove +/// the session cached instead of re-asking (and that a region never asks at all). +@MainActor +private final class NavigableSource: ElementSource, ComponentLadderSource, ChildNavigationSource, RegionAnchorSource { + let ladder: [Element] + let anchor: Element? + /// Children keyed by the parent's id, so a test can shape a whole subtree — + /// and swap a branch mid-test to prove a re-descent does NOT re-query. + var childrenByID: [String: [Element]] + private(set) var childQueries: [String] = [] + private(set) var lastHint: CGPoint? + + init(ladder: [Element], childrenByID: [String: [Element]] = [:], anchor: Element? = nil) { + self.ladder = ladder + self.childrenByID = childrenByID + self.anchor = anchor + } + func snapshot() -> [WindowSnapshot] { [] } + func hitTest(_ point: CGPoint) -> Element? { ladder.first } + func componentLadder(at point: CGPoint) -> [Element] { ladder } + func regionAnchor(at point: CGPoint) -> Element? { anchor } + func children(of element: Element, near hint: CGPoint?) -> [Element] { + childQueries.append(element.id) + lastHint = hint + return childrenByID[element.id] ?? [] + } + func selector(for element: Element) -> String { "#\(element.id)" } + func screenshot(of element: Element?) async throws -> CapturedImage { + CapturedImage(pngData: Data(), pixelWidth: 1, pixelHeight: 1) + } +} + /// A source that serves a marquee ladder AND a point hit-test (and optionally a /// region anchor), so one test can interleave drags and clicks — the mixed flow /// the catcher actually produces, and the one that surfaces stale per-selection @@ -103,42 +136,117 @@ final class AnnotationSessionTests: XCTestCase { ) } - func testWidenSelectionStepsUpTheComponentLadder() { + func testSelectParentStepsUpTheSelectionPath() { let ladder = [makeLadderElement("Leaf"), makeLadderElement("Settings.Models"), makeLadderElement("Settings")] let session = AnnotationSession(source: LadderSource(ladder: ladder), sink: NotesFileSink(path: "/dev/null")) session.start() session.select(atAXPoint: .zero) XCTAssertEqual(session.selected?.id, "Leaf", "selection starts at the hit-test target") - XCTAssertTrue(session.canWidenSelection) + XCTAssertTrue(session.canSelectParent) + XCTAssertFalse(session.canSelectChild, "nothing below the deepest rung, and no history yet") + + XCTAssertEqual(session.selectParent()?.id, "Settings.Models", "parent -> enclosing component") + XCTAssertTrue(session.canSelectParent) + XCTAssertEqual(session.selectParent()?.id, "Settings", "parent -> broader component") + XCTAssertFalse(session.canSelectParent, "no stepping past the broadest rung") + XCTAssertNil(session.selectParent(), "parent at the top is a no-op") + } + + /// The overshoot case the one-way Widen button could not undo: every upward + /// step must be walkable back down through HISTORY, with no source involved. + func testSelectChildWalksBackDownThroughHistory() { + let ladder = [makeLadderElement("Leaf"), makeLadderElement("Settings.Models"), makeLadderElement("Settings")] + let source = NavigableSource(ladder: ladder) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + session.selectParent() + session.selectParent() + XCTAssertEqual(session.selected?.id, "Settings") + XCTAssertTrue(session.canSelectChild, "an ascended selection can always come back down") + + XCTAssertEqual(session.selectChild()?.id, "Settings.Models", "child -> back down one rung") + XCTAssertEqual(session.selectChild()?.id, "Leaf", "child -> back to the original target") + XCTAssertFalse(session.canSelectChild, "the leaf has no children and no history left") + XCTAssertNil(session.selectChild(), "child at a childless deepest rung is a no-op") + } + + /// Below the deepest KNOWN rung there is no history, so the source is asked — + /// and the answer is PREPENDED, keeping index 0 "the deepest known rung" and + /// leaving the original target reachable with Parent. + func testSelectChildDescendsBelowTheDeepestRungAndKeepsTheParentReachable() { + let target = makeLadderElement("Card") + let row = makeLadderElement("Card.Row") + let source = NavigableSource(ladder: [target, makeLadderElement("Page")], + childrenByID: ["Card": [row]]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: CGPoint(x: 7, y: 9)) + XCTAssertEqual(source.lastHint, CGPoint(x: 7, y: 9), "the click point is handed on as the ordering hint") + XCTAssertTrue(session.canSelectChild, "a cached child enables descent") + + XCTAssertEqual(session.selectChild()?.id, "Card.Row", "child -> the source's most likely child") + XCTAssertTrue(session.canSelectParent, "the original target is still one rung up") + XCTAssertEqual(session.selectParent()?.id, "Card", "parent returns to the original target") + XCTAssertEqual(session.selectParent()?.id, "Page", "and keeps climbing the original ladder") + } + + /// The reason descent PREPENDS instead of re-querying: under a live UI the + /// source's "most likely child" can change between presses, so a re-query would + /// make Parent-then-Child land somewhere the user never chose. + func testReDescentReplaysHistoryRatherThanReQueryingTheSource() { + let target = makeLadderElement("Card") + let source = NavigableSource(ladder: [target], childrenByID: ["Card": [makeLadderElement("Card.Row")]]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + XCTAssertEqual(session.selectChild()?.id, "Card.Row") - XCTAssertEqual(session.widenSelection()?.id, "Settings.Models", "widen -> enclosing component") - XCTAssertTrue(session.canWidenSelection) - XCTAssertEqual(session.widenSelection()?.id, "Settings", "widen -> broader component") - XCTAssertFalse(session.canWidenSelection, "no widening past the broadest rung") - XCTAssertNil(session.widenSelection(), "widen at the top is a no-op") + // The UI moves on: the same query would now answer differently. + source.childrenByID["Card"] = [makeLadderElement("Card.Chevron")] + XCTAssertEqual(session.selectParent()?.id, "Card") + XCTAssertEqual(session.selectChild()?.id, "Card.Row", + "the round trip returns to the child actually visited, not to a fresh guess") } - func testWideningResetsOnNewSelectionAndCapture() { + func testNavigationResetsOnNewSelectionAndCapture() { let ladder = [makeLadderElement("Leaf"), makeLadderElement("Card")] let session = AnnotationSession(source: LadderSource(ladder: ladder), sink: NotesFileSink(path: "/dev/null")) session.start() session.select(atAXPoint: .zero) - session.widenSelection() + session.selectParent() XCTAssertEqual(session.selected?.id, "Card") - // A fresh selection restarts the ladder at the leaf. + // A fresh selection restarts the path at the leaf — including the history + // that would otherwise let Child step down into the PREVIOUS selection. session.select(atAXPoint: .zero) XCTAssertEqual(session.selected?.id, "Leaf") - XCTAssertTrue(session.canWidenSelection) - // Capturing clears the ladder (no widening with nothing selected). + XCTAssertTrue(session.canSelectParent) + XCTAssertFalse(session.canSelectChild, "a new selection carries no navigation history") + // Capturing clears the path (nothing to navigate with nothing selected). session.addNote(comment: "note") - XCTAssertFalse(session.canWidenSelection) + XCTAssertFalse(session.canSelectParent) + XCTAssertFalse(session.canSelectChild) + } + + /// `canSelectChild` is read on every SwiftUI render, so it must answer from the + /// cache. One query per bound-element change is the budget; a per-render walk + /// would be a serious regression. + func testChildAvailabilityIsAnsweredFromCacheNotTheSource() { + let source = NavigableSource(ladder: [makeLadderElement("Card")], + childrenByID: ["Card": [makeLadderElement("Card.Row")]]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + XCTAssertEqual(source.childQueries, ["Card"], "exactly one query when the selection is made") + for _ in 0 ..< 50 { _ = session.canSelectChild } + XCTAssertEqual(source.childQueries, ["Card"], "reading the property must never reach the source") } /// An unseeded target whose enclosing card is only reachable via the GEOMETRIC - /// ladder (a `.axCardSurface` sibling, not an ancestor) still gets its - /// `component` set to that card — the ladder's tightest enclosing entry, not + /// path (a `.axCardSurface` sibling, not an ancestor) still gets its + /// `component` set to that card — the path's tightest enclosing entry, not /// the target's ancestry. - func testUnseededTargetTakesComponentFromGeometricLadder() { + func testUnseededTargetTakesComponentFromGeometricPath() { // Leaf with no identifier of its own; the card is a separate ladder entry // (as a sibling surface would be), not in the leaf's path. let leaf = Element( @@ -162,7 +270,62 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertEqual(note?.elementText, "9:00 Standup") } - func testRegionSelectionCannotWiden() { + /// Descending below an UNSEEDED target must not hand the agent a grep target + /// that matches nothing. + /// + /// This pins the bug prepending created. `component` used to be + /// `path.dropFirst().first?.id` — index 0 is the target, index 1 is its + /// enclosing component — which held only while the path was strictly upward, + /// because every upward rung is seeded by construction. Insert a child at 0 + /// and index 1 becomes the ORIGINAL TARGET, which may carry no identifier at + /// all; an unseeded `Element.id` is a slash-joined path string, so the note + /// would have exported `#AXWindow[0]/AXGroup[0]` as the thing to grep for and + /// looked entirely plausible doing it. The derivation must skip to the first + /// SEEDED rung above the bound one. + func testDescendingBelowAnUnseededTargetStillNamesASeededComponent() { + let unseededTarget = Element( + id: "AXWindow[0]/AXGroup[0]", role: "AXGroup", type: "AXGroup", + label: "", value: "", frame: CGRect(x: 0, y: 0, width: 40, height: 40), + isVisible: true, isActionable: false, + path: [ + PathComponent(role: "AXWindow", label: "", identifier: nil, indexAmongRole: 0), + PathComponent(role: "AXGroup", label: "", identifier: nil, indexAmongRole: 0), + ] + ) + let child = Element( + id: "AXWindow[0]/AXGroup[0]/AXStaticText[0]", role: "AXStaticText", type: "AXStaticText", + label: "", value: "Connect", frame: CGRect(x: 4, y: 4, width: 20, height: 10), + isVisible: true, isActionable: false, + path: [ + PathComponent(role: "AXWindow", label: "", identifier: nil, indexAmongRole: 0), + PathComponent(role: "AXGroup", label: "", identifier: nil, indexAmongRole: 0), + PathComponent(role: "AXStaticText", label: "", identifier: nil, indexAmongRole: 0), + ] + ) + let seededCard = makeLadderElement("Dashboard.FirstRun") + let source = NavigableSource( + ladder: [unseededTarget, seededCard], + childrenByID: [unseededTarget.id: [child]] + ) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + XCTAssertEqual(session.selectChild()?.id, child.id, "descends below the unseeded target") + + let note = session.addNote(comment: "wrong copy here") + XCTAssertEqual(note?.component, "Dashboard.FirstRun", + "skips the unseeded target to the first SEEDED rung above the bound one") + XCTAssertNotEqual(note?.component, unseededTarget.id, + "a slash-path id must never be exported as the component to grep") + XCTAssertFalse(note?.component?.contains("/") ?? false, "no component is ever a path string") + XCTAssertEqual(note?.unseeded, true, "the bound child carries no identifier of its own") + } + + /// A region is a synthetic marker, not a node in anyone's tree, so BOTH + /// navigation controls stay disabled. The empty `selectionPath` is what + /// guarantees it: `canSelectChild` refuses to query a source for the children + /// of an element that does not exist in the hierarchy. + func testRegionSelectionCannotNavigate() { let anchor = makeElement() let session = AnnotationSession( source: EmptyWithAnchorSource(anchor: anchor), sink: NotesFileSink(path: "/dev/null") @@ -170,7 +333,8 @@ final class AnnotationSessionTests: XCTestCase { session.start() session.select(atAXPoint: CGPoint(x: 5, y: 5)) XCTAssertEqual(session.selected?.role, "AXRegion") - XCTAssertFalse(session.canWidenSelection, "region notes have no widening ladder") + XCTAssertFalse(session.canSelectParent, "a region has no selection path to climb") + XCTAssertFalse(session.canSelectChild, "and none to descend into") } private func makeElement() -> Element { @@ -240,7 +404,7 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertEqual(session.select(inAXRect: drawn)?.id, "Card", "a marquee binds to the ladder's first rung") XCTAssertEqual(session.selectedMarqueeRect, drawn, "the drawn frame is kept absolute") XCTAssertNil(session.selectedRegionOffset, "a framed note carries no point locator") - XCTAssertTrue(session.canWidenSelection, "the marquee ladder drives widening like the point ladder") + XCTAssertTrue(session.canSelectParent, "the marquee ladder drives navigation like the point ladder") } func testMarqueeNormalizesABackwardsDrag() { @@ -296,7 +460,7 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertNotNil(note?.regionRect) } - func testWideningAfterAMarqueeReRelativizesTheFrame() { + func testSelectParentAfterAMarqueeReRelativizesTheFrame() { // The frame is stored ABSOLUTE precisely so widening stays correct: the // note rebinds to a bigger element with a different origin AFTER the drag, // so the persisted rect must be measured against the widened element. A @@ -309,7 +473,7 @@ final class AnnotationSessionTests: XCTestCase { session.start() let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) session.select(inAXRect: drawn) - XCTAssertEqual(session.widenSelection()?.id, "Card") + XCTAssertEqual(session.selectParent()?.id, "Card") XCTAssertEqual(session.selectedMarqueeRect, drawn, "widening keeps the absolute drawn frame") let note = session.addNote(comment: "framed then widened") XCTAssertEqual(note?.selector, "#Card") From 862a1a01539b1b508ef6674469bdf4ec6fae03db Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 19:47:09 -0500 Subject: [PATCH 14/24] feat(frame-mode): no hover highlight, and the drawn frame is the anchor Frame mode was only half a mode: underneath, it still behaved like point mode. Two distinct wrongs, both visible in one dogfooding screenshot of a whole dashboard card lit up with its name tag while the user was in frame mode and had drawn nothing. 1. During the gesture it hover-highlighted. `hover(atAXPoint:)` gated only on `mode == .annotating`, so every pointer motion in frame mode ran an AX hit-test and advertised a point-selection that mode can never make. Now gated on `tool == .point` in the SESSION (the view keeps its narrower during-the-drag guard as defence in depth), which also removes one cross-process AX query per motion event. `setTool` additionally clears `hovered`, so a highlight resolved in point mode no longer survives the switch; the rest of its "touch nothing else" contract -- open composer, selection, pending notes -- is unchanged. 2. After the gesture the drawn frame was thrown away visually: every anchor derived from `selected.frame`, so the highlight snapped from the swept rectangle to the resolved element and the composer and pin followed it. `selectionAnchorFrame` now exposes the drawn rect (absolute AX coordinates, like `Element.frame`) while it is still the truth of the selection, and the highlight, `composerPlacement` and the pin anchor all prefer it. The committed frame renders SOLID (the in-progress band stays dashed) with no name tag; the resolved element is named in the composer header instead ("Frame -> Dashboard.Today"), so a note is never captured against a target the user could not see, and no second rectangle appears. Pressing Parent/Child is the user explicitly asking WHICH element, so the anchor drops to that element and the drawn frame stays on screen dimmed beneath it -- it is still what the note records. The anchoring flag is cleared at the top of both `select` paths, in the `selected` didSet nil clear, and in `bindCurrentRung` (the single funnel both navigation directions pass through), because a frame -> navigate -> click sequence that missed any one of them would anchor the next note to a stale rectangle. The region-fallback branch deliberately gets no flag: its synthetic element's frame IS the drawn rect. The note payload is untouched -- `regionRect` plus `selector`/`component` round-trip exactly as before, no new fields. Co-Authored-By: Claude Opus 5 (1M context) --- .../AnnotKit/Overlay/AnnotationSession.swift | 96 ++++++++-- Sources/AnnotKit/Overlay/OverlayView.swift | 167 +++++++++++----- .../AnnotationSessionTests.swift | 179 +++++++++++++++++- 3 files changed, 378 insertions(+), 64 deletions(-) diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index 3b31bf1..028ce48 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -44,15 +44,16 @@ public final class AnnotationSession: ObservableObject { @Published public private(set) var pending: [AnnotationNote] = [] @Published public private(set) var hovered: Element? @Published public private(set) var selected: Element? { - // The region offset, the drawn marquee frame, and the navigation path only - // make sense while their selection is alive; clearing the selection - // (capture, cancel, stop, pin editing) must never leave a stale offset, - // frame, or path for the NEXT note. + // The region offset, the drawn marquee frame, the frame-anchoring flag and + // the navigation path only make sense while their selection is alive; + // clearing the selection (capture, cancel, stop, pin editing) must never + // leave a stale offset, frame, anchor or path for the NEXT note. didSet { if selected == nil { selectedRegionOffset = nil selectedMarqueeRect = nil marqueeRegionOrigin = nil + anchorsToDrawnFrame = false selectionPath = [] pathIndex = 0 cachedChildren = [] @@ -77,6 +78,34 @@ public final class AnnotationSession: ObservableObject { /// selections, which measure from the selected element's origin. private var marqueeRegionOrigin: CGPoint? + /// Whether the overlay should anchor to the frame the user DREW rather than to + /// the element that frame resolved to. Set ONLY where a drag resolves to a real + /// element — the one branch where the drawn rectangle would otherwise VANISH on + /// release, the highlight snapping to a box the user never swept. The region + /// fallback deliberately has no flag: its synthetic element's `frame` IS the + /// drawn rect, so anchoring to `selected.frame` there already anchors to the + /// frame, and a second mechanism would be two sources of truth for one rect. + /// + /// Like ``selectedMarqueeRect``, deliberately NOT `@Published`: it is only ever + /// mutated alongside an assignment to `selected`, which IS published, so every + /// change is delivered to the UI by that emission. A future mutation that does + /// NOT coincide with a `selected` assignment would change the anchor without + /// redrawing — the composer would stay glued to the wrong rectangle. + private var anchorsToDrawnFrame = false + + /// The rect the overlay should anchor to: the frame the user DREW, while it is + /// still the truth of the selection; nil once the binding is an element the user + /// chose (a click, or a navigated frame selection), in which case the UI anchors + /// to `selected.frame` as it always has. + /// + /// ABSOLUTE AX screen coordinates — the same space as ``Element/frame`` — so + /// every consumer applies the identical `- axOrigin` transform it already + /// applies to the selected element. Deliberately not a new coordinate space: + /// the in-progress band is window-local and the two would be trivially + /// confusable, which on a secondary display means a frame drawn in the right + /// place and rendered off by the window's screen origin. + public var selectionAnchorFrame: CGRect? { anchorsToDrawnFrame ? selectedMarqueeRect : nil } + /// The BIDIRECTIONAL navigation path for the current selection, and the index /// of the bound rung. Ordering is a CONVENTION the whole file depends on: /// index 0 is the deepest rung known so far, ascending indices are @@ -151,13 +180,23 @@ public final class AnnotationSession: ObservableObject { public func start() { mode = .annotating } - /// Switch which gesture the catcher interprets. Deliberately touches NOTHING - /// else: an open composer, the current selection and the retained notes all - /// survive, because changing how you will pick the NEXT target says nothing - /// about the note you are in the middle of writing. Discarding a half-typed - /// comment because the user reached for the other tool would be the kind of - /// data loss nobody reports — they just stop using the picker. - public func setTool(_ tool: SelectionTool) { self.tool = tool } + /// Switch which gesture the catcher interprets. Deliberately touches nothing + /// beyond the hover highlight: an open composer, the current selection and the + /// retained notes all survive, because changing how you will pick the NEXT + /// target says nothing about the note you are in the middle of writing. + /// Discarding a half-typed comment because the user reached for the other tool + /// would be the kind of data loss nobody reports — they just stop using the + /// picker. + /// + /// `hovered` is the ONE exception, and it is not housekeeping: hover is + /// point-mode-only, so a highlight resolved a moment before the switch would + /// otherwise SURVIVE into frame mode and sit there — a lit-up element with its + /// name tag, advertising a click-selection frame mode will never make — until + /// the pointer happened to leave the catcher. That is the reported symptom. + public func setTool(_ tool: SelectionTool) { + self.tool = tool + hovered = nil + } public func stop() { mode = .idle @@ -171,8 +210,17 @@ public final class AnnotationSession: ObservableObject { /// Update the hover highlight for a screen point (AX top-left coordinates). /// Throttled to ~60fps so rapid hover events do not flood the point query. + /// + /// POINT MODE ONLY. Frame mode makes its selection from a swept rectangle, so a + /// hover highlight there promises a point-selection that no press in that mode + /// can produce — the dogfooding report was a whole dashboard card lit up with + /// its name tag while the user was in frame mode and had drawn nothing. Gated + /// HERE rather than in the view (which has its own guard for the narrower + /// during-the-drag case) so no UI path can reintroduce it, and so it is + /// testable without a window. It is also a real cost, not just a visual one: + /// this skips one cross-process AX hit-test per pointer-motion event. public func hover(atAXPoint point: CGPoint) { - guard mode == .annotating else { return } + guard mode == .annotating, tool == .point else { return } let now = Date() guard now.timeIntervalSince(lastHover) >= hoverInterval else { return } lastHover = now @@ -198,15 +246,18 @@ public final class AnnotationSession: ObservableObject { // Any catcher tap dismisses an open pin editor: a tap on empty space is a // click-away close, and a tap on an element hands the stage to the composer. editingNoteID = nil - // Every selection starts offset-free, frame-free and path-free: a - // region -> element or marquee -> click re-selection (the catcher stays + // Every selection starts offset-free, frame-free, anchor-free and path-free: + // a region -> element or marquee -> click re-selection (the catcher stays // active behind an open composer) must not leak the previous region's // offset, the previous drag's drawn frame, or a stale navigation path onto // the next note — the didSet only clears them when `selected` becomes nil, - // not on replacement. + // not on replacement. `anchorsToDrawnFrame` rides with the frame for the + // same reason: a click that landed after a drag would otherwise place its + // composer and pin against the rectangle drawn for the PREVIOUS note. selectedRegionOffset = nil selectedMarqueeRect = nil marqueeRegionOrigin = nil + anchorsToDrawnFrame = false resetNavigation(hint: point) selected = source.hitTest(point) // Seed the navigation path for a real element selection. The ladder's first @@ -277,6 +328,7 @@ public final class AnnotationSession: ObservableObject { selectedRegionOffset = nil selectedMarqueeRect = nil marqueeRegionOrigin = nil + anchorsToDrawnFrame = false // The hint is set below, once the rect is normalized — a backwards drag's // raw `midX`/`midY` are still correct, but deriving it from the standardized // rect keeps one definition of "the centre of what was drawn". @@ -311,6 +363,11 @@ public final class AnnotationSession: ObservableObject { selectionHint = CGPoint(x: normalized.midX, y: normalized.midY) refreshChildCache() selectedMarqueeRect = normalized + // The frame is the truth of THIS selection until the user says + // otherwise: they drew a box, so the box is what the overlay + // anchors to. Set only here — the region branch below needs no + // flag, because its synthetic element's frame already IS this rect. + anchorsToDrawnFrame = true return selected } } @@ -407,6 +464,15 @@ public final class AnnotationSession: ObservableObject { // rect from the wrong box, and that is exactly the class of bug `7993a67` // was. marqueeRegionOrigin = nil + // Pressing Parent or Child is the user EXPLICITLY asking which element the + // note is filed against, so the answer stops being implied and becomes the + // anchor: dropping this flag re-points the highlight, composer and pin at + // the chosen element (the drawn frame stays on screen, dimmed, because it + // is still what the note records). Clearing here rather than in the two + // callers is deliberate — every navigation in both directions funnels + // through this method, so the two can never drift into clearing different + // state, which is exactly how a stale-anchor bug gets in. + anchorsToDrawnFrame = false // `selectedMarqueeRect` is deliberately KEPT: it is absolute, so it is // still the frame the user drew whichever rung is now bound, and `addNote` // re-relativizes it against that rung. diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index 81fd8eb..7dd4cfa 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -56,47 +56,7 @@ struct OverlayView: View { ZStack(alignment: .topLeading) { catcher - // The band REPLACES the element highlight while a frame is being drawn - // (same ZStack slot: above the catcher, below the chrome). Showing both - // would put a solid "this is what you get" highlight under a rectangle - // that has not resolved to anything yet. - if let marqueeRect { - // COORDINATES: `marqueeRect` is already window-local, so it is - // offset DIRECTLY — no `axOrigin` subtraction, unlike the element - // highlight one branch down, which arrives in AX screen space. - // Subtracting here "for symmetry" would slide the band off by the - // window's screen origin, invisible on the primary display at the - // global origin and badly wrong on every secondary display. - RoundedRectangle(cornerRadius: 3) - .stroke(Color.accentColor, style: StrokeStyle(lineWidth: 2, dash: [6, 4])) - .background(Color.accentColor.opacity(0.08)) - .frame(width: marqueeRect.width, height: marqueeRect.height) - .offset(x: marqueeRect.minX, y: marqueeRect.minY) - .allowsHitTesting(false) - } else if let element = session.selected ?? session.hovered { - let originX = element.frame.minX - axOrigin.x - let originY = element.frame.minY - axOrigin.y - RoundedRectangle(cornerRadius: 3) - .stroke(Color.accentColor, lineWidth: 2) - .background(Color.accentColor.opacity(0.12)) - .frame(width: element.frame.width, height: element.frame.height) - .offset(x: originX, y: originY) - .allowsHitTesting(false) - // Name tag: shows WHICH element a click binds to, so an element and - // its enclosing card (which look alike as bare rectangles) are - // distinguishable at a glance. Sits just above the highlight, or - // just inside its top when the element hugs the window's top edge. - Text(highlightName(element)) - .font(.caption2.weight(.medium)) - .lineLimit(1) - .foregroundStyle(.white) - .padding(.horizontal, 6) - .padding(.vertical, 2) - .background(Capsule().fill(Color.accentColor)) - .fixedSize() - .offset(x: originX, y: originY - 20 < 0 ? originY + 2 : originY - 20) - .allowsHitTesting(false) - } + highlightLayer // Numbered comment pins (annotate-mode-only chrome). Layered ABOVE // the catcher and BELOW the composer/toolbar, so each pin consumes its @@ -129,6 +89,92 @@ struct OverlayView: View { } } + /// What is drawn ON the app, between the catcher and the chrome. Four states, + /// and keeping them in ONE mutually-exclusive ladder is the point: each earlier + /// case is a stronger claim about what the next press/release binds to, and any + /// two of them on screen at once would show the user two different answers. + /// + /// 1. **Drawing** — the dashed band, frame-mode-only, window-local. + /// 2. **Committed frame** — the rect the user just swept, drawn SOLID (so + /// "still drawing" and "done" are distinguishable at a glance) and with NO + /// name tag: the note's bind target is named in the composer header instead, + /// because a tag here would put a second, differently-shaped claim next to + /// the rectangle the user actually drew. See ``composerHeader``. + /// 3. **Navigated** — the user pressed Parent/Child, so they have explicitly + /// asked WHICH element: the element is highlighted and named, and the drawn + /// frame stays beneath it, DIMMED, because it is still what the note records + /// (`regionRect`) even though it no longer decides the binding. + /// 4. **Point selection / hover** — the original element highlight. + @ViewBuilder + private var highlightLayer: some View { + // The band REPLACES everything else while a frame is being drawn. Showing a + // solid "this is what you get" highlight under a rectangle that has not + // resolved to anything yet would promise a binding the release may not make. + if let marqueeRect { + // COORDINATES: `marqueeRect` is already window-local, so it is offset + // DIRECTLY — no `axOrigin` subtraction, unlike every other case here, + // which arrives in AX screen space. Subtracting here "for symmetry" + // would slide the band off by the window's screen origin, invisible on + // the primary display at the global origin and badly wrong on every + // secondary display. + RoundedRectangle(cornerRadius: 3) + .stroke(Color.accentColor, style: StrokeStyle(lineWidth: 2, dash: [6, 4])) + .background(Color.accentColor.opacity(0.08)) + .frame(width: marqueeRect.width, height: marqueeRect.height) + .offset(x: marqueeRect.minX, y: marqueeRect.minY) + .allowsHitTesting(false) + } else if let frame = session.selectionAnchorFrame { + // The committed frame. It deliberately does NOT also draw the resolved + // element: the user drew a box and asked to see that box, and the whole + // reported bug was the highlight snapping to a card they never swept. + RoundedRectangle(cornerRadius: 3) + .stroke(Color.accentColor, lineWidth: 2) + .background(Color.accentColor.opacity(0.12)) + .frame(width: frame.width, height: frame.height) + .offset(x: frame.minX - axOrigin.x, y: frame.minY - axOrigin.y) + .allowsHitTesting(false) + } else { + // The drawn frame AFTER navigation moved the binding off it. Drawn + // first so it sits BENEATH the element highlight, and much weaker than + // case 2, so "this is what the note records" cannot be misread as "this + // is what the note binds to". Skipped when it coincides with the + // element's own frame — a region-fallback selection, whose synthetic + // element IS the drawn rect, would otherwise stack two strokes on one + // edge and read as a rendering glitch. + if let drawn = session.selectedMarqueeRect, drawn != session.selected?.frame { + RoundedRectangle(cornerRadius: 3) + .stroke(Color.accentColor.opacity(0.35), style: StrokeStyle(lineWidth: 1, dash: [4, 3])) + .frame(width: drawn.width, height: drawn.height) + .offset(x: drawn.minX - axOrigin.x, y: drawn.minY - axOrigin.y) + .allowsHitTesting(false) + } + if let element = session.selected ?? session.hovered { + let originX = element.frame.minX - axOrigin.x + let originY = element.frame.minY - axOrigin.y + RoundedRectangle(cornerRadius: 3) + .stroke(Color.accentColor, lineWidth: 2) + .background(Color.accentColor.opacity(0.12)) + .frame(width: element.frame.width, height: element.frame.height) + .offset(x: originX, y: originY) + .allowsHitTesting(false) + // Name tag: shows WHICH element a click binds to, so an element and + // its enclosing card (which look alike as bare rectangles) are + // distinguishable at a glance. Sits just above the highlight, or + // just inside its top when the element hugs the window's top edge. + Text(highlightName(element)) + .font(.caption2.weight(.medium)) + .lineLimit(1) + .foregroundStyle(.white) + .padding(.horizontal, 6) + .padding(.vertical, 2) + .background(Capsule().fill(Color.accentColor)) + .fixedSize() + .offset(x: originX, y: originY - 20 < 0 ? originY + 2 : originY - 20) + .allowsHitTesting(false) + } + } + } + /// The label shown on the hover/selection highlight so the user can see WHICH /// element a click will bind to — an element vs its enclosing card, which are /// otherwise indistinguishable as bare rectangles. Prefers the seeded @@ -232,7 +278,7 @@ struct OverlayView: View { /// element, with a Cancel / Add note footer. Enter submits, Escape cancels. private var composer: some View { AnnotationCard( - header: session.selectionLabel ?? "Element", + header: composerHeader, text: $comment, placement: composerPlacement, // Re-focus when the selection moves element→element (the shared card @@ -294,6 +340,19 @@ struct OverlayView: View { } } + /// The composer's header. For a committed FRAME selection this is the only + /// place on screen that names the element the note is filed against — the + /// canvas deliberately shows the drawn rectangle and no name tag — so it must + /// not read as a label for the rectangle itself. The `Frame →` prefix says the + /// name is what the frame RESOLVED to, which is the one thing a user cannot + /// otherwise verify before pressing Add note, and it disappears the moment + /// Parent/Child re-anchors to a named element on the canvas, so the name is + /// never qualified in two places at once. + private var composerHeader: String { + let label = session.selectionLabel ?? "Element" + return session.selectionAnchorFrame != nil ? "Frame → \(label)" : label + } + /// The EDIT card: the SAME shared ``AnnotationCard`` chrome as the composer, /// anchored to the tapped pin instead of an element, with a Delete / Save /// footer. Enter saves, Escape cancels, and Save/Delete/click-away all end @@ -377,14 +436,19 @@ struct OverlayView: View { } /// Capture the pending note. Snapshots the pin anchor BEFORE `addNote` clears - /// the selection (element AX top-left minus axOrigin — the same window-local - /// transform the highlight uses — so the pin lands on the element's top-left - /// corner), then resets the field. Enter submits; Shift+Enter inserts a + /// the selection (AX top-left minus axOrigin — the same window-local transform + /// the highlight uses — so the pin lands on the top-left corner of whatever was + /// highlighted), then resets the field. Enter submits; Shift+Enter inserts a /// newline (handled in the field's `onKeyPress`). + /// + /// The anchor follows the SAME rect the highlight and composer used: pinning a + /// framed note to the resolved element instead would drop the numbered pin on a + /// corner the user never swept — possibly far outside the frame, for a frame + /// that resolved to a large enclosing card. private func addNote() { guard !comment.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } - let anchor = session.selected.map { - CGPoint(x: $0.frame.minX - axOrigin.x, y: $0.frame.minY - axOrigin.y) + let anchor = (session.selectionAnchorFrame ?? session.selected?.frame).map { + CGPoint(x: $0.minX - axOrigin.x, y: $0.minY - axOrigin.y) } session.addNote(comment: comment, anchor: anchor) comment = "" @@ -400,8 +464,15 @@ struct OverlayView: View { /// and its caret. See ``ComposerPlacement``. On the primary display at the /// global origin `axOrigin == .zero`, so the previously-working case (card /// directly below the element) is unchanged. + /// + /// Anchored to the DRAWN frame while that is the truth of the selection, so the + /// card points at the rectangle the user swept rather than at the element it + /// resolved to (which may be much larger, and elsewhere). ``ComposerPlacement`` + /// already clamps both axes and flips the card when it would spill, so a frame + /// larger than any element — even one spanning the whole window — still places + /// sanely; feeding it the frame is the entire change. private var composerPlacement: ComposerPlacement { - guard let f = session.selected?.frame else { + guard let f = session.selectionAnchorFrame ?? session.selected?.frame else { return ComposerPlacement(origin: CGPoint(x: 24, y: 24), caretPointsUp: true, caretDX: 0) } return ComposerPlacement.resolve( diff --git a/Tests/AnnotKitTests/AnnotationSessionTests.swift b/Tests/AnnotKitTests/AnnotationSessionTests.swift index 585dcb6..fe36c5d 100644 --- a/Tests/AnnotKitTests/AnnotationSessionTests.swift +++ b/Tests/AnnotKitTests/AnnotationSessionTests.swift @@ -4,12 +4,20 @@ import XCTest @testable import AnnotKit /// A canned ElementSource so the session can be tested without a window. +/// +/// Counts hit-tests, because "no highlight appeared" is a weaker assertion than the +/// behaviour actually wanted: a suppressed highlight that still ran the query would +/// pass it while burning one cross-process AX round trip per pointer-motion event. @MainActor private final class StubSource: ElementSource { let element: Element + private(set) var hitTests = 0 init(_ element: Element) { self.element = element } func snapshot() -> [WindowSnapshot] { [] } - func hitTest(_ point: CGPoint) -> Element? { element } + func hitTest(_ point: CGPoint) -> Element? { + hitTests += 1 + return element + } func selector(for element: Element) -> String { "#\(element.id)" } func screenshot(of element: Element?) async throws -> CapturedImage { CapturedImage(pngData: Data(), pixelWidth: 1, pixelHeight: 1) @@ -590,6 +598,175 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertEqual(session.tool, .frame, "re-entering annotate mode keeps the chosen tool") } + /// Frame mode selects from a swept rectangle, so a hover highlight there + /// advertises a point-selection no press in that mode can make — the reported + /// symptom was a whole card lit up with its name tag before anything was drawn. + /// The hit-test count is the load-bearing half: suppressing the highlight while + /// still running the query would leave one cross-process AX round trip per + /// pointer-motion event. + func testHoverIsInertInFrameMode() { + let source = StubSource(makeElement()) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.setTool(.frame) + + session.hover(atAXPoint: .zero) + XCTAssertNil(session.hovered, "frame mode must not resolve a hover highlight") + XCTAssertEqual(source.hitTests, 0, "and must not even ask the source — the AX query is the cost") + + session.setTool(.point) + session.hover(atAXPoint: .zero) + XCTAssertEqual(session.hovered?.id, "SaveButton", "point mode hovers exactly as before") + XCTAssertEqual(source.hitTests, 1) + } + + /// A highlight resolved in point mode must not SURVIVE the switch to frame mode: + /// it would sit there — lit element, name tag, no relation to the next gesture — + /// until the pointer happened to leave the catcher. Everything else about the + /// tool's "touch nothing" contract still holds, because the user may be + /// mid-note when they reach for the other tool. + func testSetToolClearsTheHoverHighlightAndNothingElse() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + session.addNote(comment: "already captured") + session.hover(atAXPoint: .zero) + session.select(atAXPoint: .zero) + XCTAssertEqual(session.hovered?.id, "SaveButton") + + session.setTool(.frame) + XCTAssertNil(session.hovered, "the stale point-mode highlight must go with the tool") + XCTAssertEqual(session.tool, .frame) + XCTAssertEqual(session.selected?.id, "SaveButton", "an open composer survives a tool change") + XCTAssertEqual(session.pending.count, 1, "and so do the retained notes") + } + + // MARK: - Frame anchoring + + /// The other half of the report: the drawn frame was thrown away VISUALLY the + /// moment it resolved, because every anchor derived from `selected.frame`. The + /// frame is the anchor while it is still the truth of the selection — and the + /// note payload is untouched by any of it. + func testMarqueeAnchorsTheOverlayToTheDrawnFrame() { + let leaf = makeLadderElement("Leaf", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let card = makeLadderElement("Card", frame: CGRect(x: 80, y: 60, width: 200, height: 150)) + let session = AnnotationSession( + source: MarqueeSource(ladder: [leaf, card]), sink: NotesFileSink(path: "/dev/null") + ) + session.start() + let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) + session.select(inAXRect: drawn) + + XCTAssertEqual(session.selectionAnchorFrame, drawn, "the overlay anchors to the frame the user drew") + XCTAssertNotEqual(session.selectionAnchorFrame, session.selected?.frame, + "and NOT to the element it resolved to — the whole bug") + + let note = session.addNote(comment: "framed") + XCTAssertEqual(note?.selector, "#Leaf", "the binding is unchanged: still the resolved element") + XCTAssertEqual(note?.component, "Leaf") + XCTAssertEqual(note?.regionRect, CGRect(x: 10, y: 20, width: 40, height: 20), + "and the payload is unchanged: the frame relative to the bound element") + XCTAssertNil(session.selectionAnchorFrame, "capture clears the anchor with the selection") + } + + /// A click carries no drawn frame, so it must never claim one — otherwise the + /// composer would point at a rectangle from an earlier gesture. + func testPointSelectionHasNoAnchorFrame() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + XCTAssertNil(session.selectionAnchorFrame) + } + + /// The region fallback needs NO anchor flag, and this pins that: its synthetic + /// element's own `frame` IS the drawn rect, so `selected.frame` already anchors + /// everything to the frame. A later refactor that "fixed" this branch by + /// setting the flag too would give one rectangle two sources of truth. + func testRegionFallbackFrameAnchorsThroughItsSyntheticElement() { + let anchor = makeElement() + let session = AnnotationSession( + source: EmptyWithAnchorSource(anchor: anchor), sink: NotesFileSink(path: "/dev/null") + ) + session.start() + let drawn = CGRect(x: 22, y: 14, width: 30, height: 12) + session.select(inAXRect: drawn) + + XCTAssertNil(session.selectionAnchorFrame, "no flag here by design") + XCTAssertEqual(session.selected?.frame, drawn, "because the selection's own frame is already the drawn rect") + } + + /// Pressing Parent/Child is the user explicitly asking WHICH element, so the + /// answer becomes visible: the anchor drops to the chosen element. The drawn + /// frame itself stays — it is still what the note records, and the overlay draws + /// it dimmed underneath. + func testNavigationDropsTheFrameAnchorButKeepsTheDrawnFrame() { + let leaf = makeLadderElement("Leaf", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let card = makeLadderElement("Card", frame: CGRect(x: 80, y: 60, width: 200, height: 150)) + let session = AnnotationSession( + source: MarqueeSource(ladder: [leaf, card]), sink: NotesFileSink(path: "/dev/null") + ) + session.start() + let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) + session.select(inAXRect: drawn) + XCTAssertEqual(session.selectionAnchorFrame, drawn) + + XCTAssertEqual(session.selectParent()?.id, "Card") + XCTAssertNil(session.selectionAnchorFrame, "the chosen element becomes the anchor") + XCTAssertEqual(session.selectedMarqueeRect, drawn, "but the note still records the drawn frame") + + XCTAssertEqual(session.selectChild()?.id, "Leaf", "coming back down is still a CHOSEN element") + XCTAssertNil(session.selectionAnchorFrame, "so the anchor stays on the element, not the frame") + XCTAssertEqual(session.selectedMarqueeRect, drawn) + + // The note is unaffected by any of the anchoring: still the bound element's + // selector, still the frame measured against it. + let note = session.addNote(comment: "framed then navigated") + XCTAssertEqual(note?.selector, "#Leaf") + XCTAssertEqual(note?.regionRect, CGRect(x: 10, y: 20, width: 40, height: 20)) + } + + /// The `7993a67` regression, anchor edition: frame -> navigate -> new click. The + /// navigation clears the flag and the click clears the frame, so if either site + /// were missed the next note would place its composer and pin against a + /// rectangle drawn for the previous one. + func testFrameThenNavigateThenClickLeavesNoStaleAnchor() { + let leaf = makeLadderElement("Leaf", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let card = makeLadderElement("Card", frame: CGRect(x: 80, y: 60, width: 200, height: 150)) + let source = MarqueeSource(ladder: [leaf, card]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(inAXRect: CGRect(x: 110, y: 120, width: 40, height: 20)) + session.selectParent() + + source.hit = makeElement() + source.ladder = [] + session.select(atAXPoint: .zero) + XCTAssertEqual(session.selected?.id, "SaveButton", "the click re-selects a real element") + XCTAssertNil(session.selectionAnchorFrame, "no anchor may survive into the next selection") + XCTAssertNil(session.selectedMarqueeRect) + XCTAssertNil(session.addNote(comment: "click after frame and navigate")?.regionRect, + "and the next note carries no trace of the drawn frame") + } + + /// The anchor is per-SELECTION, so every path that ends a selection must drop it + /// — cancel and stop go through the `selected` didSet, which is the one clear + /// site no `select` call can cover. + func testCancellingAndStoppingClearTheFrameAnchor() { + let leaf = makeLadderElement("Leaf", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let session = AnnotationSession( + source: MarqueeSource(ladder: [leaf]), sink: NotesFileSink(path: "/dev/null") + ) + session.start() + let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) + session.select(inAXRect: drawn) + session.cancelSelection() + XCTAssertNil(session.selectionAnchorFrame, "cancelling the composer drops the anchor") + + session.select(inAXRect: drawn) + session.stop() + XCTAssertNil(session.selectionAnchorFrame, "and so does leaving annotate mode") + } + func testClearHoverDropsHighlightButKeepsSelection() { let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) session.start() From d7cc0eb05e1c862dbc62da38872ab2b8c259f219 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 19:50:09 -0500 Subject: [PATCH 15/24] fix(marquee): reject a degenerate drag before touching any state A perfectly axis-aligned drag (dy == 0) clears the gesture layer's travel threshold, so select(inAXRect:) really does receive a zero-height rect in normal use. Rejecting it after the per-selection clears silently stripped an open frame selection's anchor -- re-anchoring a composer mid-typing -- and did so without assigning selected, so the @Published change driving the re-render never fired and the overlay kept stale geometry. Co-Authored-By: Claude Opus 5 (1M context) --- .../AnnotKit/Overlay/AnnotationSession.swift | 42 +++++++++++-------- .../AnnotationSessionTests.swift | 26 ++++++++++++ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index 028ce48..9b69af6 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -318,6 +318,31 @@ public final class AnnotationSession: ObservableObject { @discardableResult public func select(inAXRect rect: CGRect) -> Element? { guard mode == .annotating else { return nil } + + // Normalize and reject a degenerate rect BEFORE touching any state, so a + // drag that framed nothing is a TRUE no-op. + // + // A zero-width or zero-height rect is a click that never moved, not a + // marquee. Returning nil is a DELIBERATE divergence from "fall back to the + // region path when resolution yields nothing": that fallback is for a drag + // that framed nothing annotatable, whereas this is not a drag at all. + // Falling through would anchor a zero-area frame to whatever sits near the + // press and plant a note the user never asked for — worse than nothing, + // because it looks deliberate. This also bails BEFORE consulting the + // source, so a degenerate rect never reaches `MarqueeTargetRule.resolve` + // or `regionAnchor(at:)`. + // + // Bailing FIRST is load-bearing, not tidiness. A perfectly axis-aligned + // drag (dy == 0) clears the gesture layer's travel threshold and arrives + // here with zero height, so this branch IS reachable in normal use. Clear + // per-selection state before the guard and such a drag would silently strip + // an open frame selection's anchor — re-anchoring a composer the user was + // still typing into, and doing it WITHOUT assigning `selected`, so the + // @Published change that drives the re-render never fires and the overlay + // is left rendering stale geometry. + let normalized = rect.standardized + guard normalized.width > 0, normalized.height > 0 else { return nil } + // A drag on the catcher dismisses an open pin editor for the same reason a // tap does: the composer is about to take the stage. editingNoteID = nil @@ -334,23 +359,6 @@ public final class AnnotationSession: ObservableObject { // rect keeps one definition of "the centre of what was drawn". resetNavigation(hint: nil) - // Normalize once: a right-to-left or bottom-to-top drag arrives with - // negative extents. - // - // A zero-width or zero-height rect is a click that never moved, not a - // marquee. Returning nil here is a DELIBERATE divergence from "the caller - // falls back to the region path when resolution yields nothing": that - // fallback is for a drag that framed nothing annotatable, whereas this is - // not a drag at all. Falling through would anchor a zero-area frame to - // whatever happens to sit near the press and plant a note the user never - // asked for — worse than nothing, because it looks deliberate. The gesture - // recognizer routes this case to ``select(atAXPoint:)`` instead (see the - // caller contract above). Note this also bails BEFORE consulting the - // source, so a degenerate rect never reaches `MarqueeTargetRule.resolve` - // or `regionAnchor(at:)`. - let normalized = rect.standardized - guard normalized.width > 0, normalized.height > 0 else { return nil } - if let marqueeSource = source as? MarqueeTargetSource { let ladder = marqueeSource.marqueeLadder(in: normalized) if let target = ladder.first { diff --git a/Tests/AnnotKitTests/AnnotationSessionTests.swift b/Tests/AnnotKitTests/AnnotationSessionTests.swift index fe36c5d..9a8dada 100644 --- a/Tests/AnnotKitTests/AnnotationSessionTests.swift +++ b/Tests/AnnotKitTests/AnnotationSessionTests.swift @@ -427,6 +427,32 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertEqual(session.selectedMarqueeRect, CGRect(x: 110, y: 120, width: 40, height: 20)) } + /// A degenerate drag must be a TRUE no-op, not a partial one. + /// + /// A perfectly axis-aligned drag (dy == 0) clears the gesture layer's travel + /// threshold, so it really does arrive here with zero height — this is normal + /// use, not a synthetic edge case. Rejecting it AFTER the per-selection clears + /// would silently strip an open frame selection's anchor, re-anchoring a + /// composer the user was still typing into, and would do it without assigning + /// `selected`, so the @Published change that drives the re-render never fires + /// and the overlay keeps rendering the old geometry. + func testDegenerateDragLeavesAnOpenFrameSelectionUntouched() { + let card = makeLadderElement("Card", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let source = MarqueeSource(ladder: [card]) + let session = AnnotationSession(source: source, sink: NotesFileSink(path: "/dev/null")) + session.start() + let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) + session.select(inAXRect: drawn) + XCTAssertEqual(session.selectionAnchorFrame, drawn, "the frame is the anchor before the stray drag") + + // A long, perfectly horizontal drag: passes the travel threshold, zero height. + XCTAssertNil(session.select(inAXRect: CGRect(x: 200, y: 300, width: 80, height: 0))) + + XCTAssertEqual(session.selected?.id, "Card", "the open selection survives") + XCTAssertEqual(session.selectionAnchorFrame, drawn, "and so does its frame anchor") + XCTAssertEqual(session.selectedMarqueeRect, drawn, "the note's record of the drag is intact") + } + func testMarqueeThenClickDropsTheStaleFrame() { // The 7993a67 hazard, marquee edition: the catcher stays live behind an // open composer, so drag-then-click WITHOUT capturing is a supported flow. From 752e9b5657f917c5de74751c8112d629bea7a37a Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 20:03:31 -0500 Subject: [PATCH 16/24] docs(navigation): record selection navigation + frame anchoring, cover them in the probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DECISIONS.md gains two rows and two sections: why Parent/Child replaced the one-way "Widen", why descent replays history instead of re-querying a live tree, why prepending the frontier child is what makes the round trip hold both ways, the `component` consequence that follows from prepending (first SEEDED rung above the BOUND one, read from the identifier — an unseeded id is a slash-joined path that greps to nothing), and the open question F1 left about whether the parent chain stays seeded-only. Frame anchoring records why the drawn box outranks the resolved element visually, why the element is named in the composer rather than drawn, why navigating reveals it, and why the hover gate lives in the session. PARITY.md gains rows for child navigation (one shared pure ChildNavigationRule on all three adapters; each supplies only candidates) and for frame-mode anchoring / hover gating (shared session + shared view, so no code asymmetry — but the live-tree probe is macOS-only, recorded as a verification gap). AnnotKitOverlayProbe grows Phase 8: round trip up (climb 3, descend 3, land on the original), round trip down through a real ChildNavigationSource query, history-not-re-query (proved by the path depth NOT growing on a second descent), the component fix against the live tree (unseeded child, unseeded parent, component resolves to the seeded grandparent with no slash), an inert frame-mode hover on a point proven live in point mode, and the frame anchor surviving until navigation drops it while the drawn rect persists. Behaviour is unchanged; the probe reports nine phases, all PASS. Co-Authored-By: Claude Opus 5 (1M context) --- DECISIONS.md | 84 ++++++ PARITY.md | 2 + Sources/AnnotKitOverlayProbe/main.swift | 323 +++++++++++++++++++++++- 3 files changed, 407 insertions(+), 2 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index 52d527f..1fe7ebc 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -11,6 +11,8 @@ Resolves the open decisions from the plan (planning/annotkit in the cli repo). P | Default element source | Accessibility hierarchy | The only strategy that surfaces SwiftUI `accessibilityIdentifier` values. | | Annotation target rule | Deepest actionable, else deepest meaningful; anchor the selector to the nearest identifier | One rule on both platforms. Supersedes the earlier macOS "deepest meaningful" and iOS "nearest identified" split. See below. | | Marquee target rule | Largest meaningful element ≥85% surrounded; else the tightest element enclosing the drawn frame | Rect selection, the deliberate inverse of the point rule's deepest-wins. See below. | +| Selection navigation | Bidirectional Parent/Child over one path; descent replays history and only queries the source at the deepest rung | Replaces the one-way "Widen". Prepending the frontier child shifts every rung, so a note's `component` is the first SEEDED rung above the BOUND one. Whether the parent chain stays seeded-only is OPEN. See below. | +| Frame mode anchoring | The frame the user DREW anchors the overlay until they navigate; the resolved element is NAMED in the composer, not drawn on the canvas | Hover is point-mode-only, gated in the session rather than the view. See below. | | Opt-in element source | View tree (NSView/UIView) | Surfaces concrete view class names; richer for AppKit/UIKit hosts. Collapses to hosting views in pure SwiftUI. | | `pathname` mapping | Host-supplied route, inferred fallback | A native app has no URL routes; the host sets a route, else infer from the key window title or identifier. | | Overlay coverage | Primary screen (MVP) | The overlay covers the primary display; SwiftUI-local points map to AX screen coordinates there. Full multi-display placement is deferred (cli-a99qm.4.2). | @@ -129,6 +131,88 @@ the optional `MarqueeTargetSource` capability, which returns a component-widenin ladder identical in contract to `ComponentLadderSource`, so widening and the note's `component` field work unchanged. +## Selection navigation (VRT-mijf.1) + +The composer's one-way "Widen" button is replaced by Parent and Child over a +single path: index 0 is the deepest rung known so far, ascending indices are +progressively broader, and one index marks the rung the note is bound to. + +The rename is not cosmetic. "Widen" named the MECHANISM — the highlighted area +gets bigger — when the act is choosing which component the note is filed against. +A button that makes things bigger implies no inverse, so a user who overshot, or +whom the target rule bound coarser than they meant, had nothing to press. + +**Descent prefers HISTORY over re-querying.** Above the deepest rung, Child just +steps the index back down what the user climbed. Only AT the deepest rung does it +ask the source for children, and it then PREPENDS the one it takes, so index 0 +still means "deepest known rung". Re-querying on every press would be less code +and wrong: the source's answer is a heuristic over a LIVE tree, so a hover state +resolving or a list reflowing between two presses makes the same key produce a +different result. Prepending is what makes the round trip hold in BOTH +directions — after descending to child C, Parent returns to the original target +and Child returns to C ITSELF rather than re-running the heuristic against a tree +that has moved on. + +**The consequence that bit us.** Prepending shifts every existing rung up one, so +"the note's `component` is the rung above the target" stopped being true: index 1 +is now the ORIGINAL target, which is frequently unseeded. `component` is +therefore the first SEEDED rung strictly above the BOUND rung, and it is read +from that rung's IDENTIFIER, never from its `Element.id`. An unseeded element's +`id` is a slash-joined path (`AXWindow[0]/AXGroup[0]/AXStaticText[1]`); exported +as a `component` it hands the consuming agent a grep target that matches nothing +while looking entirely plausible in the note — a silent miss, not a visible one. +The same path is rooted differently depending on which entry point produced the +element (`snapshot()` roots at the window, the hit-test and marquee paths at the +application), so the id is not even stable for one node, which is a second reason +it can never be a code locator. + +**Open, pending dogfood: should the parent chain stay seeded-only?** It is today — +every rung above the target is an identified component, so every rung locates +code and no press can bind a note to something that names nothing. The cost is +that it skips structural levels the user can SEE: a row inside an unseeded stack +offers no rung for the stack, so Parent jumps from the row straight to the card +and the level the user was aiming at is unreachable. Admitting unseeded rungs +would fix the navigation and degrade the notes. Which failure is worse is not +decidable from the design; it needs real use, so this is recorded as unresolved +rather than settled. + +## Frame mode anchoring (VRT-mijf.2) + +When a drawn frame resolves to a real element, the overlay anchors its highlight, +composer and pin to the FRAME the user drew — not to the element — until the user +presses Parent or Child, at which point the bound element becomes the anchor and +the frame stays on screen, dimmed. + +**Why the frame outranks the resolved element.** The user drew a box, so the box +is the truth of the selection until they say otherwise. Anchoring to the +resolution instead makes the rectangle vanish the instant the mouse comes up and +the highlight snap to a card that was never swept, which reads as the tool having +ignored the gesture. + +**Why the element is NAMED rather than DRAWN.** A note must never be captured +against a target the user could not see, so the binding has to appear somewhere. +But a second rectangle on the canvas is exactly what "show me only the frame I +drew" rules out, and two boxes of different shapes leave it ambiguous which one +the note records. The composer header carries the name behind a `Frame →` prefix, +so it reads as what the frame RESOLVED to rather than as a label for the +rectangle, and the prefix disappears the moment navigation puts a named element +back on the canvas — the name is never qualified in two places at once. + +**Why navigating reveals the element.** Pressing Parent or Child IS the question +"which element is this filed against?", so the answer has to become visible; +moving the binding while the highlight stays on the drawn rect would give no +feedback at all. The frame survives, weaker, because it is still what the note +records (`regionRect`) even once it no longer decides the binding. + +**Why hover is gated in the SESSION, not the view.** Frame mode selects from a +swept rectangle, so a hover highlight there advertises a click-selection no press +in that mode can produce — the dogfooding report was a whole card lit up with its +name tag while nothing had been drawn. The view keeps its own guard for the +narrower during-the-drag case; the MODE gate belongs one level down because there +it is unit-testable without a window, no future UI path can reintroduce it, and +it removes a cross-process AX hit-test per pointer-motion event. It is a cost +decision as much as a visual one. + ## IP hygiene (carried into the F7 legal gate) - Do not copy original Agentation source (PolyForm Shield 1.0.0, non-compete). Only the `AGENTATION_NOTES.md` file format is reused, reimplemented clean-room. diff --git a/PARITY.md b/PARITY.md index 62ce317..78610eb 100644 --- a/PARITY.md +++ b/PARITY.md @@ -14,6 +14,8 @@ row; each asymmetry is closed by code or has a tracked mitigation. | Annotation target rule | shared `AnnotationTargetRule` over an AX candidate chain | shared `AnnotationTargetRule` over a UIView candidate chain | none — both build a `[TargetCandidate]` chain and apply the SAME rule (deepest actionable, else deepest meaningful). Closes the earlier split (macOS "deepest meaningful" vs iOS "nearest identified"), cli-got28.2 | | Component widening | `ComponentLadderSource` (AX chain) | `ComponentLadderSource` (UIView chain) | none — same ladder (target, then enclosing identified components) | | Marquee selection (drawn frame → element) | `MarqueeTargetSource`: shared `MarqueeTargetRule` over `[MarqueeCandidate]` read from the AX tree | `MarqueeTargetSource`: shared `MarqueeTargetRule` over `[MarqueeCandidate]` read from the UIView tree | none — the DECISION is one pure rule (largest ≥85%-surrounded element, else the tightest enclosing one); the adapters differ only in how they read candidates out of their own tree. Both do ONE walk from a single root so depth (the rule's tie-break) is numbered comparably, both collect the subtree WHOLE (an intersects-the-frame filter would discard the enclosing-pass candidates), and both return the SAME target-first, broadest-last ladder as `ComponentLadderSource`, so the session's widening and `component` field work unchanged from a framed selection | +| Child navigation (select child) | `ChildNavigationSource` over the AX tree; the opt-in view-tree source implements it over `NSView` | `ChildNavigationSource` over the `UIView` tree | none — the ORDERING is the one pure `ChildNavigationRule` (contains the gesture's hint, then seeded, then larger area, then lowest index) on all three adapters, which supply only `[ChildCandidate]`. All three also share the cost shape the protocol mandates: re-find the bound element by descending its containing window along its own frame centre (bounded by tree DEPTH, not tree size), then collect the NEAREST meaningful descendants, each branch stopping at its first meaningful node — so a SwiftUI host's unidentified layout wrappers are descended THROUGH rather than offered as children, which is what keeps the Child control alive under pure SwiftUI on both platforms. macOS additionally skips window chrome and its own overlay window during that descent; UIKit has no chrome, and the overlay is already excluded by the shared window lookup | +| Frame-mode anchoring + hover gating | shared `AnnotationSession` (`selectionAnchorFrame`, the `tool == .point` hover gate, `setTool` clearing `hovered`) rendered by the shared `OverlayView` | same | none in the code — all of it is session-level and platform-free, and one SwiftUI view renders it. ASYMMETRIC VERIFICATION, recorded as a gap rather than closed: `AnnotKitOverlayProbe` Phase 8 drives navigation, the note's `component`, the hover gate and the frame anchor against a REAL accessibility tree, and it is macOS-only (`#if os(macOS)`, AppKit + `AXUIElement`), so the iOS adapter's live behaviour is covered only by unit tests over the pure rules. Mitigated, not fixed, by the fact that everything Phase 8 asserts about anchoring and hover lives in the shared session; what remains unverified on iOS is the ADAPTER's candidate collection. Note the hover gate is also moot on touch-only iOS — hover exists there only with a trackpad or pencil — so the reported symptom cannot arise without a pointer | | Marquee drag threshold | cursor slop (a mouse does not move on a deliberate click) | larger touch slop | ASYMMETRIC BY DESIGN, owned by the drag UI, not the adapters: a finger rolls several points on a deliberate tap, so the macOS threshold on iOS would turn taps into marquees. Below the threshold both platforms route the gesture to the point path (`select(atAXPoint:)`), per the caller contract on `select(inAXRect:)` | | Overlay excluded from element lookup | AX window identifier (`AXIntrospection.overlayWindowIdentifier`) filtered out of every `kAXWindows` read | `PassThroughWindow` TYPE identity filtered out of `IOSElementSource.windows()` | ASYMMETRIC BY NECESSITY — the hosts are different window kinds. macOS's overlay is a separate `NSPanel` matched by the identifier the controller stamps on it; iOS's is a `UIWindow` in the HOST's scene sharing its pid, so no pid/scene filter separates it and a type check (internal to the module) cannot drift the way an identifier convention can. Both filter in the shared window lookup, so snapshot / hit-test / region-anchor / marquee agree; leaving it in would let a marquee bind the user's note to AnnotKit's own UI | | Coordinate space | Cocoa bottom-left to AX top-left flip | UIKit top-left native | iOS needs no flip; shared `ScreenSpace` used only on macOS | diff --git a/Sources/AnnotKitOverlayProbe/main.swift b/Sources/AnnotKitOverlayProbe/main.swift index 9d3c7e2..d135c62 100644 --- a/Sources/AnnotKitOverlayProbe/main.swift +++ b/Sources/AnnotKitOverlayProbe/main.swift @@ -1352,10 +1352,287 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { self.marqueeController?.unmount() window.orderOut(nil) - self.finish() + self.phase8Navigation() } } + // ---- Phase 8: selection navigation (F1 + F2) ---------------------------- + // Parent/Child is BIDIRECTIONAL navigation over one path, and the property + // that makes it usable is that it round-trips: whatever you climbed, you can + // walk back down to, and vice versa. That is a claim about a LIVE tree — the + // whole reason descent replays history instead of re-querying is that a live + // UI would answer the same question differently on consecutive presses — so it + // has to be asserted here rather than only against hand-built fixtures. + var navController: OverlayController? + var navSession: AnnotationSession? + var navHost: NSWindow? + var passNav = true + func check8(_ cond: Bool, _ msg: String) { + passNav = passNav && cond + print(" " + (cond ? "ok " : "FAIL ") + msg) + } + + /// Climb to the broadest rung, returning how many rungs were actually walked. + /// Doubles as the probe's only view of the path's SHAPE: the session keeps the + /// path private, so "how many rungs sit above the bound one" is observable only + /// by walking them — which is what lets 8c prove no extra rung was inserted. + func climbToTop(_ session: AnnotationSession) -> Int { + var climbed = 0 + while session.selectParent() != nil { climbed += 1 } + return climbed + } + + /// Walk `count` rungs back down, returning how many actually moved. + @discardableResult + func descend(_ session: AnnotationSession, _ count: Int) -> Int { + var moved = 0 + for _ in 0 ..< count where session.selectChild() != nil { moved += 1 } + return moved + } + + /// Identity for a round-trip assertion: id AND frame. The id alone is not + /// enough — an UNSEEDED element's id is its slash-joined path, which two + /// sibling rows of the same role and depth can share — and the frame alone is + /// not enough either, because a coextensive surface shares it with the content + /// group in front of it. + func same(_ lhs: Element?, _ rhs: Element?) -> Bool { + guard let lhs, let rhs else { return false } + return lhs.id == rhs.id && approxEqual(lhs.frame, rhs.frame, tol: 0.5) + } + + func label(_ element: Element?) -> String { + guard let element else { return "nil" } + let text = element.value.isEmpty ? element.label : element.value + return "#\(element.id) \(element.role)\(text.isEmpty ? "" : " \"\(text)\"") \(fmt(element.frame))" + } + + func phase8Navigation() { + print("\n--- Phase 8: selection navigation (parent/child round trips, frame anchoring) ---") + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 520, height: 700), + styleMask: [.titled, .closable], + backing: .buffered, + defer: false + ) + window.title = "AnnotKit Harness W8 (navigation)" + window.contentView = NSHostingView(rootView: ProbeNavigationView()) + window.makeKeyAndOrderFront(nil) + navHost = window + + let session = AnnotationSession( + source: MacElementSource(), + sink: NotesFileSink(path: NSTemporaryDirectory() + "annotkit-navigation.md") + ) + let controller = OverlayController(session: session) + controller.mount(on: window) + controller.start() + navController = controller + navSession = session + + DispatchQueue.main.asyncAfter(deadline: .now() + 0.9) { [weak self] in + guard let self else { return } + let source = MacElementSource() + let roots = source.snapshot().map(\.root) + guard let card = self.findElement(id: "Spec.Card", in: roots), + let cardText = self.findElement(id: "Spec.CardText", in: roots), + let section = self.findElement(id: "Spec.Section", in: roots), + let list = self.findFirst(in: roots, where: { $0.label == "Nav list" }) else { + check8(false, "snapshot exposes Spec.Card/CardText/Section + the Nav list container") + self.navController?.unmount() + window.orderOut(nil) + self.finish() + return + } + print(" fixture: card=\(fmt(card.frame)) cardText=\(fmt(cardText.frame)) " + + "section=\(fmt(section.frame)) list=\(self.label(list))") + + self.phase8aRoundTripUp(session: session, source: source, cardText: cardText) + self.phase8bcdDescent(session: session, source: source, list: list) + self.phase8eHover(session: session, cardText: cardText) + self.phase8fAnchoring(session: session, card: card) + + // The hover re-check runs on a later turn ON PURPOSE: `hover` is + // throttled to ~60fps, so a re-hover issued in this same runloop turn + // would be dropped by the throttle and "still nil" would prove the + // throttle, not the tool gate. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { [weak self] in + guard let self else { return } + self.phase8eHoverRevived(session: session, cardText: cardText) + self.navController?.unmount() + window.orderOut(nil) + self.finish() + } + } + } + + /// 8a — climbing N rungs and descending N returns to the ORIGINAL element. + func phase8aRoundTripUp(session: AnnotationSession, source: MacElementSource, cardText: Element) { + print("\n 8a — round trip UP: climb N, descend N, land on the original element:") + let point = center(of: cardText.frame) + let origin = session.select(atAXPoint: point) + print(" click \(fmt(CGRect(origin: point, size: .zero))) -> \(self.label(origin))") + check8(origin?.id == "Spec.CardText", "click selects #Spec.CardText (got \(self.label(origin)))") + + let climbed = climbToTop(session) + let top = session.selected + print(" climbed \(climbed) rung(s) -> \(self.label(top))") + // Without this the round trip is vacuous: descending zero rungs trivially + // "returns" to where it started. Two rungs, not one, so the descent has to + // replay a sequence rather than a single undo. + check8(climbed >= 2, "the click's ladder offered >=2 rungs to climb (got \(climbed) — a real multi-rung path)") + check8(!same(top, origin), "climbing actually moved the binding off the original element") + check8(!session.canSelectParent, "the climb stopped at the BROADEST rung (Select Parent is spent)") + + let descended = descend(session, climbed) + print(" descended \(descended) rung(s) -> \(self.label(session.selected))") + check8(descended == climbed, "descending walks back exactly as many rungs as were climbed") + check8(same(session.selected, origin), + "N up then N down returns to the ORIGINAL element (got \(self.label(session.selected)))") + } + + /// 8b/8c/8d — descending BELOW the target: a real `ChildNavigationSource` query + /// against the live tree, the history replay, and the `component` fix. + /// + /// Why this needs the Nav-list container rather than the Spec fixtures: every + /// element in `ProbeSpecificityView` is an AX LEAF (the card and section are + /// `children: .ignore` surfaces; the button and texts have no AX children), so + /// a child query there can only ever return [] and every assertion built on it + /// would pass while proving nothing. That is asserted below rather than assumed, + /// so the day the fixture grows children this comment does not quietly rot. + func phase8bcdDescent(session: AnnotationSession, source: MacElementSource, list: Element) { + print("\n 8b/8c/8d — descend BELOW the target (live child query), history replay, component:") + + // The frame IS how this selection is made: the list container's AX frame is + // the union of its rows (the documented `children: .contain` behaviour), so + // there is no point inside it that hit-tests to the container itself. A + // drawn frame binds to it by the marquee rule's largest-surrounded pass. + let drawn = list.frame.insetBy(dx: -6, dy: -6) + let target = session.select(inAXRect: drawn) + print(" frame \(fmt(drawn)) -> \(self.label(target))") + // Matched on label + frame, NOT on id, and that is not laziness: an unseeded + // element's id is its slash-joined path, and the path is rooted differently + // depending on which entry point produced the element — `snapshot()` roots + // its trees at the window, while the hit-test / marquee / navigation paths + // build the chain from the AXApplication. So the SAME live node is + // `AXWindow[0]/AXGroup[0]/…` here and `AXApplication[0]/AXWindow[1]/…` + // there. Pre-existing and orthogonal to this phase — but it is the concrete + // reason a note's `component` must be an IDENTIFIER and never an id (8d). + check8(target?.label == list.label && approxEqual(target?.frame ?? .zero, list.frame, tol: 0.5), + "the drawn frame binds to the Nav list container (got \(self.label(target)))") + check8((target?.path.last?.identifier ?? "").isEmpty, + "the bound target is UNSEEDED (so `component` must come from a rung above it)") + check8(target?.id.contains("/") == true, + "the unseeded target's id IS a slash-joined path (got \(target?.id ?? "nil")) — the string that must never be exported as a grep target") + + let children = source.children(of: list, near: center(of: drawn)) + print(" live children(of: list) = \(children.isEmpty ? "[]" : children.map { self.label($0) }.joined(separator: " | "))") + check8(!children.isEmpty, "the live tree really offers children under the bound target (the query is not returning [])") + + // 8b — descend below the target, then Parent returns to the target. + let child = session.selectChild() + print(" selectChild() -> \(self.label(child))") + check8(child != nil, "selectChild() descends below the deepest known rung") + check8(!same(child, target), "the descent landed on a DIFFERENT element than the target") + check8(same(child, children.first), + "the descent landed on the rule's most-likely child (got \(self.label(child)), rule ranked \(self.label(children.first)))") + let backUp = session.selectParent() + print(" selectParent() -> \(self.label(backUp))") + check8(same(backUp, target), "Parent from the prepended child returns to the ORIGINAL target (round trip DOWN)") + + // 8c — history, not a re-query. `descend` measures the path's shape by + // walking it: a second descent that RE-QUERIED would prepend a second rung, + // so the number of rungs above the child would grow by one. That count is + // the only externally visible witness of the prepend, which is why the + // check is phrased as a depth comparison rather than "the ids match" alone. + let secondChild = session.selectChild() + print(" selectChild() again -> \(self.label(secondChild))") + check8(same(secondChild, child), "the second descent lands on the SAME child (history, not a fresh heuristic)") + let depthAfterFirst = climbToTop(session) + descend(session, depthAfterFirst) + let thirdChild = session.selected + session.selectParent() + let fourthChild = session.selectChild() + let depthAfterThird = climbToTop(session) + descend(session, depthAfterThird) + print(" rungs above the child: after descent \(depthAfterFirst), after an up-down replay \(depthAfterThird)") + check8(depthAfterFirst >= 2, "the descended path has >=2 rungs above the child (a real path to compare)") + check8(same(thirdChild, child), "a full climb-and-return still lands on that same child") + check8(same(fourthChild, child), "the replayed descent lands on that same child") + check8(depthAfterThird == depthAfterFirst, + "re-descending did NOT prepend another rung (\(depthAfterFirst) -> \(depthAfterThird)) — it replayed history rather than re-querying the live tree") + + // 8d — the component fix, against the live tree. The bound element is + // unseeded, so `component` is searched UPWARD from the bound rung — and it + // must be an identifier, never an unseeded element's slash-joined id, which + // would export as a grep target that matches nothing while looking + // perfectly plausible in the note. + check8(same(session.selected, child), "still bound to the descended child going into the capture") + check8((session.selected?.path.last?.identifier ?? "").isEmpty, + "the descended child is itself UNSEEDED (so the note's component is not just its own id)") + let note = session.addNote(comment: "phase 8 descended note") + print(" note component=\(note?.component ?? "nil") unseeded=\(note?.unseeded.map(String.init) ?? "nil") selector=\(note?.selector ?? "nil")") + check8(note != nil, "a note captured from the descended selection") + check8(note?.component != nil, "the descended note names a component to grep") + check8(note?.component?.contains("/") != true, + "the component is NOT a slash-joined path (got \(note?.component ?? "nil")) — an unseeded id must never be exported as a grep target") + check8(note?.component == "Nav.Section", + "the component is the first SEEDED rung above the bound one (got \(note?.component ?? "nil"))") + } + + /// 8e — frame mode makes hover inert, gated in the SESSION. + func phase8eHover(session: AnnotationSession, cardText: Element) { + print("\n 8e — hover is POINT-MODE ONLY (gated in the session, not the view):") + let point = center(of: cardText.frame) + session.setTool(.point) + session.hover(atAXPoint: point) + print(" point-mode hover at \(String(format: "(%.0f, %.0f)", point.x, point.y)) -> \(self.label(session.hovered))") + // Non-vacuity: a point that hits nothing would leave `hovered` nil in BOTH + // modes, and the frame-mode assertion below would prove nothing at all. + check8(session.hovered?.id == "Spec.CardText", + "sanity: this point DOES resolve to a real element in point mode (got \(self.label(session.hovered)))") + + session.setTool(.frame) + check8(session.hovered == nil, "switching to frame mode drops the standing highlight") + session.hover(atAXPoint: point) + print(" frame-mode hover at the SAME live point -> \(self.label(session.hovered))") + check8(session.hovered == nil, "hover() in frame mode is inert (no highlight, and no AX hit-test spent)") + } + + /// 8e (continued, a later runloop turn) — and it comes back in point mode, so + /// the gate is the TOOL and not a point that went dead. + func phase8eHoverRevived(session: AnnotationSession, cardText: Element) { + session.setTool(.point) + session.hover(atAXPoint: center(of: cardText.frame)) + print(" back in point mode, same point -> \(self.label(session.hovered))") + check8(session.hovered?.id == "Spec.CardText", + "hover resolves again once the tool is point (the gate was the TOOL, not a dead point)") + } + + /// 8f — the drawn frame is the anchor until the user navigates. + func phase8fAnchoring(session: AnnotationSession, card: Element) { + print("\n 8f — frame anchoring: the drawn rect anchors the overlay until Parent/Child is pressed:") + let drawn = card.frame.insetBy(dx: -8, dy: -8) + let target = session.select(inAXRect: drawn) + print(" frame \(fmt(drawn)) -> \(self.label(target)) anchor=\(session.selectionAnchorFrame.map(fmt) ?? "nil")") + check8(target?.id == "Spec.Card", "the drawn frame resolves to a real element (got \(self.label(target)))") + // Non-vacuity: the anchor must differ from the resolved element's own frame, + // or "anchors to the frame" and "anchors to the element" are the same claim. + check8(!approxEqual(drawn, target?.frame ?? .zero, tol: 0.5), + "sanity: the drawn frame is NOT the resolved element's own frame (8pt proud of it)") + check8(session.selectionAnchorFrame.map { approxEqual($0, drawn, tol: 0.5) } ?? false, + "selectionAnchorFrame IS the drawn rect (got \(session.selectionAnchorFrame.map(fmt) ?? "nil"))") + + let parent = session.selectParent() + print(" selectParent() -> \(self.label(parent)) anchor=\(session.selectionAnchorFrame.map(fmt) ?? "nil") " + + "marquee=\(session.selectedMarqueeRect.map(fmt) ?? "nil")") + check8(parent != nil, "sanity: the framed selection really had a parent rung to navigate to") + check8(session.selectionAnchorFrame == nil, + "navigating drops the frame anchor, revealing the bound element (got \(session.selectionAnchorFrame.map(fmt) ?? "nil"))") + check8(session.selectedMarqueeRect.map { approxEqual($0, drawn, tol: 0.5) } ?? false, + "the drawn rect SURVIVES navigation (it is still what the note records)") + session.cancelSelection() + } + /// First element matching `predicate`, depth-first. func findFirst(in elements: [Element], where predicate: (Element) -> Bool) -> Element? { for element in elements { @@ -1407,8 +1684,9 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { print(" Phase 5 (seeded container resolves on body hover): \(passCard ? "PASS" : "FAIL")") print(" Phase 6 (positional specificity by cursor position): \(passSpec ? "PASS" : "FAIL")") print(" Phase 7 (marquee frame selection: drawn rect -> element): \(passMarquee ? "PASS" : "FAIL")") + print(" Phase 8 (selection navigation: round trips, history, component, frame anchor): \(passNav ? "PASS" : "FAIL")") print("\n=== AnnotKitOverlayProbe complete ===") - exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec && passMarquee ? 0 : 1) + exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec && passMarquee && passNav ? 0 : 1) } func collectIDs(_ elements: [Element]) -> [String] { @@ -1478,6 +1756,47 @@ struct ProbeSpecificityView: View { } } +/// Phase 8 host content: the specificity fixture (reused unchanged — its card / +/// card text / section are the nesting the UPWARD navigation and the frame anchor +/// need) plus ONE addition it cannot supply: a container with real children. +/// +/// The addition is deliberate and minimal. `ProbeSpecificityView`'s elements are +/// all AX leaves, so `ChildNavigationSource` can only return [] for them and every +/// descent assertion built on it would be vacuously green. This container is: +/// +/// * MEANINGFUL but UNSEEDED (a label, no identifier) — so descending below it +/// exercises the `component` search that must skip unseeded rungs, and its own +/// `Element.id` is the slash-joined path that must never reach a note; and +/// * a real AX parent (`children: .contain`) of two UNSEEDED rows — so the child +/// the descent lands on is unseeded too, and the note's component has to be +/// found further up, at `Nav.Section`. +/// +/// Both fixtures share one window so the phase reads one snapshot; they are +/// independent subtrees, so neither one's geometry disturbs the other. +struct ProbeNavigationView: View { + var body: some View { + VStack(spacing: 24) { + ProbeSpecificityView() + + VStack(alignment: .leading, spacing: 8) { + Text("Row one") + Text("Row two") + } + .padding(20) + .accessibilityElement(children: .contain) + .accessibilityLabel("Nav list") + } + .padding(24) + .background( + Color.clear + .contentShape(Rectangle()) + .accessibilityElement(children: .ignore) + .accessibilityIdentifier("Nav.Section") + ) + .padding(16) + } +} + let app = NSApplication.shared app.setActivationPolicy(.accessory) let delegate = OverlayProbeDelegate() From fd9a00a204079cd33028bfe1facee64b300918be Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:18:58 -0500 Subject: [PATCH 17/24] feat(icons): real elliptical arcs in the Lucide d-string parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `A`/`a` branch drew a straight line to the arc endpoint. That survived only while every arc in the set was a 1-2 unit corner round; Lucide `undo-2` is a 5.5-radius semicircular loop whose endpoint sits directly below its start, so the shortcut collapses the whole glyph to a vertical line. Implements SVG 1.1 F.6.5 endpoint -> centre parameterisation, emitted as cubics split at 90 degrees (k = 4/3 tan(theta/4)), with x-axis-rotation applied to the control points, both flags honoured, F.6.6.1 radius abs, F.6.6.2 radius scale-up so undersized radii still land on the authored endpoint, and F.6.2 degenerate handling. The parser's make-progress-or-bail contract is unchanged. Adds 18 geometric tests (sampled along the curves, not on bounding rects that may include control points), including a pinned 5.5-radius semicircle that fails against the old straight-line branch. Side effect: pencil, mouse-pointer-2 and square-dashed carry `a` commands and now render their true curves for the first time — the pencil's eraser butt is the visible one (r=1 across a 5.6-unit chord, scaled up per F.6.6.2). All stay inside the 24-unit grid. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/Overlay/PillStyle.swift | 178 ++++++++++-- Tests/AnnotKitTests/LucideArcTests.swift | 332 +++++++++++++++++++++++ 2 files changed, 487 insertions(+), 23 deletions(-) create mode 100644 Tests/AnnotKitTests/LucideArcTests.swift diff --git a/Sources/AnnotKit/Overlay/PillStyle.swift b/Sources/AnnotKit/Overlay/PillStyle.swift index 0b6fa0d..2339a30 100644 --- a/Sources/AnnotKit/Overlay/PillStyle.swift +++ b/Sources/AnnotKit/Overlay/PillStyle.swift @@ -65,16 +65,19 @@ enum IconPart { } /// The Lucide glyphs the toolbar pill uses, authored on the 24x24 viewBox. The -/// `d` strings are copied from lucide.dev so the rendered shape matches the -/// sibling Agentation nav bar; the rest use primitives (`copy`'s rounded rect and -/// the straight strokes of `pencil`/`download`/`close`) so no elliptical-arc -/// parsing is needed. +/// `d` strings are copied verbatim from lucide.dev so the rendered shape matches +/// the sibling Agentation nav bar — including their elliptical arcs, which the +/// parser now converts to real cubics. A few glyphs (`copy`'s rounded rect, +/// `download`'s tray) stay hand-built from primitives because the primitive form +/// is simpler to read, not because the parser cannot handle their `d`. struct LucideIcon { let parts: [IconPart] - /// Lucide `pencil` — the idle pill's ENTER-annotate-mode glyph. Uses the real - /// Lucide `d` strings; the parser approximates the small corner arcs (`a`) as - /// a line to the arc endpoint, which reads identically at 16pt. + /// Lucide `pencil` — the idle pill's ENTER-annotate-mode glyph. The real + /// Lucide `d` strings, arcs included. Its eraser end is an `a` with r=1 across + /// a 5.6-unit chord, so it only closes into a round butt once the parser + /// applies the F.6.6.2 radius scale-up; the nib and shoulder arcs are ordinary + /// small rounds. static let pencil = LucideIcon(parts: [ .path("M21.174 6.812a1 1 0 0 0-3.986-3.987L3.842 16.174a2 2 0 0 0-.5.83l-1.321 4.352a.5.5 0 0 0 .623.622l4.353-1.32a2 2 0 0 0 .83-.497z"), .path("m15 5 4 4"), @@ -82,9 +85,11 @@ struct LucideIcon { static let check = LucideIcon(parts: [.path("M20 6 9 17l-5-5")]) - /// Lucide `download` — export to a file. Drawn with straight strokes only - /// (the real glyph's rounded tray uses SVG arc commands the primitive parser - /// does not implement): an open-top tray plus a down arrow into it. + /// Lucide `download` — export to a file. A deliberate simplification of the + /// real glyph: an open-top tray plus a down arrow, with square tray corners + /// instead of Lucide's arc-rounded ones. The corners are square by choice (the + /// parser handles arcs now); swapping in the upstream `d` is a glyph change, + /// not a parser one. static let download = LucideIcon(parts: [ .line(CGPoint(x: 4, y: 15), CGPoint(x: 4, y: 20)), .line(CGPoint(x: 4, y: 20), CGPoint(x: 20, y: 20)), @@ -113,9 +118,8 @@ struct LucideIcon { ]) /// Lucide `mouse-pointer-2` — the POINT tool: select by clicking. The real - /// Lucide `d` string; its two tiny corner arcs (`a`) are approximated by the - /// parser as a line to the arc endpoint, which is invisible at 16pt on a shape - /// this angular. + /// Lucide `d` string; its four corner arcs (`a`) render as true curves, which + /// is what keeps the cursor's tail and notch from reading as hard mitres. static let mousePointer = LucideIcon(parts: [ .path("M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"), ]) @@ -176,7 +180,7 @@ struct LucideShape: Shape { return path } - // MARK: SVG `d`-string parser (M/L/H/V/C/Q/Z, absolute + relative) + // MARK: SVG `d`-string parser (M/L/H/V/C/Q/A/Z, absolute + relative) private enum Token { case command(Character); case number(CGFloat) } @@ -237,16 +241,21 @@ struct LucideShape: Shape { current = end path.addQuadCurve(to: scaled(end), control: scaled(ctrl)) case "A", "a": - // Elliptical arc. The parser has no arc-to-bezier, so it draws a - // straight segment to the arc ENDPOINT — the only arcs in use are - // Lucide's pencil corner rounds and its flat eraser diagonal, - // which read the same at 16pt. Consume all 7 params (rx ry rot - // large sweep x y). - guard nextNumber() != nil, nextNumber() != nil, nextNumber() != nil, - nextNumber() != nil, nextNumber() != nil, + // Elliptical arc (rx ry x-rotation large-arc sweep x y), converted + // to cubics by ``appendArc``. This used to draw a straight line to + // the endpoint, which was survivable only while every arc in the + // set was a 1-2 unit corner round; a glyph whose whole identity is + // a loop (Lucide `undo-2`, 5.5-radius semicircles) collapses to a + // vertical line under that shortcut, so the flattening is gone. + guard let rx = nextNumber(), let ry = nextNumber(), let rotation = nextNumber(), + let largeArc = nextNumber(), let sweep = nextNumber(), let ax = nextNumber(), let ay = nextNumber() else { return } - current = command == "a" ? CGPoint(x: current.x + ax, y: current.y + ay) : CGPoint(x: ax, y: ay) - path.addLine(to: scaled(current)) + let end = command == "a" ? CGPoint(x: current.x + ax, y: current.y + ay) : CGPoint(x: ax, y: ay) + appendArc( + from: current, to: end, rx: rx, ry: ry, rotationDegrees: rotation, + largeArc: largeArc != 0, sweep: sweep != 0, to: &path, scale: scale + ) + current = end case "Z", "z": path.closeSubpath() current = subStart @@ -257,6 +266,129 @@ struct LucideShape: Shape { } } + /// SVG 1.1 Appendix F.6.5 endpoint -> centre parameterisation, emitted as + /// cubic Béziers. Everything here is in 24-grid units until the final + /// `scaled` on each control point, so the ellipse maths never has to know the + /// render size. + private static func appendArc( + from start: CGPoint, + to end: CGPoint, + rx rxIn: CGFloat, + ry ryIn: CGFloat, + rotationDegrees: CGFloat, + largeArc: Bool, + sweep: Bool, + to path: inout Path, + scale: CGFloat + ) { + func scaled(_ p: CGPoint) -> CGPoint { CGPoint(x: p.x * scale, y: p.y * scale) } + + // F.6.2 out-of-range handling. Coincident endpoints mean "omit the + // segment entirely" — emitting a zero-length line instead would round-cap + // into a stray dot. A zero radius is a plain lineto. Both branches also + // keep the divisions below from producing NaN control points, which would + // silently blank the whole glyph. + if start == end { return } + var rx = abs(rxIn), ry = abs(ryIn) + guard rx > 0, ry > 0 else { + path.addLine(to: scaled(end)) + return + } + + let phi = rotationDegrees.truncatingRemainder(dividingBy: 360) * .pi / 180 + let cosPhi = cos(phi), sinPhi = sin(phi) + + // F.6.5.1 — the chord half-vector expressed in the ellipse's own frame. + let dx = (start.x - end.x) / 2, dy = (start.y - end.y) / 2 + let x1 = cosPhi * dx + sinPhi * dy + let y1 = -sinPhi * dx + cosPhi * dy + + // F.6.6.2 — radii too small to span the endpoints are scaled UP until they + // just reach, rather than rejected: falling back to a line here is what + // makes an authored glyph fall short of its own endpoint and break the + // subpath. Lucide relies on this (`pencil` asks for r=1 across a 5.6-unit + // chord), so this branch is hot, not defensive. + let lambda = (x1 * x1) / (rx * rx) + (y1 * y1) / (ry * ry) + if lambda > 1 { + let correction = sqrt(lambda) + rx *= correction + ry *= correction + } + + // F.6.5.2/3 — centre, first in the rotated frame then back to user space. + let rx2 = rx * rx, ry2 = ry * ry + let denominator = rx2 * y1 * y1 + ry2 * x1 * x1 + let numerator = rx2 * ry2 - denominator + var factor = denominator > 0 ? sqrt(max(0, numerator) / denominator) : 0 + if largeArc == sweep { factor = -factor } + let cxp = factor * rx * y1 / ry + let cyp = -factor * ry * x1 / rx + let cx = cosPhi * cxp - sinPhi * cyp + (start.x + end.x) / 2 + let cy = sinPhi * cxp + cosPhi * cyp + (start.y + end.y) / 2 + + // F.6.5.5/6 — start angle and swept angle, then the flag fix-up that turns + // the raw [-pi, pi] result into the direction `sweep` actually asked for. + func angle(_ ux: CGFloat, _ uy: CGFloat, _ vx: CGFloat, _ vy: CGFloat) -> CGFloat { + let lengths = sqrt((ux * ux + uy * uy) * (vx * vx + vy * vy)) + guard lengths > 0 else { return 0 } + let value = acos(min(1, max(-1, (ux * vx + uy * vy) / lengths))) + return (ux * vy - uy * vx) < 0 ? -value : value + } + let ux = (x1 - cxp) / rx, uy = (y1 - cyp) / ry + let vx = (-x1 - cxp) / rx, vy = (-y1 - cyp) / ry + let theta1 = angle(1, 0, ux, uy) + var delta = angle(ux, uy, vx, vy) + if !sweep, delta > 0 { + delta -= 2 * .pi + } else if sweep, delta < 0 { + delta += 2 * .pi + } + + // A single cubic cannot hold more than a quarter turn without visible + // error, so split the sweep into <=90 degree pieces. `k` is the standard + // control magnitude (4/3)*tan(theta/4), exact at the segment endpoints and + // tangents. + let segments = max(1, Int(ceil(abs(delta) / (.pi / 2) - 1e-9))) + let step = delta / CGFloat(segments) + let k = 4.0 / 3.0 * tan(step / 4) + + // Points and tangents are evaluated on the unrotated ellipse and then run + // through the rotation individually — rotating a bounding box, or applying + // phi only to the endpoints, skews the control points and gives an ellipse + // that is the wrong shape rather than merely the wrong orientation. + func point(_ theta: CGFloat) -> CGPoint { + let c = cos(theta), s = sin(theta) + return CGPoint( + x: cx + rx * c * cosPhi - ry * s * sinPhi, + y: cy + rx * c * sinPhi + ry * s * cosPhi + ) + } + func derivative(_ theta: CGFloat) -> CGPoint { + let c = cos(theta), s = sin(theta) + return CGPoint( + x: -rx * s * cosPhi - ry * c * sinPhi, + y: -rx * s * sinPhi + ry * c * cosPhi + ) + } + + var theta = theta1 + for segment in 0.. [Token] { let commands = Set("MmLlHhVvCcSsQqTtAaZz") var tokens: [Token] = [] diff --git a/Tests/AnnotKitTests/LucideArcTests.swift b/Tests/AnnotKitTests/LucideArcTests.swift new file mode 100644 index 0000000..d978cec --- /dev/null +++ b/Tests/AnnotKitTests/LucideArcTests.swift @@ -0,0 +1,332 @@ +import CoreGraphics +import SwiftUI +import XCTest +@testable import AnnotKit + +/// Geometry tests for the `d`-parser's elliptical-arc support (SVG 1.1 F.6.5). +/// +/// These assert on points sampled ALONG the emitted curves rather than on +/// `Path.boundingRect`, because a Bézier's bounding rect is allowed to include +/// control points — a bulge assertion made against it could pass on a path whose +/// drawn curve never goes there. Sampling also makes every failure here a real +/// visual failure: the old straight-line arc handling reproduces the same +/// endpoints and often the same bounding box, and is only distinguishable by +/// where the stroke actually travels. +final class LucideArcTests: XCTestCase { + private let grid = CGRect(x: 0, y: 0, width: 24, height: 24) + + // MARK: Sampling helpers + + /// Flatten a path into points on the drawn curve (endpoints plus interior + /// samples of every line/quad/cubic). + private func samples(_ d: String, per: Int = 24) -> [CGPoint] { + let path = LucideShape(parts: [.path(d)]).path(in: grid) + var points: [CGPoint] = [] + var current = CGPoint.zero + var subStart = CGPoint.zero + + func lerp(_ a: CGPoint, _ b: CGPoint, _ t: CGFloat) -> CGPoint { + CGPoint(x: a.x + (b.x - a.x) * t, y: a.y + (b.y - a.y) * t) + } + func cubic(_ p0: CGPoint, _ p1: CGPoint, _ p2: CGPoint, _ p3: CGPoint, _ t: CGFloat) -> CGPoint { + let u = 1 - t + let x = u * u * u * p0.x + 3 * u * u * t * p1.x + 3 * u * t * t * p2.x + t * t * t * p3.x + let y = u * u * u * p0.y + 3 * u * u * t * p1.y + 3 * u * t * t * p2.y + t * t * t * p3.y + return CGPoint(x: x, y: y) + } + + path.forEach { element in + switch element { + case .move(let to): + current = to + subStart = to + points.append(to) + case .line(let to): + for step in 1...per { points.append(lerp(current, to, CGFloat(step) / CGFloat(per))) } + current = to + case .quadCurve(let to, let control): + // Promote to a cubic so one evaluator covers both. + let c1 = lerp(current, control, 2.0 / 3.0) + let c2 = lerp(to, control, 2.0 / 3.0) + for step in 1...per { + points.append(cubic(current, c1, c2, to, CGFloat(step) / CGFloat(per))) + } + current = to + case .curve(let to, let control1, let control2): + for step in 1...per { + points.append(cubic(current, control1, control2, to, CGFloat(step) / CGFloat(per))) + } + current = to + case .closeSubpath: + for step in 1...per { points.append(lerp(current, subStart, CGFloat(step) / CGFloat(per))) } + current = subStart + } + } + return points + } + + /// Tight bounding box of the DRAWN curve (control points excluded). + private func drawnBounds(_ d: String) -> CGRect { + let points = samples(d) + guard let first = points.first else { return .null } + var box = CGRect(origin: first, size: .zero) + for point in points.dropFirst() { + box = box.union(CGRect(origin: point, size: .zero)) + } + return box + } + + private func distanceToCurve(_ d: String, _ target: CGPoint) -> CGFloat { + samples(d, per: 64).map { hypot($0.x - target.x, $0.y - target.y) }.min() ?? .infinity + } + + private func assertBounds( + _ d: String, _ expected: CGRect, accuracy: CGFloat = 0.02, + _ message: String = "", file: StaticString = #filePath, line: UInt = #line + ) { + let box = drawnBounds(d) + XCTAssertEqual(box.minX, expected.minX, accuracy: accuracy, "minX \(message)", file: file, line: line) + XCTAssertEqual(box.minY, expected.minY, accuracy: accuracy, "minY \(message)", file: file, line: line) + XCTAssertEqual(box.maxX, expected.maxX, accuracy: accuracy, "maxX \(message)", file: file, line: line) + XCTAssertEqual(box.maxY, expected.maxY, accuracy: accuracy, "maxY \(message)", file: file, line: line) + } + + // MARK: The regression that motivated arc support + + /// Lucide `undo-2`'s loop is a 5.5-radius semicircle written as a single `a` + /// whose endpoint sits DIRECTLY BELOW its start. A straight line to that + /// endpoint has zero width — the curl vanishes and the glyph stops reading as + /// "undo" — so this pins the loop's full 5.5-unit bulge. + func testSemicircleLoopBulgesTheFullRadius() { + let box = drawnBounds("M14.5 9a5.5 5.5 0 0 1 0 11") + XCTAssertEqual(box.width, 5.5, accuracy: 0.02, "the loop collapsed to a straight segment") + XCTAssertEqual(box.minX, 14.5, accuracy: 0.02) + XCTAssertEqual(box.maxX, 20, accuracy: 0.02) + XCTAssertEqual(box.minY, 9, accuracy: 0.02) + XCTAssertEqual(box.maxY, 20, accuracy: 0.02) + // The extreme of the loop, a quarter turn in: dead centre of the bulge. + XCTAssertLessThan(distanceToCurve("M14.5 9a5.5 5.5 0 0 1 0 11", CGPoint(x: 20, y: 14.5)), 0.02) + } + + /// The same loop as Lucide actually authors it — two chained quarter arcs, so + /// the flattened form is a triangle rather than a line and the bounding box + /// alone would NOT catch the regression. The 45-degree points are what + /// separates a real curve from its chords. + func testUndoStyleTwoArcLoopFollowsTheCircle() { + let d = "M4 9h10.5a5.5 5.5 0 0 1 5.5 5.5a5.5 5.5 0 0 1-5.5 5.5H11" + let box = drawnBounds(d) + XCTAssertEqual(box.maxX, 20, accuracy: 0.02) + XCTAssertEqual(box.maxY, 20, accuracy: 0.02) + // Centre (14.5, 14.5), radius 5.5: the two 45-degree points sit ~1.1 units + // outside the chords a line approximation would draw. + let offset: CGFloat = 5.5 / CGFloat(2).squareRoot() + XCTAssertLessThan(distanceToCurve(d, CGPoint(x: 14.5 + offset, y: 14.5 - offset)), 0.02) + XCTAssertLessThan(distanceToCurve(d, CGPoint(x: 14.5 + offset, y: 14.5 + offset)), 0.02) + } + + // MARK: Basic sweeps + + func testQuarterCircleSpansItsRadiiAndBulges() { + let d = "M0 0 A10 10 0 0 1 10 10" + assertBounds(d, CGRect(x: 0, y: 0, width: 10, height: 10), "quarter circle") + // Centre (0,10): the mid-sweep point is at 45 degrees, 2.93 units off the + // chord. A straight line shares this bounding box, so the chord distance — + // not the box — is the assertion that has teeth. + let radial: CGFloat = 10 / CGFloat(2).squareRoot() + let mid = CGPoint(x: radial, y: 10 - radial) + XCTAssertLessThan(distanceToCurve(d, mid), 0.02) + XCTAssertGreaterThan(distanceToCurve(d, CGPoint(x: 5, y: 5)), 2.9, "curve hugged the chord") + XCTAssertEqual(samples(d).last!.x, 10, accuracy: 0.0001) + XCTAssertEqual(samples(d).last!.y, 10, accuracy: 0.0001) + } + + func testHalfCircleBulgesToTheFullRadiusOnTheSweptSide() { + // Start (5,4) -> end (5,14), r=5: a semicircle bulging right for sweep=1. + assertBounds("M5 4 A5 5 0 0 1 5 14", CGRect(x: 5, y: 4, width: 5, height: 10), "sweep=1 half") + // sweep=0 mirrors it to the left of the chord. + assertBounds("M5 4 A5 5 0 0 0 5 14", CGRect(x: 0, y: 4, width: 5, height: 10), "sweep=0 half") + } + + // MARK: Flags + + /// All four flag combinations from one pair of endpoints. Two centres times + /// two directions: minor arcs stay inside the chord's half-plane, major arcs + /// wrap past both poles. + func testAllFourLargeArcSweepCombinationsDiffer() { + let paths = [ + "M5 10 A6 6 0 0 0 15 10", // minor, counter-sweep -> below the chord + "M5 10 A6 6 0 0 1 15 10", // minor, sweep -> above the chord + "M5 10 A6 6 0 1 0 15 10", // major, counter-sweep -> wraps below + "M5 10 A6 6 0 1 1 15 10", // major, sweep -> wraps above + ] + let boxes = paths.map { drawnBounds($0) } + + for i in boxes.indices { + for j in boxes.indices where j > i { + XCTAssertNotEqual(boxes[i], boxes[j], "flag combination \(i) and \(j) drew the same arc") + } + } + + // Centres are (10, 6.683) and (10, 13.317); half-height of the bulge is + // 6 - 3.317 = 2.683 for the minor arcs, 6 + 3.317 = 9.317 for the major. + XCTAssertEqual(boxes[0].minY, 10, accuracy: 0.02, "minor sweep=0 must not cross above the chord") + XCTAssertEqual(boxes[0].maxY, 12.683, accuracy: 0.02) + XCTAssertEqual(boxes[1].maxY, 10, accuracy: 0.02, "minor sweep=1 must not cross below the chord") + XCTAssertEqual(boxes[1].minY, 7.317, accuracy: 0.02) + XCTAssertEqual(boxes[2].maxY, 19.317, accuracy: 0.02, "major sweep=0 must wrap the far pole") + XCTAssertEqual(boxes[3].minY, 0.683, accuracy: 0.02, "major sweep=1 must wrap the far pole") + // Both major arcs pass the ellipse's left and right extremes. + XCTAssertEqual(boxes[2].minX, 4, accuracy: 0.02) + XCTAssertEqual(boxes[3].maxX, 16, accuracy: 0.02) + } + + // MARK: Rotation + + func testXAxisRotationReshapesTheEllipse() { + let upright = drawnBounds("M4 12 A8 3 0 0 1 20 12") + let rotated = drawnBounds("M4 12 A8 3 45 0 1 20 12") + XCTAssertNotEqual(upright, rotated, "x-axis-rotation was ignored") + // Upright: the top half of an 8x3 ellipse centred on the chord, so it rises + // exactly ry above the chord and stays inside the endpoints in x. + XCTAssertEqual(upright.minX, 4, accuracy: 0.02) + XCTAssertEqual(upright.minY, 9, accuracy: 0.02) + XCTAssertEqual(upright.maxY, 12, accuracy: 0.02) + // Rotated 45 degrees the arc leans: it climbs far past the chord's own + // y-range and reaches left of the start point. Rotating the RESULT (or a + // bounding box) instead of the ellipse frame cannot produce this. + XCTAssertEqual(rotated.minX, -0.167, accuracy: 0.02) + XCTAssertEqual(rotated.minY, -0.167, accuracy: 0.02) + XCTAssertEqual(rotated.maxX, 20, accuracy: 0.02) + } + + /// The strongest rotation check available without a second implementation: + /// half an ellipse whose endpoints are the two ends of its major axis, so the + /// centre is the chord midpoint and every point on the curve — control points + /// included, since they are what shapes the samples — must satisfy the + /// rotated ellipse equation. + func testRotatedEllipseSamplesSatisfyTheEllipseEquation() { + // Centre (10,10), rx 10, ry 5, phi 30 degrees. + let phi = CGFloat.pi / 6 + let d = "M1.339746 5 A10 5 30 0 1 18.660254 15" + let points = samples(d, per: 40) + XCTAssertGreaterThan(points.count, 40) + for point in points { + let dx = point.x - 10, dy = point.y - 10 + let x = cos(phi) * dx + sin(phi) * dy + let y = -sin(phi) * dx + cos(phi) * dy + XCTAssertEqual(x * x / 100 + y * y / 25, 1, accuracy: 0.01, "point \(point) is off the ellipse") + } + } + + /// Rotating an ellipse by 90 degrees is the same shape as swapping its radii. + /// An implementation that rotated only the endpoints, or applied phi to the + /// control points inconsistently, breaks this identity. + func testNinetyDegreeRotationEqualsSwappedRadii() { + let rotated = samples("M2 3 A10 5 90 0 1 14 9", per: 40) + let swapped = samples("M2 3 A5 10 0 0 1 14 9", per: 40) + XCTAssertEqual(rotated.count, swapped.count) + for (a, b) in zip(rotated, swapped) { + XCTAssertEqual(a.x, b.x, accuracy: 0.0001) + XCTAssertEqual(a.y, b.y, accuracy: 0.0001) + } + } + + func testRotationOfACircleIsANoOp() { + // A circle is rotation-invariant; if phi leaked into the radii instead of + // the frame, these would diverge. + assertBounds("M0 0 A10 10 0 0 1 10 10", CGRect(x: 0, y: 0, width: 10, height: 10)) + assertBounds("M0 0 A10 10 37 0 1 10 10", CGRect(x: 0, y: 0, width: 10, height: 10)) + } + + // MARK: Degenerate input (F.6.2 / F.6.6) + + func testZeroRadiusDegradesToAStraightLine() { + let d = "M2 2 A0 0 0 0 1 10 10" + assertBounds(d, CGRect(x: 2, y: 2, width: 8, height: 8)) + // Every sample sits on the chord: y == x for this segment. + for point in samples(d) { + XCTAssertEqual(point.y, point.x, accuracy: 0.0001, "zero-radius arc left the straight line") + } + } + + func testCoincidentEndpointsDrawNothing() { + // F.6.2: an arc whose endpoints coincide is omitted entirely. The move is + // still recorded, so the path is a single point, not a stray loop. + let box = drawnBounds("M6 6 A4 4 0 1 1 6 6") + XCTAssertEqual(box.width, 0, accuracy: 0.0001) + XCTAssertEqual(box.height, 0, accuracy: 0.0001) + } + + func testNegativeRadiiUseTheirAbsoluteValue() { + assertBounds("M0 0 A-10 -10 0 0 1 10 10", CGRect(x: 0, y: 0, width: 10, height: 10)) + } + + /// F.6.6.2: radii too small to span the endpoints are scaled up, so the arc + /// still LANDS on its endpoint. Falling back to "closest reachable" would + /// leave a visible gap before the next command. + func testTooSmallRadiiScaleUpAndStillReachTheEndpoint() { + let d = "M0 0 A1 1 0 0 1 10 10" + let end = samples(d).last! + XCTAssertEqual(end.x, 10, accuracy: 0.0001) + XCTAssertEqual(end.y, 10, accuracy: 0.0001) + // Scaled to r = 7.0711 the arc is a half circle centred on the chord's + // midpoint (5,5), so it sweeps through the circle's top and right extremes + // — well outside the endpoints' own 10x10 box. + let radius: CGFloat = 5 * CGFloat(2).squareRoot() + assertBounds(d, CGRect(x: 0, y: 5 - radius, width: 5 + radius, height: 5 + radius)) + XCTAssertLessThan(distanceToCurve(d, CGPoint(x: 10, y: 0)), 0.02) + } + + // MARK: Parser contract + + func testRelativeArcResolvesAgainstTheCurrentPoint() { + // Same circle authored absolutely and relatively. + XCTAssertEqual(drawnBounds("M5 4 A5 5 0 0 1 5 14"), drawnBounds("M5 4 a5 5 0 0 1 0 10")) + } + + func testArcLeavesTheCurrentPointForTheNextCommand() { + // The command after the arc must start from the arc's endpoint, not from + // where the arc began. + assertBounds("M0 0 A10 10 0 0 1 10 10 L10 20", CGRect(x: 0, y: 0, width: 10, height: 20)) + } + + func testTruncatedArcBailsWithoutSpinning() { + // Missing the final parameter: the parser must return, not loop forever. + let path = LucideShape(parts: [.path("M2 2 A4 4 0 0 1 10")]).path(in: grid) + XCTAssertFalse(path.isEmpty) // the move survived + XCTAssertEqual(path.boundingRect.width, 0, accuracy: 0.0001) + } + + // MARK: Shipped glyphs + + /// `pencil`, `mouse-pointer-2` and `square-dashed` all carry `a` commands and + /// therefore change shape with real arcs. Their curves must still land inside + /// the 24-unit design grid — the F.6.6.2 scale-up on `pencil`'s r=1 eraser + /// arc is the one that could plausibly push a glyph out of bounds. + func testArcCarryingGlyphsStayOnTheDesignGrid() { + let glyphs: [(String, LucideIcon)] = [ + ("pencil", .pencil), ("mousePointer", .mousePointer), ("squareDashed", .squareDashed), + ] + for (name, icon) in glyphs { + var points: [CGPoint] = [] + for case .path(let d) in icon.parts { points += samples(d) } + XCTAssertFalse(points.isEmpty, "\(name) drew no arcs or lines") + for point in points { + XCTAssertGreaterThanOrEqual(point.x, -0.5, "\(name) spills left") + XCTAssertGreaterThanOrEqual(point.y, -0.5, "\(name) spills up") + XCTAssertLessThanOrEqual(point.x, 24.5, "\(name) spills right") + XCTAssertLessThanOrEqual(point.y, 24.5, "\(name) spills down") + } + } + } + + /// `square-dashed`'s corner strokes are 90-degree rounds, previously drawn as + /// diagonals. Pinning one corner's mid-sweep point keeps a future parser + /// change from silently flattening them again. + func testSquareDashedCornersRenderAsRounds() { + let d = "M5 3a2 2 0 0 0-2 2" + let offset: CGFloat = 2 - 2 / CGFloat(2).squareRoot() // 0.586 units off the chord + XCTAssertLessThan(distanceToCurve(d, CGPoint(x: 3 + offset, y: 3 + offset)), 0.02) + assertBounds(d, CGRect(x: 3, y: 3, width: 2, height: 2)) + } +} From 61151ea727db00e603ac64f14fdcd2d5518a1514 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:19:13 -0500 Subject: [PATCH 18/24] feat(escape): back out one level at a time via a local key monitor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Escape previously did almost nothing: the only handler was `.onExitCommand` on the shared card, which fires only when the overlay PANEL is key — and the panel is made key solely by a card focusing its text field. In annotate mode with nothing open (the state a user most wants to leave) the HOST window is key, so no view in the panel ever saw the keystroke. - `EscapeRule` (pure, platform-free, unit-tested): drag -> card -> mode, and pass-through when idle so the host's own Escape is untouched. Drag beats card because the catcher stays live behind an open composer, so the two coexist. `EscapeAction.consumesEvent` pins the swallow contract: AnnotKit is in-process with its host, so a handled-and-forwarded Escape would be acted on twice. - `AnnotationSession` gains `isDrawingFrame` / `frameDragGeneration` / `hasOpenCard` so the handler, which lives outside the view, can see an in-flight gesture. Only the FLAG moves: the band's rect stays window-local `@State`, since everything in the session is AX screen space. - `OverlayController` owns an `NSEvent` local key monitor, installed in `start()` and removed in BOTH `stop()` and `unmount()` — a leaked monitor would keep eating the host app's Escape for the life of the process. - The composer draft is now cleared by the composer CLOSING rather than by the Cancel button, so a dismissal from anywhere no longer leaves a half-typed comment to reappear over the next selection. `.onExitCommand` is deleted, leaving Escape exactly one owner. Co-Authored-By: Claude Opus 5 (1M context) --- PARITY.md | 1 + .../AnnotKit/Overlay/AnnotationSession.swift | 60 ++++++++++ Sources/AnnotKit/Overlay/EscapeRule.swift | 58 ++++++++++ Sources/AnnotKit/Overlay/OverlayView.swift | 106 +++++++++++++---- .../AnnotKit/macOS/OverlayController.swift | 91 +++++++++++++++ .../AnnotationSessionTests.swift | 108 ++++++++++++++++++ .../EscapeMonitorLifecycleTests.swift | 85 ++++++++++++++ Tests/AnnotKitTests/EscapeRuleTests.swift | 88 ++++++++++++++ 8 files changed, 575 insertions(+), 22 deletions(-) create mode 100644 Sources/AnnotKit/Overlay/EscapeRule.swift create mode 100644 Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift create mode 100644 Tests/AnnotKitTests/EscapeRuleTests.swift diff --git a/PARITY.md b/PARITY.md index 78610eb..3d57fcb 100644 --- a/PARITY.md +++ b/PARITY.md @@ -17,6 +17,7 @@ row; each asymmetry is closed by code or has a tracked mitigation. | Child navigation (select child) | `ChildNavigationSource` over the AX tree; the opt-in view-tree source implements it over `NSView` | `ChildNavigationSource` over the `UIView` tree | none — the ORDERING is the one pure `ChildNavigationRule` (contains the gesture's hint, then seeded, then larger area, then lowest index) on all three adapters, which supply only `[ChildCandidate]`. All three also share the cost shape the protocol mandates: re-find the bound element by descending its containing window along its own frame centre (bounded by tree DEPTH, not tree size), then collect the NEAREST meaningful descendants, each branch stopping at its first meaningful node — so a SwiftUI host's unidentified layout wrappers are descended THROUGH rather than offered as children, which is what keeps the Child control alive under pure SwiftUI on both platforms. macOS additionally skips window chrome and its own overlay window during that descent; UIKit has no chrome, and the overlay is already excluded by the shared window lookup | | Frame-mode anchoring + hover gating | shared `AnnotationSession` (`selectionAnchorFrame`, the `tool == .point` hover gate, `setTool` clearing `hovered`) rendered by the shared `OverlayView` | same | none in the code — all of it is session-level and platform-free, and one SwiftUI view renders it. ASYMMETRIC VERIFICATION, recorded as a gap rather than closed: `AnnotKitOverlayProbe` Phase 8 drives navigation, the note's `component`, the hover gate and the frame anchor against a REAL accessibility tree, and it is macOS-only (`#if os(macOS)`, AppKit + `AXUIElement`), so the iOS adapter's live behaviour is covered only by unit tests over the pure rules. Mitigated, not fixed, by the fact that everything Phase 8 asserts about anchoring and hover lives in the shared session; what remains unverified on iOS is the ADAPTER's candidate collection. Note the hover gate is also moot on touch-only iOS — hover exists there only with a trackpad or pencil — so the reported symptom cannot arise without a pointer | | Marquee drag threshold | cursor slop (a mouse does not move on a deliberate click) | larger touch slop | ASYMMETRIC BY DESIGN, owned by the drag UI, not the adapters: a finger rolls several points on a deliberate tap, so the macOS threshold on iOS would turn taps into marquees. Below the threshold both platforms route the gesture to the point path (`select(atAXPoint:)`), per the caller contract on `select(inAXRect:)` | +| Escape (back out one level) | `NSEvent.addLocalMonitorForEvents(matching: .keyDown)` owned by `OverlayController`, resolving the shared pure `EscapeRule` (drag → card → mode, pass-through when idle) | none — no Escape key exists on iOS | ASYMMETRIC BY THE HARDWARE, not by the code: a touch device has no Escape key, so there is nothing to bind. The DECISION is platform-free and unit-tested (`EscapeRule`), so an iOS back-out affordance (a swipe, a hardware-keyboard binding on iPad) can adopt it without re-deriving the precedence; only the macOS delivery mechanism is `#if os(macOS)`. The mechanism is a LOCAL KEY MONITOR rather than a SwiftUI modifier (`.onExitCommand`) because a panel-scoped modifier only fires while the overlay panel is KEY, and the panel is made key solely by a card focusing its text field — so in annotate mode with nothing open, the state a user most wants to leave, the HOST window is key and no view in the panel ever sees the keystroke. A local monitor works precisely because AnnotKit is in-process with its host: the Escape headed for the host window passes through it first, and it can swallow what it acted on (`EscapeAction.consumesEvent`), which a global monitor cannot. It is installed in `start()` and removed in BOTH `stop()` and `unmount()` — a monitor outliving the overlay would keep eating the host app's own Escape for the life of the process | | Overlay excluded from element lookup | AX window identifier (`AXIntrospection.overlayWindowIdentifier`) filtered out of every `kAXWindows` read | `PassThroughWindow` TYPE identity filtered out of `IOSElementSource.windows()` | ASYMMETRIC BY NECESSITY — the hosts are different window kinds. macOS's overlay is a separate `NSPanel` matched by the identifier the controller stamps on it; iOS's is a `UIWindow` in the HOST's scene sharing its pid, so no pid/scene filter separates it and a type check (internal to the module) cannot drift the way an identifier convention can. Both filter in the shared window lookup, so snapshot / hit-test / region-anchor / marquee agree; leaving it in would let a marquee bind the user's note to AnnotKit's own UI | | Coordinate space | Cocoa bottom-left to AX top-left flip | UIKit top-left native | iOS needs no flip; shared `ScreenSpace` used only on macOS | | Screenshot | ScreenCaptureKit / `cacheDisplay` | `UIGraphicsImageRenderer` + `drawHierarchy` | both capture own hierarchy only; no cross-window or secure overlays | diff --git a/Sources/AnnotKit/Overlay/AnnotationSession.swift b/Sources/AnnotKit/Overlay/AnnotationSession.swift index 9b69af6..fe2d84b 100644 --- a/Sources/AnnotKit/Overlay/AnnotationSession.swift +++ b/Sources/AnnotKit/Overlay/AnnotationSession.swift @@ -152,6 +152,33 @@ public final class AnnotationSession: ObservableObject { /// closes the other — so exactly one card is ever on screen. @Published public private(set) var editingNoteID: String? + /// True while the catcher is drawing a frame. UI-only state, and deliberately + /// only the FLAG: the band's rect stays `@State` in the view because it is + /// WINDOW-LOCAL while everything else here is AX screen space, and a rect that + /// looks like the others but is measured from a different origin is the exact + /// confusion `OverlayView`'s header warns about (invisible on the primary + /// display, off by the window origin on every other one). + /// + /// It lives here anyway because the Escape handler is OUTSIDE the view — a + /// process-wide key monitor owned by the host controller — and cannot otherwise + /// tell an in-flight gesture from an idle catcher. + @Published public private(set) var isDrawingFrame: Bool = false + + /// Bumped when an in-flight frame drag is cancelled. The cancel signal has to be + /// an EDGE rather than a flag because AppKit delivers the gesture's `onEnded` + /// regardless — cancelling cannot suppress it — so the view compares the + /// generation it captured at drag start against this one on release and skips + /// resolution when they differ. A plain `isDrawingFrame == false` test would also + /// swallow the release of a NEW drag started in the same run, and a boolean + /// "wasCancelled" would need clearing by whoever read it last. + @Published public private(set) var frameDragGeneration: Int = 0 + + /// True when a composer or a pin editor is on screen. These two are mutually + /// exclusive by construction (``beginEditing(id:)`` nils `selected`; the `select` + /// paths nil `editingNoteID`), so this is "a card is up" — the single question + /// ``EscapeRule`` asks. + public var hasOpenCard: Bool { selected != nil || editingNoteID != nil } + private let source: ElementSource private let sink: AnnotationSink private let route: () -> String? @@ -206,6 +233,13 @@ public final class AnnotationSession: ObservableObject { selected = nil // Leaving annotate mode hides the pins, so any open pin editor must go too. editingNoteID = nil + // A drag in flight when the mode ends never receives its `onEnded`: the + // catcher is gated on annotate mode, so it is REMOVED from the view tree and + // SwiftUI drops the gesture silently. Without this, `isDrawingFrame` latches + // true into the next session — where Escape would "cancel" a drag nobody is + // making instead of exiting — and the drawn band, whose layer is not gated on + // mode, keeps painting over the idle overlay. + if isDrawingFrame { cancelFrameDrag() } } /// Update the hover highlight for a screen point (AX top-left coordinates). @@ -233,6 +267,32 @@ public final class AnnotationSession: ObservableObject { hovered = nil } + /// The catcher's band just became visible (the press cleared the travel + /// threshold), so a frame drag is in flight. + public func beginFrameDrag() { + isDrawingFrame = true + } + + /// The drag reached its natural end (the release). Deliberately does NOT bump the + /// generation: a normal release must still resolve into a selection, and the + /// generation is precisely the signal that says "don't". + public func endFrameDrag() { + isDrawingFrame = false + } + + /// Abandon the frame being drawn. Bumping the generation is what actually + /// cancels: the release still arrives, and the view skips resolution because the + /// generation moved under it. + /// + /// Touches NOTHING else — not `selected`, not `pending`. The catcher stays live + /// behind an open composer, so the drag being abandoned may have started while a + /// previous note was half-typed; clearing the selection here would make Escape + /// destroy a draft the user was not even interacting with. + public func cancelFrameDrag() { + frameDragGeneration &+= 1 + isDrawingFrame = false + } + /// Select the element under a screen point (AX top-left coordinates). /// /// When nothing resolves (decoration, dividers, gaps beyond any container's diff --git a/Sources/AnnotKit/Overlay/EscapeRule.swift b/Sources/AnnotKit/Overlay/EscapeRule.swift new file mode 100644 index 0000000..bb9d8f9 --- /dev/null +++ b/Sources/AnnotKit/Overlay/EscapeRule.swift @@ -0,0 +1,58 @@ +/// What a press of Escape means, given only what is on screen. +/// +/// Modelled as a value rather than handled inline in the key monitor for the same +/// reason ``SelectionGesture`` is: a decision buried in an AppKit event closure can +/// only be checked by a human with a keyboard, and this one is reachable from four +/// different UI states that a human tester will not think to enumerate. +public enum EscapeAction: Sendable, Hashable { + /// Throw away the frame the user is mid-way through drawing; the selection that + /// was live before the drag is untouched. + case cancelDrag + /// Close the open composer or pin editor, discarding its draft. + case dismissCard + /// Leave annotate mode entirely. + case exitAnnotateMode + /// AnnotKit has no claim on this keystroke; the host must see it. + case passThrough + + /// Whether the key monitor should SWALLOW the event. + /// + /// This is the monitor's whole contract with the host, and getting it wrong is + /// not cosmetic: AnnotKit runs IN-PROCESS with its host, so an Escape it acts on + /// and also forwards is delivered twice — the overlay leaves annotate mode while + /// the host closes its own sheet, from one press. Only ``passThrough`` (where + /// AnnotKit deliberately did nothing) may forward. + public var consumesEvent: Bool { self != .passThrough } +} + +/// The pure Escape decision: given the three pieces of overlay state that a press +/// of Escape can plausibly refer to, name the ONE thing it backs out of. +/// +/// The design is "undo one level at a time", and the ordering below is that policy. +/// A first Escape that exited annotate mode outright would take a half-typed comment +/// with it — the single irreversible thing in the flow, since a dismissed mode can be +/// re-entered with one click but a discarded draft cannot be retyped from anywhere. +public enum EscapeRule { + /// Resolve a press of Escape. Precedence is strict and the order is the whole + /// rule: + /// + /// 1. **Not annotating → ``EscapeAction/passThrough``.** AnnotKit is not active, + /// so the host's own Escape handling — closing ITS sheets, dismissing ITS + /// menus — must be exactly as it was before AnnotKit was installed. An + /// overlay that ate Escape while idle would break the host app permanently + /// and look like the host's bug. + /// 2. **Drawing a frame → ``EscapeAction/cancelDrag``.** The in-flight gesture is + /// the most immediate thing on screen. Tested BEFORE the card rather than + /// after because the two genuinely COEXIST: the catcher stays live behind an + /// open composer, so a user can begin drawing a new frame while the previous + /// note's composer is still up. Card-first would then discard the draft the + /// user never touched and leave the band they are actively dragging on screen. + /// 3. **An open card → ``EscapeAction/dismissCard``.** + /// 4. **Otherwise → ``EscapeAction/exitAnnotateMode``.** + public static func resolve(isAnnotating: Bool, isDrawingFrame: Bool, hasOpenCard: Bool) -> EscapeAction { + guard isAnnotating else { return .passThrough } + if isDrawingFrame { return .cancelDrag } + if hasOpenCard { return .dismissCard } + return .exitAnnotateMode + } +} diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index 7dd4cfa..fe664f1 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -51,6 +51,16 @@ struct OverlayView: View { /// selection — that comes back as `session.selected` and is drawn by the /// highlight branch. @State private var marqueeRect: CGRect? + /// `session.frameDragGeneration` as it stood when the current drag began, and + /// re-synced to the session's at every release (and whenever a fresh catcher + /// appears). The difference between this and the live value IS the cancellation: + /// AppKit delivers the gesture's `onEnded` whether or not Escape was pressed, and + /// nothing in SwiftUI can suppress it, so the release has to ask. + /// + /// Re-syncing at the END of every press is what makes the NEXT press start + /// uncancelled — without it, one cancelled drag would leave the two values apart + /// forever and kill frame mode for the rest of the session. + @State private var frameDragGeneration = 0 var body: some View { ZStack(alignment: .topLeading) { @@ -87,6 +97,25 @@ struct OverlayView: View { editDraft = note.comment } } + // The draft belongs to the composer, so it dies WITH the composer — for any + // reason it closes, not just the Cancel button. The button used to clear it + // itself, which made "no selection implies no draft" a property of one code + // path rather than an invariant: dismissing from anywhere else (Escape, which + // now goes through the key monitor, or `stop()`) left `comment` populated and + // it reappeared, pre-filled, over the NEXT element the user selected. + // + // Deliberately not `if session.selected == nil` inside the capture path: a + // capture also nils `selected`, and `addNote()` clearing the field itself is + // redundant with this rather than in conflict with it. + .onChange(of: session.selected) { _, element in + if element == nil { comment = "" } + } + // Drop the band the instant a drag is cancelled, rather than waiting for the + // release: the whole point of Escape here is that the rectangle stops being + // dragged around under the cursor. + .onChange(of: session.frameDragGeneration) { _, _ in + marqueeRect = nil + } } /// What is drawn ON the app, between the catcher and the chrome. Four states, @@ -194,6 +223,12 @@ struct OverlayView: View { if session.mode == .annotating { Color.clear .contentShape(Rectangle()) + // A freshly-inserted catcher has no press in flight, so it must start + // in sync. This is not belt-and-braces: leaving annotate mode DURING a + // drag cancels it and then removes this view, so the release that + // would have re-armed the generation never arrives — and the first + // frame drag of the next session would be born cancelled. + .onAppear { frameDragGeneration = session.frameDragGeneration } .onContinuousHover { phase in switch phase { case .active(let point): @@ -232,11 +267,35 @@ struct OverlayView: View { guard session.tool == .frame, SelectionGesture.travelledFarEnough(from: value.startLocation, to: value.location) else { return } + // A press Escape has already cancelled stays dead for the + // REST of the press. The button is usually still held when + // Escape lands, so the next twitch of the mouse arrives + // here — and without this the band would spring back under + // the cursor and the release would resolve it, which is + // exactly what was just cancelled. + guard frameDragGeneration == session.frameDragGeneration else { return } + // Announce the drag on the band's FIRST appearance, when + // it becomes a thing the user can see and therefore a + // thing Escape can refer to. + if marqueeRect == nil { + frameDragGeneration = session.frameDragGeneration + session.beginFrameDrag() + } marqueeRect = SelectionGesture.localRect(from: value.startLocation, to: value.location) session.clearHover() } .onEnded { value in marqueeRect = nil + // A cancelled drag still delivers this release; the moved + // generation is how we know to drop it on the floor. + // Resolving anyway would plant exactly the note Escape + // was pressed to prevent. + let cancelled = frameDragGeneration != session.frameDragGeneration + session.endFrameDrag() + // Re-arm for the next press BEFORE bailing out, so a + // cancellation costs one drag and not the mode. + frameDragGeneration = session.frameDragGeneration + guard !cancelled else { return } switch SelectionGesture.resolve( tool: session.tool, from: value.startLocation, @@ -275,7 +334,8 @@ struct OverlayView: View { } /// The WRITE card: a shared ``AnnotationCard`` anchored to the selected - /// element, with a Cancel / Add note footer. Enter submits, Escape cancels. + /// element, with a Cancel / Add note footer. Enter submits; Escape dismisses via + /// the host's key monitor (``EscapeRule``), not from inside this view. private var composer: some View { AnnotationCard( header: composerHeader, @@ -285,10 +345,6 @@ struct OverlayView: View { // is not re-inserted, so `.onAppear` won't refire). focusKey: session.selected?.id ?? "", onSubmit: { addNote() }, - onCancel: { - comment = "" - session.cancelSelection() - }, onFocusRequest: onFocusRequest, // Tree navigation: rebind the note to the enclosing component, or to a // component inside the current one. Its own row rather than a third @@ -328,10 +384,11 @@ struct OverlayView: View { } ) { HStack { - Button("Cancel") { - comment = "" - session.cancelSelection() - } + // No `comment = ""` here any more: the draft is cleared by the + // composer CLOSING (see the `session.selected` hook on the ZStack), so + // every dismissal path — this button, Escape, leaving the mode — + // clears it identically. + Button("Cancel") { session.cancelSelection() } Spacer() Button("Add note") { addNote() } .buttonStyle(.borderedProminent) @@ -355,7 +412,8 @@ struct OverlayView: View { /// The EDIT card: the SAME shared ``AnnotationCard`` chrome as the composer, /// anchored to the tapped pin instead of an element, with a Delete / Save - /// footer. Enter saves, Escape cancels, and Save/Delete/click-away all end + /// footer. Enter saves, Escape (via the host's key monitor) closes it, and + /// Save/Delete/click-away all end /// editing. Because it lives in the overlay panel (not a system `.popover`), /// it inherits the composer's reliable `panel.makeKey()` focus. private func editCard(note: AnnotationNote, anchor: CGPoint) -> some View { @@ -366,7 +424,6 @@ struct OverlayView: View { // Re-focus when the editor moves pin→pin without re-insertion. focusKey: note.id, onSubmit: { saveEdit(note) }, - onCancel: { session.endEditing() }, onFocusRequest: onFocusRequest, // No navigation row: this card edits a note that has ALREADY been // captured, whose selector, component and element path were frozen at @@ -508,10 +565,13 @@ struct OverlayView: View { /// ``ComposerCaret`` pointing at its anchor, the drop shadow, the /// ``ComposerPlacement`` offset, first-responder focus (host `makeKey` + /// `@FocusState` + a next-tick re-assert to beat the insertion race), and the -/// keyboard contract (Enter submits, Shift+Enter inserts a newline, Escape -/// cancels on macOS). The two flows differ ONLY in the `header` text, the `text` -/// binding, the `placement` anchor, the optional `navigation` row, and the -/// `footer` button row. +/// keyboard contract it can actually honour: Enter submits and Shift+Enter inserts +/// a newline. Escape is NOT handled here — it is the host key monitor's, because a +/// view-level handler only ever sees it once the overlay panel is key (see the note +/// at the bottom of `body`). +/// +/// The two flows differ ONLY in the `header` text, the `text` binding, the +/// `placement` anchor, the optional `navigation` row, and the `footer` button row. private struct AnnotationCard: View { /// Header label: the composer shows the element's selection label; the editor /// shows the note's selector. @@ -527,8 +587,6 @@ private struct AnnotationCard: View { let focusKey: String /// Enter (no Shift) and the trailing footer button. let onSubmit: () -> Void - /// Escape (macOS) and the leading footer button (Cancel for the composer). - let onCancel: () -> Void /// Make the host panel key so the field accepts keystrokes (`panel.makeKey()` /// on macOS; a no-op on iOS, where `@FocusState` alone raises the keyboard). let onFocusRequest: () -> Void @@ -549,7 +607,7 @@ private struct AnnotationCard: View { .font(.headline) .lineLimit(1) Spacer(minLength: 8) - // Enter submits; Shift+Enter inserts a newline; Esc cancels. + // Enter submits; Shift+Enter inserts a newline. Text("⏎ save · ⇧⏎ newline") .font(.caption2) .foregroundStyle(.secondary) @@ -599,10 +657,14 @@ private struct AnnotationCard: View { // the view (so `.onAppear` won't refire). .onAppear { focus() } .onChange(of: focusKey) { _, _ in focus() } - #if os(macOS) - // Escape cancels without committing (macOS-only API; iOS has no Esc key). - .onExitCommand { onCancel() } - #endif + // NO `.onExitCommand` here, deliberately. It only ever fired when the overlay + // PANEL was key — which happens only once a card has focused its text field — + // so it could not dismiss anything in the state a user most wants out of + // (annotate mode, nothing open, host window key). Escape now has exactly ONE + // owner, the controller's local key monitor, which sees the keystroke wherever + // it is delivered. Restoring this modifier would not be redundancy: with the + // panel key BOTH handlers would run on one press, closing the card and then + // acting again on the state that leaves behind. } /// Focus the text field, making the host panel key FIRST so the non-activating diff --git a/Sources/AnnotKit/macOS/OverlayController.swift b/Sources/AnnotKit/macOS/OverlayController.swift index 5f00483..c8dc636 100644 --- a/Sources/AnnotKit/macOS/OverlayController.swift +++ b/Sources/AnnotKit/macOS/OverlayController.swift @@ -42,6 +42,13 @@ public final class OverlayController: NSObject { /// host stops instead of re-syncing against a stale (or detached) window. private var settleGeneration = 0 + /// The Escape key monitor's token, non-nil ONLY while annotate mode is running. + /// Held so it can be removed in both exits (``stop()`` and ``unmount()``): AppKit + /// keeps a local monitor alive until it is explicitly removed, so a leaked one + /// would keep swallowing the host app's own Escape long after the overlay was + /// gone — a bug that presents as "this app's dialogs stopped closing". + private var escapeMonitor: Any? + public init(session: AnnotationSession) { self.session = session super.init() @@ -75,6 +82,11 @@ public final class OverlayController: NSObject { public func unmount() { NotificationCenter.default.removeObserver(self) + // Unmounting can happen mid-annotate (a host window closing), which never + // routes through `stop()`. A monitor outliving the overlay it belongs to is + // the one failure here that damages the HOST rather than AnnotKit: it keeps + // eating Escape for the life of the process. + removeEscapeMonitor() // Invalidate any in-flight settle poll so it cannot re-sync a detached host. settleGeneration += 1 lastSyncedHostFrame = .null @@ -101,9 +113,14 @@ public final class OverlayController: NSObject { host?.orderFront(nil) session.start() syncFrameAndOrigin() + installEscapeMonitor() } public func stop() { + // Remove FIRST: the monitor exists to serve annotate mode, and leaving it + // installed for even the rest of this call would let a keystroke arrive while + // the session is half-torn-down. + removeEscapeMonitor() session.stop() syncFrameAndOrigin() } @@ -116,6 +133,80 @@ public final class OverlayController: NSObject { try? ClipboardSink().flush(session.pending) } + // MARK: - Escape + + /// Start watching for Escape while annotate mode runs. + /// + /// A LOCAL monitor, not a SwiftUI modifier and not a global one, and both halves + /// of that matter: + /// + /// * A panel-scoped modifier (`.onExitCommand`) only fires when the overlay panel + /// is KEY, and the panel is made key solely by a card focusing its text field. + /// In the state a user most wants to leave — annotate mode with nothing open — + /// the HOST window is key, so no view in the panel ever sees the keystroke. + /// * A local monitor sees events on their way to THIS process's windows, which + /// works precisely because AnnotKit is in-process with its host: the Escape + /// headed for the host window passes through here first. (A global monitor + /// watches OTHER apps, cannot consume the event, and would need accessibility + /// permission — all three wrong for this.) + private func installEscapeMonitor() { + guard escapeMonitor == nil else { return } + escapeMonitor = NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] event in + guard let self else { return event } + return self.handleKeyDown(event) + } + } + + private func removeEscapeMonitor() { + guard let escapeMonitor else { return } + NSEvent.removeMonitor(escapeMonitor) + self.escapeMonitor = nil + } + + /// Decide one key-down. EVERY event in the app flows through here while annotate + /// mode runs, so the non-Escape path does nothing but return the event: any work + /// on this path is work added to every keystroke the user types into their own + /// app, and any early `return nil` is a character they never see. + private func handleKeyDown(_ event: NSEvent) -> NSEvent? { + // 53 is Escape's virtual key code — a hardware position, so it is + // layout-independent (a character comparison would miss on layouts that + // remap, and `charactersIgnoringModifiers` is empty for some IMEs). + guard event.keyCode == 53 else { return event } + + let action = EscapeRule.resolve( + isAnnotating: session.mode == .annotating, + isDrawingFrame: session.isDrawingFrame, + hasOpenCard: session.hasOpenCard + ) + switch action { + case .cancelDrag: + session.cancelFrameDrag() + case .dismissCard: + // The composer and the pin editor are mutually exclusive, and the composer + // wins the same theoretical tie the view's render ladder gives it, so the + // card that closes is always the card that is drawn. + if session.selected != nil { + session.cancelSelection() + } else { + session.endEditing() + } + case .exitAnnotateMode: + // The CONTROLLER's stop, never `session.stop()`: leaving the mode also + // shrinks the panel back to the toolbar corner and re-syncs the AX origin. + // Stopping the session alone would leave a full-window transparent panel + // over the host, swallowing every click with no visible overlay to explain + // why — and this monitor would stay installed on top of that. + stop() + case .passThrough: + break + } + // Swallow anything we acted on. Forwarding a handled Escape means the host + // ALSO acts on it (we are in its process, and the event is still on its way to + // its key window), so one press would both leave annotate mode and close the + // host's sheet. + return action.consumesEvent ? nil : event + } + // MARK: - Host window /// Pick the host window once, excluding our own panels so we never attach to diff --git a/Tests/AnnotKitTests/AnnotationSessionTests.swift b/Tests/AnnotKitTests/AnnotationSessionTests.swift index 9a8dada..4aabcba 100644 --- a/Tests/AnnotKitTests/AnnotationSessionTests.swift +++ b/Tests/AnnotKitTests/AnnotationSessionTests.swift @@ -793,6 +793,114 @@ final class AnnotationSessionTests: XCTestCase { XCTAssertNil(session.selectionAnchorFrame, "and so does leaving annotate mode") } + // MARK: - Frame drag lifecycle (what Escape reads) + + func testBeginAndEndFrameDragToggleTheFlag() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + XCTAssertFalse(session.isDrawingFrame, "no band, no drag") + session.beginFrameDrag() + XCTAssertTrue(session.isDrawingFrame) + session.endFrameDrag() + XCTAssertFalse(session.isDrawingFrame, "the release ends the drag") + } + + func testEndingADragDoesNotLookLikeACancellation() { + // The generation is the CANCEL signal, so an ordinary release must leave it + // alone. Bumping it here would make the view discard every completed frame + // drag — the feature would appear to do nothing at all. + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + let before = session.frameDragGeneration + session.beginFrameDrag() + session.endFrameDrag() + XCTAssertEqual(session.frameDragGeneration, before, "a normal release must still resolve") + } + + func testCancelFrameDragBumpsTheGenerationAndEndsTheDrag() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + let before = session.frameDragGeneration + session.beginFrameDrag() + session.cancelFrameDrag() + XCTAssertEqual(session.frameDragGeneration, before + 1, "the moved generation IS the cancellation") + XCTAssertFalse(session.isDrawingFrame, "and the drag is over") + } + + func testCancellingADragLeavesAPreviousSelectionUntouched() { + // The catcher stays live BEHIND an open composer, so the drag being abandoned + // may have been started while an earlier note was half-typed. Escape on that + // drag must cost the drag and nothing else — clearing the selection here would + // destroy a draft the user was not even interacting with. + let leaf = makeLadderElement("Leaf", frame: CGRect(x: 100, y: 100, width: 60, height: 40)) + let session = AnnotationSession( + source: MarqueeSource(ladder: [leaf]), sink: NotesFileSink(path: "/dev/null") + ) + session.start() + let drawn = CGRect(x: 110, y: 120, width: 40, height: 20) + session.select(inAXRect: drawn) + XCTAssertEqual(session.selected?.id, "Leaf") + + session.beginFrameDrag() + session.cancelFrameDrag() + XCTAssertEqual(session.selected?.id, "Leaf", "the open composer survives a cancelled drag") + XCTAssertEqual(session.selectionAnchorFrame, drawn, "and so does its anchor") + } + + func testLeavingAnnotateModeMidDragCancelsIt() { + // A drag in flight when the mode ends never gets its release: the catcher is + // gated on annotate mode and SwiftUI drops the gesture with the view. A + // latched flag would tell the next session's Escape to cancel a drag nobody + // is making, and the moved generation is what clears the drawn band. + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + let before = session.frameDragGeneration + session.beginFrameDrag() + session.stop() + XCTAssertFalse(session.isDrawingFrame, "the mode took the drag with it") + XCTAssertEqual(session.frameDragGeneration, before + 1, "and cancelled it, so the band clears") + } + + func testStoppingWithNoDragInFlightDoesNotBumpTheGeneration() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + let before = session.frameDragGeneration + session.stop() + XCTAssertEqual(session.frameDragGeneration, before, "nothing to cancel, nothing to signal") + } + + // MARK: - hasOpenCard + + func testHasOpenCardIsFalseWithNothingOnScreen() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + XCTAssertFalse(session.hasOpenCard, "idle shows no card") + session.start() + XCTAssertFalse(session.hasOpenCard, "and neither does an empty annotate mode") + } + + func testHasOpenCardIsTrueForTheComposer() { + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + XCTAssertTrue(session.hasOpenCard, "a selection is an open composer") + session.cancelSelection() + XCTAssertFalse(session.hasOpenCard, "which closes with the selection") + } + + func testHasOpenCardIsTrueForThePinEditor() { + // The other card. Missing this branch would make Escape skip straight past an + // open pin editor and leave annotate mode with the editor still on screen. + let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) + session.start() + session.select(atAXPoint: .zero) + guard let note = session.addNote(comment: "fix this") else { return XCTFail("expected a note") } + XCTAssertFalse(session.hasOpenCard, "capturing closed the composer") + session.beginEditing(id: note.id) + XCTAssertTrue(session.hasOpenCard, "a pin editor is a card too") + session.endEditing() + XCTAssertFalse(session.hasOpenCard) + } + func testClearHoverDropsHighlightButKeepsSelection() { let session = AnnotationSession(source: StubSource(makeElement()), sink: NotesFileSink(path: "/dev/null")) session.start() diff --git a/Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift b/Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift new file mode 100644 index 0000000..df38a3a --- /dev/null +++ b/Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift @@ -0,0 +1,85 @@ +#if os(macOS) +import AppKit +import CoreGraphics +import Foundation +import XCTest +@testable import AnnotKit + +/// A minimal source so a controller can be built without an accessibility tree. +@MainActor +private final class NullSource: ElementSource { + func snapshot() -> [WindowSnapshot] { [] } + func hitTest(_ point: CGPoint) -> Element? { nil } + func selector(for element: Element) -> String { "#none" } + func screenshot(of element: Element?) async throws -> CapturedImage { + CapturedImage(pngData: Data(), pixelWidth: 1, pixelHeight: 1) + } +} + +/// The Escape key monitor's INSTALL/REMOVE lifecycle, which is the one part of this +/// feature that can damage the host app: `NSEvent` keeps a local monitor alive until +/// it is explicitly removed, so one left behind keeps swallowing the host's own +/// Escape for the life of the process — presenting as "this app's dialogs stopped +/// closing", with nothing on screen to connect it to AnnotKit. +/// +/// Read through a `Mirror` because there is no API to ask AppKit what monitors are +/// installed. That is a deliberate trade: the alternative is exposing the token just +/// so a test can see it, and this contract (nil whenever the overlay is not +/// annotating) is worth more than the coupling costs. +@MainActor +final class EscapeMonitorLifecycleTests: XCTestCase { + private func monitorIsInstalled(_ controller: OverlayController) -> Bool { + for child in Mirror(reflecting: controller).children where child.label == "escapeMonitor" { + // The property is `Any?`; a nil optional still reflects as a child, so + // unwrap through `Optional` rather than testing for its presence. + return (child.value as? Any?) .map { $0 != nil } ?? false + } + XCTFail("escapeMonitor is gone — this test is asserting nothing") + return false + } + + private func makeController() -> OverlayController { + // Force NSApp into existence: `start()` activates the app, and `NSApp` is an + // implicitly-unwrapped optional that is nil until something touches it. + _ = NSApplication.shared + return OverlayController( + session: AnnotationSession(source: NullSource(), sink: NotesFileSink(path: "/dev/null")) + ) + } + + func testMonitorIsInstalledOnlyWhileAnnotating() { + let controller = makeController() + XCTAssertFalse(monitorIsInstalled(controller), "idle must not watch the host's keystrokes") + controller.start() + XCTAssertTrue(monitorIsInstalled(controller), "annotate mode needs to see Escape") + controller.stop() + XCTAssertFalse(monitorIsInstalled(controller), "leaving the mode must give Escape back to the host") + } + + func testUnmountRemovesTheMonitorEvenMidAnnotate() { + // Unmounting can happen while annotate mode is still running (a host window + // closing), and that path never touches `stop()`. + let controller = makeController() + controller.start() + XCTAssertTrue(monitorIsInstalled(controller)) + controller.unmount() + XCTAssertFalse(monitorIsInstalled(controller), "an unmounted overlay must own no monitor") + } + + func testRepeatedStartsInstallExactlyOneMonitor() { + // `installEscapeMonitor` is guarded, so a second `start()` cannot strand the + // first token — which would be unremovable, since only the latest is kept. + let controller = makeController() + controller.start() + let first = Mirror(reflecting: controller).children.first { $0.label == "escapeMonitor" }?.value + controller.start() + let second = Mirror(reflecting: controller).children.first { $0.label == "escapeMonitor" }?.value + XCTAssertTrue( + (first as AnyObject) === (second as AnyObject), + "the second start must reuse the installed monitor, not leak the first" + ) + controller.stop() + XCTAssertFalse(monitorIsInstalled(controller)) + } +} +#endif diff --git a/Tests/AnnotKitTests/EscapeRuleTests.swift b/Tests/AnnotKitTests/EscapeRuleTests.swift new file mode 100644 index 0000000..7e5886a --- /dev/null +++ b/Tests/AnnotKitTests/EscapeRuleTests.swift @@ -0,0 +1,88 @@ +import XCTest +@testable import AnnotKit + +/// Pins the Escape precedence and the swallow contract. Both are things a human +/// tester can only check by getting into four different UI states with a keyboard, +/// and one of them (drag WITH a composer open) is easy to forget exists at all. +final class EscapeRuleTests: XCTestCase { + // MARK: - Every state combination + + func testIdlePassesThroughWhateverElseIsTrue() { + // The host's Escape must behave exactly as it did before AnnotKit was + // installed. This is checked with the other two flags in every combination + // because "not annotating" outranks them ABSOLUTELY — a leftover flag from a + // previous session must not resurrect a handler after the mode is gone. + for drawing in [false, true] { + for card in [false, true] { + XCTAssertEqual( + EscapeRule.resolve(isAnnotating: false, isDrawingFrame: drawing, hasOpenCard: card), + .passThrough, + "idle must never claim Escape (drawing: \(drawing), card: \(card))" + ) + } + } + } + + func testAnnotatingWithNothingOpenExitsTheMode() { + XCTAssertEqual( + EscapeRule.resolve(isAnnotating: true, isDrawingFrame: false, hasOpenCard: false), + .exitAnnotateMode + ) + } + + func testAnOpenCardIsDismissedBeforeTheModeIsLeft() { + // The half-typed comment is the only irreversible thing in the flow: a mode + // is one click to re-enter, a draft cannot be retyped from anywhere. + XCTAssertEqual( + EscapeRule.resolve(isAnnotating: true, isDrawingFrame: false, hasOpenCard: true), + .dismissCard + ) + } + + func testAnInFlightDragIsCancelledBeforeTheModeIsLeft() { + XCTAssertEqual( + EscapeRule.resolve(isAnnotating: true, isDrawingFrame: true, hasOpenCard: false), + .cancelDrag + ) + } + + func testADragBeatsAnOpenCard() { + // THE ordering that is easy to get backwards, and the reason the two are + // tested together: the catcher stays live BEHIND an open composer, so a user + // can start framing a second note while the first is still being typed. Card + // first would throw away the draft they never touched and leave the band they + // are actively dragging on screen. + XCTAssertEqual( + EscapeRule.resolve(isAnnotating: true, isDrawingFrame: true, hasOpenCard: true), + .cancelDrag + ) + } + + // MARK: - The swallow contract + + func testOnlyPassThroughForwardsTheEvent() { + // AnnotKit is in-process with its host, so a handled-and-forwarded Escape is + // delivered TWICE: the overlay leaves annotate mode while the host closes its + // own sheet, from one press. Only the action that deliberately did nothing + // may forward. + XCTAssertFalse(EscapeAction.passThrough.consumesEvent) + XCTAssertTrue(EscapeAction.cancelDrag.consumesEvent) + XCTAssertTrue(EscapeAction.dismissCard.consumesEvent) + XCTAssertTrue(EscapeAction.exitAnnotateMode.consumesEvent) + } + + func testEveryResolvedActionInAnnotateModeIsConsumed() { + // Restates the above against the RULE rather than the enum, so a future + // action that resolves in annotate mode cannot quietly start leaking + // keystrokes to the host. + for drawing in [false, true] { + for card in [false, true] { + let action = EscapeRule.resolve(isAnnotating: true, isDrawingFrame: drawing, hasOpenCard: card) + XCTAssertTrue( + action.consumesEvent, + "annotate mode always acts, so it must always swallow (drawing: \(drawing), card: \(card))" + ) + } + } + } +} From 516aa115f50e4da37c36477bd425e2174f536a53 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:21:31 -0500 Subject: [PATCH 19/24] fix(overlay): re-arm the Escape monitor when re-mounting into an annotating session unmount() removes the monitor but the session's mode outlives the panel, so a host that unmounts and re-mounts a live overlay came back annotating with Escape silently dead -- the one state with no other keyboard way out. Co-Authored-By: Claude Opus 5 (1M context) --- .../AnnotKit/macOS/OverlayController.swift | 8 ++++++++ .../EscapeMonitorLifecycleTests.swift | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/Sources/AnnotKit/macOS/OverlayController.swift b/Sources/AnnotKit/macOS/OverlayController.swift index c8dc636..e35bccf 100644 --- a/Sources/AnnotKit/macOS/OverlayController.swift +++ b/Sources/AnnotKit/macOS/OverlayController.swift @@ -259,6 +259,14 @@ public final class OverlayController: NSObject { // grows a runloop turn or two later without posting a move/resize; poll until // the host frame settles so the panel and axOrigin reflect the FINAL frame. scheduleSettleResync() + // Re-arm Escape if we are attaching INTO an already-annotating session. + // `unmount()` removes the monitor, and the session's mode outlives the + // controller's panel — so a host that unmounts and re-mounts a live + // overlay (window recycling, a host that tears down on hide) would come + // back with annotate mode on and Escape silently dead, the one state with + // no other keyboard way out. `installEscapeMonitor()` is idempotent, so + // the ordinary mount-then-start path is unaffected. + if session.mode == .annotating { installEscapeMonitor() } } private func makeRootView() -> OverlayView { diff --git a/Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift b/Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift index df38a3a..6aa0916 100644 --- a/Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift +++ b/Tests/AnnotKitTests/EscapeMonitorLifecycleTests.swift @@ -66,6 +66,26 @@ final class EscapeMonitorLifecycleTests: XCTestCase { XCTAssertFalse(monitorIsInstalled(controller), "an unmounted overlay must own no monitor") } + /// Re-mounting into a session that is ALREADY annotating must re-arm Escape. + /// + /// `unmount()` removes the monitor, but the SESSION's mode outlives the + /// controller's panel — so a host that unmounts and re-mounts a live overlay + /// (window recycling, a host tearing the overlay down on hide) comes back with + /// annotate mode on. Without re-arming, Escape is silently dead in exactly the + /// state that has no other keyboard way out, and nothing on screen hints why. + func testReMountingWhileStillAnnotatingReArmsTheMonitor() { + let controller = makeController() + controller.start() + controller.unmount() + XCTAssertFalse(monitorIsInstalled(controller)) + XCTAssertEqual(controller.session.mode, .annotating, "unmount does not leave the mode — that is the trap") + + controller.mount(on: NSWindow(contentRect: NSRect(x: 0, y: 0, width: 320, height: 240), + styleMask: [.titled], backing: .buffered, defer: true)) + XCTAssertTrue(monitorIsInstalled(controller), "a re-mounted annotating overlay must hear Escape again") + controller.unmount() + } + func testRepeatedStartsInstallExactlyOneMonitor() { // `installEscapeMonitor` is guarded, so a second `start()` cannot strand the // first token — which would be unremovable, since only the latest is kept. From fc7c701ccb21b897e427010534ce41964824765a Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:42:57 -0500 Subject: [PATCH 20/24] fix(overlay): clamp the pill to the visible screen and stop swallowing scroll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported from dogfooding: "on scrollable screens, the menu in the bottom right disappears." Reproduced in AnnotKitOverlayProbe (new Phase 9) before fixing: * A tall/scrollable host is a window taller than the display. AppKit constrains a window's TOP under the menu bar but never lifts its bottom, so its bottom edge ends up below `visibleFrame` — and BOTH overlay modes anchored the toolbar to that bottom edge, drawing the pill under the Dock or off the display entirely. Measured: `windowNumber(at: pill center) == 0`, i.e. not clickable at all. * Worse, the expanded catcher swallowed every wheel event: it covers the host with `ignoresMouseEvents = false`, and an event no view handles walks the PANEL's responder chain, never the window beneath. The host could not be scrolled AT ALL while annotating, so nothing below the fold could be reached on exactly the screens the report is about. Placement now goes through `OverlayPlacement`, which narrows both modes to the intersection of the host frame and `host.screen.visibleFrame` (falling back to the primary display, and keeping the unclamped frame on an EMPTY intersection so an off-display host is not collapsed to nothing). The idle panel is ANCHORED to that region's bottom-right at full size rather than intersected down to it, since shrinking it would clip the pill it exists to carry. `axOrigin` and `surfaceSize` are re-derived from the CLAMPED panel frame, not the host: `OverlayView`'s contract is that they describe the surface it draws into, so a host-derived origin would offset every click, highlight and card by exactly the clipped amount. Clamping the bottom leaves the origin alone (it hangs off the frame's top edge) and only shrinks the surface, which is right — the composer should clamp cards to the VISIBLE region. Clamping the TOP really does move the origin, so Phase 9d builds a host tucked under the menu bar and asserts a click still resolves there; unfixed, that same click resolves to nil. `KeyablePanel` now forwards an unconsumed `scrollWheel` to the view under the pointer in the host, re-aimed through screen space (the panel is no longer the host's frame once clamped). Only events that reached the WINDOW are forwarded, so overlay chrome that legitimately scrolls still keeps its own wheel. swift build: no warnings. swift test: 181 (was 172). Probe: all phases PASS. Co-Authored-By: Claude Opus 5 (1M context) --- .../AnnotKit/macOS/OverlayController.swift | 151 +++++- Sources/AnnotKitOverlayProbe/main.swift | 430 +++++++++++++++++- .../AnnotKitTests/OverlayPlacementTests.swift | 134 ++++++ 3 files changed, 687 insertions(+), 28 deletions(-) create mode 100644 Tests/AnnotKitTests/OverlayPlacementTests.swift diff --git a/Sources/AnnotKit/macOS/OverlayController.swift b/Sources/AnnotKit/macOS/OverlayController.swift index e35bccf..4e09856 100644 --- a/Sources/AnnotKit/macOS/OverlayController.swift +++ b/Sources/AnnotKit/macOS/OverlayController.swift @@ -12,7 +12,9 @@ import SwiftUI /// /// The panel is sized to just the toolbar corner of the host window when idle (so /// the rest of the app stays usable) and to the host window's full frame while -/// annotating (so the SwiftUI catcher receives hover and clicks over the app). +/// annotating (so the SwiftUI catcher receives hover and clicks over the app) — +/// both narrowed to the display's `visibleFrame` by ``OverlayPlacement``, because a +/// host taller than the screen would otherwise draw its toolbar under the Dock. /// Because it is pinned to one window, all coordinates collapse to a single fixed /// AX origin (`axOrigin`), which also makes placement correct on any display — /// no `NSScreen.main` (active-screen) assumption. Child windows follow the @@ -26,11 +28,12 @@ public final class OverlayController: NSObject { private var hostingView: NSHostingView? private weak var host: NSWindow? - /// AX top-left origin of the host window, threaded into `OverlayView` so click, - /// highlight, and composer share one transform. Recomputed on every geometry - /// change. + /// AX top-left origin of the overlay PANEL — the surface `OverlayView` draws into, + /// which is the host window narrowed to the visible screen — threaded into the view + /// so click, highlight, and composer share one transform. Recomputed on every + /// geometry change. private var axOrigin: CGPoint = .zero - /// Host-window-local size, for clamping the composer on-screen. + /// Panel-local size, for clamping the composer inside the visible region. private var surfaceSize: CGSize = .zero /// Host frame captured at the last geometry sync. A SwiftUI/content-sized host @@ -291,7 +294,10 @@ public final class OverlayController: NSObject { // A child follows the parent's position automatically but does not // resize with it, and neither the child nor AppKit recomputes our AX // origin — so re-sync on every geometry change. One handler covers move - // (origin stale), resize (frame + origin stale), and screen changes. + // (origin stale), resize (frame + origin stale), and screen changes. The + // screen-parameters observer earns its keep twice over now that placement is + // clamped: a Dock that appears, or a display that changes resolution, moves + // `visibleFrame` without touching the host's frame, and the pill has to follow. center.addObserver(self, selector: #selector(hostGeometryChanged(_:)), name: NSWindow.didMoveNotification, object: host) center.addObserver(self, selector: #selector(hostGeometryChanged(_:)), @@ -350,36 +356,96 @@ public final class OverlayController: NSObject { /// and surface size, then push both into the SwiftUI view. private func syncFrameAndOrigin() { guard let panel, let host else { return } - panel.setFrame(frame(for: session.mode, on: host), display: true) + let panelFrame = frame(for: session.mode, on: host) + panel.setFrame(panelFrame, display: true) // Primary display = the origin/menu-bar screen, NOT NSScreen.main (the // active screen), which was the single-display bug. let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 - axOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: host.frame, primaryHeight: primaryHeight) - surfaceSize = host.frame.size + // Derived from the PANEL's frame, never the host's. `OverlayView`'s contract is + // that these two describe the SURFACE it draws into — the catcher ADDS + // `axOrigin` to turn a panel-local click into an AX screen point, the highlight + // and composer SUBTRACT it — so once placement is clamped to the visible + // region, a host-derived origin would offset every click, highlight and card by + // exactly the clipped amount. + // + // Clamping the BOTTOM (the reported bug) leaves `axOrigin` alone, since it + // hangs off the frame's TOP edge, and only shrinks `surfaceSize` — which is + // itself the right answer, because the composer clamp should keep cards inside + // the VISIBLE region rather than inside a window that runs off the display. + // Clamping the TOP (a host tucked under the menu bar) genuinely does move the + // origin, and that is the case a host-derived origin breaks silently. + axOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: panelFrame, primaryHeight: primaryHeight) + surfaceSize = panelFrame.size lastSyncedHostFrame = host.frame hostingView?.rootView = makeRootView() } private func frame(for mode: AnnotationSession.Mode, on host: NSWindow) -> NSRect { + OverlayPlacement.panelFrame( + for: mode, + hostFrame: host.frame, + // `host.screen` is the display the window is mostly on, so the clamp + // follows the host across displays. A window AppKit reports no screen for + // (entirely off-display, or mid-teardown) falls back to the primary rather + // than skipping the clamp, so the pill still lands somewhere reachable. + visibleFrame: (host.screen ?? NSScreen.screens.first)?.visibleFrame + ) + } +} + +/// Where the overlay panel goes, as pure geometry — no window, no display, so the +/// rules below are unit-testable instead of only observable on a real screen. +enum OverlayPlacement { + /// The idle panel's size: fits the widest pill state (toggle + count badge + copy + /// + export + clear, ~180pt) plus its 20pt inset and the drop shadow, and the 44pt + /// pill plus the same inset. + static let idleSize = CGSize(width: 240, height: 104) + + /// The part of the host the user can actually see and click: its frame narrowed to + /// the screen's `visibleFrame`. + /// + /// `visibleFrame` and not `frame`, because the Dock and menu bar cover the + /// display's edges and a pill under the Dock is exactly as unreachable as one off + /// the display. + /// + /// This is the whole of the "on scrollable screens the menu in the bottom right + /// disappears" bug: BOTH modes anchor the toolbar to the host's BOTTOM edge, and + /// AppKit constrains a window's TOP under the menu bar but never lifts its bottom — + /// so a host taller than the display (the shape a content-sized/scrollable window + /// grows into, the same growth ``OverlayController/scheduleSettleResync()`` exists + /// to catch) hangs its bottom edge below `visibleFrame`, and an unclamped anchor + /// draws the pill under the Dock or off the display entirely. + /// + /// An EMPTY intersection means the host is off-display altogether (another Space, a + /// window parked off-screen). Keep the unclamped frame there rather than collapse + /// the panel to nothing: a zero-sized panel would have to be rebuilt to come back, + /// whereas an off-screen one simply reappears with its window. + static func region(hostFrame: CGRect, visibleFrame: CGRect?) -> CGRect { + guard let visibleFrame else { return hostFrame } + let region = hostFrame.intersection(visibleFrame) + return region.isEmpty ? hostFrame : region + } + + static func panelFrame(for mode: AnnotationSession.Mode, hostFrame: CGRect, visibleFrame: CGRect?) -> CGRect { + let region = region(hostFrame: hostFrame, visibleFrame: visibleFrame) switch mode { case .annotating: - // The host window's full outer frame (includes the title bar so the - // window-local y aligns with AX y). - return host.frame + // The host's outer frame minus whatever hangs off the display (the title + // bar is included, so the window-local y still aligns with AX y). Clipping + // the catcher costs nothing: the part that was cut cannot be hovered or + // clicked anyway. + return region case .idle: - // A small panel sized to the pill's real bounds, pinned to the host - // window's bottom-right corner, so the idle overlay covers only the - // toolbar and never swallows clicks meant for the host. Width fits the - // widest pill state (toggle + count badge + copy + export + clear, - // ~180pt) plus its 20pt inset and the drop shadow; height fits the 44pt - // pill plus the same inset. Anchored to the - // window so it rides along on move and is recomputed on resize. - let size = NSSize(width: 240, height: 104) - return NSRect( - x: host.frame.maxX - size.width, - y: host.frame.minY, - width: size.width, - height: size.height + // Anchored to the visible region's bottom-right corner at FULL size, not + // intersected down to it: shrinking this panel would clip the pill it + // exists to carry. The pill is drawn against the panel's BOTTOM edge, so + // pinning that edge inside the region is what keeps it reachable — only the + // panel's empty upper part can spill past a region shorter than 104pt. + return CGRect( + x: region.maxX - idleSize.width, + y: region.minY, + width: idleSize.width, + height: idleSize.height ) } } @@ -392,5 +458,40 @@ public final class OverlayController: NSObject { /// into the composer focus the field. Used as a child window per host window. final class KeyablePanel: NSPanel { override var canBecomeKey: Bool { true } + + /// Hand an unconsumed scroll down to the host window. + /// + /// Measured, not assumed: while annotating this panel covers the host's whole frame + /// with `ignoresMouseEvents = false`, and an event no view handles walks THIS + /// window's responder chain — never the window beneath it — where `NSWindow`'s + /// do-nothing default ate it. So the host could not be scrolled AT ALL in annotate + /// mode, which is fatal on exactly the tall scrollable screens this placement fix is + /// about: nothing below the fold can be annotated if it cannot be brought into view. + /// + /// Only events that reached the WINDOW are forwarded, and that is the correct filter + /// by construction rather than by a mode check: anything in the overlay that + /// legitimately scrolls (a long note in the composer) consumes the wheel in the view + /// tree, so it never arrives here. + override func scrollWheel(with event: NSEvent) { + guard let host = parent else { return } + // Re-aim through screen space rather than passing `locationInWindow` along: it + // is PANEL-local, and the panel is no longer the host's frame once placement is + // clamped to the visible region, so handing it over unconverted would scroll + // whatever sits at the wrong point (the wrong scroller, in a window with two). + let hostPoint = host.convertPoint(fromScreen: convertPoint(toScreen: event.locationInWindow)) + // Falls back to the content view when the point is over no host view at all + // (the title-bar strip, or a host smaller than the panel): a wheel that hit + // AnnotKit must never simply vanish. `NSView`'s default `scrollWheel` walks the + // event UP to the enclosing scroller from wherever it lands, so aiming at the + // deepest view under the pointer is enough — no scroll-view search here. + // + // The event itself is passed along unchanged, so the target reads a + // `locationInWindow` that is still PANEL-local (an `NSEvent`'s location cannot + // be rewritten). Scroll handling uses the DELTAS, and the location's one real + // job — choosing the target — is done above, so this costs nothing short of a + // host view that positions something off the wheel's own coordinates. + + (host.contentView?.hitTest(hostPoint) ?? host.contentView)?.scrollWheel(with: event) + } } #endif diff --git a/Sources/AnnotKitOverlayProbe/main.swift b/Sources/AnnotKitOverlayProbe/main.swift index d135c62..6bdea51 100644 --- a/Sources/AnnotKitOverlayProbe/main.swift +++ b/Sources/AnnotKitOverlayProbe/main.swift @@ -273,6 +273,120 @@ func grownFrame(from base: CGRect) -> CGRect { CGRect(origin: base.origin, size: resizeLayout().grownSize) } +// MARK: - Hosts that hang off the visible screen (Phase 9) + +/// A window that really goes where it is put, bypassing AppKit's +/// keep-the-title-bar-below-the-menu-bar constraint. Needed ONLY by 9d: `setFrame` +/// silently pins any window's top to `visibleFrame.maxY` (measured — a borderless one +/// too), so a host tucked UNDER the menu bar, the one direction in which clamping moves +/// the AX origin, cannot be built any other way. The window is genuinely there — real +/// backing store at a real frame — so every assertion still reads real geometry. +final class UnconstrainedWindow: NSWindow { + override func constrainFrameRect(_ frameRect: NSRect, to screen: NSScreen?) -> NSRect { frameRect } +} + +/// Counts the wheel events that reach the HOST's view tree, and from WHERE. +/// +/// The scroll question cannot be answered by watching an `NSScrollView`'s offset: +/// measured, AppKit's scroll views ignore a synthesized `NSEvent` outright (a synthetic +/// wheel moves nothing even when delivered straight to the scroll view). So 9c measures +/// the ROUTING instead, which is the mechanism actually in question — does a wheel that +/// lands on AnnotKit's panel reach the host at all, and does it reach the right view. +final class ScrollSpyView: NSView { + let name: String + var scrollCount = 0 + init(name: String, frame: NSRect) { + self.name = name + super.init(frame: frame) + } + required init?(coder: NSCoder) { fatalError("unused") } + override func scrollWheel(with event: NSEvent) { scrollCount += 1 } +} + +@MainActor +struct ClampedHost { + let window: NSWindow + /// Lower and upper halves of the host content. Which one a forwarded wheel lands in + /// is the only witness available for the panel→host coordinate conversion: the + /// forwarded event object still carries its PANEL-local `locationInWindow` (an + /// `NSEvent`'s location cannot be rewritten), so the conversion shows up in the + /// choice of target view, not in what the target reads off the event. Harmless for + /// real scroll handling, which uses the deltas. + let lowerSpy: ScrollSpyView + let upperSpy: ScrollSpyView + let button: NSButton +} + +/// Build a host and stock its content: two stacked scroll spies and one identified +/// button placed inside `visibleBand` (in screen coordinates) so 9b's click assertion is +/// about a control the user can actually reach. +@MainActor +func makeClampedHost(title: String, frame: NSRect, unconstrained: Bool, visibleBand: NSRect) -> ClampedHost { + let window: NSWindow = unconstrained + ? UnconstrainedWindow(contentRect: frame, styleMask: [.titled, .closable], backing: .buffered, defer: false) + : NSWindow(contentRect: frame, styleMask: [.titled, .closable], backing: .buffered, defer: false) + window.title = title + window.makeKeyAndOrderFront(nil) + window.setFrame(frame, display: true) + + let size = window.contentView?.bounds.size ?? frame.size + let content = NSView(frame: NSRect(origin: .zero, size: size)) + let lower = ScrollSpyView(name: "lower", frame: NSRect(x: 0, y: 0, width: size.width, height: size.height / 2)) + let upper = ScrollSpyView(name: "upper", frame: NSRect(x: 0, y: size.height / 2, width: size.width, height: size.height / 2)) + content.addSubview(lower) + content.addSubview(upper) + + // The button's y is chosen in SCREEN space and converted back, so it sits in the + // band that survives the clamp whichever edge is being clipped. + let button = NSButton(title: "Clamped action", target: nil, action: nil) + button.setAccessibilityIdentifier("Clamp.Button") + let buttonScreenY = visibleBand.midY + let contentOriginScreenY = window.convertPoint(toScreen: .zero).y + button.frame = NSRect(x: 60, y: buttonScreenY - contentOriginScreenY, width: 200, height: 32) + content.addSubview(button) + + window.contentView = content + return ClampedHost(window: window, lowerSpy: lower, upperSpy: upper, button: button) +} + +/// The pill's own rect inside a panel at `frame`, mirroring `OverlayView.toolbar`: the +/// bottom-right corner, 20pt padding, and the pill's real bounds (~180x44 at its widest, +/// the same numbers the 240x104 idle panel is built from). Asserting on the PILL and not +/// the panel is the whole point — the idle panel is deliberately taller than the pill, +/// so "the panel overlaps the screen" would pass while the pill itself sat under the +/// Dock. +func pillRect(inPanel frame: CGRect) -> CGRect { + CGRect(x: frame.maxX - 20 - 180, y: frame.minY + 20, width: 180, height: 44) +} + +/// Read the overlay's private `axOrigin` / `surfaceSize` by reflection — the same trade +/// `EscapeMonitorLifecycleTests` makes for the Escape monitor. Clamping the panel severs +/// the witness Phase 3 could rely on (the panel frame equalling the host frame), and +/// widening the library's public API purely so a probe can look is a worse deal than +/// this coupling. Returns nil if the property is renamed, and every caller FAILS on nil, +/// so this can never silently assert nothing. +@MainActor +func overlayValue(_ controller: OverlayController, _ label: String, as type: T.Type) -> T? { + for child in Mirror(reflecting: controller).children where child.label == label { + return child.value as? T + } + return nil +} + +/// A wheel event whose `locationInWindow` is exactly `panelLocal` (Cocoa, y-up, relative +/// to the panel). `NSEvent(cgEvent:)` yields `windowNumber == 0`, and AppKit reports such +/// an event's `locationInWindow` as the CG location flipped into Cocoa screen space — so +/// writing the flipped panel-local point into the CG event reproduces exactly what a real +/// wheel delivered to the panel carries. (Verified: the round trip is exact.) +@MainActor +func makeScrollEvent(panelLocal: CGPoint) -> NSEvent? { + guard let cg = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 1, + wheel1: -120, wheel2: 0, wheel3: 0) else { return nil } + let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 + cg.location = CGPoint(x: panelLocal.x, y: primaryHeight - panelLocal.y) + return NSEvent(cgEvent: cg) +} + // MARK: - SwiftUI host (mirrors AnnotKitDemo's DemoView) for issue-2 coverage /// A SwiftUI control surface identical in shape to `AnnotKitDemo.DemoView`, so the @@ -1440,7 +1554,7 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { check8(false, "snapshot exposes Spec.Card/CardText/Section + the Nav list container") self.navController?.unmount() window.orderOut(nil) - self.finish() + self.phase9Clamp() return } print(" fixture: card=\(fmt(card.frame)) cardText=\(fmt(cardText.frame)) " + @@ -1460,7 +1574,7 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { self.phase8eHoverRevived(session: session, cardText: cardText) self.navController?.unmount() window.orderOut(nil) - self.finish() + self.phase9Clamp() } } } @@ -1675,6 +1789,315 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { return nil } + // ---- Phase 9: a host that hangs off the visible screen ------------------ + // The dogfooding report: "on scrollable screens, the menu in the bottom right + // disappears." A tall/scrollable host is a window taller than the display, and + // AppKit constrains a window's TOP under the menu bar but never lifts its bottom — + // so its bottom edge ends up below `visibleFrame`. BOTH overlay modes anchor the + // pill to that bottom edge (idle: a 240x104 panel at the host's bottom-right corner; + // annotate: the pill drawn at the bottom-right INSIDE a full-host-frame panel), so + // the toolbar is drawn under the Dock or off the display entirely and there is no + // way to reach it. Every sub-phase here first PROVES the host really extends past + // the visible area, or it would be asserting nothing. + var clampController: OverlayController? + var clampSession: AnnotationSession? + var clampHost: ClampedHost? + var passClamp = true + func check9(_ cond: Bool, _ msg: String) { + passClamp = passClamp && cond + print(" " + (cond ? "ok " : "FAIL ") + msg) + } + + /// The display the clamp is measured against. `NSScreen.main` is the screen the + /// probe's own windows land on, which is what `host.screen` will report back. + var visibleFrame: NSRect { + (NSScreen.main ?? NSScreen.screens.first)?.visibleFrame ?? NSRect(x: 0, y: 0, width: 1440, height: 900) + } + + /// How far the fixtures hang past the visible edge. Large enough that a stale, + /// unclamped placement is unambiguously off-screen rather than a rounding artifact. + let overhang: CGFloat = 300 + + func phase9Clamp() { + print("\n--- Phase 9: the pill on a host that hangs BELOW the visible screen (scrollable-window report) ---") + let visible = visibleFrame + // Top pinned to the visible top (so AppKit does not fight the placement) and + // TALLER than the visible height by `overhang` — exactly the shape a window + // takes when its content grows past the display. + let frame = NSRect(x: visible.midX - 310, y: visible.minY - overhang, + width: 620, height: visible.height + overhang) + let host = makeClampedHost(title: "AnnotKit Harness W9 (below the fold)", + frame: frame, unconstrained: false, + visibleBand: visible.intersection(frame)) + clampHost = host + + let session = AnnotationSession( + source: MacElementSource(), + sink: NotesFileSink(path: NSTemporaryDirectory() + "annotkit-clamp.md") + ) + let controller = OverlayController(session: session) + controller.mount(on: host.window) + clampController = controller + clampSession = session + + DispatchQueue.main.asyncAfter(deadline: .now() + 0.6) { [weak self] in + self?.phase9aIdle() + } + } + + /// The precondition every assertion in this phase rests on: the host really does + /// extend past the bottom of the visible area. Printed with the numbers so a run on + /// a different display arrangement is diagnosable. + @discardableResult + func assertHangsBelow(_ host: NSWindow) -> Bool { + let visible = visibleFrame + let hangs = host.frame.minY < visible.minY - 100 + print(" host=\(fmt(host.frame)) visibleFrame=\(fmt(visible)) " + + "bottom hangs \(String(format: "%.0f", visible.minY - host.frame.minY))pt BELOW the visible area") + check9(hangs, "sanity: the host really extends past the bottom of the visible screen (else this phase is vacuous)") + return hangs + } + + // ---- 9a: the IDLE pill stays on the visible screen ---------------------- + func phase9aIdle() { + print("\n 9a — IDLE pill on a host whose bottom is below the visible area:") + guard let host = clampHost, let panel = host.window.childWindows?.first else { + check9(false, "the overlay mounted a child panel on the clamped host") + return phase9dTucked() + } + assertHangsBelow(host.window) + + let visible = visibleFrame + let unclamped = idleFrame(host.window.frame) + print(" idle panel=\(fmt(panel.frame)) pill=\(fmt(pillRect(inPanel: panel.frame))) " + + "(unclamped placement would be \(fmt(unclamped)), pill \(fmt(pillRect(inPanel: unclamped))))") + check9(!visible.contains(pillRect(inPanel: unclamped)), + "sanity: the UNCLAMPED bottom-right placement really is off the visible screen (the reported bug)") + check9(visible.contains(pillRect(inPanel: panel.frame)), + "the idle pill is fully inside the visible screen") + // Size, not just position: clamping by intersecting the panel down to the + // visible region would leave the pill's own panel too short to draw it. + check9(approxEqual(panel.frame, CGRect(origin: panel.frame.origin, size: CGSize(width: 240, height: 104))), + "the idle panel keeps its full 240x104 size (a clipped panel would clip the pill)") + // Hit-testability, the property the user actually lost: AppKit's own + // "which window would a click here land on" answer must be OUR panel. + let pillCenter = center(of: pillRect(inPanel: panel.frame)) + let hitNumber = NSWindow.windowNumber(at: pillCenter, belowWindowWithWindowNumber: 0) + print(" windowNumber(at: pill center \(String(format: "(%.0f, %.0f)", pillCenter.x, pillCenter.y)))=\(hitNumber) " + + "panel=\(panel.windowNumber) host=\(host.window.windowNumber)") + check9(hitNumber == panel.windowNumber, "a click at the pill's center lands on the overlay panel (it is hit-testable)") + + clampController?.start() + DispatchQueue.main.asyncAfter(deadline: .now() + 0.6) { [weak self] in + self?.phase9bAnnotate() + } + } + + // ---- 9b: the ANNOTATE pill stays on screen AND the hit-test stays true --- + func phase9bAnnotate() { + print("\n 9b — ANNOTATE pill + hit-test on the same clamped host:") + guard let host = clampHost, let controller = clampController, + let panel = host.window.childWindows?.first else { + check9(false, "the overlay is still mounted in annotate mode") + return phase9dTucked() + } + let visible = visibleFrame + assertHangsBelow(host.window) + print(" annotate panel=\(fmt(panel.frame)) pill=\(fmt(pillRect(inPanel: panel.frame))) " + + "(unclamped would be the host frame, pill \(fmt(pillRect(inPanel: host.window.frame))))") + check9(!visible.contains(pillRect(inPanel: host.window.frame)), + "sanity: the pill drawn at the bottom-right of the FULL host frame is off the visible screen") + check9(visible.contains(pillRect(inPanel: panel.frame)), + "the annotate pill is fully inside the visible screen") + + // The part that would make a naive fix worse than the bug: the surface the + // overlay transforms clicks through must be the surface it is now DRAWN on. + let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 + let panelAXOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: panel.frame, primaryHeight: primaryHeight) + guard let axOrigin = overlayValue(controller, "axOrigin", as: CGPoint.self), + let surfaceSize = overlayValue(controller, "surfaceSize", as: CGSize.self) else { + check9(false, "the controller still has axOrigin/surfaceSize to read (this phase asserts nothing otherwise)") + return phase9cScroll() + } + print(" axOrigin=\(String(format: "(%.1f, %.1f)", axOrigin.x, axOrigin.y)) " + + "panel-derived=\(String(format: "(%.1f, %.1f)", panelAXOrigin.x, panelAXOrigin.y)) " + + "surfaceSize=\(String(format: "%.0fx%.0f", surfaceSize.width, surfaceSize.height)) " + + "panel=\(String(format: "%.0fx%.0f", panel.frame.width, panel.frame.height)) " + + "host=\(String(format: "%.0fx%.0f", host.window.frame.width, host.window.frame.height))") + check9(approxEqualPt(axOrigin, panelAXOrigin), "axOrigin is derived from the CLAMPED panel frame") + check9(abs(surfaceSize.height - panel.frame.height) < 2, + "surfaceSize is the CLAMPED panel's size (the composer clamps cards to the VISIBLE region)") + check9(surfaceSize.height < host.window.frame.height - 100, + "the clamped surface is materially shorter than the host (the clip is real, not a no-op)") + + // And now the click itself, along the real path: SwiftUI hands the catcher a + // PANEL-local point, the catcher ADDS axOrigin, the source resolves that. + let source = MacElementSource() + guard let buttonFrame = frame(ofID: "Clamp.Button", in: source.snapshot()) else { + check9(false, "the host's button is in the AX tree (needed to test the click through the clamped panel)") + return phase9cScroll() + } + let buttonAXCenter = center(of: buttonFrame) + let local = CGPoint(x: buttonAXCenter.x - panelAXOrigin.x, y: buttonAXCenter.y - panelAXOrigin.y) + let queried = CGPoint(x: local.x + axOrigin.x, y: local.y + axOrigin.y) + let hit = source.hitTest(queried) + print(" button AX \(fmt(buttonFrame)) -> panel-local \(String(format: "(%.0f, %.0f)", local.x, local.y)) " + + "-> catcher queries \(String(format: "(%.0f, %.0f)", queried.x, queried.y)) -> hit=\(hit?.id ?? "nil")") + check9(panel.frame.contains(CGPoint(x: buttonAXCenter.x, y: primaryHeight - buttonAXCenter.y)), + "sanity: the button really is under the clamped panel (a click on it goes through the catcher)") + check9(hit?.id == "Clamp.Button", + "a click on the clamped host still resolves to the element under it (got \(hit?.id ?? "nil"))") + + phase9cScroll() + } + + // ---- 9c: can the host be scrolled AT ALL while annotating? -------------- + // The other half of the report: the expanded catcher covers the host with + // `ignoresMouseEvents = false`, and an event no view handles walks the PANEL's own + // responder chain, never the window beneath. If the wheel dies there, nothing below + // the fold can be annotated — on precisely the screens the report is about. + func phase9cScroll() { + print("\n 9c — scroll WHILE ANNOTATING: does a wheel over the catcher reach the host?") + guard let host = clampHost, let panel = host.window.childWindows?.first else { + check9(false, "the overlay panel is present for the scroll measurement") + return phase9dTucked() + } + // Aim at the vertical middle of the VISIBLE band. Chosen so the correct + // (converted) host-local point lands in the UPPER spy while the unconverted + // panel-local point would land in the LOWER one — the two answers are + // distinguishable, so this measures the conversion and not just the routing. + let visible = visibleFrame + let aimScreen = CGPoint(x: panel.frame.midX, y: visible.midY) + let panelLocal = CGPoint(x: aimScreen.x - panel.frame.minX, y: aimScreen.y - panel.frame.minY) + let hostLocal = CGPoint(x: aimScreen.x - host.window.frame.minX, y: aimScreen.y - host.window.frame.minY) + let split = (host.window.contentView?.bounds.height ?? 0) / 2 + print(" aim screen=\(String(format: "(%.0f, %.0f)", aimScreen.x, aimScreen.y)) " + + "panel-local=\(String(format: "(%.0f, %.0f)", panelLocal.x, panelLocal.y)) " + + "host-local=\(String(format: "(%.0f, %.0f)", hostLocal.x, hostLocal.y)) spy split at y=\(String(format: "%.0f", split))") + check9(panel.frame.contains(aimScreen), "sanity: the annotate catcher really covers the point being scrolled") + check9((hostLocal.y > split) != (panelLocal.y > split), + "sanity: converted and unconverted points land in DIFFERENT spies (so the target proves the conversion)") + + guard let event = makeScrollEvent(panelLocal: panelLocal) else { + check9(false, "a synthesized wheel event could be built") + return phase9dTucked() + } + check9(approxEqualPt(event.locationInWindow, panelLocal), + "sanity: the synthesized event carries the panel-local location a real wheel would") + + // Baseline: the spies are live and reachable when an event is handed to them + // directly, so a zero count later means "swallowed", not "broken fixture". + host.upperSpy.scrollWheel(with: event) + check9(host.upperSpy.scrollCount == 1, "sanity: the host's spy DOES record a wheel delivered straight to it") + let base = (lower: host.lowerSpy.scrollCount, upper: host.upperSpy.scrollCount) + + // Deliver the way AppKit does once it has picked the window: hit-test the + // panel's content and hand the event to the deepest view. + let target = panel.contentView?.hitTest(panelLocal) + var chain: [String] = [] + var responder: NSResponder? = target + while let current = responder, chain.count < 8 { + chain.append("\(type(of: current))") + responder = current.nextResponder + } + print(" panel hit-test -> \(target.map { "\(type(of: $0))" } ?? "nil"); responder chain: \(chain)") + check9(target != nil, "sanity: the wheel lands on the overlay's own content view (it is over the catcher)") + check9(chain.contains { $0.contains("KeyablePanel") }, + "sanity: nothing in the overlay's view tree consumes the wheel — it reaches the panel WINDOW") + + target?.scrollWheel(with: event) + let lower = host.lowerSpy.scrollCount - base.lower + let upper = host.upperSpy.scrollCount - base.upper + print(" after the wheel: lower spy +\(lower), upper spy +\(upper)") + check9(lower + upper == 1, "a wheel over the annotate catcher reaches the HOST (it is not swallowed by the panel)") + check9(upper == 1, "it reaches the view actually under the pointer (the panel→host conversion is applied)") + + clampController?.unmount() + clampHost?.window.orderOut(nil) + phase9dTucked() + } + + // ---- 9d: the OTHER clamp direction — a host tucked under the menu bar ---- + // Clamping the bottom leaves `axOrigin` untouched (it hangs off the frame's TOP + // edge), so 9b cannot tell a fix that re-derives the origin from one that forgot to. + // A host whose TOP is clipped can: there the origin really moves, and a host-derived + // origin offsets every click by exactly the clipped amount. + func phase9dTucked() { + print("\n 9d — the other direction: a host whose TOP is tucked under the menu bar:") + let visible = visibleFrame + let frame = NSRect(x: visible.midX - 310, y: visible.minY + 80, + width: 620, height: visible.height - 80 + overhang) + let host = makeClampedHost(title: "AnnotKit Harness W9 (under the menu bar)", + frame: frame, unconstrained: true, + visibleBand: visible.intersection(frame)) + clampHost = host + + let session = AnnotationSession( + source: MacElementSource(), + sink: NotesFileSink(path: NSTemporaryDirectory() + "annotkit-clamp-top.md") + ) + let controller = OverlayController(session: session) + controller.mount(on: host.window) + controller.start() + clampController = controller + clampSession = session + + DispatchQueue.main.asyncAfter(deadline: .now() + 0.7) { [weak self] in + self?.phase9dChecks() + } + } + + func phase9dChecks() { + guard let host = clampHost, let controller = clampController, + let panel = host.window.childWindows?.first else { + check9(false, "the overlay mounted on the menu-bar-tucked host") + return finish() + } + let visible = visibleFrame + let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 + print(" host=\(fmt(host.window.frame)) visibleFrame=\(fmt(visible)) " + + "top is \(String(format: "%.0f", host.window.frame.maxY - visible.maxY))pt ABOVE the visible top") + check9(host.window.frame.maxY > visible.maxY + 50, + "sanity: the host's top really is under the menu bar (else the origin never moves and this proves nothing)") + + let hostAXOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: host.window.frame, primaryHeight: primaryHeight) + let panelAXOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: panel.frame, primaryHeight: primaryHeight) + guard let axOrigin = overlayValue(controller, "axOrigin", as: CGPoint.self) else { + check9(false, "the controller still has an axOrigin to read") + return finish() + } + print(" panel=\(fmt(panel.frame)) axOrigin=\(String(format: "(%.1f, %.1f)", axOrigin.x, axOrigin.y)) " + + "panel-derived=\(String(format: "(%.1f, %.1f)", panelAXOrigin.x, panelAXOrigin.y)) " + + "host-derived=\(String(format: "(%.1f, %.1f)", hostAXOrigin.x, hostAXOrigin.y))") + check9(!approxEqualPt(panelAXOrigin, hostAXOrigin), + "sanity: clamping the TOP really does move the AX origin (the two candidates differ)") + check9(approxEqualPt(axOrigin, panelAXOrigin), + "axOrigin follows the CLAMPED panel, not the host (a host-derived origin offsets every click here)") + + let source = MacElementSource() + guard let buttonFrame = frame(ofID: "Clamp.Button", in: source.snapshot()) else { + check9(false, "the tucked host's button is in the AX tree") + return finish() + } + let buttonAXCenter = center(of: buttonFrame) + let local = CGPoint(x: buttonAXCenter.x - panelAXOrigin.x, y: buttonAXCenter.y - panelAXOrigin.y) + let queried = CGPoint(x: local.x + axOrigin.x, y: local.y + axOrigin.y) + let hit = source.hitTest(queried) + // What the SAME click would resolve to if the origin had stayed host-derived: + // named explicitly so the failure mode has a number next to it, not a shrug. + let stale = CGPoint(x: local.x + hostAXOrigin.x, y: local.y + hostAXOrigin.y) + print(" button AX \(fmt(buttonFrame)) -> panel-local \(String(format: "(%.0f, %.0f)", local.x, local.y)) " + + "-> queries \(String(format: "(%.0f, %.0f)", queried.x, queried.y)) hit=\(hit?.id ?? "nil"); " + + "a host-derived origin would query \(String(format: "(%.0f, %.0f)", stale.x, stale.y)) " + + "-> \(source.hitTest(stale)?.id ?? "nil")") + check9(hit?.id == "Clamp.Button", + "a click on the menu-bar-tucked host still resolves to the element under it (got \(hit?.id ?? "nil"))") + + clampController?.unmount() + clampHost?.window.orderOut(nil) + finish() + } + func finish() { print("\n issue-2 (per-control hit-test through the expanded overlay): \(passIssue2 ? "PASS" : "FAIL")") print(" issue-1 (retention / copy / export / pill persistence): \(pass1 ? "PASS" : "FAIL")") @@ -1685,8 +2108,9 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { print(" Phase 6 (positional specificity by cursor position): \(passSpec ? "PASS" : "FAIL")") print(" Phase 7 (marquee frame selection: drawn rect -> element): \(passMarquee ? "PASS" : "FAIL")") print(" Phase 8 (selection navigation: round trips, history, component, frame anchor): \(passNav ? "PASS" : "FAIL")") + print(" Phase 9 (pill + hit-test + scroll on a host hanging off the visible screen): \(passClamp ? "PASS" : "FAIL")") print("\n=== AnnotKitOverlayProbe complete ===") - exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec && passMarquee && passNav ? 0 : 1) + exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec && passMarquee && passNav && passClamp ? 0 : 1) } func collectIDs(_ elements: [Element]) -> [String] { diff --git a/Tests/AnnotKitTests/OverlayPlacementTests.swift b/Tests/AnnotKitTests/OverlayPlacementTests.swift new file mode 100644 index 0000000..c379d5d --- /dev/null +++ b/Tests/AnnotKitTests/OverlayPlacementTests.swift @@ -0,0 +1,134 @@ +#if os(macOS) +import CoreGraphics +import XCTest +@testable import AnnotKit + +/// Where the overlay panel is placed when the host window does not fit on the display. +/// +/// The reported bug: "on scrollable screens, the menu in the bottom right disappears." +/// A tall/scrollable host is a window taller than the display, AppKit constrains a +/// window's TOP under the menu bar but never lifts its bottom, and BOTH overlay modes +/// anchor the toolbar to the host's BOTTOM edge — so the pill was drawn under the Dock +/// or off the display entirely. These are the placement rules that fix it, tested as +/// pure geometry so they hold on display arrangements no test machine has. +@MainActor +final class OverlayPlacementTests: XCTestCase { + /// A 1512x982 display with a 33pt menu bar and a 61pt Dock — the shape of a real + /// laptop screen, so "visible" and "screen" are never accidentally interchangeable. + private let visible = CGRect(x: 0, y: 61, width: 1512, height: 888) + + private func annotating(_ host: CGRect, _ visible: CGRect?) -> CGRect { + OverlayPlacement.panelFrame(for: .annotating, hostFrame: host, visibleFrame: visible) + } + + private func idle(_ host: CGRect, _ visible: CGRect?) -> CGRect { + OverlayPlacement.panelFrame(for: .idle, hostFrame: host, visibleFrame: visible) + } + + // MARK: - The unchanged case + + /// A host that fits on the display must be placed EXACTLY as before the clamp, or + /// the fix has moved the overlay for every user who never had the bug. + func testHostInsideTheVisibleAreaIsPlacedExactlyAsBefore() { + let host = CGRect(x: 200, y: 200, width: 800, height: 600) + XCTAssertEqual(annotating(host, visible), host) + XCTAssertEqual(idle(host, visible), CGRect(x: host.maxX - 240, y: host.minY, width: 240, height: 104)) + } + + // MARK: - The reported direction: the bottom hangs off + + func testBottomOverhangPullsBothModesUpToTheVisibleBottom() { + // 300pt of the host is below the visible area — the shape a content-sized + // window grows into when its content outgrows the display. + let host = CGRect(x: 400, y: visible.minY - 300, width: 620, height: visible.height + 300) + + let annotate = annotating(host, visible) + XCTAssertEqual(annotate, host.intersection(visible)) + XCTAssertEqual(annotate.minY, visible.minY, "the catcher's bottom edge — where the pill is drawn — is on screen") + + let corner = idle(host, visible) + XCTAssertEqual(corner.minY, visible.minY) + XCTAssertEqual(corner.maxX, host.maxX, "the pill stays anchored to the host's right edge, which is on screen") + } + + /// The idle panel is anchored, never intersected: shrinking it to the visible region + /// would clip the pill it exists to carry. + func testIdlePanelKeepsItsFullSizeWhenTheHostHangsOff() { + let host = CGRect(x: 400, y: visible.minY - 300, width: 620, height: visible.height + 300) + XCTAssertEqual(idle(host, visible).size, OverlayPlacement.idleSize) + } + + /// The coordinate half of the fix, in the direction that is easy to get right by + /// accident: clipping only the BOTTOM leaves the AX origin (which hangs off the + /// frame's TOP edge) untouched, and only shrinks the surface — which is itself + /// correct, because the composer should clamp its cards to the VISIBLE region. + func testBottomClampLeavesTheAXOriginAloneAndOnlyShrinksTheSurface() { + let host = CGRect(x: 400, y: visible.minY - 300, width: 620, height: visible.height + 300) + let panel = annotating(host, visible) + let primaryHeight: CGFloat = 982 + XCTAssertEqual( + ScreenSpace.windowAXOrigin(cocoaFrame: panel, primaryHeight: primaryHeight), + ScreenSpace.windowAXOrigin(cocoaFrame: host, primaryHeight: primaryHeight) + ) + XCTAssertEqual(panel.height, host.height - 300) + } + + // MARK: - The other direction: the top is tucked under the menu bar + + /// The case a naive fix breaks silently. Clipping the TOP moves the AX origin, so an + /// origin still derived from the host offsets every click, highlight and card by + /// exactly the clipped amount — a fix that makes the pill reachable while shifting + /// the whole hit-test is worse than the bug it replaces. + func testTopClampMovesTheAXOriginByExactlyTheClippedAmount() { + let host = CGRect(x: 400, y: 300, width: 620, height: visible.maxY - 300 + 200) + let panel = annotating(host, visible) + XCTAssertEqual(panel.maxY, visible.maxY) + + let primaryHeight: CGFloat = 982 + let panelOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: panel, primaryHeight: primaryHeight) + let hostOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: host, primaryHeight: primaryHeight) + XCTAssertEqual(panelOrigin.y - hostOrigin.y, 200, "the origin moves down by the clipped 200pt") + XCTAssertNotEqual(panelOrigin, hostOrigin, "sanity: the two candidate origins really do differ here") + } + + // MARK: - Horizontal overhang + + func testRightOverhangPullsTheIdlePillInsideTheDisplay() { + let host = CGRect(x: 1200, y: 300, width: 800, height: 400) // 488pt off the right edge + XCTAssertEqual(idle(host, visible).maxX, visible.maxX) + XCTAssertEqual(annotating(host, visible).maxX, visible.maxX) + } + + // MARK: - Degenerate hosts + + /// A host entirely off-display (another Space, a window parked off-screen) keeps its + /// UNCLAMPED placement. Collapsing the panel to the empty intersection would leave a + /// zero-sized overlay that has to be rebuilt; an off-screen one just reappears with + /// its window. + func testHostEntirelyOffDisplayKeepsTheUnclampedPlacement() { + let host = CGRect(x: -12000, y: -12000, width: 600, height: 400) + XCTAssertEqual(annotating(host, visible), host) + XCTAssertEqual(idle(host, visible), CGRect(x: host.maxX - 240, y: host.minY, width: 240, height: 104)) + XCTAssertFalse(annotating(host, visible).isEmpty) + } + + /// No screen to clamp against (AppKit reports none mid-teardown) is not a reason to + /// place the panel nowhere. + func testNoScreenFallsBackToTheHostFrame() { + let host = CGRect(x: 100, y: -400, width: 600, height: 1400) + XCTAssertEqual(annotating(host, nil), host) + XCTAssertEqual(idle(host, nil), CGRect(x: host.maxX - 240, y: host.minY, width: 240, height: 104)) + } + + /// A host with only a sliver on screen still gets a full-size idle panel whose + /// BOTTOM edge — the edge the pill is drawn against — is inside the visible region. + func testSliverOfHostOnScreenStillYieldsAReachablePill() { + let host = CGRect(x: 400, y: visible.minY - 1000, width: 620, height: 1040) // 40pt visible + let corner = idle(host, visible) + XCTAssertEqual(corner.size, OverlayPlacement.idleSize) + XCTAssertEqual(corner.minY, visible.minY) + XCTAssertTrue(visible.contains(CGRect(x: corner.maxX - 200, y: corner.minY + 20, width: 180, height: 44)), + "the pill itself (bottom-right, 20pt inset) is inside the visible region") + } +} +#endif From bb92fe3e11a749fbec9cee6b566e39368b7907fd Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:54:37 -0500 Subject: [PATCH 21/24] feat(cards): one icon row per note card, shared with the pill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both cards showed their actions as two rows of TEXT buttons — a Parent/Child navigation row above a Cancel/Add note footer — because four labelled buttons do not fit across the card's 260pt. Four 28pt Lucide glyphs do, so the split row was a symptom of the labels rather than of the grouping. - Four glyphs added to `LucideIcon` from lucide.dev's current `d` strings: `arrow-up`, `arrow-down`, `undo-2`, `send`. `undo-2` ships as two chained 5.5-radius quarter arcs, which only render as a loop because of the arc support added earlier in this epic; `LucideArcTests` now pins the SHIPPED glyph's 45-degree points, since a flattened arc keeps its endpoints and would pass every other check. - `PillButton` becomes `IconButton`, parameterised by an `IconButtonPalette`. One implementation of the interaction logic (hover wash, disabled dim, press scale, tooltip + matching accessibility label) now serves the dark pill and the `.regularMaterial` cards; the pill's palette reproduces its previous colours exactly, so its appearance is unchanged. - Composer: [arrow-up] [arrow-down] … [undo-2] [send]. Editor: [trash] … [send]. Icon-only, each with the full label in both a tooltip and an `accessibilityLabel`. The commit action keeps its prominence through an accent tint now that `.borderedProminent` is gone. Every disabled rule, the destructive treatment, the `⏎ save · ⇧⏎ newline` hint, and the session-driven draft clearing are unchanged. - `AnnotationCard` loses its `navigation` slot: it existed only to hold the second row. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/Overlay/OverlayView.swift | 243 +++++++++++++-------- Sources/AnnotKit/Overlay/PillStyle.swift | 112 +++++++++- Tests/AnnotKitTests/LucideArcTests.swift | 47 +++- Tests/AnnotKitTests/LucideIconTests.swift | 4 + 4 files changed, 302 insertions(+), 104 deletions(-) diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index fe664f1..e84430f 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -334,8 +334,16 @@ struct OverlayView: View { } /// The WRITE card: a shared ``AnnotationCard`` anchored to the selected - /// element, with a Cancel / Add note footer. Enter submits; Escape dismisses via - /// the host's key monitor (``EscapeRule``), not from inside this view. + /// element, with a single icon row — tree navigation on the left, dismiss and + /// commit on the right. Enter submits; Escape dismisses via the host's key + /// monitor (``EscapeRule``), not from inside this view. + /// + /// The four controls used to be two rows of TEXT buttons (a Parent/Child + /// navigation row above a Cancel/Add note footer), because four labelled + /// buttons do not fit across 260pt. Four 28pt glyphs do, with room to spare, so + /// the split row was a symptom of the labels rather than of the grouping — and + /// a two-row footer on a card whose whole job is one text field read as heavier + /// than the thing it was framing. private var composer: some View { AnnotationCard( header: composerHeader, @@ -345,54 +353,65 @@ struct OverlayView: View { // is not re-inserted, so `.onAppear` won't refire). focusKey: session.selected?.id ?? "", onSubmit: { addNote() }, - onFocusRequest: onFocusRequest, - // Tree navigation: rebind the note to the enclosing component, or to a - // component inside the current one. Its own row rather than a third - // footer button — the card is 260pt wide and Cancel/Add note already - // fill it, so a third control there would be cramped enough to misread. - // - // DISABLED, never hidden. The control this replaces (a lone "Widen" - // button) appeared and disappeared with availability, and that is a - // large part of why it was unlearnable: a control you have never seen - // is a control you cannot predict. Both buttons are always present, so - // "you can move the binding up and down the tree" is visible from the - // first note, and greying tells you where you are in the tree. - // - // No element name here: the card HEADER already shows the bound - // element (`session.selectionLabel`) and re-renders as you navigate, so - // it is the "where am I" indicator and repeating it would be noise. - navigation: { - HStack(spacing: 6) { - Button { session.selectParent() } label: { - Label("Parent", systemImage: "chevron.up") - } - .disabled(!session.canSelectParent) - // Tooltips name the EFFECT, not the mechanism. "Widen" described - // what the code did to the ladder and read as "make the - // highlight bigger"; what the user is choosing is which - // component the note is FILED AGAINST. - .help("Bind this note to the enclosing component") - Button { session.selectChild() } label: { - Label("Child", systemImage: "chevron.down") - } - .disabled(!session.canSelectChild) - .help("Bind this note to a component inside") - Spacer() - } - .controlSize(.small) - .frame(width: 260) - } + onFocusRequest: onFocusRequest ) { - HStack { + HStack(spacing: 4) { + // Tree navigation: rebind the note to the enclosing component, or to + // a component inside the current one. Leading, and separated from + // the commit pair by the Spacer, because they change WHAT the note + // is filed against — they are inputs to the note, not ways of + // ending it. + // + // DISABLED, never hidden. The control this replaces (a lone "Widen" + // button) appeared and disappeared with availability, and that is a + // large part of why it was unlearnable: a control you have never + // seen is a control you cannot predict. Both buttons are always + // present, so "you can move the binding up and down the tree" is + // visible from the first note, and the dim tells you where you are + // in the tree. + // + // No element name in these labels: the card HEADER already shows the + // bound element (`session.selectionLabel`) and re-renders as you + // navigate, so it is the "where am I" indicator and repeating it + // would be noise. + IconButton( + icon: .arrowUp, + palette: .card, + isDisabled: !session.canSelectParent, + tooltip: "Select parent component", + action: { session.selectParent() } + ) + IconButton( + icon: .arrowDown, + palette: .card, + isDisabled: !session.canSelectChild, + tooltip: "Select child component", + action: { session.selectChild() } + ) + Spacer() // No `comment = ""` here any more: the draft is cleared by the // composer CLOSING (see the `session.selected` hook on the ZStack), so // every dismissal path — this button, Escape, leaving the mode — // clears it identically. - Button("Cancel") { session.cancelSelection() } - Spacer() - Button("Add note") { addNote() } - .buttonStyle(.borderedProminent) - .disabled(comment.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + IconButton( + icon: .undo2, + palette: .card, + tooltip: "Cancel", + action: { session.cancelSelection() } + ) + // The commit action, and the ONLY tinted glyph on the card. It lost + // `.borderedProminent` along with its label, and without the tint it + // would be one more grey 16pt glyph in a row of four — the user would + // have to read four tooltips to find out which one files the note. + // Colour carries the prominence that the filled capsule used to. + IconButton( + icon: .send, + palette: .card, + isDisabled: comment.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + glyphTint: .accentColor, + tooltip: "Add note", + action: { addNote() } + ) } } } @@ -411,11 +430,21 @@ struct OverlayView: View { } /// The EDIT card: the SAME shared ``AnnotationCard`` chrome as the composer, - /// anchored to the tapped pin instead of an element, with a Delete / Save - /// footer. Enter saves, Escape (via the host's key monitor) closes it, and + /// anchored to the tapped pin instead of an element, with a Delete / Save icon + /// row. Enter saves, Escape (via the host's key monitor) closes it, and /// Save/Delete/click-away all end /// editing. Because it lives in the overlay panel (not a system `.popover`), /// it inherits the composer's reliable `panel.makeKey()` focus. + /// + /// Its row is icons for the same reason the composer's is, and it changed at the + /// same time on purpose: two cards that share every pixel of their chrome but + /// disagree about whether actions are words or glyphs would read as a bug in + /// whichever one the user opened second. + /// + /// No navigation pair here: this card edits a note that has ALREADY been + /// captured, whose selector, component and element path were frozen at capture. + /// Offering Parent/Child would move a highlight and change nothing about the + /// record, which is worse than no control at all. private func editCard(note: AnnotationNote, anchor: CGPoint) -> some View { AnnotationCard( header: note.selector, @@ -424,26 +453,34 @@ struct OverlayView: View { // Re-focus when the editor moves pin→pin without re-insertion. focusKey: note.id, onSubmit: { saveEdit(note) }, - onFocusRequest: onFocusRequest, - // No navigation row: this card edits a note that has ALREADY been - // captured, whose selector, component and element path were frozen at - // capture. Offering Parent/Child here would move a highlight and change - // nothing about the record, which is worse than no control at all. - navigation: { EmptyView() } + onFocusRequest: onFocusRequest ) { - HStack { - Button(role: .destructive) { - session.deleteNote(id: note.id) - session.endEditing() - } label: { - Label("Delete", systemImage: "trash") - .foregroundStyle(.red) - } - .tint(.red) + HStack(spacing: 4) { + // Destructive treatment survives the loss of the word "Delete": red + // at rest and a full red wash on hover (see ``IconButtonPalette.card``), + // so the one irreversible control on either card is still the only + // coloured thing besides the commit action. + IconButton( + icon: .trash, + palette: .card, + isDestructive: true, + tooltip: "Delete", + action: { + session.deleteNote(id: note.id) + session.endEditing() + } + ) Spacer() - Button("Save") { saveEdit(note) } - .buttonStyle(.borderedProminent) - .disabled(editDraft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + // Same glyph and same accent as the composer's Add note: both cards + // commit with `send`, so the gesture is learned once. + IconButton( + icon: .send, + palette: .card, + isDisabled: editDraft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + glyphTint: .accentColor, + tooltip: "Save", + action: { saveEdit(note) } + ) } } } @@ -571,8 +608,8 @@ struct OverlayView: View { /// at the bottom of `body`). /// /// The two flows differ ONLY in the `header` text, the `text` binding, the -/// `placement` anchor, the optional `navigation` row, and the `footer` button row. -private struct AnnotationCard: View { +/// `placement` anchor, and the `footer` icon row. +private struct AnnotationCard: View { /// Header label: the composer shows the element's selection label; the editor /// shows the note's selector. let header: String @@ -590,12 +627,9 @@ private struct AnnotationCard: View { /// Make the host panel key so the field accepts keystrokes (`panel.makeKey()` /// on macOS; a no-op on iOS, where `@FocusState` alone raises the keyboard). let onFocusRequest: () -> Void - /// Row between the field and the footer: the composer's Parent/Child tree - /// navigation, `EmptyView` for the pin editor (whose binding is already fixed). - /// A separate slot rather than more footer buttons, so the 260pt footer keeps - /// exactly its two primary actions. - @ViewBuilder let navigation: () -> Navigation - /// The differing two-button row (Cancel/Add note vs Delete/Save). + /// The differing icon row (Parent/Child/Cancel/Add note vs Delete/Save). One + /// slot, not the two it used to be: the separate `navigation` row existed only + /// because four TEXT buttons could not share 260pt, and four glyphs can. @ViewBuilder let footer: () -> Footer @FocusState private var focused: Bool @@ -626,11 +660,6 @@ private struct AnnotationCard: View { onSubmit() return .handled } - // No `.frame(width:)` here, unlike the rows around it: the editor - // passes `EmptyView`, and a frame modifier would give that nothing a - // 260pt-wide slot and an 8pt VStack gap on a card that has no nav row. - // The composer's row carries its own width instead. - navigation() footer() .frame(width: 260) } @@ -735,13 +764,13 @@ private struct ToolbarView: View { // control to its right, these change how the NEXT press behaves, so // they must stay live in an empty session, which is exactly when a // user is choosing how to make their first selection. - PillButton( + IconButton( icon: .mousePointer, isActive: session.tool == .point, tooltip: "Select by clicking", action: { session.setTool(.point) } ) - PillButton( + IconButton( icon: .squareDashed, isActive: session.tool == .frame, tooltip: "Select by drawing a frame", @@ -755,28 +784,28 @@ private struct ToolbarView: View { .frame(width: 1, height: 16) .padding(.horizontal, 3) .allowsHitTesting(false) - PillButton( + IconButton( icon: justCopied ? .check : .copy, isDisabled: !hasNotes, glyphTint: justCopied ? PillStyle.success : nil, tooltip: justCopied ? "Copied" : "Copy notes (Markdown)", action: { onCopy(); flashCopied() } ) - PillButton( + IconButton( icon: .download, isDisabled: !hasNotes, tooltip: "Export to AGENTATION_NOTES.md", action: onExport ) - PillButton(icon: .trash, isDestructive: true, isDisabled: !hasNotes, tooltip: "Clear notes") { + IconButton(icon: .trash, isDestructive: true, isDisabled: !hasNotes, tooltip: "Clear notes") { session.clear() } // Deliberately NOT gated on `hasNotes`: with zero notes every other // control is inert, so the X is the only live thing left and must // still work. - PillButton(icon: .close, tooltip: "Stop annotating", action: onToggle) + IconButton(icon: .close, tooltip: "Stop annotating", action: onToggle) } else { - PillButton(icon: .pencil, tooltip: "Annotate", action: onToggle) + IconButton(icon: .pencil, tooltip: "Annotate", action: onToggle) } } // Idle shows ONE 28pt button, so the inset must be EVEN (8pt all around -> @@ -834,11 +863,23 @@ private struct ToolbarView: View { } } -/// A single 28pt circular icon button in the pill: transparent when idle, a faint -/// white wash on hover (red for destructive). Each carries a tooltip (`.help`) -/// and a matching accessibility label. -private struct PillButton: View { +/// A single 28pt circular Lucide icon button: transparent when idle, a faint wash +/// on hover (the destructive fill for destructive actions), dimmed and inert when +/// disabled, pressed in slightly on tap. Each carries a tooltip (`.help`) and a +/// MATCHING accessibility label — an icon-only control with a tooltip alone is +/// simply unlabelled to VoiceOver, which is why the two are one parameter here +/// rather than two that can be filled in independently. +/// +/// ONE implementation serves both the dark toolbar pill and the `.regularMaterial` +/// note cards, differing only by ``IconButtonPalette``. The colours are the cheap +/// part; the reason this is not two views is the interaction logic above, where a +/// second copy would drift silently (hover that survives a disable, a press scale +/// that stops honouring reduce-motion) and only ever be noticed on one surface. +private struct IconButton: View { let icon: LucideIcon + /// Defaulted to the pill so the toolbar's seven call sites stay exactly as they + /// were — this is a refactor for the pill, not a restyle of it. + var palette: IconButtonPalette = .pill var isDestructive: Bool = false /// Dims the glyph and makes the button a true no-op (used by copy/export/clear /// while there are no notes to act on). @@ -850,7 +891,15 @@ private struct PillButton: View { /// A segment with no lit member is worse than no segment at all, because the /// user cannot tell whether a click will select a point or do nothing. var isActive: Bool = false + /// Overrides the glyph colour outright, for a button whose MEANING is a colour: + /// the pill's copied-check flash, and the cards' accent-tinted commit action, + /// which is how `send` keeps the prominence `.borderedProminent` used to give it. + /// Ranked below `isDisabled` on purpose — an accent glyph on a dead button would + /// advertise a commit the empty field cannot make. var glyphTint: Color? = nil + /// Shown on hover AND read by VoiceOver. Name the effect in full: the glyph no + /// longer carries a word beside it, so this string is the only place the action + /// is named at all. let tooltip: String let action: () -> Void @@ -860,14 +909,14 @@ private struct PillButton: View { private var glyphColor: Color { // Disabled outranks every other state: a dimmed glyph with no hover/active // treatment reads as unavailable. - if isDisabled { return PillStyle.iconIdle.opacity(0.4) } + if isDisabled { return palette.disabled } if let glyphTint { return glyphTint } // Active outranks hover: hovering the ALREADY-active tool must not dim it // toward the inactive treatment, which would read as "clicking this turns // it off" for a segment that has no off. - if isActive { return PillStyle.iconActive } - if isDestructive && hovering { return .white } - return hovering ? PillStyle.iconHover : PillStyle.iconIdle + if isActive { return palette.active } + if isDestructive { return hovering ? palette.destructiveHover : palette.destructiveIdle } + return hovering ? palette.hover : palette.idle } // Hover is the ONLY fill state, even for the active tool: the segment is @@ -875,7 +924,7 @@ private struct PillButton: View { // active tool cannot be mistaken for the hover wash sitting on a neighbour. private var fillColor: Color { // No hover wash while disabled — the button must look inert. - if hovering && !isDisabled { return isDestructive ? PillStyle.destructive : PillStyle.hoverBackground } + if hovering && !isDisabled { return isDestructive ? palette.destructiveFill : palette.hoverFill } return .clear } @@ -889,9 +938,9 @@ private struct PillButton: View { .background(Circle().fill(fillColor)) .contentShape(Circle()) } - .buttonStyle(PressablePillButtonStyle(reduceMotion: reduceMotion)) + .buttonStyle(PressableIconButtonStyle(reduceMotion: reduceMotion)) .disabled(isDisabled) - .pillToolTip(tooltip) + .iconToolTip(tooltip) .accessibilityLabel(tooltip) .onHover { value in // Ignore hover entirely while disabled so no wash/glyph change leaks in. @@ -904,7 +953,7 @@ private struct PillButton: View { /// Presses the glyph in slightly on tap (`scaleEffect(0.96)`), honoring /// reduce-motion by dropping the animation. -private struct PressablePillButtonStyle: ButtonStyle { +private struct PressableIconButtonStyle: ButtonStyle { let reduceMotion: Bool func makeBody(configuration: Configuration) -> some View { diff --git a/Sources/AnnotKit/Overlay/PillStyle.swift b/Sources/AnnotKit/Overlay/PillStyle.swift index 2339a30..588d17b 100644 --- a/Sources/AnnotKit/Overlay/PillStyle.swift +++ b/Sources/AnnotKit/Overlay/PillStyle.swift @@ -50,6 +50,70 @@ enum PillStyle { static let divider = Color.white.opacity(0.1) } +// MARK: - Icon-button palette + +/// The colours a single ``IconButton`` needs, so ONE implementation of the button's +/// mechanics (hover wash, disabled dim, press scale, tooltip + accessibility label) +/// can serve two surfaces that must not look alike. The pill is an opaque `#1A1A1A` +/// capsule, where white-on-dark is the only legible treatment; the note cards are +/// `.regularMaterial`, where that same white glyph would all but vanish against a +/// light desktop showing through. Parameterising the eight colours is what keeps +/// the second surface from becoming a second copy of the interaction logic — which +/// is the part that would actually drift. +struct IconButtonPalette: Sendable { + let idle: Color + let hover: Color + /// The lit member of a segmented control (the pill's tool pair). The cards have + /// no persistent-state control, so this is simply never reached there. + let active: Color + /// Disabled is a colour, not an opacity modifier, because the two surfaces dim + /// from different starting points: white-at-0.4 on the pill, the system's + /// secondary label on the cards. + let disabled: Color + let destructiveIdle: Color + /// Glyph colour once the destructive fill is behind it — it has to survive a + /// saturated red, so it is not simply ``hover``. + let destructiveHover: Color + let hoverFill: Color + let destructiveFill: Color +} + +extension IconButtonPalette { + /// The pill's palette, byte-identical to what ``PillStyle`` already drove: the + /// destructive glyph at rest is deliberately the SAME dim white as every other + /// glyph, because on the pill "this one deletes" is announced by the red hover + /// wash alone and a permanently red glyph in that row would read as an error. + static let pill = IconButtonPalette( + idle: PillStyle.iconIdle, + hover: PillStyle.iconHover, + active: PillStyle.iconActive, + disabled: PillStyle.iconIdle.opacity(0.4), + destructiveIdle: PillStyle.iconIdle, + destructiveHover: .white, + hoverFill: PillStyle.hoverBackground, + destructiveFill: PillStyle.destructive + ) + + /// The note cards' palette: system label colours, so the glyphs track the + /// viewer's appearance the way the `.regularMaterial` behind them already does. + /// Hard-coding the pill's white here is the specific failure this exists to + /// prevent — it is invisible on a light background, which is most of them. + /// + /// The card's destructive glyph IS red at rest, unlike the pill's: it replaces a + /// `Label("Delete")` that was already `.red`, and the card has no second red + /// element for it to be confused with. + static let card = IconButtonPalette( + idle: .secondary, + hover: .primary, + active: .primary, + disabled: Color.secondary.opacity(0.4), + destructiveIdle: .red, + destructiveHover: .white, + hoverFill: Color.primary.opacity(0.08), + destructiveFill: .red + ) +} + // MARK: - Lucide icon model /// A primitive on Lucide's 24x24 design grid. Modeling each glyph as a small @@ -124,6 +188,45 @@ struct LucideIcon { .path("M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"), ]) + /// Lucide `arrow-up` / `arrow-down` — the composer's tree navigation: bind the + /// note to the enclosing component, or to one inside it. Full-shaft arrows + /// rather than the bare `chevron.up`/`chevron.down` they replace: a lone + /// chevron at 16pt reads as "expand/collapse a disclosure", which is the wrong + /// promise for a control that MOVES the binding, and the shaft is what makes + /// the pair read as travel along an axis. + static let arrowUp = LucideIcon(parts: [ + .path("m5 12 7-7 7 7"), + .path("M12 19V5"), + ]) + + static let arrowDown = LucideIcon(parts: [ + .path("M12 5v14"), + .path("m19 12-7 7-7-7"), + ]) + + /// Lucide `undo-2` — dismiss the composer without capturing. Its identity is + /// the semicircular loop, authored upstream as TWO chained 5.5-radius quarter + /// arcs; both quarters must render as real curves or the glyph degrades to a + /// triangular pennant that reads as nothing in particular. `LucideArcTests` + /// pins the loop's 45-degree points for exactly that reason. + /// + /// Chosen over `x` for Cancel because the card's other neutral glyphs are all + /// directional: an X would be the only "destroy" mark on a row whose + /// destructive slot (the editor's trash) is a different button entirely. + static let undo2 = LucideIcon(parts: [ + .path("M9 14 4 9l5-5"), + .path("M4 9h10.5a5.5 5.5 0 0 1 5.5 5.5a5.5 5.5 0 0 1-5.5 5.5H11"), + ]) + + /// Lucide `send` — the commit action on both cards (Add note / Save). The real + /// `d`, whose body is one closed subpath of `a`-rounded corners, so it is a + /// filled-looking dart only because the corners are true arcs; flattened it + /// collapses into a scalene triangle with a nick in it. + static let send = LucideIcon(parts: [ + .path("M14.536 21.686a.5.5 0 0 0 .937-.024l6.5-19a.496.496 0 0 0-.635-.635l-19 6.5a.5.5 0 0 0-.024.937l7.93 3.18a2 2 0 0 1 1.112 1.11z"), + .path("m21.854 2.147-10.94 10.939"), + ]) + /// Lucide `square-dashed` — the FRAME tool: select by drawing a frame. Twelve /// short strokes rather than one outline, and that is the point: the gaps echo /// the dashed rubber band the tool draws, so the button previews its own @@ -453,10 +556,13 @@ private struct ToolTipBacking: NSViewRepresentable { #endif extension View { - /// Hover tooltip for a pill control: SwiftUI `.help` plus (on macOS) an - /// NSView-backed `toolTip` for reliability inside the overlay panel. + /// Hover tooltip for an icon-only control (pill or note card): SwiftUI `.help` + /// plus (on macOS) an NSView-backed `toolTip` for reliability inside the overlay + /// panel. Both surfaces need the AppKit backing — the cards live in the same + /// borderless, non-activating panel the pill does, where `.help` alone can fail + /// to render, and a card button is now the ONLY place its action is named. @ViewBuilder - func pillToolTip(_ text: String) -> some View { + func iconToolTip(_ text: String) -> some View { #if os(macOS) help(text).background(ToolTipBacking(text: text)) #else diff --git a/Tests/AnnotKitTests/LucideArcTests.swift b/Tests/AnnotKitTests/LucideArcTests.swift index d978cec..22a45f3 100644 --- a/Tests/AnnotKitTests/LucideArcTests.swift +++ b/Tests/AnnotKitTests/LucideArcTests.swift @@ -76,6 +76,14 @@ final class LucideArcTests: XCTestCase { return box } + /// The `d` strings of a SHIPPED glyph, so a test can assert on the icon the UI + /// actually draws rather than on a copy of its path retyped into the test. + private func pathData(_ icon: LucideIcon) -> [String] { + var strings: [String] = [] + for case .path(let d) in icon.parts { strings.append(d) } + return strings + } + private func distanceToCurve(_ d: String, _ target: CGPoint) -> CGFloat { samples(d, per: 64).map { hypot($0.x - target.x, $0.y - target.y) }.min() ?? .infinity } @@ -299,13 +307,44 @@ final class LucideArcTests: XCTestCase { // MARK: Shipped glyphs - /// `pencil`, `mouse-pointer-2` and `square-dashed` all carry `a` commands and - /// therefore change shape with real arcs. Their curves must still land inside - /// the 24-unit design grid — the F.6.6.2 scale-up on `pencil`'s r=1 eraser - /// arc is the one that could plausibly push a glyph out of bounds. + /// The shipped `undo-2`, asserted on the glyph itself rather than on a `d` + /// string retyped here: what has to hold is that the icon the note cards draw + /// still contains a real loop. This is the one failure the rest of the suite + /// cannot see — a flattened arc keeps its ENDPOINTS, so the glyph would still + /// render, still fill the grid, still pass every non-empty/in-bounds check, and + /// simply stop looking like undo. Off-screen, nothing else would notice. + func testShippedUndoGlyphDrawsARealLoop() { + let strings = pathData(.undo2) + XCTAssertEqual(strings.count, 2, "undo2 is an arrowhead plus a loop") + let loop = strings[1] + + // The loop spans from the shaft's start out to the circle's right and + // bottom extremes — centre (14.5, 14.5), radius 5.5. + let box = drawnBounds(loop) + XCTAssertEqual(box.minX, 4, accuracy: 0.02) + XCTAssertEqual(box.maxX, 20, accuracy: 0.02, "the loop never reached the circle's right extreme") + XCTAssertEqual(box.minY, 9, accuracy: 0.02) + XCTAssertEqual(box.maxY, 20, accuracy: 0.02, "the loop never reached the circle's bottom extreme") + + // Both quarters' 45-degree points are ON the curve... + let offset: CGFloat = 5.5 / CGFloat(2).squareRoot() + XCTAssertLessThan(distanceToCurve(loop, CGPoint(x: 14.5 + offset, y: 14.5 - offset)), 0.02) + XCTAssertLessThan(distanceToCurve(loop, CGPoint(x: 14.5 + offset, y: 14.5 + offset)), 0.02) + // ...and the chord midpoints a straight-line arc would pass through are + // 1.61 units AWAY from it. This pair is the assertion with teeth: the + // bounding box above survives flattening, these do not. + XCTAssertGreaterThan(distanceToCurve(loop, CGPoint(x: 17.25, y: 11.75)), 1.5, "the first quarter is a chord") + XCTAssertGreaterThan(distanceToCurve(loop, CGPoint(x: 17.25, y: 17.25)), 1.5, "the second quarter is a chord") + } + + /// Every `a`-carrying glyph must still land inside the 24-unit design grid — + /// the F.6.6.2 scale-up on `pencil`'s r=1 eraser arc and on `send`'s r=.5 + /// corners is what could plausibly push one out of bounds, and a glyph that + /// spills is clipped by the 16pt frame rather than drawn small. func testArcCarryingGlyphsStayOnTheDesignGrid() { let glyphs: [(String, LucideIcon)] = [ ("pencil", .pencil), ("mousePointer", .mousePointer), ("squareDashed", .squareDashed), + ("undo2", .undo2), ("send", .send), ] for (name, icon) in glyphs { var points: [CGPoint] = [] diff --git a/Tests/AnnotKitTests/LucideIconTests.swift b/Tests/AnnotKitTests/LucideIconTests.swift index 0845cf0..b9f62eb 100644 --- a/Tests/AnnotKitTests/LucideIconTests.swift +++ b/Tests/AnnotKitTests/LucideIconTests.swift @@ -19,6 +19,10 @@ final class LucideIconTests: XCTestCase { ("close", .close), ("mousePointer", .mousePointer), ("squareDashed", .squareDashed), + ("arrowUp", .arrowUp), + ("arrowDown", .arrowDown), + ("undo2", .undo2), + ("send", .send), ] func testEachIconRendersNonEmptyPath() { From 8d3091b95ddb8ac248c5e0491280a43c40ea893f Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:20:27 -0500 Subject: [PATCH 22/24] fix(overlay): stop the hover-driven relayout storm that hid the pill Two multipliers behind 'the menu disappears when I hover over it and then off it, on scrollable screens'. AXIntrospection.appElement() WROTE AXEnhancedUserInterface on every call, and the catcher calls it from the hover hit-test at up to 60Hz. That attribute announces an assistive client, and AppKit answers by re-evaluating and relaying out its windows -- so a moving pointer drove a resize storm in the host. Hovering ONTO the pill stops the writes (the pill consumes hover, so the catcher sees .ended and queries nothing) and moving OFF restarts them, which is exactly the reported trigger. Scrollable screens surfaced it first because a large scroll view is a large AX tree, so materialising it costs a real layout pass. It is now set once per process and the element is cached. Each resulting resize then pushed a fresh SwiftUI root view, tearing the pill down and rebuilding it mid-hover. syncFrameAndOrigin() now skips that when neither axOrigin nor surfaceSize changed, while still recording the host frame for the settle poll and still rebuilding on a REAL change. Both pinned by mutation-verified tests. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/macOS/AXIntrospection.swift | 33 +++++- .../AnnotKit/macOS/OverlayController.swift | 20 +++- Tests/AnnotKitTests/HoverStormTests.swift | 107 ++++++++++++++++++ 3 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 Tests/AnnotKitTests/HoverStormTests.swift diff --git a/Sources/AnnotKit/macOS/AXIntrospection.swift b/Sources/AnnotKit/macOS/AXIntrospection.swift index 6a752ac..bf6886a 100644 --- a/Sources/AnnotKit/macOS/AXIntrospection.swift +++ b/Sources/AnnotKit/macOS/AXIntrospection.swift @@ -97,12 +97,39 @@ enum AXIntrospection { // MARK: - Application element - /// Our own application AX element, with `AXEnhancedUserInterface` set so - /// AppKit/SwiftUI materializes the full semantic tree for us. Self-query - /// needs no accessibility-trust prompt. + /// The process's application AX element, cached, with `AXEnhancedUserInterface` + /// set EXACTLY ONCE so AppKit/SwiftUI materializes the full semantic tree for us. + /// Self-query needs no accessibility-trust prompt. + /// + /// Setting that attribute once is load-bearing, not an optimization. It was + /// previously re-set on every call — and this is called from the HOVER hit-test, + /// which the catcher runs at up to 60Hz while the pointer moves. Writing + /// `AXEnhancedUserInterface` tells AppKit an assistive client just attached, and + /// AppKit responds by re-evaluating (and, on SwiftUI hosts, relaying out) its + /// windows; doing that 60 times a second drove a resize storm in the host, each + /// resize firing `didResize` -> `syncFrameAndOrigin()` -> a fresh SwiftUI root + /// view, which is what made the toolbar pill visibly vanish. + /// + /// The reported trigger pinpointed it: hovering ONTO the pill stops the storm + /// (the pill consumes hover, so the catcher sees `.ended` and queries nothing) + /// and moving OFF it restarts them. It showed up on scrollable screens because a + /// large scroll view is a large AX tree, so materializing it is far more likely + /// to cost a real layout pass. + /// + /// The element itself is stable for the life of the process, so caching it also + /// drops an `AXUIElementCreateApplication` per query. + private static var cachedAppElement: AXUIElement? + /// How many times `AXEnhancedUserInterface` has been written. Must never exceed 1; + /// the probe asserts it across a hover storm so the regression cannot come back + /// silently. + private(set) static var enhancedUserInterfaceWrites = 0 + private static func appElement() -> AXUIElement { + if let cachedAppElement { return cachedAppElement } let app = AXUIElementCreateApplication(ProcessInfo.processInfo.processIdentifier) AXUIElementSetAttributeValue(app, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + enhancedUserInterfaceWrites += 1 + cachedAppElement = app return app } diff --git a/Sources/AnnotKit/macOS/OverlayController.swift b/Sources/AnnotKit/macOS/OverlayController.swift index 4e09856..63f7848 100644 --- a/Sources/AnnotKit/macOS/OverlayController.swift +++ b/Sources/AnnotKit/macOS/OverlayController.swift @@ -35,6 +35,10 @@ public final class OverlayController: NSObject { private var axOrigin: CGPoint = .zero /// Panel-local size, for clamping the composer inside the visible region. private var surfaceSize: CGSize = .zero + /// How many times a fresh SwiftUI root view has been pushed into the hosting view. + /// Rebuilding it mid-hover is what makes the pill flicker or vanish, so the probe + /// asserts a storm of redundant geometry notifications pushes none. + private(set) var rootViewPushes = 0 /// Host frame captured at the last geometry sync. A SwiftUI/content-sized host /// can attach at its PRE-LAYOUT frame and grow to its final size a runloop turn @@ -374,9 +378,21 @@ public final class OverlayController: NSObject { // the VISIBLE region rather than inside a window that runs off the display. // Clamping the TOP (a host tucked under the menu bar) genuinely does move the // origin, and that is the case a host-derived origin breaks silently. - axOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: panelFrame, primaryHeight: primaryHeight) - surfaceSize = panelFrame.size + let newAXOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: panelFrame, primaryHeight: primaryHeight) + // Always record the host frame, even on the early-out below: the settle poll + // decides "has the host stopped growing" by comparing against this, so leaving + // it stale would keep the poll re-syncing a window that has already settled. lastSyncedHostFrame = host.frame + // Push a new SwiftUI root view ONLY when something it renders from actually + // changed. `syncFrameAndOrigin()` runs on every move/resize/screen-parameter + // notification, and a host that emits a burst of them (a relayout storm, a live + // resize drag) would otherwise replace the root view on each one — tearing down + // and rebuilding the pill mid-hover, which reads as the toolbar flickering or + // vanishing. Cheap guard, and it makes a redundant notification free. + guard newAXOrigin != axOrigin || panelFrame.size != surfaceSize else { return } + axOrigin = newAXOrigin + surfaceSize = panelFrame.size + rootViewPushes += 1 hostingView?.rootView = makeRootView() } diff --git a/Tests/AnnotKitTests/HoverStormTests.swift b/Tests/AnnotKitTests/HoverStormTests.swift new file mode 100644 index 0000000..e0cb287 --- /dev/null +++ b/Tests/AnnotKitTests/HoverStormTests.swift @@ -0,0 +1,107 @@ +#if os(macOS) +import AppKit +import CoreGraphics +import Foundation +import XCTest +@testable import AnnotKit + +/// The vanishing-pill regression, reported as: "the menu disappears when I hover over +/// it and then off it, on scrollable screens." +/// +/// Two multipliers, both fixed, both pinned here: +/// +/// 1. `AXIntrospection.appElement()` WROTE `AXEnhancedUserInterface` on every call, +/// and the catcher calls it from the hover hit-test at up to 60Hz. That attribute +/// announces an assistive client; AppKit answers by re-evaluating and relaying out +/// its windows, so a moving pointer drove a resize storm in the host. +/// 2. Every resulting resize pushed a FRESH SwiftUI root view, tearing the pill down +/// and rebuilding it mid-hover. +/// +/// The reported trigger is what identified it: hovering ONTO the pill stops the +/// writes — the pill consumes hover, so the catcher sees `.ended` and queries nothing +/// — and moving OFF restarts them. Scrollable screens showed it first because a large +/// scroll view is a large AX tree, so materializing it costs a real layout pass. +@MainActor +final class HoverStormTests: XCTestCase { + /// A hover storm must write the AX attribute ZERO extra times. + /// + /// `snapshot()` goes through the same `appElement()` every hover hit-test uses, so + /// this exercises the real path without needing a live window. The first call in + /// the process may legitimately write once; what must never happen again is a + /// second write, let alone one per query. + func testHoverStormDoesNotRewriteTheEnhancedUIAttribute() { + _ = NSApplication.shared + // Prime the cache so this test is independent of whichever test ran first. + _ = AXIntrospection.snapshot() + let writesAfterPriming = AXIntrospection.enhancedUserInterfaceWrites + XCTAssertLessThanOrEqual(writesAfterPriming, 1, "the attribute is set once per process, not per query") + + // ~3 seconds of real 60Hz hovering. + for _ in 0 ..< 200 { _ = AXIntrospection.snapshot() } + + XCTAssertEqual(AXIntrospection.enhancedUserInterfaceWrites, writesAfterPriming, + "200 queries wrote AXEnhancedUserInterface again — this is the resize storm that ate the pill") + } + + /// Redundant geometry notifications must not rebuild the overlay's SwiftUI root. + /// + /// A relayout storm arrives as a burst of `didResize`/`didMove`. None of them + /// change the panel's geometry, so none may tear down and rebuild the pill — that + /// rebuild is what the user SEES as the toolbar vanishing. + func testRedundantGeometryNotificationsDoNotRebuildTheOverlay() { + _ = NSApplication.shared + let host = NSWindow(contentRect: NSRect(x: 200, y: 200, width: 640, height: 480), + styleMask: [.titled, .resizable], backing: .buffered, defer: false) + host.orderFront(nil) + defer { host.orderOut(nil) } + + let controller = OverlayController( + session: AnnotationSession(source: MacElementSource(), sink: NotesFileSink(path: "/dev/null")) + ) + controller.mount(on: host) + defer { controller.unmount() } + + let pushesBefore = controller.rootViewPushes + let frameBefore = host.childWindows?.first?.frame + + for _ in 0 ..< 50 { + NotificationCenter.default.post(name: NSWindow.didResizeNotification, object: host) + NotificationCenter.default.post(name: NSWindow.didMoveNotification, object: host) + } + + XCTAssertEqual(controller.rootViewPushes, pushesBefore, + "100 redundant notifications rebuilt the pill — mid-hover that reads as it vanishing") + XCTAssertEqual(host.childWindows?.first?.frame, frameBefore, "and the panel must not have moved") + } + + /// The guard must skip REDUNDANT syncs, not all of them. + /// + /// An idempotence check that silently stopped syncing would be a far worse bug + /// than the flicker it fixes: the panel would drift away from a host that really + /// did move. + func testARealResizeStillRebuildsTheOverlay() { + _ = NSApplication.shared + let host = NSWindow(contentRect: NSRect(x: 200, y: 200, width: 640, height: 480), + styleMask: [.titled, .resizable], backing: .buffered, defer: false) + host.orderFront(nil) + defer { host.orderOut(nil) } + + let controller = OverlayController( + session: AnnotationSession(source: MacElementSource(), sink: NotesFileSink(path: "/dev/null")) + ) + controller.mount(on: host) + defer { controller.unmount() } + + let pushesBefore = controller.rootViewPushes + // MOVE the host rather than growing it. In idle mode the panel is a fixed-size + // corner pinned to the host's BOTTOM edge, so growing the window upward leaves + // the panel exactly where it was — the guard correctly skips that, and a test + // built on it would be asserting the guard is broken. + host.setFrameOrigin(NSPoint(x: host.frame.minX + 140, y: host.frame.minY + 90)) + NotificationCenter.default.post(name: NSWindow.didMoveNotification, object: host) + + XCTAssertGreaterThan(controller.rootViewPushes, pushesBefore, + "a real resize must still push a new root view, or the overlay stops tracking its host") + } +} +#endif From 3267df72c5e84dfc0f900497378c6db86f230329 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:10 -0500 Subject: [PATCH 23/24] fix(overlay): re-assert the clamped panel frame after AppKit's parent-follow THE vanishing-toolbar root cause, measured against a live host rather than reasoned about. AppKit repositions a CHILD window to preserve its offset from its parent, and it does so AFTER the didMove notification the controller reacts to. So placement computed the correct visible-frame-clamped rect, applied it, and was then silently dragged back a runloop turn later. With forensics on: host=(221,-200,1291,889) computed=(1272,60,240,104) afterSet=(1272,60,240,104) next-turn panel=(1272,-200,240,104) clobbered=true On a host whose bottom hangs below the display -- a tall scrollable window, which is what 'on scrollable screens' meant -- that parks the pill under the Dock or off the display entirely. Verified end to end: moving the host down 260pt now leaves the pill at y=818 (on screen) instead of 1078 (off it). Also fixes the scroll hand-off. Forwarding the NSEvent object into the host's view tree engaged NSScrollView's responsive-scroll event tracking against event.window -- the PANEL -- and that cross-window tracking wedged the panel's event delivery and display: the overlay stopped rendering AND stopped hit-testing while its window sat there, which is why a plain mouse wheel never triggered it but a trackpad always did. The panel now drives the enclosing scroller's clip directly by the deltas; the event never crosses. Both mutation-verified. Panel forensics kept behind ANNOTKIT_PANEL_FORENSICS. Co-Authored-By: Claude Opus 5 (1M context) --- .../AnnotKit/macOS/OverlayController.swift | 119 +++++++- Sources/AnnotKitOverlayProbe/main.swift | 254 ++++++++++++++---- .../PanelFrameEnforcementTests.swift | 106 ++++++++ Tests/AnnotKitTests/ScrollForwardTests.swift | 101 +++++++ 4 files changed, 515 insertions(+), 65 deletions(-) create mode 100644 Tests/AnnotKitTests/PanelFrameEnforcementTests.swift create mode 100644 Tests/AnnotKitTests/ScrollForwardTests.swift diff --git a/Sources/AnnotKit/macOS/OverlayController.swift b/Sources/AnnotKit/macOS/OverlayController.swift index 63f7848..4731619 100644 --- a/Sources/AnnotKit/macOS/OverlayController.swift +++ b/Sources/AnnotKit/macOS/OverlayController.swift @@ -45,6 +45,9 @@ public final class OverlayController: NSObject { /// or two later without posting `didMove`/`didResize`; comparing against this /// lets the post-attach settle poll tell when the frame has stabilized. private var lastSyncedHostFrame: NSRect = .null + /// The clamped frame the panel is SUPPOSED to occupy, re-asserted after AppKit's + /// parent-follow repositioning. `.null` until the first sync. + private var desiredPanelFrame: NSRect = .null /// Bumped on every attach/unmount so an in-flight settle poll for a previous /// host stops instead of re-syncing against a stale (or detached) window. private var settleGeneration = 0 @@ -361,7 +364,22 @@ public final class OverlayController: NSObject { private func syncFrameAndOrigin() { guard let panel, let host else { return } let panelFrame = frame(for: session.mode, on: host) + desiredPanelFrame = panelFrame panel.setFrame(panelFrame, display: true) + // AppKit repositions a CHILD window to follow its parent, and it does so AFTER + // the `didMove` notification we are reacting to — so the clamped frame we just + // applied is silently dragged back to the host's own corner a runloop turn + // later. On a host whose bottom hangs below the display that puts the pill + // off-screen: the toolbar "disappears", exactly as reported, with placement + // that computed the right answer and a panel that no longer sits there. + // + // Measured, not assumed — with forensics on: + // computed=(1272, 60, 240, 104) afterSet=(1272, 60, 240, 104) + // next-turn panel=(1272, -200, 240, 104) clobbered=true + // + // Re-assert once AppKit has finished. `enforcePanelFrame()` is a no-op when it + // left us alone, so the common case costs one runloop hop and nothing else. + DispatchQueue.main.async { [weak self] in self?.enforcePanelFrame() } // Primary display = the origin/menu-bar screen, NOT NSScreen.main (the // active screen), which was the single-display bug. let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 @@ -396,6 +414,21 @@ public final class OverlayController: NSObject { hostingView?.rootView = makeRootView() } + /// Put the panel back where placement said it belongs, if AppKit moved it. + /// + /// Child windows are repositioned by AppKit to preserve their offset from the + /// parent, which undoes the visible-frame clamp on every host move. Comparing + /// before setting keeps this free when nothing fought us, and keeps it from + /// looping: a `setFrame` to the frame the window already has posts no move. + private func enforcePanelFrame() { + guard let panel, desiredPanelFrame != .null, panel.frame != desiredPanelFrame else { return } + if KeyablePanel.forensics { + FileHandle.standardError.write(Data( + "[sync] re-asserting clamped frame: \(panel.frame) -> \(desiredPanelFrame)\n".utf8)) + } + panel.setFrame(desiredPanelFrame, display: true) + } + private func frame(for mode: AnnotationSession.Mode, on host: NSWindow) -> NSRect { OverlayPlacement.panelFrame( for: mode, @@ -475,6 +508,40 @@ enum OverlayPlacement { final class KeyablePanel: NSPanel { override var canBecomeKey: Bool { true } + /// Forensics for "the pill vanished": every path AppKit can take to hide a + /// window funnels through `orderWindow`/`setIsVisible`/`close`, so logging the + /// call stack at each one names the culprit instead of leaving a symptom. + /// Opt-in via `ANNOTKIT_PANEL_FORENSICS=1`; costs one env lookup otherwise. + static let forensics = ProcessInfo.processInfo.environment["ANNOTKIT_PANEL_FORENSICS"] == "1" + + private func forensic(_ what: String) { + guard Self.forensics else { return } + FileHandle.standardError.write(Data(""" + [panel-forensics] \(what) visible=\(isVisible) frame=\(frame) parent=\(parent.map { "\($0.title)" } ?? "nil") + \(Thread.callStackSymbols.prefix(14).joined(separator: "\n"))\n\n + """.utf8)) + } + + override func order(_ place: NSWindow.OrderingMode, relativeTo otherWin: Int) { + if place == .out { forensic("order(.out)") } + super.order(place, relativeTo: otherWin) + } + + override func orderOut(_ sender: Any?) { + forensic("orderOut(sender: \(sender.map { String(describing: type(of: $0)) } ?? "nil"))") + super.orderOut(sender) + } + + override func setIsVisible(_ flag: Bool) { + if !flag { forensic("setIsVisible(false)") } + super.setIsVisible(flag) + } + + override func close() { + forensic("close()") + super.close() + } + /// Hand an unconsumed scroll down to the host window. /// /// Measured, not assumed: while annotating this panel covers the host's whole frame @@ -495,19 +562,47 @@ final class KeyablePanel: NSPanel { // clamped to the visible region, so handing it over unconverted would scroll // whatever sits at the wrong point (the wrong scroller, in a window with two). let hostPoint = host.convertPoint(fromScreen: convertPoint(toScreen: event.locationInWindow)) - // Falls back to the content view when the point is over no host view at all - // (the title-bar strip, or a host smaller than the panel): a wheel that hit - // AnnotKit must never simply vanish. `NSView`'s default `scrollWheel` walks the - // event UP to the enclosing scroller from wherever it lands, so aiming at the - // deepest view under the pointer is enough — no scroll-view search here. + // Hit-test from the window's ROOT view (the content view's superview, AppKit's + // border/theme frame), not from the content view: `hitTest(_:)` takes a point + // in the receiver's SUPERVIEW space, and the border view is flipped — so + // handing window-base coordinates to `contentView.hitTest` mirrors the y and + // misses everything, silently. The root view's "superview space" is defined + // as window base, which is exactly what `convertPoint(fromScreen:)` yields. + let root = host.contentView?.superview ?? host.contentView + guard let target = root?.hitTest(hostPoint) ?? host.contentView else { return } + + // Scroll the host's scroller DIRECTLY by the event's deltas. The event object + // itself must NEVER cross into the host's view tree: an earlier version did + // `targetView.scrollWheel(with: event)`, and for a TRACKPAD stream — phase + // `began`/`changed`/`ended` plus momentum, which is every real-world scroll — + // NSScrollView's responsive scrolling responds to `began` by engaging an + // event-tracking loop against `event.window`. That window is THIS PANEL, not + // the scroll view's own, and the cross-window tracking never terminates: + // it wedged the panel's event delivery and display, so the overlay silently + // stopped rendering AND stopped hit-testing while its window sat there — + // observed as "the toolbar vanishes after I scroll, then hover on and off it", + // with a plain mouse wheel (no phases) never triggering it. Reproduced + // against a live host and pinned by the probe's scroll phase. // - // The event itself is passed along unchanged, so the target reads a - // `locationInWindow` that is still PANEL-local (an `NSEvent`'s location cannot - // be rewritten). Scroll handling uses the DELTAS, and the location's one real - // job — choosing the target — is done above, so this costs nothing short of a - // host view that positions something off the wheel's own coordinates. - - (host.contentView?.hitTest(hostPoint) ?? host.contentView)?.scrollWheel(with: event) + // Driving the clip view by deltas keeps everything inside the HOST's own + // machinery, no event identity involved. Momentum events still arrive here + // carrying deltas, so inertia is preserved; only the edge rubber-band is + // lost, because `constrainBoundsRect` clamps at the document bounds. + var view: NSView? = target + while let current = view, !(current is NSScrollView) { view = current.superview } + guard let scrollView = view as? NSScrollView else { return } + + let clip = scrollView.contentView + // Non-precise deltas (an external mouse wheel) are in LINES; convert to + // points the same way NSScrollView itself does. + let scale: CGFloat = event.hasPreciseScrollingDeltas ? 1 : scrollView.verticalLineScroll + var origin = clip.bounds.origin + origin.x -= event.scrollingDeltaX * scale + // A flipped clip (the AppKit default for scroll content) grows y downward, so + // natural-scroll deltas subtract; an unflipped one is the mirror. + origin.y += (clip.isFlipped ? -1 : 1) * event.scrollingDeltaY * scale + clip.scroll(to: clip.constrainBoundsRect(NSRect(origin: origin, size: clip.bounds.size)).origin) + scrollView.reflectScrolledClipView(clip) } } #endif diff --git a/Sources/AnnotKitOverlayProbe/main.swift b/Sources/AnnotKitOverlayProbe/main.swift index 6bdea51..e7c204f 100644 --- a/Sources/AnnotKitOverlayProbe/main.swift +++ b/Sources/AnnotKitOverlayProbe/main.swift @@ -285,35 +285,40 @@ final class UnconstrainedWindow: NSWindow { override func constrainFrameRect(_ frameRect: NSRect, to screen: NSScreen?) -> NSRect { frameRect } } -/// Counts the wheel events that reach the HOST's view tree, and from WHERE. +/// A real scroller whose clip offset is the 9c witness. /// -/// The scroll question cannot be answered by watching an `NSScrollView`'s offset: -/// measured, AppKit's scroll views ignore a synthesized `NSEvent` outright (a synthetic -/// wheel moves nothing even when delivered straight to the scroll view). So 9c measures -/// the ROUTING instead, which is the mechanism actually in question — does a wheel that -/// lands on AnnotKit's panel reach the host at all, and does it reach the right view. -final class ScrollSpyView: NSView { - let name: String - var scrollCount = 0 - init(name: String, frame: NSRect) { - self.name = name - super.init(frame: frame) - } - required init?(coder: NSCoder) { fatalError("unused") } - override func scrollWheel(with event: NSEvent) { scrollCount += 1 } +/// The panel now scrolls the host by driving the enclosing `NSScrollView`'s clip view +/// DIRECTLY by the event's deltas — never by handing the event object into the host's +/// view tree, because NSScrollView answers a phase `began` (every trackpad scroll) by +/// engaging event tracking against `event.window`, and cross-window that tracking +/// wedges the panel's event delivery and display. A welcome side effect for the probe: +/// the clip offset moves for a SYNTHESIZED event too (the old event-delivery path +/// ignored synthetic wheels), so 9c can assert actual scrolling, not mere routing. +/// Flipped like every real host document (SwiftUI documents are flipped); an +/// unflipped one starts at the content's end, where a downward scroll is a no-op. +final class FlippedProbeDocument: NSView { + override var isFlipped: Bool { true } +} + +@MainActor +func makeScrollPane(frame: NSRect) -> NSScrollView { + let scroll = NSScrollView(frame: frame) + let document = FlippedProbeDocument(frame: NSRect(x: 0, y: 0, width: frame.width, height: frame.height * 6)) + scroll.documentView = document + scroll.hasVerticalScroller = true + return scroll } @MainActor struct ClampedHost { let window: NSWindow - /// Lower and upper halves of the host content. Which one a forwarded wheel lands in - /// is the only witness available for the panel→host coordinate conversion: the - /// forwarded event object still carries its PANEL-local `locationInWindow` (an - /// `NSEvent`'s location cannot be rewritten), so the conversion shows up in the - /// choice of target view, not in what the target reads off the event. Harmless for - /// real scroll handling, which uses the deltas. - let lowerSpy: ScrollSpyView - let upperSpy: ScrollSpyView + /// Lower and upper halves of the host content, each an independent scroller. + /// Which one MOVES is the witness for the panel→host coordinate conversion: the + /// panel re-aims the wheel through screen space before picking a target, so aiming + /// at a point whose panel-local and host-local interpretations fall in different + /// halves shows whether the conversion was applied. + let lowerScroll: NSScrollView + let upperScroll: NSScrollView let button: NSButton } @@ -331,8 +336,8 @@ func makeClampedHost(title: String, frame: NSRect, unconstrained: Bool, visibleB let size = window.contentView?.bounds.size ?? frame.size let content = NSView(frame: NSRect(origin: .zero, size: size)) - let lower = ScrollSpyView(name: "lower", frame: NSRect(x: 0, y: 0, width: size.width, height: size.height / 2)) - let upper = ScrollSpyView(name: "upper", frame: NSRect(x: 0, y: size.height / 2, width: size.width, height: size.height / 2)) + let lower = makeScrollPane(frame: NSRect(x: 0, y: 0, width: size.width, height: size.height / 2)) + let upper = makeScrollPane(frame: NSRect(x: 0, y: size.height / 2, width: size.width, height: size.height / 2)) content.addSubview(lower) content.addSubview(upper) @@ -346,7 +351,7 @@ func makeClampedHost(title: String, frame: NSRect, unconstrained: Bool, visibleB content.addSubview(button) window.contentView = content - return ClampedHost(window: window, lowerSpy: lower, upperSpy: upper, button: button) + return ClampedHost(window: window, lowerScroll: lower, upperScroll: upper, button: button) } /// The pill's own rect inside a panel at `frame`, mirroring `OverlayView.toolbar`: the @@ -379,9 +384,15 @@ func overlayValue(_ controller: OverlayController, _ label: String, as type: /// writing the flipped panel-local point into the CG event reproduces exactly what a real /// wheel delivered to the panel carries. (Verified: the round trip is exact.) @MainActor -func makeScrollEvent(panelLocal: CGPoint) -> NSEvent? { +func makeScrollEvent(panelLocal: CGPoint, phase: Int64 = 0) -> NSEvent? { guard let cg = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 1, wheel1: -120, wheel2: 0, wheel3: 0) else { return nil } + if phase != 0 { + // A trackpad stream: phase-tagged, continuous. This is the shape that engaged + // NSScrollView's cross-window event tracking under the old forwarding design. + cg.setIntegerValueField(.scrollWheelEventScrollPhase, value: phase) + cg.setIntegerValueField(.scrollWheelEventIsContinuous, value: 1) + } let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 cg.location = CGPoint(x: panelLocal.x, y: primaryHeight - panelLocal.y) return NSEvent(cgEvent: cg) @@ -565,6 +576,15 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { } func applicationDidFinishLaunching(_ note: Notification) { + // Opt-in interactive reproduction of the vanishing-pill report: drives REAL + // window-server events (scroll, hover on the pill, hover off) against a tall + // SwiftUI ScrollView host, sampling the panel's visibility throughout. Kept + // out of the default run because it needs Accessibility trust to post events + // and takes over the pointer. + if ProcessInfo.processInfo.environment["ANNOTKIT_PROBE_HOVERSCROLL"] == "1" { + runHoverScrollRepro() + return + } print("=== AnnotKitOverlayProbe: EXPANDED-overlay AX diagnostic ===") h1 = makeSwiftUIHost(title: "AnnotKit Harness W1") // SwiftUI needs a beat to render before its AX tree materializes. @@ -573,6 +593,124 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { } } + // ---- Interactive reproduction: scroll, hover on, hover off ------------ + + var reproHost: NSWindow? + var reproController: OverlayController? + + func post(_ event: CGEvent?) { event?.post(tap: .cghidEventTap) } + + func moveMouse(to cocoaPoint: CGPoint) { + // CGEvent uses top-left global coords; Cocoa is bottom-left. + let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 + let cg = CGPoint(x: cocoaPoint.x, y: primaryHeight - cocoaPoint.y) + post(CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: cg, mouseButton: .left)) + } + + func scroll(lines: Int32) { + post(CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 1, wheel1: lines, wheel2: 0, wheel3: 0)) + } + + func sample(_ label: String) { + guard let host = reproHost else { return } + let panel = host.childWindows?.first + let mode = reproController?.session.mode + print("[\(label)] panel: exists=\(panel != nil) visible=\(panel?.isVisible ?? false) " + + "frame=\(panel.map { fmt($0.frame) } ?? "-") parentSet=\(panel?.parent != nil) " + + "hostChildren=\(host.childWindows?.count ?? 0) mode=\(mode.map { "\($0)" } ?? "-") " + + "hostFrame=\(fmt(host.frame))") + } + + func runHoverScrollRepro() { + print("=== HOVER/SCROLL REPRO: open menu -> scroll -> hover on pill -> hover off ===") + print("AXIsProcessTrusted=\(AXIsProcessTrusted()) (event posting needs trust; if false, events will not land)") + NSApp.setActivationPolicy(.regular) + NSApp.activate(ignoringOtherApps: true) + + // A tall scrollable SwiftUI host, like the report's "scrollable screens". + // TALL=1 makes it taller than the display so the CLAMPED placement path runs — + // AppKit constrains the top under the menu bar but lets the bottom hang. + let tall = ProcessInfo.processInfo.environment["TALL"] == "1" + let window = UnconstrainedWindow( + contentRect: tall ? NSRect(x: 300, y: -500, width: 560, height: 1800) + : NSRect(x: 300, y: 80, width: 560, height: 760), + styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false + ) + window.title = "AnnotKit Scroll Host" + window.contentView = NSHostingView(rootView: ScrollView { + LazyVStack(alignment: .leading, spacing: 12) { + ForEach(0 ..< 120, id: \.self) { i in + Text("Row \(i) — scrollable content that makes a large AX tree") + .padding(8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.gray.opacity(0.12)) + .accessibilityIdentifier("Repro.Row\(i)") + } + }.padding(16) + }) + window.makeKeyAndOrderFront(nil) + reproHost = window + + let controller = OverlayController(session: AnnotationSession( + source: MacElementSource(), sink: NotesFileSink(path: "/dev/null") + )) + reproController = controller + + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { [self] in + controller.mount(on: window) + controller.start() // "I open the menu" + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { [self] in + sample("after start") + guard let panel = window.childWindows?.first else { + print("NO PANEL — cannot continue"); exit(2) + } + // The pill sits 20pt in from the panel's bottom-right; aim at its center. + let pill = CGPoint(x: panel.frame.maxX - 20 - 90, y: panel.frame.minY + 20 + 22) + let content = CGPoint(x: window.frame.midX, y: window.frame.midY) + // EXIT decides where "off the menu" goes: back onto the catcher + // (inside), or OUT of the window — the pill hugs the corner, so a + // real pointer plausibly leaves the window entirely. + let exitMode = ProcessInfo.processInfo.environment["EXIT"] ?? "inside" + let offPill: CGPoint + switch exitMode { + case "right": offPill = CGPoint(x: window.frame.maxX + 60, y: pill.y) + case "below": offPill = CGPoint(x: pill.x, y: max(2, panel.frame.minY - 40)) + default: offPill = CGPoint(x: pill.x, y: pill.y + 120) + } + print("exit mode = \(exitMode), offPill = \(offPill), tall = \(ProcessInfo.processInfo.environment["TALL"] ?? "0")") + + var step = 0 + let script: [(String, () -> Void)] = [ + ("move to content", { self.moveMouse(to: content) }), + ("scroll x5", { for _ in 0 ..< 5 { self.scroll(lines: -3) } }), + ("hover ON pill", { self.moveMouse(to: pill) }), + ("hover OFF pill", { self.moveMouse(to: offPill) }), + ("hover ON pill 2", { self.moveMouse(to: pill) }), + ("hover OFF pill 2", { self.moveMouse(to: offPill) }), + ] + @MainActor func advance() { + guard step < script.count else { + sample("FINAL") + let panelNow = window.childWindows?.first + let gone = panelNow == nil || !(panelNow?.isVisible ?? false) + print(gone ? "\n*** REPRODUCED: the panel is gone ***" : "\n*** NOT reproduced: panel still visible ***") + exit(gone ? 3 : 0) + } + let (name, action) = script[step] + step += 1 + action() + DispatchQueue.main.asyncAfter(deadline: .now() + 0.8) { + Task { @MainActor in + self.sample(name) + advance() + } + } + } + advance() + } + } + } + // ---- Phase 1: baseline (idle) ----------------------------------------- func phase1Baseline() { print("\n--- Phase 1a: BASELINE (no overlay installed, idle) ---") @@ -1881,8 +2019,19 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { "the idle panel keeps its full 240x104 size (a clipped panel would clip the pill)") // Hit-testability, the property the user actually lost: AppKit's own // "which window would a click here land on" answer must be OUR panel. + // + // Ask only about THIS PROCESS's windows. `windowNumber(at:)` answers globally, + // so any unrelated app that happens to cover the point — a full-screen + // terminal running the probe, most obviously — makes the assertion fail for a + // reason that has nothing to do with AnnotKit. Walking our own window list + // front-to-back keeps the check about the panel-vs-host layering it is + // actually testing, and keeps the probe from flaking on whatever is frontmost. let pillCenter = center(of: pillRect(inPanel: panel.frame)) - let hitNumber = NSWindow.windowNumber(at: pillCenter, belowWindowWithWindowNumber: 0) + let ownNumbers = Set(NSApp.windows.map(\.windowNumber)) + var hitNumber = NSWindow.windowNumber(at: pillCenter, belowWindowWithWindowNumber: 0) + while hitNumber != 0, !ownNumbers.contains(hitNumber) { + hitNumber = NSWindow.windowNumber(at: pillCenter, belowWindowWithWindowNumber: hitNumber) + } print(" windowNumber(at: pill center \(String(format: "(%.0f, %.0f)", pillCenter.x, pillCenter.y)))=\(hitNumber) " + "panel=\(panel.windowNumber) host=\(host.window.windowNumber)") check9(hitNumber == panel.windowNumber, "a click at the pill's center lands on the overlay panel (it is hit-testable)") @@ -1985,32 +2134,31 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { check9(approxEqualPt(event.locationInWindow, panelLocal), "sanity: the synthesized event carries the panel-local location a real wheel would") - // Baseline: the spies are live and reachable when an event is handed to them - // directly, so a zero count later means "swallowed", not "broken fixture". - host.upperSpy.scrollWheel(with: event) - check9(host.upperSpy.scrollCount == 1, "sanity: the host's spy DOES record a wheel delivered straight to it") - let base = (lower: host.lowerSpy.scrollCount, upper: host.upperSpy.scrollCount) - - // Deliver the way AppKit does once it has picked the window: hit-test the - // panel's content and hand the event to the deepest view. - let target = panel.contentView?.hitTest(panelLocal) - var chain: [String] = [] - var responder: NSResponder? = target - while let current = responder, chain.count < 8 { - chain.append("\(type(of: current))") - responder = current.nextResponder + // The panel drives the enclosing scroller's CLIP directly — the event object + // never enters the host's view tree (a phase `began` would otherwise engage + // NSScrollView's event tracking against the PANEL window and wedge it; that + // was the vanishing-toolbar bug). Offset movement is therefore observable + // even for a synthesized event, which the old event-delivery path ignored. + let upperBefore = host.upperScroll.contentView.bounds.origin.y + let lowerBefore = host.lowerScroll.contentView.bounds.origin.y + panel.scrollWheel(with: event) + let upperMoved = abs(host.upperScroll.contentView.bounds.origin.y - upperBefore) + let lowerMoved = abs(host.lowerScroll.contentView.bounds.origin.y - lowerBefore) + print(" after the wheel: upper clip moved \(String(format: "%.1f", upperMoved))pt, lower \(String(format: "%.1f", lowerMoved))pt") + check9(upperMoved > 0, "a wheel over the annotate catcher SCROLLS the host (it is not swallowed by the panel)") + check9(lowerMoved == 0, "and it scrolls the pane actually under the pointer (the panel→host conversion is applied)") + + // The trackpad case that used to KILL the overlay: a phase-tagged event. It + // must scroll like any other — and, mechanically, must never reach the host's + // own scrollWheel, which is what the direct-clip design guarantees. + if let phased = makeScrollEvent(panelLocal: panelLocal, phase: 1) { + let before = host.upperScroll.contentView.bounds.origin.y + panel.scrollWheel(with: phased) + check9(abs(host.upperScroll.contentView.bounds.origin.y - before) > 0, + "a PHASE-tagged (trackpad) wheel scrolls too — the stream that wedged the old event-forwarding design") + } else { + check9(false, "a phase-tagged wheel event could be built") } - print(" panel hit-test -> \(target.map { "\(type(of: $0))" } ?? "nil"); responder chain: \(chain)") - check9(target != nil, "sanity: the wheel lands on the overlay's own content view (it is over the catcher)") - check9(chain.contains { $0.contains("KeyablePanel") }, - "sanity: nothing in the overlay's view tree consumes the wheel — it reaches the panel WINDOW") - - target?.scrollWheel(with: event) - let lower = host.lowerSpy.scrollCount - base.lower - let upper = host.upperSpy.scrollCount - base.upper - print(" after the wheel: lower spy +\(lower), upper spy +\(upper)") - check9(lower + upper == 1, "a wheel over the annotate catcher reaches the HOST (it is not swallowed by the panel)") - check9(upper == 1, "it reaches the view actually under the pointer (the panel→host conversion is applied)") clampController?.unmount() clampHost?.window.orderOut(nil) diff --git a/Tests/AnnotKitTests/PanelFrameEnforcementTests.swift b/Tests/AnnotKitTests/PanelFrameEnforcementTests.swift new file mode 100644 index 0000000..69ae2ae --- /dev/null +++ b/Tests/AnnotKitTests/PanelFrameEnforcementTests.swift @@ -0,0 +1,106 @@ +#if os(macOS) +import AppKit +import XCTest +@testable import AnnotKit + +/// The vanishing-toolbar bug, at its root. +/// +/// AppKit repositions a CHILD window to preserve its offset from the parent — and it +/// does so AFTER the `didMove` notification the controller reacts to. So the +/// visible-frame clamp was computed correctly, applied correctly, and then silently +/// dragged back a runloop turn later. On a host whose bottom hangs below the display +/// (a tall scrollable window) that puts the pill off-screen: "the menu disappears". +/// +/// Measured against a live host with forensics on, before the fix: +/// +/// computed=(1272, 60, 240, 104) afterSet=(1272, 60, 240, 104) +/// next-turn panel=(1272, -200, 240, 104) clobbered=true +/// +/// The controller re-asserts the clamped frame on the next turn. These tests pin both +/// halves: that the clamp survives a parent move, and that the re-assert does not fire +/// when AppKit left the panel alone (which would mean it is fighting normal placement). +@MainActor +final class PanelFrameEnforcementTests: XCTestCase { + /// A host whose bottom hangs below the visible frame, so placement must clamp. + private func makeHangingHost() -> NSWindow { + let visible = (NSScreen.screens.first?.visibleFrame) ?? NSRect(x: 0, y: 0, width: 1512, height: 900) + let host = NSWindow(contentRect: NSRect(x: visible.minX + 100, y: visible.minY - 260, + width: 600, height: 800), + styleMask: [.titled, .resizable], backing: .buffered, defer: false) + host.orderFront(nil) + // AppKit constrains a TITLED window's top under the menu bar but never lifts + // its bottom, so this really does end up hanging off the display. + host.setFrame(NSRect(x: visible.minX + 100, y: visible.minY - 260, width: 600, height: 800), + display: true) + return host + } + + private func makeController(on host: NSWindow) -> OverlayController { + _ = NSApplication.shared + let controller = OverlayController( + session: AnnotationSession(source: MacElementSource(), sink: NotesFileSink(path: "/dev/null")) + ) + controller.mount(on: host) + return controller + } + + /// Moving the host must not drag the pill off the visible screen. + func testTheClampSurvivesAParentMove() async { + let host = makeHangingHost() + defer { host.orderOut(nil) } + let controller = makeController(on: host) + defer { controller.unmount() } + + guard let panel = host.childWindows?.first, + let visible = (host.screen ?? NSScreen.screens.first)?.visibleFrame else { + return XCTFail("no panel/screen to assert against") + } + XCTAssertGreaterThanOrEqual(panel.frame.minY, visible.minY - 0.5, + "precondition: the clamp put the panel inside the visible frame") + + // Reproduce the OBSERVED ORDERING, which is the whole bug: the controller + // handles `didMove` and applies the clamped frame, and only THEN does AppKit + // drag the child to follow its parent. Posting the notification and moving the + // panel afterwards is exactly that sequence — and it is why a test that merely + // moves the host proves nothing: there, the controller's own `setFrame` is the + // last writer and the assertion passes with or without the fix (verified by + // deleting the re-assert and watching it still pass). + host.setFrameOrigin(NSPoint(x: host.frame.minX, y: host.frame.minY - 120)) + NotificationCenter.default.post(name: NSWindow.didMoveNotification, object: host) + let clobbered = NSRect(x: panel.frame.minX, y: visible.minY - 200, + width: panel.frame.width, height: panel.frame.height) + panel.setFrame(clobbered, display: false) // AppKit's parent-follow, after the fact + XCTAssertLessThan(panel.frame.minY, visible.minY, "precondition: the panel really is off-screen now") + + await Task.yield() + try? await Task.sleep(nanoseconds: 250_000_000) + + XCTAssertGreaterThanOrEqual(panel.frame.minY, visible.minY - 0.5, + "AppKit dragged the panel below the visible frame and it was not put back — the pill is off-screen") + } + + /// The re-assert must be a no-op when nothing fought us: a controller that + /// re-set the frame unconditionally would post a move notification for every + /// move it handled, and chase its own tail. + func testTheReAssertDoesNotFireWhenAppKitLeavesThePanelAlone() async { + let visible = (NSScreen.screens.first?.visibleFrame) ?? NSRect(x: 0, y: 0, width: 1512, height: 900) + // A host comfortably INSIDE the visible frame: placement and AppKit agree, so + // there is nothing to correct. + let host = NSWindow(contentRect: NSRect(x: visible.minX + 120, y: visible.minY + 120, + width: 500, height: 400), + styleMask: [.titled], backing: .buffered, defer: false) + host.orderFront(nil) + defer { host.orderOut(nil) } + let controller = makeController(on: host) + defer { controller.unmount() } + + guard let panel = host.childWindows?.first else { return XCTFail("no panel") } + let settled = panel.frame + NotificationCenter.default.post(name: NSWindow.didMoveNotification, object: host) + await Task.yield() + try? await Task.sleep(nanoseconds: 250_000_000) + + XCTAssertEqual(panel.frame, settled, "the panel moved when nothing asked it to") + } +} +#endif diff --git a/Tests/AnnotKitTests/ScrollForwardTests.swift b/Tests/AnnotKitTests/ScrollForwardTests.swift new file mode 100644 index 0000000..ce8f899 --- /dev/null +++ b/Tests/AnnotKitTests/ScrollForwardTests.swift @@ -0,0 +1,101 @@ +#if os(macOS) +import AppKit +import XCTest +@testable import AnnotKit + +/// The panel's scroll hand-off, at the unit level. +/// +/// The overlay panel covers the host while annotating, so wheel events land on the +/// panel. It must scroll the host WITHOUT the event object ever entering the host's +/// view tree: NSScrollView answers a phase `began` (every trackpad scroll) by engaging +/// event tracking against `event.window` — the PANEL — and that cross-window tracking +/// wedges the panel's event delivery and display. That was the vanishing-toolbar bug. +/// The design under test drives the enclosing scroller's clip directly by the deltas. +/// Flipped like every real host document (SwiftUI and NSHostingView documents are +/// flipped): a non-flipped document starts with its origin at the content's END, so +/// a downward scroll is correctly a no-op there and the fixture would prove nothing. +private final class FlippedDocument: NSView { + override var isFlipped: Bool { true } +} + +@MainActor +final class ScrollForwardTests: XCTestCase { + private func makeHost() -> (NSWindow, NSScrollView) { + let host = NSWindow(contentRect: NSRect(x: 300, y: 200, width: 400, height: 600), + styleMask: [.titled], backing: .buffered, defer: false) + host.orderFront(nil) + let scroll = NSScrollView(frame: NSRect(x: 0, y: 0, width: 400, height: 600)) + scroll.documentView = FlippedDocument(frame: NSRect(x: 0, y: 0, width: 400, height: 4000)) + host.contentView?.addSubview(scroll) + return (host, scroll) + } + + private func makePanel(over host: NSWindow) -> KeyablePanel { + let panel = KeyablePanel(contentRect: host.frame, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, defer: false) + host.addChildWindow(panel, ordered: .above) + return panel + } + + private func wheel(at panelLocal: CGPoint, deltaY: Int32, phase: Int64 = 0) -> NSEvent { + let cg = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 1, + wheel1: deltaY, wheel2: 0, wheel3: 0)! + if phase != 0 { + cg.setIntegerValueField(.scrollWheelEventScrollPhase, value: phase) + cg.setIntegerValueField(.scrollWheelEventIsContinuous, value: 1) + } + let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 + cg.location = CGPoint(x: panelLocal.x, y: primaryHeight - panelLocal.y) + return NSEvent(cgEvent: cg)! + } + + func testWheelOverThePanelScrollsTheHostScroller() { + let (host, scroll) = makeHost() + defer { host.orderOut(nil) } + let panel = makePanel(over: host) + + let before = scroll.contentView.bounds.origin.y + panel.scrollWheel(with: wheel(at: CGPoint(x: 200, y: 300), deltaY: -120)) + XCTAssertGreaterThan(scroll.contentView.bounds.origin.y, before, + "a wheel that landed on the panel must move the host's clip") + } + + /// The trackpad stream that used to wedge the overlay: phase-tagged events must + /// scroll like any other, and the mechanism (direct clip drive) guarantees the + /// host's own `scrollWheel` never sees the cross-window event. + func testPhaseTaggedWheelScrollsToo() { + let (host, scroll) = makeHost() + defer { host.orderOut(nil) } + let panel = makePanel(over: host) + + let before = scroll.contentView.bounds.origin.y + panel.scrollWheel(with: wheel(at: CGPoint(x: 200, y: 300), deltaY: -120, phase: 1)) + XCTAssertGreaterThan(scroll.contentView.bounds.origin.y, before, + "a phase-tagged (trackpad) wheel must scroll — this stream wedged the old design") + } + + /// The event object must never be delivered into the host's view tree — that IS + /// the bug. A scroller subclass records whether its `scrollWheel` ran. + func testTheEventObjectNeverEntersTheHostViewTree() { + final class RecordingScrollView: NSScrollView { + nonisolated(unsafe) static var sawEvent = false + override func scrollWheel(with event: NSEvent) { Self.sawEvent = true } + } + let host = NSWindow(contentRect: NSRect(x: 300, y: 200, width: 400, height: 600), + styleMask: [.titled], backing: .buffered, defer: false) + host.orderFront(nil) + defer { host.orderOut(nil) } + let scroll = RecordingScrollView(frame: NSRect(x: 0, y: 0, width: 400, height: 600)) + scroll.documentView = FlippedDocument(frame: NSRect(x: 0, y: 0, width: 400, height: 4000)) + host.contentView?.addSubview(scroll) + let panel = makePanel(over: host) + + RecordingScrollView.sawEvent = false + panel.scrollWheel(with: wheel(at: CGPoint(x: 200, y: 300), deltaY: -120, phase: 1)) + XCTAssertFalse(RecordingScrollView.sawEvent, + "the cross-window event reached the host's scrollWheel — the wedge is back") + XCTAssertGreaterThan(scroll.contentView.bounds.origin.y, 0, "yet the clip still moved") + } +} +#endif From be624b45963db46724dc4efeb60a6e2cc60046b9 Mon Sep 17 00:00:00 2001 From: Angus Bezzina <37071175+angusbezzina@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:16:27 -0500 Subject: [PATCH 24/24] feat(overlay): the toolbar gets its own window, so it can never move or vanish The pill was drawn inside the catcher's ZStack, which made its position a function of the catcher panel's frame. That frame changes size on every open/close, is narrowed to the visible screen, and is dragged around by AppKit whenever the host moves -- three ways for a control that must never move to move -- and anything that stalled the catcher (a wedged scroll, a rebuilt SwiftUI root) took the pill down with it. The toolbar now has its own permanently mounted panel: fixed size, pinned to the host's bottom-right, re-placed only when the host moves, resizes or changes screen. Opening the menu creates a SEPARATE catcher panel over the host and closing it tears that panel down. The menu is now genuinely just open or closed, and the toolbar itself is untouched by the transition. Ordering matters and is not free: re-adding an existing child window does NOT re-stack it (measured -- childWindows still ended with the catcher, and the catcher sat on top, eating the click that closes the menu). The toolbar is detached and re-added, then ordered explicitly above the catcher. Probe updated for the two-panel world: childWindows.first is no longer "the overlay", and BOTH panels carry the AX identifier -- correctly, since the point query must skip both -- so 7g now picks the panel that actually spans the host rather than the first match. 191 tests, probe 11/11, iOS cross-compile clean. Co-Authored-By: Claude Opus 5 (1M context) --- Sources/AnnotKit/Overlay/OverlayView.swift | 31 ++- .../AnnotKit/macOS/OverlayController.swift | 257 +++++++++++++----- Sources/AnnotKitOverlayProbe/main.swift | 88 +++++- .../PanelFrameEnforcementTests.swift | 32 +++ 4 files changed, 322 insertions(+), 86 deletions(-) diff --git a/Sources/AnnotKit/Overlay/OverlayView.swift b/Sources/AnnotKit/Overlay/OverlayView.swift index e84430f..4bf662f 100644 --- a/Sources/AnnotKit/Overlay/OverlayView.swift +++ b/Sources/AnnotKit/Overlay/OverlayView.swift @@ -86,7 +86,6 @@ struct OverlayView: View { editCard(note: note, anchor: anchor) } - toolbar } .ignoresSafeArea() .accessibilityHidden(true) @@ -577,9 +576,29 @@ struct OverlayView: View { ) } - /// Pins the compact pill to the host window's bottom-right corner (margin 20, - /// matching the idle child-window frame the controller sizes). - private var toolbar: some View { +} + +/// The toolbar pill, in a window of its OWN. +/// +/// The pill used to be drawn inside the catcher's ZStack, which meant its position +/// was a function of the catcher panel's frame — and that frame changes size on +/// every open/close, gets narrowed to the visible screen, and is dragged around by +/// AppKit whenever the host window moves. Three ways for a control that must never +/// move to move, and a fourth failure on top: anything that stalled the catcher +/// (a wedged scroll, a rebuilt SwiftUI root) took the pill down with it. +/// +/// Giving it a separate, permanently mounted panel makes the guarantee structural +/// rather than incidental: the pill is always present, always at the host's +/// bottom-right corner, and the catcher's geometry and lifetime cannot touch it. +/// The menu is now genuinely just OPEN or CLOSED — the toolbar itself never moves +/// between the two. +struct ToolbarOverlayView: View { + @ObservedObject var session: AnnotationSession + let onToggle: () -> Void + let onCopy: () -> Void + let onExport: () -> Void + + var body: some View { VStack { Spacer() HStack { @@ -593,6 +612,8 @@ struct OverlayView: View { .padding(20) } } + .ignoresSafeArea() + .accessibilityHidden(true) } } @@ -741,7 +762,7 @@ private struct AnnotationCard: View { /// The pill itself is rendered unconditionally and is NEVER gated on an entrance /// flag, so it stays visible across idle<->annotate; only its contents swap as /// annotate mode toggles. -private struct ToolbarView: View { +struct ToolbarView: View { @ObservedObject var session: AnnotationSession let onToggle: () -> Void let onCopy: () -> Void diff --git a/Sources/AnnotKit/macOS/OverlayController.swift b/Sources/AnnotKit/macOS/OverlayController.swift index 4731619..006f89b 100644 --- a/Sources/AnnotKit/macOS/OverlayController.swift +++ b/Sources/AnnotKit/macOS/OverlayController.swift @@ -24,8 +24,20 @@ import SwiftUI @MainActor public final class OverlayController: NSObject { public let session: AnnotationSession - private var panel: KeyablePanel? - private var hostingView: NSHostingView? + /// The TOOLBAR panel: permanently mounted, fixed size, pinned to the host's + /// bottom-right. It holds only the pill, and its frame changes for exactly one + /// reason — the host moved, resized, or changed screen. Opening and closing the + /// menu does not touch it, which is what makes the pill "always visible, always + /// in the same spot" a structural property rather than a coincidence. + private var toolbarPanel: KeyablePanel? + private var toolbarHosting: NSHostingView? + /// The CATCHER panel: exists ONLY while the menu is open. Covers the host so the + /// SwiftUI catcher can receive hover and clicks, and carries the highlight, the + /// marquee band, the pins and the cards. Ordered BELOW the toolbar panel so the + /// pill stays clickable, and torn down on close so nothing of it can outlive the + /// open state. + private var catcherPanel: KeyablePanel? + private var catcherHosting: NSHostingView? private weak var host: NSWindow? /// AX top-left origin of the overlay PANEL — the surface `OverlayView` draws into, @@ -45,9 +57,11 @@ public final class OverlayController: NSObject { /// or two later without posting `didMove`/`didResize`; comparing against this /// lets the post-attach settle poll tell when the frame has stabilized. private var lastSyncedHostFrame: NSRect = .null - /// The clamped frame the panel is SUPPOSED to occupy, re-asserted after AppKit's - /// parent-follow repositioning. `.null` until the first sync. - private var desiredPanelFrame: NSRect = .null + /// The clamped frames each panel is SUPPOSED to occupy, re-asserted after + /// AppKit's parent-follow repositioning. `.null` when there is nothing to hold + /// (no sync yet, or the catcher is closed). + private var desiredToolbarFrame: NSRect = .null + private var desiredCatcherFrame: NSRect = .null /// Bumped on every attach/unmount so an in-flight settle poll for a previous /// host stops instead of re-syncing against a stale (or detached) window. private var settleGeneration = 0 @@ -65,7 +79,7 @@ public final class OverlayController: NSObject { } public func mount() { - guard panel == nil else { return } + guard toolbarPanel == nil else { return } guard let host = hostWindow() else { // Embedded tools always get a window eventually; retry when one // becomes main rather than dropping the install on the floor. @@ -86,7 +100,7 @@ public final class OverlayController: NSObject { /// (`NSApp.mainWindow ?? keyWindow ?? first visible non-panel`) can otherwise /// resolve to a floating panel when several windows are visible. public func mount(on host: NSWindow) { - guard panel == nil else { return } + guard toolbarPanel == nil else { return } attach(to: host) } @@ -100,12 +114,13 @@ public final class OverlayController: NSObject { // Invalidate any in-flight settle poll so it cannot re-sync a detached host. settleGeneration += 1 lastSyncedHostFrame = .null - if let panel { - host?.removeChildWindow(panel) - panel.orderOut(nil) + dismissCatcher() + if let toolbarPanel { + host?.removeChildWindow(toolbarPanel) + toolbarPanel.orderOut(nil) } - panel = nil - hostingView = nil + toolbarPanel = nil + toolbarHosting = nil host = nil } @@ -122,6 +137,7 @@ public final class OverlayController: NSObject { NSApp.activate(ignoringOtherApps: true) host?.orderFront(nil) session.start() + if let host { presentCatcher(on: host) } syncFrameAndOrigin() installEscapeMonitor() } @@ -132,6 +148,7 @@ public final class OverlayController: NSObject { // the session is half-torn-down. removeEscapeMonitor() session.stop() + dismissCatcher() syncFrameAndOrigin() } @@ -230,7 +247,7 @@ public final class OverlayController: NSObject { self.host = host let panel = KeyablePanel( - contentRect: frame(for: .idle, on: host), + contentRect: OverlayPlacement.toolbarFrame(hostFrame: host.frame, visibleFrame: visibleFrame(for: host)), styleMask: [.borderless, .nonactivatingPanel], backing: .buffered, defer: false @@ -246,7 +263,7 @@ public final class OverlayController: NSObject { // resolves to the overlay's hosting view instead of the control beneath. panel.setAccessibilityIdentifier(AXIntrospection.overlayWindowIdentifier) - let hostingView = NSHostingView(rootView: makeRootView()) + let hostingView = NSHostingView(rootView: makeToolbarView()) // Keep the overlay out of the app's own AX tree so the point query sees // through it (the SwiftUI root is also `accessibilityHidden`). hostingView.setAccessibilityElement(false) @@ -257,8 +274,10 @@ public final class OverlayController: NSObject { // orderFrontRegardless. host.addChildWindow(panel, ordered: .above) - self.panel = panel - self.hostingView = hostingView + self.toolbarPanel = panel + self.toolbarHosting = hostingView + // Re-open the catcher if we are attaching INTO an already-open menu. + if session.mode == .annotating { presentCatcher(on: host) } // We have a host now, so drop the retry observer and start tracking // geometry. @@ -279,6 +298,75 @@ public final class OverlayController: NSObject { if session.mode == .annotating { installEscapeMonitor() } } + /// Build a transparent, non-activating child panel. Shared so the toolbar and the + /// catcher cannot drift in the properties that make an overlay behave — + /// transparency, shadowlessness, and the AX identifier the point query skips. + private func makePanel(frame: NSRect) -> KeyablePanel { + let panel = KeyablePanel( + contentRect: frame, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = false + panel.ignoresMouseEvents = false + // Tag the panel WINDOW so `AXIntrospection` skips it in the point query and + // the snapshot: a full-frame panel is a live `AXWindow` the query would hit + // first, resolving every click to the overlay instead of the app beneath. + panel.setAccessibilityIdentifier(AXIntrospection.overlayWindowIdentifier) + return panel + } + + /// Open the catcher over the host. Idempotent. + private func presentCatcher(on host: NSWindow) { + guard catcherPanel == nil else { return } + let panel = makePanel(frame: OverlayPlacement.catcherFrame(hostFrame: host.frame, + visibleFrame: visibleFrame(for: host))) + let hosting = NSHostingView(rootView: makeRootView()) + hosting.setAccessibilityElement(false) + panel.contentView = hosting + host.addChildWindow(panel, ordered: .above) + catcherPanel = panel + catcherHosting = hosting + // The toolbar must stay ABOVE the catcher, or the full-frame catcher swallows + // every click meant for the pill — including the one that closes the menu. + // + // Re-adding an existing child does NOT re-stack it (measured: `childWindows` + // still ended with the catcher, and the catcher sat on top). Detaching first + // is what actually moves the toolbar to the end of the child order, and + // ordering it explicitly above the catcher pins the window-server z-order the + // clicks actually follow. + if let toolbarPanel { + host.removeChildWindow(toolbarPanel) + host.addChildWindow(toolbarPanel, ordered: .above) + toolbarPanel.order(.above, relativeTo: panel.windowNumber) + } + } + + /// Close the catcher. Idempotent; leaves the toolbar untouched. + private func dismissCatcher() { + guard let panel = catcherPanel else { return } + host?.removeChildWindow(panel) + panel.orderOut(nil) + catcherPanel = nil + catcherHosting = nil + } + + private func visibleFrame(for host: NSWindow) -> NSRect? { + (host.screen ?? NSScreen.screens.first)?.visibleFrame + } + + private func makeToolbarView() -> ToolbarOverlayView { + ToolbarOverlayView( + session: session, + onToggle: { [weak self] in self?.toggle() }, + onCopy: { [weak self] in self?.copy() }, + onExport: { [weak self] in self?.export() } + ) + } + private func makeRootView() -> OverlayView { OverlayView( session: session, @@ -290,7 +378,7 @@ public final class OverlayController: NSObject { // Make the non-activating child panel key so the composer/pin-editor // text fields accept keystrokes (a plain borderless panel that is not // key silently drops typing). - onFocusRequest: { [weak self] in self?.panel?.makeKey() } + onFocusRequest: { [weak self] in self?.catcherPanel?.makeKey() } ) } @@ -355,78 +443,79 @@ public final class OverlayController: NSObject { } @objc private func hostWindowAppeared(_ note: Notification) { - guard panel == nil, hostWindow() != nil else { return } + guard toolbarPanel == nil, hostWindow() != nil else { return } mount() } - /// Resize the child to the current mode's frame and recompute the AX origin - /// and surface size, then push both into the SwiftUI view. + /// Re-place BOTH panels for the current host geometry and push the catcher's + /// coordinates into its SwiftUI root. + /// + /// The toolbar's frame does not depend on the mode, so opening or closing the + /// menu leaves the pill exactly where it was — that is the whole point of giving + /// it its own window. private func syncFrameAndOrigin() { - guard let panel, let host else { return } - let panelFrame = frame(for: session.mode, on: host) - desiredPanelFrame = panelFrame - panel.setFrame(panelFrame, display: true) - // AppKit repositions a CHILD window to follow its parent, and it does so AFTER - // the `didMove` notification we are reacting to — so the clamped frame we just - // applied is silently dragged back to the host's own corner a runloop turn - // later. On a host whose bottom hangs below the display that puts the pill - // off-screen: the toolbar "disappears", exactly as reported, with placement - // that computed the right answer and a panel that no longer sits there. - // - // Measured, not assumed — with forensics on: - // computed=(1272, 60, 240, 104) afterSet=(1272, 60, 240, 104) - // next-turn panel=(1272, -200, 240, 104) clobbered=true - // - // Re-assert once AppKit has finished. `enforcePanelFrame()` is a no-op when it - // left us alone, so the common case costs one runloop hop and nothing else. - DispatchQueue.main.async { [weak self] in self?.enforcePanelFrame() } - // Primary display = the origin/menu-bar screen, NOT NSScreen.main (the - // active screen), which was the single-display bug. - let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 - // Derived from the PANEL's frame, never the host's. `OverlayView`'s contract is - // that these two describe the SURFACE it draws into — the catcher ADDS - // `axOrigin` to turn a panel-local click into an AX screen point, the highlight - // and composer SUBTRACT it — so once placement is clamped to the visible - // region, a host-derived origin would offset every click, highlight and card by - // exactly the clipped amount. - // - // Clamping the BOTTOM (the reported bug) leaves `axOrigin` alone, since it - // hangs off the frame's TOP edge, and only shrinks `surfaceSize` — which is - // itself the right answer, because the composer clamp should keep cards inside - // the VISIBLE region rather than inside a window that runs off the display. - // Clamping the TOP (a host tucked under the menu bar) genuinely does move the - // origin, and that is the case a host-derived origin breaks silently. - let newAXOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: panelFrame, primaryHeight: primaryHeight) + guard let host else { return } + let visible = visibleFrame(for: host) + + let toolbarFrame = OverlayPlacement.toolbarFrame(hostFrame: host.frame, visibleFrame: visible) + desiredToolbarFrame = toolbarFrame + toolbarPanel?.setFrame(toolbarFrame, display: true) + + let catcherFrame = OverlayPlacement.catcherFrame(hostFrame: host.frame, visibleFrame: visible) + desiredCatcherFrame = catcherPanel == nil ? .null : catcherFrame + catcherPanel?.setFrame(catcherFrame, display: true) + // Always record the host frame, even on the early-out below: the settle poll // decides "has the host stopped growing" by comparing against this, so leaving // it stale would keep the poll re-syncing a window that has already settled. lastSyncedHostFrame = host.frame - // Push a new SwiftUI root view ONLY when something it renders from actually - // changed. `syncFrameAndOrigin()` runs on every move/resize/screen-parameter - // notification, and a host that emits a burst of them (a relayout storm, a live - // resize drag) would otherwise replace the root view on each one — tearing down - // and rebuilding the pill mid-hover, which reads as the toolbar flickering or - // vanishing. Cheap guard, and it makes a redundant notification free. - guard newAXOrigin != axOrigin || panelFrame.size != surfaceSize else { return } + + // AppKit repositions a CHILD window to follow its parent, and does so AFTER + // the `didMove` notification we are reacting to — so the clamped frames we + // just applied get dragged back a runloop turn later. Measured, with + // forensics on: computed=(1272,60,240,104) then next-turn=(1272,-200,...). + // Re-assert once AppKit has finished; it is a no-op when nothing fought us. + DispatchQueue.main.async { [weak self] in self?.enforcePanelFrames() } + + // Primary display = the origin/menu-bar screen, NOT NSScreen.main (the active + // screen), which was the single-display bug. + let primaryHeight = NSScreen.screens.first?.frame.height ?? 0 + // Derived from the CATCHER's frame, never the host's: `OverlayView`'s contract + // is that these describe the surface it draws into — the catcher ADDS + // `axOrigin` to turn a panel-local click into an AX screen point, the + // highlight and composer SUBTRACT it — so a host-derived origin would offset + // every click, highlight and card by exactly the clipped amount. + let newAXOrigin = ScreenSpace.windowAXOrigin(cocoaFrame: catcherFrame, primaryHeight: primaryHeight) + + // Push a new SwiftUI root ONLY when something it renders from changed. A host + // emitting a burst of move/resize notifications would otherwise rebuild the + // catcher on each one; the pill is no longer in that view, so this can no + // longer flicker the toolbar, but the churn is still wasted work. + guard newAXOrigin != axOrigin || catcherFrame.size != surfaceSize else { return } axOrigin = newAXOrigin - surfaceSize = panelFrame.size + surfaceSize = catcherFrame.size rootViewPushes += 1 - hostingView?.rootView = makeRootView() + catcherHosting?.rootView = makeRootView() } - /// Put the panel back where placement said it belongs, if AppKit moved it. + /// Put both panels back where placement said they belong, if AppKit moved them. /// - /// Child windows are repositioned by AppKit to preserve their offset from the - /// parent, which undoes the visible-frame clamp on every host move. Comparing - /// before setting keeps this free when nothing fought us, and keeps it from - /// looping: a `setFrame` to the frame the window already has posts no move. - private func enforcePanelFrame() { - guard let panel, desiredPanelFrame != .null, panel.frame != desiredPanelFrame else { return } + /// Child windows are repositioned to preserve their offset from the parent, which + /// undoes the visible-frame clamp on every host move. Comparing before setting + /// keeps this free when nothing fought us, and keeps it from looping: a `setFrame` + /// to the frame a window already has posts no move. + private func enforcePanelFrames() { + enforce(toolbarPanel, desiredToolbarFrame, "toolbar") + enforce(catcherPanel, desiredCatcherFrame, "catcher") + } + + private func enforce(_ panel: KeyablePanel?, _ desired: NSRect, _ label: String) { + guard let panel, desired != .null, panel.frame != desired else { return } if KeyablePanel.forensics { FileHandle.standardError.write(Data( - "[sync] re-asserting clamped frame: \(panel.frame) -> \(desiredPanelFrame)\n".utf8)) + "[sync] re-asserting \(label) frame: \(panel.frame) -> \(desired)\n".utf8)) } - panel.setFrame(desiredPanelFrame, display: true) + panel.setFrame(desired, display: true) } private func frame(for mode: AnnotationSession.Mode, on host: NSWindow) -> NSRect { @@ -475,6 +564,28 @@ enum OverlayPlacement { return region.isEmpty ? hostFrame : region } + /// The TOOLBAR panel's frame: a fixed-size corner, pinned to the visible region's + /// bottom-right. Identical in both states — the menu opening must never move the + /// pill — and dependent only on where the host is, never on what mode it is in. + static func toolbarFrame(hostFrame: CGRect, visibleFrame: CGRect?) -> CGRect { + let region = region(hostFrame: hostFrame, visibleFrame: visibleFrame) + // Anchored at FULL size rather than intersected down to the region: shrinking + // this panel would clip the pill it exists to carry. Pinning its BOTTOM edge + // inside the region is what keeps the pill reachable; only the panel's empty + // upper part may spill past a region shorter than itself. + return CGRect( + x: region.maxX - idleSize.width, + y: region.minY, + width: idleSize.width, + height: idleSize.height + ) + } + + /// The CATCHER panel's frame: the host, narrowed to what is on screen. + static func catcherFrame(hostFrame: CGRect, visibleFrame: CGRect?) -> CGRect { + region(hostFrame: hostFrame, visibleFrame: visibleFrame) + } + static func panelFrame(for mode: AnnotationSession.Mode, hostFrame: CGRect, visibleFrame: CGRect?) -> CGRect { let region = region(hostFrame: hostFrame, visibleFrame: visibleFrame) switch mode { diff --git a/Sources/AnnotKitOverlayProbe/main.swift b/Sources/AnnotKitOverlayProbe/main.swift index e7c204f..bb4e66b 100644 --- a/Sources/AnnotKitOverlayProbe/main.swift +++ b/Sources/AnnotKitOverlayProbe/main.swift @@ -466,6 +466,26 @@ func makeSwiftUIHost(title: String) -> NSWindow { /// Depth-first search for the AX frame (AX top-left screen coords) of the element /// with `id` inside a public snapshot. Used to derive each control's true center /// as the AX point to fire the queries at. +/// The overlay now mounts TWO child panels: a permanently present TOOLBAR panel that +/// carries the pill (fixed size, pinned to the host's bottom-right, unchanged by the +/// menu opening) and a CATCHER panel that exists only while the menu is open and +/// covers the host. `childWindows.first` is therefore no longer "the overlay" — these +/// pick the one each assertion actually means. +@MainActor +func overlayCatcher(of host: NSWindow) -> NSWindow? { + // The catcher is the host-sized one; the toolbar is the small fixed corner. + host.childWindows?.max { lhs, rhs in + (lhs.frame.width * lhs.frame.height) < (rhs.frame.width * rhs.frame.height) + } +} + +@MainActor +func overlayToolbar(of host: NSWindow) -> NSWindow? { + host.childWindows?.min { lhs, rhs in + (lhs.frame.width * lhs.frame.height) < (rhs.frame.width * rhs.frame.height) + } +} + @MainActor func frame(ofID id: String, in windows: [WindowSnapshot]) -> CGRect? { func walk(_ element: Element) -> CGRect? { @@ -942,7 +962,7 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { check1(session.mode == .idle, "controller.stop() -> idle mode") check1(session.pending.count == 2, "retained notes survive leaving annotate mode (pending still 2)") - guard let panel = h2.window.childWindows?.first else { + guard let panel = overlayCatcher(of: h2.window) else { check1(false, "child overlay panel STILL PRESENT after stop() (pill persists)") verifyPinModel(session: session) return @@ -1582,7 +1602,13 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { let app = AXUIElementCreateApplication(ProcessInfo.processInfo.processIdentifier) AXUIElementSetAttributeValue(app, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) let rawWindows = AX.windows(app) - let overlayPanel = rawWindows.first { AX.string($0, kAXIdentifierAttribute) == overlayWindowIdentifier } + // BOTH overlay panels carry the identifier — correctly, since the point + // query has to skip the toolbar as well as the catcher — so pick the one + // that actually spans the host: the catcher. Taking `.first` here grabbed + // the small fixed toolbar and made the span check below fail. + let overlayPanel = rawWindows + .filter { AX.string($0, kAXIdentifierAttribute) == overlayWindowIdentifier } + .max { AX.frame($0).width * AX.frame($0).height < AX.frame($1).width * AX.frame($1).height } check7(overlayPanel != nil, "the overlay panel IS a live AX window during the drag (a real shadowing risk)") if let overlayPanel { let panelFrame = AX.frame(overlayPanel) @@ -1999,7 +2025,7 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { // ---- 9a: the IDLE pill stays on the visible screen ---------------------- func phase9aIdle() { print("\n 9a — IDLE pill on a host whose bottom is below the visible area:") - guard let host = clampHost, let panel = host.window.childWindows?.first else { + guard let host = clampHost, let panel = overlayCatcher(of: host.window) else { check9(false, "the overlay mounted a child panel on the clamped host") return phase9dTucked() } @@ -2046,7 +2072,7 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { func phase9bAnnotate() { print("\n 9b — ANNOTATE pill + hit-test on the same clamped host:") guard let host = clampHost, let controller = clampController, - let panel = host.window.childWindows?.first else { + let panel = overlayCatcher(of: host.window) else { check9(false, "the overlay is still mounted in annotate mode") return phase9dTucked() } @@ -2107,7 +2133,7 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { // the fold can be annotated — on precisely the screens the report is about. func phase9cScroll() { print("\n 9c — scroll WHILE ANNOTATING: does a wheel over the catcher reach the host?") - guard let host = clampHost, let panel = host.window.childWindows?.first else { + guard let host = clampHost, let panel = overlayCatcher(of: host.window) else { check9(false, "the overlay panel is present for the scroll measurement") return phase9dTucked() } @@ -2197,7 +2223,7 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { func phase9dChecks() { guard let host = clampHost, let controller = clampController, - let panel = host.window.childWindows?.first else { + let panel = overlayCatcher(of: host.window) else { check9(false, "the overlay mounted on the menu-bar-tucked host") return finish() } @@ -2243,7 +2269,52 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { clampController?.unmount() clampHost?.window.orderOut(nil) - finish() + phase10Escape() + } + + // ---- Phase 10: Escape closes the menu ----------------------------------- + // The one claim no unit test can make. A local key monitor is invoked by + // NSApplication's event DISPATCH, so it needs a real app with a running + // runloop — `NSApp.postEvent` puts a real key-down into that queue, which is + // as close to a keypress as a process can get to itself. + var passEscape = true + func check10(_ c: Bool, _ m: String) { + print(" " + (c ? "ok " : "FAIL ") + m) + passEscape = passEscape && c + } + + func escapeEvent() -> NSEvent? { + NSEvent.keyEvent(with: .keyDown, location: .zero, modifierFlags: [], + timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: 0, + context: nil, characters: "\u{1B}", charactersIgnoringModifiers: "\u{1B}", + isARepeat: false, keyCode: 53) + } + + func phase10Escape() { + print("\n--- Phase 10: Escape closes the menu ---") + let host = makeHostWindow(title: "AnnotKit Escape W1") + let controller = OverlayController(session: AnnotationSession( + source: MacElementSource(), sink: NotesFileSink(path: "/dev/null") + )) + controller.mount(on: host.window) + controller.start() + check10(controller.session.mode == .annotating, "sanity: the menu is OPEN before the keypress") + + guard let event = escapeEvent() else { + check10(false, "a real Escape key-down could be built") + controller.unmount(); host.window.orderOut(nil); return finish() + } + NSApp.postEvent(event, atStart: true) + // Dispatch happens on the runloop, not inline: let it turn. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { [weak self] in + guard let self else { return } + let mode = controller.session.mode + print(" after posting Escape: mode=\(mode)") + self.check10(mode == .idle, "Escape closed the menu (annotate mode exited)") + controller.unmount() + host.window.orderOut(nil) + self.finish() + } } func finish() { @@ -2257,8 +2328,9 @@ final class OverlayProbeDelegate: NSObject, NSApplicationDelegate { print(" Phase 7 (marquee frame selection: drawn rect -> element): \(passMarquee ? "PASS" : "FAIL")") print(" Phase 8 (selection navigation: round trips, history, component, frame anchor): \(passNav ? "PASS" : "FAIL")") print(" Phase 9 (pill + hit-test + scroll on a host hanging off the visible screen): \(passClamp ? "PASS" : "FAIL")") + print(" Phase 10 (Escape closes the menu): \(passEscape ? "PASS" : "FAIL")") print("\n=== AnnotKitOverlayProbe complete ===") - exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec && passMarquee && passNav && passClamp ? 0 : 1) + exit(pass1 && passIssue2 && passPins && passResize && passChrome && passCard && passSpec && passMarquee && passNav && passClamp && passEscape ? 0 : 1) } func collectIDs(_ elements: [Element]) -> [String] { diff --git a/Tests/AnnotKitTests/PanelFrameEnforcementTests.swift b/Tests/AnnotKitTests/PanelFrameEnforcementTests.swift index 69ae2ae..15dd874 100644 --- a/Tests/AnnotKitTests/PanelFrameEnforcementTests.swift +++ b/Tests/AnnotKitTests/PanelFrameEnforcementTests.swift @@ -79,6 +79,38 @@ final class PanelFrameEnforcementTests: XCTestCase { "AppKit dragged the panel below the visible frame and it was not put back — the pill is off-screen") } + /// The pill must not move when the menu opens or closes. + /// + /// It lives in its own permanently mounted panel precisely so that opening the + /// menu — which creates a second, host-sized catcher panel — cannot shift it. The + /// old design drew the pill inside that resizing panel, so every open/close moved + /// the thing the user is aiming at. + func testTheToolbarDoesNotMoveWhenTheMenuOpensOrCloses() { + let visible = (NSScreen.screens.first?.visibleFrame) ?? NSRect(x: 0, y: 0, width: 1512, height: 900) + let host = NSWindow(contentRect: NSRect(x: visible.minX + 120, y: visible.minY + 120, + width: 700, height: 500), + styleMask: [.titled, .resizable], backing: .buffered, defer: false) + host.orderFront(nil) + defer { host.orderOut(nil) } + let controller = makeController(on: host) + defer { controller.unmount() } + + guard let toolbar = host.childWindows?.first else { return XCTFail("no toolbar panel") } + let closed = toolbar.frame + XCTAssertEqual(host.childWindows?.count, 1, "closed: only the toolbar is mounted") + + controller.start() + XCTAssertEqual(toolbar.frame, closed, "the pill moved when the menu opened") + XCTAssertEqual(host.childWindows?.count, 2, "open: the catcher joins the toolbar") + // The toolbar must be ABOVE the catcher, or the full-frame catcher eats the + // click that closes the menu. + XCTAssertEqual(host.childWindows?.last, toolbar, "the toolbar must stay on top of the catcher") + + controller.stop() + XCTAssertEqual(toolbar.frame, closed, "the pill moved when the menu closed") + XCTAssertEqual(host.childWindows?.count, 1, "closed again: the catcher is gone") + } + /// The re-assert must be a no-op when nothing fought us: a controller that /// re-set the frame unconditionally would post a move notification for every /// move it handled, and chase its own tail.