diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index e0d87ed6d..b0726e022 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -140,6 +140,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedTextCommitProgressWalksExpectedPrefixOnly \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index ae140fc47..5617f5516 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -379,9 +379,13 @@ extension RunnerTests { ) let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) - XCTAssertFalse( + // A precondition, not a product claim. The fixture's empty `inputView` is what keeps the + // keyboard down, but nothing in this bundle owns the simulator's own keyboard settings, so an + // ambient flip that raised one here would be an environment fact — and reporting it as a + // failed assertion is what made this read as a product regression on unrelated PRs (#1874). + try XCTSkipIf( isKeyboardVisible(app: app), - "the test must exercise a focused responder with the software keyboard hidden" + "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" ) let failureCountBefore = currentXCTestFailureCount() @@ -407,6 +411,56 @@ extension RunnerTests { XCTAssertEqual(String(describing: textField.value ?? ""), "hardware-keyboard") } + // `waitForTextEntryReadiness`'s hardware-keyboard fallback returns early only on confirmed + // focus (#1874), and `keyboardFocusConfirmed` reads that from the app-wide focus predicate this + // bundle otherwise refuses to trust. Two XCTest facts it rests on, neither a repository + // invariant: the predicate reports a responder that shows NO software keyboard at all, and it + // names the element well enough to tell the tapped field from another one. The fixture field is + // the exact shape the fallback exists for — a real responder with an empty `inputView` — so this + // is where both are observable. If either regressed, readiness would silently stop taking the + // fallback and spend the full readinessTimeout on every hardware-keyboard field, which no other + // assertion would notice. + func testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let otherElement = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: textField), + "an untapped field must not confirm focus, or the fallback would fire immediately" + ) + + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + try XCTSkipIf( + isKeyboardVisible(app: app), + "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" + ) + + let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) + var confirmed = keyboardFocusConfirmed(app: app, element: textField) + while !confirmed && Date() < deadline { + sleepFor(TextEntryTiming.pollInterval) + confirmed = keyboardFocusConfirmed(app: app, element: textField) + } + XCTAssertTrue(confirmed, "a tapped responder must confirm its own keyboard focus") + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: otherElement), + "focus held by another element must read as a refusal, never as this element's focus" + ) + } + func testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand() throws { app.launchArguments = [ "--agent-device-text-entry-regression", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift new file mode 100644 index 000000000..b88d2d5aa --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift @@ -0,0 +1,206 @@ +import XCTest + +// The synthesized text-entry commit wait, end to end: the deadline that bounds it, the two waits +// that run it, the observation/pacing they poll through, and the value-free cadence line that path +// is allowed to log. Split from RunnerTests+SynthesizedTextEntry.swift, which keeps the +// private-XCTest synthesis boundary and the route policies; the pure outcome functions these wrap +// stay there next to the rules they encode. Everything that touches the polled field value now +// lives in this one file, which is the surface apple-runner-log-redaction.test.ts guards. +extension RunnerTests { + /// One commit wait's running deadline. + /// + /// A synthesized burst can be *throttled* rather than dropped: on a loaded simulator the + /// characters keep landing, just slowly, and everything touching the input system slows with + /// them (#1874). A flat wall-clock budget cannot tell that apart from a wedged pipeline, so it + /// condemned both at the same instant — and the throttled case is a working command reported as + /// `TEXT_INPUT_COMMIT_NOT_OBSERVED`, which is what turned an environment episode into a red + /// lane on branches touching no iOS code. + /// + /// So time is granted against *progress* — the observed value's expected-prefix growing, the + /// same length-only evidence `logCommitCadence` already emits — with an absolute `ceiling`, so a + /// pipeline delivering one character per stall window cannot hold a command open forever. A + /// pipeline making no progress at all still expires at exactly the `stallBudget` the flat + /// deadline used, so nothing that fails today starts passing merely by waiting longer. + /// + /// Started by the poll loop from its own first `now()`, and held as a local `var` there. The + /// wait's setup reads the field's placeholder first, an AX round-trip that takes seconds on + /// exactly the loaded host this exists for; charging that to the deadline made the wait give up + /// sooner than the flat budget it replaced. + struct SynthesizedCommitDeadline { + private let startedAt: Date + private let stallBudget: TimeInterval + private let ceiling: TimeInterval + // Nothing has landed yet, and an observation of "still nothing" must not read as progress. + private var bestPrefixLength = 0 + private var lastProgressAt: Date + + init(startedAt: Date, stallBudget: TimeInterval, ceiling: TimeInterval) { + self.startedAt = startedAt + self.stallBudget = stallBudget + self.ceiling = ceiling + self.lastProgressAt = startedAt + } + + /// Records one observation's expected-prefix length. Only forward movement counts: a shorter + /// read — the app clearing the field mid-flight, or a value that could not be read at all, + /// which measures as 0 — is not evidence the burst is still landing, so it neither buys time + /// nor takes any back. + mutating func record(expectedPrefixLength: Int, at now: Date) { + guard expectedPrefixLength > bestPrefixLength else { return } + bestPrefixLength = expectedPrefixLength + lastProgressAt = now + } + + func isExpired(at now: Date) -> Bool { + now.timeIntervalSince(startedAt) >= ceiling + || now.timeIntervalSince(lastProgressAt) >= stallBudget + } + } + + /// Blocks until the synthesized bare-type text is observable in the target field, so `type` + /// cannot report ok while trailing characters are still uncommitted on a slow simulator. + /// + /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the + /// event stream, so re-synthesizing the difference risks committing it twice after the command + /// already reported success (#1676 rejected exactly that repair). Reporting `.notObserved` is + /// what the caller does instead: the partial value is the agent's to resolve, and a named + /// failure beats a success that misdescribes the field. Text carrying a submit key is skipped + /// outright: the app may clear or rewrite the field on submit, so `textBefore + typedText` is + /// not the value to wait for. + func awaitSynthesizedFirstResponderCommit( + app: XCUIApplication, + target: TextEntryTarget, + textBefore: String?, + typedText: String + ) -> SynthesizedTextCommitOutcome { + guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else { + return .unobservable + } + let expectedText = textBefore + typedText + let waitStartedAt = Date() + NSLog("[DEBUG-1874] wait start expectedLen=%ld route=append", expectedText.count) + let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) + let outcome = Self.awaitSynthesizedCommitOutcome( + expectedText: expectedText, + placeholder: ingredients.placeholder, + now: { Date() }, + observe: ingredients.observe, + waitForNextObservation: ingredients.waitForNextObservation + ) + NSLog( + "[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f route=append", + String(describing: outcome), + waitStartedAt.timeIntervalSinceNow * -1000 + ) + return outcome + } + + /// Blocks until the synthesized replacement text (`fill`) is observable in the target field, so + /// `fill` cannot report ok while the select-all-and-retype it posted is still uncommitted — or + /// silently wrong — on a slow or channel-penalized simulator (this route runs only when the + /// XCTest channel is already penalized, and never resolves an `XCUIElement`, so it previously had + /// no verification at all). + /// + /// Unlike the append route, the expected value is the final text itself — replacement mode + /// clears the field first, so there is no `textBefore` prefix to account for — and unlike the + /// append route, a settled mismatch is always reported rather than trusted: see + /// `awaitSynthesizedReplacementCommitOutcome`'s doc comment. + func awaitSynthesizedReplacementCommit( + app: XCUIApplication, + target: TextEntryTarget, + expectedText: String + ) -> SynthesizedTextCommitOutcome { + guard !expectedText.contains("\n"), !expectedText.contains("\r") else { + return .unobservable + } + let waitStartedAt = Date() + NSLog("[DEBUG-1874] wait start expectedLen=%ld route=replacement", expectedText.count) + let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( + expectedText: expectedText, + placeholder: ingredients.placeholder, + now: { Date() }, + observe: ingredients.observe, + waitForNextObservation: ingredients.waitForNextObservation + ) + NSLog( + "[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f route=replacement", + String(describing: outcome), + waitStartedAt.timeIntervalSinceNow * -1000 + ) + return outcome + } + + /// The emitted cadence line, as a pure function so its output is assertable. Only lengths and + /// a timestamp are representable here; there is no String parameter, so observed field + /// contents cannot reach runner.log through this boundary whatever they contain. + static func commitCadenceLogLine( + elapsedMs: Int, + observedLen: Int, + expectedPrefixLen: Int + ) -> String { + "[DEBUG-1874] poll t=\(elapsedMs)ms observedLen=\(observedLen) expectedPrefixLen=\(expectedPrefixLen)" + } + + /// The typed boundary for commit-wait cadence evidence. The poll path must log through this + /// function and never through a raw NSLog: every parameter is an Int, so the polled value's + /// contents are unrepresentable at the call site. + static func logCommitCadence( + elapsedMs: Int, + observedLen: Int, + expectedPrefixLen: Int + ) { + NSLog( + "%@", + commitCadenceLogLine( + elapsedMs: elapsedMs, + observedLen: observedLen, + expectedPrefixLen: expectedPrefixLen + ) + ) + } + + /// The placeholder/observe/pacing ingredients shared by the append route + /// (`awaitSynthesizedFirstResponderCommit`) and the replacement route + /// (`awaitSynthesizedReplacementCommit`). What must NOT be shared is which outcome function + /// consumes them: see `awaitSynthesizedReplacementCommitOutcome`'s doc comment for why append + /// mode's "trust a diverged value" rule is wrong for replacement mode. Each caller therefore + /// calls its own named outcome function directly, with real argument labels — deliberately not + /// a stored closure/function-value parameter here, which would erase those labels at the call + /// site and make the observe closure unrecognizable to the static content-redaction check in + /// `apple-runner-log-redaction.test.ts` (`extractObserveClosure` locates the labeled closure + /// literal by its text; a closure passed as a plain function value carries no such label). + private func synthesizedCommitPollingIngredients( + app: XCUIApplication, + target: TextEntryTarget, + expectedText: String + ) -> ( + placeholder: String?, + observe: () -> String?, + waitForNextObservation: () -> Void + ) { + let placeholder = resolveTextEntryElement(app: app, target: target)?.placeholderValue + let waitStartedAt = Date() + return ( + placeholder: placeholder, + observe: { + let observedText = self.editableTextValue( + for: self.resolveTextEntryElement(app: app, target: target), + treatingPlaceholderAsEmpty: true + ) + // Cadence evidence stays value-free: the polled value is user content typed through + // `type`/`fill` and must never reach runner.log. Lengths and the expected-prefix walk + // are enough to distinguish throttling (prefix grows slowly) from a wedge (it freezes). + Self.logCommitCadence( + elapsedMs: Int(waitStartedAt.timeIntervalSinceNow * -1000), + observedLen: observedText?.count ?? -1, + expectedPrefixLen: observedText.map { Self.commonPrefixLength($0, expectedText) } ?? -1 + ) + return observedText + }, + // XCUI resolution shares the automation channel with the in-flight synthesized event. + // Sparse reads let the target consume that event instead of continuously interrupting it. + waitForNextObservation: { self.sleepFor(TextEntryTiming.synthesizedCommitPollInterval) } + ) + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 0b75db4d3..357af4f8d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -213,35 +213,6 @@ extension RunnerTests { return length } - /// The emitted cadence line, as a pure function so its output is assertable. Only lengths and - /// a timestamp are representable here; there is no String parameter, so observed field - /// contents cannot reach runner.log through this boundary whatever they contain. - static func commitCadenceLogLine( - elapsedMs: Int, - observedLen: Int, - expectedPrefixLen: Int - ) -> String { - "[DEBUG-1874] poll t=\(elapsedMs)ms observedLen=\(observedLen) expectedPrefixLen=\(expectedPrefixLen)" - } - - /// The typed boundary for commit-wait cadence evidence. The poll path must log through this - /// function and never through a raw NSLog: every parameter is an Int, so the polled value's - /// contents are unrepresentable at the call site. - static func logCommitCadence( - elapsedMs: Int, - observedLen: Int, - expectedPrefixLen: Int - ) { - NSLog( - "%@", - commitCadenceLogLine( - elapsedMs: elapsedMs, - observedLen: observedLen, - expectedPrefixLen: expectedPrefixLen - ) - ) - } - /// How the commit wait ended. Distinct from `SynthesizedTextCommitProgress`, which classifies a /// single observation: this is the whole wait's verdict, and it exists so the deadline can be /// told apart from success. The wait used to return `Void`, which made an expired deadline @@ -260,13 +231,20 @@ extension RunnerTests { case notObserved } - /// The commit wait's decision, with observation, pacing and the clock injected so the deadline - /// branch is exercisable without a simulator (the macOS host lane runs this; the member wrapper - /// below binds the real XCUI reads). + /// The commit wait's decision, with observation, pacing and the clock injected so both deadline + /// branches are exercisable without a simulator (the macOS host lane runs this; the member + /// wrapper below binds the real XCUI reads). + /// + /// The deadline is a local `var`, started from this loop's own first `now()` and advanced from + /// the same observation the progress check reads, so "did the burst move" and "is time up" are + /// two statements in one loop. The budget defaults to the shipped one, so only a test that is + /// asking about time has to name it. static func awaitSynthesizedCommitOutcome( expectedText: String, placeholder: String?, - isExpired: () -> Bool, + stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout, + ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling, + now: () -> Date = { Date() }, observe: () -> String?, waitForNextObservation: () -> Void ) -> SynthesizedTextCommitOutcome { @@ -276,15 +254,24 @@ extension RunnerTests { if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) { return .notObserved } + var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling) // The deadline is checked AFTER an observation, never before one, so the last thing that // happens before condemning a commit is a read. Checking first would condemn a commit that // landed during the final poll sleep — the exact loaded-host timing this wait exists for. while true { - switch synthesizedTextCommitProgress(observedText: observe(), expectedText: expectedText) { + let observedText = observe() + switch synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) { case .committed, .diverged: return .settled case .pending: - if isExpired() { return .notObserved } + // One clock sample, so the instant an observation is recorded at is the instant it is + // judged against. + let sampledAt = now() + deadline.record( + expectedPrefixLength: Self.commonPrefixLength(observedText ?? "", expectedText), + at: sampledAt + ) + if deadline.isExpired(at: sampledAt) { return .notObserved } waitForNextObservation() } } @@ -323,143 +310,34 @@ extension RunnerTests { static func awaitSynthesizedReplacementCommitOutcome( expectedText: String, placeholder: String?, - isExpired: () -> Bool, + stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout, + ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling, + now: () -> Date = { Date() }, observe: () -> String?, waitForNextObservation: () -> Void ) -> SynthesizedTextCommitOutcome { if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) { return .notObserved } + var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling) while true { - if observe() == expectedText { + let observedText = observe() + if observedText == expectedText { return .settled } - if isExpired() { return .notObserved } + // Prefix growth cannot settle this wait — a value with a hole in the middle is still a + // failure, see the doc comment above — but it is the same evidence that the burst is still + // landing, so it buys the same time here as it does in append mode. + let sampledAt = now() + deadline.record( + expectedPrefixLength: Self.commonPrefixLength(observedText ?? "", expectedText), + at: sampledAt + ) + if deadline.isExpired(at: sampledAt) { return .notObserved } waitForNextObservation() } } - /// The placeholder/deadline/observe/pacing ingredients shared by the append route - /// (`awaitSynthesizedFirstResponderCommit`) and the replacement route - /// (`awaitSynthesizedReplacementCommit`). What must NOT be shared is which outcome function - /// consumes them: see `awaitSynthesizedReplacementCommitOutcome`'s doc comment for why append - /// mode's "trust a diverged value" rule is wrong for replacement mode. Each caller therefore - /// calls its own named outcome function directly, with real argument labels — deliberately not - /// a stored closure/function-value parameter here, which would erase those labels at the call - /// site and make the observe closure unrecognizable to the static content-redaction check in - /// `apple-runner-log-redaction.test.ts` (`extractObserveClosure` locates the labeled closure - /// literal by its text; a closure passed as a plain function value carries no such label). - private func synthesizedCommitPollingIngredients( - app: XCUIApplication, - target: TextEntryTarget, - expectedText: String - ) -> ( - placeholder: String?, - isExpired: () -> Bool, - observe: () -> String?, - waitForNextObservation: () -> Void - ) { - let placeholder = resolveTextEntryElement(app: app, target: target)?.placeholderValue - let deadline = Date().addingTimeInterval(TextEntryTiming.synthesizedCommitTimeout) - let waitStartedAt = Date() - return ( - placeholder: placeholder, - isExpired: { Date() >= deadline }, - observe: { - let observedText = self.editableTextValue( - for: self.resolveTextEntryElement(app: app, target: target), - treatingPlaceholderAsEmpty: true - ) - // Cadence evidence stays value-free: the polled value is user content typed through - // `type`/`fill` and must never reach runner.log. Lengths and the expected-prefix walk - // are enough to distinguish throttling (prefix grows slowly) from a wedge (it freezes). - Self.logCommitCadence( - elapsedMs: Int(waitStartedAt.timeIntervalSinceNow * -1000), - observedLen: observedText?.count ?? -1, - expectedPrefixLen: observedText.map { Self.commonPrefixLength($0, expectedText) } ?? -1 - ) - return observedText - }, - // XCUI resolution shares the automation channel with the in-flight synthesized event. - // Sparse reads let the target consume that event instead of continuously interrupting it. - waitForNextObservation: { self.sleepFor(TextEntryTiming.synthesizedCommitPollInterval) } - ) - } - - /// Blocks until the synthesized bare-type text is observable in the target field, so `type` - /// cannot report ok while trailing characters are still uncommitted on a slow simulator. - /// - /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the - /// event stream, so re-synthesizing the difference risks committing it twice after the command - /// already reported success (#1676 rejected exactly that repair). Reporting `.notObserved` is - /// what the caller does instead: the partial value is the agent's to resolve, and a named - /// failure beats a success that misdescribes the field. Text carrying a submit key is skipped - /// outright: the app may clear or rewrite the field on submit, so `textBefore + typedText` is - /// not the value to wait for. - func awaitSynthesizedFirstResponderCommit( - app: XCUIApplication, - target: TextEntryTarget, - textBefore: String?, - typedText: String - ) -> SynthesizedTextCommitOutcome { - guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else { - return .unobservable - } - let expectedText = textBefore + typedText - let waitStartedAt = Date() - NSLog("[DEBUG-1874] wait start expectedLen=%ld route=append", expectedText.count) - let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: expectedText, - placeholder: ingredients.placeholder, - isExpired: ingredients.isExpired, - observe: ingredients.observe, - waitForNextObservation: ingredients.waitForNextObservation - ) - NSLog( - "[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f route=append", - String(describing: outcome), - waitStartedAt.timeIntervalSinceNow * -1000 - ) - return outcome - } - - /// Blocks until the synthesized replacement text (`fill`) is observable in the target field, so - /// `fill` cannot report ok while the select-all-and-retype it posted is still uncommitted — or - /// silently wrong — on a slow or channel-penalized simulator (this route runs only when the - /// XCTest channel is already penalized, and never resolves an `XCUIElement`, so it previously had - /// no verification at all). - /// - /// Unlike the append route, the expected value is the final text itself — replacement mode - /// clears the field first, so there is no `textBefore` prefix to account for — and unlike the - /// append route, a settled mismatch is always reported rather than trusted: see - /// `awaitSynthesizedReplacementCommitOutcome`'s doc comment. - func awaitSynthesizedReplacementCommit( - app: XCUIApplication, - target: TextEntryTarget, - expectedText: String - ) -> SynthesizedTextCommitOutcome { - guard !expectedText.contains("\n"), !expectedText.contains("\r") else { - return .unobservable - } - let waitStartedAt = Date() - NSLog("[DEBUG-1874] wait start expectedLen=%ld route=replacement", expectedText.count) - let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) - let outcome = Self.awaitSynthesizedReplacementCommitOutcome( - expectedText: expectedText, - placeholder: ingredients.placeholder, - isExpired: ingredients.isExpired, - observe: ingredients.observe, - waitForNextObservation: ingredients.waitForNextObservation - ) - NSLog( - "[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f route=replacement", - String(describing: outcome), - waitStartedAt.timeIntervalSinceNow * -1000 - ) - return outcome - } - static func shouldUseResolvedCoordinateTextEntryRoute( repairMode: TextTypingRepairMode, hasX: Bool, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index d4a849343..661550362 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -45,7 +45,14 @@ extension RunnerTests { static let pollInterval: TimeInterval = 0.02 static let warmupValueTimeout: TimeInterval = 0.4 static let verificationStabilityWindow: TimeInterval = 0.2 - static let synthesizedCommitTimeout: TimeInterval = 3.0 + /// How long the commit wait tolerates seeing NO further progress toward the expected value. + /// Numerically the flat deadline this replaced, so a pipeline that delivers nothing is + /// condemned at exactly the same instant it always was (see `SynthesizedCommitDeadline`). + static let synthesizedCommitStallTimeout: TimeInterval = 3.0 + /// The commit wait's absolute bound, however long characters keep arriving. Sits well inside + /// the daemon's per-command budget (`RUNNER_COMMAND_TIMEOUT_MS`, 45s), which also has to cover + /// focus, clear and verification around this wait. + static let synthesizedCommitCeiling: TimeInterval = 10.0 static let synthesizedCommitPollInterval: TimeInterval = 0.2 } @@ -127,211 +134,6 @@ extension RunnerTests { element.typeText(deletes) } - func focusedTextInput(app: XCUIApplication) -> XCUIElement? { -#if os(iOS) - // iOS focus predicates can return stale or misleading text-input matches - // under XCUITest, so text entry readiness is driven by tap/keyboard state. - return nil -#else - return safely("FOCUSED_INPUT_QUERY") { - let candidates = app - .descendants(matching: .any) - .matching(NSPredicate(format: "hasKeyboardFocus == 1")) - .allElementsBoundByIndex - for candidate in candidates where candidate.exists { - switch candidate.elementType { - case .textField, .secureTextField, .searchField, .textView: - return candidate - default: - continue - } - } - return nil - } -#endif - } - - func rememberTextEntryTap(_ element: XCUIElement?) { - guard let element, isTextEntryElement(element) else { - clearRememberedTextEntryTap() - return - } - textEntryTapWitness = TextEntryTapWitness( - element: element, - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier - ) - } - - func clearRememberedTextEntryTap() { - textEntryTapWitness = nil - } - - private func rememberedTextEntryTarget() -> TextEntryTarget? { - guard let witness = textEntryTapWitness else { - return nil - } - // The tap is proof for one immediately-following bare type only. Consume it before checking - // the element so a failed or interrupted type cannot reuse stale focus evidence. - clearRememberedTextEntryTap() - guard witness.matches( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier - ) else { - return nil - } - let element = witness.element - // XCUIElement is query-backed rather than a stable node identity. A same-identifier field - // introduced by app-side navigation between tap and this immediate type can therefore - // re-resolve here; keep the witness one-shot and fail closed on every observable identity - // boundary instead of using frame equality, which would reject legitimate layout changes. - guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else { - return nil - } - // Keep the target scoped to the element that the preceding tap actually selected. Do not - // attach a refresh point: if that element disappeared, bare type must fail closed rather - // than rediscovering a different field or dispatching unscoped app.typeText. - return TextEntryTarget( - element: element, - refreshPoint: nil, - prefersFocusedElement: false, - fromTapWitness: true - ) - } - - func stabilizeTextInputBeforeTyping( - app: XCUIApplication, - target: XCUIElement?, - keyboardVisibleBeforeTap: Bool? = nil - ) -> TextEntryStabilization { -#if os(tvOS) - return TextEntryStabilization(element: target, focusConfirmed: true) -#else - let latest = target - let keyboardVisibleAtEntry = keyboardVisibleBeforeTap ?? isKeyboardVisible(app: app) - let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout) - while Date() < deadline { - if let focused = focusedTextInput(app: app) { - return TextEntryStabilization(element: focused, focusConfirmed: true) - } - // focusedTextInput is intentionally nil on iOS; treat the keyboard transitioning to - // visible after our tap as the focus-moved signal. Don't fast-path when it was already up. - if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) { - return TextEntryStabilization(element: latest, focusConfirmed: true) - } - sleepFor(TextEntryTiming.pollInterval) - } - return TextEntryStabilization(element: latest, focusConfirmed: false) -#endif - } - - func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { - guard let x, let y else { - let softwareKeyboardVisible = isKeyboardVisible(app: app) - if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() { - return rememberedTarget - } - // Bare `type` targets the current first responder. On iOS we intentionally do not trust - // `hasKeyboardFocus`, but an already-visible software keyboard is sufficient evidence that - // app.typeText has a receiver; waiting the full readiness timeout cannot prove a stronger - // target because there is no selector/coordinate focus move to validate. - if softwareKeyboardVisible { - return TextEntryTarget( - element: focusedTextInput(app: app), - refreshPoint: nil, - prefersFocusedElement: true - ) - } - let focused = waitForTextEntryReadiness( - app: app, - target: TextEntryTarget( - element: focusedTextInput(app: app), - refreshPoint: nil, - prefersFocusedElement: true - ) - ) - return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) - } - - let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let target = textInputAt(app: app, x: x, y: y) - let requestedPoint = CGPoint(x: x, y: y) - if let target { - let frame = target.frame - if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) - } else { - _ = tapAt(app: app, x: x, y: y) - } - } else { - _ = tapAt(app: app, x: x, y: y) - } - // A visible keyboard is not enough evidence for app.typeText, because focus may still - // belong to a previous field. With a concrete target we type through XCUIElement.typeText, - // so after tapping it the iOS readiness timeout cannot prove anything stronger. - if keyboardVisibleBeforeTap, let target { - return TextEntryTarget( - element: target, - refreshPoint: textEntryRefreshPoint(for: target) ?? requestedPoint, - prefersFocusedElement: false - ) - } - let stabilized = stabilizeTextInputBeforeTyping( - app: app, - target: target, - keyboardVisibleBeforeTap: keyboardVisibleBeforeTap - ) - let readyTarget = TextEntryTarget( - element: stabilized.element ?? target, - refreshPoint: requestedPoint, - prefersFocusedElement: false - ) - let concreteTargetReady = keyboardVisibleBeforeTap && readyTarget.element != nil - let element = stabilized.focusConfirmed || concreteTargetReady - ? (stabilized.element ?? target) - : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? target) - return TextEntryTarget( - element: element, - refreshPoint: textEntryRefreshPoint(for: element) ?? requestedPoint, - prefersFocusedElement: false - ) - } - - func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget { - let point = textEntryRefreshPoint(for: element) - let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - if let point { - _ = tapAt(app: app, x: point.x, y: point.y) - } - // See the coordinate-target path above: direct element typing keeps this scoped to the - // tapped target, while the first-character warmup and final verify still catch dropped input. - if keyboardVisibleBeforeTap { - return TextEntryTarget( - element: element, - refreshPoint: textEntryRefreshPoint(for: element) ?? point, - prefersFocusedElement: false - ) - } - let stabilized = stabilizeTextInputBeforeTyping( - app: app, - target: element, - keyboardVisibleBeforeTap: keyboardVisibleBeforeTap - ) - let readyTarget = TextEntryTarget( - element: stabilized.element ?? element, - refreshPoint: point, - prefersFocusedElement: false - ) - let resolved = stabilized.focusConfirmed - ? (stabilized.element ?? element) - : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? element) - return TextEntryTarget( - element: resolved, - refreshPoint: textEntryRefreshPoint(for: resolved) ?? point, - prefersFocusedElement: false - ) - } - func isTextEntryElement(_ element: XCUIElement) -> Bool { switch element.elementType { case .textField, .secureTextField, .searchField, .textView: @@ -375,103 +177,6 @@ extension RunnerTests { return nil } - private func waitForTextEntryReadiness( - app: XCUIApplication, - target: TextEntryTarget, - timeout: TimeInterval = TextEntryTiming.readinessTimeout - ) -> XCUIElement? { -#if os(iOS) - var latest = resolveTextEntryElement(app: app, target: target) - let keyboardVisibleAtEntry = isKeyboardVisible(app: app) - let deadline = Date().addingTimeInterval(timeout) - let hardwareKeyboardFallback = Date().addingTimeInterval( - min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout) - ) - var sawSoftwareKeyboard = false - while Date() < deadline { - if let focused = focusedTextInput(app: app) { - latest = focused - if isKeyboardVisible(app: app) { - return focused - } - } - // Fast-path on a keyboard hidden->visible transition: our tapped field gained focus, so - // return immediately instead of burning the full readinessTimeout (warmup-first-char echo - // + post-type verify/repair remain as drop safety nets). When the keyboard was ALREADY up - // (back-to-back fills), this isn't a focus signal — fall through to the settle/timeout so - // text isn't sent to the previously-focused field. - if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) { - return latest - } - sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app) - if !sawSoftwareKeyboard && Date() >= hardwareKeyboardFallback && latest != nil { - return latest - } - sleepFor(TextEntryTiming.pollInterval) - } - return focusedTextInput(app: app) ?? latest -#else - return resolveTextEntryElement(app: app, target: target) -#endif - } - - func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { -#if os(iOS) - switch element.elementType { - case .textField, .secureTextField, .searchField, .textView: - if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil { - return - } - let frame = element.frame - if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) - _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) - } - default: - return - } -#endif - } - - private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? { - let deadline = Date().addingTimeInterval(timeout) - while Date() < deadline { - if let focused = focusedTextInput(app: app) { - return focused - } - sleepFor(TextEntryTiming.pollInterval) - } - return focusedTextInput(app: app) - } - - private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? { - guard let element else { - return nil - } - let frame = element.frame - guard !frame.isEmpty else { - return nil - } - return CGPoint(x: frame.midX, y: frame.midY) - } - - /// A focus-moved signal for iOS text entry, where `focusedTextInput` is intentionally nil. - /// The software keyboard TRANSITIONING from hidden (at entry) to visible means the field we - /// just tapped gained first-responder. If the keyboard was ALREADY up (e.g. back-to-back - /// fills into different fields), its visibility is not evidence focus moved to the new field, - /// so callers must keep waiting rather than typing into the previously-focused field. - private func keyboardBecameVisible(app: XCUIApplication, wasVisibleAtEntry: Bool) -> Bool { - return !wasVisibleAtEntry && isKeyboardVisible(app: app) - } - - private func keyboardElementExists(app: XCUIApplication) -> Bool { -#if os(iOS) - return safely("KEYBOARD_EXISTS", false) { app.keyboards.firstMatch.exists } -#else - return false -#endif - } - private func moveCaretToEnd(element: XCUIElement) { #if os(tvOS) return diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift new file mode 100644 index 000000000..da780c827 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -0,0 +1,206 @@ +import XCTest + +// Text-entry target acquisition: choosing the element a `type`/`fill` will address and getting +// focus onto it — the one-shot tap witness, the post-tap stabilization, and the two +// `focusTextInputForTextEntry` entry points the command layer calls. Whether that target is ready +// is RunnerTests+TextEntryReadiness.swift's question, and this file asks it rather than answering +// it. +extension RunnerTests { + func rememberTextEntryTap(_ element: XCUIElement?) { + guard let element, isTextEntryElement(element) else { + clearRememberedTextEntryTap() + return + } + textEntryTapWitness = TextEntryTapWitness( + element: element, + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier + ) + } + + func clearRememberedTextEntryTap() { + textEntryTapWitness = nil + } + + private func rememberedTextEntryTarget() -> TextEntryTarget? { + guard let witness = textEntryTapWitness else { + return nil + } + // The tap is proof for one immediately-following bare type only. Consume it before checking + // the element so a failed or interrupted type cannot reuse stale focus evidence. + clearRememberedTextEntryTap() + guard witness.matches( + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier + ) else { + return nil + } + let element = witness.element + // XCUIElement is query-backed rather than a stable node identity. A same-identifier field + // introduced by app-side navigation between tap and this immediate type can therefore + // re-resolve here; keep the witness one-shot and fail closed on every observable identity + // boundary instead of using frame equality, which would reject legitimate layout changes. + guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else { + return nil + } + // Keep the target scoped to the element that the preceding tap actually selected. Do not + // attach a refresh point: if that element disappeared, bare type must fail closed rather + // than rediscovering a different field or dispatching unscoped app.typeText. + return TextEntryTarget( + element: element, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ) + } + + func stabilizeTextInputBeforeTyping( + app: XCUIApplication, + target: XCUIElement?, + keyboardVisibleBeforeTap: Bool? = nil + ) -> TextEntryStabilization { +#if os(tvOS) + return TextEntryStabilization(element: target, focusConfirmed: true) +#else + let latest = target + let keyboardVisibleAtEntry = keyboardVisibleBeforeTap ?? isKeyboardVisible(app: app) + let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout) + while Date() < deadline { + if let focused = focusedTextInput(app: app) { + return TextEntryStabilization(element: focused, focusConfirmed: true) + } + // focusedTextInput is intentionally nil on iOS; treat the keyboard transitioning to + // visible after our tap as the focus-moved signal. Don't fast-path when it was already up. + if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) { + return TextEntryStabilization(element: latest, focusConfirmed: true) + } + sleepFor(TextEntryTiming.pollInterval) + } + return TextEntryStabilization(element: latest, focusConfirmed: false) +#endif + } + + func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { + guard let x, let y else { + let softwareKeyboardVisible = isKeyboardVisible(app: app) + if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() { + return rememberedTarget + } + // Bare `type` targets the current first responder. On iOS we intentionally do not trust + // `hasKeyboardFocus`, but an already-visible software keyboard is sufficient evidence that + // app.typeText has a receiver; waiting the full readiness timeout cannot prove a stronger + // target because there is no selector/coordinate focus move to validate. + if softwareKeyboardVisible { + return TextEntryTarget( + element: focusedTextInput(app: app), + refreshPoint: nil, + prefersFocusedElement: true + ) + } + let focused = waitForTextEntryReadiness( + app: app, + target: TextEntryTarget( + element: focusedTextInput(app: app), + refreshPoint: nil, + prefersFocusedElement: true + ) + ) + return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) + } + + let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) + let target = textInputAt(app: app, x: x, y: y) + let requestedPoint = CGPoint(x: x, y: y) + if let target { + let frame = target.frame + if !frame.isEmpty { + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + } else { + _ = tapAt(app: app, x: x, y: y) + } + } else { + _ = tapAt(app: app, x: x, y: y) + } + // A visible keyboard is not enough evidence for app.typeText, because focus may still + // belong to a previous field. With a concrete target we type through XCUIElement.typeText, + // so after tapping it the iOS readiness timeout cannot prove anything stronger. + if keyboardVisibleBeforeTap, let target { + return TextEntryTarget( + element: target, + refreshPoint: textEntryRefreshPoint(for: target) ?? requestedPoint, + prefersFocusedElement: false + ) + } + let stabilized = stabilizeTextInputBeforeTyping( + app: app, + target: target, + keyboardVisibleBeforeTap: keyboardVisibleBeforeTap + ) + let readyTarget = TextEntryTarget( + element: stabilized.element ?? target, + refreshPoint: requestedPoint, + prefersFocusedElement: false + ) + let concreteTargetReady = keyboardVisibleBeforeTap && readyTarget.element != nil + let element = stabilized.focusConfirmed || concreteTargetReady + ? (stabilized.element ?? target) + : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? target) + return TextEntryTarget( + element: element, + refreshPoint: textEntryRefreshPoint(for: element) ?? requestedPoint, + prefersFocusedElement: false + ) + } + + func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget { + let point = textEntryRefreshPoint(for: element) + let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) + if let point { + _ = tapAt(app: app, x: point.x, y: point.y) + } + // See the coordinate-target path above: direct element typing keeps this scoped to the + // tapped target, while the first-character warmup and final verify still catch dropped input. + if keyboardVisibleBeforeTap { + return TextEntryTarget( + element: element, + refreshPoint: textEntryRefreshPoint(for: element) ?? point, + prefersFocusedElement: false + ) + } + let stabilized = stabilizeTextInputBeforeTyping( + app: app, + target: element, + keyboardVisibleBeforeTap: keyboardVisibleBeforeTap + ) + let readyTarget = TextEntryTarget( + element: stabilized.element ?? element, + refreshPoint: point, + prefersFocusedElement: false + ) + let resolved = stabilized.focusConfirmed + ? (stabilized.element ?? element) + : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? element) + return TextEntryTarget( + element: resolved, + refreshPoint: textEntryRefreshPoint(for: resolved) ?? point, + prefersFocusedElement: false + ) + } + + private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? { + guard let element else { + return nil + } + let frame = element.frame + guard !frame.isEmpty else { + return nil + } + return CGPoint(x: frame.midX, y: frame.midY) + } + + /// A focus-moved signal for iOS text entry, where `focusedTextInput` is intentionally nil. + /// The software keyboard TRANSITIONING from hidden (at entry) to visible means the field we + /// just tapped gained first-responder. If the keyboard was ALREADY up (e.g. back-to-back + /// fills into different fields), its visibility is not evidence focus moved to the new field, + /// so callers must keep waiting rather than typing into the previously-focused field. +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift new file mode 100644 index 000000000..3404d0adc --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -0,0 +1,158 @@ +import XCTest + +// Text-entry readiness: whether the target is actually able to receive text yet. The waits, the +// keyboard signals they read, and the focus corroboration that ends the hardware-keyboard window. +// RunnerTests+TextEntryFocus.swift is the only caller of the two non-private entries here; nothing +// in this file reaches back into it. +extension RunnerTests { + func focusedTextInput(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + // iOS focus predicates can return stale or misleading text-input matches + // under XCUITest, so text entry readiness is driven by tap/keyboard state. + return nil +#else + return safely("FOCUSED_INPUT_QUERY") { + let candidates = app + .descendants(matching: .any) + .matching(NSPredicate(format: "hasKeyboardFocus == 1")) + .allElementsBoundByIndex + for candidate in candidates where candidate.exists { + switch candidate.elementType { + case .textField, .secureTextField, .searchField, .textView: + return candidate + default: + continue + } + } + return nil + } +#endif + } + + func waitForTextEntryReadiness( + app: XCUIApplication, + target: TextEntryTarget, + timeout: TimeInterval = TextEntryTiming.readinessTimeout + ) -> XCUIElement? { +#if os(iOS) + var latest = resolveTextEntryElement(app: app, target: target) + let keyboardVisibleAtEntry = isKeyboardVisible(app: app) + let deadline = Date().addingTimeInterval(timeout) + var hardwareKeyboardFallback = Date().addingTimeInterval( + min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout) + ) + var sawSoftwareKeyboard = false + while Date() < deadline { + if let focused = focusedTextInput(app: app) { + latest = focused + if isKeyboardVisible(app: app) { + return focused + } + } + // Fast-path on a keyboard hidden->visible transition: our tapped field gained focus, so + // return immediately instead of burning the full readinessTimeout (warmup-first-char echo + // + post-type verify/repair remain as drop safety nets). When the keyboard was ALREADY up + // (back-to-back fills), this isn't a focus signal — fall through to the settle/timeout so + // text isn't sent to the previously-focused field. + if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) { + return latest + } + sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app) + // A responder that takes no software keyboard (hardware keyboard connected, or a custom + // `inputView`) would otherwise burn the whole readinessTimeout waiting for one that is never + // coming. Leaving that window a bare wall-clock guess made readiness a function of ambient + // simulator state (#1874): on a loaded host the keyboard is merely late, and returning here + // handed the caller an element that had not taken focus yet. Ask the target itself instead, + // and re-arm rather than re-asking every poll — the query is cheap, not free. + if !sawSoftwareKeyboard, Date() >= hardwareKeyboardFallback, let candidate = latest { + if keyboardFocusConfirmed(app: app, element: candidate) { + return candidate + } + hardwareKeyboardFallback = Date().addingTimeInterval( + TextEntryTiming.hardwareKeyboardFallbackTimeout + ) + } + sleepFor(TextEntryTiming.pollInterval) + } + return focusedTextInput(app: app) ?? latest +#else + return resolveTextEntryElement(app: app, target: target) +#endif + } + + func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { +#if os(iOS) + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil { + return + } + let frame = element.frame + if !frame.isEmpty { + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) + } + default: + return + } +#endif + } + + private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if let focused = focusedTextInput(app: app) { + return focused + } + sleepFor(TextEntryTiming.pollInterval) + } + return focusedTextInput(app: app) + } + + func keyboardBecameVisible(app: XCUIApplication, wasVisibleAtEntry: Bool) -> Bool { + return !wasVisibleAtEntry && isKeyboardVisible(app: app) + } + + /// Positive evidence that this element — the one readiness is about to hand its caller — holds + /// keyboard focus. Readable even with no software keyboard on screen, which is what lets the + /// hardware-keyboard fallback stop guessing from a wall clock. + /// + /// This is the same app-wide predicate `focusedTextInput` refuses to trust on iOS, used the + /// other way round. There, the query PICKS the target, so a stale or unrelated match becomes + /// the field that gets typed into. Here the target is already chosen and the query only + /// corroborates it: at most one element holds keyboard focus, so an answer that is not this + /// element is a refusal, not a substitution. Every way of being wrong therefore ends as `false` + /// and costs the remaining readiness timeout — exactly what the wait would spend with no + /// fallback at all. + func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { +#if os(iOS) + return safely("TEXT_ENTRY_FOCUS_CONFIRMED", false) { + // An element that no longer resolves reads as identifier "" and frame `.zero`, which would + // match any focused element that also reports an empty frame. Require a real frame first, + // so a dead handle cannot corroborate anything. + let frame = element.frame + guard !frame.isEmpty else { + return false + } + let focused = app + .descendants(matching: .any) + .matching(NSPredicate(format: "hasKeyboardFocus == 1")) + .firstMatch + guard focused.exists else { + return false + } + return focused.identifier == element.identifier && focused.frame == frame + } +#else + return false +#endif + } + + private func keyboardElementExists(app: XCUIApplication) -> Bool { +#if os(iOS) + return safely("KEYBOARD_EXISTS", false) { app.keyboards.firstMatch.exists } +#else + return false +#endif + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedCommitDeadlineTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedCommitDeadlineTests.swift new file mode 100644 index 000000000..840a6b0ef --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedCommitDeadlineTests.swift @@ -0,0 +1,164 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + /// A hand-driven clock for the commit waits. Time moves only where the wait sleeps, which is + /// what makes "the burst kept landing" and "the pipeline froze" expressible as two sequences of + /// the same length rather than as wall-clock luck — and a test that never advances it cannot + /// expire any budget, so only a test asking about time names `stallBudget`/`ceiling`. + final class CommitWaitClock { + private let origin = Date(timeIntervalSinceReferenceDate: 0) + private var current = Date(timeIntervalSinceReferenceDate: 0) + + var read: () -> Date { { self.current } } + + func advance(_ seconds: TimeInterval) { + current = current.addingTimeInterval(seconds) + } + + /// Seconds elapsed on this clock, for asserting *when* a wait gave up. + var elapsed: TimeInterval { current.timeIntervalSince(origin) } + } + + // #1874, through the shipped wait rather than a detached policy object: a burst that keeps + // landing must outlive the flat 3s deadline that used to govern it. Each poll advances the + // clock 2s and delivers one more character, so the wait is never idle for a full stall budget + // and must walk all the way to the match at t=6s. Reverting the wait to a flat deadline turns + // this red at the third poll. + func testCommitWaitOutlivesTheFlatDeadlineWhileTheExpectedPrefixGrows() { + let expected = "hardware" + let clock = CommitWaitClock() + var landed = 0 + let outcome = Self.awaitSynthesizedCommitOutcome( + expectedText: expected, + placeholder: nil, + stallBudget: 3, + ceiling: 10, + now: clock.read, + observe: { String(expected.prefix(landed * 2)) }, + waitForNextObservation: { + landed += 1 + clock.advance(2) + } + ) + XCTAssertEqual(outcome, .settled) + XCTAssertEqual(clock.elapsed, 8, "the wait must still be polling well past the 3s stall budget") + } + + // Review finding: the wait's setup reads the field's placeholder first, and that is an AX + // round-trip which takes seconds on exactly the loaded host this budget exists for. An earlier + // revision started the clock before that read, so slow setup spent the budget and the wait gave + // up after a single poll — sooner than the flat deadline it replaced, in the one condition it + // was written for. The clock now starts inside the loop, so a slow setup buys nothing and costs + // nothing. + func testCommitWaitBudgetStartsAtTheLoopRatherThanBeforeIt() { + let clock = CommitWaitClock() + clock.advance(60) + var polls = 0 + let outcome = Self.awaitSynthesizedCommitOutcome( + expectedText: "hardware", + placeholder: nil, + stallBudget: 3, + ceiling: 10, + now: clock.read, + observe: { "ha" }, + waitForNextObservation: { + polls += 1 + clock.advance(1) + } + ) + XCTAssertEqual(outcome, .notObserved) + XCTAssertEqual(polls, 3, "the wait must get its whole stall budget however slow its setup was") + } + + // The other half, and the reason the stall budget keeps the flat deadline's number: a pipeline + // that delivers nothing is condemned at exactly the instant it always was, so nothing that + // fails today starts passing merely by waiting longer. + func testCommitWaitCondemnsAFrozenPipelineAtTheStallBudget() { + let clock = CommitWaitClock() + let outcome = Self.awaitSynthesizedCommitOutcome( + expectedText: "hardware", + placeholder: nil, + stallBudget: 3, + ceiling: 10, + now: clock.read, + observe: { "ha" }, + waitForNextObservation: { clock.advance(1) } + ) + XCTAssertEqual(outcome, .notObserved) + XCTAssertEqual(clock.elapsed, 3, "a frozen prefix must give up on the stall budget, not the ceiling") + } + + // Progress buys time, but not without bound: one character per stall window would otherwise + // hold the command open until the daemon's own 45s budget killed the request. Here every poll + // lands a character, so only the ceiling can stop it. + func testCommitWaitCeilingStopsAnIndefinitelyThrottledPipeline() { + let clock = CommitWaitClock() + var landed = 0 + let outcome = Self.awaitSynthesizedCommitOutcome( + expectedText: String(repeating: "a", count: 100), + placeholder: nil, + stallBudget: 3, + ceiling: 10, + now: clock.read, + observe: { String(repeating: "a", count: landed) }, + waitForNextObservation: { + landed += 1 + clock.advance(2) + } + ) + XCTAssertEqual(outcome, .notObserved) + XCTAssertEqual(clock.elapsed, 10, "the ceiling is absolute, however long characters keep arriving") + } + + // Only forward movement is evidence the burst is still landing. A field the app clears + // mid-flight would otherwise reset the stall clock on every poll and hold every wedged wait + // open to the ceiling. Replacement mode, because that is where a non-matching value keeps + // polling rather than settling as `.diverged`. + func testCommitWaitTreatsARetreatingValueAsNoProgress() { + let clock = CommitWaitClock() + let observations = ["ada@", "", "ada@", "", "ada@"] + var index = 0 + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( + expectedText: "ada@example", + placeholder: nil, + stallBudget: 3, + ceiling: 10, + now: clock.read, + observe: { observations[min(index, observations.count - 1)] }, + waitForNextObservation: { + index += 1 + clock.advance(1) + } + ) + XCTAssertEqual(outcome, .notObserved) + XCTAssertEqual(clock.elapsed, 3, "churn between two values is not progress and must not buy time") + } + + // The replacement route earns time the same way, and is bounded the same way — it is the route + // `fill` takes when the XCTest channel is penalized, i.e. the one that runs on a loaded host. + // Here every poll lands one more character of a value that never completes, so the wait can only + // end at the ceiling: it proves the growing prefix carried it past the 3s stall budget (a flat + // deadline stops at t=3) and that the ceiling still stops it. + func testReplacementCommitWaitOutlivesTheFlatDeadlineThenStopsAtTheCeiling() { + let expected = "ada@example.com" + let clock = CommitWaitClock() + var landed = 0 + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( + expectedText: expected, + placeholder: nil, + stallBudget: 3, + ceiling: 10, + now: clock.read, + observe: { String(expected.prefix(landed)) }, + waitForNextObservation: { + landed += 1 + clock.advance(2) + } + ) + XCTAssertEqual(outcome, .notObserved) + XCTAssertEqual(clock.elapsed, 10, "progress must carry the wait past 3s, and the ceiling must end it") + XCTAssertEqual(landed, 5, "one character per poll, five polls inside the ceiling") + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 2635b1ba3..5d7f61413 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -94,13 +94,19 @@ extension RunnerTests { // indistinguishable from a commit and `type` reported ok over a partially committed field. The // CI signature was a field holding "h" out of "hardware-keyboard" with the command successful. func testSynthesizedCommitDeadlineIsNotReportedAsACommit() { + let clock = CommitWaitClock() var observations = 0 let outcome = Self.awaitSynthesizedCommitOutcome( expectedText: "hardware-keyboard", placeholder: nil, - isExpired: { observations >= 3 }, + stallBudget: 3, + ceiling: 10, + now: clock.read, observe: { "h" }, - waitForNextObservation: { observations += 1 } + waitForNextObservation: { + observations += 1 + clock.advance(1) + } ) XCTAssertEqual(outcome, .notObserved) XCTAssertEqual(observations, 3, "a pending prefix must keep polling until the deadline") @@ -112,7 +118,6 @@ extension RunnerTests { let outcome = Self.awaitSynthesizedCommitOutcome( expectedText: "hardware-keyboard", placeholder: nil, - isExpired: { false }, observe: { observed }, waitForNextObservation: { polls += 1 } ) @@ -129,7 +134,6 @@ extension RunnerTests { let outcome = Self.awaitSynthesizedCommitOutcome( expectedText: "hardware-keyboard", placeholder: nil, - isExpired: { false }, observe: { steps[min(index, steps.count - 1)] }, waitForNextObservation: { index += 1 } ) @@ -141,13 +145,21 @@ extension RunnerTests { // landing during the final poll sleep was condemned as never observed — a false failure under // exactly the loaded-host timing this wait exists for. Red against that ordering. func testCommitLandingDuringTheFinalSleepIsStillObserved() { + let clock = CommitWaitClock() var polls = 0 let outcome = Self.awaitSynthesizedCommitOutcome( expectedText: "hardware-keyboard", placeholder: nil, - isExpired: { polls >= 1 }, + stallBudget: 3, + ceiling: 10, + now: clock.read, + // The value lands during the sleep that takes the clock past the stall budget: the read + // happens first, so it is still observed. observe: { polls == 0 ? "hardware-" : "hardware-keyboard" }, - waitForNextObservation: { polls += 1 } + waitForNextObservation: { + polls += 1 + clock.advance(9) + } ) XCTAssertEqual(outcome, .settled) } @@ -163,7 +175,6 @@ extension RunnerTests { let outcome = Self.awaitSynthesizedCommitOutcome( expectedText: expectedText, placeholder: "0.00", - isExpired: { false }, observe: { observations += 1 return "0.00" @@ -191,7 +202,6 @@ extension RunnerTests { let outcome = Self.awaitSynthesizedCommitOutcome( expectedText: testCase.expectedText, placeholder: testCase.placeholder, - isExpired: { false }, observe: { testCase.expectedText }, waitForNextObservation: {} ) @@ -219,20 +229,25 @@ extension RunnerTests { Self.awaitSynthesizedCommitOutcome( expectedText: corruption.expected, placeholder: nil, - isExpired: { false }, observe: { corruption.observedAfterDrop }, waitForNextObservation: {} ), .settled, "append-mode's diverge-trusting outcome must stay unchanged by this fix" ) + let clock = CommitWaitClock() var polls = 0 let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: corruption.expected, placeholder: nil, - isExpired: { polls >= 2 }, + stallBudget: 2, + ceiling: 10, + now: clock.read, observe: { corruption.observedAfterDrop }, - waitForNextObservation: { polls += 1 } + waitForNextObservation: { + polls += 1 + clock.advance(1) + } ) XCTAssertEqual(outcome, .notObserved, "expected \(corruption.expected), dropped to \(corruption.observedAfterDrop)") XCTAssertEqual(polls, 2, "a settled-but-wrong value must be polled until the deadline, not trusted early") @@ -250,7 +265,6 @@ extension RunnerTests { let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: "ada@example", placeholder: nil, - isExpired: { false }, observe: { steps[min(index, steps.count - 1)] }, waitForNextObservation: { index += 1 } ) @@ -261,13 +275,19 @@ extension RunnerTests { // Same ordering guarantee as `testCommitLandingDuringTheFinalSleepIsStillObserved`: the deadline // is checked AFTER an observation, so a match landing during the final poll sleep is still caught. func testSynthesizedReplacementCommitLandingDuringTheFinalSleepIsStillObserved() { + let clock = CommitWaitClock() var polls = 0 let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: "ada@example", placeholder: nil, - isExpired: { polls >= 1 }, + stallBudget: 3, + ceiling: 10, + now: clock.read, observe: { polls == 0 ? "ada@exampl" : "ada@example" }, - waitForNextObservation: { polls += 1 } + waitForNextObservation: { + polls += 1 + clock.advance(9) + } ) XCTAssertEqual(outcome, .settled) } @@ -279,7 +299,6 @@ extension RunnerTests { let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: "0.00", placeholder: "0.00", - isExpired: { false }, observe: { observations += 1 return "0.00" @@ -416,11 +435,13 @@ extension RunnerTests { // poll and the value never becomes "abc" — under the replacement-mode outcome function that is // correctly a failure (see `testSynthesizedReplacementCommitCatchesDroppedMiddleCharacters` for // why it must NOT be waved through as success), so this call runs the real 3-second deadline - // (`TextEntryTiming.synthesizedCommitTimeout`) before returning. That is deliberate here, not a - // flake: this test only runs in the nightly XCUITest lane (see `runner-xctest-local-run-gotchas` - // memory / ios.yml's `-only-testing:` allowlist), where a few extra seconds is a non-issue, and - // the alternative — asserting `nil` on a wiring path that can never actually observe the - // expected text — would silently reintroduce the exact bug this fix closes. + // (`TextEntryTiming.synthesizedCommitStallTimeout`; a nil read never advances the expected + // prefix, so `SynthesizedCommitDeadline` grants it no extra time) before returning. That is + // deliberate here, not a flake: this test only runs in the nightly XCUITest lane (see + // `runner-xctest-local-run-gotchas` memory / ios.yml's `-only-testing:` allowlist), where a + // few extra seconds is a non-issue, and the alternative — asserting `nil` on a wiring path + // that can never actually observe the expected text — would silently reintroduce the exact + // bug this fix closes. XCTAssertEqual(result.failure, .commitNotObserved) } diff --git a/examples/test-app/src/screens/FormScreen.tsx b/examples/test-app/src/screens/FormScreen.tsx index 42802194c..e9c8273a6 100644 --- a/examples/test-app/src/screens/FormScreen.tsx +++ b/examples/test-app/src/screens/FormScreen.tsx @@ -108,7 +108,7 @@ export function FormScreen(props: FormScreenProps) { accessibilityLabel="Full name" label="Full name" onChangeText={(value) => props.onChange('name', value)} - placeholder="Ada Lovelace" + placeholder="Type your full name" testID="field-name" value={props.form.name} /> @@ -119,7 +119,7 @@ export function FormScreen(props: FormScreenProps) { label="Email" onChangeText={(value) => props.onChange('email', value)} onSubmitEditing={() => Keyboard.dismiss()} - placeholder="ada@example.com" + placeholder="Type your email" returnKeyType="done" testID="field-email" value={props.form.email} diff --git a/src/__tests__/apple-runner-log-redaction.test.ts b/src/__tests__/apple-runner-log-redaction.test.ts index 1d72336ec..3cb18245b 100644 --- a/src/__tests__/apple-runner-log-redaction.test.ts +++ b/src/__tests__/apple-runner-log-redaction.test.ts @@ -5,9 +5,9 @@ import { fileURLToPath } from 'node:url'; import { test } from 'vitest'; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); -const synthesizedTextEntryPath = path.join( +const commitWaitPath = path.join( repoRoot, - 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift', + 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift', ); // The synthesized bare-type commit wait polls the target field's live value on the shipped @@ -34,7 +34,7 @@ function extractObserveClosure(source: string): string { } test('commit-wait cadence logging goes through the typed value-free boundary', () => { - const source = fs.readFileSync(synthesizedTextEntryPath, 'utf8'); + const source = fs.readFileSync(commitWaitPath, 'utf8'); assert.match( source,