From 3f87bc260d8575a1d577c467fcee79752740cdaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 24 Aug 2026 10:25:04 +0200 Subject: [PATCH 1/5] fix(ci): repair nightly XCTest and conformance lanes --- .../RunnerTests+TextEntryPolicyTests.swift | 2 +- package.json | 2 +- .../differential/flows/settle-after-tap.yaml | 13 +++++-------- .../conformance/differential/invariants.test.ts | 11 +++++++++++ .../maestro-conformance/format-generated-json.mjs | 14 ++++++++++++++ .../format-generated-json.test.mjs | 10 ++++++++++ scripts/maestro-conformance/regenerate.mjs | 3 ++- 7 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 scripts/maestro-conformance/format-generated-json.mjs create mode 100644 scripts/maestro-conformance/format-generated-json.test.mjs diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 9aefeffe6..bd92abf21 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -448,7 +448,7 @@ extension RunnerTests { // Divergence stops the count: the app transformed the input, and the walk must not // resume matching after the first differing character. XCTAssertEqual(Self.commonPrefixLength("hx", "hardware-keyboard"), 1) - XCTAssertEqual(Self.commonPrefixLength("hardware-keyboarx", "hardware-keyboard"), 15) + XCTAssertEqual(Self.commonPrefixLength("hardware-keyboarx", "hardware-keyboard"), 16) } func testCommitCadenceLogLineEmitsLengthsOnlyNeverContents() { diff --git a/package.json b/package.json index 85f4ea931..5b5744709 100644 --- a/package.json +++ b/package.json @@ -113,7 +113,7 @@ "release:publish": "pnpm release:prepare && npm publish --ignore-scripts .tmp/release/*.tgz", "ad": "node bin/agent-device.mjs", "bench:help-conformance": "node scripts/help-conformance-bench.mjs", - "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", + "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", "maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs", "maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts", "size": "node scripts/size-report.mjs", diff --git a/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml b/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml index 2934a1ab9..00e8c101f 100644 --- a/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml +++ b/packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml @@ -1,16 +1,13 @@ # Layer-3 device flow (bug class 4). A real tap on a real control of the lab app, # so the post-tap settle loop actually runs and the timing invariant is meaningful. -# home-open-form sits below the fold, so scroll it into view first — the app's own -# helper flow (examples/test-app/maestro/helpers/open-checkout-form.yaml) does -# exactly this; tapping it directly fails with "element not found". +# Use the always-visible Settings tab so scrolling is not an unrelated +# precondition for the tap detector. open-inert-surface proves navigation finished. appId: com.callstack.agentdevicelab --- - launchApp: clearState: true - assertVisible: Agent Device Tester -- scrollUntilVisible: - element: - id: home-open-form - tapOn: - id: home-open-form -- assertVisible: Checkout form + text: Settings +- assertVisible: + id: open-inert-surface diff --git a/packages/maestro/test/conformance/differential/invariants.test.ts b/packages/maestro/test/conformance/differential/invariants.test.ts index 15615aca9..e32688afa 100644 --- a/packages/maestro/test/conformance/differential/invariants.test.ts +++ b/packages/maestro/test/conformance/differential/invariants.test.ts @@ -90,6 +90,17 @@ test('bug class 4 has a machine-checkable invariant, not just outcome parity', ( ); }); +test('the settle detector reaches its tap without an unrelated scroll precondition', () => { + const flow = fs.readFileSync( + path.join(import.meta.dirname, 'flows/settle-after-tap.yaml'), + 'utf8', + ); + + assert.doesNotMatch(flow, /scrollUntilVisible/); + assert.match(flow, /text: Settings/); + assert.match(flow, /id: open-inert-surface/); +}); + // --- metricAtLeast: proves a code path actually ran, not just that it passed --- const RETRY_INVARIANT: Invariant = { diff --git a/scripts/maestro-conformance/format-generated-json.mjs b/scripts/maestro-conformance/format-generated-json.mjs new file mode 100644 index 000000000..f4271b5f7 --- /dev/null +++ b/scripts/maestro-conformance/format-generated-json.mjs @@ -0,0 +1,14 @@ +import { execFileSync } from 'node:child_process'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const REPOSITORY_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const OXFMT_BIN = path.join(REPOSITORY_ROOT, 'node_modules/oxfmt/bin/oxfmt'); + +export function formatGeneratedJson(value, target = 'generated.json') { + return execFileSync(process.execPath, [OXFMT_BIN, `--stdin-filepath=${target}`], { + cwd: REPOSITORY_ROOT, + encoding: 'utf8', + input: `${JSON.stringify(value, null, 2)}\n`, + }); +} diff --git a/scripts/maestro-conformance/format-generated-json.test.mjs b/scripts/maestro-conformance/format-generated-json.test.mjs new file mode 100644 index 000000000..4b7f3dfcb --- /dev/null +++ b/scripts/maestro-conformance/format-generated-json.test.mjs @@ -0,0 +1,10 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { formatGeneratedJson } from './format-generated-json.mjs'; + +test('generated fixtures use the repository JSON format', () => { + const formatted = formatGeneratedJson({ matches: ['card'], selected: 'card' }); + + assert.equal(formatted, '{\n "matches": ["card"],\n "selected": "card"\n}\n'); + assert.deepEqual(JSON.parse(formatted), { matches: ['card'], selected: 'card' }); +}); diff --git a/scripts/maestro-conformance/regenerate.mjs b/scripts/maestro-conformance/regenerate.mjs index dff6c9b00..42d60f94c 100644 --- a/scripts/maestro-conformance/regenerate.mjs +++ b/scripts/maestro-conformance/regenerate.mjs @@ -21,6 +21,7 @@ import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { writeManifest } from './build-manifest.mjs'; +import { formatGeneratedJson } from './format-generated-json.mjs'; import { fixtureContentHash } from '../../packages/maestro/test/conformance/fixture-seal.ts'; const HERE = path.dirname(fileURLToPath(import.meta.url)); @@ -91,7 +92,7 @@ function writeFixture(name, pin, content) { // on any byte difference, which forgery cannot survive. const wrappedWithSeal = { ...wrapped, contentHash: fixtureContentHash(wrapped) }; const target = path.join(FIXTURES_DIR, name); - fs.writeFileSync(target, `${JSON.stringify(wrappedWithSeal, null, 2)}\n`); + fs.writeFileSync(target, formatGeneratedJson(wrappedWithSeal, target)); console.log(`wrote ${path.relative(HERE, target)}`); } From 85c963de680d4a678af7a7df72439f6baba4a5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 24 Aug 2026 10:55:42 +0200 Subject: [PATCH 2/5] fix(ci): harden nightly failure classification --- .../RunnerTests+TextEntryPolicyTests.swift | 6 +- package.json | 2 +- packages/contracts/src/replay.ts | 2 + .../differential/engine-process.test.ts | 22 +++ .../differential/engine-process.ts | 70 ++++++++ .../differential/invariants.test.ts | 49 ++++- .../differential/report-output.test.ts | 19 ++ .../conformance/differential/report-output.ts | 77 ++++++++ .../test/conformance/differential/run.test.ts | 19 +- .../test/conformance/differential/run.ts | 169 +++++++----------- packages/maestro/test/conformance/harness.ts | 8 +- .../src/internal/session-test-attempt.ts | 3 + src/cli-schema/cli-help-topics.test.ts | 2 + src/cli-schema/cli-help.ts | 1 + .../session-test-infrastructure.test.ts | 13 ++ .../__tests__/session-test-suite.test.ts | 1 + .../__tests__/boot-diagnostics.test.ts | 5 + .../apple/core/runner/runner-lease.ts | 1 + src/platforms/boot-diagnostics.ts | 4 + 19 files changed, 353 insertions(+), 120 deletions(-) create mode 100644 packages/maestro/test/conformance/differential/engine-process.test.ts create mode 100644 packages/maestro/test/conformance/differential/engine-process.ts create mode 100644 packages/maestro/test/conformance/differential/report-output.test.ts create mode 100644 packages/maestro/test/conformance/differential/report-output.ts diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index bd92abf21..2635b1ba3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -448,7 +448,11 @@ extension RunnerTests { // Divergence stops the count: the app transformed the input, and the walk must not // resume matching after the first differing character. XCTAssertEqual(Self.commonPrefixLength("hx", "hardware-keyboard"), 1) - XCTAssertEqual(Self.commonPrefixLength("hardware-keyboarx", "hardware-keyboard"), 16) + let sharedPrefix = "hardware-keyboar" + XCTAssertEqual( + Self.commonPrefixLength("\(sharedPrefix)x", "\(sharedPrefix)d"), + sharedPrefix.count + ) } func testCommitCadenceLogLineEmitsLengthsOnlyNeverContents() { diff --git a/package.json b/package.json index 5b5744709..29a33f10e 100644 --- a/package.json +++ b/package.json @@ -113,7 +113,7 @@ "release:publish": "pnpm release:prepare && npm publish --ignore-scripts .tmp/release/*.tgz", "ad": "node bin/agent-device.mjs", "bench:help-conformance": "node scripts/help-conformance-bench.mjs", - "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", + "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", "maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs", "maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts", "size": "node scripts/size-report.mjs", diff --git a/packages/contracts/src/replay.ts b/packages/contracts/src/replay.ts index 21de0768e..4a610131f 100644 --- a/packages/contracts/src/replay.ts +++ b/packages/contracts/src/replay.ts @@ -108,6 +108,8 @@ export type ReplaySuiteTestFailed = { attempts: number; artifactsDir?: string; error: DaemonError; + /** Present when the owning runtime classified the failure as device/runner infrastructure. */ + infrastructure?: true; shardIndex?: number; shardCount?: number; deviceId?: string; diff --git a/packages/maestro/test/conformance/differential/engine-process.test.ts b/packages/maestro/test/conformance/differential/engine-process.test.ts new file mode 100644 index 000000000..b45b69454 --- /dev/null +++ b/packages/maestro/test/conformance/differential/engine-process.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { classifyAgentDeviceFailure } from './engine-process.ts'; + +test('agent-device JSON distinguishes infrastructure from behavioral failures', () => { + const result = (infrastructure?: true) => + JSON.stringify({ + success: true, + data: { + failures: [ + { + status: 'failed', + ...(infrastructure ? { infrastructure } : {}), + }, + ], + }, + }); + + assert.equal(classifyAgentDeviceFailure(result(true)), 'infrastructure'); + assert.equal(classifyAgentDeviceFailure(result()), 'behavioral'); + assert.equal(classifyAgentDeviceFailure('not-json'), 'infrastructure'); +}); diff --git a/packages/maestro/test/conformance/differential/engine-process.ts b/packages/maestro/test/conformance/differential/engine-process.ts new file mode 100644 index 000000000..429accb46 --- /dev/null +++ b/packages/maestro/test/conformance/differential/engine-process.ts @@ -0,0 +1,70 @@ +import { spawnSync } from 'node:child_process'; +import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; + +export type EngineResult = { + engine: 'maestro' | 'agent-device'; + outcome: 'pass' | 'fail'; + exitCode: number; + /** Failure provenance stays distinct from the behavioral comparison. */ + failureKind?: 'behavioral' | 'infrastructure'; +}; + +export function runEngine( + engine: EngineResult['engine'], + command: string, + args: string[], +): EngineResult { + const [bin = '', ...rest] = command.split(' ').filter(Boolean); + const result = spawnSync(bin, [...rest, ...args], { stdio: 'inherit', cwd: process.cwd() }); + const exitCode = result.status ?? 1; + const infrastructureFailed = result.status === null || result.error !== undefined; + return { + engine, + outcome: exitCode === 0 ? 'pass' : 'fail', + exitCode, + ...(exitCode === 0 + ? {} + : { + failureKind: infrastructureFailed ? ('infrastructure' as const) : ('behavioral' as const), + }), + }; +} + +export function classifyAgentDeviceFailure(stdout: string): 'behavioral' | 'infrastructure' { + try { + const envelope = JSON.parse(stdout) as { data?: ReplaySuiteResult }; + const failures = envelope.data?.failures; + if (!Array.isArray(failures) || failures.length === 0) return 'infrastructure'; + return failures.some((failure) => failure.infrastructure === true) + ? 'infrastructure' + : 'behavioral'; + } catch { + // A non-zero process that did not return the promised suite envelope never reached a + // classifiable behavioral oracle. Keep it red, but do not call it a divergence. + return 'infrastructure'; + } +} + +export function runAgentDeviceEngine(command: string, args: string[]): EngineResult { + const [bin = '', ...rest] = command.split(' ').filter(Boolean); + const result = spawnSync(bin, [...rest, ...args, '--json'], { + cwd: process.cwd(), + encoding: 'utf8', + }); + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + const exitCode = result.status ?? 1; + const infrastructureFailed = result.status === null || result.error !== undefined; + return { + engine: 'agent-device', + outcome: exitCode === 0 ? 'pass' : 'fail', + exitCode, + ...(exitCode === 0 + ? {} + : { + failureKind: infrastructureFailed + ? ('infrastructure' as const) + : classifyAgentDeviceFailure(result.stdout), + }), + }; +} diff --git a/packages/maestro/test/conformance/differential/invariants.test.ts b/packages/maestro/test/conformance/differential/invariants.test.ts index e32688afa..1a4ab96ee 100644 --- a/packages/maestro/test/conformance/differential/invariants.test.ts +++ b/packages/maestro/test/conformance/differential/invariants.test.ts @@ -6,7 +6,11 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { test } from 'node:test'; -import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS } from '../harness.ts'; +import { + type CanonicalCommand, + MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, + parseMaestroConformanceSource, +} from '../harness.ts'; import { DIFFERENTIAL_SCENARIOS } from './scenarios.ts'; import { type Invariant, evaluateInvariant, readTrace } from './invariants.ts'; @@ -90,15 +94,42 @@ test('bug class 4 has a machine-checkable invariant, not just outcome parity', ( ); }); -test('the settle detector reaches its tap without an unrelated scroll precondition', () => { - const flow = fs.readFileSync( - path.join(import.meta.dirname, 'flows/settle-after-tap.yaml'), - 'utf8', - ); +const SETTLE_FLOW_PATH = path.join(import.meta.dirname, 'flows/settle-after-tap.yaml'); +const EXPECTED_SETTLE_COMMANDS: CanonicalCommand[] = [ + { kind: 'launchApp', appId: 'com.callstack.agentdevicelab', clearState: true }, + { + kind: 'assert', + mode: 'visible', + timed: false, + selector: { text: 'Agent Device Tester' }, + }, + { + kind: 'tap', + longPress: false, + repeat: 1, + target: { selector: { text: 'Settings' } }, + }, + { + kind: 'assert', + mode: 'visible', + timed: false, + selector: { id: 'open-inert-surface' }, + }, +]; - assert.doesNotMatch(flow, /scrollUntilVisible/); - assert.match(flow, /text: Settings/); - assert.match(flow, /id: open-inert-surface/); +function assertSettleFlowSemantics(source: string): void { + const parsed = parseMaestroConformanceSource(source, SETTLE_FLOW_PATH); + assert.deepEqual(parsed.commands, EXPECTED_SETTLE_COMMANDS); +} + +test('the settle detector reaches its tap without an unrelated setup command', () => { + assertSettleFlowSemantics(fs.readFileSync(SETTLE_FLOW_PATH, 'utf8')); +}); + +test('the settle flow guard rejects a changed tap target or inserted scroll', () => { + const flow = fs.readFileSync(SETTLE_FLOW_PATH, 'utf8'); + assert.throws(() => assertSettleFlowSemantics(flow.replace('text: Settings', 'text: Home'))); + assert.throws(() => assertSettleFlowSemantics(flow.replace('- tapOn:', '- scroll\n- tapOn:'))); }); // --- metricAtLeast: proves a code path actually ran, not just that it passed --- diff --git a/packages/maestro/test/conformance/differential/report-output.test.ts b/packages/maestro/test/conformance/differential/report-output.test.ts new file mode 100644 index 000000000..5090c85ab --- /dev/null +++ b/packages/maestro/test/conformance/differential/report-output.test.ts @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { writeReports } from './report-output.ts'; + +test('writes the typed differential report envelope', () => { + const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'differential-report-')); + try { + writeReports(outDir, 'ios', []); + const report = JSON.parse( + fs.readFileSync(path.join(outDir, 'differential-report.json'), 'utf8'), + ); + assert.deepEqual(report, { platform: 'ios', reports: [] }); + } finally { + fs.rmSync(outDir, { recursive: true, force: true }); + } +}); diff --git a/packages/maestro/test/conformance/differential/report-output.ts b/packages/maestro/test/conformance/differential/report-output.ts new file mode 100644 index 000000000..f750e20ab --- /dev/null +++ b/packages/maestro/test/conformance/differential/report-output.ts @@ -0,0 +1,77 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { EngineResult } from './engine-process.ts'; +import type { InvariantResult } from './invariants.ts'; +import type { DifferentialScenario } from './scenarios.ts'; + +export type ScenarioReport = { + id: string; + flow: string; + maestro: EngineResult; + agentDevice: EngineResult; + outcomeDiverged: boolean; + invariants: InvariantResult[]; + status: 'ok' | 'failed' | 'infrastructure-failed' | 'known-divergence' | 'stale-declaration'; + tracking?: string; + failed: boolean; +}; + +export function printScenarioReport(report: ScenarioReport): void { + console.log( + `${report.status.padEnd(17)} ${report.id} maestro=${report.maestro.outcome} agent-device=${report.agentDevice.outcome}`, + ); + for (const result of report.invariants) { + console.log(` invariant ${result.status}: ${result.detail}`); + } + if (report.status === 'known-divergence') { + console.log(` declared divergence, tracked: ${report.tracking}`); + } + if (report.status === 'stale-declaration') { + console.log( + ` passed while declared divergent — remove knownDivergence (${report.tracking}) so this stays enforced`, + ); + } + if (report.status === 'infrastructure-failed') { + console.log(' oracle did not complete because engine infrastructure failed'); + } +} + +export function printDryRun(scenarios: DifferentialScenario[]): void { + for (const scenario of scenarios) { + const invariants = scenario.engineInvariants?.length ?? 0; + const declared = scenario.knownDivergence + ? `\tdeclared-divergence=${scenario.knownDivergence.tracking}` + : ''; + console.log( + `${scenario.id}\t${scenario.flow}\texpect=${scenario.expect}\tinvariants=${invariants}${declared}`, + ); + } + const known = scenarios.filter((scenario) => scenario.knownDivergence).length; + console.log(`\n${scenarios.length} scenario(s) validated, ${known} declared divergence(s).`); +} + +export function writeReports( + outDir: string | undefined, + platform: string | undefined, + reports: ScenarioReport[], +): void { + if (!outDir) return; + fs.mkdirSync(outDir, { recursive: true }); + fs.writeFileSync( + path.join(outDir, 'differential-report.json'), + `${JSON.stringify({ platform, reports }, null, 2)}\n`, + ); +} + +export function printRunSummary(reports: ScenarioReport[]): void { + const known = reports.filter((report) => report.status === 'known-divergence'); + if (known.length > 0) { + console.log( + `\n${known.length} declared divergence(s), not enforced: ${known.map((r) => r.id).join(', ')}`, + ); + } + const failed = reports.filter((report) => report.failed); + if (failed.length === 0) return; + console.error(`\n${failed.length} scenario(s) failed: ${failed.map((r) => r.id).join(', ')}`); + process.exitCode = 1; +} diff --git a/packages/maestro/test/conformance/differential/run.test.ts b/packages/maestro/test/conformance/differential/run.test.ts index e556db844..a2f932ea2 100644 --- a/packages/maestro/test/conformance/differential/run.test.ts +++ b/packages/maestro/test/conformance/differential/run.test.ts @@ -11,6 +11,7 @@ import { DIFFERENTIAL_SCENARIOS, type DivergenceSignature, } from './scenarios.ts'; +import { parseMaestroConformanceSource } from '../harness.ts'; import { matchesSignature, parseRunnerArgs, selectScenarios, validateScenarios } from './run.ts'; const CONFORMANCE_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); @@ -120,6 +121,15 @@ describe('knownDivergence signature matching', () => { ); }); + test('infrastructure failure is NOT covered by a behavioral waiver', () => { + assert.equal( + matchesSignature(sig, engine('pass'), { ...engine('fail'), failureKind: 'infrastructure' }, [ + inv('no-data'), + ]), + false, + ); + }); + test('every declaration states its expected signature', () => { for (const scenario of DIFFERENTIAL_SCENARIOS) { const declared = scenario.knownDivergence; @@ -144,10 +154,11 @@ describe('knownDivergence signature matching', () => { test('every device flow targets the fixture app the workflow installs', () => { for (const scenario of DIFFERENTIAL_SCENARIOS) { - const body = fs.readFileSync(path.join(CONFORMANCE_DIR, scenario.flow), 'utf8'); - assert.match( - body, - new RegExp(`^appId:\\s*${DIFFERENTIAL_APP_ID}$`, 'm'), + const flowPath = path.join(CONFORMANCE_DIR, scenario.flow); + const parsed = parseMaestroConformanceSource(fs.readFileSync(flowPath, 'utf8'), flowPath); + assert.equal( + parsed.appId, + DIFFERENTIAL_APP_ID, `${scenario.id} must target ${DIFFERENTIAL_APP_ID}; a flow against any other app cannot run on the CI simulator`, ); } diff --git a/packages/maestro/test/conformance/differential/run.ts b/packages/maestro/test/conformance/differential/run.ts index 47766448a..92f7b70b6 100644 --- a/packages/maestro/test/conformance/differential/run.ts +++ b/packages/maestro/test/conformance/differential/run.ts @@ -9,7 +9,6 @@ // // `--dry-run` validates the scenario registry without a device (exercised by // run.test.ts in unit CI, the same shape as help-conformance-bench). -import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -19,6 +18,14 @@ import { type DivergenceSignature, } from './scenarios.ts'; import { type InvariantResult, evaluateInvariants, readTrace } from './invariants.ts'; +import { type EngineResult, runAgentDeviceEngine, runEngine } from './engine-process.ts'; +import { + printDryRun, + printRunSummary, + printScenarioReport, + type ScenarioReport, + writeReports, +} from './report-output.ts'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const CONFORMANCE_DIR = path.resolve(HERE, '..'); @@ -73,41 +80,6 @@ export function validateScenarios(): void { } } -type EngineResult = { - engine: 'maestro' | 'agent-device'; - outcome: 'pass' | 'fail'; - exitCode: number; -}; - -function runEngine(engine: EngineResult['engine'], command: string, args: string[]): EngineResult { - const [bin = '', ...rest] = command.split(' ').filter(Boolean); - const result = spawnSync(bin, [...rest, ...args], { stdio: 'inherit', cwd: process.cwd() }); - const exitCode = result.status ?? 1; - return { engine, outcome: exitCode === 0 ? 'pass' : 'fail', exitCode }; -} - -export type ScenarioReport = { - id: string; - flow: string; - maestro: EngineResult; - agentDevice: EngineResult; - /** Outcome parity across the two engines. */ - outcomeDiverged: boolean; - /** Engine-side invariants over agent-device's own timing trace. */ - invariants: InvariantResult[]; - /** - * ok — behaved as expected. - * failed — an UNDECLARED divergence. The signal this exists for. - * known-divergence — failed exactly as declared; tracked, so the run stays green. - * stale-declaration — passed while still declared divergent: remove the - * declaration (the gap is closed). Fails, so a fix cannot - * land while leaving the oracle blind to a regression. - */ - status: 'ok' | 'failed' | 'known-divergence' | 'stale-declaration'; - tracking?: string; - failed: boolean; -}; - /** * Locate the timing trace agent-device wrote for this run. The test runtime * writes `replay-timing.ndjson` under the run's artifacts directory; we take the @@ -140,6 +112,8 @@ export function matchesSignature( agentDevice: EngineResult, invariants: InvariantResult[], ): boolean { + if (maestro.failureKind === 'infrastructure' || agentDevice.failureKind === 'infrastructure') + return false; if (maestro.outcome !== expected.maestro) return false; if (agentDevice.outcome !== expected.agentDevice) return false; const expectedInvariants = expected.invariants ?? []; @@ -147,6 +121,45 @@ export function matchesSignature( return expectedInvariants.every((status, index) => invariants[index]?.status === status); } +function resolveScenarioStatus(input: { + infrastructureFailed: boolean; + declared: boolean; + matchesDeclared: boolean; + misbehaved: boolean; +}): ScenarioReport['status'] { + if (input.infrastructureFailed) return 'infrastructure-failed'; + if (!input.declared) return input.misbehaved ? 'failed' : 'ok'; + if (input.matchesDeclared) return 'known-divergence'; + return input.misbehaved ? 'failed' : 'stale-declaration'; +} + +const FAILED_SCENARIO_STATUSES = new Set([ + 'failed', + 'infrastructure-failed', + 'stale-declaration', +]); + +function evaluateScenarioInvariants( + scenario: DifferentialScenario, + traceRoot: string | undefined, +): InvariantResult[] { + if (!scenario.engineInvariants) return []; + const trace = findTimingTrace(traceRoot); + return evaluateInvariants(trace ? readTrace(trace) : [], scenario.engineInvariants); +} + +function matchesDeclaredDivergence( + scenario: DifferentialScenario, + maestro: EngineResult, + agentDevice: EngineResult, + invariants: InvariantResult[], +): boolean { + const declaration = scenario.knownDivergence; + return declaration + ? matchesSignature(declaration.expected, maestro, agentDevice, invariants) + : false; +} + function runScenario(scenario: DifferentialScenario, options: RunnerOptions): ScenarioReport { const flowPath = path.join(CONFORMANCE_DIR, scenario.flow); const platformArgs = options.platform ? ['--platform', options.platform] : []; @@ -154,7 +167,7 @@ function runScenario(scenario: DifferentialScenario, options: RunnerOptions): Sc const maestro = runEngine('maestro', options.maestroBin, ['test', flowPath, ...platformArgs]); // `--maestro` is required: without it `test` rejects a .yaml flow outright // ("test does not support this file type"). Matches scripts/run-test-app-maestro-suite.mjs. - const agentDevice = runEngine('agent-device', `node ${options.agentDeviceCli}`, [ + const agentDevice = runAgentDeviceEngine(`node ${options.agentDeviceCli}`, [ 'test', flowPath, '--maestro', @@ -164,14 +177,12 @@ function runScenario(scenario: DifferentialScenario, options: RunnerOptions): Sc const outcomeDiverged = maestro.outcome !== scenario.expect || agentDevice.outcome !== scenario.expect; - // Outcome parity cannot see settle ordering or timing; assert engine-side - // invariants over agent-device's own trace where the scenario declares them. - const trace = findTimingTrace(options.traceRoot); - const invariants = scenario.engineInvariants - ? evaluateInvariants(trace ? readTrace(trace) : [], scenario.engineInvariants) - : []; + // Outcome parity cannot see settle ordering or timing; assert engine-side invariants too. + const invariants = evaluateScenarioInvariants(scenario, options.traceRoot); const invariantFailed = invariants.some((result) => result.status !== 'held'); const misbehaved = outcomeDiverged || invariantFailed; + const infrastructureFailed = + maestro.failureKind === 'infrastructure' || agentDevice.failureKind === 'infrastructure'; // A declared divergence is an expected, tracked gap: it keeps the run green so // the oracle is not blocked on the engine bug it just found. But the waiver @@ -181,18 +192,13 @@ function runScenario(scenario: DifferentialScenario, options: RunnerOptions): Sc // the fix PR has to delete it; that is what turns the differential into the // acceptance test for its own findings. const declared = scenario.knownDivergence; - const matchesDeclared = - declared !== undefined && matchesSignature(declared.expected, maestro, agentDevice, invariants); - const status = !declared - ? misbehaved - ? ('failed' as const) - : ('ok' as const) - : matchesDeclared - ? ('known-divergence' as const) - : misbehaved - ? // Diverged, but not the way the waiver describes: not covered. - ('failed' as const) - : ('stale-declaration' as const); + const matchesDeclared = matchesDeclaredDivergence(scenario, maestro, agentDevice, invariants); + const status = resolveScenarioStatus({ + infrastructureFailed, + declared: declared !== undefined, + matchesDeclared, + misbehaved, + }); return { id: scenario.id, @@ -203,7 +209,7 @@ function runScenario(scenario: DifferentialScenario, options: RunnerOptions): Sc invariants, status, ...(declared ? { tracking: declared.tracking } : {}), - failed: status === 'failed' || status === 'stale-declaration', + failed: FAILED_SCENARIO_STATUSES.has(status), }; } @@ -213,59 +219,16 @@ function main(argv: readonly string[]): void { const scenarios = selectScenarios(options.only); if (options.dryRun) { - for (const scenario of scenarios) { - const invariants = scenario.engineInvariants?.length ?? 0; - const declared = scenario.knownDivergence - ? `\tdeclared-divergence=${scenario.knownDivergence.tracking}` - : ''; - console.log( - `${scenario.id}\t${scenario.flow}\texpect=${scenario.expect}\tinvariants=${invariants}${declared}`, - ); - } - const known = scenarios.filter((scenario) => scenario.knownDivergence).length; - console.log(`\n${scenarios.length} scenario(s) validated, ${known} declared divergence(s).`); + printDryRun(scenarios); return; } const reports = scenarios.map((scenario) => runScenario(scenario, options)); - if (options.outDir) { - fs.mkdirSync(options.outDir, { recursive: true }); - fs.writeFileSync( - path.join(options.outDir, 'differential-report.json'), - `${JSON.stringify({ platform: options.platform, reports }, null, 2)}\n`, - ); - } - + writeReports(options.outDir, options.platform, reports); for (const report of reports) { - console.log( - `${report.status.padEnd(17)} ${report.id} maestro=${report.maestro.outcome} agent-device=${report.agentDevice.outcome}`, - ); - for (const result of report.invariants) { - console.log(` invariant ${result.status}: ${result.detail}`); - } - if (report.status === 'known-divergence') { - console.log(` declared divergence, tracked: ${report.tracking}`); - } - if (report.status === 'stale-declaration') { - console.log( - ` passed while declared divergent — remove knownDivergence (${report.tracking}) so this stays enforced`, - ); - } - } - - // Keep declared gaps visible: a green run must still say what it is not proving. - const known = reports.filter((report) => report.status === 'known-divergence'); - if (known.length > 0) { - console.log( - `\n${known.length} declared divergence(s), not enforced: ${known.map((r) => r.id).join(', ')}`, - ); - } - - const failed = reports.filter((report) => report.failed); - if (failed.length > 0) { - console.error(`\n${failed.length} scenario(s) failed: ${failed.map((r) => r.id).join(', ')}`); - process.exitCode = 1; + printScenarioReport(report); } + printRunSummary(reports); } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { diff --git a/packages/maestro/test/conformance/harness.ts b/packages/maestro/test/conformance/harness.ts index 111252100..9377d2b5c 100644 --- a/packages/maestro/test/conformance/harness.ts +++ b/packages/maestro/test/conformance/harness.ts @@ -27,11 +27,15 @@ export { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS }; export function parseMaestroConformanceSource( source: string, sourcePath: string, -): { commands: CanonicalCommand[]; kinds: Set } { +): { appId?: string; commands: CanonicalCommand[]; kinds: Set } { const program = parseMaestroProgram(source, { sourcePath }); const kinds = new Set(); collectCommandKinds(program.commands, kinds); - return { commands: canonicalizeAgentCommands(program), kinds }; + return { + ...(program.config.appId ? { appId: program.config.appId } : {}), + commands: canonicalizeAgentCommands(program), + kinds, + }; } function collectCommandKinds( diff --git a/packages/replay-test/src/internal/session-test-attempt.ts b/packages/replay-test/src/internal/session-test-attempt.ts index 2086bd5a7..e68cb9bfc 100644 --- a/packages/replay-test/src/internal/session-test-attempt.ts +++ b/packages/replay-test/src/internal/session-test-attempt.ts @@ -388,6 +388,9 @@ function buildReplayTestFailedResult( attempts: outcome.attempts, artifactsDir: context.testArtifactsDir, error, + ...(attemptOutcome?.status === 'failed' && attemptOutcome.infrastructure + ? { infrastructure: true as const } + : {}), ...(attemptOutcome?.snapshotDiagnostics ? { snapshotDiagnostics: attemptOutcome.snapshotDiagnostics } : {}), diff --git a/src/cli-schema/cli-help-topics.test.ts b/src/cli-schema/cli-help-topics.test.ts index 0331a9667..8aa8b147f 100644 --- a/src/cli-schema/cli-help-topics.test.ts +++ b/src/cli-schema/cli-help-topics.test.ts @@ -229,6 +229,8 @@ test('usageForCommand resolves scripting help topic', async () => { assert.match(help, /agent-device fill 'id="password"' "\$AD_VAR_PASSWORD" --record-as PASSWORD/); assert.match(help, /published script contain only \$\{PASSWORD\}/); assert.match(help, /Do not record passwords\/tokens without --record-as/); + assert.match(help, /test --json marks a failed test with infrastructure: true/); + assert.match(help, /It remains a failed test/); assert.match(help, /REPLAY_DIVERGENCE with a bounded report/); assert.match(help, /replay --from --plan-digest /); assert.match(help, /resume never re-executes skipped steps/); diff --git a/src/cli-schema/cli-help.ts b/src/cli-schema/cli-help.ts index 267107509..00e480d3c 100644 --- a/src/cli-schema/cli-help.ts +++ b/src/cli-schema/cli-help.ts @@ -205,6 +205,7 @@ Use this for reusable .ad script authoring (save-script), scripted destination g Script paths are the caller's: replay and test resolve and read on the machine running the command, then send the script content (Maestro runFlow includes too) with the request. The same flows therefore run against a local daemon and against a remote one (AGENT_DEVICE_DAEMON_BASE_URL) with no copy step, and a missing script fails immediately, naming the path you typed. --save-script writes on the DAEMON host and is rejected against a remote daemon. + test --json marks a failed test with infrastructure: true only when the owning runtime classified a device, runner, boot, or transport failure. It remains a failed test; consumers may use the tag to distinguish "the oracle did not run" from a behavioral replay divergence without weakening either gate. Reusable open-to-destination scripts: Arm recording on the first open, perform the full journey, verify the destination with a selector-targeted wait, then publish without closing: diff --git a/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts b/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts index e4257b99a..b2e50c3b4 100644 --- a/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts +++ b/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts @@ -41,6 +41,19 @@ test('isReplayInfrastructureFailure accepts replay timeout cleanup races', () => assert.equal(isReplayInfrastructureFailure(response), true); }); +test('isReplayInfrastructureFailure accepts a typed foreign runner owner', () => { + const response: DaemonResponse = { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'Runner is busy', + details: { reason: 'IOS_RUNNER_OWNED_BY_OTHER_DAEMON' }, + }, + }; + + assert.equal(isReplayInfrastructureFailure(response), true); +}); + test('isReplayInfrastructureFailure rejects normal replay failures', () => { const response: DaemonResponse = { ok: false, diff --git a/src/daemon/handlers/__tests__/session-test-suite.test.ts b/src/daemon/handlers/__tests__/session-test-suite.test.ts index 418696a82..9edfc874b 100644 --- a/src/daemon/handlers/__tests__/session-test-suite.test.ts +++ b/src/daemon/handlers/__tests__/session-test-suite.test.ts @@ -101,6 +101,7 @@ test('test does not retry infrastructure startup failures and stops the suite', const tests = data.tests as Array>; expect(tests[0]?.status).toBe('failed'); expect(tests[0]?.attempts).toBe(1); + expect(tests[0]?.infrastructure).toBe(true); }); test('test --fail-fast stops the suite after the first failure and leaves the rest notRun', async () => { diff --git a/src/platforms/__tests__/boot-diagnostics.test.ts b/src/platforms/__tests__/boot-diagnostics.test.ts index 22e94865b..67e80ebb3 100644 --- a/src/platforms/__tests__/boot-diagnostics.test.ts +++ b/src/platforms/__tests__/boot-diagnostics.test.ts @@ -90,6 +90,11 @@ test('a provisioning failure is not treated as retryable infrastructure', () => assert.equal(isInfrastructureBootFailureReason('IOS_RUNNER_DEVICE_NOT_PROVISIONED'), false); }); +test('a live foreign runner owner is typed infrastructure', () => { + assert.equal(isInfrastructureBootFailureReason('IOS_RUNNER_OWNED_BY_OTHER_DAEMON'), true); + assert.match(bootFailureHint('IOS_RUNNER_OWNED_BY_OTHER_DAEMON'), /owning agent-device session/); +}); + test.each([ 'Provisioning profile "Agent Device" has expired.', 'Failed to install embedded profile: signing certificate is not valid.', diff --git a/src/platforms/apple/core/runner/runner-lease.ts b/src/platforms/apple/core/runner/runner-lease.ts index 4b95948f4..db0b46bcf 100644 --- a/src/platforms/apple/core/runner/runner-lease.ts +++ b/src/platforms/apple/core/runner/runner-lease.ts @@ -164,6 +164,7 @@ export async function prepareRunnerLeaseForStartup( ? `iOS runner for ${deviceId} is busy after device lease admission` : `iOS runner for ${deviceId} is already owned by another agent-device daemon`, { + reason: 'IOS_RUNNER_OWNED_BY_OTHER_DAEMON', deviceId, logicalLeaseContext, ownerPid: state.lease.ownerPid, diff --git a/src/platforms/boot-diagnostics.ts b/src/platforms/boot-diagnostics.ts index 7ec77be30..a1b7d4ace 100644 --- a/src/platforms/boot-diagnostics.ts +++ b/src/platforms/boot-diagnostics.ts @@ -3,6 +3,7 @@ import { asAppError } from '@agent-device/kernel/errors'; export type BootFailureReason = | 'IOS_BOOT_TIMEOUT' | 'IOS_RUNNER_CONNECT_TIMEOUT' + | 'IOS_RUNNER_OWNED_BY_OTHER_DAEMON' | 'IOS_RUNNER_DEVICE_NOT_PROVISIONED' | 'IOS_TOOL_MISSING' | 'ANDROID_BOOT_TIMEOUT' @@ -14,6 +15,7 @@ export type BootFailureReason = const INFRASTRUCTURE_BOOT_FAILURE_REASONS = new Set([ 'IOS_BOOT_TIMEOUT', 'IOS_RUNNER_CONNECT_TIMEOUT', + 'IOS_RUNNER_OWNED_BY_OTHER_DAEMON', 'IOS_TOOL_MISSING', 'ANDROID_BOOT_TIMEOUT', 'ADB_TRANSPORT_UNAVAILABLE', @@ -143,6 +145,8 @@ export function bootFailureHint(reason: BootFailureReason): string { return 'Retry simulator boot and inspect simctl bootstatus logs; in CI reduce parallel jobs or use a larger runner.'; case 'IOS_RUNNER_CONNECT_TIMEOUT': return 'Retry runner startup, inspect xcodebuild logs, and verify simulator responsiveness before command execution.'; + case 'IOS_RUNNER_OWNED_BY_OTHER_DAEMON': + return 'Close the owning agent-device session or stop its daemon with retained-runner cleanup before retrying.'; case 'IOS_RUNNER_DEVICE_NOT_PROVISIONED': return 'The XCTest runner cannot be installed on this device: its provisioning profile does not cover it. Register the device with the signing team (Xcode > Settings > Accounts, or add its UDID to the provisioning profile) and retry. Retrying without that will keep failing.'; case 'ANDROID_BOOT_TIMEOUT': From 7982e5554988a766930034072d417650995266c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 24 Aug 2026 11:35:20 +0200 Subject: [PATCH 3/5] fix(ci): stabilize macOS replay cleanup --- .../differential/engine-process.test.ts | 20 ++++- .../differential/engine-process.ts | 47 ++++------- .../differential/invariants.test.ts | 47 +++++------ .../conformance/differential/report-output.ts | 2 +- .../test/conformance/differential/run.ts | 6 +- .../src/internal/session-test-runtime.ts | 2 +- .../session-replay-runtime-failure.test.ts | 2 + .../session-test-infrastructure.test.ts | 32 ++++++++ ...session-replay-runtime-failure-response.ts | 2 +- .../handlers/session-test-infrastructure.ts | 7 +- .../apple/core/__tests__/apps.test.ts | 2 +- src/platforms/apple/os/macos/apps.test.ts | 78 +++++++++++++++++++ src/platforms/apple/os/macos/apps.ts | 26 +++++-- .../replays/macos/01-system-settings.ad | 12 ++- 14 files changed, 205 insertions(+), 80 deletions(-) create mode 100644 src/platforms/apple/os/macos/apps.test.ts diff --git a/packages/maestro/test/conformance/differential/engine-process.test.ts b/packages/maestro/test/conformance/differential/engine-process.test.ts index b45b69454..e8b3f6606 100644 --- a/packages/maestro/test/conformance/differential/engine-process.test.ts +++ b/packages/maestro/test/conformance/differential/engine-process.test.ts @@ -1,6 +1,9 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { test } from 'node:test'; -import { classifyAgentDeviceFailure } from './engine-process.ts'; +import { classifyAgentDeviceFailure, runAgentDeviceEngine } from './engine-process.ts'; test('agent-device JSON distinguishes infrastructure from behavioral failures', () => { const result = (infrastructure?: true) => @@ -20,3 +23,18 @@ test('agent-device JSON distinguishes infrastructure from behavioral failures', assert.equal(classifyAgentDeviceFailure(result()), 'behavioral'); assert.equal(classifyAgentDeviceFailure('not-json'), 'infrastructure'); }); + +test('agent-device execution accepts a CLI path containing spaces', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-engine path-')); + const cliPath = path.join(root, 'agent device.mjs'); + try { + fs.writeFileSync(cliPath, ''); + assert.deepEqual(runAgentDeviceEngine(cliPath, []), { + engine: 'agent-device', + outcome: 'pass', + exitCode: 0, + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/packages/maestro/test/conformance/differential/engine-process.ts b/packages/maestro/test/conformance/differential/engine-process.ts index 429accb46..21a93ba29 100644 --- a/packages/maestro/test/conformance/differential/engine-process.ts +++ b/packages/maestro/test/conformance/differential/engine-process.ts @@ -9,25 +9,10 @@ export type EngineResult = { failureKind?: 'behavioral' | 'infrastructure'; }; -export function runEngine( - engine: EngineResult['engine'], - command: string, - args: string[], -): EngineResult { +export function runMaestroEngine(command: string, args: string[]): EngineResult { const [bin = '', ...rest] = command.split(' ').filter(Boolean); const result = spawnSync(bin, [...rest, ...args], { stdio: 'inherit', cwd: process.cwd() }); - const exitCode = result.status ?? 1; - const infrastructureFailed = result.status === null || result.error !== undefined; - return { - engine, - outcome: exitCode === 0 ? 'pass' : 'fail', - exitCode, - ...(exitCode === 0 - ? {} - : { - failureKind: infrastructureFailed ? ('infrastructure' as const) : ('behavioral' as const), - }), - }; + return buildEngineResult('maestro', result, () => 'behavioral'); } export function classifyAgentDeviceFailure(stdout: string): 'behavioral' | 'infrastructure' { @@ -45,26 +30,28 @@ export function classifyAgentDeviceFailure(stdout: string): 'behavioral' | 'infr } } -export function runAgentDeviceEngine(command: string, args: string[]): EngineResult { - const [bin = '', ...rest] = command.split(' ').filter(Boolean); - const result = spawnSync(bin, [...rest, ...args, '--json'], { +export function runAgentDeviceEngine(cliPath: string, args: string[]): EngineResult { + const result = spawnSync(process.execPath, [cliPath, ...args, '--json'], { cwd: process.cwd(), encoding: 'utf8', }); if (result.stdout) process.stdout.write(result.stdout); if (result.stderr) process.stderr.write(result.stderr); + return buildEngineResult('agent-device', result, () => classifyAgentDeviceFailure(result.stdout)); +} + +function buildEngineResult( + engine: EngineResult['engine'], + result: { status: number | null; error?: Error }, + classifyFailure: () => NonNullable, +): EngineResult { const exitCode = result.status ?? 1; - const infrastructureFailed = result.status === null || result.error !== undefined; + if (exitCode === 0) return { engine, outcome: 'pass', exitCode }; return { - engine: 'agent-device', - outcome: exitCode === 0 ? 'pass' : 'fail', + engine, + outcome: 'fail', exitCode, - ...(exitCode === 0 - ? {} - : { - failureKind: infrastructureFailed - ? ('infrastructure' as const) - : classifyAgentDeviceFailure(result.stdout), - }), + failureKind: + result.status === null || result.error !== undefined ? 'infrastructure' : classifyFailure(), }; } diff --git a/packages/maestro/test/conformance/differential/invariants.test.ts b/packages/maestro/test/conformance/differential/invariants.test.ts index 1a4ab96ee..22c04b0d0 100644 --- a/packages/maestro/test/conformance/differential/invariants.test.ts +++ b/packages/maestro/test/conformance/differential/invariants.test.ts @@ -6,11 +6,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { test } from 'node:test'; -import { - type CanonicalCommand, - MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, - parseMaestroConformanceSource, -} from '../harness.ts'; +import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, parseMaestroConformanceSource } from '../harness.ts'; import { DIFFERENTIAL_SCENARIOS } from './scenarios.ts'; import { type Invariant, evaluateInvariant, readTrace } from './invariants.ts'; @@ -95,31 +91,28 @@ test('bug class 4 has a machine-checkable invariant, not just outcome parity', ( }); const SETTLE_FLOW_PATH = path.join(import.meta.dirname, 'flows/settle-after-tap.yaml'); -const EXPECTED_SETTLE_COMMANDS: CanonicalCommand[] = [ - { kind: 'launchApp', appId: 'com.callstack.agentdevicelab', clearState: true }, - { - kind: 'assert', - mode: 'visible', - timed: false, - selector: { text: 'Agent Device Tester' }, - }, - { - kind: 'tap', - longPress: false, - repeat: 1, - target: { selector: { text: 'Settings' } }, - }, - { - kind: 'assert', - mode: 'visible', - timed: false, - selector: { id: 'open-inert-surface' }, - }, -]; function assertSettleFlowSemantics(source: string): void { const parsed = parseMaestroConformanceSource(source, SETTLE_FLOW_PATH); - assert.deepEqual(parsed.commands, EXPECTED_SETTLE_COMMANDS); + assert.equal( + parsed.commands.some( + (command) => command.kind === 'scroll' || command.kind === 'scrollUntilVisible', + ), + false, + ); + assert.deepEqual( + parsed.commands.filter((command) => command.kind === 'tap'), + [{ kind: 'tap', longPress: false, repeat: 1, target: { selector: { text: 'Settings' } } }], + ); + assert.equal( + parsed.commands.some( + (command) => + command.kind === 'assert' && + command.mode === 'visible' && + command.selector?.id === 'open-inert-surface', + ), + true, + ); } test('the settle detector reaches its tap without an unrelated setup command', () => { diff --git a/packages/maestro/test/conformance/differential/report-output.ts b/packages/maestro/test/conformance/differential/report-output.ts index f750e20ab..c4d0956f9 100644 --- a/packages/maestro/test/conformance/differential/report-output.ts +++ b/packages/maestro/test/conformance/differential/report-output.ts @@ -18,7 +18,7 @@ export type ScenarioReport = { export function printScenarioReport(report: ScenarioReport): void { console.log( - `${report.status.padEnd(17)} ${report.id} maestro=${report.maestro.outcome} agent-device=${report.agentDevice.outcome}`, + `${report.status.padEnd('infrastructure-failed'.length)} ${report.id} maestro=${report.maestro.outcome} agent-device=${report.agentDevice.outcome}`, ); for (const result of report.invariants) { console.log(` invariant ${result.status}: ${result.detail}`); diff --git a/packages/maestro/test/conformance/differential/run.ts b/packages/maestro/test/conformance/differential/run.ts index 92f7b70b6..4654a1434 100644 --- a/packages/maestro/test/conformance/differential/run.ts +++ b/packages/maestro/test/conformance/differential/run.ts @@ -18,7 +18,7 @@ import { type DivergenceSignature, } from './scenarios.ts'; import { type InvariantResult, evaluateInvariants, readTrace } from './invariants.ts'; -import { type EngineResult, runAgentDeviceEngine, runEngine } from './engine-process.ts'; +import { type EngineResult, runAgentDeviceEngine, runMaestroEngine } from './engine-process.ts'; import { printDryRun, printRunSummary, @@ -164,10 +164,10 @@ function runScenario(scenario: DifferentialScenario, options: RunnerOptions): Sc const flowPath = path.join(CONFORMANCE_DIR, scenario.flow); const platformArgs = options.platform ? ['--platform', options.platform] : []; - const maestro = runEngine('maestro', options.maestroBin, ['test', flowPath, ...platformArgs]); + const maestro = runMaestroEngine(options.maestroBin, ['test', flowPath, ...platformArgs]); // `--maestro` is required: without it `test` rejects a .yaml flow outright // ("test does not support this file type"). Matches scripts/run-test-app-maestro-suite.mjs. - const agentDevice = runAgentDeviceEngine(`node ${options.agentDeviceCli}`, [ + const agentDevice = runAgentDeviceEngine(options.agentDeviceCli, [ 'test', flowPath, '--maestro', diff --git a/packages/replay-test/src/internal/session-test-runtime.ts b/packages/replay-test/src/internal/session-test-runtime.ts index ecdf22b2f..49436a637 100644 --- a/packages/replay-test/src/internal/session-test-runtime.ts +++ b/packages/replay-test/src/internal/session-test-runtime.ts @@ -4,7 +4,6 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { normalizeError } from '@agent-device/kernel/errors'; import { replayTestAttemptFailure, - type ReplayTestAttemptFailed, type ReplayTestAttemptOutcome, type ReplayTestAttemptStepSink, type ReplayTestEmitDiagnostic, @@ -174,6 +173,7 @@ export async function runReplayTestAttempt( }); } catch (error) { const appErr = normalizeError(error); + outcome = markReplayTestCleanupFailed(outcome, appErr); appendReplayTestTimingEvent(tracePath, { type: 'replay_test_cleanup_stop', ts: new Date().toISOString(), diff --git a/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts b/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts index 522457505..383fc947e 100644 --- a/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts @@ -181,6 +181,7 @@ test('a normalized nested failure preserves typed recovery signals on REPLAY_DIV message: 'The device is temporarily leased.', retriable: true, supportedOn: 'ios', + details: { recovery: 'runner_recycle_budget_exhausted' }, }, }), }); @@ -190,6 +191,7 @@ test('a normalized nested failure preserves typed recovery signals on REPLAY_DIV expect(response.error.code).toBe('REPLAY_DIVERGENCE'); expect(response.error.retriable).toBe(true); expect(response.error.supportedOn).toBe('ios'); + expect(response.error.details?.recovery).toBe('runner_recycle_budget_exhausted'); }); test('a failing replay step captures an available screen digest with blessed refs', async () => { diff --git a/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts b/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts index b2e50c3b4..27d0db200 100644 --- a/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts +++ b/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts @@ -2,6 +2,7 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { isReplayInfrastructureFailure } from '../session-test-infrastructure.ts'; import type { DaemonResponse } from '../../types.ts'; +import type { ReplaySuiteTestResult } from '@agent-device/contracts/replay'; test('isReplayInfrastructureFailure accepts shared boot diagnostic reasons', () => { const response: DaemonResponse = { @@ -41,6 +42,23 @@ test('isReplayInfrastructureFailure accepts replay timeout cleanup races', () => assert.equal(isReplayInfrastructureFailure(response), true); }); +test('isReplayInfrastructureFailure accepts the owning runtime verdict', () => { + const result: ReplaySuiteTestResult = { + file: 'cleanup.ad', + session: 'default:test:cleanup', + status: 'failed', + durationMs: 1, + attempts: 1, + error: { + code: 'COMMAND_FAILED', + message: 'Replay test cleanup failed', + }, + infrastructure: true, + }; + + assert.equal(isReplayInfrastructureFailure(result), true); +}); + test('isReplayInfrastructureFailure accepts a typed foreign runner owner', () => { const response: DaemonResponse = { ok: false, @@ -54,6 +72,20 @@ test('isReplayInfrastructureFailure accepts a typed foreign runner owner', () => assert.equal(isReplayInfrastructureFailure(response), true); }); +test('isReplayInfrastructureFailure accepts typed runner recycle exhaustion', () => { + const response: DaemonResponse = { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: + 'iOS runner was already restarted during this request and "snapshot" still failed, so agent-device stopped instead of paying for another runner boot.', + details: { recovery: 'runner_recycle_budget_exhausted' }, + }, + }; + + assert.equal(isReplayInfrastructureFailure(response), true); +}); + test('isReplayInfrastructureFailure rejects normal replay failures', () => { const response: DaemonResponse = { ok: false, diff --git a/src/daemon/handlers/session-replay-runtime-failure-response.ts b/src/daemon/handlers/session-replay-runtime-failure-response.ts index 48825651f..10fc5857c 100644 --- a/src/daemon/handlers/session-replay-runtime-failure-response.ts +++ b/src/daemon/handlers/session-replay-runtime-failure-response.ts @@ -111,7 +111,7 @@ function readStringDetail( return typeof value === 'string' && value.length > 0 ? value : undefined; } -const SAFE_CAUSE_DETAIL_KEYS = ['reason', 'retriable', 'supportedOn'] as const; +const SAFE_CAUSE_DETAIL_KEYS = ['reason', 'recovery', 'retriable', 'supportedOn'] as const; function pickSafeCauseDetails( details: Record | undefined, diff --git a/src/daemon/handlers/session-test-infrastructure.ts b/src/daemon/handlers/session-test-infrastructure.ts index 736b032cb..1062440a7 100644 --- a/src/daemon/handlers/session-test-infrastructure.ts +++ b/src/daemon/handlers/session-test-infrastructure.ts @@ -16,10 +16,12 @@ type ReplayFailureError = Extract['error']; export function isReplayInfrastructureFailure( result: DaemonResponse | ReplaySuiteTestResult, ): boolean { + if (!('ok' in result) && result.status === 'failed' && result.infrastructure === true) + return true; const error = readReplayFailureError(result); if (!error) return false; return ( - hasInfrastructureFailureReason(error.details) || + hasInfrastructureFailureDetails(error.details) || hasInfrastructureFailureMessage(error.code, error.message) ); } @@ -31,7 +33,8 @@ function readReplayFailureError( return result.status === 'failed' ? result.error : null; } -function hasInfrastructureFailureReason(details: Record | undefined): boolean { +function hasInfrastructureFailureDetails(details: Record | undefined): boolean { + if (details?.recovery === 'runner_recycle_budget_exhausted') return true; const reason = typeof details?.reason === 'string' ? details.reason : ''; if (reason === 'timeout_cleanup_pending') return true; return reason ? isInfrastructureBootFailureReason(reason) : false; diff --git a/src/platforms/apple/core/__tests__/apps.test.ts b/src/platforms/apple/core/__tests__/apps.test.ts index 69633f187..30a00fe67 100644 --- a/src/platforms/apple/core/__tests__/apps.test.ts +++ b/src/platforms/apple/core/__tests__/apps.test.ts @@ -505,7 +505,7 @@ test('closeIosApp on macOS uses helper quit for bundle identifiers', async () => '#!/bin/sh', 'printf "%s\\n" "$@" > "$AGENT_DEVICE_TEST_ARGS_FILE"', "cat <<'JSON'", - '{"ok":true,"data":{"bundleId":"com.example.foobar","running":true,"terminated":true,"forceTerminated":false}}', + '{"ok":true,"data":{"bundleId":"com.example.foobar","running":false,"terminated":false,"forceTerminated":false}}', 'JSON', '', ].join('\n'), diff --git a/src/platforms/apple/os/macos/apps.test.ts b/src/platforms/apple/os/macos/apps.test.ts new file mode 100644 index 000000000..6a9361d57 --- /dev/null +++ b/src/platforms/apple/os/macos/apps.test.ts @@ -0,0 +1,78 @@ +import { afterEach, expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; + +vi.mock('node:timers/promises', () => ({ setTimeout: vi.fn(async () => undefined) })); +vi.mock('./helper.ts', () => ({ quitMacOsApp: vi.fn() })); + +import { quitMacOsApp } from './helper.ts'; +import { closeMacOsApp } from './apps.ts'; + +const MACOS_DEVICE: DeviceInfo = { + platform: 'apple', + appleOs: 'macos', + id: 'host-macos-local', + name: 'Host Mac', + kind: 'device', + target: 'desktop', + booted: true, +}; + +const mockQuitMacOsApp = vi.mocked(quitMacOsApp); + +afterEach(() => { + mockQuitMacOsApp.mockReset(); +}); + +test('closeMacOsApp waits until an accepted termination has actually completed', async () => { + mockQuitMacOsApp + .mockResolvedValueOnce({ + bundleId: 'com.apple.systempreferences', + running: true, + terminated: true, + forceTerminated: false, + }) + .mockResolvedValueOnce({ + bundleId: 'com.apple.systempreferences', + running: true, + terminated: false, + forceTerminated: false, + }) + .mockResolvedValueOnce({ + bundleId: 'com.apple.systempreferences', + running: false, + terminated: false, + forceTerminated: false, + }); + + await closeMacOsApp(MACOS_DEVICE, 'com.apple.systempreferences'); + + expect(mockQuitMacOsApp).toHaveBeenCalledTimes(3); +}); + +test('closeMacOsApp fails immediately when neither termination request is accepted', async () => { + mockQuitMacOsApp.mockResolvedValue({ + bundleId: 'com.apple.systempreferences', + running: true, + terminated: false, + forceTerminated: false, + }); + + await expect(closeMacOsApp(MACOS_DEVICE, 'com.apple.systempreferences')).rejects.toThrow( + 'Failed to close macOS app com.apple.systempreferences', + ); + expect(mockQuitMacOsApp).toHaveBeenCalledTimes(1); +}); + +test('closeMacOsApp bounds termination confirmation', async () => { + mockQuitMacOsApp.mockResolvedValue({ + bundleId: 'com.apple.systempreferences', + running: true, + terminated: true, + forceTerminated: false, + }); + + await expect(closeMacOsApp(MACOS_DEVICE, 'com.apple.systempreferences')).rejects.toMatchObject({ + details: { reason: 'MACOS_APP_TERMINATION_TIMEOUT', attempts: 20 }, + }); + expect(mockQuitMacOsApp).toHaveBeenCalledTimes(20); +}); diff --git a/src/platforms/apple/os/macos/apps.ts b/src/platforms/apple/os/macos/apps.ts index 8746296a4..78809758d 100644 --- a/src/platforms/apple/os/macos/apps.ts +++ b/src/platforms/apple/os/macos/apps.ts @@ -2,6 +2,7 @@ import type { AppsFilter } from '@agent-device/contracts/device'; import { isDeepLinkTarget } from '@agent-device/contracts/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; +import { setTimeout as sleep } from 'node:timers/promises'; import { parseAppearanceAction } from '../../../appearance.ts'; import { createAppResolutionCache, @@ -16,6 +17,8 @@ const MACOS_ALIASES: Record = { }; const MACOS_BUNDLE_ID_PATTERN = /^[a-z0-9-]+(?:\.[a-z0-9-]+)+$/; +const MACOS_APP_TERMINATION_POLL_MS = 100; +const MACOS_APP_TERMINATION_MAX_ATTEMPTS = 20; // macOS currently has no install/uninstall flow; add cache invalidation if that changes. const MACOS_APP_RESOLUTION_CACHE_SCOPE = { @@ -81,13 +84,24 @@ export async function openMacOsApp( export async function closeMacOsApp(_device: DeviceInfo, app: string): Promise { const bundleId = await resolveMacOsApp(app); - const result = await quitMacOsApp(bundleId); - if (!result.running || result.terminated || result.forceTerminated) return; - throw new AppError('COMMAND_FAILED', `Failed to close macOS app ${app}`, { + const request = await quitMacOsApp(bundleId); + if (!request.running) return; + if (!request.terminated && !request.forceTerminated) { + throw new AppError('COMMAND_FAILED', `Failed to close macOS app ${app}`, { + bundleId, + running: request.running, + terminated: request.terminated, + forceTerminated: request.forceTerminated, + }); + } + for (let attempt = 1; attempt < MACOS_APP_TERMINATION_MAX_ATTEMPTS; attempt += 1) { + await sleep(MACOS_APP_TERMINATION_POLL_MS); + if (!(await quitMacOsApp(bundleId)).running) return; + } + throw new AppError('COMMAND_FAILED', `Timed out waiting for macOS app ${app} to close`, { + reason: 'MACOS_APP_TERMINATION_TIMEOUT', bundleId, - running: result.running, - terminated: result.terminated, - forceTerminated: result.forceTerminated, + attempts: MACOS_APP_TERMINATION_MAX_ATTEMPTS, }); } diff --git a/test/integration/replays/macos/01-system-settings.ad b/test/integration/replays/macos/01-system-settings.ad index d43f79093..bb8264640 100644 --- a/test/integration/replays/macos/01-system-settings.ad +++ b/test/integration/replays/macos/01-system-settings.ad @@ -4,12 +4,10 @@ open "System Settings" --relaunch screenshot "./test/screenshots/replays/macos-system-settings.png" appstate snapshot -i -wait "role=button label=About" 5000 -snapshot -i -is exists "role=button label=About" -click "role=button label=About" +wait "label=About" 5000 +# Keep one predicate as the macOS live-coverage owner for `is`; the wait makes it deterministic. +is exists "label=About" +click "label=About" wait "label=\"System Report...\" || label=\"Serial number\" || label=Chip || label=Processor || label=macOS" 5000 -snapshot -i back -wait "role=button label=About" 5000 -is exists "role=button label=About" +wait "label=About" 5000 From 0eb4414880e63055b4c771600d875e884613cad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 24 Aug 2026 17:04:28 +0200 Subject: [PATCH 4/5] fix(ci): close nightly review gaps --- .../differential/engine-process.ts | 4 +- .../test/conformance/differential/run.test.ts | 54 +++++++++++++++---- .../test/conformance/differential/run.ts | 5 +- .../src/internal/session-test-runtime.ts | 2 +- .../replays/macos/01-system-settings.ad | 11 ++-- 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/packages/maestro/test/conformance/differential/engine-process.ts b/packages/maestro/test/conformance/differential/engine-process.ts index 21a93ba29..75b265cb1 100644 --- a/packages/maestro/test/conformance/differential/engine-process.ts +++ b/packages/maestro/test/conformance/differential/engine-process.ts @@ -12,7 +12,9 @@ export type EngineResult = { export function runMaestroEngine(command: string, args: string[]): EngineResult { const [bin = '', ...rest] = command.split(' ').filter(Boolean); const result = spawnSync(bin, [...rest, ...args], { stdio: 'inherit', cwd: process.cwd() }); - return buildEngineResult('maestro', result, () => 'behavioral'); + // Maestro does not expose typed failure provenance. Until it does, a non-zero exit cannot + // safely satisfy a behavioral divergence waiver. + return buildEngineResult('maestro', result, () => 'infrastructure'); } export function classifyAgentDeviceFailure(stdout: string): 'behavioral' | 'infrastructure' { diff --git a/packages/maestro/test/conformance/differential/run.test.ts b/packages/maestro/test/conformance/differential/run.test.ts index a2f932ea2..2c1b2e5c1 100644 --- a/packages/maestro/test/conformance/differential/run.test.ts +++ b/packages/maestro/test/conformance/differential/run.test.ts @@ -3,6 +3,7 @@ // conformance-differential workflow. import assert from 'node:assert/strict'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, test } from 'node:test'; @@ -12,7 +13,13 @@ import { type DivergenceSignature, } from './scenarios.ts'; import { parseMaestroConformanceSource } from '../harness.ts'; -import { matchesSignature, parseRunnerArgs, selectScenarios, validateScenarios } from './run.ts'; +import { + matchesSignature, + parseRunnerArgs, + runScenario, + selectScenarios, + validateScenarios, +} from './run.ts'; const CONFORMANCE_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); @@ -121,15 +128,6 @@ describe('knownDivergence signature matching', () => { ); }); - test('infrastructure failure is NOT covered by a behavioral waiver', () => { - assert.equal( - matchesSignature(sig, engine('pass'), { ...engine('fail'), failureKind: 'infrastructure' }, [ - inv('no-data'), - ]), - false, - ); - }); - test('every declaration states its expected signature', () => { for (const scenario of DIFFERENTIAL_SCENARIOS) { const declared = scenario.knownDivergence; @@ -152,6 +150,42 @@ describe('knownDivergence signature matching', () => { }); }); +test('an ordinary Maestro process failure cannot satisfy a behavioral waiver', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-failure-')); + const maestroCli = path.join(root, 'maestro.mjs'); + const agentDeviceCli = path.join(root, 'agent-device.mjs'); + try { + fs.writeFileSync(maestroCli, 'process.exit(1);\n'); + fs.writeFileSync(agentDeviceCli, 'process.exit(0);\n'); + + const report = runScenario( + { + id: 'maestro-infrastructure-failure', + flow: 'differential/flows/settle-after-tap.yaml', + comparesAcrossEngines: 'test fixture', + expect: 'pass', + divergenceMeans: 'test fixture', + knownDivergence: { + reason: 'A behavioral Maestro failure is temporarily accepted for this test fixture.', + tracking: 'https://github.com/callstack/agent-device/issues/1', + expected: { maestro: 'fail', agentDevice: 'pass' }, + }, + }, + { + dryRun: false, + maestroBin: `${process.execPath} ${maestroCli}`, + agentDeviceCli, + }, + ); + + assert.equal(report.maestro.failureKind, 'infrastructure'); + assert.equal(report.status, 'infrastructure-failed'); + assert.equal(report.failed, true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + test('every device flow targets the fixture app the workflow installs', () => { for (const scenario of DIFFERENTIAL_SCENARIOS) { const flowPath = path.join(CONFORMANCE_DIR, scenario.flow); diff --git a/packages/maestro/test/conformance/differential/run.ts b/packages/maestro/test/conformance/differential/run.ts index 4654a1434..286db1da2 100644 --- a/packages/maestro/test/conformance/differential/run.ts +++ b/packages/maestro/test/conformance/differential/run.ts @@ -160,7 +160,10 @@ function matchesDeclaredDivergence( : false; } -function runScenario(scenario: DifferentialScenario, options: RunnerOptions): ScenarioReport { +export function runScenario( + scenario: DifferentialScenario, + options: RunnerOptions, +): ScenarioReport { const flowPath = path.join(CONFORMANCE_DIR, scenario.flow); const platformArgs = options.platform ? ['--platform', options.platform] : []; diff --git a/packages/replay-test/src/internal/session-test-runtime.ts b/packages/replay-test/src/internal/session-test-runtime.ts index 49436a637..ecdf22b2f 100644 --- a/packages/replay-test/src/internal/session-test-runtime.ts +++ b/packages/replay-test/src/internal/session-test-runtime.ts @@ -4,6 +4,7 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { normalizeError } from '@agent-device/kernel/errors'; import { replayTestAttemptFailure, + type ReplayTestAttemptFailed, type ReplayTestAttemptOutcome, type ReplayTestAttemptStepSink, type ReplayTestEmitDiagnostic, @@ -173,7 +174,6 @@ export async function runReplayTestAttempt( }); } catch (error) { const appErr = normalizeError(error); - outcome = markReplayTestCleanupFailed(outcome, appErr); appendReplayTestTimingEvent(tracePath, { type: 'replay_test_cleanup_stop', ts: new Date().toISOString(), diff --git a/test/integration/replays/macos/01-system-settings.ad b/test/integration/replays/macos/01-system-settings.ad index bb8264640..eab4b1041 100644 --- a/test/integration/replays/macos/01-system-settings.ad +++ b/test/integration/replays/macos/01-system-settings.ad @@ -4,10 +4,11 @@ open "System Settings" --relaunch screenshot "./test/screenshots/replays/macos-system-settings.png" appstate snapshot -i -wait "label=About" 5000 +wait "role=button label=About" 5000 # Keep one predicate as the macOS live-coverage owner for `is`; the wait makes it deterministic. -is exists "label=About" -click "label=About" -wait "label=\"System Report...\" || label=\"Serial number\" || label=Chip || label=Processor || label=macOS" 5000 +is exists "role=button label=About" +click "role=button label=About" +# Leave enough time for the macOS runner to recycle after the interaction. +wait "label=\"System Report...\" || label=\"Serial number\" || label=Chip || label=Processor || label=macOS" 45000 back -wait "label=About" 5000 +wait "role=button label=About" 5000 From 37d7cfe02957ce6c3fa70d75cf595efa68bb4404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 24 Aug 2026 18:12:41 +0200 Subject: [PATCH 5/5] fix(ci): classify device claims as infrastructure --- src/daemon/device-claim-conflict.ts | 19 +++- .../session-test-infrastructure.test.ts | 25 ++++ .../session-test-suite-infrastructure.test.ts | 107 ++++++++++++++++++ .../__tests__/session-test-suite.fixtures.ts | 16 +++ .../__tests__/session-test-suite.test.ts | 15 +-- .../handlers/session-test-infrastructure.ts | 2 + 6 files changed, 168 insertions(+), 16 deletions(-) create mode 100644 src/daemon/handlers/__tests__/session-test-suite-infrastructure.test.ts create mode 100644 src/daemon/handlers/__tests__/session-test-suite.fixtures.ts diff --git a/src/daemon/device-claim-conflict.ts b/src/daemon/device-claim-conflict.ts index 24efe8cb5..e751c966f 100644 --- a/src/daemon/device-claim-conflict.ts +++ b/src/daemon/device-claim-conflict.ts @@ -13,6 +13,21 @@ import { import type { DaemonResponse } from './types.ts'; import { errorResponse } from './handlers/response.ts'; +export type DeviceClaimConflictReason = + | 'DEVICE_CLAIM_LIVE_OWNER' + | 'DEVICE_CLAIM_RECOVERY_PENDING' + | 'DEVICE_CLAIM_OWNER_UNCERTAIN'; + +const DEVICE_CLAIM_CONFLICT_REASONS = new Set([ + 'DEVICE_CLAIM_LIVE_OWNER', + 'DEVICE_CLAIM_RECOVERY_PENDING', + 'DEVICE_CLAIM_OWNER_UNCERTAIN', +]); + +export function isDeviceClaimConflictReason(value: unknown): value is DeviceClaimConflictReason { + return DEVICE_CLAIM_CONFLICT_REASONS.has(value as DeviceClaimConflictReason); +} + export function buildDeviceClaimInspectionCommand( device: DeviceInfo, conflict: Pick, @@ -80,9 +95,7 @@ export function buildDeviceClaimConflictError( return errorResponse(error.code, error.message, details, { hint, retriable }); } -function conflictReason( - classification: DeviceClaimClassification, -): 'DEVICE_CLAIM_LIVE_OWNER' | 'DEVICE_CLAIM_RECOVERY_PENDING' | 'DEVICE_CLAIM_OWNER_UNCERTAIN' { +function conflictReason(classification: DeviceClaimClassification): DeviceClaimConflictReason { switch (classification) { case 'live': return 'DEVICE_CLAIM_LIVE_OWNER'; diff --git a/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts b/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts index 27d0db200..017cb9b41 100644 --- a/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts +++ b/src/daemon/handlers/__tests__/session-test-infrastructure.test.ts @@ -86,6 +86,31 @@ test('isReplayInfrastructureFailure accepts typed runner recycle exhaustion', () assert.equal(isReplayInfrastructureFailure(response), true); }); +test('isReplayInfrastructureFailure does not infer a device claim from message text', () => { + const response: DaemonResponse = { + ok: false, + error: { + code: 'REPLAY_DIVERGENCE', + message: 'macOS device host-macos-local is owned by another session.', + }, + }; + + assert.equal(isReplayInfrastructureFailure(response), false); +}); + +test('isReplayInfrastructureFailure keeps untyped DEVICE_IN_USE failures behavioral', () => { + const response: DaemonResponse = { + ok: false, + error: { + code: 'DEVICE_IN_USE', + message: 'The requested device is busy with another session.', + retriable: true, + }, + }; + + assert.equal(isReplayInfrastructureFailure(response), false); +}); + test('isReplayInfrastructureFailure rejects normal replay failures', () => { const response: DaemonResponse = { ok: false, diff --git a/src/daemon/handlers/__tests__/session-test-suite-infrastructure.test.ts b/src/daemon/handlers/__tests__/session-test-suite-infrastructure.test.ts new file mode 100644 index 000000000..ab47d9e6f --- /dev/null +++ b/src/daemon/handlers/__tests__/session-test-suite-infrastructure.test.ts @@ -0,0 +1,107 @@ +import { expect, test, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { mkdtempForTestSync } from '../../../__tests__/test-utils/tmp-dir.ts'; +import type { DaemonResponse } from '../../types.ts'; +import { handleSessionCommands } from './session-command-harness.ts'; +import { expectOkData, makeSessionStore } from './session-test-suite.fixtures.ts'; + +vi.mock('../snapshot-interactor-capture.ts', () => ({ + captureSnapshotWithInteractor: vi.fn(async () => { + throw new Error('no device runner available in this test'); + }), +})); + +test('test --json marks a typed live device claim as infrastructure without retrying', async () => { + const sessionStore = makeSessionStore(); + const root = mkdtempForTestSync('agent-device-test-suite-live-device-claim-'); + fs.writeFileSync(path.join(root, '01-claim.ad'), 'context platform=macos\nopen "Demo"\n'); + + let attempts = 0; + const response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-live-device-claim' }, + flags: { retries: 3 }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async () => { + attempts += 1; + return { + ok: false, + error: { + code: 'DEVICE_IN_USE', + message: 'macOS device host-macos-local is owned by another session.', + details: { reason: 'DEVICE_CLAIM_LIVE_OWNER' }, + }, + }; + }, + }); + + const json = JSON.parse(JSON.stringify(response)) as DaemonResponse; + const data = expectOkData(json); + expect((data.tests as Array>)[0]).toMatchObject({ + status: 'failed', + attempts: 1, + infrastructure: true, + error: { + code: 'REPLAY_DIVERGENCE', + details: { reason: 'DEVICE_CLAIM_LIVE_OWNER' }, + }, + }); + expect(attempts).toBe(1); + expect(data.executed).toBe(1); + expect(data.failed).toBe(1); +}); + +test('test --json retries DEVICE_IN_USE without typed device-claim provenance', async () => { + const sessionStore = makeSessionStore(); + const root = mkdtempForTestSync('agent-device-test-suite-session-busy-'); + fs.writeFileSync(path.join(root, '01-busy.ad'), 'context platform=macos\nopen "Demo"\n'); + + let attempts = 0; + const response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-session-busy' }, + flags: { retries: 3 }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async () => { + attempts += 1; + return { + ok: false, + error: { + code: 'DEVICE_IN_USE', + message: 'The requested device is busy with another session.', + retriable: true, + }, + }; + }, + }); + + const json = JSON.parse(JSON.stringify(response)) as DaemonResponse; + const data = expectOkData(json); + const result = (data.tests as Array>)[0]; + expect(result).toMatchObject({ + status: 'failed', + attempts: 4, + error: { + code: 'REPLAY_DIVERGENCE', + retriable: true, + }, + }); + expect(result).not.toHaveProperty('infrastructure'); + expect(attempts).toBe(4); +}); diff --git a/src/daemon/handlers/__tests__/session-test-suite.fixtures.ts b/src/daemon/handlers/__tests__/session-test-suite.fixtures.ts new file mode 100644 index 000000000..96bf3382e --- /dev/null +++ b/src/daemon/handlers/__tests__/session-test-suite.fixtures.ts @@ -0,0 +1,16 @@ +import { expect } from 'vitest'; +import path from 'node:path'; +import { mkdtempForTestSync } from '../../../__tests__/test-utils/tmp-dir.ts'; +import { SessionStore } from '../../session-store.ts'; +import type { DaemonResponse, DaemonResponseData } from '../../types.ts'; + +export function makeSessionStore(): SessionStore { + const root = mkdtempForTestSync('agent-device-session-test-suite-'); + return new SessionStore(path.join(root, 'sessions')); +} + +export function expectOkData(response: DaemonResponse | null | undefined): DaemonResponseData { + expect(response?.ok, JSON.stringify(response)).toBeTruthy(); + if (!response || !response.ok) throw new Error('Expected successful daemon response.'); + return response.data ?? {}; +} diff --git a/src/daemon/handlers/__tests__/session-test-suite.test.ts b/src/daemon/handlers/__tests__/session-test-suite.test.ts index 9edfc874b..b3150faf3 100644 --- a/src/daemon/handlers/__tests__/session-test-suite.test.ts +++ b/src/daemon/handlers/__tests__/session-test-suite.test.ts @@ -18,8 +18,8 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { handleSessionCommands, mockInspectDeviceRuntimeFacts } from './session-command-harness.ts'; -import { SessionStore } from '../../session-store.ts'; -import type { DaemonRequest, DaemonResponse, DaemonResponseData } from '../../types.ts'; +import type { DaemonRequest } from '../../types.ts'; +import { expectOkData, makeSessionStore } from './session-test-suite.fixtures.ts'; import { withRequestProgressSink } from '../../../request/progress.ts'; import { clearRequestCanceled, @@ -34,17 +34,6 @@ import { makeMacOsSession, } from '../../../__tests__/test-utils/session-factories.ts'; -function makeSessionStore(): SessionStore { - const root = mkdtempForTestSync('agent-device-session-test-suite-'); - return new SessionStore(path.join(root, 'sessions')); -} - -function expectOkData(response: DaemonResponse | null | undefined): DaemonResponseData { - expect(response?.ok, JSON.stringify(response)).toBeTruthy(); - if (!response || !response.ok) throw new Error('Expected successful daemon response.'); - return response.data ?? {}; -} - const ANDROID_ONE: DeviceInfo = { platform: 'android', id: 'emulator-5554', diff --git a/src/daemon/handlers/session-test-infrastructure.ts b/src/daemon/handlers/session-test-infrastructure.ts index 1062440a7..e73a82ca7 100644 --- a/src/daemon/handlers/session-test-infrastructure.ts +++ b/src/daemon/handlers/session-test-infrastructure.ts @@ -1,6 +1,7 @@ import { isInfrastructureBootFailureReason } from '../../platforms/boot-diagnostics.ts'; import type { DaemonResponse } from '../types.ts'; import type { ReplaySuiteTestResult } from '@agent-device/contracts/replay'; +import { isDeviceClaimConflictReason } from '../device-claim-conflict.ts'; const REPLAY_INFRASTRUCTURE_FAILURE_MESSAGE_PATTERNS = [ 'failed to start daemon', @@ -37,6 +38,7 @@ function hasInfrastructureFailureDetails(details: Record | unde if (details?.recovery === 'runner_recycle_budget_exhausted') return true; const reason = typeof details?.reason === 'string' ? details.reason : ''; if (reason === 'timeout_cleanup_pending') return true; + if (isDeviceClaimConflictReason(reason)) return true; return reason ? isInfrastructureBootFailureReason(reason) : false; }