fix(ci): repair nightly XCTest and conformance lanes

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift

@@ -448,7 +448,11 @@ extension RunnerTests {
     // Divergence stops the count: the app transformed the input, and the walk must not
     // resume matching after the first differing character.
     XCTAssertEqual(Self.commonPrefixLength("hx", "hardware-keyboard"), 1)
-    XCTAssertEqual(Self.commonPrefixLength("hardware-keyboarx", "hardware-keyboard"), 15)
+    let sharedPrefix = "hardware-keyboar"
+    XCTAssertEqual(
+      Self.commonPrefixLength("\(sharedPrefix)x", "\(sharedPrefix)d"),
+      sharedPrefix.count
+    )
   }
 
   func testCommitCadenceLogLineEmitsLengthsOnlyNeverContents() {

package.json

@@ -113,7 +113,7 @@
     "release:publish": "pnpm release:prepare && npm publish --ignore-scripts .tmp/release/*.tgz",
     "ad": "node bin/agent-device.mjs",
     "bench:help-conformance": "node scripts/help-conformance-bench.mjs",
-    "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts",
+    "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts",
     "maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs",
     "maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts",
     "size": "node scripts/size-report.mjs",

packages/contracts/src/replay.ts

@@ -108,6 +108,8 @@ export type ReplaySuiteTestFailed = {
   attempts: number;
   artifactsDir?: string;
   error: DaemonError;
+  /** Present when the owning runtime classified the failure as device/runner infrastructure. */
+  infrastructure?: true;
   shardIndex?: number;
   shardCount?: number;
   deviceId?: string;

packages/maestro/test/conformance/differential/engine-process.test.ts

@@ -0,0 +1,40 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { test } from 'node:test';
+import { classifyAgentDeviceFailure, runAgentDeviceEngine } from './engine-process.ts';
+
+test('agent-device JSON distinguishes infrastructure from behavioral failures', () => {
+  const result = (infrastructure?: true) =>
+    JSON.stringify({
+      success: true,
+      data: {
+        failures: [
+          {
+            status: 'failed',
+            ...(infrastructure ? { infrastructure } : {}),
+          },
+        ],
+      },
+    });
+
+  assert.equal(classifyAgentDeviceFailure(result(true)), 'infrastructure');
+  assert.equal(classifyAgentDeviceFailure(result()), 'behavioral');
+  assert.equal(classifyAgentDeviceFailure('not-json'), 'infrastructure');
+});
+
+test('agent-device execution accepts a CLI path containing spaces', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-engine path-'));
+  const cliPath = path.join(root, 'agent device.mjs');
+  try {
+    fs.writeFileSync(cliPath, '');
+    assert.deepEqual(runAgentDeviceEngine(cliPath, []), {
+      engine: 'agent-device',
+      outcome: 'pass',
+      exitCode: 0,
+    });
+  } finally {
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});

packages/maestro/test/conformance/differential/engine-process.ts

@@ -0,0 +1,59 @@
+import { spawnSync } from 'node:child_process';
+import type { ReplaySuiteResult } from '@agent-device/contracts/replay';
+
+export type EngineResult = {
+  engine: 'maestro' | 'agent-device';
+  outcome: 'pass' | 'fail';
+  exitCode: number;
+  /** Failure provenance stays distinct from the behavioral comparison. */
+  failureKind?: 'behavioral' | 'infrastructure';
+};
+
+export function runMaestroEngine(command: string, args: string[]): EngineResult {
+  const [bin = '', ...rest] = command.split(' ').filter(Boolean);
+  const result = spawnSync(bin, [...rest, ...args], { stdio: 'inherit', cwd: process.cwd() });
+  // Maestro does not expose typed failure provenance. Until it does, a non-zero exit cannot
+  // safely satisfy a behavioral divergence waiver.
+  return buildEngineResult('maestro', result, () => 'infrastructure');
+}
+
+export function classifyAgentDeviceFailure(stdout: string): 'behavioral' | 'infrastructure' {
+  try {
+    const envelope = JSON.parse(stdout) as { data?: ReplaySuiteResult };
+    const failures = envelope.data?.failures;
+    if (!Array.isArray(failures) || failures.length === 0) return 'infrastructure';
+    return failures.some((failure) => failure.infrastructure === true)
+      ? 'infrastructure'
+      : 'behavioral';
+  } catch {
+    // A non-zero process that did not return the promised suite envelope never reached a
+    // classifiable behavioral oracle. Keep it red, but do not call it a divergence.
+    return 'infrastructure';
+  }
+}
+
+export function runAgentDeviceEngine(cliPath: string, args: string[]): EngineResult {
+  const result = spawnSync(process.execPath, [cliPath, ...args, '--json'], {
+    cwd: process.cwd(),
+    encoding: 'utf8',
+  });
+  if (result.stdout) process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+  return buildEngineResult('agent-device', result, () => classifyAgentDeviceFailure(result.stdout));
+}
+
+function buildEngineResult(
+  engine: EngineResult['engine'],
+  result: { status: number | null; error?: Error },
+  classifyFailure: () => NonNullable<EngineResult['failureKind']>,
+): EngineResult {
+  const exitCode = result.status ?? 1;
+  if (exitCode === 0) return { engine, outcome: 'pass', exitCode };
+  return {
+    engine,
+    outcome: 'fail',
+    exitCode,
+    failureKind:
+      result.status === null || result.error !== undefined ? 'infrastructure' : classifyFailure(),
+  };
+}

packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml

@@ -1,16 +1,13 @@
 # Layer-3 device flow (bug class 4). A real tap on a real control of the lab app,
 # so the post-tap settle loop actually runs and the timing invariant is meaningful.
-# home-open-form sits below the fold, so scroll it into view first — the app's own
-# helper flow (examples/test-app/maestro/helpers/open-checkout-form.yaml) does
-# exactly this; tapping it directly fails with "element not found".
+# Use the always-visible Settings tab so scrolling is not an unrelated
+# precondition for the tap detector. open-inert-surface proves navigation finished.
 appId: com.callstack.agentdevicelab
 ---
 - launchApp:
     clearState: true
 - assertVisible: Agent Device Tester
-- scrollUntilVisible:
-    element:
-      id: home-open-form
 - tapOn:
-    id: home-open-form
-- assertVisible: Checkout form
+    text: Settings
+- assertVisible:
+    id: open-inert-surface

packages/maestro/test/conformance/differential/invariants.test.ts

@@ -6,7 +6,7 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { test } from 'node:test';
-import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS } from '../harness.ts';
+import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, parseMaestroConformanceSource } from '../harness.ts';
 import { DIFFERENTIAL_SCENARIOS } from './scenarios.ts';
 import { type Invariant, evaluateInvariant, readTrace } from './invariants.ts';
 
@@ -90,6 +90,41 @@ test('bug class 4 has a machine-checkable invariant, not just outcome parity', (
   );
 });
 
+const SETTLE_FLOW_PATH = path.join(import.meta.dirname, 'flows/settle-after-tap.yaml');
+
+function assertSettleFlowSemantics(source: string): void {
+  const parsed = parseMaestroConformanceSource(source, SETTLE_FLOW_PATH);
+  assert.equal(
+    parsed.commands.some(
+      (command) => command.kind === 'scroll' || command.kind === 'scrollUntilVisible',
+    ),
+    false,
+  );
+  assert.deepEqual(
+    parsed.commands.filter((command) => command.kind === 'tap'),
+    [{ kind: 'tap', longPress: false, repeat: 1, target: { selector: { text: 'Settings' } } }],
+  );
+  assert.equal(
+    parsed.commands.some(
+      (command) =>
+        command.kind === 'assert' &&
+        command.mode === 'visible' &&
+        command.selector?.id === 'open-inert-surface',
+    ),
+    true,
+  );
+}
+
+test('the settle detector reaches its tap without an unrelated setup command', () => {
+  assertSettleFlowSemantics(fs.readFileSync(SETTLE_FLOW_PATH, 'utf8'));
+});
+
+test('the settle flow guard rejects a changed tap target or inserted scroll', () => {
+  const flow = fs.readFileSync(SETTLE_FLOW_PATH, 'utf8');
+  assert.throws(() => assertSettleFlowSemantics(flow.replace('text: Settings', 'text: Home')));
+  assert.throws(() => assertSettleFlowSemantics(flow.replace('- tapOn:', '- scroll\n- tapOn:')));
+});
+
 // --- metricAtLeast: proves a code path actually ran, not just that it passed ---
 
 const RETRY_INVARIANT: Invariant = {

packages/maestro/test/conformance/differential/report-output.test.ts

@@ -0,0 +1,19 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { test } from 'node:test';
+import { writeReports } from './report-output.ts';
+
+test('writes the typed differential report envelope', () => {
+  const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'differential-report-'));
+  try {
+    writeReports(outDir, 'ios', []);
+    const report = JSON.parse(
+      fs.readFileSync(path.join(outDir, 'differential-report.json'), 'utf8'),
+    );
+    assert.deepEqual(report, { platform: 'ios', reports: [] });
+  } finally {
+    fs.rmSync(outDir, { recursive: true, force: true });
+  }
+});

packages/maestro/test/conformance/differential/report-output.ts

@@ -0,0 +1,77 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import type { EngineResult } from './engine-process.ts';
+import type { InvariantResult } from './invariants.ts';
+import type { DifferentialScenario } from './scenarios.ts';
+
+export type ScenarioReport = {
+  id: string;
+  flow: string;
+  maestro: EngineResult;
+  agentDevice: EngineResult;
+  outcomeDiverged: boolean;
+  invariants: InvariantResult[];
+  status: 'ok' | 'failed' | 'infrastructure-failed' | 'known-divergence' | 'stale-declaration';
+  tracking?: string;
+  failed: boolean;
+};
+
+export function printScenarioReport(report: ScenarioReport): void {
+  console.log(
+    `${report.status.padEnd('infrastructure-failed'.length)} ${report.id} maestro=${report.maestro.outcome} agent-device=${report.agentDevice.outcome}`,
+  );
+  for (const result of report.invariants) {
+    console.log(`        invariant ${result.status}: ${result.detail}`);
+  }
+  if (report.status === 'known-divergence') {
+    console.log(`        declared divergence, tracked: ${report.tracking}`);
+  }
+  if (report.status === 'stale-declaration') {
+    console.log(
+      `        passed while declared divergent — remove knownDivergence (${report.tracking}) so this stays enforced`,
+    );
+  }
+  if (report.status === 'infrastructure-failed') {
+    console.log('        oracle did not complete because engine infrastructure failed');
+  }
+}
+
+export function printDryRun(scenarios: DifferentialScenario[]): void {
+  for (const scenario of scenarios) {
+    const invariants = scenario.engineInvariants?.length ?? 0;
+    const declared = scenario.knownDivergence
+      ? `\tdeclared-divergence=${scenario.knownDivergence.tracking}`
+      : '';
+    console.log(
+      `${scenario.id}\t${scenario.flow}\texpect=${scenario.expect}\tinvariants=${invariants}${declared}`,
+    );
+  }
+  const known = scenarios.filter((scenario) => scenario.knownDivergence).length;
+  console.log(`\n${scenarios.length} scenario(s) validated, ${known} declared divergence(s).`);
+}
+
+export function writeReports(
+  outDir: string | undefined,
+  platform: string | undefined,
+  reports: ScenarioReport[],
+): void {
+  if (!outDir) return;
+  fs.mkdirSync(outDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(outDir, 'differential-report.json'),
+    `${JSON.stringify({ platform, reports }, null, 2)}\n`,
+  );
+}
+
+export function printRunSummary(reports: ScenarioReport[]): void {
+  const known = reports.filter((report) => report.status === 'known-divergence');
+  if (known.length > 0) {
+    console.log(
+      `\n${known.length} declared divergence(s), not enforced: ${known.map((r) => r.id).join(', ')}`,
+    );
+  }
+  const failed = reports.filter((report) => report.failed);
+  if (failed.length === 0) return;
+  console.error(`\n${failed.length} scenario(s) failed: ${failed.map((r) => r.id).join(', ')}`);
+  process.exitCode = 1;
+}

packages/maestro/test/conformance/differential/run.test.ts

@@ -3,6 +3,7 @@
 // conformance-differential workflow.
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
+import os from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { describe, test } from 'node:test';
@@ -11,7 +12,14 @@ import {
   DIFFERENTIAL_SCENARIOS,
   type DivergenceSignature,
 } from './scenarios.ts';
-import { matchesSignature, parseRunnerArgs, selectScenarios, validateScenarios } from './run.ts';
+import { parseMaestroConformanceSource } from '../harness.ts';
+import {
+  matchesSignature,
+  parseRunnerArgs,
+  runScenario,
+  selectScenarios,
+  validateScenarios,
+} from './run.ts';
 
 const CONFORMANCE_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 
@@ -142,12 +150,49 @@ describe('knownDivergence signature matching', () => {
   });
 });
 
+test('an ordinary Maestro process failure cannot satisfy a behavioral waiver', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-failure-'));
+  const maestroCli = path.join(root, 'maestro.mjs');
+  const agentDeviceCli = path.join(root, 'agent-device.mjs');
+  try {
+    fs.writeFileSync(maestroCli, 'process.exit(1);\n');
+    fs.writeFileSync(agentDeviceCli, 'process.exit(0);\n');
+
+    const report = runScenario(
+      {
+        id: 'maestro-infrastructure-failure',
+        flow: 'differential/flows/settle-after-tap.yaml',
+        comparesAcrossEngines: 'test fixture',
+        expect: 'pass',
+        divergenceMeans: 'test fixture',
+        knownDivergence: {
+          reason: 'A behavioral Maestro failure is temporarily accepted for this test fixture.',
+          tracking: 'https://github.com/callstack/agent-device/issues/1',
+          expected: { maestro: 'fail', agentDevice: 'pass' },
+        },
+      },
+      {
+        dryRun: false,
+        maestroBin: `${process.execPath} ${maestroCli}`,
+        agentDeviceCli,
+      },
+    );
+
+    assert.equal(report.maestro.failureKind, 'infrastructure');
+    assert.equal(report.status, 'infrastructure-failed');
+    assert.equal(report.failed, true);
+  } finally {
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test('every device flow targets the fixture app the workflow installs', () => {
   for (const scenario of DIFFERENTIAL_SCENARIOS) {
-    const body = fs.readFileSync(path.join(CONFORMANCE_DIR, scenario.flow), 'utf8');
-    assert.match(
-      body,
-      new RegExp(`^appId:\\s*${DIFFERENTIAL_APP_ID}$`, 'm'),
+    const flowPath = path.join(CONFORMANCE_DIR, scenario.flow);
+    const parsed = parseMaestroConformanceSource(fs.readFileSync(flowPath, 'utf8'), flowPath);
+    assert.equal(
+      parsed.appId,
+      DIFFERENTIAL_APP_ID,
       `${scenario.id} must target ${DIFFERENTIAL_APP_ID}; a flow against any other app cannot run on the CI simulator`,
     );
   }