diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index af96f3e14b..c47b77842a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -105,7 +105,7 @@ extension RunnerTests { DataPayload(nodes: $0, truncated: result.truncated) }, qualityNotes: result.qualityNotes, - presentedSurface: SnapshotPresentedSurface.classify(acquisition, modalScope: nil) + presentedSurface: SnapshotPresentedSurface.classify(acquisition) ) } diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/ModalSurface.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/ModalSurface.swift index 8dc984011c..e662cc1ed2 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/ModalSurface.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/ModalSurface.swift @@ -58,21 +58,27 @@ extension ModalSurface { case .paired: break } + // An empty honor-modal read also reports `rootMismatch`. That is only a label: the result is + // unknown either way. guard plain.first?.element == honorModal.first?.element else { return .unknown(.rootMismatch) } guard let plainChildren = children(of: plain), let honorModalChildren = children(of: honorModal) else { return .unknown(.unorderedRead) } // Pairing matched elements under parent elements. This walk matches them under the parent node, // which also tells apart the occurrences of an element that one read lists under two parents. + // An element listed more than once under one parent pairs in order when the honor-modal read keeps + // every occurrence, and counts as removed when it keeps none. When it keeps some but not all, the + // walk reports `duplicatedChild`. var failure = ModalSurfaceUnknownReason.unstable(.reparentedElement) func surfaces(honorModalNode: Int, plainNode: Int) -> [ModalSurface]? { var removed = plainChildren[plainNode] var kept: [(honorModal: Int, plain: Int)] = [] + let listed = occurrences(of: plainChildren[plainNode], in: plain) + let keptListed = occurrences(of: honorModalChildren[honorModalNode], in: honorModal) for child in honorModalChildren[honorModalNode] { let element = honorModal[child].element - let listed = plainChildren[plainNode].filter { plain[$0].element == element }.count - let keptListed = honorModalChildren[honorModalNode].filter { honorModal[$0].element == element }.count - guard listed < 2 || keptListed >= listed else { + let listedCount = listed[element, default: 0] + guard listedCount < 2 || keptListed[element, default: 0] >= listedCount else { failure = .duplicatedChild return nil } @@ -109,6 +115,10 @@ extension ModalSurface { return (isAction ? [node] : []) + node.children.flatMap { actions(in: $0, keyboardOwned: keyboardOwned) } } + private static func occurrences(of positions: [Int], in nodes: [ModalScopeNode]) -> [String: Int] { + positions.reduce(into: [:]) { counts, position in counts[nodes[position].element, default: 0] += 1 } + } + /// Each node's children, in order. A read lists every node after its parent, so a node's parent /// is its nearest listed ancestor with the parent element. private static func children(of nodes: [ModalScopeNode]) -> [[Int]]? { diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentedSurface.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentedSurface.swift index 3204658b29..5e2f21ad88 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentedSurface.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentedSurface.swift @@ -1,11 +1,9 @@ import Foundation /// Reads a modal presentation out of a runner acquisition (#3565). A runner node carries no UIKit class -/// name, so its evidence is an element type that names a presentation, or a modal-scope pair whose -/// honor-modal read found a modal surface. Its answer is `presented` or `unknown`, never `absent`: a pair -/// that removed nothing is not evidence of absence. This is the class-blind column of the golden table -/// `contracts/fixtures/ios-presented-surface.json`, with `modalScopeExpected` for the pair; the -/// class-aware rule is `collectPresentedSurface` in +/// name, so its only evidence is an element type that names a presentation. Its answer is `presented` or +/// `unknown`, never `absent`. This is the class-blind column of the golden table +/// `contracts/fixtures/ios-presented-surface.json`; the class-aware rule is `collectPresentedSurface` in /// `packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts`. public enum SnapshotPresentedSurface { /// Element types that name a presentation, keyed by the element-type numbering every iOS producer @@ -17,10 +15,8 @@ public enum SnapshotPresentedSurface { "Dialog": 8, ] - /// `modalScope` is nil when the capture read no modal-scope pair. - public static func classify(_ acquisition: SnapshotAcquisition, modalScope: ModalSurfaceReading?) -> PresentedSurface { - if case .modal = modalScope { return .presented } - return acquisition.nodes.contains { isPresentingElementType($0.type) } ? .presented : .unknown + public static func classify(_ acquisition: SnapshotAcquisition) -> PresentedSurface { + acquisition.nodes.contains { isPresentingElementType($0.type) } ? .presented : .unknown } private static func isPresentingElementType(_ elementType: String) -> Bool { diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ModalSurfaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ModalSurfaceTests.swift index c0c5b93365..944a2fea90 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ModalSurfaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/ModalSurfaceTests.swift @@ -32,6 +32,9 @@ final class ModalSurfaceTests: XCTestCase { excluded.remove(at: try XCTUnwrap(excluded.firstIndex(of: action), "\(name): \(action.label)")) } XCTAssertEqual(excluded.map(\.label), expected.excludedActions, "\(name): excluded actions") + case .unknown(.duplicatedChild): + XCTAssertEqual(expected.reading, "unknown", name) + XCTAssertEqual(expected.reason, "duplicatedChild", name) case .unknown: XCTFail("\(name): \(reading)") } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RecordedAlertSurface.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RecordedAlertSurface.swift index f2a56a5ef5..85a0324d32 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RecordedAlertSurface.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/RecordedAlertSurface.swift @@ -24,9 +24,10 @@ struct RecordedAlertSurface: Decodable { let buttons: [String] } - /// `reading` is `modal` or `noExclusion`. A `modal` reading lists each surface by its host and root - /// positions in the honor-modal tree, with the labels of the actions it offers, and the labels of - /// the honor-modal tree's actions that no surface offers. + /// `reading` is `modal`, `noExclusion`, or `unknown` with the `reason` `duplicatedChild`. A + /// `modal` reading lists each surface by its host and root positions in the honor-modal tree, with + /// the labels of the actions it offers, and the labels of the honor-modal tree's actions that no + /// surface offers. struct ModalScopeExpectation: Decodable { struct Surface: Decodable { let host: Int @@ -35,6 +36,7 @@ struct RecordedAlertSurface: Decodable { } let reading: String + let reason: String? let surfaces: [Surface]? let excludedActions: [String]? } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/SnapshotPresentedSurfaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/SnapshotPresentedSurfaceTests.swift index 5ed6abb1a1..4fac89a524 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/SnapshotPresentedSurfaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/SnapshotPresentedSurfaceTests.swift @@ -4,9 +4,7 @@ import XCTest /// Cross-language parity for the presented-surface rule (#3565): every case in /// `contracts/fixtures/ios-presented-surface.json` runs through the runner entry point, which reads no -/// class names, so each must answer the table's `classBlindExpected`. A case that states -/// `modalScopeExpected` must give the answer for all three modal-scope readings: `modal`, -/// `noExclusion` and `unknown`. +/// class names, so each must answer the table's `classBlindExpected`. final class SnapshotPresentedSurfaceTests: XCTestCase { private struct Vocabulary: Decodable { let presentingElementTypes: [PresentingElementType] @@ -27,7 +25,6 @@ final class SnapshotPresentedSurfaceTests: XCTestCase { let name: String let truncated: Bool? let classBlindExpected: String - let modalScopeExpected: [String: String]? let nodes: [Node] } @@ -54,40 +51,19 @@ final class SnapshotPresentedSurfaceTests: XCTestCase { XCTAssertFalse(cases.isEmpty) for testCase in cases { XCTAssertEqual( - SnapshotPresentedSurface.classify(acquisition(of: testCase), modalScope: nil), + SnapshotPresentedSurface.classify(acquisition(of: testCase)), try XCTUnwrap(PresentedSurface(rawValue: testCase.classBlindExpected)), testCase.name ) } } - func testEveryModalScopeReadingClassifiesAsTheGoldenTableStates() throws { - let readings: [String: ModalSurfaceReading] = [ - "modal": .modal([ModalSurface(host: 1, roots: [6])]), - "noExclusion": .noExclusion, - "unknown": .unknown(.unstable(.screenChanged)), - ] - let cases = try readFixture().cases.filter { $0.modalScopeExpected != nil } - XCTAssertFalse(cases.isEmpty) - for testCase in cases { - let expected = try XCTUnwrap(testCase.modalScopeExpected) - XCTAssertEqual(Set(expected.keys), Set(readings.keys), testCase.name) - for (key, reading) in readings { - XCTAssertEqual( - SnapshotPresentedSurface.classify(acquisition(of: testCase), modalScope: reading), - try XCTUnwrap(expected[key].flatMap(PresentedSurface.init(rawValue:))), - "\(testCase.name): \(key)" - ) - } - } - } - func testACapDoesNotDefeatPositiveEvidence() throws { let alert = try XCTUnwrap( try readFixture().cases.first { $0.name == "the same alert through a producer whose name table stops short" } ) - XCTAssertEqual(SnapshotPresentedSurface.classify(acquisition(of: alert, truncated: false), modalScope: nil), .presented) - XCTAssertEqual(SnapshotPresentedSurface.classify(acquisition(of: alert, truncated: true), modalScope: nil), .presented) + XCTAssertEqual(SnapshotPresentedSurface.classify(acquisition(of: alert, truncated: false)), .presented) + XCTAssertEqual(SnapshotPresentedSurface.classify(acquisition(of: alert, truncated: true)), .presented) } private func acquisition(of testCase: FixtureCase, truncated: Bool? = nil) -> SnapshotAcquisition { diff --git a/apple/snapshot-presentation/Tests/AlertSurfaceCorpus/README.md b/apple/snapshot-presentation/Tests/AlertSurfaceCorpus/README.md index 40e9a99905..4ce6c2aafc 100644 --- a/apple/snapshot-presentation/Tests/AlertSurfaceCorpus/README.md +++ b/apple/snapshot-presentation/Tests/AlertSurfaceCorpus/README.md @@ -1,27 +1,144 @@ # Alert surface corpus Each file records two accessibility trees of one screen from a real simulator, and what the alert -rules in `AlertSurface` must find on that screen. `AlertSurfaceTests` replays every file on the host -with `swift test --package-path apple/snapshot-presentation`, so a rule change that loses a real -prompt fails without a device. - -## Add a fixture - -1. Put the screen in the state you want, using an agent-device session: open the app, `fold`, and - raise the prompt. For prompt recipes, see `examples/test-app/README.md`. -2. Capture the app that hosts the prompt (`com.apple.springboard` for a system alert): - `pnpm capture:alert-surface --udid --bundle --name ---`. - The capture builds its runner with its own bundle id, so it runs beside the session's runner. -3. Add `expect` from the screenshot. For each source that must find the prompt, give its title and - buttons. A source you leave out must find nothing, and `{}` marks a screen with no prompt. -4. Add `expectModalScope`, which `ModalSurfaceTests` replays through `ModalSurface`. For a screen - whose honor-modal tree removes nothing, write `{"reading": "noExclusion"}`. Otherwise write - `"reading": "modal"` and give each surface's `host` and `roots` as `honorModal` indexes, with the - labels of the `actions` it offers. Then give the `excludedActions`: the labels of the - `honorModal` actions that no surface offers. - -Don't edit the trees by hand. Capture again instead. The script keeps the existing `expect` and -`expectModalScope`, but a new capture can move the indexes in `expectModalScope`. +rules in `AlertSurface` and `ModalSurface` must find on that screen. `AlertSurfaceTests` and +`ModalSurfaceTests` replay every file on the host, so a rule change that loses a real prompt fails +without a device. + +## Before you start + +You need: + +- Xcode with an iOS simulator runtime. The capture builds its runner with the Xcode that + `xcode-select -p` prints. +- A booted simulator. `xcrun simctl list devices booted` prints its UDID. +- An agent-device session on that simulator that shows the screen you want to record. For the + "Open in app" prompts, see `examples/test-app/README.md`. + +## Add a recording + +**Write `expect` and `expectModalScope` from the screenshot: which node is the prompt on the +screen, and which buttons it offers.** The replay tests `ModalSurface` against them, so they +must not come from `ModalSurface`. `pnpm print:alert-surface` prints the rule's reading and +marks nodes. Use it only to find node indexes. Never copy its reading, a host, or a root without +checking each root and action against the screenshot. + +1. Put the screen in the state you want with the session: open the app, `fold` for a folded pose, + and raise the prompt. +2. Record the app that hosts the prompt. For a system alert, that is `com.apple.springboard`: + + ```bash + pnpm capture:alert-surface --udid --bundle com.apple.springboard --name iphone-duo-folded-springboard-open-in-app + ``` + + Name the recording `---`. The capture builds a runner with its own + bundle id, so it runs beside the session's runner. If the screen changes between its reads, the + capture fails. Wait until the prompt stops animating, then run it again. + +3. Print the honor-modal tree, so you can find the prompt's nodes: + + ```bash + pnpm print:alert-surface apple/snapshot-presentation/Tests/AlertSurfaceCorpus/iphone-duo-folded-springboard-open-in-app.json + ``` + + Each line after the first shows a node's index, type, and label. `[kept 1 of 3 children]` + marks a node where the honor-modal tree removed children. The first line is what + `ModalSurface` reads, labeled as the rule's answer, not ground truth. It is `modal`, + `noExclusion`, or `unknown()` with the node where the read failed, and nodes are + marked only when it is `modal`. To print the whole tree, add `plain` after the path. If the + recording does not load in the replay, the command stops with one line that names the problem. + +4. Add `expect` from the screenshot. For each source that must find the prompt, give its title and + buttons. The sources are `blockingSystemModal`, `appAlert`, `safariDialog`, and `dismissPopup`. + A source you leave out must find nothing. Write `{}` for a screen with no prompt. +5. Add `expectModalScope` from the screenshot: + - If the screen shows no prompt, write `{"reading": "noExclusion"}`. + - If the screen shows a prompt, write `"reading": "modal"` and one surface for each prompt. + Find the prompt in the printed tree by its type and labels. Its `roots` are the kept nodes + that hold the prompt, and its `host` is their parent: the node where the honor-modal tree + removed the rest of the screen. `actions` are the labels of the prompt's controls, in the + printed order. Empty labels count. Add `excludedActions`: the labels of the honor-modal + tree's controls that the prompt does not offer, such as a toolbar that stays visible. + - Then compare the rule's reading on the first line with what you wrote. If they differ, look + at the screenshot again. If the screenshot supports what you wrote, the rule is wrong for this + screen. Don't add the recording. Report the screen in an issue, and attach the recording. + - If the first line reads `unknown(duplicatedChild)`, a node lists one element more than once + and the honor-modal tree kept only some of the copies, so the rule can't tell which copy + lost children. The line shows the element as its raw AX `element` id, such as + `99943:4829729792.1219`. Search the recording for that id to find the copies. Write + `{"reading": "unknown", "reason": "duplicatedChild"}` only after you confirm those copies in + the recording. + - If the first line reads `unknown` with any other reason, the replay rejects the recording. + Capture the screen again. If the line stays, report it in an issue. +6. Run the replay: + + ```bash + swift test --package-path apple/snapshot-presentation + ``` + +Don't edit the trees by hand. Capture again instead. A recapture keeps `expect` and +`expectModalScope`, but it can move the indexes in `expectModalScope`. After a recapture, print the +recording again and review each index. + +## Example: the SpringBoard "Open in app" alert + +`iphone-duo-folded-springboard-open-in-app.json` records the alert that +`xcrun simctl openurl "agent-device-test-app://x"` raises on a folded iPhone Duo. The start +of its printout: + +```text +ModalSurface reads (the rule's answer, not ground truth): modal + 0 Application "" [kept 1 of 40 children] + 1 Window "" [kept 1 of 3 children] + 2 Other "" [kept 1 of 3 children] + 3 Alert "Open in “Agent Device Tester”?" + 4 Other "" + 5 Other "" + 6 Other "" + 7 Other "" + 8 ScrollView "" + 9 Other "" + 10 StaticText "Open in “Agent Device Tester”?" + 11 Other "" + 12 Other "Vertical scroll bar, 2 pages" + 13 Other "" + 14 Other "Horizontal scroll bar, 1 page" + 15 Other "" + 16 Other "Vertical scroll bar, 2 pages" + 17 Other "" + 18 Other "Horizontal scroll bar, 1 page" + 19 Other "" + 20 ScrollView "" + 21 Other "" + 22 Other "" + 23 Button "Cancel" + 24 Button "Open" +... +``` + +The screenshot shows the alert `Open in “Agent Device Tester”?` with Cancel and Open. Both the +system-modal source and the app-alert source find it, so `expect` names both: + +```json +"expect": { + "blockingSystemModal": { "title": "Open in “Agent Device Tester”?", "buttons": ["Cancel", "Open"] }, + "appAlert": { "title": "Open in “Agent Device Tester”?", "buttons": ["Cancel", "Open"] } +} +``` + +In the tree, node 3 is that alert: its type is `Alert` and its label is the title. Its buttons are +nodes 23 and 24, Cancel and Open, as on the screen. Node 2 holds the alert, and the honor-modal +tree removed node 2's other children, so `host` is 2 and `roots` is `[3]`. The screen shows no +other controls, so `excludedActions` is empty. The rule's reading on the first line is `modal`, +which agrees: + +```json +"expectModalScope": { + "reading": "modal", + "surfaces": [{ "host": 2, "roots": [3], "actions": ["Cancel", "Open"] }], + "excludedActions": [] +} +``` ## Format diff --git a/contracts/fixtures/ios-presented-surface.json b/contracts/fixtures/ios-presented-surface.json index 1dd52d7819..7665376a00 100644 --- a/contracts/fixtures/ios-presented-surface.json +++ b/contracts/fixtures/ios-presented-surface.json @@ -1,5 +1,5 @@ { - "description": "The iOS rule that decides whether a capture read a modal presentation over the app's own content (issue #3565), and the tree shapes it is pinned on. Source of truth for the TypeScript classifier (collectPresentedSurface in packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts) and the Swift runner classifier (SnapshotPresentedSurface in apple/snapshot-presentation); a change here must keep both parity tests green, and each side mirrors the vocabulary it reads by hand the way contracts/fixtures/ios-system-surface-hosts.json does. Every case states the answer for a capture whose producer reports UIKit class names (`expected`) and for the same tree read by a producer that reports none (`classBlindExpected`). The TypeScript test runs both readings; the Swift runner reports no class names, so it implements and checks only `classBlindExpected` and mirrors only presentingElementTypes. A case can also state `modalScopeExpected`: the Swift answer for the same class-blind tree when the runner also read a modal-scope pair, for each reading that pair can give. No TypeScript producer reads that pair, so the TypeScript test does not read that column. `presented` comes only from positive evidence: a dimming view, an alert-root class, an element type that names a presentation itself, or, in Swift only, a modal-scope pair whose reading is `modal`. `absent` is the claim that every window the capture read showed only its own content, so it needs class facts, a completeness statement, one content window, and one container in it; anything else answers `unknown`. Each case's `note` says whether the shape was read from a device or is anticipated, and the anticipated ones are why `unknown` exists.", + "description": "The iOS rule that decides whether a capture read a modal presentation over the app's own content (issue #3565), and the tree shapes it is pinned on. Source of truth for the TypeScript classifier (collectPresentedSurface in packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts) and the Swift runner classifier (SnapshotPresentedSurface in apple/snapshot-presentation); a change here must keep both parity tests green, and each side mirrors the vocabulary it reads by hand the way contracts/fixtures/ios-system-surface-hosts.json does. Every case states the answer for a capture whose producer reports UIKit class names (`expected`) and for the same tree read by a producer that reports none (`classBlindExpected`). The TypeScript test runs both readings; the Swift runner reports no class names, so it implements and checks only `classBlindExpected` and mirrors only presentingElementTypes. `presented` comes only from positive evidence: a dimming view, an alert-root class, or an element type that names a presentation itself. `absent` is the claim that every window the capture read showed only its own content, so it needs class facts, a completeness statement, one content window, and one container in it; anything else answers `unknown`. Each case's `note` says whether the shape was read from a device or is anticipated, and the anticipated ones are why `unknown` exists.", "vocabulary": { "presentationContainerClass": "UITransitionView", "presentationContainerNote": "UIKit appends each modal presentation as a later sibling of the container it presents over, directly under the window, and presents the app's own content inside one too. A window carrying more than one container therefore holds a presentation of some kind; one container is just the window's content.", @@ -16,7 +16,6 @@ { "name": "Dialog", "elementType": 8 } ], "presentingElementTypesNote": "Element types that name a presentation wherever they appear, keyed by the element-type numbering every iOS producer reads. A producer whose name table stops short of a name publishes `Element()` for the same node, so both spellings count and both are derived from these numbers rather than listed twice. The number must be written bare: `Element(+7)` is not a spelling of Alert, because a lenient numeric parse would let each language's integer parser decide the answer.", - "modalScopeNote": "A modal-scope pair is a plain read and an honor-modal read of one screen (ModalSurface in apple/snapshot-presentation). Its reading is `modal` when the honor-modal read removed content and kept a modal surface beside it, `noExclusion` when it removed nothing, and `unknown` when the reads cannot be compared. Only `modal` is evidence: a server that ignores the honor-modal parameter also removes nothing, so `noExclusion` never answers `absent`.", "classNameMatchingNote": "Class names are matched after normalization, not byte for byte: trim, drop every case-insensitive `XCUIElementType`, drop a leading `AX`, lowercase, then keep only what follows the last `.` or `/`. That is `normalizeType` in packages/contracts/src/snapshot-text.ts, which the occlusion cut already matches UIKit classes through. One class can be spelled differently — with a module prefix, or with the `AX` prefix an accessibility role carries — so a reader that compared raw bytes would answer a different question about the same capture than one that normalized. presentationContainerClass, dimmingViewClass, alertRootClassPrefix and systemWindowClasses are all read through it, so the spellings below name those classes however a producer writes them.", "absentRequirementsNote": "absent additionally requires: `truncated` stated false (a producer that never observed completeness proves nothing about what it may have capped), a class name on every node (a producer that reports none can show no window structure), exactly one window outside systemWindowClasses (a second is a presentation UIKit hosts in its own window, or an overlay window, and the tree cannot tell those apart), and exactly one presentation container in that window (more than one is a full-screen or custom presentation, which this rule has no positive read on, and none means there is no container structure to read), and an index that identifies each node once (window structure is read through parent links, which cannot tell two nodes sharing a number apart). Positive evidence is exempt from that last requirement because it is keyed on the node occurrence it was read off, not on its number: a repeated index costs this rule the answer it declines to make, never a false absent." }, @@ -342,15 +341,10 @@ }, { "name": "a UIKit page sheet whose honor-modal read keeps only the sheet", - "note": "Read from a device: Settings' Clear History sheet on an iPhone Duo simulator through the XCTest runner, trimmed to its window structure (apple/snapshot-presentation/Tests/AlertSurfaceCorpus). The honor-modal read removes the presenting container, which holds a dimming view of its own, and keeps the one that holds the sheet's dimming view and the sheet. No node has a presenting element type, so a runner capture without the pair answers `unknown`.", + "note": "Read from a device: Settings' Clear History sheet on an iPhone Duo simulator through the XCTest runner, trimmed to its window structure (apple/snapshot-presentation/Tests/AlertSurfaceCorpus). The honor-modal read removes the presenting container, which holds a dimming view of its own, and keeps the one that holds the sheet's dimming view and the sheet. No node has a presenting element type, so a runner capture answers `unknown`.", "truncated": false, "expected": "presented", "classBlindExpected": "unknown", - "modalScopeExpected": { - "modal": "presented", - "noExclusion": "unknown", - "unknown": "unknown" - }, "nodes": [ { "index": 0, "type": "Application", "role": "UIApplication" }, { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, @@ -366,15 +360,10 @@ }, { "name": "a screen whose honor-modal read removes nothing", - "note": "Read from a device: the Settings root screen on an iPhone Duo simulator through the XCTest runner, trimmed to its window structure (apple/snapshot-presentation/Tests/AlertSurfaceCorpus). The honor-modal read equals the plain read. The class-aware reading is `absent`, but the pair proves no absence, so the runner still answers `unknown`.", + "note": "Read from a device: the Settings root screen on an iPhone Duo simulator through the XCTest runner, trimmed to its window structure (apple/snapshot-presentation/Tests/AlertSurfaceCorpus). The honor-modal read equals the plain read. The class-aware reading is `absent`, and the runner answers `unknown`.", "truncated": false, "expected": "absent", "classBlindExpected": "unknown", - "modalScopeExpected": { - "modal": "presented", - "noExclusion": "unknown", - "unknown": "unknown" - }, "nodes": [ { "index": 0, "type": "Application", "role": "UIApplication" }, { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 0a56aab45e..96b05b33a9 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -167,15 +167,13 @@ the normal session hygiene above when validating its snapshot, wait, and selecto The app declares the URL scheme `agent-device-test-app` in `app.config.js`, so it can raise two iOS prompts that `alert` must answer. Before you change `alert`, test with the reported prompt itself, -not with a different alert. +not with a different alert. To record a prompt for the host tests, see +`apple/snapshot-presentation/Tests/AlertSurfaceCorpus/README.md`. - **SpringBoard "Open in “Agent Device Tester”?" alert.** Open the app in a session, then run `xcrun simctl openurl "agent-device-test-app://x"`. After you accept this alert once, the simulator stops showing it for this app. Rebooting or reinstalling doesn't reset this, so test - `dismiss` first. To see the alert again, use a new simulator: - `xcrun simctl create tester-fresh ` prints its UDID. For example, - `xcrun simctl create tester-fresh com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro com.apple.CoreSimulator.SimRuntime.iOS-27-1`. - `xcrun simctl list devicetypes` and `xcrun simctl list runtimes` list the ids. + `dismiss` first. To see the alert again, use a fresh simulator. - **Safari "Open this page in “Agent Device Tester”?" prompt.** Open `com.apple.mobilesafari` in a session, fill the address field with `agent-device-test-app://x`, then tap Go on the keyboard. This prompt can be raised any number of times. @@ -183,6 +181,39 @@ not with a different alert. `snapshot --raw --json` includes the Safari prompt in the page tree. For the SpringBoard alert, it returns only the alert, without the windows around it. +To raise the SpringBoard alert on a fresh simulator: + +1. Create and boot the simulator. `xcrun simctl create` prints its UDID: + + ```bash + udid=$(xcrun simctl create tester-fresh com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro com.apple.CoreSimulator.SimRuntime.iOS-27-1) + xcrun simctl boot "$udid" + ``` + + `xcrun simctl list devicetypes` and `xcrun simctl list runtimes` list the ids. + +2. Build Agent Device Tester and install it on that simulator. The build cache reuses an earlier + native build, so only the first build is slow. Keep this terminal open, because it runs Metro: + + ```bash + pnpm test-app:install + pnpm test-app:ios -- --device "$udid" + ``` + +3. In a second terminal, set `udid` to the same UDID. Open the app in a session, then raise the + alert: + + ```bash + agent-device open com.callstack.agentdevicelab --platform ios --udid "$udid" --session open-in-app + xcrun simctl openurl "$udid" "agent-device-test-app://x" + ``` + +4. When you finish, close the session: + + ```bash + agent-device close --platform ios --udid "$udid" --session open-in-app + ``` + ### Android emulator or device Install dependencies and run the development build on the target Android diff --git a/package.json b/package.json index ab4de00e2f..0b4d61d021 100644 --- a/package.json +++ b/package.json @@ -224,7 +224,8 @@ "test:replay:linux": "node --experimental-strip-types src/bin.ts test test/integration/replays/linux", "test:linux:command-evidence": "node --experimental-strip-types test/integration/linux-e2e/live-runner.ts", "prepublishOnly": "echo 'agent-device publishes from .github/workflows/release.yml; see CONTRIBUTING.md.' >&2 && exit 1", - "capture:alert-surface": "node --experimental-strip-types scripts/capture-alert-surface.ts" + "capture:alert-surface": "node --experimental-strip-types scripts/capture-alert-surface.ts", + "print:alert-surface": "node --experimental-strip-types scripts/print-alert-surface-tree.ts" }, "files": [ "bin", diff --git a/scripts/print-alert-surface-tree.ts b/scripts/print-alert-surface-tree.ts new file mode 100644 index 0000000000..2ec9338af6 --- /dev/null +++ b/scripts/print-alert-surface-tree.ts @@ -0,0 +1,203 @@ +// Prints what ModalSurface reads from an alert surface corpus recording, then one of its trees, one +// node per line, so a contributor can write its `expectModalScope` by hand. +// See apple/snapshot-presentation/Tests/AlertSurfaceCorpus/README.md. +import fs from 'node:fs'; + +type Rect = { + infinite?: boolean; + nonFinite?: boolean; + x: number; + y: number; + width: number; + height: number; +}; +type RecordedNode = { + index: number; + parentIndex?: number; + element: string; + type: string; + label: string; + rect: Rect; +}; + +function fail(message: string): never { + console.error(message); + process.exit(1); +} + +const [file, treeName = 'honorModal'] = process.argv.slice(2); +if (!file || (treeName !== 'honorModal' && treeName !== 'plain')) { + fail('Usage: pnpm print:alert-surface [honorModal|plain]'); +} +let recording: Record; +try { + recording = JSON.parse(fs.readFileSync(file, 'utf8')); +} catch (error) { + fail(`${file}: ${(error as Error).message}`); +} +if (recording.format !== 2) { + fail(`${file}: format ${JSON.stringify(recording.format)} is not 2. Capture it again.`); +} +const plain = recordedTree('plain'); +const honorModal = recordedTree('honorModal'); + +// The checks RecordedAlertSurface.load makes: each node decodes, its index is its position, and its +// parent comes before it. An empty honor-modal tree is left to the reading, which names it. +function recordedTree(name: string): RecordedNode[] { + const tree = recording[name]; + if (!Array.isArray(tree) || (tree.length === 0 && name === 'plain')) { + fail(`${file}: "${name}" is not a tree`); + } + tree.forEach((node, position) => { + if (node.parentIndex === null) delete node.parentIndex; + const parent = node.parentIndex; + const isValid = + node.index === position && + (parent === undefined + ? position === 0 + : Number.isInteger(parent) && parent >= 0 && parent < position) && + ['element', 'type', 'identifier', 'label'].every((key) => typeof node[key] === 'string') && + isRect(node.rect); + if (!isValid) fail(`${file}: "${name}" node ${position} does not load. Capture it again.`); + }); + return tree; +} + +function isRect(rect: Record | null): boolean { + if (typeof rect !== 'object' || rect === null) return false; + const flags = [rect.infinite, rect.nonFinite]; + if (flags.some((flag) => flag != null && typeof flag !== 'boolean')) return false; + return ( + flags.includes(true) || + ['x', 'y', 'width', 'height'].every((key) => typeof rect[key] === 'number') + ); +} + +// CGRect equality compares rects after it makes their sizes positive. +function frameKey({ infinite, nonFinite, x, y, width, height }: Rect): string { + if (infinite) return 'infinite'; + if (nonFinite) return 'nonFinite'; + const origin = [width < 0 ? x + width : x, height < 0 ? y + height : y]; + return [...origin, Math.abs(width), Math.abs(height)].join(); +} + +// The rest ports ModalScopePair.pair, ModalSurface.read and its children(of:) in the same order, +// so the printed reading is the one the replay checks. The replay passes the honor-modal tree as its +// own confirming read and states that no read was truncated. +function pairInstability(): string | undefined { + const keyOf = (tree: RecordedNode[], node: RecordedNode) => + JSON.stringify([ + node.element, + node.parentIndex === undefined ? null : tree[node.parentIndex].element, + ]); + const unmatched = new Map(); + for (const node of plain) { + const key = keyOf(plain, node); + unmatched.set(key, { frame: frameKey(node.rect), count: (unmatched.get(key)?.count ?? 0) + 1 }); + } + const plainElements = new Set(plain.map((node) => node.element)); + for (const node of honorModal) { + const match = unmatched.get(keyOf(honorModal, node)); + if (!match || match.count === 0) { + const reason = plainElements.has(node.element) ? 'reparentedElement' : 'unmatchedElement'; + return `unknown(unstable(${reason})): node ${node.index}`; + } + if (match.frame !== frameKey(node.rect)) + return `unknown(unstable(movedElement)): node ${node.index}`; + match.count -= 1; + } + return undefined; +} + +// A node's parent is its nearest listed ancestor with the parent element. +function childrenOf(tree: RecordedNode[]): number[][] | undefined { + const children = tree.map((): number[] => []); + const ancestors: number[] = []; + for (const node of tree) { + if (node.parentIndex !== undefined) { + const parentElement = tree[node.parentIndex].element; + while (ancestors.length > 0 && tree[ancestors.at(-1)!].element !== parentElement) { + ancestors.pop(); + } + if (ancestors.length === 0) return undefined; + children[ancestors.at(-1)!].push(node.index); + } else if (node.index !== 0) { + return undefined; + } + ancestors.push(node.index); + } + return children; +} + +function countElements(tree: RecordedNode[], positions: number[]): Map { + const counts = new Map(); + for (const position of positions) { + counts.set(tree[position].element, (counts.get(tree[position].element) ?? 0) + 1); + } + return counts; +} + +const markers = new Map(); +function readModalSurface(): string { + const instability = pairInstability(); + if (instability) return instability; + if (plain.length === honorModal.length) return 'noExclusion'; + if (honorModal.length === 0) return 'unknown(rootMismatch): the honor-modal tree is empty'; + if (plain[0].element !== honorModal[0].element) return 'unknown(rootMismatch)'; + const plainChildren = childrenOf(plain); + const keptChildren = childrenOf(honorModal); + if (!plainChildren || !keptChildren) return 'unknown(unorderedRead)'; + let failure = ''; + + // The number of surfaces under the node, or undefined when the walk fails. + function surfaces(honorModalNode: number, plainNode: number): number | undefined { + const removed = [...plainChildren![plainNode]]; + const kept: Array<[number, number]> = []; + const listed = countElements(plain, plainChildren![plainNode]); + const keptListed = countElements(honorModal, keptChildren![honorModalNode]); + for (const child of keptChildren![honorModalNode]) { + const element = honorModal[child].element; + const listedCount = listed.get(element) ?? 0; + const keptCount = keptListed.get(element) ?? 0; + if (listedCount >= 2 && keptCount < listedCount) { + failure = `unknown(duplicatedChild): node ${honorModalNode} kept ${keptCount} of ${listedCount} children with element id ${element}`; + return undefined; + } + const match = removed.findIndex((position) => plain[position].element === element); + if (match < 0) { + failure = `unknown(unstable(reparentedElement)): node ${child}`; + return undefined; + } + kept.push([child, removed.splice(match, 1)[0]]); + } + let nested = 0; + for (const [keptChild, plainChild] of kept) { + const found = surfaces(keptChild, plainChild); + if (found === undefined) return undefined; + nested += found; + } + if (removed.length === 0 || kept.length === 0) return nested; + markers.set( + honorModalNode, + `[kept ${kept.length} of ${plainChildren![plainNode].length} children]`, + ); + return nested === 0 ? 1 : nested; + } + + const found = surfaces(0, 0); + if (found === undefined) return failure; + return found === 0 ? 'unknown(noSurvivingSurface)' : 'modal'; +} + +const reading = readModalSurface(); +console.log(`ModalSurface reads (the rule's answer, not ground truth): ${reading}`); +if (reading !== 'modal' || treeName === 'plain') markers.clear(); + +const depths: number[] = []; +for (const node of treeName === 'plain' ? plain : honorModal) { + depths[node.index] = node.parentIndex === undefined ? 0 : depths[node.parentIndex] + 1; + const marker = markers.has(node.index) ? ` ${markers.get(node.index)}` : ''; + console.log( + `${String(node.index).padStart(4)} ${' '.repeat(depths[node.index])}${node.type} ${JSON.stringify(node.label)}${marker}`, + ); +}