(MOT-4295) test(harness): multi-target integration scenarios INT-014 and INT-016 plus runner enablers

harness/tests/integration/README.md

@@ -23,9 +23,17 @@ No provider key or network access is required.
 | INT-010 | `crash-recovery-507` | direct | SIGKILL and restart the engine while a controlled function is in flight, with `context::assemble` held out during boot; the side effect runs once, the interrupted call closes, and the turn completes |
 | INT-011 | `stop-cancel-cascade` | direct | stopping a running root turn cancels the root and spawned children while retaining a queued message |
 | INT-012 | `queued-message-edit-unqueue` | direct | edit one queued message in place and unqueue another while the first turn is streaming; only the edited and untouched rows drain in order |
+| INT-014 | `budget-preflight-exceeded` | direct | a frozen token budget admits generation 1, then fails the turn at budget preflight when a fat function result blows generation 2's reservation; the state ledger keeps the reconciled usage |
+| INT-016 | `notify-grant-teardown` | direct | a notify subscription fires an injected notification turn; filesystem grants round-trip; the budget ledger reconciles; `harness::teardown` sweeps the standing binding |
 | UI-001 | `console-streamed-text` | playground | a message sent by the Console streams to durable completion |
 | UI-002 | `multi-turn-traces` | playground | a native function turn and a Console turn expose distinct traces and function-call events |
 
+INT-013 (`held-call-resolve`, hook-held call released via
+`harness::function::resolve`) is authored in `src/scenarios/held_call_resolve.rs`
+but not registered: running it surfaced an apparent harness defect where a
+hook-held call leaves `harness::status` timing out and the turn record missing
+from state (MOT-4296). Register it once that defect is fixed.
+
 Each fixture is defined end to end in its own `src/scenarios/*.rs` file with a
 small typed DSL. The scenario keeps its send policy, router request matchers,
 response behavior, controlled function, function history, and verification

harness/tests/integration/src/evidence_data.rs

@@ -34,6 +34,12 @@ pub struct RunEvidence {
     pub tree_statuses: Vec<Value>,
     /// Raw scripted-router calls and abort acknowledgements.
     pub router_evidence: Value,
+    /// `{function_id, response}` per fired probe action, in fire order —
+    /// the only surface for asserting a probe-dispatched call's response.
+    pub probe_responses: Vec<Value>,
+    /// `{function_id, payload}` per scripted-hook invocation the probe
+    /// served, in arrival order — whole-run evidence like `target_calls`.
+    pub hook_calls: Vec<Value>,
 }
 
 /// Compact form published in `playground-result.json`.
@@ -209,6 +215,59 @@ impl RunEvidence {
         Ok(())
     }
 
+    /// Invocation payloads served by the scripted hook registered as `name`.
+    pub fn hook_invocations(&self, name: &str) -> Vec<&Value> {
+        let suffix = format!("::hook-{name}");
+        self.hook_calls
+            .iter()
+            .filter(|entry| {
+                entry
+                    .get("function_id")
+                    .and_then(Value::as_str)
+                    .is_some_and(|id| id.ends_with(&suffix))
+            })
+            .filter_map(|entry| entry.get("payload"))
+            .collect()
+    }
+
+    pub fn expect_hook_calls(&self, name: &str, count: usize) -> anyhow::Result<()> {
+        let actual = self.hook_invocations(name).len();
+        anyhow::ensure!(
+            actual == count,
+            "hook {name} served {actual} invocation(s), expected {count}"
+        );
+        Ok(())
+    }
+
+    /// Response body of the nth fired probe action (fire order follows the
+    /// fixture's `after_turns` sort).
+    pub fn probe_response(&self, index: usize) -> Option<&Value> {
+        self.probe_responses.get(index).and_then(|entry| entry.get("response"))
+    }
+
+    /// Assert the nth probe action targeted `function_id` and its response
+    /// satisfies `check`.
+    pub fn expect_probe_response(
+        &self,
+        index: usize,
+        function_id: &str,
+        check: impl FnOnce(&Value) -> anyhow::Result<()>,
+    ) -> anyhow::Result<()> {
+        let entry = self.probe_responses.get(index).ok_or_else(|| {
+            anyhow::anyhow!(
+                "no probe response at index {index} ({} recorded)",
+                self.probe_responses.len()
+            )
+        })?;
+        let actual = entry.get("function_id").and_then(Value::as_str);
+        anyhow::ensure!(
+            actual == Some(function_id),
+            "probe action {index} targeted {actual:?}, expected {function_id:?}"
+        );
+        check(entry.get("response").unwrap_or(&Value::Null))
+            .map_err(|error| anyhow::anyhow!("probe response {index} ({function_id}): {error}"))
+    }
+
     pub fn expect_no_duplicate_messages(&self) -> anyhow::Result<()> {
         anyhow::ensure!(
             !self.has_duplicate_messages(),
@@ -348,6 +407,8 @@ mod tests {
             tree_sessions: Vec::new(),
             tree_statuses: Vec::new(),
             router_evidence: Value::Null,
+            probe_responses: Vec::new(),
+            hook_calls: Vec::new(),
         }
     }
 

harness/tests/integration/src/fixtures.rs

@@ -9,7 +9,9 @@ mod discovery;
 mod loading;
 
 pub use discovery::scenario_fixtures;
-pub use loading::{ProbeAction, ScenarioFixture, ScenarioIntervention};
+pub use loading::{
+    HookBehavior, ProbeAction, ScenarioFixture, ScenarioHook, ScenarioIntervention,
+};
 
 #[cfg(test)]
 mod tests;

harness/tests/integration/src/fixtures/loading.rs

@@ -44,8 +44,47 @@ pub struct ScenarioFixture {
     /// outside the compiled subject scenario: it is test synchronization, not
     /// a public harness request.
     pub intervention: Option<ScenarioIntervention>,
+    /// Probe-hosted hook functions bound to `harness::hook::<point>` trigger
+    /// types before Send. Each records its invocations as whole-run evidence
+    /// and answers with its scripted behavior.
+    pub hooks: Vec<ScenarioHook>,
 }
 
+/// One scripted hook the probe registers and binds for the run.
+#[derive(Debug, Clone)]
+pub struct ScenarioHook {
+    /// Short name; the registered function id is `<run_id>::hook-<name>`.
+    pub name: String,
+    /// Hook point suffix: pre-turn, pre-generate, post-generate, pre-trigger,
+    /// or post-trigger.
+    pub point: String,
+    /// Optional function-id globs for the binding's `functions` filter.
+    pub functions: Option<Vec<String>>,
+    pub priority: i64,
+    pub timeout_ms: Option<u64>,
+    pub on_error: Option<String>,
+    pub behavior: HookBehavior,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum HookBehavior {
+    /// Always answer `null` (continue).
+    Continue,
+    /// `{"decision":"hold"}` on the first invocation, `null` afterwards — so
+    /// a post-resolve chain resume does not re-park the call.
+    HoldOnce,
+    Deny { reason: String },
+    Mutate { mutations: serde_json::Value },
+}
+
+pub const HOOK_POINTS: [&str; 5] = [
+    "pre-turn",
+    "pre-generate",
+    "post-generate",
+    "pre-trigger",
+    "post-trigger",
+];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum ScenarioIntervention {
     StopCancelCascade {
@@ -61,6 +100,10 @@ pub enum ScenarioIntervention {
         remove_message: String,
         after_message: String,
     },
+    /// Wait for a hook-held call to park the turn in `awaiting_functions`,
+    /// exercise `harness::function::resolve` no-op gates (wrong turn, unknown
+    /// call), then release the held call with `action: "execute"`.
+    HeldCallResolve { function_call_id: String },
 }
 
 /// A function the PROBE (test infra, not a model turn) invokes at a completion
@@ -144,11 +187,47 @@ impl ScenarioFixture {
             }
         }
         if self.intervention.is_none() {
+            // `cancelled` requires the intervention that performs the stop; a
+            // run may legitimately END failed (e.g. a budget-preflight
+            // rejection) — the floor still pins the durable status to it.
             anyhow::ensure!(
-                self.expected_turn_statuses.last().map(String::as_str) == Some("completed"),
-                "the last terminal turn must be completed"
+                matches!(
+                    self.expected_turn_statuses.last().map(String::as_str),
+                    Some("completed") | Some("failed")
+                ),
+                "the last terminal turn must be completed or failed"
             );
         }
+        {
+            let mut names = std::collections::BTreeSet::new();
+            for hook in &self.hooks {
+                anyhow::ensure!(
+                    !hook.name.is_empty()
+                        && hook
+                            .name
+                            .bytes()
+                            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')),
+                    "hook name {:?} is not id-safe",
+                    hook.name
+                );
+                anyhow::ensure!(
+                    names.insert(hook.name.clone()),
+                    "duplicate hook name {:?}",
+                    hook.name
+                );
+                anyhow::ensure!(
+                    HOOK_POINTS.contains(&hook.point.as_str()),
+                    "unknown hook point {:?}; expected one of {HOOK_POINTS:?}",
+                    hook.point
+                );
+                if let Some(on_error) = &hook.on_error {
+                    anyhow::ensure!(
+                        matches!(on_error.as_str(), "fail_open" | "fail_closed"),
+                        "hook on_error {on_error:?} must be fail_open or fail_closed"
+                    );
+                }
+            }
+        }
         if let Some(intervention) = &self.intervention {
             match intervention {
                 ScenarioIntervention::StopCancelCascade {
@@ -202,6 +281,26 @@ impl ScenarioFixture {
                         "queued-edit scenario must end with one completed turn"
                     );
                 }
+                ScenarioIntervention::HeldCallResolve { function_call_id } => {
+                    anyhow::ensure!(
+                        !function_call_id.is_empty(),
+                        "held-call-resolve function_call_id must not be empty"
+                    );
+                    anyhow::ensure!(
+                        self.scenario.target.is_some(),
+                        "held-call-resolve needs a controlled function to hold"
+                    );
+                    anyhow::ensure!(
+                        self.hooks
+                            .iter()
+                            .any(|hook| hook.behavior == HookBehavior::HoldOnce),
+                        "held-call-resolve needs a HoldOnce hook to park the call"
+                    );
+                    anyhow::ensure!(
+                        self.expected_turn_statuses == ["completed"],
+                        "held-call-resolve scenario must end with one completed turn"
+                    );
+                }
             }
         }
         // A direct scenario is one external send, but the harness may seed

harness/tests/integration/src/fixtures/tests.rs

@@ -11,15 +11,16 @@ fn all_selection_returns_the_checked_in_fixtures() {
         ids,
         std::collections::BTreeSet::from([
             "INT-001", "INT-002", "INT-003", "INT-004", "INT-005", "INT-006", "INT-007", "INT-008",
-            "INT-009", "INT-010", "INT-011", "INT-012", "UI-001", "UI-002"
+            "INT-009", "INT-010", "INT-011", "INT-012", "INT-014", "INT-016", "UI-001",
+            "UI-002"
         ])
     );
     assert_eq!(
         fixtures
             .iter()
             .filter(|fixture| fixture.driver == crate::scenarios::ScenarioDriver::Direct)
             .count(),
-        12
+        14
     );
 }