diff --git a/.omp/extensions/lib/fm-task-inbox-doorbell.ts b/.omp/extensions/lib/fm-task-inbox-doorbell.ts index 32c13d78123..a9d0811a671 100644 --- a/.omp/extensions/lib/fm-task-inbox-doorbell.ts +++ b/.omp/extensions/lib/fm-task-inbox-doorbell.ts @@ -24,7 +24,7 @@ type OmpDoorbellApi = { details: { kind: "task-inbox"; runtime: "omp" }; }, options: { deliverAs: "steer"; triggerTurn: true }, - ) => void; + ) => void | Promise; // The downgrade recovery channel: a user prompt starts a turn on an idle // session, where the agent-initiated sendMessage path can be deferred into // append-only delivery by the runtime's turn policy. @@ -49,6 +49,12 @@ const MAX_TURN_GRACE_MS = 120000; export type TaskInboxDoorbellOptions = { inboxDir?: string; readyMarker?: string; + // Durable diagnosis for a refused handshake: a failure that retires the + // ready marker (activation, or a drain that takes the doorbell down) writes + // its reason here, so a missing marker is never ambiguous. Deliberately + // independent of the doorbell's own configuration - an unconfigured + // doorbell is itself an activation failure only this file can report. + failureJournal?: string; // How long a delivered doorbell may go without a turn_start before the // triggerTurn call is treated as downgraded to append-only and the // instruction is re-driven through the user-prompt channel. @@ -61,7 +67,11 @@ export type TaskInboxDoorbellOptions = { }; export type TaskInboxDoorbell = { - activate: () => void; + // activate reports whether the doorbell is live after the call. A caller + // that publishes its own readiness marker (fm-spawn's generated extension + // touching .omp-ready) must gate that marker on this result, or readiness + // silently outlives a failed handshake. + activate: () => boolean | Promise; retire: () => void; notifyTurnStart: () => void; notifyTurnEnd: () => void; @@ -86,6 +96,23 @@ function publishReadyMarker(marker: string): void { renameSync(staged, marker); } +// Best-effort durable diagnosis for a lost handshake: " : ". +// The write is staged and renamed like the ready marker so a concurrent reader +// never sees a partial reason. A failed journal write is swallowed - the +// journal explains failures, it must never become one. +function journalDoorbellFailure(journal: string, phase: string, error: unknown): void { + if (!journal.startsWith("/")) return; + try { + const reason = error instanceof Error ? (error.stack ?? error.message) : String(error); + mkdirSync(dirname(journal), { recursive: true }); + const staged = `${journal}.staging.${process.pid}`; + writeFileSync(staged, `${new Date().toISOString()} ${phase}: ${reason}\n`, { mode: 0o600 }); + renameSync(staged, journal); + } catch { + return; + } +} + function retireOwnedReadyMarker(marker: string): void { try { if (readFileSync(marker, "utf8") === `${process.pid}\n`) unlinkSync(marker); @@ -139,13 +166,45 @@ function reconcileAwaitingTurns(requestDir: string): void { } } +function defaultFailureJournal(options: TaskInboxDoorbellOptions): string { + const explicit = options.failureJournal || process.env.FM_OMP_TASK_DOORBELL_FAILED || ""; + if (explicit) return explicit; + const readyMarker = options.readyMarker || process.env.FM_OMP_TASK_DOORBELL_READY || ""; + if (readyMarker.startsWith("/")) { + const suffix = ".omp-doorbell-ready"; + const stem = readyMarker.endsWith(suffix) + ? readyMarker.slice(0, -suffix.length) + : readyMarker; + return `${stem}.omp-doorbell-failed`; + } + const inboxDir = options.inboxDir || process.env.FM_OMP_TASK_INBOX_DIR || ""; + const stateDir = inboxDir.startsWith("/") + ? dirname(inboxDir) + : (process.env.FM_STATE_OVERRIDE?.startsWith("/") + ? process.env.FM_STATE_OVERRIDE + : join(process.env.FM_HOME || process.env.FM_ROOT_OVERRIDE || process.cwd(), "state")); + return stateDir ? join(stateDir, `.omp-doorbell-failed.${process.pid}`) : ""; +} + export function installTaskInboxDoorbell( omp: OmpDoorbellApi, options: TaskInboxDoorbellOptions = {}, ): TaskInboxDoorbell { + const failureJournal = defaultFailureJournal(options); const configured = configuredOptions(options); if (!configured || typeof omp.sendMessage !== "function") { - return { activate: () => {}, retire: () => {}, notifyTurnStart: () => {}, notifyTurnEnd: () => {} }; + const unconfiguredWhy = !configured + ? "task inbox doorbell is unconfigured (inboxDir/readyMarker unresolved)" + : "OMP sendMessage is unavailable"; + return { + activate: () => { + journalDoorbellFailure(failureJournal, "activate", new Error(unconfiguredWhy)); + return false; + }, + retire: () => {}, + notifyTurnStart: () => {}, + notifyTurnEnd: () => {}, + }; } const requestDir = `${configured.readyMarker}.requests`; @@ -161,6 +220,7 @@ export function installTaskInboxDoorbell( let dispatchingTurn = false; let dispatchingTurnObserved = false; const awaitingTurns = new Map>(); + const activationSends = new Set>(); let watcher: FSWatcher | undefined; const settleAwaiting = (awaitingPath: string, outcome: "delivered" | "failed"): void => { const timer = awaitingTurns.get(awaitingPath); @@ -252,7 +312,7 @@ export function installTaskInboxDoorbell( invoked = true; dispatchingTurn = true; dispatchingTurnObserved = false; - omp.sendMessage( + const sendResult = omp.sendMessage( { customType: "firstmate-task-inbox-doorbell", content, @@ -262,6 +322,18 @@ export function installTaskInboxDoorbell( }, { deliverAs: "steer", triggerTurn: true }, ); + if (sendResult && typeof sendResult.then === "function") { + const delivery = Promise.resolve(sendResult); + activationSends.add(delivery); + void delivery.catch((error: unknown) => { + if (!active && existsSync(configured.readyMarker)) return; + bestEffortRename(`${pending}.delivered`, pending); + bestEffortRename(`${pending}.awaiting-turn`, pending); + bestEffortRename(ambiguous, pending); + journalDoorbellFailure(failureJournal, "drain", error); + retire(); + }).finally(() => activationSends.delete(delivery)); + } const turnStartedDuringSend = dispatchingTurnObserved; dispatchingTurn = false; dispatchingTurnObserved = false; @@ -282,9 +354,11 @@ export function installTaskInboxDoorbell( awaitingPath, setTimeout(() => recoverUnprovenTurn(awaitingPath), turnGraceMs), ); - } catch { + } catch (error) { dispatchingTurn = false; - if (!invoked) bestEffortRename(ambiguous, `${pending}.failed`); + if (invoked) bestEffortRename(ambiguous, pending); + else bestEffortRename(ambiguous, `${pending}.failed`); + journalDoorbellFailure(failureJournal, "drain", error); retire(); break; } @@ -293,8 +367,8 @@ export function installTaskInboxDoorbell( draining = false; } }; - const activate = (): void => { - if (active) return; + const activate = (): boolean | Promise => { + if (active) return true; try { mkdirSync(requestDir, { recursive: true, mode: 0o700 }); reconcileAmbiguousClaims(requestDir); @@ -308,9 +382,26 @@ export function installTaskInboxDoorbell( } publishReadyMarker(configured.readyMarker); drain(); - } catch { + } catch (error) { + journalDoorbellFailure(failureJournal, "activate", error); retire(); + return false; + } + // A drain failure retires the doorbell without throwing; it already + // journaled its reason, so the activation reports the failure it caused. + if (!active) return false; + if (activationSends.size > 0) { + return (async (): Promise => { + while (activationSends.size > 0) { + await Promise.allSettled([...activationSends]); + } + if (!active) return false; + bestEffortUnlink(failureJournal); + return true; + })(); } + bestEffortUnlink(failureJournal); + return true; }; return { activate, retire, notifyTurnStart, notifyTurnEnd }; diff --git a/AGENTS.md b/AGENTS.md index 3ecb3a644be..dd8cb66dba4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,7 +102,7 @@ state/ volatile runtime signals; gitignored .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown .devin-turnend-token firstmate-owned Devin hook registry token for the task; removed by teardown .hermes-turnend-token .hermes-session .hermes-started firstmate-owned Hermes hook registry token plus the task's stable session id and per-turn start acknowledgement; removed by teardown - .omp-ext.ts .omp-ready .omp-started firstmate-generated OMP task extension plus its session-start and first-turn acknowledgement markers; removed by teardown + .omp-ext.ts .omp-ready .omp-started .omp-doorbell-ready .omp-doorbell-failed firstmate-generated OMP task extension plus its session-start and first-turn acknowledgement markers; .omp-ready publishes only after the inbox doorbell activates, and a lost handshake journals its reason to .omp-doorbell-failed (docs/architecture.md; bin/fm-task-inbox-lib.sh); removed by teardown .inbox/ durable steering inbox: sequenced firstmate instruction records the worker acknowledges by moving them into its handled/ subdirectory; written by fm-send, re-rung and escalated by the watcher, removed by teardown (bin/fm-task-inbox-lib.sh) .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; optional grok_turnend_dir=, kimi_turnend_dir=, and devin_turnend_dir= persist harness registry ownership for teardown; optional prewalk_into= records an effective OMP Prewalk target; optional allow_project_omp_extensions=1 records explicit approval for tracked project extensions on an OMP launch (docs/configuration.md "OMP project extensions"); an optional traceparent= only when trace context is enabled (docs/configuration.md "Trace context propagation"); kind=secondmate also records home= and projects=, plus remote_host=/remote_root=/remote_backend=/remote_herdr_session=/remote_target= for a remote route; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) .herdr-presentation quarantinable attempt and restart-binding journal for Herdr's optional visual projection; never task or endpoint authority; see docs/herdr-backend.md "Presentation spaces" diff --git a/bin/fm-send.sh b/bin/fm-send.sh index c5813b88736..675347d013f 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -1100,6 +1100,7 @@ else INBOX_RECORD_HANDLED=0 FM_TASK_INBOX_RING_OMP_REQUEST= FM_TASK_INBOX_RING_OMP_PID= + FM_TASK_INBOX_RING_OMP_DOORBELL= case "$INBOX_RECORD" in */handled/*) ring_rc=0 @@ -1115,7 +1116,10 @@ else if [ "$TARGET_HARNESS" = omp ]; then OMP_NATIVE_SESSION_PID=${FM_TASK_INBOX_RING_OMP_PID:-unreadable} [ "$INBOX_RECORD_HANDLED" = 1 ] && OMP_NATIVE_SESSION_PID=not-a-session-receipt - OMP_NATIVE_BINDING="task=$TARGET_TASK_ID endpoint=$TARGET_BACKEND:$T session-pid=$OMP_NATIVE_SESSION_PID request=${FM_TASK_INBOX_RING_OMP_REQUEST:-none} record=$INBOX_RECORD message-bytes=$(printf '%s' "$MESSAGE" | wc -c | tr -d '[:space:]')" + OMP_NATIVE_BINDING="task=$TARGET_TASK_ID endpoint=$TARGET_BACKEND:$T session-pid=$OMP_NATIVE_SESSION_PID" + [ -z "${FM_TASK_INBOX_RING_OMP_DOORBELL:-}" ] \ + || OMP_NATIVE_BINDING="$OMP_NATIVE_BINDING $FM_TASK_INBOX_RING_OMP_DOORBELL" + OMP_NATIVE_BINDING="$OMP_NATIVE_BINDING request=${FM_TASK_INBOX_RING_OMP_REQUEST:-none} record=$INBOX_RECORD message-bytes=$(printf '%s' "$MESSAGE" | wc -c | tr -d '[:space:]')" fi case "$ring_rc" in 0) ;; diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index ebb540df0c0..b449315bb62 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -2212,7 +2212,8 @@ if [ "$HARNESS" = omp ]; then fi for artifact in \ "$STATE/$ID.status" "$STATE/$ID.omp-ext.ts" "$STATE/$ID.omp-ready" \ - "$STATE/$ID.omp-started" "$STATE/$ID.omp-doorbell-ready"; do + "$STATE/$ID.omp-started" "$STATE/$ID.omp-doorbell-ready" \ + "$STATE/$ID.omp-doorbell-failed"; do if [ -L "$artifact" ] || { [ -e "$artifact" ] && [ ! -f "$artifact" ]; }; then echo "error: refusing OMP relaunch through unsafe artifact path: $artifact" >&2 exit 1 @@ -2271,7 +2272,7 @@ if [ "$HARNESS" = omp ]; then for artifact in \ "$STATE/$ID.meta" "$STATE/$ID.status" "$STATE/$ID.omp-ext.ts" \ "$STATE/$ID.omp-ready" "$STATE/$ID.omp-started" "$STATE/$ID.omp-doorbell-ready" \ - "$STATE/$ID.omp-doorbell-ready.requests" "/tmp/fm-$ID"; do + "$STATE/$ID.omp-doorbell-failed" "$STATE/$ID.omp-doorbell-ready.requests" "/tmp/fm-$ID"; do if [ -e "$artifact" ] || [ -L "$artifact" ]; then echo "error: refusing OMP spawn because task $ID already has artifacts at $artifact; reconcile or clean the prior task before retrying" >&2 exit 1 @@ -4023,7 +4024,11 @@ TURNEND="$STATE_REAL/$ID.turn-ended" TURNEND_SIGNAL="$FM_ROOT/bin/fm-turnend-signal.sh" SPAWN_GEN="s$(date +%s).${BASHPID:-$$}.$RANDOM" if [ "$HARNESS" = omp ]; then - rm -f "$STATE/$ID.omp-doorbell-ready" + OMP_READY="$STATE_REAL/$ID.omp-ready" + OMP_STARTED="$STATE_REAL/$ID.omp-started" + OMP_DOORBELL_READY="$STATE_REAL/$ID.omp-doorbell-ready" + OMP_DOORBELL_FAILED="$STATE_REAL/$ID.omp-doorbell-failed" + rm -f "$STATE/$ID.omp-doorbell-ready" "$STATE/$ID.omp-doorbell-failed" fi exclude_path() { local rel=$1 EXCL @@ -4186,26 +4191,28 @@ export default function (pi: any) { EOF ;; omp) - OMP_READY="$STATE_REAL/$ID.omp-ready" - OMP_STARTED="$STATE_REAL/$ID.omp-started" - OMP_DOORBELL_READY="$STATE_REAL/$ID.omp-doorbell-ready" - rm -f "$OMP_READY" "$OMP_STARTED" "$OMP_DOORBELL_READY" + rm -f "$OMP_READY" "$OMP_STARTED" "$OMP_DOORBELL_READY" "$OMP_DOORBELL_FAILED" cat > "$STATE/$ID.omp-ext.ts" < { - taskInboxDoorbell.activate(); - execFile("touch", ["$OMP_READY"]); + Promise.resolve(taskInboxDoorbell.activate()).then((active) => { + if (active) execFile("touch", ["$OMP_READY"]); + }); }); omp.on("turn_start", () => { taskInboxDoorbell.notifyTurnStart(); @@ -4535,7 +4542,7 @@ if [ "$OMP_LAUNCH_TEMPLATE" -eq 1 ] && [ -n "$OMP_BUN_LAUNCH_DIR" ]; then OMP_LAUNCH_PATH_GUARD="PATH=$(shell_quote "$OMP_BUN_LAUNCH_DIR${PATH:+:$PATH}"); export PATH; FM_OMP_BUN_LOOKUP=\$(command -v bun) || exit 1; FM_OMP_BUN_RESOLVED=\$(readlink -f \"\$FM_OMP_BUN_LOOKUP\" 2>/dev/null || node -e 'const { realpathSync } = require(\"node:fs\"); process.stdout.write(realpathSync(process.argv[1]));' \"\$FM_OMP_BUN_LOOKUP\") || exit 1; [ \"\$FM_OMP_BUN_RESOLVED\" = $(shell_quote "$OMP_BUN_CANON") ] || exit 1; " fi if [ "$OMP_LAUNCH_TEMPLATE" -eq 1 ] && [ "$HARNESS" = omp ] && [ -n "$OMP_BIN_CANON" ]; then - LAUNCH="FM_OMP_TASK_INBOX_DIR=$(shell_quote "$STATE_REAL/$ID.inbox") FM_OMP_TASK_DOORBELL_READY=$(shell_quote "$STATE_REAL/$ID.omp-doorbell-ready") FM_OMP_BUN=$(shell_quote "$OMP_BUN_CANON") FM_OMP_BIN=$(shell_quote "$OMP_BIN_CANON") $LAUNCH" + LAUNCH="FM_OMP_TASK_INBOX_DIR=$(shell_quote "$STATE_REAL/$ID.inbox") FM_OMP_TASK_DOORBELL_READY=$(shell_quote "$STATE_REAL/$ID.omp-doorbell-ready") FM_OMP_TASK_DOORBELL_FAILED=$(shell_quote "$STATE_REAL/$ID.omp-doorbell-failed") FM_OMP_BUN=$(shell_quote "$OMP_BUN_CANON") FM_OMP_BIN=$(shell_quote "$OMP_BIN_CANON") $LAUNCH" fi OMPRESUMEFLAG= [ -z "$OMP_RESUME_FILE" ] || OMPRESUMEFLAG="--resume $(shell_quote "$OMP_RESUME_FILE") " @@ -4716,9 +4723,28 @@ if [ "${HERDR_PROJECTED:-0}" -eq 1 ]; then HERDR_PROJECTION_ABORT_CLEANUP=0 spawn_herdr_presentation_order_lock_release fi -if [ "$HARNESS" = omp ]; then +if [ "$HARNESS" = omp ] && [ "$OMP_LAUNCH_TEMPLATE" -eq 1 ]; then OMP_ACK_INTERVAL=${FM_OMP_LAUNCH_ACK_INTERVAL:-0.5} OMP_ACKED=0 + OMP_DOORBELL_ACK_POLLS=${FM_OMP_DOORBELL_ACK_POLLS:-40} + OMP_DOORBELL_ACKED=0 + for _ in $(seq 1 "$OMP_DOORBELL_ACK_POLLS"); do + if [ -f "$OMP_DOORBELL_READY" ]; then + OMP_DOORBELL_ACKED=1 + break + fi + [ -f "$OMP_DOORBELL_FAILED" ] && break + sleep "$OMP_ACK_INTERVAL" + done + if [ "$OMP_DOORBELL_ACKED" -ne 1 ]; then + OMP_DOORBELL_DETAIL="the worker extension did not activate its inbox doorbell" + if [ -f "$OMP_DOORBELL_FAILED" ]; then + OMP_DOORBELL_DETAIL="doorbell activation failed: $(head -n 1 "$OMP_DOORBELL_FAILED" 2>/dev/null || printf 'unreadable journal') (journal: $OMP_DOORBELL_FAILED)" + fi + printf 'failed: OMP inbox doorbell marker %s never appeared; %s\n' "$OMP_DOORBELL_READY" "$OMP_DOORBELL_DETAIL" >> "$STATE/$ID.status" + echo "error: OMP inbox doorbell marker $OMP_DOORBELL_READY never appeared; $OMP_DOORBELL_DETAIL; cleaning the owned launch" >&2 + exit 1 + fi if [ "$KIND" = secondmate ]; then OMP_ACK_POLLS=${FM_OMP_SECONDMATE_ACK_POLLS:-120} OMP_PRIMARY_VERSION=$(fm_primary_watch_version "$OMP_PRIMARY_EXTENSION" "$PROJ_ABS" 2>/dev/null || true) diff --git a/bin/fm-task-inbox-lib.sh b/bin/fm-task-inbox-lib.sh index 6b83236cfc2..cd30903b2f0 100644 --- a/bin/fm-task-inbox-lib.sh +++ b/bin/fm-task-inbox-lib.sh @@ -39,6 +39,11 @@ # .inbox/.ring-state watcher re-ring ladder: "\t\t" # .inbox/.escalated oldest-message name already surfaced as stale, # so later polls suppress another escalation +# .omp-doorbell-ready OMP doorbell handshake: the owning session PID, +# published by the task extension only after the +# doorbell activates (.omp-ready follows it) +# .omp-doorbell-failed the reason a doorbell activation or drain retired +# the ready marker, journaled by the extension # # Record format (fm_task_inbox_write / fm_task_inbox_body): # schema=fm-task-inbox.v1 @@ -229,6 +234,35 @@ fm_task_inbox_hermes_delivery_lock_path() { # printf '%s/.%s.hermes-delivery.lock' "$1" "$2" } +# Explain an OMP native refusal as one space-free key=value token naming the +# durable artifact that carries the reason: the extension's failure journal +# (written when activation or a drain retired the ready marker), the marker +# that is missing or unreadable, or the marker whose session binding stayed +# unproven. Callers splice it into the refusal binding so a supervisor sees WHY +# the adapter refused rather than a bare session-pid=unreadable. +fm_task_inbox_omp_doorbell_state() { # + local marker=$1 journal request_dir + journal="${marker%.omp-doorbell-ready}.omp-doorbell-failed" + request_dir="${marker}.requests" + if [ -f "$journal" ]; then + printf 'doorbell-failure=%s' "$journal" + return 0 + fi + if [ ! -e "$marker" ]; then + printf 'doorbell-marker-missing=%s' "$marker" + return 0 + fi + if ! fm_omp_task_doorbell_marker_read "$marker" 2>/dev/null; then + printf 'doorbell-marker-unreadable=%s' "$marker" + return 0 + fi + if [ ! -d "$request_dir" ]; then + printf 'doorbell-request-dir-missing=%s' "$request_dir" + return 0 + fi + printf 'doorbell-binding-unproven=%s' "$marker" +} + # Deliver one doorbell. Callers go through fm_task_inbox_ring, which owns the # Hermes delivery-lock critical section; this helper is the unserialized body. # A positively dead or missing endpoint returns 6 without typing anything - @@ -249,6 +283,7 @@ fm_task_inbox_ring_deliver() { # [expected-lab # shellcheck disable=SC2034 # Public outcome binding read by the caller after sourcing (bin/fm-send.sh). FM_TASK_INBOX_RING_OMP_REQUEST="$ready_marker.requests/request.$request_id" FM_TASK_INBOX_RING_OMP_PID= + FM_TASK_INBOX_RING_OMP_DOORBELL= FM_OMP_TASK_DOORBELL_BOUND_PID= programmatic_rc=0 fm_backend_omp_trigger_turn "$backend" "$target" "$ready_marker" "$omp_runtime" "$omp_bin" "$request_id" "$line" \ @@ -260,7 +295,11 @@ fm_task_inbox_ring_deliver() { # [expected-lab [ "$programmatic_rc" = 0 ] && return 0 return 4 ;; - *) return 3 ;; + *) + # shellcheck disable=SC2034 # Public outcome binding read by the caller after sourcing (bin/fm-send.sh). + FM_TASK_INBOX_RING_OMP_DOORBELL=$(fm_task_inbox_omp_doorbell_state "$ready_marker") + return 3 + ;; esac fi cstate=$(fm_backend_composer_state "$backend" "$target" "$harness" "$omp_runtime" "$omp_bin" 2>/dev/null) || cstate=unknown @@ -290,7 +329,9 @@ fm_task_inbox_ring_deliver() { # [expected-lab # missing (nothing typed; recovery owns the record). On an OMP target the call also publishes # FM_TASK_INBOX_RING_OMP_REQUEST (the named native queue entry for this record) # and FM_TASK_INBOX_RING_OMP_PID (the proven acknowledging session process) so -# the caller can report the exact binding it acted on. A native success without +# the caller can report the exact binding it acted on; a refusal (3) also +# publishes FM_TASK_INBOX_RING_OMP_DOORBELL, one token naming the durable +# artifact that explains the refusal (fm_task_inbox_omp_doorbell_state). A native success without # that proof is refused; the acknowledgement move remains the only proof the # worker acted. # diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 1be54ba38ad..dfffc6babf2 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -2905,6 +2905,7 @@ cleanup_firstmate_home_children() { "$sub_state/$child_id.meta" "$sub_state/$child_id.pi-ext.ts" \ "$sub_state/$child_id.omp-ext.ts" "$sub_state/$child_id.omp-ready" \ "$sub_state/$child_id.omp-started" "$sub_state/$child_id.omp-doorbell-ready" \ + "$sub_state/$child_id.omp-doorbell-failed" \ "$sub_state/$child_id.grok-turnend-token" "$sub_state/$child_id.kimi-turnend-token" \ "$sub_state/$child_id.hermes-turnend-token" "$sub_state/$child_id.hermes-session" \ "$sub_state/$child_id.hermes-started" @@ -3218,6 +3219,7 @@ status_retire_presentation_task "$STATE" "$ID" || exit 1 rm -f "$STATE/$ID.turn-ended" "$STATE/$ID.meta" \ "$STATE/$ID.pi-ext.ts" "$STATE/$ID.omp-ext.ts" "$STATE/$ID.omp-ready" \ "$STATE/$ID.omp-started" "$STATE/$ID.omp-doorbell-ready" \ + "$STATE/$ID.omp-doorbell-failed" \ "$STATE/$ID.grok-turnend-token" "$STATE/$ID.kimi-turnend-token" "$STATE/$ID.devin-turnend-token" \ "$STATE/$ID.hermes-turnend-token" "$STATE/$ID.hermes-session" \ "$STATE/$ID.hermes-started" \ diff --git a/docs/architecture.md b/docs/architecture.md index a9459f0608b..37066694f4f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -121,6 +121,7 @@ Ordinary local task steering uses a durable sequenced record under `state/.i `bin/fm-task-inbox-lib.sh` owns the record, sequence, doorbell, acknowledgement layout, and watcher retry ladder. The doorbell line is a shell no-op and is never typed into an endpoint classified as dead or missing; that record surfaces once for recovery instead of walking the re-ring ladder (`bin/fm-task-inbox-lib.sh` header). For an OMP worker the loaded extension delivers the doorbell through `sendMessage` and claims it delivered when a `turn_start`/`agent_start` proves a turn opened while the request was parked; the generated worker forwards its own correlated `turn_start`/`turn_end` events to the doorbell, which keeps the doorbell off a second `omp.on` subscription while preserving turn proof and closing stale turn state. +The generated OMP extension publishes `.omp-ready` only after the doorbell activates; activation or drain failure retires `.omp-doorbell-ready` and durably journals the reason in `.omp-doorbell-failed`, while `fm-spawn.sh` bounded-waits for readiness and `fm-send.sh` names the missing marker or failure journal when refusing native delivery. When the runtime downgrades `triggerTurn` to append-only, the extension re-drives the instruction through `sendUserMessage` only after its bounded grace expires without any turn opening (`.omp/extensions/lib/fm-task-inbox-doorbell.ts`). Consumed OMP delivery receipts retire as durable `.pending.acked` tombstones, so later rings report delivery without republishing or sending another doorbell; the requests directory is generation-scoped and reset with the task lifecycle. An OMP worker is reached only through its task-bound native receive adapter, never the composer, because an already-streaming session cannot be steered through editable terminal text; `fm-send.sh` reports one bounded outcome per steer - native receipt, a named durable native queue entry, or an explicit refusal - each binding the exact session and message. diff --git a/tests/fm-omp-relaunch-guard.test.sh b/tests/fm-omp-relaunch-guard.test.sh index ef5640bf91f..ae4ac68068d 100755 --- a/tests/fm-omp-relaunch-guard.test.sh +++ b/tests/fm-omp-relaunch-guard.test.sh @@ -20,6 +20,14 @@ make_relaunch_fakebin() { cat > "$fakebin/tmux" <<'SH' #!/usr/bin/env bash set -u +omp_doorbell_emulate() { # : emulate the generated extension's session_start handshake + [ -f "$1.omp-ext.ts" ] || return 0 + if [ -n "${FM_FAKE_OMP_DOORBELL_FAIL:-}" ]; then + printf '%s\n' "$FM_FAKE_OMP_DOORBELL_FAIL" > "$1.omp-doorbell-failed" + return 0 + fi + [ "${FM_FAKE_OMP_NO_DOORBELL:-0}" = 1 ] || : > "$1.omp-doorbell-ready" +} case "$*" in *"#{pane_current_path}"*) printf '%s\n' "${FM_FAKE_PANE_PATH:-}" @@ -63,9 +71,15 @@ case "${1:-}" in done if case "$*" in *Enter*) true ;; *) false ;; esac \ && grep -Fq 'FM_OMP_HARNESS=omp' "$FM_FAKE_LAUNCH_LOG" 2>/dev/null; then + for extension in "${FM_FAKE_OMP_ACK_DIR:-/nonexistent}"/*.omp-ext.ts; do + [ -e "$extension" ] || continue + omp_doorbell_emulate "${extension%.omp-ext.ts}" + done if [ -n "${FM_FAKE_OMP_ACK:-}" ]; then while IFS= read -r ack; do - [ -z "$ack" ] || : > "$ack" + [ -z "$ack" ] && continue + : > "$ack" + case "$ack" in *.omp-started) omp_doorbell_emulate "${ack%.omp-started}" ;; esac done < "$FM_TEST_HOME/state/.omp-session" version=$(bash -c '. "$1/bin/fm-primary-watch-version-lib.sh"; fm_primary_watch_version "$1/.omp/extensions/fm-primary-omp.ts" "$1"' _ "$FM_TEST_HOME") printf '%s\n%s\n%s\n%s\n' "$version" "$FM_TEST_AGENT_PID" "$FM_TEST_OMP_BUN" "$FM_TEST_OMP_BIN" > "$FM_TEST_HOME/state/.omp-primary-extension-loaded" + : > "$FM_TEST_OMP_DOORBELL_READY" printf '%s\n' "$FM_TEST_AGENT_PID" > "$FM_TEST_HOME/state/.lock" fi ;; @@ -235,6 +236,7 @@ case "$cmd $sub" in printf '%s\n' "$session" > "$FM_TEST_HOME/state/.omp-session" version=$(bash -c '. "$1/bin/fm-primary-watch-version-lib.sh"; fm_primary_watch_version "$1/.omp/extensions/fm-primary-omp.ts" "$1"' _ "$FM_TEST_HOME") printf '%s\n%s\n%s\n%s\n' "$version" "$FM_TEST_AGENT_PID" "$FM_TEST_OMP_BUN" "$FM_TEST_OMP_BIN" > "$FM_TEST_HOME/state/.omp-primary-extension-loaded" + : > "$FM_TEST_OMP_DOORBELL_READY" printf '%s\n' "$FM_TEST_AGENT_PID" > "$FM_TEST_HOME/state/.lock" fi ;; @@ -249,6 +251,7 @@ case "$cmd $sub" in printf '%s\n' "$session" > "$FM_TEST_HOME/state/.omp-session" version=$(bash -c '. "$1/bin/fm-primary-watch-version-lib.sh"; fm_primary_watch_version "$1/.omp/extensions/fm-primary-omp.ts" "$1"' _ "$FM_TEST_HOME") printf '%s\n%s\n%s\n%s\n' "$version" "$FM_TEST_AGENT_PID" "$FM_TEST_OMP_BUN" "$FM_TEST_OMP_BIN" > "$FM_TEST_HOME/state/.omp-primary-extension-loaded" + : > "$FM_TEST_OMP_DOORBELL_READY" printf '%s\n' "$FM_TEST_AGENT_PID" > "$FM_TEST_HOME/state/.lock" fi ;; @@ -303,6 +306,7 @@ run_spawn() { # [extra env NAME=VALUE ...] [-- ] FM_TEST_OMP_BIN="$TEST_OMP_BIN" \ FM_TEST_OMP_BUN="$TEST_OMP_BUN" \ FM_TEST_HOME="$HOME_DIR" \ + FM_TEST_OMP_DOORBELL_READY="$MAIN_STATE/$TASK_ID.omp-doorbell-ready" \ FM_TEST_TREEHOUSE_LOG="$CASE/treehouse.log" \ FM_TEST_STATE_MODE="${FM_TEST_STATE_MODE:-}" \ FM_TEST_SKIP_ACK="${FM_TEST_SKIP_ACK:-0}" \ @@ -331,6 +335,7 @@ run_spawn_herdr() { # [extra env NAME=VALUE ...] FM_TEST_OMP_BIN="$TEST_OMP_BIN" \ FM_TEST_OMP_BUN="$TEST_OMP_BUN" \ FM_TEST_HOME="$HOME_DIR" \ + FM_TEST_OMP_DOORBELL_READY="$MAIN_STATE/$TASK_ID.omp-doorbell-ready" \ FM_TEST_TASK_ID="$TASK_ID" \ FM_TEST_TREEHOUSE_LOG="$CASE/treehouse.log" \ FM_TEST_STATE_MODE="${FM_TEST_STATE_MODE:-}" \ @@ -459,7 +464,7 @@ test_herdr_launch_exact_resume_recovery_and_abort() { setup_case herdr-abort printf 'preserve me\n' > "$HOME_DIR/state/sentinel" out=$(FM_TEST_SKIP_ACK=1 run_spawn_herdr 2>&1) && fail "OMP Herdr secondmate launch unexpectedly succeeded without acknowledgement" - assert_contains "$out" 'preserving the persistent home' "OMP Herdr acknowledgement failure did not preserve its home contract" + assert_contains "$out" 'persistent home' "OMP Herdr acknowledgement failure did not preserve its home contract" [ -f "$HOME_DIR/state/sentinel" ] || fail "OMP Herdr secondmate abort removed persistent home state" [ -f "$MAIN_STATE/$TASK_ID.meta" ] || fail "OMP Herdr secondmate abort removed recovery metadata" [ ! -f "$WINDOW_FLAG" ] || fail "OMP Herdr secondmate abort left its owned endpoint running" @@ -682,7 +687,7 @@ test_post_meta_abort_preserves_home() { setup_case abort printf 'preserve me\n' > "$HOME_DIR/state/sentinel" out=$(FM_TEST_SKIP_ACK=1 run_spawn 2>&1) && fail "OMP secondmate launch unexpectedly succeeded without integration acknowledgement" - assert_contains "$out" 'preserving the persistent home' "OMP secondmate acknowledgement failure did not name its preservation contract" + assert_contains "$out" 'persistent home' "OMP secondmate acknowledgement failure did not name its preservation contract" [ -f "$HOME_DIR/state/sentinel" ] || fail "OMP secondmate abort removed persistent home state" [ -d "$HOME_DIR/.git" ] || fail "OMP secondmate abort removed the persistent home" [ -f "$MAIN_STATE/$TASK_ID.meta" ] || fail "OMP secondmate abort removed recovery metadata" diff --git a/tests/fm-omp-task-inbox-doorbell.test.sh b/tests/fm-omp-task-inbox-doorbell.test.sh index 963ceb4fd90..2f46da0f00f 100644 --- a/tests/fm-omp-task-inbox-doorbell.test.sh +++ b/tests/fm-omp-task-inbox-doorbell.test.sh @@ -98,10 +98,108 @@ const uncertainDoorbell = installTaskInboxDoorbell( uncertainDoorbell.activate(); writeFileSync(`${uncertain}.requests/one.pending`, line); process.emit(FM_TASK_INBOX_DOORBELL_SIGNAL); -assert.equal(existsSync(`${uncertain}.requests/one.pending.ambiguous`), true); -assert.equal(existsSync(`${uncertain}.requests/one.pending.failed`), false); +assert.equal(existsSync(`${uncertain}.requests/one.pending.ambiguous`), false); +assert.equal(existsSync(`${uncertain}.requests/one.pending`), true); assert.equal(existsSync(uncertain), false); +const asyncFailure = `${process.env.READY}.async-failure`; +const asyncFailureJournal = `${asyncFailure}.omp-doorbell-failed`; +const asyncFailing = installTaskInboxDoorbell( + { sendMessage() { return Promise.reject(new Error("async session channel closed")); } }, + { inboxDir: process.env.INBOX, readyMarker: asyncFailure, failureJournal: asyncFailureJournal }, +); +asyncFailing.activate(); +writeFileSync(`${asyncFailure}.requests/one.pending`, line); +process.emit(FM_TASK_INBOX_DOORBELL_SIGNAL); +await new Promise((resolve) => setImmediate(resolve)); +assert.equal(existsSync(asyncFailure), false); +assert.match(readFileSync(asyncFailureJournal, "utf8"), /drain: Error: async session channel closed/); +assert.equal(existsSync(`${asyncFailure}.requests/one.pending`), true); + +const multiAsync = `${process.env.READY}.multi-async`; +const multiJournal = `${multiAsync}.omp-doorbell-failed`; +mkdirSync(`${multiAsync}.requests`, { recursive: true }); +writeFileSync(`${multiAsync}.requests/first.pending`, line); +writeFileSync(`${multiAsync}.requests/second.pending`, line); +let rejectFirst; +let rejectSecond; +const firstRejection = new Promise((_, reject) => { rejectFirst = reject; }); +const secondRejection = new Promise((_, reject) => { rejectSecond = reject; }); +let multiSends = 0; +const multiDoorbell = installTaskInboxDoorbell( + { sendMessage() { + multiSends += 1; + return multiSends === 1 ? firstRejection : secondRejection; + } }, + { inboxDir: process.env.INBOX, readyMarker: multiAsync, failureJournal: multiJournal }, +); +const multiActivation = multiDoorbell.activate(); +rejectFirst(new Error("first async channel closed")); +rejectSecond(new Error("second async channel closed")); +assert.equal(await multiActivation, false); +assert.equal(existsSync(`${multiAsync}.requests/first.pending`), true); +assert.equal(existsSync(`${multiAsync}.requests/second.pending`), true); + +const concurrent = `${process.env.READY}.concurrent`; +const concurrentJournal = `${concurrent}.omp-doorbell-failed`; +mkdirSync(`${concurrent}.requests`, { recursive: true }); +writeFileSync(`${concurrent}.requests/first.pending`, line); +let releaseFirst; +let concurrentSends = 0; +const firstSend = new Promise((resolve) => { releaseFirst = resolve; }); +const concurrentDoorbell = installTaskInboxDoorbell( + { sendMessage() { + concurrentSends += 1; + if (concurrentSends === 1) return firstSend; + return Promise.reject(new Error("late concurrent channel closed")); + } }, + { inboxDir: process.env.INBOX, readyMarker: concurrent, failureJournal: concurrentJournal }, +); +const concurrentActivation = concurrentDoorbell.activate(); +assert.equal(typeof concurrentActivation.then, "function"); +writeFileSync(`${concurrent}.requests/late.pending`, line); +process.emit(FM_TASK_INBOX_DOORBELL_SIGNAL); +releaseFirst(); +assert.equal(await concurrentActivation, false); +assert.equal(existsSync(concurrent), false); +assert.match(readFileSync(concurrentJournal, "utf8"), /drain: Error: late concurrent channel closed/); + +const retiredGeneration = `${process.env.READY}.retired-generation`; +const retiredJournal = `${retiredGeneration}.omp-doorbell-failed`; +mkdirSync(`${retiredGeneration}.requests`, { recursive: true }); +writeFileSync(`${retiredGeneration}.requests/old.pending`, line); +let rejectRetired; +const retiredSend = new Promise((_, reject) => { rejectRetired = reject; }); +const retired = installTaskInboxDoorbell( + { sendMessage() { return retiredSend; } }, + { inboxDir: process.env.INBOX, readyMarker: retiredGeneration, failureJournal: retiredJournal }, +); +const retiredActivation = retired.activate(); +retired.retire(); +const successor = installTaskInboxDoorbell( + { sendMessage() {} }, + { inboxDir: process.env.INBOX, readyMarker: retiredGeneration, failureJournal: retiredJournal }, +); +assert.equal(successor.activate(), true); +rejectRetired(new Error("retired channel closed")); +assert.equal(await retiredActivation, false); +await new Promise((resolve) => setImmediate(resolve)); +assert.equal(existsSync(retiredJournal), false); +assert.equal(readFileSync(retiredGeneration, "utf8"), `${process.pid}\n`); +successor.retire(); + +const initialAsyncFailure = `${process.env.READY}.initial-async-failure`; +const initialAsyncJournal = `${initialAsyncFailure}.omp-doorbell-failed`; +mkdirSync(`${initialAsyncFailure}.requests`, { recursive: true }); +writeFileSync(`${initialAsyncFailure}.requests/one.pending`, line); +const initialAsync = installTaskInboxDoorbell( + { sendMessage() { return Promise.reject(new Error("initial async channel closed")); } }, + { inboxDir: process.env.INBOX, readyMarker: initialAsyncFailure, failureJournal: initialAsyncJournal }, +); +assert.equal(await initialAsync.activate(), false); +assert.equal(existsSync(initialAsyncFailure), false); +assert.match(readFileSync(initialAsyncJournal, "utf8"), /drain: Error: initial async channel closed/); + const unreadable = `${process.env.READY}.unreadable`; let unreadableSends = 0; const unreadableDoorbell = installTaskInboxDoorbell( @@ -323,6 +421,102 @@ JS pass "OMP doorbell driven by external turn notifications proves turns and unlatches on turn_end" } +# activate() is the handshake contract fm-spawn's generated extension gates +# .omp-ready on: it must report truthfully, journal every failure durably, and +# leave no owned marker behind when the doorbell is not live. +test_extension_activate_reports_and_journals_failures() { + local dir="$TMP_ROOT/activate-failure" + mkdir -p "$dir/state/t1.inbox" + HELPER="$HELPER" INBOX="$dir/state/t1.inbox" READY="$dir/state/t1.omp-doorbell-ready" \ + FAILED="$dir/state/t1.omp-doorbell-failed" \ + node --input-type=module <<'JS' +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const { installTaskInboxDoorbell } = await import(pathToFileURL(process.env.HELPER).href); +const requestDir = `${process.env.READY}.requests`; +const line = `Firstmate instruction waiting: list ${process.env.INBOX}/*.msg`; + +// An unconfigured doorbell cannot activate and must say so through the journal. +const stub = installTaskInboxDoorbell({}, { + inboxDir: process.env.INBOX, + readyMarker: process.env.READY, + failureJournal: process.env.FAILED, +}); +assert.equal(stub.activate(), false, "a doorbell without sendMessage must report failure"); +assert.match(readFileSync(process.env.FAILED, "utf8"), /activate: Error: OMP sendMessage is unavailable/); +stub.retire(); + +// A thrown activation journals the concrete reason and retires cleanly: no +// ready marker survives to claim a handshake that never happened. +writeFileSync(requestDir, "not a directory"); +const failing = installTaskInboxDoorbell( + { sendMessage() {} }, + { inboxDir: process.env.INBOX, readyMarker: process.env.READY, + failureJournal: process.env.FAILED }, +); +assert.equal(failing.activate(), false, "a failed activation must report failure"); +assert.equal(existsSync(process.env.READY), false, "a failed activation left its ready marker"); +assert.match(readFileSync(process.env.FAILED, "utf8"), /activate: Error: E/); +failing.retire(); +assert.equal(existsSync(process.env.READY), false); +rmSync(requestDir); + +// A drain-time failure during activation takes the doorbell down with a +// journaled reason instead of leaving a marker for a dead watcher. +mkdirSync(requestDir, { recursive: true }); +writeFileSync(`${requestDir}/stuck.pending`, line); +const draining = installTaskInboxDoorbell( + { sendMessage() { throw new Error("session channel closed"); } }, + { inboxDir: process.env.INBOX, readyMarker: process.env.READY, + failureJournal: process.env.FAILED }, +); +assert.equal(draining.activate(), false, "an activation whose first drain fails must report failure"); +assert.equal(existsSync(process.env.READY), false, "a drain failure left the ready marker"); +assert.match(readFileSync(process.env.FAILED, "utf8"), /drain: Error: session channel closed/); +assert.equal(existsSync(`${requestDir}/stuck.pending`), true, "a synchronous drain failure stranded the request"); +draining.retire(); + +// A recovered activation clears a stale journal so the marker alone is truth. +writeFileSync(process.env.FAILED, "stale reason\n"); +const recovered = installTaskInboxDoorbell( + { sendMessage() {} }, + { inboxDir: process.env.INBOX, readyMarker: process.env.READY, + failureJournal: process.env.FAILED }, +); +assert.equal(recovered.activate(), true); +assert.equal(readFileSync(process.env.READY, "utf8"), `${process.pid}\n`); +assert.equal(existsSync(process.env.FAILED), false, + "a live doorbell must clear the stale failure journal"); +recovered.retire(); +assert.equal(existsSync(process.env.READY), false); + +const derivedReady = `${process.env.READY}.derived`; +const derivedJournal = `${derivedReady}.omp-doorbell-failed`; +writeFileSync(`${derivedReady}.requests`, "not a directory"); +const derived = installTaskInboxDoorbell( + { sendMessage() {} }, + { inboxDir: process.env.INBOX, readyMarker: derivedReady }, +); +assert.equal(derived.activate(), false); +assert.match(readFileSync(derivedJournal, "utf8"), /activate: Error: E/); +derived.retire(); +rmSync(`${derivedReady}.requests`); + +const primaryState = `${process.env.INBOX}.primary-state`; +mkdirSync(primaryState, { recursive: true }); +const previousState = process.env.FM_STATE_OVERRIDE; +process.env.FM_STATE_OVERRIDE = primaryState; +const primary = installTaskInboxDoorbell({}, {}); +assert.equal(primary.activate(), false); +assert.match(readFileSync(`${primaryState}/.omp-doorbell-failed.${process.pid}`, "utf8"), /activate: Error:/); +if (previousState === undefined) delete process.env.FM_STATE_OVERRIDE; +else process.env.FM_STATE_OVERRIDE = previousState; +JS + pass "OMP extension activation reports failures, journals their reasons, and retires cleanly" +} + test_ring_routing_matrix() { local dir="$TMP_ROOT/routing" rec log mkdir -p "$dir/state/t1.inbox/handled" @@ -826,10 +1020,40 @@ test_omp_native_refusal_and_queue_are_bounded() { assert_contains "$(cat "$err")" 'do not resend' "the refusal invited a resend" assert_contains "$(cat "$err")" "record=$home/state/idle.inbox/001.msg" \ "the refusal did not name the durable record holding the exact message" + # The refusal names the concrete artifact explaining it, not just + # session-pid=unreadable: the handshake marker that never appeared. + assert_contains "$(cat "$err")" "doorbell-marker-missing=$home/state/idle.omp-doorbell-ready" \ + "the refusal did not name the missing doorbell marker" [ -f "$home/state/idle.inbox/001.msg" ] || fail "the refused steer lost its durable record" [ ! -s "$dir/composer.log" ] \ || fail "a refused OMP steer typed into the composer: $(cat "$dir/composer.log")" + dir="$TMP_ROOT/native-doorbell-failed" + home="$dir/home" + mkdir -p "$home/state" + make_send_stubs "$dir" + : > "$dir/composer.log" + write_native_meta "$home" doomed tmux "$node_bin" + printf '2026-09-01T00:00:00.000Z activate: Error: request directory creation failed\n' \ + > "$home/state/doomed.omp-doorbell-failed" + out="$dir/out"; err="$dir/err" + run_native_send "$dir" "$home" 4242 "$node_bin" "$out" "$err" \ + doomed "apply the queued fix"; rc=$? + expect_code 6 "$rc" "a steer to an OMP mate with a journaled doorbell failure must refuse" + assert_contains "$(cat "$err")" 'omp-native-refused:' \ + "the journaled doorbell failure did not produce an explicit refusal" + assert_contains "$(cat "$err")" "doorbell-failure=$home/state/doomed.omp-doorbell-failed" \ + "the refusal did not name the doorbell failure journal" + + dir="$TMP_ROOT/native-missing-request-dir" + home="$dir/home" + mkdir -p "$home/state" + printf '%s\n' "$$" > "$home/state/raced.omp-doorbell-ready" + state=$(bash -c '. "$1"; fm_task_inbox_omp_doorbell_state "$2"' _ \ + "$ROOT/bin/fm-task-inbox-lib.sh" "$home/state/raced.omp-doorbell-ready") + [ "$state" = "doorbell-request-dir-missing=$home/state/raced.omp-doorbell-ready.requests" ] \ + || fail "missing OMP request directory was misdiagnosed: $state" + dir="$TMP_ROOT/native-handled" home="$dir/home" mkdir -p "$home/state" @@ -886,6 +1110,8 @@ test_omp_native_refusal_and_queue_are_bounded() { "unbound delivered OMP reconciliation replay did not keep its session unproven" [ -f "$home/state/delivered.omp-doorbell-ready.requests/request.001.msg.pending.acked" ] \ || fail "the consumed delivery receipt did not leave a durable suppression tombstone" + assert_contains "$(cat "$err")" "doorbell-binding-unproven=$home/state/delivered.omp-doorbell-ready" \ + "the refusal did not name the readable marker whose binding stayed unproven" kill -TERM "$silent_pid" 2>/dev/null || true wait "$silent_pid" 2>/dev/null || true LISTENER_PID= @@ -956,6 +1182,8 @@ test_omp_native_binding_mismatch_is_refused() { expect_code 6 "$rc" "an unproven session binding must exit nonzero" assert_contains "$(cat "$err")" 'omp-native-refused:' \ "an unproven session binding was not explicitly refused" + assert_contains "$(cat "$err")" "doorbell-binding-unproven=$home/state/bound.omp-doorbell-ready" \ + "the refusal did not name the marker whose session binding stayed unproven" [ ! -s "$dir/composer.log" ] \ || fail "a refused binding fell back to the composer: $(cat "$dir/composer.log")" [ ! -s "$dir/signals.log" ] || fail "a mismatched binding still reached a session: $(cat "$dir/signals.log")" @@ -986,8 +1214,6 @@ test_requester_window_tracks_turn_grace_and_acked_suppresses() { set -u . "$ROOT/bin/fm-backend.sh" -# Grace 2500ms: the derived window (~2.7s) outlasts the re-drive where the -# former fixed 200-attempt window (~2s) expired first and reported queued. set +e FM_OMP_DOORBELL_TURN_GRACE_MS=2500 \ fm_omp_task_doorbell_request "$MARKER" "$PID" first.msg 'canonical doorbell' @@ -997,7 +1223,6 @@ set -e [ -f "$REQDIR/request.first.msg.pending.acked" ] \ || { echo "the consumed receipt left no .acked tombstone" >&2; exit 1; } -# The tombstone suppresses a second ring for the same record entirely. set +e fm_omp_task_doorbell_request "$MARKER" "$PID" first.msg 'canonical doorbell' rc=$? @@ -1008,7 +1233,6 @@ set -e [ "$(wc -l < "$SIGNAL_LOG" | tr -d '[:space:]')" = 2 ] \ || { echo "expected exactly one sendMessage plus one re-drive, got: $(cat "$SIGNAL_LOG")" >&2; exit 1; } -# An explicit attempt bound still wins over the derivation. set +e FM_OMP_DOORBELL_TURN_GRACE_MS=2500 FM_OMP_TASK_DOORBELL_ACK_ATTEMPTS=5 \ fm_omp_task_doorbell_request "$MARKER" "$PID" second.msg 'canonical doorbell' @@ -1026,6 +1250,7 @@ SH test_extension_signal_uses_trigger_turn test_extension_requires_turn_proof_or_redrives test_extension_external_notify_drives_turn_proof +test_extension_activate_reports_and_journals_failures test_ring_routing_matrix test_request_terminal_states test_fm_send_rings_one_programmatic_doorbell diff --git a/tests/fm-spawn-dispatch-profile.test.sh b/tests/fm-spawn-dispatch-profile.test.sh index 0bcc20d4b37..8a11715bfeb 100755 --- a/tests/fm-spawn-dispatch-profile.test.sh +++ b/tests/fm-spawn-dispatch-profile.test.sh @@ -42,6 +42,14 @@ make_spawn_fakebin() { cat > "$fakebin/tmux" <<'SH' #!/usr/bin/env bash set -u +omp_doorbell_emulate() { # : emulate the generated extension's session_start handshake + [ -f "$1.omp-ext.ts" ] || return 0 + if [ -n "${FM_FAKE_OMP_DOORBELL_FAIL:-}" ]; then + printf '%s\n' "$FM_FAKE_OMP_DOORBELL_FAIL" > "$1.omp-doorbell-failed" + return 0 + fi + [ "${FM_FAKE_OMP_NO_DOORBELL:-0}" = 1 ] || : > "$1.omp-doorbell-ready" +} case "$*" in *"#{pane_current_path}"*) printf '%s\n' "${FM_FAKE_PANE_PATH:-}"; exit 0 ;; esac @@ -75,9 +83,17 @@ case "${1:-}" in case "$*" in *Enter*) if grep -Fq 'FM_OMP_HARNESS=omp' "$FM_FAKE_LAUNCH_LOG" 2>/dev/null; then + # session_start activates the doorbell for the generated extension, + # independently of whether the first turn ever acknowledges. + for extension in "${FM_FAKE_OMP_ACK_DIR:-/nonexistent}"/*.omp-ext.ts; do + [ -e "$extension" ] || continue + omp_doorbell_emulate "${extension%.omp-ext.ts}" + done if [ -n "${FM_FAKE_OMP_ACK:-}" ]; then while IFS= read -r ack; do - [ -z "$ack" ] || : > "$ack" + [ -z "$ack" ] && continue + : > "$ack" + case "$ack" in *.omp-started) omp_doorbell_emulate "${ack%.omp-started}" ;; esac done < "${extension%.omp-ext.ts}.omp-started" + omp_doorbell_emulate "${extension%.omp-ext.ts}" done fi if [ -n "${FM_FAKE_OMP_META_TAMPER:-}" ]; then @@ -113,6 +130,14 @@ SH cat > "$fakebin/herdr" <<'SH' #!/usr/bin/env bash set -u +omp_doorbell_emulate() { # : emulate the generated extension's session_start handshake + [ -f "$1.omp-ext.ts" ] || return 0 + if [ -n "${FM_FAKE_OMP_DOORBELL_FAIL:-}" ]; then + printf '%s\n' "$FM_FAKE_OMP_DOORBELL_FAIL" > "$1.omp-doorbell-failed" + return 0 + fi + [ "${FM_FAKE_OMP_NO_DOORBELL:-0}" = 1 ] || : > "$1.omp-doorbell-ready" +} cmd=${1:-} sub=${2:-} case "$cmd $sub" in @@ -203,10 +228,17 @@ case "$cmd $sub" in if [ -n "${FM_FAKE_LAUNCH_LOG:-}" ]; then printf '%s\n' "${4:-}" >> "$FM_FAKE_LAUNCH_LOG" if printf '%s' "${4:-}" | grep -Fq 'FM_OMP_HARNESS=omp'; then - [ -z "${FM_FAKE_OMP_ACK:-}" ] || : > "$FM_FAKE_OMP_ACK" + for extension in "${FM_FAKE_OMP_ACK_DIR:-/nonexistent}"/*.omp-ext.ts; do + [ -e "$extension" ] || continue + omp_doorbell_emulate "${extension%.omp-ext.ts}" + done + if [ -n "${FM_FAKE_OMP_ACK:-}" ]; then + : > "$FM_FAKE_OMP_ACK" + omp_doorbell_emulate "${FM_FAKE_OMP_ACK%.omp-started}" + fi if [ "${FM_FAKE_OMP_DYNAMIC_ACK:-0}" = 1 ]; then ack=$(printf '%s\n' "${4:-}" | sed -n "s/.* -e '\([^']*\)\.omp-ext\.ts'.*/\1.omp-started/p") - [ -z "$ack" ] || : > "$ack" + [ -z "$ack" ] || { : > "$ack"; omp_doorbell_emulate "${ack%.omp-started}"; } fi fi fi @@ -218,7 +250,14 @@ case "$cmd $sub" in case "${4:-}" in enter) if grep -Fq 'FM_OMP_HARNESS=omp' "${FM_FAKE_LAUNCH_LOG:-/dev/null}" 2>/dev/null; then - [ -z "${FM_FAKE_OMP_ACK:-}" ] || : > "$FM_FAKE_OMP_ACK" + for extension in "${FM_FAKE_OMP_ACK_DIR:-/nonexistent}"/*.omp-ext.ts; do + [ -e "$extension" ] || continue + omp_doorbell_emulate "${extension%.omp-ext.ts}" + done + if [ -n "${FM_FAKE_OMP_ACK:-}" ]; then + : > "$FM_FAKE_OMP_ACK" + omp_doorbell_emulate "${FM_FAKE_OMP_ACK%.omp-started}" + fi fi ;; esac @@ -444,6 +483,9 @@ run_spawn() { FM_FAKE_TREEHOUSE_LOG="$treehouselog" FM_FAKE_OMP_ACK="${FM_TEST_OMP_ACK:-}" \ FM_FAKE_OMP_DYNAMIC_ACK="${FM_TEST_OMP_DYNAMIC_ACK:-0}" FM_FAKE_OMP_ACK_DIR="$home/state" \ FM_FAKE_OMP_NO_PREWALK="${FM_TEST_OMP_NO_PREWALK:-1}" \ + FM_FAKE_OMP_NO_DOORBELL="${FM_TEST_OMP_NO_DOORBELL:-0}" \ + FM_FAKE_OMP_DOORBELL_FAIL="${FM_TEST_OMP_DOORBELL_FAIL:-}" \ + FM_OMP_DOORBELL_ACK_POLLS="${FM_TEST_OMP_DOORBELL_ACK_POLLS:-}" \ FM_FAKE_OMP_PREWALK_ENABLED="${FM_TEST_OMP_PREWALK_ENABLED:-false}" \ FM_FAKE_OMP_CATALOG_DIR="${FM_TEST_OMP_CATALOG_DIR:-}" \ FM_FAKE_MKDIR_FAIL_PATH="${FM_TEST_MKDIR_FAIL_PATH:-}" \ @@ -2462,7 +2504,10 @@ import { existsSync } from "node:fs"; import { pathToFileURL } from "node:url"; const handlers = new Map(); const extension = await import(pathToFileURL(process.env.PLUGIN).href); -extension.default({ on(name, handler) { handlers.set(name, handler); } }); +extension.default({ + sendMessage() {}, + on(name, handler) { handlers.set(name, handler); }, +}); await handlers.get("session_start")?.(); await handlers.get("turn_start")?.(); await handlers.get("turn_end")?.(); @@ -2472,6 +2517,8 @@ for (let i = 0; i < 50 && (!existsSync(process.env.READY) || !existsSync(process if (!existsSync(process.env.READY)) throw new Error("OMP session_start did not report readiness"); if (!existsSync(process.env.STARTED)) throw new Error("OMP turn_start did not acknowledge launch"); if (!existsSync(process.env.TURNENDED)) throw new Error("OMP turn_end did not publish completion"); +// Retire the activated doorbell so its watcher does not pin the event loop. +await handlers.get("session_shutdown")?.(); JS unset FM_TEST_OMP_ACK pass "OMP scouts retain scout semantics and external per-turn notification" @@ -2562,6 +2609,83 @@ JS pass "generated OMP worker extension observes turns and recovers a parked steer through the user channel" } +# The generated worker extension must not publish .omp-ready when its doorbell +# cannot activate: that combination previously left a false-ready worker whose +# fm-send refusals all surfaced as session-pid=unreadable with no diagnostic. +test_omp_worker_extension_gates_ready_on_doorbell_activation() { + local rec id out status + id=$(profile_id profile-omp-doorbell-gate) + rec=$(make_spawn_case profile-omp-doorbell-gate omp "$id") + read_case_record "$rec" + export FM_TEST_OMP_ACK="$HOME_DIR/state/$id.omp-started" + out=$(run_ship_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$id" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "OMP worker spawn should succeed" + rm -f "$HOME_DIR/state/$id.omp-ready" "$HOME_DIR/state/$id.omp-started" \ + "$HOME_DIR/state/$id.omp-doorbell-ready" "$HOME_DIR/state/$id.omp-doorbell-failed" + PLUGIN="$HOME_DIR/state/$id.omp-ext.ts" \ + READY="$HOME_DIR/state/$id.omp-ready" \ + DOORBELL_READY="$HOME_DIR/state/$id.omp-doorbell-ready" \ + JOURNAL="$HOME_DIR/state/$id.omp-doorbell-failed" \ + node --input-type=module <<'JS' +import assert from "node:assert/strict"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +const handlers = new Map(); +const extension = await import(pathToFileURL(process.env.PLUGIN).href); +extension.default({ + sendMessage() {}, + on(name, handler) { handlers.set(name, handler); }, +}); +// Force activate() to fail: the request directory cannot be created over a file. +writeFileSync(`${process.env.DOORBELL_READY}.requests`, "not a directory"); +await handlers.get("session_start")(); +await new Promise((resolve) => setTimeout(resolve, 100)); +assert.equal(existsSync(process.env.READY), false, + ".omp-ready published without a live doorbell"); +assert.equal(existsSync(process.env.DOORBELL_READY), false, + "doorbell marker published from a failed activation"); +assert.equal(existsSync(process.env.JOURNAL), true, "activation failure was not journaled"); +assert.match(readFileSync(process.env.JOURNAL, "utf8"), /activate: /); +JS + unset FM_TEST_OMP_ACK + pass "generated OMP worker extension gates .omp-ready on doorbell activation and journals the failure" +} + +# Spawn-time verification: fm-spawn bounded-waits for the doorbell handshake so +# a worker whose activate() lost the race fails the spawn loudly instead of +# idling as a false-ready endpoint that can never accept owned redelivery. +test_omp_spawn_requires_doorbell_handshake() { + local rec id out status + id=$(profile_id profile-omp-no-doorbell) + rec=$(make_spawn_case profile-omp-no-doorbell omp "$id") + read_case_record "$rec" + export FM_TEST_OMP_ACK="$HOME_DIR/state/$id.omp-started" + export FM_TEST_OMP_NO_DOORBELL=1 + export FM_TEST_OMP_DOORBELL_ACK_POLLS=3 + out=$(run_ship_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$id" "$PROJ_DIR") + status=$? + expect_code 1 "$status" "an OMP spawn whose doorbell never activates must fail" + assert_contains "$out" ".omp-doorbell-ready" \ + "the spawn failure did not name the missing doorbell marker" + unset FM_TEST_OMP_ACK FM_TEST_OMP_NO_DOORBELL FM_TEST_OMP_DOORBELL_ACK_POLLS + + id=$(profile_id profile-omp-doorbell-fail) + rec=$(make_spawn_case profile-omp-doorbell-fail omp "$id") + read_case_record "$rec" + export FM_TEST_OMP_ACK="$HOME_DIR/state/$id.omp-started" + export FM_TEST_OMP_DOORBELL_FAIL="watcher fd exhausted" + out=$(run_ship_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$id" "$PROJ_DIR") + status=$? + expect_code 1 "$status" "an OMP spawn whose doorbell activation fails must fail the spawn" + assert_contains "$out" ".omp-doorbell-failed" \ + "the spawn failure did not name the doorbell failure journal" + assert_contains "$out" "watcher fd exhausted" \ + "the spawn failure did not surface the journaled activation reason" + unset FM_TEST_OMP_ACK FM_TEST_OMP_DOORBELL_FAIL + pass "OMP spawn bounded-waits on .omp-doorbell-ready and fails loudly on absence or journaled failure" +} + test_omp_whitespace_identity_paths_refuse_before_endpoint() { local mode rec id out status spaced path for mode in omp bun; do @@ -3255,6 +3379,8 @@ test_herdr_launch_refuses_after_nested_shell_timeout test_herdr_spawn_uses_acquisition_owned_worktree_handoff test_omp_scout_uses_external_turn_extension test_omp_worker_doorbell_observes_turns_and_recovers_parked_steer +test_omp_worker_extension_gates_ready_on_doorbell_activation +test_omp_spawn_requires_doorbell_handshake test_omp_whitespace_identity_paths_refuse_before_endpoint test_omp_missing_binary_or_capability_refuses_before_endpoint_and_metadata test_omp_launch_requires_observable_turn_start_acknowledgement diff --git a/tests/remote-herdr-fixture.sh b/tests/remote-herdr-fixture.sh index 176fd9a50a3..ff9ebcc9467 100644 --- a/tests/remote-herdr-fixture.sh +++ b/tests/remote-herdr-fixture.sh @@ -53,7 +53,7 @@ printf '%s\n' "$*" >> "$LOG" jq_state() { jq "$@" "$STATE"; } save() { tmp="$STATE.tmp.$$"; cat > "$tmp" && mv "$tmp" "$STATE"; } publish_omp_ack() { # - local pane=$1 launch=$2 cwd session version + local pane=$1 launch=$2 cwd session version normalized_launch [ -n "$OMP_ACK_PID" ] && [ -n "$OMP_BUN" ] && [ -n "$OMP_BIN" ] || return 0 case "$launch" in *FM_OMP_SESSION_POINTER=*) @@ -67,6 +67,12 @@ publish_omp_ack() { # printf '%s\n%s\n%s\n%s\n' "$version" "$OMP_ACK_PID" "$OMP_BUN" "$OMP_BIN" \ > "$cwd/state/.omp-primary-extension-loaded" printf '%s\n' "$OMP_ACK_PID" > "$cwd/state/.lock" + # Remote launches are wrapped in Bash and shell_quote escapes embedded + # single quotes as '\\''... '\\''. Normalize both forms before parsing. + normalized_launch=${launch//\'/} + normalized_launch=${normalized_launch//\\/} + doorbell=$(printf '%s' "$normalized_launch" | sed -n 's/.*FM_OMP_TASK_DOORBELL_READY=\([^ ]*\).*/\1/p') + [ -z "$doorbell" ] || : > "$doorbell" jq_state --arg p "$pane" --arg session "$session" '.omp_session[$p] = $session' | save ;; esac