diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 506f398cce9..7a84e40965f 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -81,13 +81,15 @@ FM_CLASSIFY_CAPTAIN_RE_DEFAULT='done:|needs-decision:|blocked:|failed:|PR ready| # paused: # to declare it is intentionally idling on a KNOWN external dependency - an # upstream release, a vendor rate-limit reset, a scheduled window. Unlike -# `blocked:` (stuck, firstmate must help) an idle `paused:` pane is EXPECTED, so -# the stale path absorbs it instead of escalating a possible wedge. It is +# `blocked:` (stuck, firstmate must help) a `paused:` pane is EXPECTED to look +# unattended - idle, or busy in a polling or watch loop - so the wedge path +# absorbs it instead of escalating (status_pause_damps_wedge below). It is # deliberately NOT in the captain-relevant set above: a pause is a "stop -# wedge-nagging this idle pane" signal, not work to keep surfacing. This constant -# is the ONE definition of the verb; both the watcher and the daemon read it here -# (status_is_paused) rather than hardcoding the literal, so the vocabulary cannot -# drift between the two consumers. FM_CLASSIFY_PAUSED_VERB overrides it. +# wedge-nagging this unattended pane" signal, not work to keep surfacing. This +# constant is the ONE definition of the verb; both the watcher and the daemon +# read it here (status_is_paused) rather than hardcoding the literal, so the +# vocabulary cannot drift between the two consumers. FM_CLASSIFY_PAUSED_VERB +# overrides it. FM_CLASSIFY_PAUSED_VERB_DEFAULT='paused' # Bounded re-surface cadence for a declared pause or a verified captain hold. @@ -160,6 +162,21 @@ status_is_paused() { # [ "$verb" = "${FM_CLASSIFY_PAUSED_VERB:-$FM_CLASSIFY_PAUSED_VERB_DEFAULT}" ] } +# 0 if declares the current external wait that damps wedge aging. +# The always-on watcher applies this predicate before choosing the shorter wedge +# cadence, including when a reason already carries wedge decoration; the +# away-mode daemon applies the same precedence through its own declared-wait +# verdict (status_is_paused_or_captain_held below). +# Only the latest status line is passed, so any newer non-pause line restores +# ordinary wedge detection. Callers that already hold an authoritative run-step +# verdict keep its existing precedence over the status declaration. Deliberately +# narrower than status_is_paused_or_captain_held below: a pause predicts an +# unattended pane outright, while a captain hold earns the shared bounded cadence +# only through each caller's own liveness or idleness evidence. +status_pause_damps_wedge() { # + status_is_paused "$1" +} + # 0 if a status line's leading verb is the verified captain-held transfer verb. # The same pure verb read as status_is_paused, and the discriminator a supervisor # needs once a declared wait has already been recognized: the two declarations get diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index caa39443b3b..3f685d54ce6 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -417,7 +417,7 @@ classify_stale() { # [ ] fi if [ -n "$last" ] && status_is_paused_or_captain_held "$last"; then # A DECLARED external-wait pause or a verified captain-held transfer - # (fm-classify-lib.sh owns which declarations qualify): an idle pane is + # (fm-classify-lib.sh owns which declarations qualify): an unattended pane is # EXPECTED, so this is not a wedge. The caller records a pause marker (long # re-surface cadence in housekeeping) rather than a wedge stale marker. Cheap: # reuses the status line already read, no fm-crew-state.sh call, mirroring the diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 23042e9c4b7..f6e72bdf22a 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -9,7 +9,7 @@ # since the previous poll. Every other no-verb wake surfaces, so a crew # that finishes (or stops and waits) is never silently swallowed. A declared wait, # either a paused: external wait or a verified captain-held transfer, is the -# separate idle absorb case and re-surfaces only on its long bounded cadence, +# separate absorb case and re-surfaces only on its long bounded cadence, # although its initial no-verb status signal still surfaces in normal mode. # While state/.afk exists, the daemon owns triage and this watcher queues and exits # on every wake. Printed reason lines: @@ -45,10 +45,10 @@ # (window_is_busy true) is exempt from the above, but # only up to BUSY_TURN_MAX_SECS with no completed turn # (state/.turn-ended, or the spawn record before any -# turn completes). Past that bound, a declared external -# wait or verified captain-held transfer uses the long -# pause recheck cadence (under afk it is instead handed -# to the daemon as this plain reason, once per +# turn completes). Past that bound, a current declared +# external wait or verified captain-held transfer uses +# the long pause recheck cadence (under afk it is instead +# handed to the daemon as this plain reason, once per # declaration; busy_turn_bound_check owns that handoff); # every other pane goes through the same wedge timer and # surfaces with the identical "stale: ..." reason, @@ -217,15 +217,19 @@ BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600} # A local secondmate's foreign queue is checked on every poll, but only after this # bounded age can it produce a parent notification. SECONDMATE_WAKE_STALL_SECS=${FM_SECONDMATE_WAKE_STALL_SECS:-60} -# A crew that declared a pause is idling on a known external wait, so its stale -# pane is absorbed rather than wedge-escalated. -# A captain-held or paused crew whose agent has confidently exited uses the same -# bounded cadence, while a live or ambiguously read agent surfaces on first sight -# and is then held to that same cadence; a secondmate earns the cadence on its -# declaration alone, because its endpoint liveness is deliberately never read -# (pause_state_class owns that split). +# A crew whose latest status declares a pause is waiting on a known external +# dependency, idle or busy in its own poll loop, so it takes this bounded cadence +# on that declaration alone rather than a wedge escalation; a newer non-pause +# status restores ordinary wedge detection. +# A captain-held crew earns the same cadence on the stale path only once its +# agent has confidently exited, while a live or ambiguously read agent surfaces +# on first sight and is then held to that same cadence; a secondmate's hold earns +# it on the declaration alone, because its endpoint liveness is deliberately +# never read (pause_state_class owns that split, and busy_turn_bound_check owns +# the busy-pane bound). # These cases re-surface once for a recheck every PAUSE_RESURFACE_SECS - far -# longer than the wedge threshold, but finite so a forgotten hold cannot rot invisibly. +# longer than the wedge threshold, but finite so a forgotten pause or hold cannot +# rot invisibly. PAUSE_RESURFACE_SECS=${FM_PAUSE_RESURFACE_SECS:-$FM_PAUSE_RESURFACE_SECS_DEFAULT} # Consecutive event-path failures (fm_backend_wait_transition returning 2 - # connect/subscribe failure) before the push fast-path is disabled for the rest @@ -868,10 +872,10 @@ busy_turn_bound_check() { # here: the re-surface throttle belongs to the +# declaration, not to one absorb, so a pane that flickers busy mid-wait (a short +# turn that appends no status line) re-enters the cadence without an off-schedule +# recheck. The main loop drops the throttle once the latest status stops +# declaring a wait (tests/fm-watch-triage.test.sh pins the flicker case). clear_pause_state() { # local key=$1 - rm -f "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" "$STATE/.paused-resurfaced-$key" + rm -f "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" } # The hash-scoped half of clear_pause_tracking: the stale suppressor, its wedge @@ -924,9 +933,11 @@ clear_pause_tracking() { # } # Reconcile a declared pause or captain-held status with authoritative crew state. -# After fm-crew-state has fallen back to stopped or unknown, paused classification is -# recovered only for a confidently dead ordinary crew, or for a secondmate, whose -# endpoint liveness this function deliberately never reads. +# A current declared pause owns the long cadence unless an authoritative run-step +# says work resumed. After fm-crew-state has fallen back to stopped or unknown, a +# captain-held classification is recovered only for a confidently dead ordinary +# crew, or for a secondmate, whose endpoint liveness this function deliberately +# never reads. pause_state_class() { # local win=$1 task=$2 key last recheck_file class agent_alive kind key=$(window_key "$win") @@ -942,6 +953,10 @@ pause_state_class() { # # far more common no-declaration path above still costs none. kind=$(window_kind "$win") if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then + if status_pause_damps_wedge "$last"; then + printf 'paused' + return + fi if [ "$kind" != secondmate ]; then agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown if [ "$agent_alive" != dead ]; then @@ -959,7 +974,9 @@ pause_state_class() { # printf 'working' return fi - if [ "$kind" != secondmate ]; then + if status_pause_damps_wedge "$last"; then + class=paused + elif [ "$kind" != secondmate ]; then agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown if [ "$agent_alive" != dead ]; then rm -f "$recheck_file" @@ -1815,8 +1832,9 @@ EOF [ -z "$task" ] || inbox_steer_check "$w" "$task" key=$(window_key "$w") last=$(last_status_line "$STATE/$task.status") - if ! status_is_paused_or_captain_held "$last" && [ -e "$STATE/.paused-$key" ]; then - clear_pause_tracking "$key" + if ! status_is_paused_or_captain_held "$last"; then + rm -f "$STATE/.paused-resurfaced-$key" + [ ! -e "$STATE/.paused-$key" ] || clear_pause_tracking "$key" fi # An idle secondmate endpoint is healthy by design, so a mate is admitted to # the pane-stale path ONLY to serve a declared wait's bounded re-surface - @@ -1913,9 +1931,11 @@ EOF # - working: an actively-running pipeline legitimately sits on a static # pane (e.g. waiting on CI), so absorb and start the wedge timer so a # genuinely frozen run still escalates past STALE_ESCALATE_SECS; - # - paused: a declared wait pause_state_class admits (its header owns which - # liveness evidence each kind of crew must supply), so absorb on the long - # PAUSE_RESURFACE_SECS cadence instead of wedge-escalating; + # - paused: a declared wait pause_state_class admits - the latest status + # still declares an external wait, or a captain hold supplied the + # liveness evidence its header requires of that kind of crew - so + # absorb on the long PAUSE_RESURFACE_SECS cadence instead of + # wedge-escalating; # - none: no running pipeline, no exact busy verdict, no admitted declared wait. # Surface immediately so firstmate inspects the inconclusive state # (it may be done via an interactive menu that wrote no done: status, @@ -1967,8 +1987,8 @@ EOF fi # A busy pane normally means real work resumed, so stale pause bookkeeping # is cleared - but not in the same poll the declared-pause cadence just - # recorded it, or the re-surface throttle it depends on would be erased and - # the pause would re-surface every poll instead of once per long cadence. + # recorded it, or a pane that flickers busy would flap the bookkeeping on + # every poll of a still-current pause. if [ "$paused_bound" -ne 0 ] && [ -e "$pf" ] && { [ "$n" -ge 2 ] || ! status_is_paused_or_captain_held "$(last_status_line "$STATE/$(window_to_task "$w" "$STATE").status")"; }; then clear_pause_tracking "$key" fi diff --git a/docs/architecture.md b/docs/architecture.md index 2d67e57086e..287f908274b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,9 +16,9 @@ That deferral re-surfaces on the same `FM_PAUSE_RESURFACE_SECS` cadence as a dec Every absence of write evidence, including a missing worktree record, a torn-down worktree, a walk that outlives its wall-clock bound on a hung mount, and a failed walk, leaves the existing escalation schedule untouched, so a crew that writes nothing still escalates exactly as before. A secondmate's recorded worktree is never probed for write activity, because it is a provisioned firstmate home whose own supervision keeps writing inside it whether or not the mate produces anything, so its panes keep escalating on the unchanged schedule. A busy pane is otherwise exempt from staleness, but only until its latest `state/.turn-ended` marker reaches `FM_BUSY_TURN_MAX_SECS`, or its `state/.meta` spawn record reaches that age before any turn completes; past that bound it is routed through the same wedge escalation, with the identical reason, escalation count, worktree-write deferral, and `demand-deep-inspection` marker, for inspection only - never an automatic interrupt, signal, or restart. -A crew that declared an external wait (`paused:`) or a verified captain-held transfer is the one exception to that bound: its busy verdict supplies liveness while identifying the long-running foreground call as the declared wait, so it takes the bounded `FM_PAUSE_RESURFACE_SECS` recheck instead of a wedge escalation. -Lifting the declaration restores the unchanged busy-pane wedge path, while a pane that is no longer busy returns to the existing idle declared-wait classification. -While away mode is active, a busy pane that crosses the bound under a declared wait is handed to the daemon as the plain wake identity instead of taking that recheck in the watcher, because the daemon owns triage there and a wake already decorated as a possible wedge would override the daemon's own declared-wait verdict; an undeclared busy pane past the bound still takes the wedge escalation in away mode. +A crew whose latest status declares an external wait (`paused:`) or a verified captain-held transfer is the one exception to that bound: its busy verdict supplies liveness while identifying the long-running foreground call as the declared wait, so it takes the bounded `FM_PAUSE_RESURFACE_SECS` recheck instead of a wedge escalation. +A newer non-pause status restores the unchanged busy-pane wedge path, while a pane that is no longer busy returns to the existing declared-wait classification. +While away mode is active, a busy pane that crosses the bound under a declared wait is handed to the daemon as the plain wake identity instead of taking that recheck in the watcher, because the daemon owns triage there and classifies the declaration itself, so the watcher's wedge ladder never climbs against a wait the daemon already tracks on the pause cadence; an undeclared busy pane past the bound still takes the wedge escalation in away mode. That handoff is keyed on the declaration itself (the status log's signature) rather than on the pane capture, so a harness footer that ticks on every poll wakes the daemon once per declaration instead of once per poll, and it clears the wedge timer, escalation count, and worktree-write deferral exactly as the normal-mode absorber does, so an undeclared busy phase's timer does not resume when the declaration lifts. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) only after generation-bound recovery evidence is published, so an interrupted watcher or handling turn can be recovered without losing the queue record. Agent endpoint liveness and queue-consumption liveness are separate: on each poll, the primary watcher reads the oldest valid row from every endpoint-recorded local secondmate home's durable wake queue without locking, consuming, or rewriting that foreign queue. @@ -41,8 +41,10 @@ The deferral is bounded per endpoint by `FM_TURNEND_CHURN_ABSORB_SECS`, tracked That bound is load-bearing rather than cosmetic: churn and staleness read the same pane, so a pane that renders continuously - a clock, a spinner, a shell heartbeat, or a harness that leaves a background renderer alive after its agent yields - never reaches the staleness backbone's two-identical-hashes test either, and an unbounded churn absorb would leave a genuinely stopped worker behind such a renderer with no path left to surface it. If two metadata records derive the same per-window marker key, including two records that name the same endpoint, that marker is not attributable churn evidence for either task, so the bare turn-ended wake surfaces without changing or migrating existing marker state. A `kind=secondmate` task's status signal is the parent-directed reply stream and is never absorbed as provably working; its bare turn-ended signal is absorbed only by the ordinary authoritative working proof because an active secondmate does not enter the staleness backbone that would resurface deferred pane-churn evidence. -A crew that declares `paused:` for a known external wait, or carries a verified `captain-held` transfer, is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge. -For an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies that same cadence to an unchanged `paused:` or durable `captain-held` endpoint; the pause classification itself is recovered only when the backend confidently reports its agent dead. +A crew whose latest status declares `paused:` for a known external wait is separately absorbed whether its pane is idle or busy in the wait's own poll loop, and is re-surfaced only on the longer pause cadence rather than being treated as a possible wedge. +A newer non-pause status restores ordinary wedge aging on the next poll, while an authoritative run-step keeps its existing precedence during idle classification. +`status_pause_damps_wedge` in `bin/fm-classify-lib.sh` owns this latest-line rule for the always-on watcher, including whether an already-decorated wedge reason may override it; the away-mode daemon applies the same precedence through its own declared-wait verdict. +A verified `captain-held` transfer takes that same cadence on stricter evidence: for an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies the cadence to an unchanged durable `captain-held` endpoint; the hold classification itself is recovered only when the backend confidently reports its agent dead. Live or inconclusive liveness remains fail-open at that initial surface, so a worker genuinely waiting on a decision is never silenced. Its later sights are still held to that same bounded cadence rather than re-alarming on every pane-hash change, because the throttle is keyed to the declaration and not to the pane an idle parked worker keeps ticking. A secondmate's endpoint liveness is still never read at all; a mate is admitted to that same cadence only to serve a declared wait's bounded re-surface, so a forgotten pause or captain hold on a mate cannot rot invisibly. diff --git a/docs/configuration.md b/docs/configuration.md index d89b5837208..44cd8fa1e6b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -867,9 +867,9 @@ FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals FM_TURNEND_CHURN_ABSORB_SECS=900 # longest one endpoint's bare turn-ends may be deferred on pane-churn evidence alone; only consulted when config/turnend-churn-absorb is present FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked -FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless admitted directly to the declared-wait cadence, while a live idle declared wait still surfaces once before that cadence bounds repeats -FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a declared external wait or verified captain-held transfer takes the FM_PAUSE_RESURFACE_SECS recheck below instead -FM_PAUSE_RESURFACE_SECS=3600 # seconds between bounded rechecks of a declared external wait or verified captain-held transfer, including a live idle pane after its first inconclusive stale wake and a live busy pane past FM_BUSY_TURN_MAX_SECS; the away-mode daemon uses the same setting, ageing its window against the crew's own latest status line rather than pane busy state +FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb or are otherwise admitted directly to the declared-wait cadence, while a live idle captain-held wait still surfaces once before that cadence bounds repeats +FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a current declared external wait or verified captain-held transfer takes the FM_PAUSE_RESURFACE_SECS recheck below instead +FM_PAUSE_RESURFACE_SECS=3600 # seconds between bounded rechecks of a current declared external wait (whether its pane is idle or busy in the wait's own poll loop) or verified captain-held transfer, including a live idle captain-held pane after its first inconclusive stale wake and a live busy pane past FM_BUSY_TURN_MAX_SECS; the away-mode daemon uses the same setting, ageing its window against the crew's own latest status line rather than pane busy state FM_SECONDMATE_WAKE_STALL_SECS=60 # minimum age of the oldest valid foreign wake-queue row before an endpoint-recorded local secondmate produces one durable parent wake-loop-stall notification; zero or invalid values use 60 FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WORKTREE_WRITE_PRUNE='.git node_modules .venv venv __pycache__ .mypy_cache .pytest_cache .ruff_cache .tox target dist build .next .cache vendor' # directory names the wedge detector's task-worktree write probe skips; the default keeps .git out so a supervisor's own read-only git command can never look like crew progress; set it to the empty string to prune nothing, which widens the probe to the whole depth-bounded tree rather than disabling it diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 2b8d2933c4b..fb1000d3794 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1928,14 +1928,97 @@ test_nonterminal_stale_paused_absorbed_then_resurfaced() { pass "a declared pause is absorbed on first sight, then re-surfaced as a recheck past the threshold, never wedge-escalated" } +# The re-surface throttle belongs to the declaration, not to any one absorb. A +# paused crew that completes a short turn without appending a status line (a +# reply to a steer, a harness whose poll is itself a turn) flips its pane busy +# for a poll, which clears the stale pause bookkeeping, then settles idle on a +# new hash. That idle poll must NOT re-surface the still-current pause merely +# because the busy poll ran: the recent throttle keeps it on the long +# FM_PAUSE_RESURFACE_SECS cadence, and the recheck still fires once the throttle +# itself ages past the cadence, so the kept throttle bounds rather than silences. +test_paused_busy_flicker_keeps_resurface_cadence() { + local dir state fakebin out capture_file window key statusf sig pid back + dir=$(make_case paused-busy-flicker); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-paused-flicker" + statusf="$state/paused-flicker.status" + printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/paused-flicker.meta" + printf 'paused: holding for the upstream tool release\n' > "$statusf" + back=$(( $(date +%s) - 500 )) + set_mtime "$back" "$statusf" + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-paused-flicker_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + # The pause was absorbed on an idle hash and re-surfaced for a recheck moments ago. + printf '%s' "$(hash_text "idle, holding for upstream")" > "$state/.hash-$key" + printf '%s' "$(hash_text "idle, holding for upstream")" > "$state/.stale-$key" + printf '2\n' > "$state/.count-$key" + : > "$state/.paused-$key" + date +%s > "$state/.paused-rechecked-$key" + date +%s > "$state/.paused-resurfaced-$key" + # A short turn just completed: the pane reads busy with a fresh completed-turn + # marker, well inside the busy-turn bound, and no new status line. + record_pi_busy "$state" paused-flicker + touch "$state/paused-flicker.turn-ended" + prime_turnend_seen "$state/paused-flicker.turn-ended" + printf 'Working... (2.1s)' > "$capture_file" + export FM_FAKE_CREW_STATE='state: paused · source: status-log · holding for the upstream tool release' + + # Phase A: the busy poll clears the stale pause bookkeeping and wakes nothing. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_BUSY_TURN_MAX_SECS=999 FM_STALE_ESCALATE_SECS=999 FM_PAUSE_RESURFACE_SECS=240 \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a paused crew's short busy turn surfaced a wake: $(cat "$out")"; } + reap "$pid" + [ ! -s "$out" ] || fail "a paused crew's short busy turn printed a wake reason: $(cat "$out")" + [ ! -e "$state/.paused-$key" ] || fail "a busy pane left the stale pause bookkeeping in place" + [ -e "$state/.paused-resurfaced-$key" ] || fail "a busy pane erased the declared pause's re-surface throttle" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional busy-flicker phase-A stop" + + # Phase B: the turn is over and the pane settles idle on a new hash while the + # pause is still current and already past the cadence. The recent throttle + # holds: absorbed again on the cadence, no wake, no wedge timer. + "$ROOT/bin/fm-busy-event.sh" apply "$state" paused-flicker idle --current-gen \ + --source pi-ext --event agent-end || fail "could not settle the fake pi pane idle" + printf 'idle again, holding for upstream' > "$capture_file" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_BUSY_TURN_MAX_SECS=999 FM_STALE_ESCALATE_SECS=999 FM_PAUSE_RESURFACE_SECS=240 \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a still-current pause re-surfaced right after a busy flicker instead of on its cadence: $(cat "$out")"; } + reap "$pid" + [ ! -s "$out" ] || fail "a busy flicker printed an off-cadence pause recheck: $(cat "$out")" + [ -e "$state/.paused-$key" ] || fail "the settled pane did not re-enter the pause cadence" + [ ! -e "$state/.stale-since-$key" ] || fail "the settled pane started a wedge timer under a declared pause" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional busy-flicker phase-B stop" + + # Phase C: once the kept throttle itself ages past the cadence, the pause still + # rechecks - bounded, never silenced, never a wedge. + set_mtime "$back" "$state/.paused-resurfaced-$key" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_BUSY_TURN_MAX_SECS=999 FM_STALE_ESCALATE_SECS=999 FM_PAUSE_RESURFACE_SECS=240 \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "the kept throttle silenced the declared pause's bounded recheck" + grep -F "stale: $window" "$out" >/dev/null || fail "the bounded recheck did not print a stale wake: $(cat "$out")" + grep -F "awaiting external" "$out" >/dev/null || fail "the bounded recheck was not labeled a paused/awaiting-external recheck: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null && fail "a declared pause was mislabeled a possible wedge after a busy flicker: $(cat "$out")" + unset FM_FAKE_CREW_STATE + pass "a paused crew's short busy turn keeps the bounded re-surface cadence instead of an immediate recheck" +} + # A captain-held crew can leave a stable backend endpoint after its agent exits. # fm-crew-state then authoritatively reports stopped rather than paused, but the # confirmed-dead agent plus the declared wait or captain-held transfer must retain # bounded pause handling. -# A still-live agent at an external-decision gate is the disconfirming case: it -# must surface once, while the unchanged hash must not append the same wake on -# every watcher re-arm. -test_exited_declared_pause_is_bounded_but_live_gate_surfaces() { +# A still-live agent with a current declared pause is the disconfirming case for +# the old liveness override: it must take the same bounded pause cadence rather +# than surfacing immediately or entering repeated wedge escalation. +test_declared_pause_and_exited_captain_hold_are_bounded() { local dir state fakebin out capture_file statusf window key pane_hash sig pid back round wakes bare dir=$(make_case exited-declared-pause); state="$dir/state"; fakebin="$dir/fakebin" out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/held.status" @@ -2020,38 +2103,38 @@ test_exited_declared_pause_is_bounded_but_live_gate_surfaces() { printf '%s' "$pane_hash" > "$state/.hash-$key" printf '1\n' > "$state/.count-$key" - # First sight must surface promptly so a live external-decision gate is not - # hidden behind the pause cadence. + # Phase A: current-main used the live agent verdict to discard this declared + # pause and surface immediately. A fresh declaration must instead be absorbed + # without arming the wedge timer. PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting at an active external-decision gate' \ FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" & pid=$! - wait_for_exit "$pid" 100 || fail "live external-decision gate did not surface immediately" - ack_stopped_cycle "$state" || fail "could not acknowledge the immediate external-decision surface" + wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a live declared pause surfaced immediately: $(cat "$out")"; } + [ -e "$state/.paused-$key" ] || { reap "$pid"; fail "a live declared pause did not enter the pause cadence"; } + [ ! -e "$state/.stale-since-$key" ] || { reap "$pid"; fail "a live declared pause armed the wedge timer"; } + reap "$pid" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional live-pause phase-A stop" - # Re-arm with the stale timer already beyond the wedge threshold. This is the - # exact unchanged-hash fallback after the immediate surface: it must retain - # the pause cadence and discard any residual wedge timer instead of emitting - # a second possible-wedge wake. - printf '%s\n' $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" + # Phase B: the same live declaration still re-surfaces once its own long cadence + # expires, and never acquires possible-wedge or deep-inspection decoration. + back=$(( $(date +%s) - 500 )) + if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf" + else touch -m -d "@$back" "$statusf"; fi + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" + : > "$out" PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting at an active external-decision gate' \ - FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" & pid=$! - if ! wait_poll_cycle "$state" "$pid"; then - reap "$pid" - fail "live external-decision gate escalated on the wedge timer after its immediate surface: $(cat "$out")" - fi - [ -e "$state/.paused-$key" ] || { reap "$pid"; fail "live external-decision gate lost its pause cadence marker"; } - [ ! -e "$state/.stale-since-$key" ] || { reap "$pid"; fail "live external-decision gate retained the wedge timer"; } - reap "$pid" - wakes=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w { n++ } END { print n + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0) - bare=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w && $5 == "stale: " w { n++ } END { print n + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0) - [ "$wakes" -eq 0 ] || fail "acknowledged external-decision surface replayed $wakes wakes" - [ "$bare" -eq 0 ] || fail "acknowledged external-decision bare stale remained queued" - pass "exited declared-pause and captain-held panes use bounded pause cadence while a live decision gate still surfaces once" + wait_for_exit "$pid" 100 || fail "a live declared pause did not re-surface on the pause cadence" + grep -F "awaiting external" "$out" >/dev/null || fail "the live declared pause recheck omitted its external-wait reason: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null && fail "a live declared pause was mislabeled a possible wedge: $(cat "$out")" + grep -F "demand-deep-inspection" "$out" >/dev/null && fail "a live declared pause demanded deep inspection: $(cat "$out")" + [ ! -e "$state/.stale-since-$key" ] || fail "the live declared pause retained a wedge timer" + pass "current declared pauses use the bounded pause cadence regardless of agent liveness, while exited captain-held panes remain bounded" } # A dead worker reaches handle_paused_stale rather than the live fallback above. @@ -2120,7 +2203,7 @@ test_absorbed_replacement_wait_does_not_inherit_the_old_throttle() { # `check: rearm-resurface` before it ever reaches the stale path, and every # absorb assertion below passes vacuously. A live agent (pane_current_command # matching the recorded harness) on an idle pane is the exact population -# pause_state_class answers `none` for. +# pause_state_class answers `none` for under a captain hold. # `exit` requires the watcher to surface and exit; `absorb` requires it to # survive whole poll cycles - enough to see the new hash, count it stable, and # reach the stale path. Returns 1 when the watcher does the other thing. @@ -2146,13 +2229,15 @@ parked_watch_round() { # > "$statusf" @@ -2242,8 +2323,7 @@ test_live_declared_wait_churn_honors_the_resurface_throttle() { "$state/.wake-queue" 2>/dev/null || echo 0) [ "$wakes" -eq 1 ] || fail "[$name] elapsed re-surface window produced $wakes wakes instead of one" [ "$bare" -eq 1 ] || fail "[$name] elapsed re-surface changed the wake identity: $(cat "$state/.wake-queue")" - done - pass "a parked live worker surfaces once, absorbs pane churn for the whole re-surface window, then re-surfaces when it elapses" + pass "a parked live captain-held worker surfaces once, absorbs pane churn for the whole re-surface window, then re-surfaces when it elapses" } test_secondmate_paused_resurfaces_in_normal_mode() { @@ -4053,7 +4133,8 @@ test_afk_busy_declared_pause_hands_off_plain_stale test_afk_busy_declared_pause_ticking_pane_hands_off_once test_nonterminal_stale_not_working_surfaced test_nonterminal_stale_paused_absorbed_then_resurfaced -test_exited_declared_pause_is_bounded_but_live_gate_surfaces +test_paused_busy_flicker_keeps_resurface_cadence +test_declared_pause_and_exited_captain_hold_are_bounded test_absorbed_replacement_wait_does_not_inherit_the_old_throttle test_live_declared_wait_churn_honors_the_resurface_throttle test_secondmate_paused_resurfaces_in_normal_mode