fix: report managed systemd unit health during reconcile

bridges/_dispatch.sh

@@ -352,11 +352,69 @@ _redact_secret_diff() {
   '
 }
 
+# _report_systemd_unit_health <unit> [<label>]
+#
+# Read and report the runtime state of one systemd unit. Reporting only —
+# this never starts, stops, or restarts anything, preserving the operator
+# boundary that _smart_update_systemd_unit documents.
+#
+# Reconciling the unit *file* while staying silent about the unit's runtime
+# state let a managed unit sit in `failed` for four weeks while every
+# upgrade reported success (#576). A desired-state reconciler that reports
+# only the half of the state it writes is not reporting state.
+#
+# Warnings are recorded in HEALTH_WARNINGS so the caller can resurface them
+# in the final summary, where they cannot scroll past.
+_report_systemd_unit_health() {
+  local unit="$1"
+  local label="${2:-$unit}"
+
+  command -v systemctl >/dev/null 2>&1 || return 0
+
+  local active enabled since
+  active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)"
+  [ -n "$active" ] || return 0
+  enabled="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
+
+  case "$active" in
+    active)
+      log "  $label: active"
+      ;;
+    failed)
+      since="$(systemctl show "$unit" -p ActiveEnterTimestamp --value 2>/dev/null || true)"
+      warn "  $label: FAILED${since:+ since $since} — this upgrade did not start it"
+      warn "  $label: inspect with 'systemctl status $unit' and 'journalctl -u $unit -n 50'"
+      # NOTE: `[ -n "${arr+x}" ]` reads an EMPTY bash array as unset, which
+      # would drop the first warning every time. Test declaration instead.
+      if declare -p HEALTH_WARNINGS >/dev/null 2>&1; then
+        HEALTH_WARNINGS+=("$label is in state 'failed' — systemctl status $unit")
+      fi
+      ;;
+    *)
+      # An enabled unit that is not running is a real finding. A disabled
+      # one is a deliberate operator choice and stays quiet.
+      if [ "$enabled" = "enabled" ]; then
+        warn "  $label: $active while enabled — this upgrade did not start it"
+        warn "  $label: start with 'systemctl start $unit'"
+        if declare -p HEALTH_WARNINGS >/dev/null 2>&1; then
+          HEALTH_WARNINGS+=("$label is enabled but '$active' — systemctl start $unit")
+        fi
+      else
+        log "  $label: $active (not enabled)"
+      fi
+      ;;
+  esac
+}
+
 # _smart_update_systemd_unit <unit_file> <new_unit> [<label>]
 #
 # Diff + write + daemon-reload a single systemd unit. Records the change in
 # the caller's UPDATED_ITEMS array. NEVER restarts the unit — operator does
 # that explicitly per the documented restart hint in the summary.
+#
+# Unit health is reported on every pass, including the unchanged and
+# dry-run paths: a long-dead unit whose file is already correct is the
+# quietest failure of all (#576).
 _smart_update_systemd_unit() {
   local unit_file="$1"
   local new_unit="$2"
@@ -367,6 +425,8 @@ _smart_update_systemd_unit() {
     return 0
   fi
 
+  _report_systemd_unit_health "$(basename "$unit_file")" "$label"
+
   if echo "$new_unit" | cmp -s - "$unit_file"; then
     log "  $(basename "$unit_file"): unchanged"
     return 0

tests/bridge-service-adapters.sh

@@ -80,4 +80,81 @@ reconciler_plan_reset
 bridge_service_adapters_plan "$INSTALLATION_OPERATION_SETUP"
 [ "${RECONCILER_PLAN_RECORDS[*]}" = bridges.kimaki ] || error "external WordPress planned local services"
 
+# Managed unit runtime health is reported, never acted on (#576). A unit whose
+# file is already correct but which has been dead for weeks is the quietest
+# failure mode, so health must not depend on the file having changed.
+mkdir -p "$TMP/bin"
+cat > "$TMP/bin/systemctl" <<'SH'
+#!/bin/sh
+case "$1" in
+  show)
+    case "$*" in
+      *ActiveState*) printf '%s\n' "${FAKE_ACTIVE:-active}" ;;
+      *ActiveEnterTimestamp*) printf '%s\n' "${FAKE_SINCE:-}" ;;
+      *) printf '\n' ;;
+    esac
+    ;;
+  is-enabled) printf '%s\n' "${FAKE_ENABLED:-enabled}" ;;
+  *) exit 0 ;;
+esac
+SH
+chmod +x "$TMP/bin/systemctl"
+PATH="$TMP/bin:$PATH"
+export PATH
+
+# Capture warnings instead of discarding them.
+CAPTURED=""
+warn() { CAPTURED="$CAPTURED$1
+"; }
+
+# Healthy unit: no warning, no summary entry.
+HEALTH_WARNINGS=()
+CAPTURED=""
+export FAKE_ACTIVE=active FAKE_ENABLED=enabled FAKE_SINCE=""
+_report_systemd_unit_health kimaki.service kimaki.service
+[ "${#HEALTH_WARNINGS[@]}" -eq 0 ] || error "active unit produced a health warning"
+
+# Failed unit: warns and records for the summary.
+HEALTH_WARNINGS=()
+CAPTURED=""
+export FAKE_ACTIVE=failed FAKE_ENABLED=enabled FAKE_SINCE="Wed 2026-08-05 14:46:59 UTC"
+_report_systemd_unit_health kimaki.service kimaki.service
+[ "${#HEALTH_WARNINGS[@]}" -eq 1 ] || error "failed unit was not recorded for the summary"
+case "${HEALTH_WARNINGS[0]}" in
+  *failed*kimaki.service*) : ;;
+  *) error "failed-unit summary entry did not name the unit and state: ${HEALTH_WARNINGS[0]}" ;;
+esac
+case "$CAPTURED" in
+  *"did not start it"*) : ;;
+  *) error "failed unit did not state that the upgrade left it alone" ;;
+esac
+
+# Enabled but inactive is a real finding.
+HEALTH_WARNINGS=()
+CAPTURED=""
+export FAKE_ACTIVE=inactive FAKE_ENABLED=enabled
+_report_systemd_unit_health kimaki.service kimaki.service
+[ "${#HEALTH_WARNINGS[@]}" -eq 1 ] || error "enabled-but-inactive unit was not recorded"
+
+# Deliberately disabled units stay quiet.
+HEALTH_WARNINGS=()
+CAPTURED=""
+export FAKE_ACTIVE=inactive FAKE_ENABLED=disabled
+_report_systemd_unit_health kimaki.service kimaki.service
+[ "${#HEALTH_WARNINGS[@]}" -eq 0 ] || error "disabled unit produced a health warning"
+
+# Health is reported even when the unit file is byte-identical, which is the
+# path that hid a four-week outage.
+HEALTH_WARNINGS=()
+CAPTURED=""
+UPDATED_ITEMS=()
+DRY_RUN=false
+TIMESTAMP=test
+UNCHANGED_UNIT="$TMP/kimaki.service"
+printf '[Service]\nExecStart=/usr/bin/kimaki\n' > "$UNCHANGED_UNIT"
+export FAKE_ACTIVE=failed FAKE_ENABLED=enabled
+_smart_update_systemd_unit "$UNCHANGED_UNIT" "$(cat "$UNCHANGED_UNIT")" kimaki.service
+[ "${#HEALTH_WARNINGS[@]}" -eq 1 ] || error "unchanged unit file suppressed the health report"
+[ "${#UPDATED_ITEMS[@]}" -eq 0 ] || error "unchanged unit file was reported as updated"
+
 echo "PASS: tests/bridge-service-adapters.sh"

upgrade.sh

@@ -563,6 +563,8 @@ fi
 UPDATED_ITEMS=()
 PENDING_ITEMS=()
 PLUGIN_UPDATE_FAILURES=()
+# Managed units found in a bad runtime state. Reported, never acted on.
+HEALTH_WARNINGS=()
 
 if [ "${SYSTEMS_CAPABILITIES_ONLY:-false}" = true ]; then
   [ -n "${SYSTEMS_CAPABILITIES_PROFILE:-}" ] || error "--systems-capabilities-only requires --systems-capabilities <profile>"
@@ -1326,6 +1328,17 @@ print_summary() {
     done
   fi
 
+  # Runtime health is reported separately from file reconciliation. "Nothing
+  # changed" describes the unit files, not whether the services are running
+  # (#576).
+  if [ ${#HEALTH_WARNINGS[@]} -gt 0 ]; then
+    echo ""
+    warn "Service health:"
+    for item in "${HEALTH_WARNINGS[@]}"; do
+      warn "  - $item"
+    done
+  fi
+
   if [ ${#PENDING_ITEMS[@]} -gt 0 ]; then
     echo ""
     warn "Pending:"