|
| 1 | +//! K8s runtime driver — projects k8s Pod signals onto the canonical model. |
| 2 | +//! |
| 3 | +//! Mapping (ADR §6, the k8s column): Pod phase + readinessProbe ⇒ Starting / |
| 4 | +//! Running; `metadata.deletionTimestamp != null` ⇒ `DesiredStatus::Stopped` |
| 5 | +//! (Terminating: preStop + grace); `Unknown` phase (node lost) ⇒ |
| 6 | +//! Unhealthy(fenced), same as a lost lease; CrashLoopBackOff ⇒ Unhealthy. |
| 7 | +//! Deliberately mirrors `ecs.rs`'s shape — this is a direct port of an |
| 8 | +//! already-approved ADR table, not a new design decision (studio#146, the |
| 9 | +//! observability follow-up `k8s_driver.rs`'s module doc flagged as deferred |
| 10 | +//! when the k8s `ProvisionDriver` write path landed). |
| 11 | +
|
| 12 | +use crate::{DesiredStatus, Discriminator, Health, RuntimeDriver}; |
| 13 | + |
| 14 | +/// The subset of k8s Pod `phase` relevant to the projection. `Unknown` is |
| 15 | +/// k8s's own "kubelet not reporting" signal — the direct analogue of ECS's |
| 16 | +/// node-loss case, not a leftover default. |
| 17 | +#[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 18 | +pub enum PodPhase { |
| 19 | + Pending, |
| 20 | + Running, |
| 21 | + Succeeded, |
| 22 | + Failed, |
| 23 | + Unknown, |
| 24 | +} |
| 25 | + |
| 26 | +/// A single k8s Pod observation (the subset the projection needs). |
| 27 | +#[derive(Debug, Clone, Copy)] |
| 28 | +pub struct K8sPod { |
| 29 | + pub phase: PodPhase, |
| 30 | + /// `metadata.deletionTimestamp != null`. |
| 31 | + pub deletion_timestamp_set: bool, |
| 32 | + /// The Pod's `Ready` condition (readiness probe result when one is |
| 33 | + /// declared; otherwise just reflects the container's running state). |
| 34 | + pub ready: bool, |
| 35 | + /// Whether a readiness probe is actually declared on the pod spec. When |
| 36 | + /// it is not, k8s's `Ready` condition tracks plain container-running |
| 37 | + /// state, not health — so `ready == false` there means "still starting", |
| 38 | + /// not a fault. Mirrors `EcsTask::health_check_defined`'s "no probe ≠ |
| 39 | + /// fault" distinction; without it, a probe-less pod that just hasn't |
| 40 | + /// finished starting would misread as permanently degraded. |
| 41 | + pub ready_check_defined: bool, |
| 42 | + /// A container in this pod is in CrashLoopBackOff (or an equivalent |
| 43 | + /// terminal restart-loop waiting reason) — a fault signal distinct from |
| 44 | + /// `ready`, since a crash-looping container can still report a phase |
| 45 | + /// that looks superficially fine between restarts. |
| 46 | + pub crash_loop_back_off: bool, |
| 47 | + /// CP-issued lease still valid (heartbeat authorized). |
| 48 | + pub lease_valid: bool, |
| 49 | + /// CP/director cordon: `false` ⇒ not admitting new work (→ Paused). |
| 50 | + pub accepting_work: bool, |
| 51 | +} |
| 52 | + |
| 53 | +/// Projects k8s Pod state onto the canonical lifecycle model. |
| 54 | +pub struct K8sDriver; |
| 55 | + |
| 56 | +impl RuntimeDriver for K8sDriver { |
| 57 | + type Native = K8sPod; |
| 58 | + /// Pod UID. |
| 59 | + type InstanceId = String; |
| 60 | + |
| 61 | + fn project(&self, pod: &K8sPod, verified_before: bool) -> Discriminator { |
| 62 | + let desired_status = if pod.deletion_timestamp_set { |
| 63 | + DesiredStatus::Stopped |
| 64 | + } else { |
| 65 | + DesiredStatus::Running |
| 66 | + }; |
| 67 | + |
| 68 | + // `identity_verified` latches once the pod has ever reported phase |
| 69 | + // Running with Ready true. |
| 70 | + let identity_verified = |
| 71 | + verified_before || (pod.phase == PodPhase::Running && pod.ready); |
| 72 | + |
| 73 | + // Node-unreachable (Unknown phase), a crash-loop, or a lost lease all |
| 74 | + // fault outright. A *declared* readiness probe failing faults too — |
| 75 | + // but an undeclared one reporting `ready == false` just means "still |
| 76 | + // starting" (see `ready_check_defined`'s doc), not a fault; that case |
| 77 | + // only matters once `identity_verified` is already true, since |
| 78 | + // `classify()` reads `Starting` ahead of `health` otherwise. |
| 79 | + let health = if pod.phase == PodPhase::Unknown |
| 80 | + || pod.crash_loop_back_off |
| 81 | + || !pod.lease_valid |
| 82 | + || (pod.ready_check_defined && !pod.ready) |
| 83 | + { |
| 84 | + Health::Faulted |
| 85 | + } else { |
| 86 | + Health::Ok |
| 87 | + }; |
| 88 | + |
| 89 | + Discriminator { |
| 90 | + desired_status, |
| 91 | + accepting_work: pod.accepting_work, |
| 92 | + health, |
| 93 | + identity_verified, |
| 94 | + } |
| 95 | + } |
| 96 | +} |
| 97 | + |
| 98 | +#[cfg(test)] |
| 99 | +mod tests { |
| 100 | + use super::*; |
| 101 | + use crate::AgentState; |
| 102 | + |
| 103 | + fn pod( |
| 104 | + phase: PodPhase, |
| 105 | + deleting: bool, |
| 106 | + ready: bool, |
| 107 | + lease: bool, |
| 108 | + accepting: bool, |
| 109 | + ) -> K8sPod { |
| 110 | + // Default: no readiness probe declared (the common case for OAB agents). |
| 111 | + pod_probe(phase, deleting, ready, lease, accepting, false, false) |
| 112 | + } |
| 113 | + |
| 114 | + fn pod_probe( |
| 115 | + phase: PodPhase, |
| 116 | + deleting: bool, |
| 117 | + ready: bool, |
| 118 | + lease: bool, |
| 119 | + accepting: bool, |
| 120 | + ready_check_defined: bool, |
| 121 | + crash_loop_back_off: bool, |
| 122 | + ) -> K8sPod { |
| 123 | + K8sPod { |
| 124 | + phase, |
| 125 | + deletion_timestamp_set: deleting, |
| 126 | + ready, |
| 127 | + ready_check_defined, |
| 128 | + crash_loop_back_off, |
| 129 | + lease_valid: lease, |
| 130 | + accepting_work: accepting, |
| 131 | + } |
| 132 | + } |
| 133 | + |
| 134 | + #[test] |
| 135 | + fn pending_pod_projects_to_starting() { |
| 136 | + let d = K8sDriver.project(&pod(PodPhase::Pending, false, false, false, false), false); |
| 137 | + assert_eq!(d.classify(), AgentState::Starting); |
| 138 | + } |
| 139 | + |
| 140 | + #[test] |
| 141 | + fn running_ready_pod_projects_to_running() { |
| 142 | + let d = K8sDriver.project(&pod(PodPhase::Running, false, true, true, true), false); |
| 143 | + assert_eq!(d.classify(), AgentState::Running); |
| 144 | + } |
| 145 | + |
| 146 | + #[test] |
| 147 | + fn cordoned_running_pod_is_paused() { |
| 148 | + let d = K8sDriver.project(&pod(PodPhase::Running, false, true, true, false), true); |
| 149 | + assert_eq!(d.classify(), AgentState::Paused); |
| 150 | + } |
| 151 | + |
| 152 | + #[test] |
| 153 | + fn declared_probe_failing_after_verified_is_unhealthy() { |
| 154 | + // Was Ready before, now the declared readiness probe fails ⇒ |
| 155 | + // Unhealthy (not Starting). |
| 156 | + let d = K8sDriver.project( |
| 157 | + &pod_probe(PodPhase::Running, false, false, true, false, true, false), |
| 158 | + true, |
| 159 | + ); |
| 160 | + assert_eq!(d.classify(), AgentState::Unhealthy); |
| 161 | + } |
| 162 | + |
| 163 | + #[test] |
| 164 | + fn node_lost_unknown_is_unhealthy_not_stopped() { |
| 165 | + // Unknown phase (kubelet not reporting) while verified ⇒ |
| 166 | + // Unhealthy(fenced), not Stopped. |
| 167 | + let d = K8sDriver.project(&pod(PodPhase::Unknown, false, false, false, false), true); |
| 168 | + assert_eq!(d.classify(), AgentState::Unhealthy); |
| 169 | + } |
| 170 | + |
| 171 | + #[test] |
| 172 | + fn crash_loop_back_off_is_unhealthy_even_if_ready() { |
| 173 | + let d = K8sDriver.project( |
| 174 | + &pod_probe(PodPhase::Running, false, true, true, true, true, true), |
| 175 | + true, |
| 176 | + ); |
| 177 | + assert_eq!(d.classify(), AgentState::Unhealthy); |
| 178 | + } |
| 179 | + |
| 180 | + #[test] |
| 181 | + fn running_without_declared_probe_and_not_ready_is_still_running() { |
| 182 | + // No readiness probe declared ⇒ k8s's Ready condition isn't a health |
| 183 | + // signal here — a running, leased instance stays Running (the ECS |
| 184 | + // side's equivalent case: `running_unknown_without_health_check_is_running`). |
| 185 | + let d = K8sDriver.project(&pod(PodPhase::Running, false, false, true, true), true); |
| 186 | + assert_eq!(d.classify(), AgentState::Running); |
| 187 | + } |
| 188 | + |
| 189 | + #[test] |
| 190 | + fn deletion_timestamp_is_stopping_while_observable() { |
| 191 | + let d = K8sDriver.project(&pod(PodPhase::Running, true, true, true, false), true); |
| 192 | + assert_eq!(d.classify(), AgentState::Stopping); |
| 193 | + } |
| 194 | +} |
0 commit comments