diff --git a/Cargo.lock b/Cargo.lock index a842852..300d4df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,6 +113,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] diff --git a/Cargo.toml b/Cargo.toml index 68214cd..6c98470 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,5 +9,5 @@ ed25519-dalek = { version = "2.1.0", features = ["rand_core"] } rand = "0.8.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -chrono = "0.4" +chrono = { version = "0.4", features = ["serde"] } clap = { version = "4.4", features = ["derive"] } \ No newline at end of file diff --git a/docs/SECRET_ROTATION_ARCHITECTURE.md b/docs/SECRET_ROTATION_ARCHITECTURE.md new file mode 100644 index 0000000..d5eedbf --- /dev/null +++ b/docs/SECRET_ROTATION_ARCHITECTURE.md @@ -0,0 +1,34 @@ +# Secret Rotation Service Architecture + +## Goals + +The Secret Rotation Service coordinates database credential and API key lifecycle management across Utility Protocol services while keeping critical-path lookups under the 100 ms P99 target. It never logs or persists secret material in application state; only metadata such as secret IDs, versions, rollout windows, and retirement timestamps are exported. + +## Components + +1. **Rotation coordinator**: evaluates `SecretRecord` metadata against a `RotationPolicy`, creates a `RolloutPlan`, and delegates material creation/retirement to a secret provider. +2. **Secret provider adapter**: production implementations wrap the cloud KMS or secret manager. The Rust trait boundary is `SecretProvider`, which supports creating the next version and retiring the previous version. +3. **Metadata store**: durable storage for the fields represented by `SecretRecord`. The store should be replicated across zones and read-through cached by services. +4. **Service reload path**: each consumer watches version changes, warms a new connection/client pool with the canary credential, then flips traffic after canary analysis. + +## Rotation Flow + +1. Scheduler reads each record and policy. +2. Coordinator rotates once `now >= expires_at - overlap`. +3. Provider creates `version + 1` without exposing material to logs. +4. Blue-green rollout starts with `canary_percent` of eligible traffic. +5. Monitoring gates full cutover on authentication error rate, dependency latency, and application error budget burn. +6. Previous version remains valid until `retire_previous_at`, then is disabled and deleted according to provider retention policy. + +## Security Controls + +- Least-privilege provider roles: create/read current version only for producers, read current/previous only for consumers during overlap. +- Secret metadata is safe to export; raw values must stay inside KMS/secret-manager APIs. +- Rotation events require structured audit records with actor, secret ID, old/new versions, and rollout decision. +- Break-glass rotations must use the same coordinator path so monitoring and retirement guarantees remain intact. + +## Availability and Performance + +- Consumers use locally cached active version metadata and provider-side client caching to avoid network calls on hot request paths. +- Overlap windows permit zero-downtime pool warming and rollback to the previous version. +- Scheduler instances should use leader election or compare-and-swap writes to avoid double rotations. diff --git a/docs/runbooks/SECRET_ROTATION_RUNBOOK.md b/docs/runbooks/SECRET_ROTATION_RUNBOOK.md new file mode 100644 index 0000000..65901b6 --- /dev/null +++ b/docs/runbooks/SECRET_ROTATION_RUNBOOK.md @@ -0,0 +1,25 @@ +# Secret Rotation Runbook + +## Normal Rotation + +1. Confirm dashboards show green dependency health and no active incident. +2. Trigger the scheduler or wait for the configured rotation window. +3. Verify a `Rotate` decision was emitted for each due secret ID. +4. Watch canary traffic for authentication failures, database connection errors, and P99 latency regressions. +5. Promote to full cutover when canary analysis passes. +6. Confirm the previous version is retired after the overlap window. + +## Emergency Rotation + +1. Open an incident and identify the compromised secret IDs. +2. Run an immediate rotation through the coordinator path, not a manual provider-only change. +3. Set canary to 100% only when the blast radius requires immediate revocation; otherwise keep overlap long enough for client reload. +4. Confirm all consumers observe the new version. +5. Retire the compromised version and record audit evidence. + +## Rollback + +1. Stop promotion if canary error budget burn exceeds the alert threshold. +2. Route traffic back to the previous version while it remains inside `previous_version_expires_at`. +3. Keep the failed version disabled for new traffic. +4. File a post-incident review before retrying rotation. diff --git a/monitoring/secret-rotation-alerts.yml b/monitoring/secret-rotation-alerts.yml new file mode 100644 index 0000000..bc57fb5 --- /dev/null +++ b/monitoring/secret-rotation-alerts.yml @@ -0,0 +1,27 @@ +groups: + - name: secret-rotation + rules: + - alert: SecretRotationFailuresHigh + expr: sum(rate(secret_rotation_failures_total[5m])) > 0 + for: 10m + labels: + severity: critical + annotations: + summary: Secret rotation failures detected + description: Rotation provider or coordinator errors have been non-zero for 10 minutes. + - alert: SecretRotationCanaryAuthErrors + expr: rate(secret_rotation_canary_auth_errors_total[5m]) > 0.01 + for: 5m + labels: + severity: warning + annotations: + summary: Canary credentials are producing authentication errors + description: Pause rollout and use the secret rotation rollback runbook. + - alert: SecretApproachingExpiry + expr: min(secret_seconds_until_expiry) < 86400 + for: 15m + labels: + severity: critical + annotations: + summary: Secret expires within 24 hours + description: Rotate immediately to preserve availability. diff --git a/monitoring/secret-rotation-dashboard.json b/monitoring/secret-rotation-dashboard.json new file mode 100644 index 0000000..2e39910 --- /dev/null +++ b/monitoring/secret-rotation-dashboard.json @@ -0,0 +1,9 @@ +{ + "title": "Secret Rotation Service", + "panels": [ + { "title": "Rotation Decisions", "type": "timeseries", "targets": [{ "expr": "sum by (decision) (rate(secret_rotation_decisions_total[5m]))" }] }, + { "title": "Rotation Failures", "type": "timeseries", "targets": [{ "expr": "sum(rate(secret_rotation_failures_total[5m]))" }] }, + { "title": "Canary Auth Errors", "type": "timeseries", "targets": [{ "expr": "rate(secret_rotation_canary_auth_errors_total[5m])" }] }, + { "title": "Seconds Until Expiry", "type": "stat", "targets": [{ "expr": "min(secret_seconds_until_expiry)" }] } + ] +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..fe5349d --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,3 @@ +//! Support libraries for utility protocol tooling. + +pub mod secret_rotation; diff --git a/src/secret_rotation.rs b/src/secret_rotation.rs new file mode 100644 index 0000000..a686817 --- /dev/null +++ b/src/secret_rotation.rs @@ -0,0 +1,256 @@ +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Identifies the secret family and backing service a credential belongs to. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub enum SecretKind { + DatabaseCredential, + ApiKey, +} + +/// Rotation policy for one class of secrets. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RotationPolicy { + pub rotate_after: Duration, + pub overlap: Duration, + pub canary_percent: u8, +} + +impl RotationPolicy { + pub fn validate(&self) -> Result<(), RotationError> { + if self.rotate_after <= Duration::zero() { + return Err(RotationError::InvalidPolicy( + "rotate_after must be positive", + )); + } + if self.overlap < Duration::zero() || self.overlap >= self.rotate_after { + return Err(RotationError::InvalidPolicy( + "overlap must be non-negative and shorter than rotate_after", + )); + } + if !(1..=100).contains(&self.canary_percent) { + return Err(RotationError::InvalidPolicy( + "canary_percent must be 1..=100", + )); + } + Ok(()) + } +} + +/// Metadata intentionally excludes secret material so it can be logged and exported safely. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct SecretRecord { + pub id: String, + pub kind: SecretKind, + pub version: u64, + pub active_from: DateTime, + pub expires_at: DateTime, + pub previous_version_expires_at: Option>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum RotationDecision { + NotDue, + Rotate { rollout: RolloutPlan }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RolloutPlan { + pub secret_id: String, + pub next_version: u64, + pub canary_percent: u8, + pub full_cutover_at: DateTime, + pub retire_previous_at: DateTime, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum RotationError { + InvalidPolicy(&'static str), + MissingSecret(String), + ProviderFailed(String), +} + +pub trait SecretProvider { + fn create_next_version(&mut self, record: &SecretRecord) -> Result; + fn retire_version(&mut self, secret_id: &str, version: u64) -> Result<(), RotationError>; +} + +/// In-memory coordinator for deterministic rotation decisions. Production callers wrap this with +/// durable storage and a KMS/secret-manager implementation of [`SecretProvider`]. +#[derive(Default)] +pub struct SecretRotationService { + records: BTreeMap, +} + +impl SecretRotationService { + pub fn insert(&mut self, record: SecretRecord) { + self.records.insert(record.id.clone(), record); + } + + pub fn record(&self, id: &str) -> Option<&SecretRecord> { + self.records.get(id) + } + + pub fn evaluate( + &self, + id: &str, + policy: &RotationPolicy, + now: DateTime, + ) -> Result { + policy.validate()?; + let record = self + .records + .get(id) + .ok_or_else(|| RotationError::MissingSecret(id.into()))?; + let rotate_at = record.expires_at - policy.overlap; + if now < rotate_at { + return Ok(RotationDecision::NotDue); + } + Ok(RotationDecision::Rotate { + rollout: RolloutPlan { + secret_id: record.id.clone(), + next_version: record.version + 1, + canary_percent: policy.canary_percent, + full_cutover_at: now, + retire_previous_at: now + policy.overlap, + }, + }) + } + + pub fn rotate_due( + &mut self, + id: &str, + policy: &RotationPolicy, + now: DateTime, + provider: &mut P, + ) -> Result { + let decision = self.evaluate(id, policy, now)?; + let RotationDecision::Rotate { rollout } = decision.clone() else { + return Ok(decision); + }; + let current = self + .records + .get(id) + .cloned() + .ok_or_else(|| RotationError::MissingSecret(id.into()))?; + let next_version = provider.create_next_version(¤t)?; + if next_version != rollout.next_version { + return Err(RotationError::ProviderFailed( + "provider returned unexpected version".into(), + )); + } + provider.retire_version(¤t.id, current.version)?; + self.records.insert( + id.into(), + SecretRecord { + version: next_version, + active_from: now, + expires_at: now + policy.rotate_after, + previous_version_expires_at: Some(rollout.retire_previous_at), + ..current + }, + ); + Ok(decision) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct FakeProvider { + retired: Vec<(String, u64)>, + } + + impl SecretProvider for FakeProvider { + fn create_next_version(&mut self, record: &SecretRecord) -> Result { + Ok(record.version + 1) + } + fn retire_version(&mut self, secret_id: &str, version: u64) -> Result<(), RotationError> { + self.retired.push((secret_id.to_owned(), version)); + Ok(()) + } + } + + fn fixture(now: DateTime) -> (SecretRotationService, RotationPolicy) { + let mut service = SecretRotationService::default(); + service.insert(SecretRecord { + id: "db/main".into(), + kind: SecretKind::DatabaseCredential, + version: 7, + active_from: now - Duration::days(29), + expires_at: now + Duration::hours(1), + previous_version_expires_at: None, + }); + ( + service, + RotationPolicy { + rotate_after: Duration::days(30), + overlap: Duration::hours(2), + canary_percent: 5, + }, + ) + } + + #[test] + fn rotates_inside_overlap_window_and_tracks_previous_retirement() { + let now = Utc::now(); + let (mut service, policy) = fixture(now); + let mut provider = FakeProvider { + retired: Vec::new(), + }; + let decision = service + .rotate_due("db/main", &policy, now, &mut provider) + .expect("rotation succeeds"); + assert!(matches!(decision, RotationDecision::Rotate { .. })); + let record = service.record("db/main").expect("record exists"); + assert_eq!(record.version, 8); + assert_eq!(record.expires_at, now + Duration::days(30)); + assert_eq!( + record.previous_version_expires_at, + Some(now + Duration::hours(2)) + ); + assert_eq!(provider.retired, vec![("db/main".into(), 7)]); + } + + #[test] + fn skips_rotation_before_overlap_window() { + let now = Utc::now(); + let mut service = SecretRotationService::default(); + service.insert(SecretRecord { + id: "api/billing".into(), + kind: SecretKind::ApiKey, + version: 2, + active_from: now, + expires_at: now + Duration::days(10), + previous_version_expires_at: None, + }); + let policy = RotationPolicy { + rotate_after: Duration::days(30), + overlap: Duration::hours(2), + canary_percent: 10, + }; + assert_eq!( + service + .evaluate("api/billing", &policy, now) + .expect("valid"), + RotationDecision::NotDue + ); + } + + #[test] + fn rejects_invalid_policy() { + let now = Utc::now(); + let (service, _) = fixture(now); + let policy = RotationPolicy { + rotate_after: Duration::hours(1), + overlap: Duration::hours(1), + canary_percent: 0, + }; + assert!(matches!( + service.evaluate("db/main", &policy, now), + Err(RotationError::InvalidPolicy(_)) + )); + } +}