fix(metrics): bound domain and method label cardinality

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/pokt-network/ring-go v0.2.0
 	github.com/pokt-network/shannon-sdk v0.0.0-20260702172744-c2af5007ed72
 	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/redis/go-redis/v9 v9.19.0
 	github.com/stretchr/testify v1.11.1
 	github.com/testcontainers/testcontainers-go v0.42.0
@@ -38,7 +39,7 @@ require (
 	github.com/viccon/sturdyc v1.1.5
 	go.uber.org/mock v0.6.0
 	golang.org/x/net v0.56.0
-	google.golang.org/grpc v1.82.0
+	google.golang.org/grpc v1.82.1
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -246,7 +247,6 @@ require (
 	github.com/pokt-network/smt v0.14.1 // indirect
 	github.com/pokt-network/smt/kvstore/pebble v0.0.0-20240822175047-21ea8639c188 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect

go.sum

@@ -1304,8 +1304,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
-google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU=
-google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
+google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
+google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

metrics/cardinality_guard.go

@@ -166,6 +166,10 @@ func packageGuards() []*cardinalityGuard {
 		hedgeSupplierGuard,
 		qosFilterRejectionGuard,
 		healthCheckStatusGuard,
+		probationEventsGuard,
+		observationPipelineGuard,
+		circuitBreakerEventsGuard,
+		rpcTypeFallbackGuard,
 	}
 }
 
@@ -410,22 +414,22 @@ func hashLabelValues(labelValues []string) uint64 {
 // delete precisely the series it reclaims (see DefaultSeriesLimit).
 var (
 	supplierSignalGuard = newCardinalityGuard("supplier_signal_total", defaultSeriesLimit).
-		withEviction(defaultGuardIdleWindow, func(lv []string) {
+				withEviction(defaultGuardIdleWindow, func(lv []string) {
 			SupplierSignalTotal.DeleteLabelValues(lv...)
 		})
 
 	supplierReputationGuard = newCardinalityGuard("supplier_reputation_score", defaultSeriesLimit).
-		withEviction(defaultGuardIdleWindow, func(lv []string) {
+				withEviction(defaultGuardIdleWindow, func(lv []string) {
 			SupplierReputationScore.DeleteLabelValues(lv...)
 		})
 
 	hedgeSupplierGuard = newCardinalityGuard("hedge_supplier_latency_seconds", defaultSeriesLimit).
-		withEviction(defaultGuardIdleWindow, func(lv []string) {
+				withEviction(defaultGuardIdleWindow, func(lv []string) {
 			HedgeSupplierOutcomeTotal.DeleteLabelValues(lv...)
 		})
 
 	qosFilterRejectionGuard = newCardinalityGuard("qos_filter_rejection_total", defaultSeriesLimit).
-		withEviction(defaultGuardIdleWindow, func(lv []string) {
+				withEviction(defaultGuardIdleWindow, func(lv []string) {
 			QoSFilterRejectionTotal.DeleteLabelValues(lv...)
 		})
 
@@ -437,7 +441,53 @@ var (
 	// `domain` values) cannot reproduce the 200K+ series this metric carried in
 	// production while completely unguarded.
 	healthCheckStatusGuard = newCardinalityGuard("health_check_status_total", defaultSeriesLimit).
-		withEviction(defaultGuardIdleWindow, func(lv []string) {
+				withEviction(defaultGuardIdleWindow, func(lv []string) {
 			HealthCheckStatus.DeleteLabelValues(lv...)
 		})
+
+	// Guards added after the 2026-08-12 cardinality regression, in which these
+	// four metrics contributed ~3.0M series (63% of PNF's entire TSDB) and took
+	// Prometheus to 89% of its memory ceiling. All four shipped unguarded.
+	//
+	// Two distinct label-source defects fed them, both fixed separately
+	// (SanitizeDomainLabel, SanitizeMethodLabel). These guards exist so that
+	// neither fix is load-bearing: a future label source that leaks unbounded
+	// values costs a capped number of series and a WARN, not a monitoring
+	// outage. That is the actual lesson of the regression — the sanitizers are
+	// hygiene, the guard is the bound.
+
+	// probationEventsGuard — 2,354,499 series in production (92× growth, 27% of
+	// the TSDB) because `domain` carried raw supplier addresses. Realistic tuple
+	// count post-fix is domain × rpc_type × service_id × event, far under cap.
+	probationEventsGuard = newCardinalityGuard("probation_events_total", defaultSeriesLimit).
+				withEviction(defaultGuardIdleWindow, func(lv []string) {
+			ProbationEventsTotal.DeleteLabelValues(lv...)
+		})
+
+	// observationPipelineGuard — the ONLY hard bound on the `method` label, and
+	// the only one of these four guards that is load-bearing rather than a
+	// backstop. `method` is attacker-controlled: it carries the JSON-RPC method
+	// name or the REST URL path, so any unauthenticated client can mint a fresh
+	// series per request by varying it. SanitizeMethodLabel normalizes the
+	// SHAPE of a value but cannot bound the SET — `/aaa`, `/aab`, … all survive
+	// as legitimate-looking static route segments. Only this cap converts a
+	// remote resource-exhaustion vector into a bounded cost plus a WARN.
+	observationPipelineGuard = newCardinalityGuard("observation_pipeline_total", defaultSeriesLimit).
+					withEviction(defaultGuardIdleWindow, func(lv []string) {
+			ObservationPipeline.DeleteLabelValues(lv...)
+		})
+
+	// circuitBreakerEventsGuard — 233,269 series (49× growth).
+	circuitBreakerEventsGuard = newCardinalityGuard("circuit_breaker_events_total", defaultSeriesLimit).
+					withEviction(defaultGuardIdleWindow, func(lv []string) {
+			DomainCircuitBreakerEventsTotal.DeleteLabelValues(lv...)
+		})
+
+	// rpcTypeFallbackGuard — backstop only. The real fix was dropping the
+	// `supplier` label (3,289 values doing essentially all of the metric's
+	// 201,068-series multiplication against 9 domains × 12 service_ids).
+	rpcTypeFallbackGuard = newCardinalityGuard("rpc_type_fallback_total", defaultSeriesLimit).
+				withEviction(defaultGuardIdleWindow, func(lv []string) {
+			RPCTypeFallbackTotal.DeleteLabelValues(lv...)
+		})
 )

metrics/cardinality_regression_test.go

@@ -0,0 +1,156 @@
+package metrics
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
+)
+
+// Regression tests for the 2026-08-12 metric cardinality incident, in which
+// path metrics reached 63% of PNF's entire Prometheus TSDB and took it to 89%
+// of its memory ceiling.
+//
+// Every assertion here goes through the PRODUCTION emit helper and reads the
+// label values the Prometheus collector actually received. Asserting on
+// SanitizeDomainLabel / SanitizeMethodLabel directly would prove only that a
+// helper the author wrote returns what the author expected — the exact mistake
+// that let three drain bugs ship green (see CLAUDE.md, "Testing Changes That
+// Affect Routing"). A sanitizer that is never called on the emit path passes
+// its own unit tests perfectly.
+
+// collectLabelValues gathers a CounterVec and returns the set of values seen
+// for one label name across every child series.
+func collectLabelValues(t *testing.T, c interface {
+	Collect(chan<- prometheus.Metric)
+}, labelName string,
+) map[string]struct{} {
+	t.Helper()
+	ch := make(chan prometheus.Metric, 1<<16)
+	c.Collect(ch)
+	close(ch)
+
+	out := map[string]struct{}{}
+	for m := range ch {
+		var pb dto.Metric
+		require.NoError(t, m.Write(&pb))
+		for _, lp := range pb.GetLabel() {
+			if lp.GetName() == labelName {
+				out[lp.GetValue()] = struct{}{}
+			}
+		}
+	}
+	return out
+}
+
+// Test_ProbationEvent_SupplierAddressNeverBecomesDomain is the core F1
+// regression: 4,172 of 4,608 distinct `domain` values in production were raw
+// bech32 supplier addresses, making path_probation_events_total 27% of the
+// whole TSDB.
+//
+// The path is subtle and is why the bug was invisible: reputation/selector.go
+// guards with `if err != nil || domain == ""`, but ExtractDomainOrHost does not
+// FAIL on a bare supplier address — a dotless host takes the
+// isPrivateOrInternalDomain branch and is returned verbatim with a nil error.
+// The fallback never fires, so the address arrives here looking like a
+// successful extraction.
+func Test_ProbationEvent_SupplierAddressNeverBecomesDomain(t *testing.T) {
+	ProbationEventsTotal.Reset()
+
+	const addr = "pokt1ylsjqcl0yunve78etutw660a327avc26fxrlfr"
+	RecordProbationEvent(addr, "json_rpc", "arb-one", ProbationEventEntered)
+
+	got := collectLabelValues(t, ProbationEventsTotal, LabelDomain)
+	require.NotContains(t, got, addr,
+		"raw supplier address reached the `domain` label; it must collapse to a sentinel")
+	require.Contains(t, got, DomainSupplierAddr)
+}
+
+// Test_RealDomainsSurviveSanitization is the counterweight: the fix must not
+// collapse the 434 legitimate domains it exists to preserve. `domain` carries
+// 202 references across our Grafana dashboards — over-collapsing here would be
+// indistinguishable, from the dashboard's side, from PNF dropping the label.
+func Test_RealDomainsSurviveSanitization(t *testing.T) {
+	ProbationEventsTotal.Reset()
+
+	// Includes hosts with a `1` in a bech32-looking position and a dotless
+	// internal hostname, both of which must NOT be mistaken for addresses.
+	keep := []string{"nodefleet.net", "example.co.uk", "web31.io", "relayminer1", "88.198.50.175"}
+	for _, d := range keep {
+		RecordProbationEvent(d, "json_rpc", "eth", ProbationEventEntered)
+	}
+
+	got := collectLabelValues(t, ProbationEventsTotal, LabelDomain)
+	for _, d := range keep {
+		require.Contains(t, got, d, "legitimate domain was collapsed by the sanitizer")
+	}
+}
+
+// Test_ObservationPipeline_AttackerMethodsAreBounded is the F2 regression.
+//
+// This asserts the property that actually matters and that SanitizeMethodLabel
+// alone does NOT provide: an unauthenticated client varying the JSON-RPC method
+// or REST path cannot mint unbounded series. The sanitizer normalizes the SHAPE
+// of a value but cannot bound the SET — `/aaa`, `/aab`, … are all well-formed
+// static route segments and survive it verbatim. Only the guard bounds them.
+func Test_ObservationPipeline_AttackerMethodsAreBounded(t *testing.T) {
+	ObservationPipeline.Reset()
+	// Swap the guard POINTER rather than copying the struct: cardinalityGuard
+	// embeds a sync.Map, so assigning through it copies a lock (govet copylocks).
+	saved := observationPipelineGuard
+	t.Cleanup(func() { observationPipelineGuard = saved })
+	observationPipelineGuard = newCardinalityGuard("test_observation_pipeline", 100)
+
+	// Far more distinct methods than the cap, in the shape a scanner produces.
+	for i := 0; i < 5000; i++ {
+		RecordObservation("nodefleet.net", "json_rpc", "eth", NetworkTypeCosmos,
+			SanitizeMethodLabel(NetworkTypeCosmos, fmt.Sprintf("/probe%d/logon.html", i)), "ok")
+	}
+
+	got := collectLabelValues(t, ObservationPipeline, LabelMethod)
+	require.LessOrEqual(t, len(got), 100,
+		"attacker-varied method values were not bounded by the cardinality guard")
+}
+
+// Test_ObservationPipeline_InjectionPayloadsCannotPanic covers the payloads PNF
+// found live in the TSDB (CRLF header injection, XSS, path traversal,
+// template injection). client_golang panics inside WithLabelValues on non-UTF-8
+// label values, so a single malformed request reaching an unsanitized label is
+// a remote crash, not just untidy data — the 2026-06-15 incident.
+func Test_ObservationPipeline_InjectionPayloadsCannotPanic(t *testing.T) {
+	ObservationPipeline.Reset()
+
+	payloads := []string{
+		"/\r\n\r\n<script>alert(1)</script>",
+		"/\r\nSet-Cookie: x=1",
+		"/%2e%2e%2f%2e%2e%2f%2e%2e%2fetc/passwd",
+		"/..%252f..%252f..%252fetc/passwd",
+		"/${13337*31337}",
+		"/+CSCOE+/logon.html",
+		"/\xff\xfe invalid utf8",
+	}
+	for _, p := range payloads {
+		require.NotPanics(t, func() {
+			RecordObservation("\xff\xfe.example", "json_rpc", "cosmoshub", NetworkTypeCosmos,
+				SanitizeMethodLabel(NetworkTypeCosmos, p), "ok")
+		}, "payload %q panicked on the metrics path", p)
+	}
+}
+
+// Test_RPCTypeFallback_SupplierLabelDropped is the F3 regression: `supplier`
+// carried 3,289 values against the metric's own 9 domains × 12 service_ids,
+// producing essentially all of its 201,068 series.
+//
+// The helper still ACCEPTS a supplier argument so no caller had to change;
+// this asserts the value does not reach the collector.
+func Test_RPCTypeFallback_SupplierLabelDropped(t *testing.T) {
+	RPCTypeFallbackTotal.Reset()
+
+	const supplier = "pokt1vmy9q5ljvs39n78ygwa85t9rsffncf90xqp2lp"
+	RecordRPCTypeFallback("example.net", supplier, "cosmoshub", "COMET_BFT", "JSON_RPC")
+
+	require.Empty(t, collectLabelValues(t, RPCTypeFallbackTotal, LabelSupplier),
+		"supplier is still being emitted as a label")
+}

metrics/domain_sanitizer.go

@@ -0,0 +1,68 @@
+package metrics
+
+import (
+	"strings"
+	"unicode/utf8"
+)
+
+const (
+	// DomainUnknown — the caller had no domain to report (empty string).
+	DomainUnknown = "unknown"
+
+	// DomainSupplierAddr — the caller passed a bech32 supplier address where a
+	// domain was expected. Collapsed to a single sentinel rather than kept
+	// verbatim: the whole point of `domain` is to COLLAPSE many suppliers onto
+	// one operator, so admitting an address inverts the label's purpose and
+	// makes it expand ~1:1 with supplier count.
+	DomainSupplierAddr = "supplier_addr"
+
+	// DomainLabelMaxLen — hard cap on any `domain` label value. A registrable
+	// domain is far shorter; this is a safety net against a pathological host.
+	DomainLabelMaxLen = 64
+)
+
+// SanitizeDomainLabel bounds the cardinality of the `domain` Prometheus label.
+// MUST be called on every value flowing into a `domain` label.
+//
+// The problem it solves: shannonmetrics.ExtractDomainOrHost is shared between
+// the metrics path and the ROUTING path (operator concentration cap, drains,
+// blocked_domains, reputation keys). It returns a bare bech32 supplier address
+// verbatim and with a nil error — a dotless host takes the
+// isPrivateOrInternalDomain branch, which returns it as-is — so callers that
+// pass an EndpointAddr rather than a URL silently emit `pokt1…` as a domain,
+// and their `err != nil` fallback never fires. Fixing that inside the shared
+// extractor would change endpoint selection; fixing it here cannot.
+//
+// Measured 2026-08-12: 4,172 of 4,608 distinct `domain` values in production
+// were raw supplier addresses, driving path_probation_events_total to 2.35M
+// series (27% of the whole TSDB).
+//
+// Deliberately NOT collapsed:
+//   - Bare IPs. Operationally useful when a supplier registers one, and few
+//     enough to be a non-issue (1 observed). The per-metric cardinality guard
+//     is what protects against an IP-registration flood.
+//   - Dotless internal hostnames (`relayminer1`). Bounded and meaningful.
+func SanitizeDomainLabel(raw string) string {
+	d := strings.ToLower(strings.TrimSpace(raw))
+	if d == "" {
+		return DomainUnknown
+	}
+
+	// Bech32 check is gated on "no dot" so it can never fire on a real domain:
+	// isBech32Like requires the post-`1` remainder to be entirely alphanumeric,
+	// and any registrable domain contains a dot in that remainder. Without the
+	// gate this would be a heuristic on hostnames; with it, it is exact.
+	if !strings.Contains(d, ".") && isBech32Like(d) {
+		return DomainSupplierAddr
+	}
+
+	if len(d) > DomainLabelMaxLen {
+		d = d[:DomainLabelMaxLen]
+	}
+	// prometheus/client_golang panics inside WithLabelValues on non-UTF-8 label
+	// values, and the truncation above can split a multibyte rune.
+	if !utf8.ValidString(d) {
+		d = strings.ToValidUTF8(d, "�")
+	}
+	return d
+}

metrics/leaderboard.go

@@ -161,7 +161,7 @@ func (lp *LeaderboardPublisher) publishLeaderboard(ctx context.Context) {
 			// Publish each entry
 			for _, entry := range entries {
 				ReputationEndpointLeaderboard.WithLabelValues(
-					entry.Domain,
+					SanitizeDomainLabel(entry.Domain),
 					entry.RPCType,
 					entry.ServiceID,
 					fmt.Sprintf("%d", entry.TierThreshold),
@@ -219,7 +219,7 @@ func (lp *LeaderboardPublisher) publishLeaderboard(ctx context.Context) {
 
 	if len(cooldownCounts) > 0 {
 		for _, entry := range cooldownCounts {
-			EndpointsInCooldown.WithLabelValues(entry.Domain, entry.RPCType, entry.ServiceID).Set(float64(entry.Count))
+			EndpointsInCooldown.WithLabelValues(SanitizeDomainLabel(entry.Domain), entry.RPCType, entry.ServiceID).Set(float64(entry.Count))
 		}
 		lp.logger.Debug().Int("entries", len(cooldownCounts)).Msg("Published cooldown counts")
 	}
@@ -236,7 +236,7 @@ func (lp *LeaderboardPublisher) publishLeaderboard(ctx context.Context) {
 
 	if len(drainedCounts) > 0 {
 		for _, entry := range drainedCounts {
-			EndpointsDrained.WithLabelValues(entry.Domain, entry.RPCType, entry.ServiceID).Set(float64(entry.Count))
+			EndpointsDrained.WithLabelValues(SanitizeDomainLabel(entry.Domain), entry.RPCType, entry.ServiceID).Set(float64(entry.Count))
 		}
 		lp.logger.Warn().Int("entries", len(drainedCounts)).Msg("⚠️ endpoints are benched by an admin drain")
 	}