diff --git a/go.mod b/go.mod index 7e740330b..6084f500f 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/pokt-network/ring-go v0.2.0 github.com/pokt-network/shannon-sdk v0.0.0-20260702172744-c2af5007ed72 github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/redis/go-redis/v9 v9.19.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.42.0 @@ -38,7 +39,7 @@ require ( github.com/viccon/sturdyc v1.1.5 go.uber.org/mock v0.6.0 golang.org/x/net v0.56.0 - google.golang.org/grpc v1.82.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) @@ -246,7 +247,6 @@ require ( github.com/pokt-network/smt v0.14.1 // indirect github.com/pokt-network/smt/kvstore/pebble v0.0.0-20240822175047-21ea8639c188 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect diff --git a/go.sum b/go.sum index 1db6dbc9a..862a20bd7 100644 --- a/go.sum +++ b/go.sum @@ -1304,8 +1304,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/metrics/cardinality_guard.go b/metrics/cardinality_guard.go index 323ef8285..a1266cbb4 100644 --- a/metrics/cardinality_guard.go +++ b/metrics/cardinality_guard.go @@ -166,6 +166,10 @@ func packageGuards() []*cardinalityGuard { hedgeSupplierGuard, qosFilterRejectionGuard, healthCheckStatusGuard, + probationEventsGuard, + observationPipelineGuard, + circuitBreakerEventsGuard, + rpcTypeFallbackGuard, } } @@ -410,22 +414,22 @@ func hashLabelValues(labelValues []string) uint64 { // delete precisely the series it reclaims (see DefaultSeriesLimit). var ( supplierSignalGuard = newCardinalityGuard("supplier_signal_total", defaultSeriesLimit). - withEviction(defaultGuardIdleWindow, func(lv []string) { + withEviction(defaultGuardIdleWindow, func(lv []string) { SupplierSignalTotal.DeleteLabelValues(lv...) }) supplierReputationGuard = newCardinalityGuard("supplier_reputation_score", defaultSeriesLimit). - withEviction(defaultGuardIdleWindow, func(lv []string) { + withEviction(defaultGuardIdleWindow, func(lv []string) { SupplierReputationScore.DeleteLabelValues(lv...) }) hedgeSupplierGuard = newCardinalityGuard("hedge_supplier_latency_seconds", defaultSeriesLimit). - withEviction(defaultGuardIdleWindow, func(lv []string) { + withEviction(defaultGuardIdleWindow, func(lv []string) { HedgeSupplierOutcomeTotal.DeleteLabelValues(lv...) }) qosFilterRejectionGuard = newCardinalityGuard("qos_filter_rejection_total", defaultSeriesLimit). - withEviction(defaultGuardIdleWindow, func(lv []string) { + withEviction(defaultGuardIdleWindow, func(lv []string) { QoSFilterRejectionTotal.DeleteLabelValues(lv...) }) @@ -437,7 +441,53 @@ var ( // `domain` values) cannot reproduce the 200K+ series this metric carried in // production while completely unguarded. healthCheckStatusGuard = newCardinalityGuard("health_check_status_total", defaultSeriesLimit). - withEviction(defaultGuardIdleWindow, func(lv []string) { + withEviction(defaultGuardIdleWindow, func(lv []string) { HealthCheckStatus.DeleteLabelValues(lv...) }) + + // Guards added after the 2026-08-12 cardinality regression, in which these + // four metrics contributed ~3.0M series (63% of PNF's entire TSDB) and took + // Prometheus to 89% of its memory ceiling. All four shipped unguarded. + // + // Two distinct label-source defects fed them, both fixed separately + // (SanitizeDomainLabel, SanitizeMethodLabel). These guards exist so that + // neither fix is load-bearing: a future label source that leaks unbounded + // values costs a capped number of series and a WARN, not a monitoring + // outage. That is the actual lesson of the regression — the sanitizers are + // hygiene, the guard is the bound. + + // probationEventsGuard — 2,354,499 series in production (92× growth, 27% of + // the TSDB) because `domain` carried raw supplier addresses. Realistic tuple + // count post-fix is domain × rpc_type × service_id × event, far under cap. + probationEventsGuard = newCardinalityGuard("probation_events_total", defaultSeriesLimit). + withEviction(defaultGuardIdleWindow, func(lv []string) { + ProbationEventsTotal.DeleteLabelValues(lv...) + }) + + // observationPipelineGuard — the ONLY hard bound on the `method` label, and + // the only one of these four guards that is load-bearing rather than a + // backstop. `method` is attacker-controlled: it carries the JSON-RPC method + // name or the REST URL path, so any unauthenticated client can mint a fresh + // series per request by varying it. SanitizeMethodLabel normalizes the + // SHAPE of a value but cannot bound the SET — `/aaa`, `/aab`, … all survive + // as legitimate-looking static route segments. Only this cap converts a + // remote resource-exhaustion vector into a bounded cost plus a WARN. + observationPipelineGuard = newCardinalityGuard("observation_pipeline_total", defaultSeriesLimit). + withEviction(defaultGuardIdleWindow, func(lv []string) { + ObservationPipeline.DeleteLabelValues(lv...) + }) + + // circuitBreakerEventsGuard — 233,269 series (49× growth). + circuitBreakerEventsGuard = newCardinalityGuard("circuit_breaker_events_total", defaultSeriesLimit). + withEviction(defaultGuardIdleWindow, func(lv []string) { + DomainCircuitBreakerEventsTotal.DeleteLabelValues(lv...) + }) + + // rpcTypeFallbackGuard — backstop only. The real fix was dropping the + // `supplier` label (3,289 values doing essentially all of the metric's + // 201,068-series multiplication against 9 domains × 12 service_ids). + rpcTypeFallbackGuard = newCardinalityGuard("rpc_type_fallback_total", defaultSeriesLimit). + withEviction(defaultGuardIdleWindow, func(lv []string) { + RPCTypeFallbackTotal.DeleteLabelValues(lv...) + }) ) diff --git a/metrics/cardinality_regression_test.go b/metrics/cardinality_regression_test.go new file mode 100644 index 000000000..a7bfeba22 --- /dev/null +++ b/metrics/cardinality_regression_test.go @@ -0,0 +1,156 @@ +package metrics + +import ( + "fmt" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +// Regression tests for the 2026-08-12 metric cardinality incident, in which +// path metrics reached 63% of PNF's entire Prometheus TSDB and took it to 89% +// of its memory ceiling. +// +// Every assertion here goes through the PRODUCTION emit helper and reads the +// label values the Prometheus collector actually received. Asserting on +// SanitizeDomainLabel / SanitizeMethodLabel directly would prove only that a +// helper the author wrote returns what the author expected — the exact mistake +// that let three drain bugs ship green (see CLAUDE.md, "Testing Changes That +// Affect Routing"). A sanitizer that is never called on the emit path passes +// its own unit tests perfectly. + +// collectLabelValues gathers a CounterVec and returns the set of values seen +// for one label name across every child series. +func collectLabelValues(t *testing.T, c interface { + Collect(chan<- prometheus.Metric) +}, labelName string, +) map[string]struct{} { + t.Helper() + ch := make(chan prometheus.Metric, 1<<16) + c.Collect(ch) + close(ch) + + out := map[string]struct{}{} + for m := range ch { + var pb dto.Metric + require.NoError(t, m.Write(&pb)) + for _, lp := range pb.GetLabel() { + if lp.GetName() == labelName { + out[lp.GetValue()] = struct{}{} + } + } + } + return out +} + +// Test_ProbationEvent_SupplierAddressNeverBecomesDomain is the core F1 +// regression: 4,172 of 4,608 distinct `domain` values in production were raw +// bech32 supplier addresses, making path_probation_events_total 27% of the +// whole TSDB. +// +// The path is subtle and is why the bug was invisible: reputation/selector.go +// guards with `if err != nil || domain == ""`, but ExtractDomainOrHost does not +// FAIL on a bare supplier address — a dotless host takes the +// isPrivateOrInternalDomain branch and is returned verbatim with a nil error. +// The fallback never fires, so the address arrives here looking like a +// successful extraction. +func Test_ProbationEvent_SupplierAddressNeverBecomesDomain(t *testing.T) { + ProbationEventsTotal.Reset() + + const addr = "pokt1ylsjqcl0yunve78etutw660a327avc26fxrlfr" + RecordProbationEvent(addr, "json_rpc", "arb-one", ProbationEventEntered) + + got := collectLabelValues(t, ProbationEventsTotal, LabelDomain) + require.NotContains(t, got, addr, + "raw supplier address reached the `domain` label; it must collapse to a sentinel") + require.Contains(t, got, DomainSupplierAddr) +} + +// Test_RealDomainsSurviveSanitization is the counterweight: the fix must not +// collapse the 434 legitimate domains it exists to preserve. `domain` carries +// 202 references across our Grafana dashboards — over-collapsing here would be +// indistinguishable, from the dashboard's side, from PNF dropping the label. +func Test_RealDomainsSurviveSanitization(t *testing.T) { + ProbationEventsTotal.Reset() + + // Includes hosts with a `1` in a bech32-looking position and a dotless + // internal hostname, both of which must NOT be mistaken for addresses. + keep := []string{"nodefleet.net", "example.co.uk", "web31.io", "relayminer1", "88.198.50.175"} + for _, d := range keep { + RecordProbationEvent(d, "json_rpc", "eth", ProbationEventEntered) + } + + got := collectLabelValues(t, ProbationEventsTotal, LabelDomain) + for _, d := range keep { + require.Contains(t, got, d, "legitimate domain was collapsed by the sanitizer") + } +} + +// Test_ObservationPipeline_AttackerMethodsAreBounded is the F2 regression. +// +// This asserts the property that actually matters and that SanitizeMethodLabel +// alone does NOT provide: an unauthenticated client varying the JSON-RPC method +// or REST path cannot mint unbounded series. The sanitizer normalizes the SHAPE +// of a value but cannot bound the SET — `/aaa`, `/aab`, … are all well-formed +// static route segments and survive it verbatim. Only the guard bounds them. +func Test_ObservationPipeline_AttackerMethodsAreBounded(t *testing.T) { + ObservationPipeline.Reset() + // Swap the guard POINTER rather than copying the struct: cardinalityGuard + // embeds a sync.Map, so assigning through it copies a lock (govet copylocks). + saved := observationPipelineGuard + t.Cleanup(func() { observationPipelineGuard = saved }) + observationPipelineGuard = newCardinalityGuard("test_observation_pipeline", 100) + + // Far more distinct methods than the cap, in the shape a scanner produces. + for i := 0; i < 5000; i++ { + RecordObservation("nodefleet.net", "json_rpc", "eth", NetworkTypeCosmos, + SanitizeMethodLabel(NetworkTypeCosmos, fmt.Sprintf("/probe%d/logon.html", i)), "ok") + } + + got := collectLabelValues(t, ObservationPipeline, LabelMethod) + require.LessOrEqual(t, len(got), 100, + "attacker-varied method values were not bounded by the cardinality guard") +} + +// Test_ObservationPipeline_InjectionPayloadsCannotPanic covers the payloads PNF +// found live in the TSDB (CRLF header injection, XSS, path traversal, +// template injection). client_golang panics inside WithLabelValues on non-UTF-8 +// label values, so a single malformed request reaching an unsanitized label is +// a remote crash, not just untidy data — the 2026-06-15 incident. +func Test_ObservationPipeline_InjectionPayloadsCannotPanic(t *testing.T) { + ObservationPipeline.Reset() + + payloads := []string{ + "/\r\n\r\n", + "/\r\nSet-Cookie: x=1", + "/%2e%2e%2f%2e%2e%2f%2e%2e%2fetc/passwd", + "/..%252f..%252f..%252fetc/passwd", + "/${13337*31337}", + "/+CSCOE+/logon.html", + "/\xff\xfe invalid utf8", + } + for _, p := range payloads { + require.NotPanics(t, func() { + RecordObservation("\xff\xfe.example", "json_rpc", "cosmoshub", NetworkTypeCosmos, + SanitizeMethodLabel(NetworkTypeCosmos, p), "ok") + }, "payload %q panicked on the metrics path", p) + } +} + +// Test_RPCTypeFallback_SupplierLabelDropped is the F3 regression: `supplier` +// carried 3,289 values against the metric's own 9 domains × 12 service_ids, +// producing essentially all of its 201,068 series. +// +// The helper still ACCEPTS a supplier argument so no caller had to change; +// this asserts the value does not reach the collector. +func Test_RPCTypeFallback_SupplierLabelDropped(t *testing.T) { + RPCTypeFallbackTotal.Reset() + + const supplier = "pokt1vmy9q5ljvs39n78ygwa85t9rsffncf90xqp2lp" + RecordRPCTypeFallback("example.net", supplier, "cosmoshub", "COMET_BFT", "JSON_RPC") + + require.Empty(t, collectLabelValues(t, RPCTypeFallbackTotal, LabelSupplier), + "supplier is still being emitted as a label") +} diff --git a/metrics/domain_sanitizer.go b/metrics/domain_sanitizer.go new file mode 100644 index 000000000..b35601560 --- /dev/null +++ b/metrics/domain_sanitizer.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "strings" + "unicode/utf8" +) + +const ( + // DomainUnknown — the caller had no domain to report (empty string). + DomainUnknown = "unknown" + + // DomainSupplierAddr — the caller passed a bech32 supplier address where a + // domain was expected. Collapsed to a single sentinel rather than kept + // verbatim: the whole point of `domain` is to COLLAPSE many suppliers onto + // one operator, so admitting an address inverts the label's purpose and + // makes it expand ~1:1 with supplier count. + DomainSupplierAddr = "supplier_addr" + + // DomainLabelMaxLen — hard cap on any `domain` label value. A registrable + // domain is far shorter; this is a safety net against a pathological host. + DomainLabelMaxLen = 64 +) + +// SanitizeDomainLabel bounds the cardinality of the `domain` Prometheus label. +// MUST be called on every value flowing into a `domain` label. +// +// The problem it solves: shannonmetrics.ExtractDomainOrHost is shared between +// the metrics path and the ROUTING path (operator concentration cap, drains, +// blocked_domains, reputation keys). It returns a bare bech32 supplier address +// verbatim and with a nil error — a dotless host takes the +// isPrivateOrInternalDomain branch, which returns it as-is — so callers that +// pass an EndpointAddr rather than a URL silently emit `pokt1…` as a domain, +// and their `err != nil` fallback never fires. Fixing that inside the shared +// extractor would change endpoint selection; fixing it here cannot. +// +// Measured 2026-08-12: 4,172 of 4,608 distinct `domain` values in production +// were raw supplier addresses, driving path_probation_events_total to 2.35M +// series (27% of the whole TSDB). +// +// Deliberately NOT collapsed: +// - Bare IPs. Operationally useful when a supplier registers one, and few +// enough to be a non-issue (1 observed). The per-metric cardinality guard +// is what protects against an IP-registration flood. +// - Dotless internal hostnames (`relayminer1`). Bounded and meaningful. +func SanitizeDomainLabel(raw string) string { + d := strings.ToLower(strings.TrimSpace(raw)) + if d == "" { + return DomainUnknown + } + + // Bech32 check is gated on "no dot" so it can never fire on a real domain: + // isBech32Like requires the post-`1` remainder to be entirely alphanumeric, + // and any registrable domain contains a dot in that remainder. Without the + // gate this would be a heuristic on hostnames; with it, it is exact. + if !strings.Contains(d, ".") && isBech32Like(d) { + return DomainSupplierAddr + } + + if len(d) > DomainLabelMaxLen { + d = d[:DomainLabelMaxLen] + } + // prometheus/client_golang panics inside WithLabelValues on non-UTF-8 label + // values, and the truncation above can split a multibyte rune. + if !utf8.ValidString(d) { + d = strings.ToValidUTF8(d, "�") + } + return d +} diff --git a/metrics/leaderboard.go b/metrics/leaderboard.go index db2bed430..67ff7e87e 100644 --- a/metrics/leaderboard.go +++ b/metrics/leaderboard.go @@ -161,7 +161,7 @@ func (lp *LeaderboardPublisher) publishLeaderboard(ctx context.Context) { // Publish each entry for _, entry := range entries { ReputationEndpointLeaderboard.WithLabelValues( - entry.Domain, + SanitizeDomainLabel(entry.Domain), entry.RPCType, entry.ServiceID, fmt.Sprintf("%d", entry.TierThreshold), @@ -219,7 +219,7 @@ func (lp *LeaderboardPublisher) publishLeaderboard(ctx context.Context) { if len(cooldownCounts) > 0 { for _, entry := range cooldownCounts { - EndpointsInCooldown.WithLabelValues(entry.Domain, entry.RPCType, entry.ServiceID).Set(float64(entry.Count)) + EndpointsInCooldown.WithLabelValues(SanitizeDomainLabel(entry.Domain), entry.RPCType, entry.ServiceID).Set(float64(entry.Count)) } lp.logger.Debug().Int("entries", len(cooldownCounts)).Msg("Published cooldown counts") } @@ -236,7 +236,7 @@ func (lp *LeaderboardPublisher) publishLeaderboard(ctx context.Context) { if len(drainedCounts) > 0 { for _, entry := range drainedCounts { - EndpointsDrained.WithLabelValues(entry.Domain, entry.RPCType, entry.ServiceID).Set(float64(entry.Count)) + EndpointsDrained.WithLabelValues(SanitizeDomainLabel(entry.Domain), entry.RPCType, entry.ServiceID).Set(float64(entry.Count)) } lp.logger.Warn().Int("entries", len(drainedCounts)).Msg("⚠️ endpoints are benched by an admin drain") } diff --git a/metrics/method_sanitizer.go b/metrics/method_sanitizer.go index 7705110ed..e916fd8ac 100644 --- a/metrics/method_sanitizer.go +++ b/metrics/method_sanitizer.go @@ -155,13 +155,51 @@ func normalizeRESTPath(p string) string { if seg == "" { continue } - if isDynamicSegment(seg) { + if isDynamicSegment(seg) || !isRouteShapedSegment(seg) { parts[i] = ":var" } } + if len(parts) > maxRESTPathSegments { + parts = append(parts[:maxRESTPathSegments], "...") + } return strings.Join(parts, "/") } +// maxRESTPathSegments bounds path depth. A deeply nested path is either a +// traversal probe or a route we do not serve; either way its depth carries no +// signal worth a distinct series. +const maxRESTPathSegments = 12 + +// isRouteShapedSegment reports whether a path segment could plausibly be a +// route component: unreserved URL characters only (RFC 3986 §2.3 plus `~`). +// +// This is the SHAPE half of bounding the `method` label, and on its own it is +// NOT sufficient — `/aaa`, `/aab`, … are all route-shaped and still unbounded, +// which is why observationPipelineGuard is the actual cap (verified by +// Test_ObservationPipeline_AttackerMethodsAreBounded, where 5,000 route-shaped +// probe paths survive this function untouched). +// +// What it does buy: the hostile values PNF found live in the TSDB +// (`/\r\n\r\nSet-Cookie: …`, `/${13337*31337}`, `/+CSCOE+/logon.html`, +// `/%2e%2e%2f…`) never reach a label at all, so they cannot appear in +// dashboards, alert annotations, or anything downstream that renders a label +// value. Character sanitization alone was the 2026-06-15 fix's mistake — +// it is worth having, it is just not the bound. +func isRouteShapedSegment(seg string) bool { + for i := 0; i < len(seg); i++ { + c := seg[i] + switch { + case c >= 'a' && c <= 'z': + case c >= 'A' && c <= 'Z': + case c >= '0' && c <= '9': + case c == '_' || c == '-' || c == '.' || c == '~': + default: + return false + } + } + return true +} + // isDynamicSegment reports whether a path segment looks like a request-specific // identifier (block height, tx hash, account address, opaque token) rather than // a stable route component. diff --git a/metrics/metrics.go b/metrics/metrics.go index af596c7ee..1837b37dd 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -387,6 +387,7 @@ func SetCircuitBreakerState(serviceID, domain string, broken bool) { if domain == "" { return } + domain = SanitizeDomainLabel(domain) v := 0.0 if broken { v = 1.0 @@ -450,12 +451,18 @@ var DomainCircuitBreakerEventsTotal = promauto.NewCounterVec( // RecordCircuitBreakerEvent increments the per-(service, domain, reason, event) // counter. Skipped silently when domain is empty. func RecordCircuitBreakerEvent(serviceID, domain, reasonCategory, event string) { + // Sanitize AFTER the empty check: SanitizeDomainLabel maps "" to + // DomainUnknown, which would defeat the documented skip. if domain == "" { return } + domain = SanitizeDomainLabel(domain) if reasonCategory == "" { reasonCategory = CircuitBreakReasonUnknown } + if !circuitBreakerEventsGuard.allow(serviceID, domain, reasonCategory, event) { + return + } DomainCircuitBreakerEventsTotal.WithLabelValues(serviceID, domain, reasonCategory, event).Inc() } @@ -597,7 +604,7 @@ var RPCTypeFallbackTotal = promauto.NewCounterVec( Name: MetricPrefix + "rpc_type_fallback_total", Help: "Count of RPC type fallbacks when supplier doesn't support requested RPC type.", }, - []string{LabelDomain, LabelSupplier, LabelServiceID, "requested_rpc_type", "fallback_rpc_type"}, + []string{LabelDomain, LabelServiceID, "requested_rpc_type", "fallback_rpc_type"}, ) // ============================================================================= @@ -909,10 +916,15 @@ var SelectionPoolOperators = promauto.NewHistogramVec( func RecordSelectionPool(serviceID, selectionPath string, operatorCounts map[string]int, poolSize int, selectedOperator string) { SelectionPoolSize.WithLabelValues(serviceID, selectionPath).Observe(float64(poolSize)) SelectionPoolOperators.WithLabelValues(serviceID, selectionPath).Observe(float64(len(operatorCounts))) + // operatorKey() derives these from ExtractDomainOrHost, which returns a bare + // bech32 supplier address verbatim for a dotless host — so `operator` is + // exposed to the same leak that made `domain` 90.5% supplier addresses. + // These two are counters with no Reset(), so an unsanitized value here is + // retained for the pod's lifetime. for op := range operatorCounts { - SelectionCandidateTotal.WithLabelValues(serviceID, op, selectionPath).Inc() + SelectionCandidateTotal.WithLabelValues(serviceID, SanitizeDomainLabel(op), selectionPath).Inc() } - SelectionSelectedTotal.WithLabelValues(serviceID, selectedOperator, selectionPath).Inc() + SelectionSelectedTotal.WithLabelValues(serviceID, SanitizeDomainLabel(selectedOperator), selectionPath).Inc() } // Selector paths for LabelSelectionPath. @@ -1344,6 +1356,7 @@ var WebsocketEndpointStallTotal = promauto.NewCounterVec( // one backend collapse onto the same (domain, …) series, which is what every // consumer of this metric already aggregates to. func RecordHealthCheck(domain, _, rpcType, serviceID, healthCheckName, reputationSignal string) { + domain = SanitizeDomainLabel(domain) if !healthCheckStatusGuard.allow(domain, rpcType, serviceID, healthCheckName, reputationSignal) { return } @@ -1357,22 +1370,31 @@ func RecordHealthCheckDeduped(serviceID string) { // RecordObservation records an observation pipeline event func RecordObservation(domain, rpcType, serviceID, networkType, method, reputationSignal string) { + domain = SanitizeDomainLabel(domain) + // `method` is attacker-controlled; this guard is the hard bound on it. See + // observationPipelineGuard. + if !observationPipelineGuard.allow(domain, rpcType, serviceID, networkType, method, reputationSignal) { + return + } ObservationPipeline.WithLabelValues(domain, rpcType, serviceID, networkType, method, reputationSignal).Inc() } // RecordLatencyReputation records a latency categorization func RecordLatencyReputation(domain, rpcType, serviceID, latencySignal string) { + domain = SanitizeDomainLabel(domain) LatencyReputation.WithLabelValues(domain, rpcType, serviceID, latencySignal).Inc() } // RecordRequest records a request with status code and latency func RecordRequest(domain, rpcType, serviceID, statusCode string, latencySeconds float64) { + domain = SanitizeDomainLabel(domain) RequestsTotal.WithLabelValues(domain, rpcType, serviceID, statusCode).Inc() RequestLatency.WithLabelValues(domain, rpcType, serviceID, statusCode).Observe(latencySeconds) } // RecordRetryDistribution records why a retry happened func RecordRetryDistribution(domain, rpcType, serviceID, reason string) { + domain = SanitizeDomainLabel(domain) RetriesDistribution.WithLabelValues(domain, rpcType, serviceID, reason).Inc() } @@ -1567,12 +1589,17 @@ func RecordRequestSize(rpcType, serviceID string, bytesReceived, bytesSent int64 // RecordProbationEvent records a probation event (entered, exited, or routed) func RecordProbationEvent(domain, rpcType, serviceID, event string) { + domain = SanitizeDomainLabel(domain) + if !probationEventsGuard.allow(domain, rpcType, serviceID, event) { + return + } ProbationEventsTotal.WithLabelValues(domain, rpcType, serviceID, event).Inc() } // RecordSupplierBlacklist records a supplier being blacklisted with the specific reason // reason should be one of the BlacklistReason* constants func RecordSupplierBlacklist(domain, supplier, serviceID, reason string) { + domain = SanitizeDomainLabel(domain) SupplierBlacklistTotal.WithLabelValues(domain, supplier, serviceID, reason).Inc() } @@ -1595,13 +1622,25 @@ func RecordSupplierPubkeyRecovered(supplier string) { } // RecordRPCTypeFallback records when a supplier doesn't support the requested RPC type -// and a fallback RPC type is used instead -func RecordRPCTypeFallback(domain, supplier, serviceID, requestedRPCType, fallbackRPCType string) { - RPCTypeFallbackTotal.WithLabelValues(domain, supplier, serviceID, requestedRPCType, fallbackRPCType).Inc() +// and a fallback RPC type is used instead. +// +// The second argument is the supplier address. It is accepted and ignored: the +// signature is kept so callers need no change, but the value is deliberately NOT +// used as a label. It carried 3,289 distinct values against the metric's own 9 +// domains × 12 service_ids, doing essentially all of the 201,068-series +// multiplication this counter reached in production. Per-supplier attribution +// belongs in traces or logs, not a Prometheus label. +func RecordRPCTypeFallback(domain, _, serviceID, requestedRPCType, fallbackRPCType string) { + domain = SanitizeDomainLabel(domain) + if !rpcTypeFallbackGuard.allow(domain, serviceID, requestedRPCType, fallbackRPCType) { + return + } + RPCTypeFallbackTotal.WithLabelValues(domain, serviceID, requestedRPCType, fallbackRPCType).Inc() } // SetMeanScore sets the mean reputation score for a domain/service/rpc_type combination func SetMeanScore(domain, serviceID, rpcType string, score float64) { + domain = SanitizeDomainLabel(domain) ReputationMeanScore.WithLabelValues(domain, serviceID, rpcType).Set(score) } @@ -1657,6 +1696,7 @@ func RecordSupplierSignal(supplier, serviceID, signalType string) { // statusCode should be the HTTP status code category (2xx, 4xx, 5xx, etc.) // reputationSignal should be the signal recorded (ok, minor_error, major_error, etc.) func RecordRelay(domain, rpcType, serviceID, statusCode, reputationSignal, relayType string, latencySeconds float64) { + domain = SanitizeDomainLabel(domain) RelaysTotal.WithLabelValues(domain, rpcType, serviceID, statusCode, reputationSignal, relayType).Inc() RelayLatency.WithLabelValues(domain, rpcType, serviceID, statusCode, reputationSignal, relayType).Observe(latencySeconds) } @@ -1666,6 +1706,7 @@ func RecordRelay(domain, rpcType, serviceID, statusCode, reputationSignal, relay // RecordWebsocketConnectionEstablished records a successful WebSocket connection establishment // and increments the active connection count func RecordWebsocketConnectionEstablished(domain, serviceID string) { + domain = SanitizeDomainLabel(domain) WebsocketConnectionsActive.WithLabelValues(domain, serviceID).Inc() WebsocketConnectionEventsTotal.WithLabelValues(domain, serviceID, WSEventEstablished).Inc() } @@ -1673,6 +1714,7 @@ func RecordWebsocketConnectionEstablished(domain, serviceID string) { // RecordWebsocketConnectionClosed records a WebSocket connection closure // and decrements the active connection count, recording the duration func RecordWebsocketConnectionClosed(domain, serviceID string, durationSeconds float64) { + domain = SanitizeDomainLabel(domain) WebsocketConnectionsActive.WithLabelValues(domain, serviceID).Dec() WebsocketConnectionEventsTotal.WithLabelValues(domain, serviceID, WSEventClosed).Inc() WebsocketConnectionDuration.WithLabelValues(domain, serviceID).Observe(durationSeconds) @@ -1698,17 +1740,20 @@ func MoveWebsocketConnection(oldDomain, newDomain, serviceID string) { // event="closed" / duration observation still comes from RecordWebsocketConnectionClosed on // the shared close path; this only labels WHY. func RecordWebsocketIdleReaped(domain, serviceID string) { + domain = SanitizeDomainLabel(domain) WebsocketIdleReapedTotal.WithLabelValues(domain, serviceID).Inc() } // RecordWebsocketConnectionFailed records a WebSocket connection failure func RecordWebsocketConnectionFailed(domain, serviceID string) { + domain = SanitizeDomainLabel(domain) WebsocketConnectionEventsTotal.WithLabelValues(domain, serviceID, WSEventFailed).Inc() } // RecordWebsocketMessage records a WebSocket message // direction should be WSDirectionClientToEndpoint or WSDirectionEndpointToClient func RecordWebsocketMessage(domain, serviceID, direction, reputationSignal string) { + domain = SanitizeDomainLabel(domain) WebsocketMessagesTotal.WithLabelValues(domain, serviceID, direction, reputationSignal).Inc() } @@ -1716,6 +1761,7 @@ func RecordWebsocketMessage(domain, serviceID, direction, reputationSignal strin // Called once per connection per sampler pass, including for connections currently at // zero — see WebsocketConnectionFrameRate for why the zeros are load-bearing. func RecordWebsocketConnectionFrameRate(domain, serviceID string, framesPerSec float64) { + domain = SanitizeDomainLabel(domain) WebsocketConnectionFrameRate.WithLabelValues(domain, serviceID).Observe(framesPerSec) } @@ -1724,6 +1770,7 @@ func RecordWebsocketConnectionFrameRate(domain, serviceID string, framesPerSec f // for the session-rebind feature. result should be one of the WSRebind* labels; trigger // one of the WSRebindTrigger* labels (what initiated the rebind). func RecordWebsocketRebind(domain, serviceID, result, trigger string, replayedSubscriptions int) { + domain = SanitizeDomainLabel(domain) WebsocketRebindTotal.WithLabelValues(domain, serviceID, result, trigger).Inc() if replayedSubscriptions > 0 { WebsocketSubscriptionsReplayedTotal.WithLabelValues(domain, serviceID).Add(float64(replayedSubscriptions)) @@ -1733,6 +1780,7 @@ func RecordWebsocketRebind(domain, serviceID, result, trigger string, replayedSu // RecordWebsocketEndpointStall records a staleness-watchdog firing. gaveUp distinguishes a // forced rebind (false) from giving up and closing the client (true). func RecordWebsocketEndpointStall(domain, serviceID string, gaveUp bool) { + domain = SanitizeDomainLabel(domain) result := WSStallRebind if gaveUp { result = WSStallGaveUp diff --git a/pnf_path_rules.yaml b/pnf_path_rules.yaml index 440e450d9..9d5f0ef18 100644 --- a/pnf_path_rules.yaml +++ b/pnf_path_rules.yaml @@ -450,7 +450,12 @@ # Block time: ~6s | Source: chainspect.app, moonscan.io # Formula: 300s / 6s = 50 blocks for 5 minutes sync_allowance: 50 - check_interval: 10s + # 30s, not 10s: health checks were 78% of this service's total relays + # (29.4 HC/s against 8.1 user relays/s, measured 2026-08-07). A 10s detection + # window buys nothing at that traffic level. Per-check rate is ~98/interval_seconds, + # independent of endpoint count, so interval is a real lever (per-service scheduling + # landed 2026-08-03; before that a service could only lower the global floor). + check_interval: 30s enabled: true checks: - name: eth_blockNumber @@ -474,19 +479,14 @@ expected_response_contains: '0x504"' timeout: 5s reputation_signal: critical_error - # Archival check - block 677,000 - # Address: 0xf89d7b9c864f589bbf53a82105107622b35eaa40 (from E2E tests) - # Block: 0xa5488 (677,000) - - name: moonbeam_archival - type: json_rpc - method: POST - path: / - body: '{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xf89d7b9c864f589bbf53a82105107622b35eaa40","0xa5488"]}' - expected_status_code: 200 - expected_response_contains: "0x109855c3ae698000" - timeout: 10s - archival: true - reputation_signal: minor_error + # REMOVED 2026-08-07: moonbeam_archival returned minor_error on 100% of runs + # (9.80/s minor_error against 0.00/s ok). The expected balance was verified + # CORRECT against rpc.api.moonbeam.network (block 0xa5488 -> 0x109855c3ae698000), + # so this was real archival-supply absence, not a stale rule value. + # A penalty applied uniformly to every endpoint distorts no ranking and alerts + # nobody, so it spent a third of this service's health-check budget for zero + # signal. Re-add (address 0xf89d7b9c864f589bbf53a82105107622b35eaa40, block + # 0xa5488) if archival supply appears. # WebSocket data-path check. Deliberately sends a request rather than only # opening the socket: a supplier can complete the handshake, answer pings, and # still deliver nothing, so connect-only would score it healthy. @@ -507,7 +507,10 @@ # Block time: ~6s | Source: moonscan.io (same as Moonbeam) # Formula: 300s / 6s = 50 blocks for 5 minutes sync_allowance: 50 - check_interval: 10s + # 30s, not 10s: health checks were 71% of this service's total relays + # (19.6 HC/s against 8.1 user relays/s, measured 2026-08-07). See moonbeam for the + # rate model — per-check rate is ~98/interval_seconds, flat in endpoint count. + check_interval: 30s enabled: true checks: - name: eth_blockNumber @@ -608,7 +611,10 @@ # Block time: ~5s | Source: celoscan.io, celo docs # Formula: 300s / 5s = 60 blocks for 5 minutes sync_allowance: 60 - check_interval: 10s + # 30s, not 10s: health checks were 87% of this service's total relays — the worst + # ratio on the fleet (22.9 HC/s against 3.3 user relays/s, measured 2026-08-07). + # See moonbeam for the rate model. + check_interval: 30s enabled: true checks: - name: eth_blockNumber @@ -632,19 +638,14 @@ expected_response_contains: '0xa4ec"' timeout: 5s reputation_signal: critical_error - # Archival check - block 20,000,000 - # Address: 0xf89d7b9c864f589bbF53a82105107622B35EaA40 (from E2E tests) - # Block: 0x1312d00 (20,000,000) - - name: celo_archival - type: json_rpc - method: POST - path: / - body: '{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xf89d7b9c864f589bbF53a82105107622B35EaA40","0x1312d00"]}' - expected_status_code: 200 - expected_response_contains: "0x9a1201de9ff1d6718714" - timeout: 10s - archival: true - reputation_signal: minor_error + # REMOVED 2026-08-07: celo_archival returned minor_error on 100% of runs + # (9.80/s minor_error against 0.00/s ok). The expected balance was verified + # CORRECT against forno.celo.org (block 0x1312d00 -> 0x9a1201de9ff1d6718714), + # so this was real archival-supply absence, not a stale rule value. Same shape as + # moonbeam_archival above. Contrast gnosis_archival (66% fail / 34% ok) and + # eth_archival (14% fail) — those discriminate between endpoints and are kept. + # Re-add (address 0xf89d7b9c864f589bbF53a82105107622B35EaA40, block 0x1312d00) + # if archival supply appears. - service_id: fantom # Block time: ~1s | Source: ftmscan.com (legacy chain, migrating to Sonic)