From 2c65c860a424d6dabacb11ea3d2392ae6c2c631a Mon Sep 17 00:00:00 2001 From: Justin Erenkrantz Date: Sun, 23 Aug 2026 20:51:35 +0000 Subject: [PATCH 1/2] Add RRCC local closure metrics Co-authored-by: c1-squire-dev[bot] --- internal/bazel/bazel.go | 11 +++ internal/daemon/grpc.go | 2 + internal/daemon/metrics.go | 7 ++ internal/daemon/metrics_test.go | 11 +++ internal/daemon/protocol.go | 6 +- internal/daemon/server.go | 23 ++++++ internal/reapi/actioncache.go | 1 + internal/reapi/reapi.go | 2 + internal/reapi/rrcc.go | 127 ++++++++++++++++++++++++++++++++ internal/reapi/rrcc_test.go | 88 ++++++++++++++++++++++ 10 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 internal/reapi/rrcc.go create mode 100644 internal/reapi/rrcc_test.go diff --git a/internal/bazel/bazel.go b/internal/bazel/bazel.go index 24b40a9..51dd07b 100644 --- a/internal/bazel/bazel.go +++ b/internal/bazel/bazel.go @@ -161,6 +161,17 @@ func (s *Store) Open(ctx context.Context, k Kind, d Digest) (*os.File, int64, bo return f, fi.Size(), true } +// OpenLocal returns a locally resident body without faulting one in from the remote tier. +// +// The Has probe refreshes the entry before Open reads it, so eviction cannot +// reclaim a body this caller is about to inspect. +func (s *Store) OpenLocal(ctx context.Context, k Kind, d Digest) (*os.File, int64, bool) { + if !s.Has(ctx, k, d) { + return nil, 0, false + } + return s.Open(ctx, k, d) +} + // Has reports whether a digest already resolves to a readable body in a // keyspace, and refreshes its last use if it does. // diff --git a/internal/daemon/grpc.go b/internal/daemon/grpc.go index 432c9fe..afc4c1a 100644 --- a/internal/daemon/grpc.go +++ b/internal/daemon/grpc.go @@ -49,6 +49,8 @@ func (s *Server) ServeBazelGRPC(ctx context.Context, ln net.Listener) error { Logf: s.logf, Verify: !s.cfg.DisableBazelVerify, }) + s.setREAPI(svc) + defer s.setREAPI(nil) g := grpc.NewServer(reapi.ServerOptions()...) svc.Register(g) diff --git a/internal/daemon/metrics.go b/internal/daemon/metrics.go index 73ee775..6992f0c 100644 --- a/internal/daemon/metrics.go +++ b/internal/daemon/metrics.go @@ -130,6 +130,13 @@ func renderMetrics(r StatusResponse) []byte { writeFamily(&b, "compactions_total", "counter", "Index compactions, which reclaim the space pruning leaves behind.", sample{value: float64(life.Compactions)}) + writeFamily(&b, "rrcc_local_closure_checks_total", "counter", + "Experimental remote repository-contents cache closures observed locally, by completeness.", + sample{labels: labels("result", "complete"), value: float64(r.RRCC.Complete)}, + sample{labels: labels("result", "marker_missing"), value: float64(r.RRCC.MarkerMissing)}, + sample{labels: labels("result", "tree_missing"), value: float64(r.RRCC.TreeMissing)}, + sample{labels: labels("result", "file_missing"), value: float64(r.RRCC.FileMissing)}, + sample{labels: labels("result", "malformed"), value: float64(r.RRCC.Malformed)}) // The shared tier's transport, which is the only part of this report about how // the cache reaches the network rather than about what it holds. Absent when diff --git a/internal/daemon/metrics_test.go b/internal/daemon/metrics_test.go index 942ef5d..89f7e26 100644 --- a/internal/daemon/metrics_test.go +++ b/internal/daemon/metrics_test.go @@ -12,6 +12,7 @@ import ( "github.com/conductorone/plaid-cache/internal/cache" "github.com/conductorone/plaid-cache/internal/index" + "github.com/conductorone/plaid-cache/internal/reapi" "github.com/conductorone/plaid-cache/internal/remote" ) @@ -203,6 +204,10 @@ func TestMetricsExposition(t *testing.T) { UploadQueueDepth: 12, UploadQueueCapacity: 512, + RRCC: reapi.RRCCMetricsSnapshot{ + Complete: 11, MarkerMissing: 2, TreeMissing: 3, FileMissing: 5, Malformed: 7, + }, + Remote: &remote.StatsSnapshot{ ConnsReused: 190, ConnsNew: 22, @@ -244,6 +249,7 @@ func TestMetricsExposition(t *testing.T) { for _, f := range []string{ "plaid_cache_gets_total", "plaid_cache_puts_total", "plaid_cache_repairs_total", "plaid_cache_uploads_total", "plaid_cache_compactions_total", + "plaid_cache_rrcc_local_closure_checks_total", // Requests that opened a connection and requests that did not: both only // ever rise, so a rate() over either is the question worth asking. "plaid_cache_remote_requests_total", @@ -278,6 +284,11 @@ func TestMetricsExposition(t *testing.T) { e.wantSample(t, `plaid_cache_uploads_total{result="dropped"}`, 2) e.wantSample(t, `plaid_cache_uploads_total{result="skipped"}`, 3) e.wantSample(t, "plaid_cache_compactions_total", 7) + e.wantSample(t, `plaid_cache_rrcc_local_closure_checks_total{result="complete"}`, 11) + e.wantSample(t, `plaid_cache_rrcc_local_closure_checks_total{result="marker_missing"}`, 2) + e.wantSample(t, `plaid_cache_rrcc_local_closure_checks_total{result="tree_missing"}`, 3) + e.wantSample(t, `plaid_cache_rrcc_local_closure_checks_total{result="file_missing"}`, 5) + e.wantSample(t, `plaid_cache_rrcc_local_closure_checks_total{result="malformed"}`, 7) e.wantSample(t, "plaid_cache_upload_queue_depth", 12) e.wantSample(t, "plaid_cache_upload_queue_capacity", 512) diff --git a/internal/daemon/protocol.go b/internal/daemon/protocol.go index 633a2cf..8fd3084 100644 --- a/internal/daemon/protocol.go +++ b/internal/daemon/protocol.go @@ -18,6 +18,7 @@ import ( "github.com/conductorone/plaid-cache/internal/cache" "github.com/conductorone/plaid-cache/internal/index" + "github.com/conductorone/plaid-cache/internal/reapi" "github.com/conductorone/plaid-cache/internal/remote" ) @@ -156,7 +157,10 @@ type StatusResponse struct { OldestAge string `json:"oldest_age,omitempty"` NewestAge string `json:"newest_age,omitempty"` Metrics cache.MetricsSnapshot `json:"metrics"` - Err string `json:"err,omitempty"` + + // RRCC records local closure observations for synthetic repository-cache entries. + RRCC reapi.RRCCMetricsSnapshot `json:"rrcc"` + Err string `json:"err,omitempty"` // UploadQueueDepth is how many uploads to the shared tier are waiting on // this daemon's pool right now, and UploadQueueCapacity how many may. diff --git a/internal/daemon/server.go b/internal/daemon/server.go index 1c12ff1..3aff39c 100644 --- a/internal/daemon/server.go +++ b/internal/daemon/server.go @@ -23,6 +23,7 @@ import ( "github.com/conductorone/plaid-cache/internal/config" "github.com/conductorone/plaid-cache/internal/ids" "github.com/conductorone/plaid-cache/internal/index" + "github.com/conductorone/plaid-cache/internal/reapi" "github.com/conductorone/plaid-cache/internal/wire" ) @@ -60,6 +61,10 @@ type Server struct { bazelStore *bazel.Store bazelErr error + // reapiMu protects the current gRPC service's process-local observability. + reapiMu sync.RWMutex + reapi *reapi.Server + // cleanOnce gates the sweep of abandoned temporaries. See cleanTemp. cleanOnce sync.Once } @@ -138,6 +143,23 @@ func (s *Server) sessionIdleTimeout() time.Duration { return sessionIdleTimeout } +// setREAPI publishes the gRPC service whose process-local metrics status reads. +func (s *Server) setREAPI(svc *reapi.Server) { + s.reapiMu.Lock() + defer s.reapiMu.Unlock() + s.reapi = svc +} + +// rrccMetrics returns the current gRPC service's RRCC observations. +func (s *Server) rrccMetrics() reapi.RRCCMetricsSnapshot { + s.reapiMu.RLock() + defer s.reapiMu.RUnlock() + if s.reapi == nil { + return reapi.RRCCMetricsSnapshot{} + } + return s.reapi.RRCCMetrics() +} + // Listen binds the unix socket. // // A socket file left behind by a daemon that died without cleaning up would @@ -603,6 +625,7 @@ func (s *Server) status() StatusResponse { Uptime: uptime.String(), UptimeSeconds: uptime.Seconds(), Metrics: s.cache.Metrics(), + RRCC: s.rrccMetrics(), } r.UploadQueueDepth, r.UploadQueueCapacity = s.cache.UploadQueue() if rs, ok := s.cache.RemoteStats(); ok { diff --git a/internal/reapi/actioncache.go b/internal/reapi/actioncache.go index 7b231da..708c3a3 100644 --- a/internal/reapi/actioncache.go +++ b/internal/reapi/actioncache.go @@ -70,6 +70,7 @@ func (a *actionCacheService) GetActionResult(ctx context.Context, req *repb.GetA a.srv.logf("bazel grpc: action result %s does not parse: %v", d, err) return nil, status.Errorf(codes.NotFound, "plaid-cache: no action result for %s", d) } + a.srv.observeRRCCLocalClosure(ctx, &res) return &res, nil } diff --git a/internal/reapi/reapi.go b/internal/reapi/reapi.go index cf3f1b6..5d658b1 100644 --- a/internal/reapi/reapi.go +++ b/internal/reapi/reapi.go @@ -97,6 +97,8 @@ type Server struct { uploads *uploads logf cache.Logf + rrccMetrics rrccMetrics + // verify mirrors the store's digest checking. It is read here to decide // whether a client may name a digest function this server cannot check, // which is the one protocol-level consequence of turning verification off. diff --git a/internal/reapi/rrcc.go b/internal/reapi/rrcc.go new file mode 100644 index 0000000..93a7113 --- /dev/null +++ b/internal/reapi/rrcc.go @@ -0,0 +1,127 @@ +// Copyright 2026 The plaid-cache authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package reapi + +import ( + "context" + "io" + "sync/atomic" + + repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "google.golang.org/protobuf/proto" + + "github.com/conductorone/plaid-cache/internal/bazel" +) + +// rrccTreeLimit bounds one observation so a malicious cache entry cannot turn +// an action-cache lookup into unbounded metadata work. +const rrccTreeLimit = 16 << 20 + +// RRCCMetricsSnapshot reports local-closure observations for Bazel's experimental +// remote repository-contents cache entries. +type RRCCMetricsSnapshot struct { + Complete int64 + MarkerMissing int64 + TreeMissing int64 + FileMissing int64 + Malformed int64 +} + +type rrccMetrics struct { + complete atomic.Int64 + markerMissing atomic.Int64 + treeMissing atomic.Int64 + fileMissing atomic.Int64 + malformed atomic.Int64 +} + +// Snapshot returns the current RRCC local-closure observation counts. +func (m *rrccMetrics) Snapshot() RRCCMetricsSnapshot { + return RRCCMetricsSnapshot{ + Complete: m.complete.Load(), + MarkerMissing: m.markerMissing.Load(), + TreeMissing: m.treeMissing.Load(), + FileMissing: m.fileMissing.Load(), + Malformed: m.malformed.Load(), + } +} + +// RRCCMetrics returns local-only closure observations accumulated by this REAPI server. +func (s *Server) RRCCMetrics() RRCCMetricsSnapshot { return s.rrccMetrics.Snapshot() } + +// observeRRCCLocalClosure records whether a synthetic repository-cache result is +// wholly available in the local cache. It deliberately does not alter the response: +// remote-only entries remain valid until remote presence checks are introduced. +func (s *Server) observeRRCCLocalClosure(ctx context.Context, result *repb.ActionResult) { + marker, tree, ok := rrccOutputs(result) + if !ok { + return + } + if !s.hasLocalCAS(ctx, marker) { + s.rrccMetrics.markerMissing.Add(1) + s.logf("bazel grpc: rrcc local closure missing marker %s", marker.GetHash()) + return + } + treeDigest, err := digest(tree) + if err != nil { + s.rrccMetrics.malformed.Add(1) + s.logf("bazel grpc: rrcc local closure has malformed tree digest: %v", err) + return + } + file, size, ok := s.store.OpenLocal(ctx, bazel.KindCAS, treeDigest) + if !ok { + s.rrccMetrics.treeMissing.Add(1) + s.logf("bazel grpc: rrcc local closure missing tree %s", tree.GetHash()) + return + } + defer func() { _ = file.Close() }() + if size > rrccTreeLimit { + s.rrccMetrics.malformed.Add(1) + s.logf("bazel grpc: rrcc local closure tree %s is %d bytes, refusing to inspect", tree.GetHash(), size) + return + } + body, err := io.ReadAll(io.LimitReader(file, rrccTreeLimit)) + if err != nil { + s.rrccMetrics.malformed.Add(1) + s.logf("bazel grpc: read rrcc tree %s: %v", tree.GetHash(), err) + return + } + var contents repb.Tree + if err := proto.Unmarshal(body, &contents); err != nil { + s.rrccMetrics.malformed.Add(1) + s.logf("bazel grpc: rrcc tree %s does not parse: %v", tree.GetHash(), err) + return + } + for _, directory := range append([]*repb.Directory{contents.GetRoot()}, contents.GetChildren()...) { + for _, node := range directory.GetFiles() { + if !s.hasLocalCAS(ctx, node.GetDigest()) { + s.rrccMetrics.fileMissing.Add(1) + s.logf("bazel grpc: rrcc local closure missing file %s", node.GetDigest().GetHash()) + return + } + } + } + s.rrccMetrics.complete.Add(1) +} + +// hasLocalCAS reports whether a valid digest is currently present in the local cache. +func (s *Server) hasLocalCAS(ctx context.Context, d *repb.Digest) bool { + parsed, err := digest(d) + return err == nil && s.store.Has(ctx, bazel.KindCAS, parsed) +} + +// rrccOutputs recognizes Bazel's synthetic remote repository-contents result shape. +func rrccOutputs(result *repb.ActionResult) (marker, tree *repb.Digest, ok bool) { + for _, output := range result.GetOutputFiles() { + if output.GetPath() == ".recorded_inputs" { + marker = output.GetDigest() + } + } + for _, output := range result.GetOutputDirectories() { + if output.GetPath() == "repo_contents" { + tree = output.GetTreeDigest() + } + } + return marker, tree, marker != nil && tree != nil +} diff --git a/internal/reapi/rrcc_test.go b/internal/reapi/rrcc_test.go new file mode 100644 index 0000000..cbcca2e --- /dev/null +++ b/internal/reapi/rrcc_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 The plaid-cache authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package reapi + +import ( + "testing" + + repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "google.golang.org/protobuf/proto" +) + +// TestRRCCLocalClosureMetrics records a complete synthetic repository closure. +func TestRRCCLocalClosureMetrics(t *testing.T) { + h := newHarness(t) + marker := []byte("recorded inputs") + file := []byte("package refactor") + putBlob(t, h, marker) + putBlob(t, h, file) + tree := &repb.Tree{Root: &repb.Directory{Files: []*repb.FileNode{{Name: "BUILD.bazel", Digest: digestOf(file)}}}} + treeDigest := putTree(t, h, tree) + action := digestOf([]byte("rrcc complete")) + + putRRCCActionResult(t, h, action, digestOf(marker), treeDigest) + if _, err := h.ac.GetActionResult(ctx(t), &repb.GetActionResultRequest{ActionDigest: action}); err != nil { + t.Fatalf("GetActionResult: %v", err) + } + if got := h.srv.RRCCMetrics(); got.Complete != 1 { + t.Fatalf("RRCCMetrics = %+v, want one complete closure", got) + } +} + +// TestRRCCLocalClosureMetricsRecordsMissingFile records a missing nested repository file. +func TestRRCCLocalClosureMetricsRecordsMissingFile(t *testing.T) { + h := newHarness(t) + marker := []byte("recorded inputs") + putBlob(t, h, marker) + missing := digestOf([]byte("missing BUILD.bazel")) + tree := &repb.Tree{Root: &repb.Directory{Files: []*repb.FileNode{{Name: "BUILD.bazel", Digest: missing}}}} + treeDigest := putTree(t, h, tree) + action := digestOf([]byte("rrcc missing file")) + + putRRCCActionResult(t, h, action, digestOf(marker), treeDigest) + if _, err := h.ac.GetActionResult(ctx(t), &repb.GetActionResultRequest{ActionDigest: action}); err != nil { + t.Fatalf("GetActionResult: %v", err) + } + if got := h.srv.RRCCMetrics(); got.FileMissing != 1 { + t.Fatalf("RRCCMetrics = %+v, want one missing file", got) + } +} + +// TestRRCCLocalClosureMetricsIgnoreOrdinaryActions keeps normal action-cache hits off RRCC metrics. +func TestRRCCLocalClosureMetricsIgnoreOrdinaryActions(t *testing.T) { + h := newHarness(t) + action := digestOf([]byte("ordinary action")) + if _, err := h.ac.UpdateActionResult(ctx(t), &repb.UpdateActionResultRequest{ActionDigest: action, ActionResult: result(0, "ordinary")}); err != nil { + t.Fatalf("UpdateActionResult: %v", err) + } + if _, err := h.ac.GetActionResult(ctx(t), &repb.GetActionResultRequest{ActionDigest: action}); err != nil { + t.Fatalf("GetActionResult: %v", err) + } + if got := h.srv.RRCCMetrics(); got != (RRCCMetricsSnapshot{}) { + t.Fatalf("RRCCMetrics = %+v, want zero", got) + } +} + +// putRRCCActionResult stores Bazel's synthetic repository-cache result shape. +func putRRCCActionResult(t *testing.T, h *harness, action, marker, tree *repb.Digest) { + t.Helper() + result := &repb.ActionResult{ + OutputFiles: []*repb.OutputFile{{Path: ".recorded_inputs", Digest: marker}}, + OutputDirectories: []*repb.OutputDirectory{{Path: "repo_contents", TreeDigest: tree}}, + } + if _, err := h.ac.UpdateActionResult(ctx(t), &repb.UpdateActionResultRequest{ActionDigest: action, ActionResult: result}); err != nil { + t.Fatalf("UpdateActionResult: %v", err) + } +} + +// putTree stores a Tree as a CAS blob and returns its digest. +func putTree(t *testing.T, h *harness, tree *repb.Tree) *repb.Digest { + t.Helper() + body, err := proto.Marshal(tree) + if err != nil { + t.Fatalf("marshal Tree: %v", err) + } + putBlob(t, h, body) + return digestOf(body) +} From a626c8ee014e2ab5f6129de1264e203c5b95a267 Mon Sep 17 00:00:00 2001 From: Justin Erenkrantz Date: Sun, 23 Aug 2026 20:55:36 +0000 Subject: [PATCH 2/2] Miss incomplete RRCC closures Co-authored-by: c1-squire-dev[bot] --- internal/reapi/actioncache.go | 4 +++- internal/reapi/rrcc.go | 26 ++++++++++++---------- internal/reapi/rrcc_test.go | 41 ++++++++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/internal/reapi/actioncache.go b/internal/reapi/actioncache.go index 708c3a3..b158602 100644 --- a/internal/reapi/actioncache.go +++ b/internal/reapi/actioncache.go @@ -70,7 +70,9 @@ func (a *actionCacheService) GetActionResult(ctx context.Context, req *repb.GetA a.srv.logf("bazel grpc: action result %s does not parse: %v", d, err) return nil, status.Errorf(codes.NotFound, "plaid-cache: no action result for %s", d) } - a.srv.observeRRCCLocalClosure(ctx, &res) + if !a.srv.validateRRCCLocalClosure(ctx, &res) { + return nil, status.Errorf(codes.NotFound, "plaid-cache: incomplete local rrcc closure for %s", d) + } return &res, nil } diff --git a/internal/reapi/rrcc.go b/internal/reapi/rrcc.go index 93a7113..020044d 100644 --- a/internal/reapi/rrcc.go +++ b/internal/reapi/rrcc.go @@ -50,59 +50,61 @@ func (m *rrccMetrics) Snapshot() RRCCMetricsSnapshot { // RRCCMetrics returns local-only closure observations accumulated by this REAPI server. func (s *Server) RRCCMetrics() RRCCMetricsSnapshot { return s.rrccMetrics.Snapshot() } -// observeRRCCLocalClosure records whether a synthetic repository-cache result is -// wholly available in the local cache. It deliberately does not alter the response: -// remote-only entries remain valid until remote presence checks are introduced. -func (s *Server) observeRRCCLocalClosure(ctx context.Context, result *repb.ActionResult) { +// validateRRCCLocalClosure accepts ordinary action results and synthetic +// repository-cache results whose complete closure is local. A missing local body +// is a cache miss: returning the ActionResult would make Bazel fail later while +// lazily reading the injected repository. +func (s *Server) validateRRCCLocalClosure(ctx context.Context, result *repb.ActionResult) bool { marker, tree, ok := rrccOutputs(result) if !ok { - return + return true } if !s.hasLocalCAS(ctx, marker) { s.rrccMetrics.markerMissing.Add(1) s.logf("bazel grpc: rrcc local closure missing marker %s", marker.GetHash()) - return + return false } treeDigest, err := digest(tree) if err != nil { s.rrccMetrics.malformed.Add(1) s.logf("bazel grpc: rrcc local closure has malformed tree digest: %v", err) - return + return false } file, size, ok := s.store.OpenLocal(ctx, bazel.KindCAS, treeDigest) if !ok { s.rrccMetrics.treeMissing.Add(1) s.logf("bazel grpc: rrcc local closure missing tree %s", tree.GetHash()) - return + return false } defer func() { _ = file.Close() }() if size > rrccTreeLimit { s.rrccMetrics.malformed.Add(1) s.logf("bazel grpc: rrcc local closure tree %s is %d bytes, refusing to inspect", tree.GetHash(), size) - return + return false } body, err := io.ReadAll(io.LimitReader(file, rrccTreeLimit)) if err != nil { s.rrccMetrics.malformed.Add(1) s.logf("bazel grpc: read rrcc tree %s: %v", tree.GetHash(), err) - return + return false } var contents repb.Tree if err := proto.Unmarshal(body, &contents); err != nil { s.rrccMetrics.malformed.Add(1) s.logf("bazel grpc: rrcc tree %s does not parse: %v", tree.GetHash(), err) - return + return false } for _, directory := range append([]*repb.Directory{contents.GetRoot()}, contents.GetChildren()...) { for _, node := range directory.GetFiles() { if !s.hasLocalCAS(ctx, node.GetDigest()) { s.rrccMetrics.fileMissing.Add(1) s.logf("bazel grpc: rrcc local closure missing file %s", node.GetDigest().GetHash()) - return + return false } } } s.rrccMetrics.complete.Add(1) + return true } // hasLocalCAS reports whether a valid digest is currently present in the local cache. diff --git a/internal/reapi/rrcc_test.go b/internal/reapi/rrcc_test.go index cbcca2e..6cc6ed7 100644 --- a/internal/reapi/rrcc_test.go +++ b/internal/reapi/rrcc_test.go @@ -7,6 +7,8 @@ import ( "testing" repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) @@ -30,7 +32,40 @@ func TestRRCCLocalClosureMetrics(t *testing.T) { } } -// TestRRCCLocalClosureMetricsRecordsMissingFile records a missing nested repository file. +// TestRRCCLocalClosureMetricsRecordsMissingMarker turns a missing repository marker into a miss. +func TestRRCCLocalClosureMetricsRecordsMissingMarker(t *testing.T) { + h := newHarness(t) + action := digestOf([]byte("rrcc missing marker")) + missingMarker := digestOf([]byte("missing marker")) + tree := putTree(t, h, &repb.Tree{}) + + putRRCCActionResult(t, h, action, missingMarker, tree) + if _, err := h.ac.GetActionResult(ctx(t), &repb.GetActionResultRequest{ActionDigest: action}); status.Code(err) != codes.NotFound { + t.Fatalf("GetActionResult = %v, want NotFound", err) + } + if got := h.srv.RRCCMetrics(); got.MarkerMissing != 1 { + t.Fatalf("RRCCMetrics = %+v, want one missing marker", got) + } +} + +// TestRRCCLocalClosureMetricsRecordsMissingTree turns a missing repository Tree into a miss. +func TestRRCCLocalClosureMetricsRecordsMissingTree(t *testing.T) { + h := newHarness(t) + marker := []byte("recorded inputs") + putBlob(t, h, marker) + action := digestOf([]byte("rrcc missing tree")) + missingTree := digestOf([]byte("missing Tree")) + + putRRCCActionResult(t, h, action, digestOf(marker), missingTree) + if _, err := h.ac.GetActionResult(ctx(t), &repb.GetActionResultRequest{ActionDigest: action}); status.Code(err) != codes.NotFound { + t.Fatalf("GetActionResult = %v, want NotFound", err) + } + if got := h.srv.RRCCMetrics(); got.TreeMissing != 1 { + t.Fatalf("RRCCMetrics = %+v, want one missing tree", got) + } +} + +// TestRRCCLocalClosureMetricsRecordsMissingFile turns a missing nested repository file into a miss. func TestRRCCLocalClosureMetricsRecordsMissingFile(t *testing.T) { h := newHarness(t) marker := []byte("recorded inputs") @@ -41,8 +76,8 @@ func TestRRCCLocalClosureMetricsRecordsMissingFile(t *testing.T) { action := digestOf([]byte("rrcc missing file")) putRRCCActionResult(t, h, action, digestOf(marker), treeDigest) - if _, err := h.ac.GetActionResult(ctx(t), &repb.GetActionResultRequest{ActionDigest: action}); err != nil { - t.Fatalf("GetActionResult: %v", err) + if _, err := h.ac.GetActionResult(ctx(t), &repb.GetActionResultRequest{ActionDigest: action}); status.Code(err) != codes.NotFound { + t.Fatalf("GetActionResult = %v, want NotFound", err) } if got := h.srv.RRCCMetrics(); got.FileMissing != 1 { t.Fatalf("RRCCMetrics = %+v, want one missing file", got)