diff --git a/cache/disk/BUILD.bazel b/cache/disk/BUILD.bazel index 603f1cc45..7bde4ea1c 100644 --- a/cache/disk/BUILD.bazel +++ b/cache/disk/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "disk.go", "findmissing.go", "lru.go", + "metrics.go", "options.go", ], importpath = "github.com/buchgr/bazel-remote/cache/disk", @@ -17,7 +18,6 @@ go_library( "//utils/tempfile:go_default_library", "@com_github_djherbis_atime//:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", - "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@org_golang_google_grpc//codes:go_default_library", "@org_golang_google_grpc//status:go_default_library", "@org_golang_google_protobuf//proto:go_default_library", @@ -39,6 +39,8 @@ go_test( "//cache/httpproxy:go_default_library", "//genproto/build/bazel/remote/execution/v2:go_default_library", "//utils:go_default_library", + "@com_github_prometheus_client_golang//prometheus:go_default_library", + "@com_github_prometheus_client_golang//prometheus/testutil:go_default_library", "@org_golang_google_protobuf//proto:go_default_library", ], ) diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 3d546fcde..1cc7b3da3 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -25,8 +25,6 @@ import ( "github.com/buchgr/bazel-remote/utils/tempfile" "github.com/djherbis/atime" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" pb "github.com/buchgr/bazel-remote/genproto/build/bazel/remote/execution/v2" "google.golang.org/protobuf/proto" @@ -34,23 +32,25 @@ import ( "golang.org/x/sync/semaphore" ) -var ( - cacheHits = promauto.NewCounter(prometheus.CounterOpts{ - Name: "bazel_remote_disk_cache_hits", - Help: "The total number of disk backend cache hits", - }) - cacheMisses = promauto.NewCounter(prometheus.CounterOpts{ - Name: "bazel_remote_disk_cache_misses", - Help: "The total number of disk backend cache misses", - }) -) - var tfc = tempfile.NewCreator() var emptyZstdBlob = []byte{40, 181, 47, 253, 32, 0, 1, 0, 0} var hashKeyRegex = regexp.MustCompile("^[a-f0-9]{64}$") +type Cache interface { + Get(ctx context.Context, kind cache.EntryKind, hash string, size int64, offset int64) (io.ReadCloser, int64, error) + GetValidatedActionResult(ctx context.Context, hash string) (*pb.ActionResult, []byte, error) + GetZstd(ctx context.Context, hash string, size int64, offset int64) (io.ReadCloser, int64, error) + Put(kind cache.EntryKind, hash string, size int64, r io.Reader) error + Contains(ctx context.Context, kind cache.EntryKind, hash string, size int64) (bool, int64) + FindMissingCasBlobs(ctx context.Context, blobs []*pb.Digest) ([]*pb.Digest, error) + + MaxSize() int64 + Stats() (totalSize int64, reservedSize int64, numItems int, uncompressedSize int64) + RegisterMetrics() +} + // lruItem is the type of the values stored in SizedLRU to keep track of items. type lruItem struct { // Size of the blob in uncompressed form. @@ -67,9 +67,9 @@ type lruItem struct { legacy bool } -// Cache is a filesystem-based LRU cache, with an optional backend proxy. +// diskCache is a filesystem-based LRU cache, with an optional backend proxy. // It is safe for concurrent use. -type Cache struct { +type diskCache struct { dir string proxy cache.Proxy storageMode casblob.CompressionType @@ -108,7 +108,7 @@ func badReqErr(format string, a ...interface{}) *cache.Error { // New returns a new instance of a filesystem-based cache rooted at `dir`, // with a maximum size of `maxSizeBytes` bytes and `opts` Options set. -func New(dir string, maxSizeBytes int64, opts ...Option) (*Cache, error) { +func New(dir string, maxSizeBytes int64, opts ...Option) (Cache, error) { err := os.MkdirAll(dir, os.ModePerm) if err != nil { @@ -120,7 +120,7 @@ func New(dir string, maxSizeBytes int64, opts ...Option) (*Cache, error) { return nil, err } - c := &Cache{ + c := diskCache{ dir: dir, // Not using config here, to avoid test import cycles. @@ -135,9 +135,34 @@ func New(dir string, maxSizeBytes int64, opts ...Option) (*Cache, error) { fileRemovalSem: semaphore.NewWeighted(5000), } + cc := CacheConfig{diskCache: &c} + + // The eviction callback deletes the file from disk. + // This function is only called while the lock is held + // by the current goroutine. + onEvict := func(key Key, value lruItem) { + ks := key.(string) + hash := ks[len(ks)-sha256.Size*2:] + var kind cache.EntryKind = cache.AC + if strings.HasPrefix(ks, "cas") { + kind = cache.CAS + } else if strings.HasPrefix(ks, "ac") { + kind = cache.AC + } else if strings.HasPrefix(ks, "raw") { + kind = cache.RAW + } + + f := filepath.Join(dir, c.FileLocation(kind, value.legacy, hash, value.size, value.random)) + + // Run in a goroutine so we can release the lock sooner. + go c.removeFile(f) + } + + c.lru = NewSizedLRU(maxSizeBytes, onEvict) + // Apply options. for _, o := range opts { - err = o(c) + err = o(&cc) if err != nil { return nil, err } @@ -163,29 +188,6 @@ func New(dir string, maxSizeBytes int64, opts ...Option) (*Cache, error) { } } - // The eviction callback deletes the file from disk. - // This function is only called while the lock is held - // by the current goroutine. - onEvict := func(key Key, value lruItem) { - ks := key.(string) - hash := ks[len(ks)-sha256.Size*2:] - var kind cache.EntryKind = cache.AC - if strings.HasPrefix(ks, "cas") { - kind = cache.CAS - } else if strings.HasPrefix(ks, "ac") { - kind = cache.AC - } else if strings.HasPrefix(ks, "raw") { - kind = cache.RAW - } - - f := filepath.Join(dir, c.FileLocation(kind, value.legacy, hash, value.size, value.random)) - - // Run in a goroutine so we can release the lock sooner. - go c.removeFile(f) - } - - c.lru = NewSizedLRU(maxSizeBytes, onEvict) - err = c.migrateDirectories() if err != nil { return nil, fmt.Errorf("Attempting to migrate the old directory structure failed: %w", err) @@ -195,10 +197,21 @@ func New(dir string, maxSizeBytes int64, opts ...Option) (*Cache, error) { return nil, fmt.Errorf("Loading of existing cache entries failed due to error: %w", err) } - return c, nil + if cc.metrics == nil { + return &c, nil + } + + cc.metrics.diskCache = &c + + return cc.metrics, nil +} + +// Non-test users must call this to expose metrics. +func (c *diskCache) RegisterMetrics() { + c.lru.RegisterMetrics() } -func (c *Cache) removeFile(f string) { +func (c *diskCache) removeFile(f string) { if err := c.fileRemovalSem.Acquire(context.Background(), 1); err != nil { log.Printf("ERROR: failed to aquire semaphore: %v, unable to remove %s", err, f) return @@ -211,7 +224,7 @@ func (c *Cache) removeFile(f string) { } } -func (c *Cache) FileLocationBase(kind cache.EntryKind, legacy bool, hash string, size int64) string { +func (c *diskCache) FileLocationBase(kind cache.EntryKind, legacy bool, hash string, size int64) string { if kind == cache.RAW { return path.Join("raw.v2", hash[:2], hash) } @@ -227,7 +240,7 @@ func (c *Cache) FileLocationBase(kind cache.EntryKind, legacy bool, hash string, return fmt.Sprintf("cas.v2/%s/%s-%d", hash[:2], hash, size) } -func (c *Cache) FileLocation(kind cache.EntryKind, legacy bool, hash string, size int64, random string) string { +func (c *diskCache) FileLocation(kind cache.EntryKind, legacy bool, hash string, size int64, random string) string { if kind == cache.RAW { return path.Join("raw.v2", hash[:2], hash+"-"+random) } @@ -243,7 +256,7 @@ func (c *Cache) FileLocation(kind cache.EntryKind, legacy bool, hash string, siz return fmt.Sprintf("cas.v2/%s/%s-%d-%s", hash[:2], hash, size, random) } -func (c *Cache) migrateDirectories() error { +func (c *diskCache) migrateDirectories() error { err := migrateDirectory(c.dir, cache.AC) if err != nil { return err @@ -415,7 +428,7 @@ func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error // loadExistingFiles lists all files in the cache directory, and adds them to the // LRU index so that they can be served. Files are sorted by access time first, // so that the eviction behavior is preserved across server restarts. -func (c *Cache) loadExistingFiles() error { +func (c *diskCache) loadExistingFiles() error { log.Printf("Loading existing files in %s.\n", c.dir) // compressed CAS items: -- @@ -521,7 +534,7 @@ func (c *Cache) loadExistingFiles() error { // If `hash` is not the empty string, and the contents don't match it, // a non-nil error is returned. All data will be read from `r` before // this function returns. -func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) (rErr error) { +func (c *diskCache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) (rErr error) { defer func() { if r != nil { _, _ = io.Copy(ioutil.Discard, r) @@ -643,7 +656,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) return nil } -func (c *Cache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) (int64, error) { +func (c *diskCache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) (int64, error) { closeFile := true defer func() { if closeFile { @@ -685,7 +698,7 @@ func (c *Cache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string } // This must be called when the lock is not held. -func (c *Cache) commit(key string, legacy bool, tempfile string, reservedSize int64, logicalSize int64, sizeOnDisk int64, random string) (unreserve bool, removeTempfile bool, err error) { +func (c *diskCache) commit(key string, legacy bool, tempfile string, reservedSize int64, logicalSize int64, sizeOnDisk int64, random string) (unreserve bool, removeTempfile bool, err error) { unreserve = reservedSize > 0 removeTempfile = true @@ -726,7 +739,7 @@ func (c *Cache) commit(key string, legacy bool, tempfile string, reservedSize in // but that we can try the proxy backend. // // This function assumes that only CAS blobs are requested in zstd form. -func (c *Cache) availableOrTryProxy(kind cache.EntryKind, hash string, size int64, offset int64, zstd bool) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { +func (c *diskCache) availableOrTryProxy(kind cache.EntryKind, hash string, size int64, offset int64, zstd bool) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { locked := true c.mu.Lock() @@ -836,18 +849,18 @@ var errOnlyCompressedCAS = &cache.Error{ // item is not found, the io.ReadCloser will be nil. If some error occurred // when processing the request, then it is returned. Callers should provide // the `size` of the item to be retrieved, or -1 if unknown. -func (c *Cache) Get(ctx context.Context, kind cache.EntryKind, hash string, size int64, offset int64) (rc io.ReadCloser, s int64, rErr error) { +func (c *diskCache) Get(ctx context.Context, kind cache.EntryKind, hash string, size int64, offset int64) (rc io.ReadCloser, s int64, rErr error) { return c.get(ctx, kind, hash, size, offset, false) } // GetZstd is just like Get, except the data available from rc is zstandard // compressed. Note that the returned `s` value still refers to the amount // of data once it has been decompressed. -func (c *Cache) GetZstd(ctx context.Context, hash string, size int64, offset int64) (rc io.ReadCloser, s int64, rErr error) { +func (c *diskCache) GetZstd(ctx context.Context, hash string, size int64, offset int64) (rc io.ReadCloser, s int64, rErr error) { return c.get(ctx, cache.CAS, hash, size, offset, true) } -func (c *Cache) get(ctx context.Context, kind cache.EntryKind, hash string, size int64, offset int64, zstd bool) (rc io.ReadCloser, s int64, rErr error) { +func (c *diskCache) get(ctx context.Context, kind cache.EntryKind, hash string, size int64, offset int64, zstd bool) (rc io.ReadCloser, s int64, rErr error) { // The hash format is checked properly in the http/grpc code. // Just perform a simple/fast check here, to catch bad tests. if len(hash) != sha256HashStrSize { @@ -855,8 +868,6 @@ func (c *Cache) get(ctx context.Context, kind cache.EntryKind, hash string, size } if kind == cache.CAS && size <= 0 && hash == emptySha256 { - cacheHits.Inc() - if zstd { return ioutil.NopCloser(bytes.NewReader(emptyZstdBlob)), 0, nil } @@ -917,12 +928,9 @@ func (c *Cache) get(ctx context.Context, kind cache.EntryKind, hash string, size unreserve = true } if f != nil { - cacheHits.Inc() return f, foundSize, nil } - cacheMisses.Inc() - if !tryProxy { return nil, -1, nil } @@ -1006,8 +1014,7 @@ func (c *Cache) get(ctx context.Context, kind cache.EntryKind, hash string, size // one) will be checked. // // Callers should provide the `size` of the item, or -1 if unknown. -func (c *Cache) Contains(ctx context.Context, kind cache.EntryKind, hash string, size int64) (bool, int64) { - +func (c *diskCache) Contains(ctx context.Context, kind cache.EntryKind, hash string, size int64) (bool, int64) { // The hash format is checked properly in the http/grpc code. // Just perform a simple/fast check here, to catch bad tests. if len(hash) != sha256HashStrSize { @@ -1043,14 +1050,14 @@ func (c *Cache) Contains(ctx context.Context, kind cache.EntryKind, hash string, } // MaxSize returns the maximum cache size in bytes. -func (c *Cache) MaxSize() int64 { +func (c *diskCache) MaxSize() int64 { // The underlying value is never modified, no need to lock. return c.lru.MaxSize() } // Stats returns the current size of the cache in bytes, and the number of // items stored in the cache. -func (c *Cache) Stats() (totalSize int64, reservedSize int64, numItems int, uncompressedSize int64) { +func (c *diskCache) Stats() (totalSize int64, reservedSize int64, numItems int, uncompressedSize int64) { c.mu.Lock() defer c.mu.Unlock() @@ -1074,8 +1081,7 @@ func ensureDirExists(path string) { // value from the CAS if it and all its dependencies are also available. If // not, nil values are returned. If something unexpected went wrong, return // an error. -func (c *Cache) GetValidatedActionResult(ctx context.Context, hash string) (*pb.ActionResult, []byte, error) { - +func (c *diskCache) GetValidatedActionResult(ctx context.Context, hash string) (*pb.ActionResult, []byte, error) { rc, sizeBytes, err := c.Get(ctx, cache.AC, hash, -1, 0) if rc != nil { defer rc.Close() diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go index 484eb3542..5a8581088 100644 --- a/cache/disk/disk_test.go +++ b/cache/disk/disk_test.go @@ -25,6 +25,9 @@ import ( pb "github.com/buchgr/bazel-remote/genproto/build/bazel/remote/execution/v2" "google.golang.org/protobuf/proto" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" ) func tempDir(t *testing.T) string { @@ -52,10 +55,11 @@ func TestCacheBasics(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(itemSize*2 + BlockSize) - testCache, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) if testCache.lru.Len() != 0 { t.Fatalf("Expected to start with an empty disk cache, found %d items", @@ -171,10 +175,11 @@ func TestCacheGetContainsWrongSizeWithProxy(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, BlockSize, WithProxyBackend(new(proxyStub)), WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, BlockSize, WithProxyBackend(new(proxyStub)), WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) var found bool var rdr io.ReadCloser @@ -280,11 +285,11 @@ func expectContentEquals(rdr io.ReadCloser, sizeBytes int64, expectedContent []b return nil } -func putGetCompare(ctx context.Context, kind cache.EntryKind, hash string, content string, testCache *Cache) error { +func putGetCompare(ctx context.Context, kind cache.EntryKind, hash string, content string, testCache *diskCache) error { return putGetCompareBytes(ctx, kind, hash, []byte(content), testCache) } -func putGetCompareBytes(ctx context.Context, kind cache.EntryKind, hash string, data []byte, testCache *Cache) error { +func putGetCompareBytes(ctx context.Context, kind cache.EntryKind, hash string, data []byte, testCache *diskCache) error { r := bytes.NewReader(data) @@ -315,10 +320,11 @@ func TestOverwrite(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, BlockSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, BlockSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) err = putGetCompare(ctx, cache.CAS, hashStr("hello"), "hello", testCache) if err != nil { @@ -415,10 +421,11 @@ func TestCacheExistingFiles(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. const cacheSize = BlockSize * 5 - testCache, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) evicted := []Key{} origOnEvict := testCache.lru.onEvict @@ -470,10 +477,11 @@ func TestCacheExistingFiles(t *testing.T) { func TestCacheBlobTooLarge(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, BlockSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, BlockSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) for k := range []cache.EntryKind{cache.AC, cache.RAW} { kind := cache.EntryKind(k) @@ -496,10 +504,11 @@ func TestCacheBlobTooLarge(t *testing.T) { func TestCacheCorruptedCASBlob(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, BlockSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, BlockSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) err = testCache.Put(cache.CAS, hashStr("foo"), int64(len(contents)), strings.NewReader(contents)) @@ -569,10 +578,11 @@ func TestMigrateFromOldDirectoryStructure(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. const cacheSize = 2560*2 + BlockSize*2 - testCache, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) _, _, numItems, _ := testCache.Stats() if numItems != 3 { @@ -667,10 +677,11 @@ func TestLoadExistingEntries(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64((blobSize + BlockSize) * numBlobs * 2) - testCache, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) _, _, numItems, _ := testCache.Stats() if int64(numItems) != numBlobs { @@ -713,10 +724,11 @@ func TestDistinctKeyspaces(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64((blobSize+BlockSize)*3) * 2 - testCache, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) blob, casHash := testutils.RandomDataAndHash(1024) @@ -839,10 +851,11 @@ func TestHttpProxyBackend(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(1024*10) * 2 - testCache, err := New(cacheDir, cacheSize, WithProxyBackend(proxy), WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, cacheSize, WithProxyBackend(proxy), WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) blobSize := int64(1024) blob, casHash := testutils.RandomDataAndHash(blobSize) @@ -877,10 +890,12 @@ func TestHttpProxyBackend(t *testing.T) { // Create a new (empty) testCache, without a proxy backend. cacheDir = testutils.TempDir(t) defer os.RemoveAll(cacheDir) - testCache, err = New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) + + testCacheI, err = New(cacheDir, cacheSize, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache = testCacheI.(*diskCache) // Confirm that it does not contain the item we added to the // first testCache and the proxy backend. @@ -942,10 +957,11 @@ func TestGetValidatedActionResult(t *testing.T) { cacheDir := testutils.TempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 1024*32, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, 1024*32, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) // Create a directory tree like so: // /bar/foo.txt @@ -1117,10 +1133,11 @@ func TestGetWithOffset(t *testing.T) { const blobSize = 2048 + 256 - testCache, err := New(cacheDir, blobSize*2, WithAccessLogger(testutils.NewSilentLogger())) + testCacheI, err := New(cacheDir, blobSize*2, WithAccessLogger(testutils.NewSilentLogger())) if err != nil { t.Fatal(err) } + testCache := testCacheI.(*diskCache) data, hash := testutils.RandomDataAndHash(blobSize) @@ -1171,3 +1188,196 @@ func TestGetWithOffset(t *testing.T) { } } } + +func count(counter *prometheus.CounterVec, kind string, status string) float64 { + gets := testutil.ToFloat64(counter.With(prometheus.Labels{"method": getMethod, "kind": kind, "status": status})) + contains := testutil.ToFloat64(counter.With(prometheus.Labels{"method": containsMethod, "kind": kind, "status": status})) + return gets + contains +} + +func TestMetricsUnvalidatedAC(t *testing.T) { + cacheDir := tempDir(t) + defer os.RemoveAll(cacheDir) + + cacheSize := int64(100000) + + testCacheI, err := New(cacheDir, cacheSize, + WithAccessLogger(testutils.NewSilentLogger()), + WithEndpointMetrics()) + if err != nil { + t.Fatal(err) + } + testCache := testCacheI.(*metricsDecorator) + + // Add an AC entry with a missing cas blob. + randomBlob, hash := testutils.RandomDataAndHash(100) + ar := pb.ActionResult{ + StdoutDigest: &pb.Digest{ + Hash: hash, + SizeBytes: int64(len(randomBlob)), + }, + } + arData, err := proto.Marshal(&ar) + if err != nil { + t.Fatal(err) + } + fakeActionHash := "8f279f9d8bc605b4d733d0ba9386de2376004ab628fee6b000144fdc7b30a6a1" + + err = testCache.Put(cache.AC, fakeActionHash, int64(len(arData)), bytes.NewReader(arData)) + if err != nil { + t.Fatal(err) + } + + contains, size := testCache.Contains(context.Background(), cache.AC, fakeActionHash, -1) + if !contains { + t.Fatalf("Expected hash %q to exist in the cache", fakeActionHash) + } + if size != int64(len(arData)) { + t.Fatalf("Expected cached blob to be of size %d, found %d", len(arData), size) + } + + acHits := count(testCache.counter, acKind, hitStatus) + if acHits != 1 { + t.Fatalf("Expected acHit counter to be 1, found %f", acHits) + } + + acMiss := count(testCache.counter, acKind, missStatus) + if acMiss != 0 { + t.Fatalf("Expected acMiss counter to be 0, found %f", acMiss) + } + + casHits := count(testCache.counter, casKind, hitStatus) + if casHits != 0 { + t.Fatalf("Expected casHit counter to be 0, found %f", casHits) + } + + casMisses := count(testCache.counter, casKind, missStatus) + if casMisses != 0 { + t.Fatalf("Expected casMiss counter to be 0, found %f", casMisses) + } + + rawHits := count(testCache.counter, rawKind, hitStatus) + if rawHits != 0 { + t.Fatalf("Expected rawHit counter to be 0, found %f", rawHits) + } + + rawMisses := count(testCache.counter, rawKind, missStatus) + if rawMisses != 0 { + t.Fatalf("Expected rawMiss counter to be 0, found %f", rawMisses) + } + + rc, _, err := testCache.Get(context.Background(), cache.AC, fakeActionHash, -1, 0) + if err != nil { + t.Fatal(err) + } + if rc == nil { + t.Fatalf("Expected %q to be found in the action cache", fakeActionHash) + } + + acHits = count(testCache.counter, acKind, hitStatus) + if acHits != 2 { + t.Fatalf("Expected acHit counter to be 2, found %f", acHits) + } + + acMiss = count(testCache.counter, acKind, missStatus) + if acMiss != 0 { + t.Fatalf("Expected acMiss counter to be 0, found %f", acMiss) + } + + casHits = count(testCache.counter, casKind, hitStatus) + if casHits != 0 { + t.Fatalf("Expected casHit counter to be 0, found %f", casHits) + } + + casMisses = count(testCache.counter, casKind, missStatus) + if casMisses != 0 { + t.Fatalf("Expected casMiss counter to be 0, found %f", casMisses) + } + + rawHits = count(testCache.counter, rawKind, hitStatus) + if rawHits != 0 { + t.Fatalf("Expected rawHit counter to be 0, found %f", rawHits) + } + + rawMisses = count(testCache.counter, rawKind, missStatus) + if rawMisses != 0 { + t.Fatalf("Expected rawMiss counter to be 0, found %f", rawMisses) + } +} + +func TestMetricsValidatedAC(t *testing.T) { + cacheDir := tempDir(t) + defer os.RemoveAll(cacheDir) + + cacheSize := int64(100000) + + testCacheI, err := New(cacheDir, cacheSize, + WithAccessLogger(testutils.NewSilentLogger()), + WithEndpointMetrics()) + if err != nil { + t.Fatal(err) + } + testCache := testCacheI.(*metricsDecorator) + + // Add an AC entry with a missing cas blob. + randomBlob, hash := testutils.RandomDataAndHash(100) + ar := pb.ActionResult{ + StdoutDigest: &pb.Digest{ + Hash: hash, + SizeBytes: int64(len(randomBlob)), + }, + } + arData, err := proto.Marshal(&ar) + if err != nil { + t.Fatal(err) + } + fakeActionHash := "8f279f9d8bc605b4d733d0ba9386de2376004ab628fee6b000144fdc7b30a6a1" + + err = testCache.Put(cache.AC, fakeActionHash, int64(len(arData)), bytes.NewReader(arData)) + if err != nil { + t.Fatal(err) + } + + // Neither Get nor Contains are supposed to be called on AC blobs in this mode. + // GetValidatedActionResult is used instead in this case. + // TODO: should those methods return errors for AC requests in that mode? + + gotAr, _, err := testCache.GetValidatedActionResult(context.Background(), fakeActionHash) + if err != nil { + t.Fatal(err) + } + if gotAr != nil { + t.Fatal("Expected a cache miss, since the referenced CAS blob is missing") + } + + acHits := count(testCache.counter, acKind, hitStatus) + if acHits != 0 { + t.Fatalf("Expected acHit counter to be 0, found %f", acHits) + } + + acMisses := count(testCache.counter, acKind, missStatus) + if acMisses != 1 { + t.Fatalf("Expected acMiss counter to be 1, found %f", acMisses) + } + + casHits := count(testCache.counter, casKind, hitStatus) + if casHits != 0 { + t.Fatalf("Expected casHit counter to be 0, found %f", casHits) + } + + casMisses := count(testCache.counter, casKind, missStatus) + if casMisses != 0 { + // The referenced stdout blob is missing, but we're only supposed to count the AC lookup. + t.Fatalf("Expected casMiss counter to be 1, found %f", casMisses) + } + + rawHits := count(testCache.counter, rawKind, hitStatus) + if rawHits != 0 { + t.Fatalf("Expected rawHit counter to be 0, found %f", rawHits) + } + + rawMisses := count(testCache.counter, rawKind, missStatus) + if rawMisses != 0 { + t.Fatalf("Expected rawMiss counter to be 0, found %f", rawMisses) + } +} diff --git a/cache/disk/findmissing.go b/cache/disk/findmissing.go index 77634ec69..92d2e8f8b 100644 --- a/cache/disk/findmissing.go +++ b/cache/disk/findmissing.go @@ -23,7 +23,7 @@ type proxyCheck struct { // Returns a slice with the blobs that are missing from the cache. // // Note that this modifies the input slice and returns a subset of it. -func (c *Cache) FindMissingCasBlobs(ctx context.Context, blobs []*pb.Digest) ([]*pb.Digest, error) { +func (c *diskCache) FindMissingCasBlobs(ctx context.Context, blobs []*pb.Digest) ([]*pb.Digest, error) { const batchSize = 20 var wg sync.WaitGroup @@ -86,7 +86,7 @@ func filterNonNil(blobs []*pb.Digest) []*pb.Digest { // Set blobs that exist in the disk cache to nil, and return the number // of missing blobs. -func (c *Cache) findMissingLocalCAS(blobs []*pb.Digest) int { +func (c *diskCache) findMissingLocalCAS(blobs []*pb.Digest) int { var exists bool var key string missing := 0 @@ -115,7 +115,7 @@ func (c *Cache) findMissingLocalCAS(blobs []*pb.Digest) int { return missing } -func (c *Cache) containsWorker() { +func (c *diskCache) containsWorker() { var ok bool for req := range c.containsQueue { ok, _ = c.proxy.Contains(req.ctx, cache.CAS, (*req.digest).Hash) @@ -131,7 +131,7 @@ func (c *Cache) containsWorker() { } } -func (c *Cache) spawnContainsQueueWorkers() { +func (c *diskCache) spawnContainsQueueWorkers() { // TODO: make these configurable? const queueSize = 2048 const numWorkers = 512 diff --git a/cache/disk/findmissing_test.go b/cache/disk/findmissing_test.go index 4590ab694..5d1e8515f 100644 --- a/cache/disk/findmissing_test.go +++ b/cache/disk/findmissing_test.go @@ -102,7 +102,7 @@ func TestContainsWorker(t *testing.T) { tp := testCWProxy{blob: "9205adc12a2c8b65e7cd77918ff8e6e20f39bdd0b7fc4b984abfd690c79d80c1"} - c := Cache{ + c := diskCache{ accessLogger: testutils.NewSilentLogger(), proxy: &tp, containsQueue: make(chan proxyCheck, 2), diff --git a/cache/disk/lru.go b/cache/disk/lru.go index ec4f3b0ee..f3de12aa9 100644 --- a/cache/disk/lru.go +++ b/cache/disk/lru.go @@ -6,29 +6,6 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - gaugeCacheSizeBytes = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "bazel_remote_disk_cache_size_bytes", - Help: "The current number of bytes in the disk backend", - }) - - gaugeCacheLogicalBytes = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "bazel_remote_disk_cache_logical_bytes", - Help: "The current number of bytes in the disk backend if they were uncompressed", - }) - - counterEvictedBytes = promauto.NewCounter(prometheus.CounterOpts{ - Name: "bazel_remote_disk_cache_evicted_bytes_total", - Help: "The total number of bytes evicted from disk backend, due to full cache", - }) - - counterOverwrittenBytes = promauto.NewCounter(prometheus.CounterOpts{ - Name: "bazel_remote_disk_cache_overwritten_bytes_total", - Help: "The total number of bytes removed from disk backend, due to put of already existing key", - }) ) // Key is the type used for identifying cache items. For non-test code, @@ -66,6 +43,11 @@ type SizedLRU struct { maxSize int64 onEvict EvictCallback + + gaugeCacheSizeBytes prometheus.Gauge + gaugeCacheLogicalBytes prometheus.Gauge + counterEvictedBytes prometheus.Counter + counterOverwrittenBytes prometheus.Counter } type entry struct { @@ -84,9 +66,33 @@ func NewSizedLRU(maxSize int64, onEvict EvictCallback) SizedLRU { ll: list.New(), cache: make(map[interface{}]*list.Element), onEvict: onEvict, + + gaugeCacheSizeBytes: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "bazel_remote_disk_cache_size_bytes", + Help: "The current number of bytes in the disk backend", + }), + gaugeCacheLogicalBytes: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "bazel_remote_disk_cache_logical_bytes", + Help: "The current number of bytes in the disk backend if they were uncompressed", + }), + counterEvictedBytes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "bazel_remote_disk_cache_evicted_bytes_total", + Help: "The total number of bytes evicted from disk backend, due to full cache", + }), + counterOverwrittenBytes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "bazel_remote_disk_cache_overwritten_bytes_total", + Help: "The total number of bytes removed from disk backend, due to put of already existing key", + }), } } +func (c *SizedLRU) RegisterMetrics() { + prometheus.MustRegister(c.gaugeCacheSizeBytes) + prometheus.MustRegister(c.gaugeCacheLogicalBytes) + prometheus.MustRegister(c.counterEvictedBytes) + prometheus.MustRegister(c.counterOverwrittenBytes) +} + // Add adds a (key, value) to the cache, evicting items as necessary. // Add returns false and does not add the item if the item size is // larger than the maximum size of the cache, or if the item cannot @@ -111,7 +117,7 @@ func (c *SizedLRU) Add(key Key, value lruItem) (ok bool) { } uncompressedSizeDelta = roundUp4k(value.size) - roundUp4k(ee.Value.(*entry).value.size) c.ll.MoveToFront(ee) - counterOverwrittenBytes.Add(float64(ee.Value.(*entry).value.sizeOnDisk)) + c.counterOverwrittenBytes.Add(float64(ee.Value.(*entry).value.sizeOnDisk)) prevValue := ee.Value.(*entry).value if c.onEvict != nil { @@ -141,8 +147,8 @@ func (c *SizedLRU) Add(key Key, value lruItem) (ok bool) { c.currentSize += sizeDelta c.uncompressedSize += uncompressedSizeDelta - gaugeCacheSizeBytes.Set(float64(c.currentSize)) - gaugeCacheLogicalBytes.Set(float64(c.uncompressedSize)) + c.gaugeCacheSizeBytes.Set(float64(c.currentSize)) + c.gaugeCacheLogicalBytes.Set(float64(c.uncompressedSize)) return true } @@ -161,8 +167,8 @@ func (c *SizedLRU) Get(key Key) (value lruItem, ok bool) { func (c *SizedLRU) Remove(key Key) { if ele, hit := c.cache[key]; hit { c.removeElement(ele) - gaugeCacheSizeBytes.Set(float64(c.currentSize)) - gaugeCacheLogicalBytes.Set(float64(c.uncompressedSize)) + c.gaugeCacheSizeBytes.Set(float64(c.currentSize)) + c.gaugeCacheLogicalBytes.Set(float64(c.uncompressedSize)) } } @@ -263,7 +269,7 @@ func (c *SizedLRU) removeElement(e *list.Element) { delete(c.cache, kv.key) c.currentSize -= roundUp4k(kv.value.sizeOnDisk) c.uncompressedSize -= roundUp4k(kv.value.size) - counterEvictedBytes.Add(float64(kv.value.sizeOnDisk)) + c.counterEvictedBytes.Add(float64(kv.value.sizeOnDisk)) if c.onEvict != nil { c.onEvict(kv.key, kv.value) diff --git a/cache/disk/metrics.go b/cache/disk/metrics.go new file mode 100644 index 000000000..c23ea3835 --- /dev/null +++ b/cache/disk/metrics.go @@ -0,0 +1,121 @@ +package disk + +import ( + "context" + "io" + + "github.com/buchgr/bazel-remote/cache" + + pb "github.com/buchgr/bazel-remote/genproto/build/bazel/remote/execution/v2" + + "github.com/prometheus/client_golang/prometheus" +) + +type metricsDecorator struct { + counter *prometheus.CounterVec + *diskCache +} + +const ( + hitStatus = "hit" + missStatus = "miss" + + containsMethod = "contains" + getMethod = "get" + //putMethod = "put" + + acKind = "ac" // This must be lowercase to match cache.EntryKind.String() + casKind = "cas" + rawKind = "raw" +) + +func (m *metricsDecorator) RegisterMetrics() { + prometheus.MustRegister(m.counter) + m.diskCache.RegisterMetrics() +} + +func (m *metricsDecorator) Get(ctx context.Context, kind cache.EntryKind, hash string, size int64, offset int64) (io.ReadCloser, int64, error) { + rc, size, err := m.diskCache.Get(ctx, kind, hash, size, offset) + + lbls := prometheus.Labels{"method": getMethod, "kind": kind.String()} + if rc != nil { + lbls["status"] = hitStatus + } else if err == nil { + lbls["status"] = missStatus + } + m.counter.With(lbls).Inc() + + return rc, size, err +} + +func (m *metricsDecorator) GetValidatedActionResult(ctx context.Context, hash string) (*pb.ActionResult, []byte, error) { + ar, data, err := m.diskCache.GetValidatedActionResult(ctx, hash) + + lbls := prometheus.Labels{"method": getMethod, "kind": acKind} + if ar != nil { + lbls["status"] = hitStatus + } else if err == nil { + lbls["status"] = missStatus + } + m.counter.With(lbls).Inc() + + return ar, data, err +} + +func (m *metricsDecorator) GetZstd(ctx context.Context, hash string, size int64, offset int64) (io.ReadCloser, int64, error) { + rc, size, err := m.diskCache.GetZstd(ctx, hash, size, offset) + + lbls := prometheus.Labels{ + "method": getMethod, + "kind": "cas", + } + if rc != nil { + lbls["status"] = hitStatus + } else if err == nil { + lbls["status"] = missStatus + } + m.counter.With(lbls).Inc() + + return rc, size, err +} + +func (m *metricsDecorator) Contains(ctx context.Context, kind cache.EntryKind, hash string, size int64) (bool, int64) { + ok, size := m.diskCache.Contains(ctx, kind, hash, size) + + lbls := prometheus.Labels{"method": containsMethod, "kind": kind.String()} + if ok { + lbls["status"] = hitStatus + } else { + lbls["status"] = missStatus + } + m.counter.With(lbls).Inc() + + return ok, size +} + +func (m *metricsDecorator) FindMissingCasBlobs(ctx context.Context, blobs []*pb.Digest) ([]*pb.Digest, error) { + numLooking := len(blobs) + digests, err := m.diskCache.FindMissingCasBlobs(ctx, blobs) + numFound := len(digests) + + numMissing := numLooking - numFound + + hitLabels := prometheus.Labels{ + "method": containsMethod, + "kind": "cas", + "status": hitStatus, + } + hits := m.counter.With(hitLabels) + + missLabels := prometheus.Labels{ + "method": containsMethod, + "kind": "cas", + "status": missStatus, + } + misses := m.counter.With(missLabels) + + hits.Add(float64(numFound)) + misses.Add(float64(numMissing)) + + return digests, err +} diff --git a/cache/disk/options.go b/cache/disk/options.go index f8a2ea78f..abaa54d76 100644 --- a/cache/disk/options.go +++ b/cache/disk/options.go @@ -6,17 +6,24 @@ import ( "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/cache/disk/casblob" + + "github.com/prometheus/client_golang/prometheus" ) -type Option func(*Cache) error +type Option func(*CacheConfig) error + +type CacheConfig struct { + diskCache *diskCache // Assumed to be non-nil. + metrics *metricsDecorator // May be nil. +} func WithStorageMode(mode string) Option { - return func(c *Cache) error { + return func(c *CacheConfig) error { if mode == "zstd" { - c.storageMode = casblob.Zstandard + c.diskCache.storageMode = casblob.Zstandard return nil } else if mode == "uncompressed" { - c.storageMode = casblob.Identity + c.diskCache.storageMode = casblob.Identity return nil } else { return fmt.Errorf("Unsupported storage mode: " + mode) @@ -25,25 +32,25 @@ func WithStorageMode(mode string) Option { } func WithMaxBlobSize(size int64) Option { - return func(c *Cache) error { + return func(c *CacheConfig) error { if size <= 0 { return fmt.Errorf("Invalid MaxBlobSize: %d", size) } - c.maxBlobSize = size + c.diskCache.maxBlobSize = size return nil } } func WithProxyBackend(proxy cache.Proxy) Option { - return func(c *Cache) error { - if c.proxy != nil && proxy != nil { + return func(c *CacheConfig) error { + if c.diskCache.proxy != nil && proxy != nil { return fmt.Errorf("Proxy backends may be set only once") } if proxy != nil { - c.proxy = proxy - c.spawnContainsQueueWorkers() + c.diskCache.proxy = proxy + c.diskCache.spawnContainsQueueWorkers() } return nil @@ -51,8 +58,26 @@ func WithProxyBackend(proxy cache.Proxy) Option { } func WithAccessLogger(logger *log.Logger) Option { - return func(c *Cache) error { - c.accessLogger = logger + return func(c *CacheConfig) error { + c.diskCache.accessLogger = logger + return nil + } +} + +func WithEndpointMetrics() Option { + return func(c *CacheConfig) error { + if c.metrics != nil { + return fmt.Errorf("WithEndpointMetrics specified multiple times") + } + + c.metrics = &metricsDecorator{ + counter: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "bazel_remote_incoming_requests_total", + Help: "The number of incoming cache requests", + }, + []string{"method", "kind", "status"}), + } + return nil } } diff --git a/go.mod b/go.mod index 82d1a8947..53223cbd4 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/mostynb/go-grpc-compression v1.1.11 github.com/mostynb/zstdpool-syncpool v0.0.8 github.com/prometheus/client_golang v1.7.1 + github.com/prometheus/client_model v0.2.0 // indirect github.com/slok/go-http-metrics v0.8.0 github.com/urfave/cli/v2 v2.2.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d @@ -38,7 +39,6 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.10.0 // indirect github.com/prometheus/procfs v0.1.3 // indirect github.com/rs/xid v1.2.1 // indirect diff --git a/main.go b/main.go index 95c0269c7..9d2345245 100644 --- a/main.go +++ b/main.go @@ -81,17 +81,25 @@ func run(ctx *cli.Context) error { rlimit.Raise() + validateAC := !c.DisableHTTPACValidation + opts := []disk.Option{ disk.WithStorageMode(c.StorageMode), disk.WithMaxBlobSize(c.MaxBlobSize), - disk.WithProxyBackend(c.ProxyBackend), disk.WithAccessLogger(c.AccessLogger), } + if c.ProxyBackend != nil { + opts = append(opts, disk.WithProxyBackend(c.ProxyBackend)) + } + if c.EnableEndpointMetrics { + opts = append(opts, disk.WithEndpointMetrics()) + } diskCache, err := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024, opts...) if err != nil { log.Fatal(err) } + diskCache.RegisterMetrics() mux := http.NewServeMux() httpServer := &http.Server{ @@ -102,7 +110,6 @@ func run(ctx *cli.Context) error { WriteTimeout: c.HTTPWriteTimeout, } - validateAC := !c.DisableHTTPACValidation checkClientCertForWrites := c.TLSCaFile != "" && c.AllowUnauthenticatedReads h := server.NewHTTPCache(diskCache, c.AccessLogger, c.ErrorLogger, validateAC, c.EnableACKeyInstanceMangling, checkClientCertForWrites, gitCommit) diff --git a/server/grpc.go b/server/grpc.go index d34be33e5..1e20bc766 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -37,7 +37,7 @@ var ( ) type grpcServer struct { - cache *disk.Cache + cache disk.Cache accessLogger cache.Logger errorLogger cache.Logger depsCheck bool @@ -53,7 +53,7 @@ func ListenAndServeGRPC(addr string, opts []grpc.ServerOption, mangleACKeys bool, enableRemoteAssetAPI bool, checkClientCertForWrites bool, - c *disk.Cache, a cache.Logger, e cache.Logger) error { + c disk.Cache, a cache.Logger, e cache.Logger) error { listener, err := net.Listen("tcp", addr) if err != nil { @@ -68,7 +68,7 @@ func serveGRPC(l net.Listener, opts []grpc.ServerOption, mangleACKeys bool, enableRemoteAssetAPI bool, checkClientCertForWrites bool, - c *disk.Cache, a cache.Logger, e cache.Logger) error { + c disk.Cache, a cache.Logger, e cache.Logger) error { srv := grpc.NewServer(opts...) s := &grpcServer{ diff --git a/server/grpc_test.go b/server/grpc_test.go index 366f777f5..1e67ec1bd 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -48,7 +48,7 @@ var ( bsClient bytestream.ByteStreamClient assetClient asset.FetchClient ctx = context.Background() - diskCache *disk.Cache + diskCache disk.Cache badDigestTestCases = []badDigest{ {digest: &pb.Digest{Hash: ""}, reason: "empty hash"}, diff --git a/server/http.go b/server/http.go index 6c7537100..1ebca4d13 100644 --- a/server/http.go +++ b/server/http.go @@ -36,7 +36,7 @@ type HTTPCache interface { } type httpCache struct { - cache *disk.Cache + cache disk.Cache accessLogger cache.Logger errorLogger cache.Logger validateAC bool @@ -60,7 +60,7 @@ type statusPageData struct { // accessLogger will print one line for each HTTP request to stdout. // errorLogger will print unexpected server errors. Inexistent files and malformed URLs will not // be reported. -func NewHTTPCache(cache *disk.Cache, accessLogger cache.Logger, errorLogger cache.Logger, validateAC bool, mangleACKeys bool, checkClientCertForWrites bool, commit string) HTTPCache { +func NewHTTPCache(cache disk.Cache, accessLogger cache.Logger, errorLogger cache.Logger, validateAC bool, mangleACKeys bool, checkClientCertForWrites bool, commit string) HTTPCache { _, _, numItems, _ := cache.Stats()