From d1a0c97218690bfff4c85923957f3fd7fb74174f Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 24 Sep 2026 09:31:49 -0500 Subject: [PATCH 1/4] Utility for detecting leaked resources --- sei-db/common/utils/must_close.go | 54 +++++++++++++++++++ .../litt/disktable/forward_iterator.go | 17 ++++-- .../litt/disktable/keymap/pebble_db_keymap.go | 6 +++ .../litt/disktable/reverse_iterator.go | 17 ++++-- sei-db/db_engine/litt/util/file_lock.go | 12 ++++- sei-db/db_engine/pebbledb/batch.go | 12 ++++- sei-db/db_engine/pebbledb/db.go | 11 +++- sei-db/db_engine/pebbledb/iterator.go | 8 +++ sei-db/db_engine/pebbledb/mvcc/iterator.go | 10 +++- .../pebbledb/mvcc/iterator_ascending.go | 11 +++- sei-db/db_engine/rocksdb/mvcc/iterator.go | 14 ++++- sei-db/db_engine/view/view_impl.go | 8 +++ sei-db/db_engine/view/view_manager_impl.go | 38 ++++++++++++- sei-db/state_db/giga/state_db.go | 9 ++++ sei-db/state_db/sc/flatkv/exporter.go | 9 +++- sei-db/state_db/sc/flatkv/state_view.go | 8 +++ sei-db/state_db/sc/flatkv/store_read.go | 5 +- .../sc/flatkv/sview/atomic_store_view.go | 12 ++++- .../state_db/sc/hashvault/pebble_hashvault.go | 11 ++++ sei-db/state_db/ss/cosmos/store.go | 11 +++- sei-db/state_db/ss/evm/store.go | 10 ++++ sei-db/state_db/ss/snapshot/manager.go | 13 ++++- sei-db/state_db/statewal/state_wal_impl.go | 6 +++ 23 files changed, 290 insertions(+), 22 deletions(-) create mode 100644 sei-db/common/utils/must_close.go diff --git a/sei-db/common/utils/must_close.go b/sei-db/common/utils/must_close.go new file mode 100644 index 0000000000..c96fe9de4c --- /dev/null +++ b/sei-db/common/utils/must_close.go @@ -0,0 +1,54 @@ +package utils + +import ( + "fmt" + "runtime" + "testing" + + "github.com/sei-protocol/seilog" +) + +var logger = seilog.NewLogger("db", "common", "utils") + +// MustClose closes obj if it becomes unreachable while isClosed() still reports it open, logging an error when it +// does. It is a safety net, not a way to close things: any close it performs is a bug in the code that leaked obj. +// In a test binary it panics instead, so a leak fails the test run. +// +// obj must point to the start of an allocation that has no finalizer. isClosed() and close() receive obj as their +// argument and must not capture it, or obj never becomes unreachable; pass method expressions such as +// (*Store).Close. An object reachable from its own goroutines or from a reference cycle is never closed. +func MustClose[T any]( + obj *T, + description string, + isClosed func(*T) bool, + close func(*T), +) { + MustCloseE(obj, description, isClosed, func(o *T) error { + close(o) + return nil + }) +} + +// MustCloseE is MustClose() for a close() that returns an error. A close error is logged. +func MustCloseE[T any]( + obj *T, + description string, + isClosed func(*T) bool, + close func(*T) error, +) { + runtime.SetFinalizer(obj, func(o *T) { + if isClosed(o) { + return + } + if testing.Testing() { + panic(fmt.Sprintf("%s became unreachable without being closed", description)) + } + logger.Error("object became unreachable without being closed, closing it now", "object", description) + // Finalizers share one goroutine, and a close may block. + go func() { + if err := close(o); err != nil { + logger.Error("failed to close an unreachable object", "object", description, "err", err) + } + }() + }) +} diff --git a/sei-db/db_engine/litt/disktable/forward_iterator.go b/sei-db/db_engine/litt/disktable/forward_iterator.go index 43872884ed..f922119b4c 100644 --- a/sei-db/db_engine/litt/disktable/forward_iterator.go +++ b/sei-db/db_engine/litt/disktable/forward_iterator.go @@ -3,6 +3,7 @@ package disktable import ( "fmt" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/segment" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/types" @@ -69,11 +70,13 @@ type forwardIterator struct { // newForwardIterator creates a forward iterator over the given snapshot of sealed segments, owned by a // live table. func newForwardIterator(table *DiskTable, segs []*segment.Segment) *forwardIterator { - return &forwardIterator{ + it := &forwardIterator{ onClose: closeLiveIterator(table, segs), segs: segs, segPos: 0, } + utils.MustCloseE(it, "littdb forward iterator", (*forwardIterator).isClosed, (*forwardIterator).Close) + return it } // newForwardIteratorAt creates a forward iterator over the given snapshot positioned so that the first @@ -87,20 +90,22 @@ func newForwardIteratorAt( keys []*types.ScopedKey, keyPos int, ) *forwardIterator { - return &forwardIterator{ + it := &forwardIterator{ onClose: closeLiveIterator(table, segs), segs: segs, segPos: segPos, keys: keys, keyPos: keyPos, } + utils.MustCloseE(it, "littdb forward iterator", (*forwardIterator).isClosed, (*forwardIterator).Close) + return it } // NewOfflineForwardIterator creates a forward iterator over the given snapshot of segments, gathered // directly from disk rather than from a live table. release is called once, by Close, in place of the // live path's segment-reservation release and control-loop notification. func NewOfflineForwardIterator(segs []*segment.Segment, release func()) litt.Iterator { - return &forwardIterator{ + it := &forwardIterator{ onClose: func() error { release() return nil @@ -108,6 +113,8 @@ func NewOfflineForwardIterator(segs []*segment.Segment, release func()) litt.Ite segs: segs, segPos: 0, } + utils.MustCloseE(it, "littdb forward iterator", (*forwardIterator).isClosed, (*forwardIterator).Close) + return it } // Next advances the iterator to the next key in insertion order. @@ -302,3 +309,7 @@ func closeLiveIterator(table *DiskTable, segs []*segment.Segment) func() error { return nil } } + +func (it *forwardIterator) isClosed() bool { + return it.closed +} diff --git a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go index b9a5876732..072f5b85a7 100644 --- a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go +++ b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go @@ -11,6 +11,7 @@ import ( "github.com/cockroachdb/pebble/v2" "github.com/cockroachdb/pebble/v2/bloom" "github.com/sei-protocol/sei-chain/sei-db/common/unit" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/types" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" ) @@ -85,6 +86,7 @@ func newPebbleDBKeymap( syncWrites: syncWrites, } kmap.alive.Store(true) + utils.MustCloseE(kmap, "littdb pebble keymap", (*PebbleDBKeymap).isStopped, (*PebbleDBKeymap).Stop) return kmap, requiresReload, nil } @@ -208,6 +210,10 @@ func (p *PebbleDBKeymap) Stop() error { return nil } +func (p *PebbleDBKeymap) isStopped() bool { + return !p.alive.Load() +} + func (p *PebbleDBKeymap) Destroy() error { err := p.Stop() if err != nil { diff --git a/sei-db/db_engine/litt/disktable/reverse_iterator.go b/sei-db/db_engine/litt/disktable/reverse_iterator.go index 3577685ba3..e5be94bc17 100644 --- a/sei-db/db_engine/litt/disktable/reverse_iterator.go +++ b/sei-db/db_engine/litt/disktable/reverse_iterator.go @@ -3,6 +3,7 @@ package disktable import ( "fmt" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/segment" "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/types" @@ -49,11 +50,13 @@ type reverseIterator struct { // newReverseIterator creates a reverse iterator over the given snapshot of sealed segments, owned by a // live table. func newReverseIterator(table *DiskTable, segs []*segment.Segment) *reverseIterator { - return &reverseIterator{ + it := &reverseIterator{ onClose: closeLiveIterator(table, segs), segs: segs, segPos: len(segs) - 1, } + utils.MustCloseE(it, "littdb reverse iterator", (*reverseIterator).isClosed, (*reverseIterator).Close) + return it } // newReverseIteratorAt creates a reverse iterator over the given snapshot positioned so that the first @@ -67,20 +70,22 @@ func newReverseIteratorAt( keys []*types.ScopedKey, keyPos int, ) *reverseIterator { - return &reverseIterator{ + it := &reverseIterator{ onClose: closeLiveIterator(table, segs), segs: segs, segPos: segPos, keys: keys, keyPos: keyPos, } + utils.MustCloseE(it, "littdb reverse iterator", (*reverseIterator).isClosed, (*reverseIterator).Close) + return it } // NewOfflineReverseIterator creates a reverse iterator over the given snapshot of segments, gathered // directly from disk rather than from a live table. release is called once, by Close, in place of the // live path's segment-reservation release and control-loop notification. func NewOfflineReverseIterator(segs []*segment.Segment, release func()) litt.Iterator { - return &reverseIterator{ + it := &reverseIterator{ onClose: func() error { release() return nil @@ -88,6 +93,8 @@ func NewOfflineReverseIterator(segs []*segment.Segment, release func()) litt.Ite segs: segs, segPos: len(segs) - 1, } + utils.MustCloseE(it, "littdb reverse iterator", (*reverseIterator).isClosed, (*reverseIterator).Close) + return it } // Next advances the iterator to the next key in reverse insertion order. @@ -170,3 +177,7 @@ func (it *reverseIterator) Close() error { it.segs = nil return closeErr } + +func (it *reverseIterator) isClosed() bool { + return it.closed +} diff --git a/sei-db/db_engine/litt/util/file_lock.go b/sei-db/db_engine/litt/util/file_lock.go index 7636aa1b01..79b50800d4 100644 --- a/sei-db/db_engine/litt/util/file_lock.go +++ b/sei-db/db_engine/litt/util/file_lock.go @@ -10,6 +10,8 @@ import ( "strings" "syscall" "time" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" ) // FileLock represents a file-based lock @@ -159,11 +161,13 @@ func NewFileLock(logger *slog.Logger, path string, fsync bool) (*FileLock, error } } - return &FileLock{ + lock := &FileLock{ logger: logger, path: path, file: file, - }, nil + } + utils.MustClose(lock, "littdb file lock", (*FileLock).isReleased, (*FileLock).Release) + return lock, nil } // Release releases the file lock by closing and removing the lock file. @@ -190,6 +194,10 @@ func (fl *FileLock) Release() { } } +func (fl *FileLock) isReleased() bool { + return fl.file == nil +} + // Path returns the path of the lock file func (fl *FileLock) Path() string { return fl.path diff --git a/sei-db/db_engine/pebbledb/batch.go b/sei-db/db_engine/pebbledb/batch.go index c96294841e..0e2da6acac 100644 --- a/sei-db/db_engine/pebbledb/batch.go +++ b/sei-db/db_engine/pebbledb/batch.go @@ -4,6 +4,8 @@ import ( "fmt" "github.com/cockroachdb/pebble/v2" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" ) @@ -14,6 +16,7 @@ type pebbleBatch struct { b *pebble.Batch operationMetrics *OperationMetrics commitMetrics *CommitMetrics + closed bool } var _ types.Batch = (*pebbleBatch)(nil) @@ -22,11 +25,13 @@ var _ types.Batch = (*pebbleBatch)(nil) // Asking pebble for a sized batch instead replaces that buffer with a fresh allocation on every call, // which is why nothing here does. func (p *pebbleDB) NewBatch() types.Batch { - return &pebbleBatch{ + pb := &pebbleBatch{ b: p.db.NewBatch(), operationMetrics: p.operationMetrics, commitMetrics: p.commitMetrics, } + utils.MustCloseE(pb, "pebbledb batch", (*pebbleBatch).isClosed, (*pebbleBatch).Close) + return pb } func (pb *pebbleBatch) Set(key, value []byte) error { @@ -75,5 +80,10 @@ func (pb *pebbleBatch) Reset() { } func (pb *pebbleBatch) Close() error { + pb.closed = true return pb.b.Close() } + +func (pb *pebbleBatch) isClosed() bool { + return pb.closed +} diff --git a/sei-db/db_engine/pebbledb/db.go b/sei-db/db_engine/pebbledb/db.go index 7179d8c005..8347d74a15 100644 --- a/sei-db/db_engine/pebbledb/db.go +++ b/sei-db/db_engine/pebbledb/db.go @@ -14,6 +14,7 @@ import ( dbm "github.com/tendermint/tm-db" errorutils "github.com/sei-protocol/sei-chain/sei-db/common/errors" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" ) @@ -99,12 +100,14 @@ func Open( metricsCancel = NewPebbleMetrics(db, filepath.Base(config.DataDir), config.MetricsScrapeInterval) } - return &pebbleDB{ + p := &pebbleDB{ db: db, metricsCancel: metricsCancel, operationMetrics: NewOperationMetrics(config.EnableReadWriteMetrics, filepath.Base(config.DataDir)), commitMetrics: NewCommitMetrics(config.EnableMetrics, filepath.Base(config.DataDir)), - }, nil + } + utils.MustCloseE(p, "pebbledb", (*pebbleDB).isClosed, (*pebbleDB).Close) + return p, nil } func (p *pebbleDB) Get(key []byte) ([]byte, error) { @@ -219,6 +222,10 @@ func (p *pebbleDB) Close() error { return db.Close() } +func (p *pebbleDB) isClosed() bool { + return p.db == nil +} + func toPebbleWriteOpts(opts types.WriteOptions) *pebble.WriteOptions { if opts.Sync { return pebble.Sync diff --git a/sei-db/db_engine/pebbledb/iterator.go b/sei-db/db_engine/pebbledb/iterator.go index 2d6e55f252..e14de9f88d 100644 --- a/sei-db/db_engine/pebbledb/iterator.go +++ b/sei-db/db_engine/pebbledb/iterator.go @@ -4,6 +4,7 @@ import ( "github.com/cockroachdb/pebble/v2" dbm "github.com/tendermint/tm-db" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" ) @@ -18,6 +19,7 @@ type pebbleIterator struct { reverse bool operationMetrics *OperationMetrics readCount int64 + closed bool } func newPebbleIterator(it *pebble.Iterator, opts *types.IterOptions, operationMetrics *OperationMetrics) *pebbleIterator { @@ -35,6 +37,7 @@ func newPebbleIterator(it *pebble.Iterator, opts *types.IterOptions, operationMe if pi.it.Valid() { pi.readCount++ } + utils.MustCloseE(pi, "pebbledb iterator", (*pebbleIterator).isClosed, (*pebbleIterator).Close) return pi } @@ -73,8 +76,13 @@ func (pi *pebbleIterator) Error() error { } func (pi *pebbleIterator) Close() error { + pi.closed = true if pi.operationMetrics != nil { pi.operationMetrics.AddRead(pi.readCount) } return pi.it.Close() } + +func (pi *pebbleIterator) isClosed() bool { + return pi.closed +} diff --git a/sei-db/db_engine/pebbledb/mvcc/iterator.go b/sei-db/db_engine/pebbledb/mvcc/iterator.go index 37aae1c0c4..96ce7ceb6c 100644 --- a/sei-db/db_engine/pebbledb/mvcc/iterator.go +++ b/sei-db/db_engine/pebbledb/mvcc/iterator.go @@ -14,6 +14,7 @@ import ( dbm "github.com/tendermint/tm-db" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" pebbledbmetrics "github.com/sei-protocol/sei-chain/sei-db/db_engine/pebbledb" ) @@ -75,7 +76,7 @@ func newPebbleDBIterator( ) *iterator { // Return invalid iterator if requested iterator height is lower than earliest version after pruning if version < earliestVersion { - return &iterator{ + itr := &iterator{ source: src, prefix: prefix, start: mvccStart, @@ -89,6 +90,8 @@ func newPebbleDBIterator( dbName: dbName, ctx: ctx, } + utils.MustCloseE(itr, "mvcc iterator", (*iterator).isClosed, (*iterator).Close) + return itr } // move the underlying PebbleDB iterator to the first key @@ -113,6 +116,7 @@ func newPebbleDBIterator( dbName: dbName, ctx: ctx, } + utils.MustCloseE(itr, "mvcc iterator", (*iterator).isClosed, (*iterator).Close) if valid { currKey, _, ok := SplitMVCCKey(itr.source.Key()) @@ -499,3 +503,7 @@ func (itr *iterator) DebugRawIterate() { } } } + +func (itr *iterator) isClosed() bool { + return itr.source == nil +} diff --git a/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go b/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go index ebc31a98a1..48eba23830 100644 --- a/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go +++ b/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go @@ -13,6 +13,7 @@ import ( dbm "github.com/tendermint/tm-db" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" pebbledbmetrics "github.com/sei-protocol/sei-chain/sei-db/db_engine/pebbledb" ) @@ -60,7 +61,7 @@ func newAscendingIterator( ) *ascendingIterator { // Return invalid iterator if requested iterator height is lower than earliest version after pruning if version < earliestVersion { - return &ascendingIterator{ + itr := &ascendingIterator{ source: src, prefix: prefix, start: mvccStart, @@ -73,6 +74,9 @@ func newAscendingIterator( dbName: dbName, ctx: ctx, } + utils.MustCloseE(itr, "mvcc ascending iterator", + (*ascendingIterator).isClosed, (*ascendingIterator).Close) + return itr } // move the underlying PebbleDB iterator to the first key @@ -96,6 +100,7 @@ func newAscendingIterator( dbName: dbName, ctx: ctx, } + utils.MustCloseE(itr, "mvcc ascending iterator", (*ascendingIterator).isClosed, (*ascendingIterator).Close) if valid { currKey, _, ok := SplitMVCCKey(itr.source.Key()) @@ -439,3 +444,7 @@ func (itr *ascendingIterator) cursorTombstoned() bool { return true } + +func (itr *ascendingIterator) isClosed() bool { + return itr.source == nil +} diff --git a/sei-db/db_engine/rocksdb/mvcc/iterator.go b/sei-db/db_engine/rocksdb/mvcc/iterator.go index 47e52d2338..e439f7a48c 100644 --- a/sei-db/db_engine/rocksdb/mvcc/iterator.go +++ b/sei-db/db_engine/rocksdb/mvcc/iterator.go @@ -9,6 +9,8 @@ import ( "github.com/linxGnu/grocksdb" dbm "github.com/tendermint/tm-db" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" ) var _ dbm.Iterator = (*iterator)(nil) @@ -26,7 +28,7 @@ type iterator struct { func NewRocksDBIterator(source *grocksdb.Iterator, readOpts *grocksdb.ReadOptions, prefix, start, end []byte, version int64, earliestVersion int64, reverse bool) *iterator { // Return invalid iterator if requested iterator height is lower than earliest version after pruning if version < earliestVersion { - return &iterator{ + itr := &iterator{ source: source, readOpts: readOpts, prefix: prefix, @@ -36,6 +38,8 @@ func NewRocksDBIterator(source *grocksdb.Iterator, readOpts *grocksdb.ReadOption reverse: reverse, invalid: true, } + utils.MustCloseE(itr, "rocksdb mvcc iterator", (*iterator).isClosed, (*iterator).Close) + return itr } if reverse { @@ -61,7 +65,7 @@ func NewRocksDBIterator(source *grocksdb.Iterator, readOpts *grocksdb.ReadOption } } - return &iterator{ + itr := &iterator{ source: source, readOpts: readOpts, prefix: prefix, @@ -71,6 +75,8 @@ func NewRocksDBIterator(source *grocksdb.Iterator, readOpts *grocksdb.ReadOption reverse: reverse, invalid: !source.Valid(), } + utils.MustCloseE(itr, "rocksdb mvcc iterator", (*iterator).isClosed, (*iterator).Close) + return itr } // Domain returns the domain of the iterator. The caller must not modify the @@ -182,3 +188,7 @@ func (itr *iterator) assertIsValid() { panic("iterator is invalid") } } + +func (itr *iterator) isClosed() bool { + return itr.source == nil +} diff --git a/sei-db/db_engine/view/view_impl.go b/sei-db/db_engine/view/view_impl.go index 6824cf9d1c..5e093a9c11 100644 --- a/sei-db/db_engine/view/view_impl.go +++ b/sei-db/db_engine/view/view_impl.go @@ -59,6 +59,14 @@ func (s *viewImpl) Release() error { return nil } +func (s *viewImpl) isReleased() bool { + return s.parentManager.isVersionReleased(s.version) +} + +func (s *viewImpl) releaseAll() error { + return s.parentManager.releaseAllReservations(s.version) +} + func (s *viewImpl) Finalize(writes []*proto.KVPair) error { return s.parentManager.FinalizeView(s.version, writes) } diff --git a/sei-db/db_engine/view/view_manager_impl.go b/sei-db/db_engine/view/view_manager_impl.go index 6d24543fc3..29cc861c7b 100644 --- a/sei-db/db_engine/view/view_manager_impl.go +++ b/sei-db/db_engine/view/view_manager_impl.go @@ -11,6 +11,7 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/sei-protocol/sei-chain/sei-db/common/threading" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" ) @@ -563,6 +564,7 @@ func (c *viewManager) commitLocked() (_ View, sealedVersion uint64, _ error) { } } + utils.MustCloseE(view, "view", (*viewImpl).isReleased, (*viewImpl).releaseAll) return view, sealedVersion, nil } @@ -673,6 +675,32 @@ func (c *viewManager) DecrementReferenceCount(version uint64) error { return nil } +// isVersionReleased reports whether every reservation on version has been released. Every version of a +// closed or bricked manager counts as released. +func (c *viewManager) isVersionReleased(version uint64) bool { + c.versionLock.Lock() + defer c.versionLock.Unlock() + + if c.ctx.Err() != nil { + // A shut-down manager holds nothing on a reservation's behalf. + return true + } + // A retired version is no longer in the map. + counter, ok := c.versionMap[version] + return !ok || counter.referenceCount == 0 +} + +// releaseAllReservations releases every reservation still held on version. Releasing the last one +// is subject to the same finalization check as DecrementReferenceCount(). +func (c *viewManager) releaseAllReservations(version uint64) error { + for !c.isVersionReleased(version) { + if err := c.DecrementReferenceCount(version); err != nil { + return fmt.Errorf("failed to release version (%d): %w", version, err) + } + } + return nil +} + // Scans for new flush eligible versions. Returns true if there is a new flush eligible version discovered. // // The Locked postfix indicates that the caller must hold the versionLock. @@ -830,7 +858,9 @@ func (c *viewManager) Iterator(opts *types.IterOptions) (dbm.Iterator, error) { for _, s := range c.shards { s.IteratorOpened() } - return &trackedIterator{Iterator: iter, manager: c}, nil + tracked := &trackedIterator{Iterator: iter, manager: c} + utils.MustCloseE(tracked, "view manager iterator", (*trackedIterator).isClosed, (*trackedIterator).Close) + return tracked, nil } // trackedIterator deregisters itself from every shard when closed, so Close can report the iterators @@ -839,11 +869,13 @@ type trackedIterator struct { dbm.Iterator manager *viewManager closeOnce sync.Once + closed bool } func (w *trackedIterator) Close() error { var err error w.closeOnce.Do(func() { + w.closed = true errs := make([]error, 0, len(w.manager.shards)+1) for _, s := range w.manager.shards { errs = append(errs, s.IteratorClosed()) @@ -854,6 +886,10 @@ func (w *trackedIterator) Close() error { return err } +func (w *trackedIterator) isClosed() bool { + return w.closed +} + // MaterializeCurrentOverrides gathers the in-memory overrides at the current version from every // shard and returns them sorted ascending by key. Each shard is responsible for its own locking; // here we just stitch the results together, and the sort runs without any shard lock held. diff --git a/sei-db/state_db/giga/state_db.go b/sei-db/state_db/giga/state_db.go index e1eb804a1e..381cba7a67 100644 --- a/sei-db/state_db/giga/state_db.go +++ b/sei-db/state_db/giga/state_db.go @@ -52,6 +52,9 @@ type StateDB struct { // Splits a commit into the three stores it writes. Driven only by CommitStateChanges, which // callers serialize. commitPhases *metrics.PhaseTimer + + // Set by Close(). + closed bool } // commitPhaseTimerName prefixes the instruments the commit phase breakdown is published on. @@ -105,6 +108,7 @@ func NewStateDB( if err := s.catchUpToWAL(ctx); err != nil { return nil, err } + utils.MustCloseE(s, "giga state DB", (*StateDB).isClosed, (*StateDB).Close) return s, nil } @@ -282,6 +286,7 @@ func (s *StateDB) truncateWAL(target int64) error { // How long each of the three took is logged, since each drains its own write queue and waits on the // compactions behind it, and those dominate the time a shutdown takes. func (s *StateDB) Close() error { + s.closed = true var errs error var timer utils.CloseTimer if s.ss != nil { @@ -303,6 +308,10 @@ func (s *StateDB) Close() error { return errs } +func (s *StateDB) isClosed() bool { + return s.closed +} + // SC returns the state commit store. func (s *StateDB) SC() *flatkv.CommitStore { return s.sc } diff --git a/sei-db/state_db/sc/flatkv/exporter.go b/sei-db/state_db/sc/flatkv/exporter.go index 0584df3125..369ea8c79e 100644 --- a/sei-db/state_db/sc/flatkv/exporter.go +++ b/sei-db/state_db/sc/flatkv/exporter.go @@ -8,6 +8,7 @@ import ( errorutils "github.com/sei-protocol/sei-chain/sei-db/common/errors" "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" ) @@ -38,10 +39,12 @@ type KVExporter struct { } func NewKVExporter(store *CommitStore, version int64) *KVExporter { - return &KVExporter{ + e := &KVExporter{ store: store, version: version, } + utils.MustCloseE(e, "flatkv exporter", (*KVExporter).isClosed, (*KVExporter).Close) + return e } func (e *KVExporter) Next() (interface{}, error) { @@ -97,3 +100,7 @@ func (e *KVExporter) Close() error { } return nil } + +func (e *KVExporter) isClosed() bool { + return e.store == nil +} diff --git a/sei-db/state_db/sc/flatkv/state_view.go b/sei-db/state_db/sc/flatkv/state_view.go index 1ada67d887..b91afdef25 100644 --- a/sei-db/state_db/sc/flatkv/state_view.go +++ b/sei-db/state_db/sc/flatkv/state_view.go @@ -21,6 +21,9 @@ type flatKVStateView struct { // Guards the release, so a second Close does not release a reservation this view no longer owns. closeOnce sync.Once + + // Set by Close. + closed bool } // GetBlockHeight returns the block height of this view. @@ -32,12 +35,17 @@ func (v *flatKVStateView) GetBlockHeight() int64 { // Idempotent. func (v *flatKVStateView) Close() { v.closeOnce.Do(func() { + v.closed = true if err := v.blockView.Release(); err != nil { panic(fmt.Sprintf("flatkv: close state view at height %d: %v", v.blockView.BlockHeight(), err)) } }) } +func (v *flatKVStateView) isClosed() bool { + return v.closed +} + func (v *flatKVStateView) Get(module string, key []byte) ([]byte, bool) { if module != keys.EVMStoreKey { return v.miscValue(module, key) diff --git a/sei-db/state_db/sc/flatkv/store_read.go b/sei-db/state_db/sc/flatkv/store_read.go index 1f270e22f7..a4fc83904e 100644 --- a/sei-db/state_db/sc/flatkv/store_read.go +++ b/sei-db/state_db/sc/flatkv/store_read.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" @@ -18,7 +19,9 @@ func (s *CommitStore) OpenView() gigatypes.StateView { if err != nil { panic(fmt.Sprintf("flatkv: OpenView: %v", err)) } - return &flatKVStateView{blockView: blockView} + v := &flatKVStateView{blockView: blockView} + utils.MustClose(v, "flatkv state view", (*flatKVStateView).isClosed, (*flatKVStateView).Close) + return v } // Get returns the value for the given key within the specified module. diff --git a/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go b/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go index d9be4d54bb..c9ae417a1c 100644 --- a/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go +++ b/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go @@ -3,6 +3,8 @@ package sview import ( "fmt" "sync" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" ) // AtomicStoreView holds one StoreView and hands it out to readers on any thread. It owns exactly one @@ -29,7 +31,9 @@ func NewAtomicStoreView(initialView *StoreView) (*AtomicStoreView, error) { if err := initialView.Reserve(); err != nil { return nil, fmt.Errorf("reserve initial view: %w", err) } - return &AtomicStoreView{currentView: initialView}, nil + asv := &AtomicStoreView{currentView: initialView} + utils.MustCloseE(asv, "atomic store view", (*AtomicStoreView).isClosed, (*AtomicStoreView).Close) + return asv, nil } // Get() returns the installed view with a reservation the caller owns and must release exactly once. @@ -101,3 +105,9 @@ func (asv *AtomicStoreView) Close() error { } return nil } + +func (asv *AtomicStoreView) isClosed() bool { + asv.mu.RLock() + defer asv.mu.RUnlock() + return asv.currentView == nil +} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault.go b/sei-db/state_db/sc/hashvault/pebble_hashvault.go index 0c1a8d7722..96063caa4a 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault.go @@ -12,6 +12,8 @@ import ( "github.com/cockroachdb/pebble/v2" "github.com/ethereum/go-ethereum/common/lru" "github.com/sei-protocol/seilog" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" ) var _ HashVault = (*PebbleHashVault)(nil) @@ -92,6 +94,9 @@ func newPebbleHashVault(_ context.Context, config HashVaultConfig) (*PebbleHashV "dataDir", config.DataDir) } + utils.MustCloseE(p, "hashvault", (*PebbleHashVault).isClosed, func(p *PebbleHashVault) error { + return p.Close(context.Background()) + }) return p, nil } @@ -238,6 +243,12 @@ func (p *PebbleHashVault) Close(_ context.Context) error { return nil } +func (p *PebbleHashVault) isClosed() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.closed +} + func (p *PebbleHashVault) logHashMismatch(blockHeight uint64, existing, incoming []byte) { logger.Error("Hashvault detected app hash mismatch; node attempted to change its mind. "+ "DO NOT RESTART WITHOUT HUMAN INVESTIGATION. If you are CERTAIN this is not a real "+ diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 35b550d452..1fa6f1e15c 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -6,6 +6,7 @@ import ( dbm "github.com/tendermint/tm-db" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -22,11 +23,14 @@ type CosmosStateStore struct { db types.StateStore snapshotMgr *sssnapshot.Manager externalPruning bool + closed bool } // NewCosmosStateStore wraps an existing StateStore as a CosmosStateStore. func NewCosmosStateStore(db types.StateStore) *CosmosStateStore { - return &CosmosStateStore{db: db} + s := &CosmosStateStore{db: db} + utils.MustCloseE(s, "cosmos state store", (*CosmosStateStore).isClosed, (*CosmosStateStore).Close) + return s } func (s *CosmosStateStore) Get(storeKey string, version int64, key []byte) ([]byte, error) { @@ -98,9 +102,14 @@ func (s *CosmosStateStore) Import(version int64, ch <-chan types.SnapshotNode) e } func (s *CosmosStateStore) Close() error { + s.closed = true return s.db.Close() } +func (s *CosmosStateStore) isClosed() bool { + return s.closed +} + func (s *CosmosStateStore) SupportsCheckpoint() bool { return sssnapshot.SupportsCheckpoint(s.db) } diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 9ace11abe4..916fa40772 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -13,6 +13,7 @@ import ( commonevm "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/common/metrics" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -46,6 +47,9 @@ type EVMStateStore struct { commitPhases *metrics.PhaseTimer externalPruning bool + + // Set by Close(). + closed bool } // NewEVMStateStore opens either a single unified MVCC DB for all EVM state @@ -63,6 +67,7 @@ func NewEVMStateStore(dir string, ssConfig config.StateStoreConfig) (*EVMStateSt _ = store.Close() return nil, err } + utils.MustCloseE(store, "EVM state store", (*EVMStateStore).isClosed, (*EVMStateStore).Close) return store, nil } @@ -557,12 +562,17 @@ func subDBPath(base string, storeType EVMStoreType) string { } func (s *EVMStateStore) Close() error { + s.closed = true // A snapshot being published reads and stamps these databases, so it has to finish before they // close rather than race the shutdown. s.stopCheckpoints() return s.closeDBs() } +func (s *EVMStateStore) isClosed() bool { + return s.closed +} + func (s *EVMStateStore) SupportsCheckpoint() bool { for _, db := range s.managedDBs { if !sssnapshot.SupportsCheckpoint(db) { diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go index b11863a838..4108a18ad6 100644 --- a/sei-db/state_db/ss/snapshot/manager.go +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -133,6 +133,7 @@ type Staged struct { version int64 tmpDir string finalDir string + finished bool } func (s *Staged) Abort() { @@ -142,6 +143,10 @@ func (s *Staged) Abort() { s.manager.abort(s) } +func (s *Staged) isFinished() bool { + return s.finished +} + // SnapshotDirName returns the directory name for a snapshot labeled with the // given version. func SnapshotDirName(version int64) string { @@ -260,12 +265,14 @@ func (m *Manager) Prepare(version int64) (*Staged, error) { if err := os.MkdirAll(m.root, 0o750); err != nil { return nil, fmt.Errorf("create %s snapshot root: %w", m.name, err) } - return &Staged{ + staged := &Staged{ manager: m, version: version, tmpDir: tmpDir, finalDir: finalDir, - }, nil + } + utils.MustClose(staged, "staged state store snapshot", (*Staged).isFinished, (*Staged).Abort) + return staged, nil } // Schedule queues staged's checkpoint behind the writes already enqueued on this member's backend. @@ -282,6 +289,7 @@ func (m *Manager) Commit(staged *Staged) error { if staged == nil || staged.manager != m { return fmt.Errorf("%s staged snapshot belongs to a different manager", m.name) } + staged.finished = true apparentBytes, sizeErr := snapshotDirApparentBytes(staged.tmpDir) if sizeErr != nil { logger.Error("failed to measure state store snapshot", @@ -328,6 +336,7 @@ func (m *Manager) abort(staged *Staged) { if staged == nil || staged.manager != m { return } + staged.finished = true if err := os.RemoveAll(staged.tmpDir); err != nil { logger.Error("failed to remove aborted state store snapshot", "store", m.name, "dir", staged.tmpDir, "error", err) diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index 12a0479b16..1e8b412e2a 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -5,6 +5,7 @@ import ( "fmt" "sync/atomic" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/seiwal" ) @@ -113,6 +114,7 @@ func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { w.lastBlock.Store(last) w.hasBlock = true } + utils.MustCloseE(w, "state WAL", (*stateWALImpl).isClosed, (*stateWALImpl).Close) return w, nil } @@ -234,6 +236,10 @@ func (w *stateWALImpl) Close() error { return nil } +func (w *stateWALImpl) isClosed() bool { + return w.closed +} + // fail records err as the first fatal error that bricks the WAL and returns it. Once set, every // subsequent operation fails fast rather than touching the underlying WAL. func (w *stateWALImpl) fail(err error) error { From aa556bad4e88322809c4cfde36923412741bdc2a Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 24 Sep 2026 09:56:19 -0500 Subject: [PATCH 2/4] fix unit tests --- sei-db/bootstrap/recovery_test.go | 4 ++- sei-db/common/utils/must_close.go | 31 +++++++++++++++++-- sei-db/db_engine/litt/disktable/disk_table.go | 12 +++++-- .../db_engine/litt/disktable/iterator_test.go | 2 ++ .../litt/test/keymap_migration_test.go | 2 ++ sei-db/db_engine/litt/test/table_test.go | 2 ++ sei-db/db_engine/test/storage_test_suite.go | 1 + .../state_db/sc/flatkv/import_export_test.go | 5 +-- sei-db/state_db/sc/flatkv/importer.go | 6 ++++ sei-db/state_db/sc/flatkv/snapshot_test.go | 3 ++ sei-db/state_db/sc/flatkv/store_test.go | 1 + sei-db/state_db/sc/flatkv/testutil_test.go | 1 + .../state_db/statewal/state_wal_brick_test.go | 1 + 13 files changed, 62 insertions(+), 9 deletions(-) diff --git a/sei-db/bootstrap/recovery_test.go b/sei-db/bootstrap/recovery_test.go index 34ea37be12..bebe11a247 100644 --- a/sei-db/bootstrap/recovery_test.go +++ b/sei-db/bootstrap/recovery_test.go @@ -451,7 +451,9 @@ func TestRecoverReplaysForwardFromTheStoreOwnHeight(t *testing.T) { require.Equal(t, int64(9), manager.SC().Version()) requireWALTail(t, manager, 9) require.DirExists(t, scSnapshotDir(cfg.FlatKVConfig.DataDir, 1)) - marker, ok := manager.SC().OpenView().Get(evm.EVMStoreKey, evmNonceKey(108)) + view := manager.SC().OpenView() + defer view.Close() + marker, ok := view.Get(evm.EVMStoreKey, evmNonceKey(108)) require.True(t, ok, "SC replayed forward from 8, so the working copy it held there is still open") require.Equal(t, evmNonce(108), marker) } diff --git a/sei-db/common/utils/must_close.go b/sei-db/common/utils/must_close.go index c96fe9de4c..ded56b13c9 100644 --- a/sei-db/common/utils/must_close.go +++ b/sei-db/common/utils/must_close.go @@ -3,6 +3,7 @@ package utils import ( "fmt" "runtime" + "strings" "testing" "github.com/sei-protocol/seilog" @@ -10,9 +11,12 @@ import ( var logger = seilog.NewLogger("db", "common", "utils") +// The deepest call stack recorded for where an object passed to MustClose() was created. +const mustCloseStackDepth = 32 + // MustClose closes obj if it becomes unreachable while isClosed() still reports it open, logging an error when it // does. It is a safety net, not a way to close things: any close it performs is a bug in the code that leaked obj. -// In a test binary it panics instead, so a leak fails the test run. +// In a test binary it panics instead, naming where obj was created, so a leak fails the test run. // // obj must point to the start of an allocation that has no finalizer. isClosed() and close() receive obj as their // argument and must not capture it, or obj never becomes unreachable; pass method expressions such as @@ -36,12 +40,22 @@ func MustCloseE[T any]( isClosed func(*T) bool, close func(*T) error, ) { + + // Capture a stack trace, but only if running in a test environment. Too costly for production use. + var createdAt []uintptr + if testing.Testing() { + createdAt = make([]uintptr, mustCloseStackDepth) + // Skips runtime.Callers() and MustCloseE() itself. + createdAt = createdAt[:runtime.Callers(2, createdAt)] + } + runtime.SetFinalizer(obj, func(o *T) { if isClosed(o) { return } if testing.Testing() { - panic(fmt.Sprintf("%s became unreachable without being closed", description)) + panic(fmt.Sprintf("%s became unreachable without being closed; created at:\n%s", + description, formatStack(createdAt))) } logger.Error("object became unreachable without being closed, closing it now", "object", description) // Finalizers share one goroutine, and a close may block. @@ -52,3 +66,16 @@ func MustCloseE[T any]( }() }) } + +// formatStack renders the program counters recorded by runtime.Callers() as one function and file:line per frame. +func formatStack(pcs []uintptr) string { + var b strings.Builder + frames := runtime.CallersFrames(pcs) + for { + frame, more := frames.Next() + fmt.Fprintf(&b, " %s\n %s:%d\n", frame.Function, frame.File, frame.Line) + if !more { + return b.String() + } + } +} diff --git a/sei-db/db_engine/litt/disktable/disk_table.go b/sei-db/db_engine/litt/disktable/disk_table.go index 09a52a0227..de8d4a7831 100644 --- a/sei-db/db_engine/litt/disktable/disk_table.go +++ b/sei-db/db_engine/litt/disktable/disk_table.go @@ -106,7 +106,7 @@ type DiskTable struct { flushCoordinator *flushCoordinator } -// NewDiskTable creates a new DiskTable. +// NewDiskTable creates a new DiskTable, which takes ownership of keymap. If it fails, keymap is stopped. func NewDiskTable( config *litt.Config, runtimeConfig *litt.RuntimeConfig, @@ -117,7 +117,15 @@ func NewDiskTable( keymapTypeFile *keymap.KeymapTypeFile, roots []string, reloadKeymap bool, - metrics *metrics.LittDBMetrics) (litt.ManagedTable, error) { + metrics *metrics.LittDBMetrics) (_ litt.ManagedTable, err error) { + + defer func() { + if err != nil { + if stopErr := keymap.Stop(); stopErr != nil { + err = errors.Join(err, fmt.Errorf("failed to stop keymap: %w", stopErr)) + } + } + }() if config.GCPeriod <= 0 { return nil, errors.New("garbage collection period must be greater than 0") diff --git a/sei-db/db_engine/litt/disktable/iterator_test.go b/sei-db/db_engine/litt/disktable/iterator_test.go index da623a4046..6e45a9dcfe 100644 --- a/sei-db/db_engine/litt/disktable/iterator_test.go +++ b/sei-db/db_engine/litt/disktable/iterator_test.go @@ -870,6 +870,8 @@ func TestIteratorAccessorsRejectMisuse(t *testing.T) { _, err = it.GetValue() require.ErrorContains(t, err, "not positioned") + require.NoError(t, it.Close()) + // Closed. Re-position first, so the guard being tested is `closed` and not exhaustion. it, err = table.Iterator(reverse) require.NoError(t, err) diff --git a/sei-db/db_engine/litt/test/keymap_migration_test.go b/sei-db/db_engine/litt/test/keymap_migration_test.go index cc0e508db5..d516b142da 100644 --- a/sei-db/db_engine/litt/test/keymap_migration_test.go +++ b/sei-db/db_engine/litt/test/keymap_migration_test.go @@ -291,4 +291,6 @@ func TestFailedKeymapMigration(t *testing.T) { require.True(t, ok) require.Equal(t, expectedValue, value) } + + require.NoError(t, db.Close()) } diff --git a/sei-db/db_engine/litt/test/table_test.go b/sei-db/db_engine/litt/test/table_test.go index 3ce6a906c4..31d59ee37d 100644 --- a/sei-db/db_engine/litt/test/table_test.go +++ b/sei-db/db_engine/litt/test/table_test.go @@ -505,6 +505,8 @@ func TestInvalidTableName(t *testing.T) { table, err = db.BuildTable(litt.DefaultTableConfig(tableName)) require.Error(t, err) require.Nil(t, table) + + require.NoError(t, db.Close()) } // secondaryKeyBasicsTest runs against every table implementation registered in tableBuilders. It diff --git a/sei-db/db_engine/test/storage_test_suite.go b/sei-db/db_engine/test/storage_test_suite.go index d21d0f6e00..cfe2ba4392 100644 --- a/sei-db/db_engine/test/storage_test_suite.go +++ b/sei-db/db_engine/test/storage_test_suite.go @@ -711,6 +711,7 @@ func (s *StorageTestSuite) TestDatabasePrune() { itr, err := db.Iterator(storeKey1, 25, []byte("key000"), nil) s.Require().NoError(err) s.Require().False(itr.Valid()) + s.Require().NoError(itr.Close()) // prune the latest version which should prune the entire dataset s.Require().NoError(db.Prune(50)) diff --git a/sei-db/state_db/sc/flatkv/import_export_test.go b/sei-db/state_db/sc/flatkv/import_export_test.go index 9f02a5b443..af34441958 100644 --- a/sei-db/state_db/sc/flatkv/import_export_test.go +++ b/sei-db/state_db/sc/flatkv/import_export_test.go @@ -857,10 +857,7 @@ func TestExporterCorruptAccountValueInDB(t *testing.T) { } require.Len(t, nodes, 1, "corrupt value should still be exported as raw bytes") require.Equal(t, []byte{0xDE, 0xAD}, nodes[0].Value) - if exp.iter != nil { - _ = exp.iter.Close() - exp.iter = nil - } + require.NoError(t, exp.Close()) } // TestExporterImporterNonEVMMiscRoundTrip drives non-EVM module data diff --git a/sei-db/state_db/sc/flatkv/importer.go b/sei-db/state_db/sc/flatkv/importer.go index 2d8836a4db..6a0135907d 100644 --- a/sei-db/state_db/sc/flatkv/importer.go +++ b/sei-db/state_db/sc/flatkv/importer.go @@ -173,6 +173,12 @@ func (w *dbWorker) flush() (err error) { addImportKVPairs(w.ctx, w.dir, pairCount) w.flushes++ w.pairs += int64(pairCount) + closeErr := w.batch.Close() + // Cleared before the error check, so run() does not close the batch a second time. + w.batch = nil + if closeErr != nil { + return fmt.Errorf("%s close batch: %w", w.dir, closeErr) + } w.batch = w.db.NewBatch() w.ltMutations = w.ltMutations[:0] return nil diff --git a/sei-db/state_db/sc/flatkv/snapshot_test.go b/sei-db/state_db/sc/flatkv/snapshot_test.go index 7c41e0fdd2..3d4afb0164 100644 --- a/sei-db/state_db/sc/flatkv/snapshot_test.go +++ b/sei-db/state_db/sc/flatkv/snapshot_test.go @@ -1790,6 +1790,7 @@ func TestSingleDBOpenFailure(t *testing.T) { require.NoError(t, err) err = s2.LoadLatest() require.Error(t, err, "open should fail when storageDB is corrupted in both working and snapshot") + require.NoError(t, s2.Close()) } // ============================================================================= @@ -1878,6 +1879,7 @@ func TestLocalMetaCorruption(t *testing.T) { err = s2.LoadLatest() require.Error(t, err, "open should fail when meta version is corrupted") require.Contains(t, err.Error(), "invalid _meta/version length") + require.NoError(t, s2.Close()) } // TestWALSegmentCorruption simulates WAL data loss caused by segment corruption. @@ -1936,6 +1938,7 @@ func TestWALSegmentCorruption(t *testing.T) { require.NoError(t, err) require.Error(t, s2.LoadLatest(), "opening must fail loudly rather than silently skipping the corrupted WAL segment") + require.NoError(t, s2.Close()) } // ============================================================================= diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index ae5f98069b..7348d834b7 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -759,6 +759,7 @@ func TestReadOnlyLoadFromUnopenedStore(t *testing.T) { fresh, err := newCommitStoreWithWAL(t.Context(), cfg) require.NoError(t, err) + defer func() { require.NoError(t, fresh.Close()) }() ro, err := fresh.LoadVersionReadOnly(0) require.NoError(t, err) defer ro.Close() diff --git a/sei-db/state_db/sc/flatkv/testutil_test.go b/sei-db/state_db/sc/flatkv/testutil_test.go index 97f1edfc8e..550cc647d5 100644 --- a/sei-db/state_db/sc/flatkv/testutil_test.go +++ b/sei-db/state_db/sc/flatkv/testutil_test.go @@ -136,6 +136,7 @@ func setupTestDB(t *testing.T) types.KeyValueDB { cfg := pebbledb.DefaultTestConfig(t) db, err := pebbledb.Open(t.Context(), &cfg) require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) return db } diff --git a/sei-db/state_db/statewal/state_wal_brick_test.go b/sei-db/state_db/statewal/state_wal_brick_test.go index 51a377492a..41a0f35857 100644 --- a/sei-db/state_db/statewal/state_wal_brick_test.go +++ b/sei-db/state_db/statewal/state_wal_brick_test.go @@ -67,6 +67,7 @@ func newFakeStateWAL(t *testing.T, f *fakeWAL) StateWAL { t.Helper() w, err := newStateWAL(f) require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, w.Close()) }) return w } From fba04f10d39bb2796e4272da45c4ec5e26bd12f0 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 25 Sep 2026 14:47:36 -0500 Subject: [PATCH 3/4] change pattern to be less foot-gun-ish --- sei-db/common/utils/must_close.go | 118 +++++++++++------- sei-db/common/utils/must_close_test.go | 43 +++++++ .../litt/disktable/forward_iterator.go | 24 ++-- .../litt/disktable/keymap/pebble_db_keymap.go | 9 +- .../litt/disktable/reverse_iterator.go | 24 ++-- sei-db/db_engine/litt/util/file_lock.go | 10 +- sei-db/db_engine/pebbledb/batch.go | 12 +- sei-db/db_engine/pebbledb/db.go | 10 +- sei-db/db_engine/pebbledb/iterator.go | 12 +- sei-db/db_engine/pebbledb/mvcc/iterator.go | 12 +- .../pebbledb/mvcc/iterator_ascending.go | 13 +- sei-db/db_engine/rocksdb/mvcc/iterator.go | 12 +- sei-db/db_engine/view/brick_test.go | 6 +- sei-db/db_engine/view/shutdown_test.go | 13 +- sei-db/db_engine/view/view_impl.go | 20 +-- sei-db/db_engine/view/view_lifecycle_test.go | 9 +- sei-db/db_engine/view/view_manager.go | 5 + sei-db/db_engine/view/view_manager_impl.go | 57 +++------ sei-db/state_db/giga/state_db.go | 12 +- sei-db/state_db/sc/flatkv/exporter.go | 10 +- .../sc/flatkv/finalization_manager.go | 29 ++++- .../sc/flatkv/lthash/block_gatherer.go | 7 +- .../state_db/sc/flatkv/lthash/hash_engine.go | 7 +- .../sc/flatkv/lthash/hash_engine_messages.go | 2 +- .../sc/flatkv/lthash/hash_engine_test.go | 1 + sei-db/state_db/sc/flatkv/snapshot.go | 3 +- sei-db/state_db/sc/flatkv/snapshot_writer.go | 16 ++- .../sc/flatkv/snapshot_writer_test.go | 2 + sei-db/state_db/sc/flatkv/state_view.go | 11 +- .../sc/flatkv/state_view_physkey_test.go | 4 + sei-db/state_db/sc/flatkv/store_read.go | 2 +- sei-db/state_db/sc/flatkv/store_write.go | 45 +++++-- .../sc/flatkv/sview/atomic_store_view.go | 12 +- sei-db/state_db/sc/flatkv/sview/store_view.go | 22 +++- .../sc/flatkv/sview/store_view_test.go | 2 + .../state_db/sc/flatkv/sview/testutil_test.go | 2 + .../state_db/sc/hashvault/pebble_hashvault.go | 22 ++-- sei-db/state_db/ss/cosmos/store.go | 11 +- sei-db/state_db/ss/evm/store.go | 12 +- sei-db/state_db/ss/snapshot/manager.go | 14 +-- sei-db/state_db/statewal/state_wal_impl.go | 28 ++--- 41 files changed, 400 insertions(+), 285 deletions(-) create mode 100644 sei-db/common/utils/must_close_test.go diff --git a/sei-db/common/utils/must_close.go b/sei-db/common/utils/must_close.go index 71497feb3c..43027d47fa 100644 --- a/sei-db/common/utils/must_close.go +++ b/sei-db/common/utils/must_close.go @@ -4,7 +4,9 @@ import ( "fmt" "runtime" "strings" + "sync/atomic" "testing" + "weak" "github.com/sei-protocol/seilog" ) @@ -14,57 +16,83 @@ var logger = seilog.NewLogger("db", "common", "utils") // The deepest call stack recorded for where an object passed to MustClose() was created. const mustCloseStackDepth = 32 -// MustClose closes obj if it becomes unreachable while isClosed() still reports it open, logging an error when it -// does. It is a safety net, not a way to close things: any close it performs is a bug in the code that leaked obj. -// In a test binary it panics instead, naming where obj was created, so a leak fails the test run. -// -// obj must point to the start of an allocation that has no finalizer. isClosed() and close() receive obj as their -// argument and must not capture it, or obj never becomes unreachable; pass method expressions such as -// (*Store).Close. An object reachable from its own goroutines or from a reference cycle is never closed. -func MustClose[T any]( - obj *T, - description string, - isClosed func(*T) bool, - close func(*T), -) { - MustCloseE(obj, description, isClosed, func(o *T) error { - close(o) - return nil - }) +// CloseMarker records whether the object it was issued for by MustClose() has been closed. Copies of a marker +// share its state, so it may be stored by value. +type CloseMarker[T any] struct { + // The state every copy of this marker shares. Nil for a marker not issued by MustClose(). + state *closeState[T] } -// MustCloseE is MustClose() for a close() that returns an error. A close error is logged. -func MustCloseE[T any]( - obj *T, - description string, - isClosed func(*T) bool, - close func(*T) error, -) { +// closeState is the state a CloseMarker and its copies share. +type closeState[T any] struct { + // The object this marker was issued for. + owner weak.Pointer[T] - // Capture a stack trace, but only if running in a test environment. Too costly for production use. - var createdAt []uintptr + // Set by CloseMarker.Close(). + closed atomic.Bool + + // What the object is, named in reports. + description string + + // Where the object was registered. Recorded only in a test binary. + createdAt []uintptr +} + +// MustClose reports obj as leaked if it becomes unreachable before the returned marker is closed. It logs an error, +// or in a test binary panics naming where obj was created. Every path that closes obj must call the marker's +// Close(). An object reachable from its own running goroutines is never reported. +func MustClose[T any](obj *T, description string) CloseMarker[T] { + state := &closeState[T]{ + owner: weak.Make(obj), + description: description, + } if testing.Testing() { - createdAt = make([]uintptr, mustCloseStackDepth) - // Skips runtime.Callers() and MustCloseE() itself. - createdAt = createdAt[:runtime.Callers(2, createdAt)] + createdAt := make([]uintptr, mustCloseStackDepth) + // Skips runtime.Callers() and MustClose() itself. + state.createdAt = createdAt[:runtime.Callers(2, createdAt)] } + runtime.AddCleanup(obj, reportIfOpen[T], state) + return CloseMarker[T]{state: state} +} - runtime.SetFinalizer(obj, func(o *T) { - if isClosed(o) { - return - } - if testing.Testing() { - panic(fmt.Sprintf("%s became unreachable without being closed; created at:\n%s", - description, formatStack(createdAt))) - } - logger.Error("object became unreachable without being closed, closing it now", "object", description) - // Finalizers share one goroutine, and a close may block. - go func() { - if err := close(o); err != nil { - logger.Error("failed to close an unreachable object", "object", description, "err", err) - } - }() - }) +// Close records that owner has been closed. owner must be the object this marker was issued for. Idempotent. +func (m CloseMarker[T]) Close(owner *T) { + state := m.issuedState() + if weak.Make(owner) != state.owner { + state.report("was closed through a marker issued for another object") + } + state.closed.Store(true) + // owner must stay reachable until it reads as closed, or its cleanup could run in between. + runtime.KeepAlive(owner) +} + +// IsClosed reports whether Close() has been called. +func (m CloseMarker[T]) IsClosed() bool { + return m.issuedState().closed.Load() +} + +// issuedState returns the marker's shared state, panicking if the marker was not issued by MustClose(). +func (m CloseMarker[T]) issuedState() *closeState[T] { + if m.state == nil { + panic("close marker was not issued by MustClose()") + } + return m.state +} + +// reportIfOpen reports the object state was issued for as leaked, unless it has been closed. +func reportIfOpen[T any](state *closeState[T]) { + if state.closed.Load() { + return + } + state.report("became unreachable without being closed") +} + +// report panics with problem and the object's creation stack in a test binary, and logs problem otherwise. +func (s *closeState[T]) report(problem string) { + if testing.Testing() { + panic(fmt.Sprintf("%s %s; created at:\n%s", s.description, problem, formatStack(s.createdAt))) + } + logger.Error("object "+problem, "object", s.description) } // formatStack renders the program counters recorded by runtime.Callers() as one function and file:line per frame. diff --git a/sei-db/common/utils/must_close_test.go b/sei-db/common/utils/must_close_test.go new file mode 100644 index 0000000000..5c47ed3afb --- /dev/null +++ b/sei-db/common/utils/must_close_test.go @@ -0,0 +1,43 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// mustCloseTestObject is an object registered with MustClose() by these tests. It holds a pointer so the runtime +// does not batch it into a shared allocation, where its cleanup might never run. +type mustCloseTestObject struct { + // Unused; present only to give the object a pointer. + payload *int +} + +// A marker closed with an object other than the one it was issued for is reported. +func TestCloseMarkerRejectsAnotherOwner(t *testing.T) { + owner := &mustCloseTestObject{} + marker := MustClose(owner, "test object") + + require.Panics(t, func() { marker.Close(&mustCloseTestObject{}) }) + require.False(t, marker.IsClosed()) + + marker.Close(owner) + require.True(t, marker.IsClosed()) +} + +// Every copy of a marker reports the close made through any of them. +func TestCloseMarkerCopiesShareState(t *testing.T) { + owner := &mustCloseTestObject{} + marker := MustClose(owner, "test object") + markerCopy := marker + + markerCopy.Close(owner) + require.True(t, marker.IsClosed()) +} + +// A marker that MustClose() did not issue refuses to be used. +func TestCloseMarkerNotIssuedPanics(t *testing.T) { + var marker CloseMarker[mustCloseTestObject] + require.Panics(t, func() { marker.Close(&mustCloseTestObject{}) }) + require.Panics(t, func() { marker.IsClosed() }) +} diff --git a/sei-db/db_engine/litt/disktable/forward_iterator.go b/sei-db/db_engine/litt/disktable/forward_iterator.go index f922119b4c..d01376226c 100644 --- a/sei-db/db_engine/litt/disktable/forward_iterator.go +++ b/sei-db/db_engine/litt/disktable/forward_iterator.go @@ -52,8 +52,8 @@ type forwardIterator struct { // readerSeg is the segment that reader was created for, so we can detect when to recreate it. readerSeg *segment.Segment - // closed is true once Close has been called. - closed bool + // closed records whether Close has been called. + closed utils.CloseMarker[forwardIterator] // groupValid is true once a primary key in the current group has been visited, meaning groupAddr and // groupValue describe that group's primary value. @@ -75,7 +75,7 @@ func newForwardIterator(table *DiskTable, segs []*segment.Segment) *forwardItera segs: segs, segPos: 0, } - utils.MustCloseE(it, "littdb forward iterator", (*forwardIterator).isClosed, (*forwardIterator).Close) + it.closed = utils.MustClose(it, "littdb forward iterator") return it } @@ -97,7 +97,7 @@ func newForwardIteratorAt( keys: keys, keyPos: keyPos, } - utils.MustCloseE(it, "littdb forward iterator", (*forwardIterator).isClosed, (*forwardIterator).Close) + it.closed = utils.MustClose(it, "littdb forward iterator") return it } @@ -113,13 +113,13 @@ func NewOfflineForwardIterator(segs []*segment.Segment, release func()) litt.Ite segs: segs, segPos: 0, } - utils.MustCloseE(it, "littdb forward iterator", (*forwardIterator).isClosed, (*forwardIterator).Close) + it.closed = utils.MustClose(it, "littdb forward iterator") return it } // Next advances the iterator to the next key in insertion order. func (it *forwardIterator) Next() (bool, error) { - if it.closed { + if it.closed.IsClosed() { return false, fmt.Errorf("iterator is closed") } @@ -166,7 +166,7 @@ func (it *forwardIterator) Next() (bool, error) { // GetKey returns the current key and whether it is a primary key. func (it *forwardIterator) GetKey() (key []byte, isPrimary bool, err error) { - if it.closed { + if it.closed.IsClosed() { return nil, false, fmt.Errorf("iterator is closed") } if it.current == nil { @@ -177,7 +177,7 @@ func (it *forwardIterator) GetKey() (key []byte, isPrimary bool, err error) { // GetValue reads and returns the value associated with the current key. func (it *forwardIterator) GetValue() (value []byte, err error) { - if it.closed { + if it.closed.IsClosed() { // Close released the snapshot's segment reservations, so reading now would open a new // reader on segments GC is free to have deleted. return nil, fmt.Errorf("iterator is closed") @@ -263,10 +263,10 @@ func (it *forwardIterator) secondaryWithinGroup(addr types.Address) bool { // Close releases the resources held by the iterator, via onClose. func (it *forwardIterator) Close() error { - if it.closed { + if it.closed.IsClosed() { return nil } - it.closed = true + it.closed.Close(it) // Close the buffered reader. var readerErr error @@ -309,7 +309,3 @@ func closeLiveIterator(table *DiskTable, segs []*segment.Segment) func() error { return nil } } - -func (it *forwardIterator) isClosed() bool { - return it.closed -} diff --git a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go index 072f5b85a7..7b7bf320ad 100644 --- a/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go +++ b/sei-db/db_engine/litt/disktable/keymap/pebble_db_keymap.go @@ -26,6 +26,8 @@ type PebbleDBKeymap struct { doubleWriteProtection bool keymapPath string alive atomic.Bool + // closed records whether Stop has been called. + closed utils.CloseMarker[PebbleDBKeymap] // This is a "test mode only" flag. Should be true in production use cases or anywhere that data consistency // is critical. Unit tests write lots of little values, and syncing each one is slow, so it may be desirable // to set this to false in some tests. @@ -86,7 +88,7 @@ func newPebbleDBKeymap( syncWrites: syncWrites, } kmap.alive.Store(true) - utils.MustCloseE(kmap, "littdb pebble keymap", (*PebbleDBKeymap).isStopped, (*PebbleDBKeymap).Stop) + kmap.closed = utils.MustClose(kmap, "littdb pebble keymap") return kmap, requiresReload, nil } @@ -202,6 +204,7 @@ func (p *PebbleDBKeymap) Stop() error { if !alive { return nil } + p.closed.Close(p) err := p.db.Close() if err != nil { @@ -210,10 +213,6 @@ func (p *PebbleDBKeymap) Stop() error { return nil } -func (p *PebbleDBKeymap) isStopped() bool { - return !p.alive.Load() -} - func (p *PebbleDBKeymap) Destroy() error { err := p.Stop() if err != nil { diff --git a/sei-db/db_engine/litt/disktable/reverse_iterator.go b/sei-db/db_engine/litt/disktable/reverse_iterator.go index e5be94bc17..7c5ee8477a 100644 --- a/sei-db/db_engine/litt/disktable/reverse_iterator.go +++ b/sei-db/db_engine/litt/disktable/reverse_iterator.go @@ -43,8 +43,8 @@ type reverseIterator struct { // currentSeg is the segment that current was read from. currentSeg *segment.Segment - // closed is true once Close has been called. - closed bool + // closed records whether Close has been called. + closed utils.CloseMarker[reverseIterator] } // newReverseIterator creates a reverse iterator over the given snapshot of sealed segments, owned by a @@ -55,7 +55,7 @@ func newReverseIterator(table *DiskTable, segs []*segment.Segment) *reverseItera segs: segs, segPos: len(segs) - 1, } - utils.MustCloseE(it, "littdb reverse iterator", (*reverseIterator).isClosed, (*reverseIterator).Close) + it.closed = utils.MustClose(it, "littdb reverse iterator") return it } @@ -77,7 +77,7 @@ func newReverseIteratorAt( keys: keys, keyPos: keyPos, } - utils.MustCloseE(it, "littdb reverse iterator", (*reverseIterator).isClosed, (*reverseIterator).Close) + it.closed = utils.MustClose(it, "littdb reverse iterator") return it } @@ -93,13 +93,13 @@ func NewOfflineReverseIterator(segs []*segment.Segment, release func()) litt.Ite segs: segs, segPos: len(segs) - 1, } - utils.MustCloseE(it, "littdb reverse iterator", (*reverseIterator).isClosed, (*reverseIterator).Close) + it.closed = utils.MustClose(it, "littdb reverse iterator") return it } // Next advances the iterator to the next key in reverse insertion order. func (it *reverseIterator) Next() (bool, error) { - if it.closed { + if it.closed.IsClosed() { return false, fmt.Errorf("iterator is closed") } @@ -138,7 +138,7 @@ func (it *reverseIterator) Next() (bool, error) { // GetKey returns the current key and whether it is a primary key. func (it *reverseIterator) GetKey() (key []byte, isPrimary bool, err error) { - if it.closed { + if it.closed.IsClosed() { return nil, false, fmt.Errorf("iterator is closed") } if it.current == nil { @@ -151,7 +151,7 @@ func (it *reverseIterator) GetKey() (key []byte, isPrimary bool, err error) { // directly from the value file (the forward secondary-key optimization does not apply because a // secondary is reached before its primary). func (it *reverseIterator) GetValue() (value []byte, err error) { - if it.closed { + if it.closed.IsClosed() { // Close released the snapshot's segment reservations, so reading now would touch segments // GC is free to have deleted. return nil, fmt.Errorf("iterator is closed") @@ -168,16 +168,12 @@ func (it *reverseIterator) GetValue() (value []byte, err error) { // Close releases the resources held by the iterator, via onClose. func (it *reverseIterator) Close() error { - if it.closed { + if it.closed.IsClosed() { return nil } - it.closed = true + it.closed.Close(it) closeErr := it.onClose() it.segs = nil return closeErr } - -func (it *reverseIterator) isClosed() bool { - return it.closed -} diff --git a/sei-db/db_engine/litt/util/file_lock.go b/sei-db/db_engine/litt/util/file_lock.go index 79b50800d4..42bfdc49b2 100644 --- a/sei-db/db_engine/litt/util/file_lock.go +++ b/sei-db/db_engine/litt/util/file_lock.go @@ -19,6 +19,9 @@ type FileLock struct { logger *slog.Logger path string file *os.File + + // closed records whether Release has been called. + closed utils.CloseMarker[FileLock] } // IsProcessAlive checks if a process with the given PID is still running @@ -166,7 +169,7 @@ func NewFileLock(logger *slog.Logger, path string, fsync bool) (*FileLock, error path: path, file: file, } - utils.MustClose(lock, "littdb file lock", (*FileLock).isReleased, (*FileLock).Release) + lock.closed = utils.MustClose(lock, "littdb file lock") return lock, nil } @@ -176,6 +179,7 @@ func (fl *FileLock) Release() { if fl.file == nil { return } + fl.closed.Close(fl) // Close the file first err := fl.file.Close() @@ -194,10 +198,6 @@ func (fl *FileLock) Release() { } } -func (fl *FileLock) isReleased() bool { - return fl.file == nil -} - // Path returns the path of the lock file func (fl *FileLock) Path() string { return fl.path diff --git a/sei-db/db_engine/pebbledb/batch.go b/sei-db/db_engine/pebbledb/batch.go index 0e2da6acac..20b4d170df 100644 --- a/sei-db/db_engine/pebbledb/batch.go +++ b/sei-db/db_engine/pebbledb/batch.go @@ -16,7 +16,9 @@ type pebbleBatch struct { b *pebble.Batch operationMetrics *OperationMetrics commitMetrics *CommitMetrics - closed bool + + // closed records whether Close has been called. + closed utils.CloseMarker[pebbleBatch] } var _ types.Batch = (*pebbleBatch)(nil) @@ -30,7 +32,7 @@ func (p *pebbleDB) NewBatch() types.Batch { operationMetrics: p.operationMetrics, commitMetrics: p.commitMetrics, } - utils.MustCloseE(pb, "pebbledb batch", (*pebbleBatch).isClosed, (*pebbleBatch).Close) + pb.closed = utils.MustClose(pb, "pebbledb batch") return pb } @@ -80,10 +82,6 @@ func (pb *pebbleBatch) Reset() { } func (pb *pebbleBatch) Close() error { - pb.closed = true + pb.closed.Close(pb) return pb.b.Close() } - -func (pb *pebbleBatch) isClosed() bool { - return pb.closed -} diff --git a/sei-db/db_engine/pebbledb/db.go b/sei-db/db_engine/pebbledb/db.go index 8347d74a15..68359e61b0 100644 --- a/sei-db/db_engine/pebbledb/db.go +++ b/sei-db/db_engine/pebbledb/db.go @@ -24,6 +24,9 @@ type pebbleDB struct { metricsCancel context.CancelFunc operationMetrics *OperationMetrics commitMetrics *CommitMetrics + + // closed records whether Close has been called. + closed utils.CloseMarker[pebbleDB] } var _ types.KeyValueDB = (*pebbleDB)(nil) @@ -106,7 +109,7 @@ func Open( operationMetrics: NewOperationMetrics(config.EnableReadWriteMetrics, filepath.Base(config.DataDir)), commitMetrics: NewCommitMetrics(config.EnableMetrics, filepath.Base(config.DataDir)), } - utils.MustCloseE(p, "pebbledb", (*pebbleDB).isClosed, (*pebbleDB).Close) + p.closed = utils.MustClose(p, "pebbledb") return p, nil } @@ -210,6 +213,7 @@ func (p *pebbleDB) Close() error { if p.db == nil { return nil } + p.closed.Close(p) if p.metricsCancel != nil { p.metricsCancel() @@ -222,10 +226,6 @@ func (p *pebbleDB) Close() error { return db.Close() } -func (p *pebbleDB) isClosed() bool { - return p.db == nil -} - func toPebbleWriteOpts(opts types.WriteOptions) *pebble.WriteOptions { if opts.Sync { return pebble.Sync diff --git a/sei-db/db_engine/pebbledb/iterator.go b/sei-db/db_engine/pebbledb/iterator.go index e14de9f88d..1be799a892 100644 --- a/sei-db/db_engine/pebbledb/iterator.go +++ b/sei-db/db_engine/pebbledb/iterator.go @@ -19,7 +19,9 @@ type pebbleIterator struct { reverse bool operationMetrics *OperationMetrics readCount int64 - closed bool + + // closed records whether Close has been called. + closed utils.CloseMarker[pebbleIterator] } func newPebbleIterator(it *pebble.Iterator, opts *types.IterOptions, operationMetrics *OperationMetrics) *pebbleIterator { @@ -37,7 +39,7 @@ func newPebbleIterator(it *pebble.Iterator, opts *types.IterOptions, operationMe if pi.it.Valid() { pi.readCount++ } - utils.MustCloseE(pi, "pebbledb iterator", (*pebbleIterator).isClosed, (*pebbleIterator).Close) + pi.closed = utils.MustClose(pi, "pebbledb iterator") return pi } @@ -76,13 +78,9 @@ func (pi *pebbleIterator) Error() error { } func (pi *pebbleIterator) Close() error { - pi.closed = true + pi.closed.Close(pi) if pi.operationMetrics != nil { pi.operationMetrics.AddRead(pi.readCount) } return pi.it.Close() } - -func (pi *pebbleIterator) isClosed() bool { - return pi.closed -} diff --git a/sei-db/db_engine/pebbledb/mvcc/iterator.go b/sei-db/db_engine/pebbledb/mvcc/iterator.go index 96ce7ceb6c..d89ab36f98 100644 --- a/sei-db/db_engine/pebbledb/mvcc/iterator.go +++ b/sei-db/db_engine/pebbledb/mvcc/iterator.go @@ -42,6 +42,9 @@ type iterator struct { err error closeSync sync.Once + + // closed records whether Close has been called. + closed utils.CloseMarker[iterator] } func abortIfCancelled(ctx context.Context) error { @@ -90,7 +93,7 @@ func newPebbleDBIterator( dbName: dbName, ctx: ctx, } - utils.MustCloseE(itr, "mvcc iterator", (*iterator).isClosed, (*iterator).Close) + itr.closed = utils.MustClose(itr, "mvcc iterator") return itr } @@ -116,7 +119,7 @@ func newPebbleDBIterator( dbName: dbName, ctx: ctx, } - utils.MustCloseE(itr, "mvcc iterator", (*iterator).isClosed, (*iterator).Close) + itr.closed = utils.MustClose(itr, "mvcc iterator") if valid { currKey, _, ok := SplitMVCCKey(itr.source.Key()) @@ -387,6 +390,7 @@ func (itr *iterator) Error() error { func (itr *iterator) Close() error { itr.closeSync.Do(func() { + itr.closed.Close(itr) _ = itr.source.Close() itr.source = nil itr.valid = false @@ -503,7 +507,3 @@ func (itr *iterator) DebugRawIterate() { } } } - -func (itr *iterator) isClosed() bool { - return itr.source == nil -} diff --git a/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go b/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go index 48eba23830..66c19211bc 100644 --- a/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go +++ b/sei-db/db_engine/pebbledb/mvcc/iterator_ascending.go @@ -46,6 +46,9 @@ type ascendingIterator struct { err error closeSync sync.Once + + // closed records whether Close has been called. + closed utils.CloseMarker[ascendingIterator] } func newAscendingIterator( @@ -74,8 +77,7 @@ func newAscendingIterator( dbName: dbName, ctx: ctx, } - utils.MustCloseE(itr, "mvcc ascending iterator", - (*ascendingIterator).isClosed, (*ascendingIterator).Close) + itr.closed = utils.MustClose(itr, "mvcc ascending iterator") return itr } @@ -100,7 +102,7 @@ func newAscendingIterator( dbName: dbName, ctx: ctx, } - utils.MustCloseE(itr, "mvcc ascending iterator", (*ascendingIterator).isClosed, (*ascendingIterator).Close) + itr.closed = utils.MustClose(itr, "mvcc ascending iterator") if valid { currKey, _, ok := SplitMVCCKey(itr.source.Key()) @@ -386,6 +388,7 @@ func (itr *ascendingIterator) Error() error { func (itr *ascendingIterator) Close() error { itr.closeSync.Do(func() { + itr.closed.Close(itr) _ = itr.source.Close() itr.source = nil itr.valid = false @@ -444,7 +447,3 @@ func (itr *ascendingIterator) cursorTombstoned() bool { return true } - -func (itr *ascendingIterator) isClosed() bool { - return itr.source == nil -} diff --git a/sei-db/db_engine/rocksdb/mvcc/iterator.go b/sei-db/db_engine/rocksdb/mvcc/iterator.go index e439f7a48c..f6948f9df3 100644 --- a/sei-db/db_engine/rocksdb/mvcc/iterator.go +++ b/sei-db/db_engine/rocksdb/mvcc/iterator.go @@ -23,6 +23,9 @@ type iterator struct { reverse bool invalid bool closeOnce sync.Once + + // closed records whether Close has been called. + closed utils.CloseMarker[iterator] } func NewRocksDBIterator(source *grocksdb.Iterator, readOpts *grocksdb.ReadOptions, prefix, start, end []byte, version int64, earliestVersion int64, reverse bool) *iterator { @@ -38,7 +41,7 @@ func NewRocksDBIterator(source *grocksdb.Iterator, readOpts *grocksdb.ReadOption reverse: reverse, invalid: true, } - utils.MustCloseE(itr, "rocksdb mvcc iterator", (*iterator).isClosed, (*iterator).Close) + itr.closed = utils.MustClose(itr, "rocksdb mvcc iterator") return itr } @@ -75,7 +78,7 @@ func NewRocksDBIterator(source *grocksdb.Iterator, readOpts *grocksdb.ReadOption reverse: reverse, invalid: !source.Valid(), } - utils.MustCloseE(itr, "rocksdb mvcc iterator", (*iterator).isClosed, (*iterator).Close) + itr.closed = utils.MustClose(itr, "rocksdb mvcc iterator") return itr } @@ -167,6 +170,7 @@ func (itr *iterator) Error() error { func (itr *iterator) Close() error { itr.closeOnce.Do(func() { + itr.closed.Close(itr) src := itr.source ro := itr.readOpts itr.source = nil @@ -188,7 +192,3 @@ func (itr *iterator) assertIsValid() { panic("iterator is invalid") } } - -func (itr *iterator) isClosed() bool { - return itr.source == nil -} diff --git a/sei-db/db_engine/view/brick_test.go b/sei-db/db_engine/view/brick_test.go index 17a65574ce..d1839d8d79 100644 --- a/sei-db/db_engine/view/brick_test.go +++ b/sei-db/db_engine/view/brick_test.go @@ -54,6 +54,9 @@ func TestFlushFailureBricksManagerCleanly(t *testing.T) { case <-time.After(2 * time.Second): t.Fatal("AwaitFlush waiter did not unblock after the brick") } + + // view2 is never finalized, so it cannot be released. + view2.Abandon() } func TestCloseAfterBrickReportsFatalError(t *testing.T) { @@ -166,7 +169,8 @@ func TestBadVersionReferenceCountErrorsDoNotBrick(t *testing.T) { e := manager.(*viewManager) require.Error(t, e.IncrementReferenceCount(9999)) - require.Error(t, e.DecrementReferenceCount(9999)) + _, err := e.DecrementReferenceCount(9999) + require.Error(t, err) require.NoError(t, e.ctx.Err(), "a bogus version must not brick the manager") // Still fully usable: reads serve, and a view completes its whole lifecycle. diff --git a/sei-db/db_engine/view/shutdown_test.go b/sei-db/db_engine/view/shutdown_test.go index 28ff2db35b..5211756d58 100644 --- a/sei-db/db_engine/view/shutdown_test.go +++ b/sei-db/db_engine/view/shutdown_test.go @@ -51,6 +51,8 @@ func TestCloseWaitsForLifecycleMidCommit(t *testing.T) { case <-time.After(2 * time.Second): t.Fatal("Close did not return after the stalled commit was released") } + // The view outlives its manager, so it is given up rather than released. + view.Abandon() } // Close must release AwaitFlush waiters with errors wrapping ErrViewManagerClosed, both for a view @@ -94,6 +96,9 @@ func TestCloseUnblocksFlushWaiters(t *testing.T) { t.Fatalf("%s AwaitFlush waiter did not unblock after Close", name) } } + // The views outlive their manager, so they are given up rather than released. + unfinalized.Abandon() + finalized.Abandon() } // A Commit() call blocked on lifecycle backpressure must not outlive Close. Depending on how @@ -114,7 +119,11 @@ func TestCloseUnblocksBackpressuredCommit(t *testing.T) { blockedDone := make(chan error, 1) go func() { - _, err := manager.Commit() + view, err := manager.Commit() + if view != nil { + // A view the drain let through outlives its manager, so it is given up rather than released. + view.Abandon() + } blockedDone <- err }() select { @@ -205,6 +214,8 @@ func TestMethodsAfterCloseReportManagerClosed(t *testing.T) { _, err = manager.BatchGet([][]byte{[]byte("k")}) require.ErrorIs(t, err, ErrViewManagerClosed) + // The view outlives its manager, so it is given up rather than released. + view.Abandon() } // Every goroutine the manager owns (lifecycle runner, metrics scrape loop) must be gone once diff --git a/sei-db/db_engine/view/view_impl.go b/sei-db/db_engine/view/view_impl.go index 5e093a9c11..4d45643430 100644 --- a/sei-db/db_engine/view/view_impl.go +++ b/sei-db/db_engine/view/view_impl.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/proto" ) @@ -14,6 +15,9 @@ var _ View = (*viewImpl)(nil) type viewImpl struct { version uint64 parentManager *viewManager + + // closed records whether the view's last reservation has been released, or the view abandoned. + closed utils.CloseMarker[viewImpl] } func (s *viewImpl) Name() string { @@ -52,19 +56,21 @@ func (s *viewImpl) Reserve() error { } func (s *viewImpl) Release() error { - err := s.parentManager.DecrementReferenceCount(s.version) + lastReleased, err := s.parentManager.DecrementReferenceCount(s.version) if err != nil { + // Every failure leaves the version already dropped or the manager bricked, so nothing more is + // released through this view. + s.closed.Close(s) return fmt.Errorf("failed to decrement reference count: %w", err) } + if lastReleased { + s.closed.Close(s) + } return nil } -func (s *viewImpl) isReleased() bool { - return s.parentManager.isVersionReleased(s.version) -} - -func (s *viewImpl) releaseAll() error { - return s.parentManager.releaseAllReservations(s.version) +func (s *viewImpl) Abandon() { + s.closed.Close(s) } func (s *viewImpl) Finalize(writes []*proto.KVPair) error { diff --git a/sei-db/db_engine/view/view_lifecycle_test.go b/sei-db/db_engine/view/view_lifecycle_test.go index 4b063beceb..c03e3a771c 100644 --- a/sei-db/db_engine/view/view_lifecycle_test.go +++ b/sei-db/db_engine/view/view_lifecycle_test.go @@ -226,7 +226,7 @@ func TestCloseDoesNotFlush(t *testing.T) { // v1 is never finalized, which deterministically keeps the background flusher away from v2: // the flush frontier stops at the first unfinalized version. require.NoError(t, manager.Set([]byte("k1"), []byte("v1"))) - _, err := manager.Commit() + view1, err := manager.Commit() require.NoError(t, err) require.NoError(t, manager.Set([]byte("k2"), []byte("v2"))) @@ -238,6 +238,8 @@ func TestCloseDoesNotFlush(t *testing.T) { require.NoError(t, manager.Close()) require.False(t, db.has("k1"), "Close must not flush unfinalized views") require.False(t, db.has("k2"), "Close must not flush finalized views either") + // view1 outlives its manager, so it is given up rather than released. + view1.Abandon() } func TestCloseIsIdempotent(t *testing.T) { @@ -251,9 +253,12 @@ func TestCloseSkipsUnfinalizedView(t *testing.T) { db := newTestDB(nil) manager := newTestManagerWithDB(t, db, 1, 4096) require.NoError(t, manager.Set([]byte("k"), []byte("v"))) - _, err := manager.Commit() + view, err := manager.Commit() require.NoError(t, err) // Never finalized. require.NoError(t, manager.Close()) require.False(t, db.has("k"), "Close must not flush unfinalized views") + + // The view outlives its manager, so it is given up rather than released. + view.Abandon() } diff --git a/sei-db/db_engine/view/view_manager.go b/sei-db/db_engine/view/view_manager.go index 3e2815ee8c..7d69021f10 100644 --- a/sei-db/db_engine/view/view_manager.go +++ b/sei-db/db_engine/view/view_manager.go @@ -238,6 +238,11 @@ type View interface { // failing to release a view will stall flushes of all later views indefinitely. Release() error + // Abandon gives up on this view without releasing its reservations, so the view is not reported as + // leaked. The reservations stay held and the version stays pinned. It is for failure paths on which + // the node is going down. Idempotent. + Abandon() + // Finalize attaches this view's metadata — whatever the consumer wants recorded alongside // the block, such as its content hash. It must be called exactly once, by a consumer that // currently holds a reservation, and must return before that consumer issues its final Release diff --git a/sei-db/db_engine/view/view_manager_impl.go b/sei-db/db_engine/view/view_manager_impl.go index 1b0fe02e4c..70f5ec42aa 100644 --- a/sei-db/db_engine/view/view_manager_impl.go +++ b/sei-db/db_engine/view/view_manager_impl.go @@ -530,7 +530,7 @@ func (c *viewManager) Commit() (View, error) { c.metrics.setViewPhase("submit_diff_sort") c.materializeDiffAtVersion(sealedVersion) - utils.MustCloseE(view, "view", (*viewImpl).isReleased, (*viewImpl).releaseAll) + view.closed = utils.MustClose(view, "view") return view, nil } @@ -637,26 +637,27 @@ func (c *viewManager) IncrementReferenceCount(version uint64) error { return nil } -// Decrement the reference count for the given version. -func (c *viewManager) DecrementReferenceCount(version uint64) error { +// DecrementReferenceCount releases one reservation on version, reporting whether it was the last one. +func (c *viewManager) DecrementReferenceCount(version uint64) (lastReleased bool, err error) { c.versionLock.Lock() defer c.versionLock.Unlock() if version < c.oldestVersion { - return fmt.Errorf("version (%d) is less than the oldest version (%d)", version, c.oldestVersion) + return false, fmt.Errorf("version (%d) is less than the oldest version (%d)", version, c.oldestVersion) } if version >= c.currentVersion { - return fmt.Errorf("version (%d) must be less than the current version (%d)", version, c.currentVersion) + return false, fmt.Errorf("version (%d) must be less than the current version (%d)", + version, c.currentVersion) } counter, ok := c.versionMap[version] if !ok { // Should be impossible since version retirement never leaves gaps - return fmt.Errorf("version (%d) not found", version) + return false, fmt.Errorf("version (%d) not found", version) } if counter.referenceCount == 0 { - return fmt.Errorf("version (%d) has already been dropped", version) + return false, fmt.Errorf("version (%d) has already been dropped", version) } if counter.referenceCount == 1 && !counter.finalized { @@ -671,40 +672,14 @@ func (c *viewManager) DecrementReferenceCount(version uint64) error { // and leave manager state untouched, so that caller can retry with the right one. err := fmt.Errorf("version (%d) was fully released without first being finalized", version) c.brickLocked(err) - return err + return false, err } counter.referenceCount-- c.maybeWakeLifecycleLocked() - return nil -} - -// isVersionReleased reports whether every reservation on version has been released. Every version of a -// closed or bricked manager counts as released. -func (c *viewManager) isVersionReleased(version uint64) bool { - c.versionLock.Lock() - defer c.versionLock.Unlock() - - if c.ctx.Err() != nil { - // A shut-down manager holds nothing on a reservation's behalf. - return true - } - // A retired version is no longer in the map. - counter, ok := c.versionMap[version] - return !ok || counter.referenceCount == 0 -} - -// releaseAllReservations releases every reservation still held on version. Releasing the last one -// is subject to the same finalization check as DecrementReferenceCount(). -func (c *viewManager) releaseAllReservations(version uint64) error { - for !c.isVersionReleased(version) { - if err := c.DecrementReferenceCount(version); err != nil { - return fmt.Errorf("failed to release version (%d): %w", version, err) - } - } - return nil + return counter.referenceCount == 0, nil } // Scans for new flush eligible versions. Returns true if there is a new flush eligible version discovered. @@ -865,7 +840,7 @@ func (c *viewManager) Iterator(opts *types.IterOptions) (dbm.Iterator, error) { s.IteratorOpened() } tracked := &trackedIterator{Iterator: iter, manager: c} - utils.MustCloseE(tracked, "view manager iterator", (*trackedIterator).isClosed, (*trackedIterator).Close) + tracked.closed = utils.MustClose(tracked, "view manager iterator") return tracked, nil } @@ -875,13 +850,15 @@ type trackedIterator struct { dbm.Iterator manager *viewManager closeOnce sync.Once - closed bool + + // closed records whether Close has been called. + closed utils.CloseMarker[trackedIterator] } func (w *trackedIterator) Close() error { var err error w.closeOnce.Do(func() { - w.closed = true + w.closed.Close(w) errs := make([]error, 0, len(w.manager.shards)+1) for _, s := range w.manager.shards { errs = append(errs, s.IteratorClosed()) @@ -892,10 +869,6 @@ func (w *trackedIterator) Close() error { return err } -func (w *trackedIterator) isClosed() bool { - return w.closed -} - // MaterializeCurrentOverrides gathers the in-memory overrides at the current version from every // shard and returns them sorted ascending by key. Each shard is responsible for its own locking; // here we just stitch the results together, and the sort runs without any shard lock held. diff --git a/sei-db/state_db/giga/state_db.go b/sei-db/state_db/giga/state_db.go index 381cba7a67..a804051f1e 100644 --- a/sei-db/state_db/giga/state_db.go +++ b/sei-db/state_db/giga/state_db.go @@ -53,8 +53,8 @@ type StateDB struct { // callers serialize. commitPhases *metrics.PhaseTimer - // Set by Close(). - closed bool + // Closed by Close(). + closed utils.CloseMarker[StateDB] } // commitPhaseTimerName prefixes the instruments the commit phase breakdown is published on. @@ -84,6 +84,7 @@ func NewStateDB( commitPhases: metrics.NewPhaseTimerFactory(otel.Meter(gigaMeterName), commitPhaseTimerName). RecordLatencies().Build(), } + s.closed = utils.MustClose(s, "giga state DB") defer s.closeOnFailure(&retErr) wal, err := s.storedWALRange() @@ -108,7 +109,6 @@ func NewStateDB( if err := s.catchUpToWAL(ctx); err != nil { return nil, err } - utils.MustCloseE(s, "giga state DB", (*StateDB).isClosed, (*StateDB).Close) return s, nil } @@ -286,7 +286,7 @@ func (s *StateDB) truncateWAL(target int64) error { // How long each of the three took is logged, since each drains its own write queue and waits on the // compactions behind it, and those dominate the time a shutdown takes. func (s *StateDB) Close() error { - s.closed = true + s.closed.Close(s) var errs error var timer utils.CloseTimer if s.ss != nil { @@ -308,10 +308,6 @@ func (s *StateDB) Close() error { return errs } -func (s *StateDB) isClosed() bool { - return s.closed -} - // SC returns the state commit store. func (s *StateDB) SC() *flatkv.CommitStore { return s.sc } diff --git a/sei-db/state_db/sc/flatkv/exporter.go b/sei-db/state_db/sc/flatkv/exporter.go index 369ea8c79e..5f25fc04d5 100644 --- a/sei-db/state_db/sc/flatkv/exporter.go +++ b/sei-db/state_db/sc/flatkv/exporter.go @@ -36,6 +36,9 @@ type KVExporter struct { version int64 iter dbm.Iterator headerSent bool + + // closed records whether Close has been called. + closed utils.CloseMarker[KVExporter] } func NewKVExporter(store *CommitStore, version int64) *KVExporter { @@ -43,7 +46,7 @@ func NewKVExporter(store *CommitStore, version int64) *KVExporter { store: store, version: version, } - utils.MustCloseE(e, "flatkv exporter", (*KVExporter).isClosed, (*KVExporter).Close) + e.closed = utils.MustClose(e, "flatkv exporter") return e } @@ -89,6 +92,7 @@ func (e *KVExporter) Next() (interface{}, error) { } func (e *KVExporter) Close() error { + e.closed.Close(e) if e.iter != nil { _ = e.iter.Close() e.iter = nil @@ -100,7 +104,3 @@ func (e *KVExporter) Close() error { } return nil } - -func (e *KVExporter) isClosed() bool { - return e.store == nil -} diff --git a/sei-db/state_db/sc/flatkv/finalization_manager.go b/sei-db/state_db/sc/flatkv/finalization_manager.go index de8692312e..62320e5558 100644 --- a/sei-db/state_db/sc/flatkv/finalization_manager.go +++ b/sei-db/state_db/sc/flatkv/finalization_manager.go @@ -155,16 +155,24 @@ func (fm *FinalizationManager) enqueue(message any) error { } select { case fm.messageChan <- message: - return nil case <-fm.ctx.Done(): return fmt.Errorf("finalization manager is stopping: %w", fm.ctx.Err()) } + if fm.ctx.Err() != nil { + // The manager stopped around this send, so its exit drain may already have run and nothing else + // will take the message off the queue. + fm.discardQueued() + } + return nil } // run finalizes blocks until the manager is stopped or a block fails. It cancels the manager's context // on the way out, whatever the reason: everything waiting on this manager waits under that context, and // this goroutine is the only thing that can release it. func (fm *FinalizationManager) run() { + // Deferred first so it runs after the cancel: a message enqueued once this drain has run finds the + // manager stopped, and enqueue() drains it instead. + defer fm.discardQueued() defer fm.cancel() for { @@ -182,11 +190,30 @@ func (fm *FinalizationManager) run() { } } +// discardQueued empties the queue, abandoning the view of each block still on it. The blocks are +// unfinalized, so their reservations stay held rather than released. +func (fm *FinalizationManager) discardQueued() { + for { + select { + case message := <-fm.messageChan: + if pending, ok := message.(*pendingFinalization); ok { + pending.blockView.Abandon() + } + default: + return + } + } +} + // handle deals with one message, reporting whether the manager may continue. func (fm *FinalizationManager) handle(message any) bool { switch request := message.(type) { case *pendingFinalization: stopped, err := fm.finalize(request) + if err != nil || stopped { + // The block is dropped: a stopping manager never records it, and a failure bricks this one. + request.blockView.Abandon() + } if err != nil { fm.brick(err) return false diff --git a/sei-db/state_db/sc/flatkv/lthash/block_gatherer.go b/sei-db/state_db/sc/flatkv/lthash/block_gatherer.go index 98cbb51fb4..539b19f290 100644 --- a/sei-db/state_db/sc/flatkv/lthash/block_gatherer.go +++ b/sei-db/state_db/sc/flatkv/lthash/block_gatherer.go @@ -90,10 +90,15 @@ func (g *blockGatherer) run() { } } -// Drain the queue without hashing it, releasing each block's reservation. +// Drain the queue without hashing it and stop the combiner. func (g *blockGatherer) teardown() { defer close(g.combineJobChan) + g.discardQueued() +} +// discardQueued empties the queue without hashing it, releasing each block's reservation. It runs when +// the gatherer stops, and from the engine's enqueue() for a message that reaches the queue after that. +func (g *blockGatherer) discardQueued() { for { select { case message := <-g.scheduledBlockChan: diff --git a/sei-db/state_db/sc/flatkv/lthash/hash_engine.go b/sei-db/state_db/sc/flatkv/lthash/hash_engine.go index 2a3c5da127..3a162c4653 100644 --- a/sei-db/state_db/sc/flatkv/lthash/hash_engine.go +++ b/sei-db/state_db/sc/flatkv/lthash/hash_engine.go @@ -170,10 +170,15 @@ func (he *HashEngine) enqueue(message any) error { } select { case he.gatherer.scheduledBlockChan <- message: - return nil case <-he.ctx.Done(): return fmt.Errorf("hash engine is stopping: %w", he.ctx.Err()) } + if he.ctx.Err() != nil { + // The engine stopped around this send, so the gatherer's exit drain may already have run and + // nothing else will take the message off the queue. + he.gatherer.discardQueued() + } + return nil } // brick latches err as the engine's fatal error and stops it. diff --git a/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go b/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go index e708db644c..b52b22266a 100644 --- a/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go +++ b/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go @@ -24,7 +24,7 @@ type hashRequest struct { } // reserve takes this request's own reservation on both views, so that neither can be torn down while the -// engine still has to read it. On failure it releases whatever it took. +// engine still has to read it. On failure it gives up whatever it took. func (r *hashRequest) reserve() error { if err := r.current.Reserve(); err != nil { return fmt.Errorf("reserve block %d: %w", r.blockNumber, err) diff --git a/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go b/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go index 5971454b8c..05165d72b8 100644 --- a/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go +++ b/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go @@ -82,6 +82,7 @@ func (v *pipeView) BatchGet(keys [][]byte) (map[string][]byte, error) { func (v *pipeView) Reserve() error { v.reserves++; return nil } func (v *pipeView) Release() error { v.releases++; return nil } +func (v *pipeView) Abandon() {} func (v *pipeView) Get([]byte, bool) ([]byte, bool, error) { panic("pipeView: unexpected Get") } func (v *pipeView) Finalize([]*proto.KVPair) error { panic("pipeView: unexpected Finalize") } diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 2c505a8038..6e0dfaa20d 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -432,7 +432,8 @@ func (s *CommitStore) outOfBandSnapshot() (err error) { tmpPath, err := checkpointDatabases( s.ctx, s.flatkvDir(), blockView, s.checkpointables(), s.phaseTimer) if err != nil { - // Error is fatal; leaking reservations doesn't make it worse. + // Error is fatal; the reservation is given up rather than released. + blockView.Abandon() return fmt.Errorf("checkpoint databases at version %d: %w", version, err) } if err := blockView.Release(); err != nil { diff --git a/sei-db/state_db/sc/flatkv/snapshot_writer.go b/sei-db/state_db/sc/flatkv/snapshot_writer.go index bb2952ac04..e510ba2d2d 100644 --- a/sei-db/state_db/sc/flatkv/snapshot_writer.go +++ b/sei-db/state_db/sc/flatkv/snapshot_writer.go @@ -247,7 +247,7 @@ func (w *SnapshotWriter) enqueue(message any) error { return fmt.Errorf("enqueue to snapshot writer: %w", w.stoppedError()) } - return otelMetrics.SnapshotQueue.SendVia( + err := otelMetrics.SnapshotQueue.SendVia( func() bool { select { case w.messages <- message: @@ -265,6 +265,15 @@ func (w *SnapshotWriter) enqueue(message any) error { } }, ) + if err != nil { + return err + } + if w.ctx.Err() != nil { + // The writer stopped around this send, so its exit drain may already have run and nothing else + // will take the message off the queue. + w.discardQueued() + } + return nil } // onSnapshotInterval reports whether a committed block becomes a snapshot on this writer's own @@ -405,9 +414,8 @@ func (w *SnapshotWriter) checkpointBlock(request *snapshotRequest) error { } // discardQueued empties the queue, handing back what each snapshot request holds and answering each -// flush so its caller is not left waiting. A message enqueued after this has run is stranded, which -// only happens once the writer has stopped — the view managers are closing by then, and closing one -// releases everything it holds. +// flush so its caller is not left waiting. It runs when the writer stops, and from enqueue() for a +// message that reaches the queue after that. func (w *SnapshotWriter) discardQueued() { for { select { diff --git a/sei-db/state_db/sc/flatkv/snapshot_writer_test.go b/sei-db/state_db/sc/flatkv/snapshot_writer_test.go index 0e2d1a59cf..2349098539 100644 --- a/sei-db/state_db/sc/flatkv/snapshot_writer_test.go +++ b/sei-db/state_db/sc/flatkv/snapshot_writer_test.go @@ -84,6 +84,8 @@ func (v *fakeView) Release() error { return v.releaseErr } +func (v *fakeView) Abandon() {} + func (v *fakeView) Get([]byte, bool) ([]byte, bool, error) { panic("fakeView: unexpected Get") } diff --git a/sei-db/state_db/sc/flatkv/state_view.go b/sei-db/state_db/sc/flatkv/state_view.go index b91afdef25..c38877168b 100644 --- a/sei-db/state_db/sc/flatkv/state_view.go +++ b/sei-db/state_db/sc/flatkv/state_view.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/db_engine/view" gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" @@ -22,8 +23,8 @@ type flatKVStateView struct { // Guards the release, so a second Close does not release a reservation this view no longer owns. closeOnce sync.Once - // Set by Close. - closed bool + // Closed by Close. + closed utils.CloseMarker[flatKVStateView] } // GetBlockHeight returns the block height of this view. @@ -35,17 +36,13 @@ func (v *flatKVStateView) GetBlockHeight() int64 { // Idempotent. func (v *flatKVStateView) Close() { v.closeOnce.Do(func() { - v.closed = true + v.closed.Close(v) if err := v.blockView.Release(); err != nil { panic(fmt.Sprintf("flatkv: close state view at height %d: %v", v.blockView.BlockHeight(), err)) } }) } -func (v *flatKVStateView) isClosed() bool { - return v.closed -} - func (v *flatKVStateView) Get(module string, key []byte) ([]byte, bool) { if module != keys.EVMStoreKey { return v.miscValue(module, key) diff --git a/sei-db/state_db/sc/flatkv/state_view_physkey_test.go b/sei-db/state_db/sc/flatkv/state_view_physkey_test.go index a30c7b0a41..0b30a624f8 100644 --- a/sei-db/state_db/sc/flatkv/state_view_physkey_test.go +++ b/sei-db/state_db/sc/flatkv/state_view_physkey_test.go @@ -43,6 +43,10 @@ func (v *recordingView) Release() error { panic("unexpected call") } +func (v *recordingView) Abandon() { + panic("unexpected call") +} + func (v *recordingView) Finalize([]*proto.KVPair) error { panic("unexpected call") } diff --git a/sei-db/state_db/sc/flatkv/store_read.go b/sei-db/state_db/sc/flatkv/store_read.go index a4fc83904e..41668460be 100644 --- a/sei-db/state_db/sc/flatkv/store_read.go +++ b/sei-db/state_db/sc/flatkv/store_read.go @@ -20,7 +20,7 @@ func (s *CommitStore) OpenView() gigatypes.StateView { panic(fmt.Sprintf("flatkv: OpenView: %v", err)) } v := &flatKVStateView{blockView: blockView} - utils.MustClose(v, "flatkv state view", (*flatKVStateView).isClosed, (*flatKVStateView).Close) + v.closed = utils.MustClose(v, "flatkv state view") return v } diff --git a/sei-db/state_db/sc/flatkv/store_write.go b/sei-db/state_db/sc/flatkv/store_write.go index 0f8a68e890..ed2f46ae06 100644 --- a/sei-db/state_db/sc/flatkv/store_write.go +++ b/sei-db/state_db/sc/flatkv/store_write.go @@ -151,34 +151,40 @@ func (s *CommitStore) sealBlock( // The replay skip list: the height each database had already reached when replay started, or nil // outside replay. A database listed at or above version keeps the metadata it already has. alreadyHave map[string]int64, -) error { +) (err error) { + var blockView *sview.StoreView + var previous *sview.StoreView + defer func() { + if err != nil { + // Error is fatal; the reservations are given up rather than released. + blockView.Abandon() + previous.Abandon() + } + }() + s.phaseTimer.SetPhase("commit_seal_stores") - blockView, err := s.commitStores(version) + blockView, err = s.commitStores(version) if err != nil { return err } - previous, err := s.lastSealed.Get() + previous, err = s.lastSealed.Get() if err != nil { - // Error is fatal; leaking reservations doesn't make it worse. return fmt.Errorf("read previous block's view: %w", err) } if err := s.lastSealed.Set(blockView); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. return fmt.Errorf("install block %d: %w", version, err) } s.phaseTimer.SetPhase("commit_offer_finalization") if err := s.finalizer.Offer(version, blockView, alreadyHave); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. return err } s.phaseTimer.SetPhase("commit_schedule_hash") if err := s.hashEngine.ScheduleHash(blockView, previous); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. return err } @@ -235,6 +241,13 @@ func (s *CommitStore) commitStores(version int64) (*sview.StoreView, error) { wg.Wait() if err := errors.Join(accountErr, codeErr, storageErr, miscErr); err != nil { + // Error is fatal; the reservations the stores that did seal handed out are given up rather than + // released. + for _, dbView := range []view.View{account, code, storage, misc} { + if dbView != nil { + dbView.Abandon() + } + } return nil, err } return sview.NewStoreView(version, account, code, storage, misc) @@ -248,7 +261,8 @@ func (s *CommitStore) offerToSnapshotWriter() error { return fmt.Errorf("read latest sealed view: %w", err) } if err := s.snapshotWriter.Offer(blockView); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. + // Error is fatal; the reservation is given up rather than released. + blockView.Abandon() return err } if err := blockView.Release(); err != nil { @@ -291,7 +305,8 @@ func (s *CommitStore) flushLatestVersion() error { return fmt.Errorf("read latest sealed view: %w", err) } if err := blockView.AwaitFlush(s.ctx); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. + // Error is fatal; the reservation is given up rather than released. + blockView.Abandon() return fmt.Errorf("await flush: %w", err) } if err := blockView.Release(); err != nil { @@ -405,13 +420,15 @@ func (s *CommitStore) sealSeededVersion(seededVersion int64) error { for _, dbView := range blockView.Views() { if err := finalizeStore(dbView, seededVersion, nil, s.loadedHashes); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. + // Error is fatal; the reservations are given up rather than released. + blockView.Abandon() return fmt.Errorf("%s finalize seeded version: %w", dbView.Name(), err) } } if err := s.replaceSealedView(blockView); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. + // Error is fatal; the reservations are given up rather than released. + blockView.Abandon() return fmt.Errorf("install seeded version: %w", err) } if err := blockView.Release(); err != nil { @@ -430,13 +447,15 @@ func (s *CommitStore) sealBaseline() error { for _, dbView := range blockView.Views() { if err := dbView.Finalize(nil); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. + // Error is fatal; the reservations are given up rather than released. + blockView.Abandon() return fmt.Errorf("%s finalize baseline: %w", dbView.Name(), err) } } if err := s.replaceSealedView(blockView); err != nil { - // Error is fatal; leaking reservations doesn't make it worse. + // Error is fatal; the reservations are given up rather than released. + blockView.Abandon() return fmt.Errorf("install baseline: %w", err) } if err := blockView.Release(); err != nil { diff --git a/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go b/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go index c9ae417a1c..b998341194 100644 --- a/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go +++ b/sei-db/state_db/sc/flatkv/sview/atomic_store_view.go @@ -21,6 +21,9 @@ type AtomicStoreView struct { // The most recently installed view. Nil once closed. currentView *StoreView + + // closed records whether Close() has been called. + closed utils.CloseMarker[AtomicStoreView] } // NewAtomicStoreView() installs initialView, which must be non-nil. @@ -32,7 +35,7 @@ func NewAtomicStoreView(initialView *StoreView) (*AtomicStoreView, error) { return nil, fmt.Errorf("reserve initial view: %w", err) } asv := &AtomicStoreView{currentView: initialView} - utils.MustCloseE(asv, "atomic store view", (*AtomicStoreView).isClosed, (*AtomicStoreView).Close) + asv.closed = utils.MustClose(asv, "atomic store view") return asv, nil } @@ -97,6 +100,7 @@ func (asv *AtomicStoreView) Close() error { if asv.currentView == nil { return nil } + asv.closed.Close(asv) previous := asv.currentView asv.currentView = nil @@ -105,9 +109,3 @@ func (asv *AtomicStoreView) Close() error { } return nil } - -func (asv *AtomicStoreView) isClosed() bool { - asv.mu.RLock() - defer asv.mu.RUnlock() - return asv.currentView == nil -} diff --git a/sei-db/state_db/sc/flatkv/sview/store_view.go b/sei-db/state_db/sc/flatkv/sview/store_view.go index 68541066d3..1f1fe4b567 100644 --- a/sei-db/state_db/sc/flatkv/sview/store_view.go +++ b/sei-db/state_db/sc/flatkv/sview/store_view.go @@ -96,28 +96,42 @@ func (sv *StoreView) Views() []view.View { return sv.viewSlice } -// Reserve() takes one reservation on every store's view. A failure stops there, leaving what it -// already took held: a view manager error is unrecoverable, so the node is going down anyway. +// Reserve() takes one reservation on every store's view. A failure stops there and abandons every view, +// leaving what it already took held: a view manager error is unrecoverable, so the node is going down +// anyway. func (sv *StoreView) Reserve() error { for _, dbView := range sv.viewSlice { if err := dbView.Reserve(); err != nil { + sv.Abandon() return fmt.Errorf("reserve %s view at height %d: %w", dbView.Name(), sv.blockHeight, err) } } return nil } -// Releases one reservation on every store's view. A failure stops there, for the same reason -// Reserve() does. +// Releases one reservation on every store's view. A failure stops there and abandons every view, for +// the same reason Reserve() does. func (sv *StoreView) Release() error { for _, dbView := range sv.viewSlice { if err := dbView.Release(); err != nil { + sv.Abandon() return fmt.Errorf("release %s view at height %d: %w", dbView.Name(), sv.blockHeight, err) } } return nil } +// Abandon() gives up on every store's view without releasing its reservations. It is for failure paths +// on which the node is going down. A no-op on a nil receiver. +func (sv *StoreView) Abandon() { + if sv == nil { + return + } + for _, dbView := range sv.viewSlice { + dbView.Abandon() + } +} + // AwaitFlush() blocks until every store has written this view's block to disk. The caller must hold a // reservation across the call. func (sv *StoreView) AwaitFlush(ctx context.Context) error { diff --git a/sei-db/state_db/sc/flatkv/sview/store_view_test.go b/sei-db/state_db/sc/flatkv/sview/store_view_test.go index 9a8bbe0758..830e31ae79 100644 --- a/sei-db/state_db/sc/flatkv/sview/store_view_test.go +++ b/sei-db/state_db/sc/flatkv/sview/store_view_test.go @@ -36,6 +36,8 @@ func (s *stubView) Release() error { return s.releaseErr } +func (s *stubView) Abandon() {} + func (s *stubView) Get(key []byte, updateLru bool) ([]byte, bool, error) { panic("stubView: unexpected Get") } diff --git a/sei-db/state_db/sc/flatkv/sview/testutil_test.go b/sei-db/state_db/sc/flatkv/sview/testutil_test.go index 090d3128aa..cc79267ce3 100644 --- a/sei-db/state_db/sc/flatkv/sview/testutil_test.go +++ b/sei-db/state_db/sc/flatkv/sview/testutil_test.go @@ -62,6 +62,8 @@ func (v *fakeView) Release() error { return nil } +func (v *fakeView) Abandon() {} + func (v *fakeView) Get([]byte, bool) ([]byte, bool, error) { panic("fakeView: unexpected Get") } diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault.go b/sei-db/state_db/sc/hashvault/pebble_hashvault.go index 96063caa4a..853e227b70 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault.go @@ -27,8 +27,8 @@ type PebbleHashVault struct { writeOpts *pebble.WriteOptions mu sync.Mutex - // closed is true after Close. Every other public method returns ErrClosed once set. - closed bool + // closed records whether Close has been called. Every other public method returns ErrClosed once it has. + closed utils.CloseMarker[PebbleHashVault] // pruneBoundary is the lowest height that may still be committed. pruneBoundary uint64 cache *lru.Cache[uint64, []byte] @@ -94,9 +94,7 @@ func newPebbleHashVault(_ context.Context, config HashVaultConfig) (*PebbleHashV "dataDir", config.DataDir) } - utils.MustCloseE(p, "hashvault", (*PebbleHashVault).isClosed, func(p *PebbleHashVault) error { - return p.Close(context.Background()) - }) + p.closed = utils.MustClose(p, "hashvault") return p, nil } @@ -140,7 +138,7 @@ func (p *PebbleHashVault) CommitToHash(ctx context.Context, blockHeight uint64, p.mu.Lock() defer p.mu.Unlock() - if p.closed { + if p.closed.IsClosed() { return ErrClosed } if blockHeight < p.pruneBoundary { @@ -202,7 +200,7 @@ func (p *PebbleHashVault) Prune(ctx context.Context, blockHeight uint64) error { p.mu.Lock() defer p.mu.Unlock() - if p.closed { + if p.closed.IsClosed() { return ErrClosed } if blockHeight <= p.pruneBoundary { @@ -232,10 +230,10 @@ func (p *PebbleHashVault) Prune(ctx context.Context, blockHeight uint64) error { func (p *PebbleHashVault) Close(_ context.Context) error { p.mu.Lock() defer p.mu.Unlock() - if p.closed { + if p.closed.IsClosed() { return nil } - p.closed = true + p.closed.Close(p) p.cache.Purge() if err := p.db.Close(); err != nil { return fmt.Errorf("failed to close hashvault pebble db: %w", err) @@ -243,12 +241,6 @@ func (p *PebbleHashVault) Close(_ context.Context) error { return nil } -func (p *PebbleHashVault) isClosed() bool { - p.mu.Lock() - defer p.mu.Unlock() - return p.closed -} - func (p *PebbleHashVault) logHashMismatch(blockHeight uint64, existing, incoming []byte) { logger.Error("Hashvault detected app hash mismatch; node attempted to change its mind. "+ "DO NOT RESTART WITHOUT HUMAN INVESTIGATION. If you are CERTAIN this is not a real "+ diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 1fa6f1e15c..35b550d452 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -6,7 +6,6 @@ import ( dbm "github.com/tendermint/tm-db" - "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -23,14 +22,11 @@ type CosmosStateStore struct { db types.StateStore snapshotMgr *sssnapshot.Manager externalPruning bool - closed bool } // NewCosmosStateStore wraps an existing StateStore as a CosmosStateStore. func NewCosmosStateStore(db types.StateStore) *CosmosStateStore { - s := &CosmosStateStore{db: db} - utils.MustCloseE(s, "cosmos state store", (*CosmosStateStore).isClosed, (*CosmosStateStore).Close) - return s + return &CosmosStateStore{db: db} } func (s *CosmosStateStore) Get(storeKey string, version int64, key []byte) ([]byte, error) { @@ -102,14 +98,9 @@ func (s *CosmosStateStore) Import(version int64, ch <-chan types.SnapshotNode) e } func (s *CosmosStateStore) Close() error { - s.closed = true return s.db.Close() } -func (s *CosmosStateStore) isClosed() bool { - return s.closed -} - func (s *CosmosStateStore) SupportsCheckpoint() bool { return sssnapshot.SupportsCheckpoint(s.db) } diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 916fa40772..9d9ea1bbfd 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -48,8 +48,8 @@ type EVMStateStore struct { externalPruning bool - // Set by Close(). - closed bool + // Closed by Close(). + closed utils.CloseMarker[EVMStateStore] } // NewEVMStateStore opens either a single unified MVCC DB for all EVM state @@ -63,11 +63,11 @@ func NewEVMStateStore(dir string, ssConfig config.StateStoreConfig) (*EVMStateSt commitPhases: metrics.NewPhaseTimer(otel.Meter("seidb_ss_evm"), "ss_evm_commit"), externalPruning: ssConfig.ExternalPruning, } + store.closed = utils.MustClose(store, "EVM state store") if err := store.openDBs(); err != nil { _ = store.Close() return nil, err } - utils.MustCloseE(store, "EVM state store", (*EVMStateStore).isClosed, (*EVMStateStore).Close) return store, nil } @@ -562,17 +562,13 @@ func subDBPath(base string, storeType EVMStoreType) string { } func (s *EVMStateStore) Close() error { - s.closed = true + s.closed.Close(s) // A snapshot being published reads and stamps these databases, so it has to finish before they // close rather than race the shutdown. s.stopCheckpoints() return s.closeDBs() } -func (s *EVMStateStore) isClosed() bool { - return s.closed -} - func (s *EVMStateStore) SupportsCheckpoint() bool { for _, db := range s.managedDBs { if !sssnapshot.SupportsCheckpoint(db) { diff --git a/sei-db/state_db/ss/snapshot/manager.go b/sei-db/state_db/ss/snapshot/manager.go index 4108a18ad6..e43713713c 100644 --- a/sei-db/state_db/ss/snapshot/manager.go +++ b/sei-db/state_db/ss/snapshot/manager.go @@ -133,7 +133,9 @@ type Staged struct { version int64 tmpDir string finalDir string - finished bool + + // closed records whether the snapshot has been committed or aborted. + closed utils.CloseMarker[Staged] } func (s *Staged) Abort() { @@ -143,10 +145,6 @@ func (s *Staged) Abort() { s.manager.abort(s) } -func (s *Staged) isFinished() bool { - return s.finished -} - // SnapshotDirName returns the directory name for a snapshot labeled with the // given version. func SnapshotDirName(version int64) string { @@ -271,7 +269,7 @@ func (m *Manager) Prepare(version int64) (*Staged, error) { tmpDir: tmpDir, finalDir: finalDir, } - utils.MustClose(staged, "staged state store snapshot", (*Staged).isFinished, (*Staged).Abort) + staged.closed = utils.MustClose(staged, "staged state store snapshot") return staged, nil } @@ -289,7 +287,7 @@ func (m *Manager) Commit(staged *Staged) error { if staged == nil || staged.manager != m { return fmt.Errorf("%s staged snapshot belongs to a different manager", m.name) } - staged.finished = true + staged.closed.Close(staged) apparentBytes, sizeErr := snapshotDirApparentBytes(staged.tmpDir) if sizeErr != nil { logger.Error("failed to measure state store snapshot", @@ -336,7 +334,7 @@ func (m *Manager) abort(staged *Staged) { if staged == nil || staged.manager != m { return } - staged.finished = true + staged.closed.Close(staged) if err := os.RemoveAll(staged.tmpDir); err != nil { logger.Error("failed to remove aborted state store snapshot", "store", m.name, "dir", staged.tmpDir, "error", err) diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index 1e8b412e2a..654a7bd53d 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -21,13 +21,13 @@ type stateWALImpl struct { // The underlying generic WAL, keyed by block number, whose payload is a block's changesets. wal seiwal.WAL[[]*proto.NamedChangeSet] - // Set by Close() so subsequent calls fail fast. A plain field: like the write-ordering state below, it - // is only ever touched by the single caller, which must not invoke methods concurrently. - closed bool + // Closed by Close() so subsequent calls fail fast. + closed utils.CloseMarker[stateWALImpl] // The first fatal error from the underlying WAL that bricked this one, surfaced to the caller by every // subsequent operation. Once set, no operation touches the underlying WAL, so a corrupt WAL never - // limps onward. Caller-serialized like closed. + // limps onward. A plain field: like hasBlock, it is only ever touched by the single caller, which must + // not invoke methods concurrently. fatalErr error // The highest block number written. Atomic because the garbage collector reads it off-goroutine @@ -38,7 +38,7 @@ type stateWALImpl struct { lastBlock atomic.Uint64 // Whether any block has been written (this session or recovered from disk), which is what tells an - // empty WAL apart from one holding only block 0. Caller-serialized like closed. + // empty WAL apart from one holding only block 0. Caller-serialized like fatalErr. hasBlock bool } @@ -114,13 +114,13 @@ func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { w.lastBlock.Store(last) w.hasBlock = true } - utils.MustCloseE(w, "state WAL", (*stateWALImpl).isClosed, (*stateWALImpl).Close) + w.closed = utils.MustClose(w, "state WAL") return w, nil } // Write appends a block's changesets to the WAL as a single record. func (w *stateWALImpl) Write(blockNumber uint64, cs []*proto.NamedChangeSet) error { - if w.closed { + if w.closed.IsClosed() { return fmt.Errorf("state WAL is closed") } if w.fatalErr != nil { @@ -162,7 +162,7 @@ func (w *stateWALImpl) checkBlockOrder(blockNumber uint64) error { // Flush blocks until all previously scheduled writes are durable. func (w *stateWALImpl) Flush() error { - if w.closed { + if w.closed.IsClosed() { return fmt.Errorf("state WAL is closed") } if w.fatalErr != nil { @@ -176,7 +176,7 @@ func (w *stateWALImpl) Flush() error { // GetStoredRange reports the range of complete blocks stored in the WAL. func (w *stateWALImpl) GetStoredRange() (bool, uint64, uint64, error) { - if w.closed { + if w.closed.IsClosed() { return false, 0, 0, fmt.Errorf("state WAL is closed") } if w.fatalErr != nil { @@ -192,7 +192,7 @@ func (w *stateWALImpl) GetStoredRange() (bool, uint64, uint64, error) { // Prune schedules removal of whole underlying files below lowestBlockNumberToKeep. It does not block on // completion. func (w *stateWALImpl) Prune(lowestBlockNumberToKeep uint64) error { - if w.closed { + if w.closed.IsClosed() { return fmt.Errorf("state WAL is closed") } if w.fatalErr != nil { @@ -210,7 +210,7 @@ func (w *stateWALImpl) Iterator( startingBlockNumber uint64, endingBlockNumber uint64, ) (seiwal.Iterator[[]*proto.NamedChangeSet], error) { - if w.closed { + if w.closed.IsClosed() { return nil, fmt.Errorf("state WAL is closed") } if w.fatalErr != nil { @@ -229,17 +229,13 @@ func (w *stateWALImpl) Iterator( // Close flushes pending writes, closes the underlying WAL, and releases resources. func (w *stateWALImpl) Close() error { - w.closed = true + w.closed.Close(w) if err := w.wal.Close(); err != nil { return fmt.Errorf("failed to close state WAL: %w", err) } return nil } -func (w *stateWALImpl) isClosed() bool { - return w.closed -} - // fail records err as the first fatal error that bricks the WAL and returns it. Once set, every // subsequent operation fails fast rather than touching the underlying WAL. func (w *stateWALImpl) fail(err error) error { From 5945a330c00700222e26de106d2b6ef3eb0b1beb Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 28 Sep 2026 14:47:18 -0500 Subject: [PATCH 4/4] fix test leak --- sei-db/state_db/ss/snapshot/manager_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sei-db/state_db/ss/snapshot/manager_test.go b/sei-db/state_db/ss/snapshot/manager_test.go index 010e137d71..b8e304fb7b 100644 --- a/sei-db/state_db/ss/snapshot/manager_test.go +++ b/sei-db/state_db/ss/snapshot/manager_test.go @@ -230,6 +230,8 @@ func TestManagerPrepareRefusesExistingStagingDir(t *testing.T) { _, err := manager.Prepare(10) require.Error(t, err) require.DirExists(t, staged.tmpDir, "the first attempt's checkpoint must survive the refusal") + + staged.Abort() } func openManager(t *testing.T, root string, scheduler *controlledScheduler, keepRecent int, external bool) *Manager {