diff --git a/base/bootstrap.go b/base/bootstrap.go index a43a2772bc..2b9cb6be56 100644 --- a/base/bootstrap.go +++ b/base/bootstrap.go @@ -86,6 +86,9 @@ type BootstrapConnection interface { // information about whether a new DB on this bucket has opted in (e.g., PUT // with // use_system_metadata_collection: true) so the very first bootstrap doc lands correctly. SetBucketBootstrapTargetHint(ctx context.Context, bucket string, optInHint bool) error + // CachedBootstrapTargets returns the cached bootstrap doc target for each bucket, for observability purposes. Values are "system_mobile", "default". + // The snapshot is not guaranteed to be consistent across concurrent updates. + CachedBootstrapTargets() map[string]string // Close releases any long-lived connections Close() } @@ -116,9 +119,9 @@ type CouchbaseCluster struct { useSystemMetadataCollection bool // When true, bootstrap metadata is stored in _system._mobile, with read-fallback to _default._default during migration migrationComplete atomic.Bool // When set, fallback reads are skipped even if useSystemMetadataCollection is true // bucketBootstrapTargets caches the resolved bootstrap-doc location for each bucket. - // Resolved lazily on first interaction (or eagerly via SetBucketBootstrapTargetHint). Values - // are bucketBootstrapTarget; absence of an entry means "fall back to the connection-wide flag." - bucketBootstrapTargets sync.Map + // Resolved lazily on first interaction (or eagerly via SetBucketBootstrapTargetHint). + // Absence of an entry means "fall back to the connection-wide flag." + bucketBootstrapTargets SyncMap[string, bucketBootstrapTarget] useGOCBFastFailRetry bool // When true, readiness checks fail fast instead of using the best-effort retry strategy } @@ -131,6 +134,16 @@ const ( bucketTargetSystemMobile // _system._mobile primary; _default._default fallback until migration complete ) +func (t bucketBootstrapTarget) String() string { + switch t { + case bucketTargetSystemMobile: + return "_system._mobile" + case bucketTargetDefault: + return "_default._default" + } // exhaustive:enforce + return "" +} + type BucketConnectionMode int const ( @@ -372,7 +385,7 @@ func (cc *CouchbaseCluster) metadataCollections(b *gocb.Bucket) (primary, fallba cached, hasCachedTarget := cc.bucketBootstrapTargets.Load(b.Name()) useSystemMobile := cc.useSystemMetadataCollection if hasCachedTarget { - useSystemMobile = cached.(bucketBootstrapTarget) == bucketTargetSystemMobile + useSystemMobile = cached == bucketTargetSystemMobile } if !useSystemMobile { // When this bucket has any opt-in indication — either cached as bucketTargetDefault @@ -399,7 +412,7 @@ func (cc *CouchbaseCluster) metadataCollections(b *gocb.Bucket) (primary, fallba // the bucket-level migration). A no-op when the cache already says _system._mobile, since the // systemMobile→default fallback direction is the legitimate in-progress-migration legacy path. func (cc *CouchbaseCluster) noteBucketFallbackHit(bucketName string) { - if cached, ok := cc.bucketBootstrapTargets.Load(bucketName); ok && cached.(bucketBootstrapTarget) == bucketTargetSystemMobile { + if cached, ok := cc.bucketBootstrapTargets.Load(bucketName); ok && cached == bucketTargetSystemMobile { return } cc.bucketBootstrapTargets.Store(bucketName, bucketTargetSystemMobile) @@ -436,6 +449,20 @@ func (cc *CouchbaseCluster) SetBucketBootstrapTargetHint(ctx context.Context, bu return nil } +// CachedBootstrapTargets returns the cached bootstrap doc target for each bucket, for observability purposes. Values are "system_mobile", "default". +// The snapshot is not guaranteed to be consistent across concurrent updates. +func (cc *CouchbaseCluster) CachedBootstrapTargets() map[string]string { + targets := make(map[string]string) + for key, value := range cc.bucketBootstrapTargets.Range { + if s := value.String(); s != "" { + targets[key] = s + } else { + targets[key] = "unknown" + } + } + return targets +} + // probeRegistryLocation checks both collections for an existing _sync:registry doc. Returns // (target, found): when found, target identifies the collection; when not found, target is // undefined and the caller picks based on its own policy (cluster flag or per-DB hint). diff --git a/base/dual_metadata_store.go b/base/dual_metadata_store.go index 69f647e40c..b3ac7a9d57 100644 --- a/base/dual_metadata_store.go +++ b/base/dual_metadata_store.go @@ -62,6 +62,32 @@ func (ms *MetadataStore) MigrationComplete() bool { return ms.migrationComplete.Load() } +// MetadataStoreMode classifies a database's metadata store for support / observability. +type MetadataStoreMode string + +const ( + // MetadataStoreModeFallbackActive means the dual store is wrapping primary+fallback and + // fallback reads are still in play (migration not yet complete). + MetadataStoreModeFallbackActive MetadataStoreMode = "fallback_active" + // MetadataStoreModeFallbackInactive means the dual store is wrapping primary+fallback but + // migration has completed, so reads no longer fall back. + MetadataStoreModeFallbackInactive MetadataStoreMode = "fallback_inactive" +) + +// GetMetadataStoreMode classifies the given datastore. Returns an empty string when the +// datastore isn't a *MetadataStore (single-store databases); callers should use JSON +// omitempty so the field is dropped in that case rather than emitted as "". +func GetMetadataStoreMode(ds DataStore) MetadataStoreMode { + ms, ok := ds.(*MetadataStore) + if !ok { + return "" + } + if ms.MigrationComplete() { + return MetadataStoreModeFallbackInactive + } + return MetadataStoreModeFallbackActive +} + // readFromFallback returns true when err indicates the key was not found in primary and metadata migration has // not yet complete, meaning the operation should be retried against the fallback DataStore. func (ms *MetadataStore) readFromFallback(ctx context.Context, err error) bool { diff --git a/base/rosmar_cluster.go b/base/rosmar_cluster.go index b208440632..e1cedda3b9 100644 --- a/base/rosmar_cluster.go +++ b/base/rosmar_cluster.go @@ -16,7 +16,6 @@ import ( "os" "runtime" "strings" - "sync" "sync/atomic" sgbucket "github.com/couchbase/sg-bucket" @@ -33,7 +32,7 @@ type RosmarCluster struct { migrationComplete atomic.Bool // When set, fallback reads are skipped even if useSystemMetadataCollection is true // bucketBootstrapTargets caches the resolved bootstrap-doc target for each bucket. // Mirrors CouchbaseCluster.bucketBootstrapTargets — see that comment for the decision tree. - bucketBootstrapTargets sync.Map + bucketBootstrapTargets SyncMap[string, bucketBootstrapTarget] } // NewRosmarCluster creates a from a given URL. useSystemMetadataCollection mirrors @@ -162,7 +161,7 @@ func (c *RosmarCluster) metadataDataStores(ctx context.Context, bucketName strin cached, hasCachedTarget := c.bucketBootstrapTargets.Load(bucketName) useSystemMobile := c.useSystemMetadataCollection if hasCachedTarget { - useSystemMobile = cached.(bucketBootstrapTarget) == bucketTargetSystemMobile + useSystemMobile = cached == bucketTargetSystemMobile } if !useSystemMobile { // Reads/writes route to default first. When this bucket has any opt-in indication — @@ -197,7 +196,7 @@ func (c *RosmarCluster) metadataDataStores(ctx context.Context, bucketName strin // the bucket-level migration). A no-op when the cache already says _system._mobile, since the // systemMobile→default fallback direction is the legitimate in-progress-migration legacy path. func (c *RosmarCluster) noteBucketFallbackHit(bucketName string) { - if cached, ok := c.bucketBootstrapTargets.Load(bucketName); ok && cached.(bucketBootstrapTarget) == bucketTargetSystemMobile { + if cached, ok := c.bucketBootstrapTargets.Load(bucketName); ok && cached == bucketTargetSystemMobile { return } c.bucketBootstrapTargets.Store(bucketName, bucketTargetSystemMobile) @@ -672,6 +671,20 @@ func (c *RosmarCluster) MigrateBootstrapDocs(ctx context.Context, bucket string, return nil } +// CachedBootstrapTargets returns the cached bootstrap doc target for each bucket, for observability purposes. Values are "system_mobile", "default". +// The snapshot is not guaranteed to be consistent across concurrent updates. +func (c *RosmarCluster) CachedBootstrapTargets() map[string]string { + targets := make(map[string]string) + for key, value := range c.bucketBootstrapTargets.Range { + if s := value.String(); s != "" { + targets[key] = s + } else { + targets[key] = "unknown" + } + } + return targets +} + // Close calls teardown for any cached buckets and removes from cachedBucketConnections func (c *RosmarCluster) Close() { } diff --git a/base/stats.go b/base/stats.go index 5d775698eb..f9f186136d 100644 --- a/base/stats.go +++ b/base/stats.go @@ -928,6 +928,13 @@ type MigrationStats struct { SeqPoisonPillApplied *SgwIntStat `json:"seq_poison_pill_applied"` // Cumulative count of MigrateMetadata pass invocations. Passes *SgwIntStat `json:"passes"` + // AbandonedRuns counts metadata migration runs that the orchestrator gave up on after the + // bounded pass loop exhausted itself without a clean pass (zero unknown-prefix remaining + // AND zero per-doc errors on the same pass). Runs that exit early via a hard error from + // MigrateMetadata, or are stopped cooperatively via the terminator, are not counted here — + // this stat is specifically the "we hit the retry ceiling" failure mode, useful for + // distinguishing transient/abortive errors from buckets that need operator intervention. + AbandonedRuns *SgwIntStat `json:"abandoned_runs"` } type SgwStatWrapper interface { @@ -2577,6 +2584,10 @@ func (d *DbStats) InitMigrationStats() error { if err != nil { return err } + resUtil.AbandonedRuns, err = NewIntStat(SubsystemMetadataMigration, "abandoned_runs", StatUnitNoUnits, MetadataMigrationAbandonedRunsDesc, StatAddedVersion4dot1dot0, StatDeprecatedVersionNotDeprecated, StatStabilityCommitted, labelKeys, labelVals, prometheus.CounterValue, 0) + if err != nil { + return err + } d.MigrationStats = resUtil return nil } @@ -2589,6 +2600,7 @@ func (d *DbStats) unregisterMigrationStats() { prometheus.Unregister(d.MigrationStats.Errors) prometheus.Unregister(d.MigrationStats.SeqPoisonPillApplied) prometheus.Unregister(d.MigrationStats.Passes) + prometheus.Unregister(d.MigrationStats.AbandonedRuns) } func (d *DbStats) MetadataMigration() *MigrationStats { diff --git a/base/stats_descriptions.go b/base/stats_descriptions.go index a9e37170a8..6d10b43891 100644 --- a/base/stats_descriptions.go +++ b/base/stats_descriptions.go @@ -426,6 +426,13 @@ const ( MetadataMigrationSeqPoisonPillAppliedDesc = "The total number of times this node applied the seq-counter poison pill to initiate fallback→primary sequence handoff. Typically 0 or 1 per migration run." MetadataMigrationPassesDesc = "The total number of MigrateMetadata range-scan passes executed for this database." + + MetadataMigrationAbandonedRunsDesc = "The total number of metadata migration runs that the " + + "orchestrator gave up on after the bounded pass loop exhausted itself without a clean pass. " + + "Runs that exit early via a hard error from MigrateMetadata, or are stopped cooperatively via " + + "the terminator, are not counted here. This stat specifically captures the \"hit the retry " + + "ceiling\" failure mode, useful for distinguishing transient/abortive errors from buckets that " + + "need operator intervention." ) // DB Replicators stats descriptions (ISGR Specific) diff --git a/base/sync_map.go b/base/sync_map.go new file mode 100644 index 0000000000..e827c9e0d4 --- /dev/null +++ b/base/sync_map.go @@ -0,0 +1,50 @@ +/* +Copyright 2026-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package base + +import ( + "iter" + "sync" +) + +// SyncMap is a type-safe wrapper around sync.Map that eliminates runtime type assertions. +type SyncMap[K comparable, V any] struct { + m sync.Map +} + +func (s *SyncMap[K, V]) Load(key K) (V, bool) { + v, ok := s.m.Load(key) + if !ok { + var zero V + return zero, false + } + return v.(V), true +} + +func (s *SyncMap[K, V]) Store(key K, value V) { + s.m.Store(key, value) +} + +func (s *SyncMap[K, V]) LoadOrStore(key K, value V) (V, bool) { + actual, loaded := s.m.LoadOrStore(key, value) + return actual.(V), loaded +} + +func (s *SyncMap[K, V]) Range(yield func(K, V) bool) { + s.m.Range(func(key, value any) bool { + return yield(key.(K), value.(V)) + }) +} + +// All returns an iterator over all key-value pairs. +func (s *SyncMap[K, V]) All() iter.Seq2[K, V] { + return s.Range +} diff --git a/db/background_mgr_metadata_migration.go b/db/background_mgr_metadata_migration.go index b8a14c897a..b9ad0cd6f7 100644 --- a/db/background_mgr_metadata_migration.go +++ b/db/background_mgr_metadata_migration.go @@ -242,6 +242,9 @@ func (m *MetadataMigrationManager) Run(ctx context.Context, options map[string]a if pass+1 >= maxPasses { base.WarnfCtx(ctx, "[%s] gave up after %d passes with %d unknown-prefix doc(s) and %d per-doc error(s) on the last pass", metadataMigrationLoggingID, maxPasses, remaining, passErrors) + if promStats != nil { + promStats.AbandonedRuns.Add(1) + } return fmt.Errorf("%s still not clear of metadata after %d passes: %d unknown-prefix doc(s), %d per-doc error(s) remain", ms.Fallback().GetName(), maxPasses, remaining, passErrors) } } diff --git a/db/util_testing.go b/db/util_testing.go index cfd7be1732..436091a5ac 100644 --- a/db/util_testing.go +++ b/db/util_testing.go @@ -1191,3 +1191,10 @@ func (db *DatabaseContext) RestartChangeListener(t testing.TB, flushCache bool) func (db *DatabaseContext) FlushChannelCache(t testing.TB) { db.RestartChangeListener(t, true) } + +// MigrateSeqCounterForTest exposes the unexported migrateSeqCounter for cross-package tests. +func MigrateSeqCounterForTest(t testing.TB, ctx context.Context, ms *base.MetadataStore, seqKey string) { + t.Helper() + stats := &MigrationStats{} + require.NoError(t, migrateSeqCounter(ctx, ms, seqKey, stats)) +} diff --git a/docs/api/components/schemas.yaml b/docs/api/components/schemas.yaml index 25a49cdd1d..b9301306d1 100644 --- a/docs/api/components/schemas.yaml +++ b/docs/api/components/schemas.yaml @@ -3044,6 +3044,18 @@ Status: host: description: The nodes host name. type: string + metadata_store_mode: + type: string + enum: + - fallback_active + - fallback_inactive + x-enumDescriptions: + fallback_active: Dual store is in place and reads fall back to `_default._default` when not found in `_system._mobile` (migration not yet complete). + fallback_inactive: Dual store is in place but migration has completed and fallback reads are skipped. + description: |- + Classification of this database's dual metadata store. Enabled via the + `use_system_metadata_collection` option in either the database or bootstrap + configuration. Omitted when this database does not use a dual metadata store. version: description: |- The product version including the build number and edition (ie. `EE` or `CE`). @@ -3512,9 +3524,102 @@ BucketInfo: properties: registry: $ref: '#/GatewayRegistry' + migration_status: + allOf: + - $ref: '#/MetadataMigrationStatus' + description: |- + Bucket-level metadata-migration status. Omitted when the status doc does not yet exist + on the bucket (no migration has been observed or initiated). enable_cross_cluster_versioning: description: Indicates if cross-cluster versioning is enabled for this bucket. type: boolean + bootstrap_target: + type: string + enum: + - _system._mobile + - _default._default + - unknown + x-enumDescriptions: + _system._mobile: Bootstrap docs live in the `_system._mobile` collection (opted-in or migrated). + _default._default: Bootstrap docs live in the `_default._default` collection (legacy registry or no opt-in). + unknown: Target was cached but could not be resolved to a known collection. + description: |- + Cached location of this bucket's bootstrap documents (registry, dbconfig, cbgt cfg) on + this node. Omitted when the target has not yet been resolved on this node. + +MetadataMigrationStatus: + type: object + description: |- + Per-bucket metadata-migration lifecycle. One document per bucket, tracking bucket-global + bootstrap state and per-database completion. Per-database entries are keyed by metadata ID + so database renames do not break tracking. + properties: + bucket_migration_id: + type: string + description: Stable identifier for this bucket's migration lifecycle. + databases: + type: object + description: Map of database metadata ID to per-database migration state. + additionalProperties: + x-additionalPropertiesName: metadataid + $ref: '#/DatabaseMigrationStatus' + bootstrap: + $ref: '#/BootstrapMigrationStatus' + +DatabaseMigrationStatus: + type: object + description: Per-database entry in the bucket migration status document. + properties: + state: + type: string + enum: + - pending + - in_progress + - complete + x-enumDescriptions: + pending: Migration has not yet started for this database. + in_progress: A node currently holds the BackgroundManager heartbeat lease and is actively migrating this database. + complete: All metadata for this database has been migrated to `_system._mobile`. + description: Per-database migration state. + started_at: + type: string + format: date-time + description: Time the per-database migration was first observed in progress. + completed_at: + type: string + format: date-time + description: Time the per-database migration reached the complete state. + +BootstrapMigrationStatus: + type: object + description: |- + Bucket-global bootstrap migration state covering `_sync:registry`, `_sync:dbconfig:*`, and + cbgt cfg documents. `attempts` and `last_attempted_at` are observability fields written by + every node that runs the migration loop, so operators can distinguish "no one has tried" from + repeated failure. + properties: + state: + type: string + enum: + - pending + - complete + x-enumDescriptions: + pending: Bootstrap migration has not yet completed; one or more per-database entries are still pending or in progress. + complete: All per-database migrations are complete and bootstrap docs have been moved to `_system._mobile`. + description: |- + Bucket-level bootstrap state. Transitions from `pending` to `complete` only after every + per-database entry is complete; it never reports `in_progress`. + completed_at: + type: string + format: date-time + description: Time the bootstrap migration reached the complete state. + last_attempted_at: + type: string + format: date-time + description: Time the migration loop most recently ran on this bucket, regardless of outcome. + attempts: + type: integer + description: Number of times the migration loop has run on this bucket. GatewayRegistry: type: object diff --git a/rest/admin_api.go b/rest/admin_api.go index 492200bd1e..f356d8c4c1 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -1665,6 +1665,7 @@ type DatabaseStatus struct { RequireResync []string `json:"require_resync"` ReplicationStatus []*db.ReplicationStatus `json:"replication_status"` SGRCluster *db.SGRCluster `json:"cluster"` + MetadataStoreMode base.MetadataStoreMode `json:"metadata_store_mode,omitempty"` } type RuntimeStatus struct { @@ -1728,6 +1729,7 @@ func (h *handler) handleGetStatus() error { ReplicationStatus: replicationsStatus, SGRCluster: cluster, RequireResync: database.RequireResync.ScopeAndCollectionNames(), + MetadataStoreMode: base.GetMetadataStoreMode(database.MetadataStore), } } @@ -2465,8 +2467,10 @@ type ClusterInfo struct { } type BucketInfo struct { - Registry *GatewayRegistry `json:"registry,omitempty"` - EnableCrossClusterVersioning bool `json:"enable_cross_cluster_versioning"` + Registry *GatewayRegistry `json:"registry,omitempty"` + MigrationStatus *base.MetadataMigrationStatus `json:"migration_status,omitempty"` + EnableCrossClusterVersioning bool `json:"enable_cross_cluster_versioning"` + BootstrapTarget string `json:"bootstrap_target,omitempty"` } // Get SG cluster information. Iterates over all buckets associated with the server, and returns cluster @@ -2500,6 +2504,22 @@ func (h *handler) handleGetClusterInfo() error { Registry: registry, EnableCrossClusterVersioning: eccv[bucketName], } + + // Grab metadata migration status for bucket, if it exists, and add to response + migrationStatus, _, err := h.server.BootstrapContext.Connection.GetMetadataMigrationStatus(h.ctx(), bucketName) + if err != nil && !base.IsDocNotFoundError(err) { + base.InfofCtx(h.ctx(), base.KeyAll, "Unable to retrieve metadata migration status for bucket %s during /_cluster_info: %v. Returning response without this information.", base.MD(bucketName), err) + } else { + bucketInfo.MigrationStatus = migrationStatus + } + + // If there's a cached bootstrap target for this bucket, add it to the response. getGatewayRegistry above + // can populate the cache, so we need to refresh for each iteration. + cachedTargets := h.server.BootstrapContext.Connection.CachedBootstrapTargets() + if target, ok := cachedTargets[bucketName]; ok { + bucketInfo.BootstrapTarget = target + } + clusterInfo.Buckets[bucketName] = bucketInfo } } else { diff --git a/rest/adminapitest/admin_api_test.go b/rest/adminapitest/admin_api_test.go index b33b1eb1b7..098995c2e7 100644 --- a/rest/adminapitest/admin_api_test.go +++ b/rest/adminapitest/admin_api_test.go @@ -4189,3 +4189,76 @@ func RequireEventCount(t *testing.T, runtimeConfig *rest.RuntimeDatabaseConfig, } require.Equal(t, expectedCount, actualCount) } + +func TestRetrieveMetadataMigrationStatusInClusterInfo(t *testing.T) { + rt := rest.NewRestTesterPersistentConfig(t) + defer rt.Close() + + metadID := rt.GetDatabase().Options.MetadataID + + resp := rt.SendAdminRequest(http.MethodGet, "/_cluster_info", "") + rest.RequireStatus(t, resp, http.StatusOK) + + var clusterInfoResponse rest.ClusterInfo + err := json.Unmarshal(resp.BodyBytes(), &clusterInfoResponse) + require.NoError(t, err) + assert.Nil(t, clusterInfoResponse.Buckets[rt.Bucket().GetName()].MigrationStatus) + + // update doc to trigger migration status update + dbCfg := rt.NewDbConfig() + dbCfg.UseSystemMobileMetadataCollection = base.Ptr(true) + rest.RequireStatus(t, rt.UpsertDbConfig("db", dbCfg), http.StatusCreated) + + // flush to ensure config is applied before starting migration + rt.ServerContext().ForceClusterCompatRefresh(t, rt.Context()) + + // start migration + resp = rt.SendAdminRequest(http.MethodPost, "/{{.db}}/_metadata_migration?action=start", "") + rest.RequireStatus(t, resp, http.StatusOK) + rt.WaitForMetadataMigrationStatus(db.BackgroundProcessStateCompleted) + + resp = rt.SendAdminRequest(http.MethodGet, "/_cluster_info", "") + rest.RequireStatus(t, resp, http.StatusOK) + err = json.Unmarshal(resp.BodyBytes(), &clusterInfoResponse) + require.NoError(t, err) + require.NotNil(t, clusterInfoResponse.Buckets[rt.Bucket().GetName()].MigrationStatus) + assert.Equal(t, base.MigrationStateComplete, clusterInfoResponse.Buckets[rt.Bucket().GetName()].MigrationStatus.Databases[metadID].State) +} + +// TestRetrieveMetadataStoreModeInStatus tests the three-state metadata_store_mode signal on /_status: +// - default DB: not a dual store, field omitted (empty string post-unmarshal) +// - after opt-in, pre-migration: dual store with fallback reads active +// - after migration completes: dual store with fallback reads inactive +func TestRetrieveMetadataStoreModeInStatus(t *testing.T) { + rt := rest.NewRestTesterPersistentConfig(t) + defer rt.Close() + + resp := rt.SendAdminRequest(http.MethodGet, "/_status", "") + rest.RequireStatus(t, resp, http.StatusOK) + + var statusResponse rest.Status + require.NoError(t, json.Unmarshal(resp.BodyBytes(), &statusResponse)) + require.Contains(t, statusResponse.Databases, "db") + assert.Empty(t, statusResponse.Databases["db"].MetadataStoreMode, "default DB should not report a dual-store mode") + + // Opt in to the dual metadata store but do not yet start the migration. + dbCfg := rt.NewDbConfig() + dbCfg.UseSystemMobileMetadataCollection = base.Ptr(true) + rest.RequireStatus(t, rt.UpsertDbConfig("db", dbCfg), http.StatusCreated) + rt.ServerContext().ForceClusterCompatRefresh(t, rt.Context()) + + resp = rt.SendAdminRequest(http.MethodGet, "/_status", "") + rest.RequireStatus(t, resp, http.StatusOK) + require.NoError(t, json.Unmarshal(resp.BodyBytes(), &statusResponse)) + assert.Equal(t, base.MetadataStoreModeFallbackActive, statusResponse.Databases["db"].MetadataStoreMode) + + // Run the migration to completion — mode should flip to fallback_inactive. + resp = rt.SendAdminRequest(http.MethodPost, "/{{.db}}/_metadata_migration?action=start", "") + rest.RequireStatus(t, resp, http.StatusOK) + rt.WaitForMetadataMigrationStatus(db.BackgroundProcessStateCompleted) + + resp = rt.SendAdminRequest(http.MethodGet, "/_status", "") + rest.RequireStatus(t, resp, http.StatusOK) + require.NoError(t, json.Unmarshal(resp.BodyBytes(), &statusResponse)) + assert.Equal(t, base.MetadataStoreModeFallbackInactive, statusResponse.Databases["db"].MetadataStoreMode) +} diff --git a/rest/metadatamigrationtest/metadata_migration_test.go b/rest/metadatamigrationtest/metadata_migration_test.go index 756d8fe5f4..28450dc3a4 100644 --- a/rest/metadatamigrationtest/metadata_migration_test.go +++ b/rest/metadatamigrationtest/metadata_migration_test.go @@ -614,6 +614,23 @@ func TestMetadataMigrationListsPrincipalsAfterCompletion(t *testing.T) { assert.Contains(t, resp.Body.String(), "observer", "roles must be listable after migration completes") } +// TestBootstrapTargetOmittedWhenNoCachedValue verifies that bootstrap_target is omitted from the +// /_cluster_info response when no database has been created on the bucket. Without a database, no +// _sync:registry exists, so probeRegistryLocation finds nothing authoritative and the per-bucket +// target cache stays empty — matching the window between node startup and the first DB creation. +func TestBootstrapTargetOmittedWhenNoCachedValue(t *testing.T) { + rt := rest.NewRestTesterPersistentConfigNoDB(t) + defer rt.Close() + + var clusterInfoResponse rest.ClusterInfo + resp := rt.SendAdminRequest(http.MethodGet, "/_cluster_info", "") + rest.RequireStatus(t, resp, http.StatusOK) + assert.NotContains(t, resp.Body.String(), "bootstrap_target", "cluster info must not be empty") + err := base.JSONUnmarshal(resp.BodyBytes(), &clusterInfoResponse) + require.NoError(t, err) + assert.Empty(t, clusterInfoResponse.Buckets[rt.Bucket().GetName()].BootstrapTarget) +} + // TestMetadataMigrationEndToEndBucketComplete is a full end-to-end check of the bucket-level // completion handoff: migrate a single database's metadata (a user doc), then assert that once // it is the last database in the bucket to finish, PostMetadataMigrationCompleteFunc migrates @@ -630,6 +647,13 @@ func TestMetadataMigrationEndToEndBucketComplete(t *testing.T) { resp := rt.CreateDatabase("db", dbConfig) rest.RequireStatus(t, resp, http.StatusCreated) + var clusterInfoResponse rest.ClusterInfo + resp = rt.SendAdminRequest(http.MethodGet, "/_cluster_info", "") + rest.RequireStatus(t, resp, http.StatusOK) + err := base.JSONUnmarshal(resp.BodyBytes(), &clusterInfoResponse) + require.NoError(t, err) + assert.Equal(t, "_default._default", clusterInfoResponse.Buckets[rt.Bucket().GetName()].BootstrapTarget) + resp = rt.SendAdminRequest(http.MethodPut, "/{{.db}}/_user/alice", `{"name":"alice","password":"letmein","admin_channels":["public"]}`) rest.RequireStatus(t, resp, http.StatusCreated) @@ -667,6 +691,13 @@ func TestMetadataMigrationEndToEndBucketComplete(t *testing.T) { assert.Equal(c, base.MigrationStateComplete, entry.State, "per-DB migration entry should be complete") assert.Equal(c, base.MigrationStateComplete, status.Bootstrap.State, "bucket bootstrap migration should be complete once the last DB finishes") }, 30*time.Second, 200*time.Millisecond) + + clusterInfoResponse = rest.ClusterInfo{} + resp = rt.SendAdminRequest(http.MethodGet, "/_cluster_info", "") + rest.RequireStatus(t, resp, http.StatusOK) + err = base.JSONUnmarshal(resp.BodyBytes(), &clusterInfoResponse) + require.NoError(t, err) + assert.Equal(t, "_system._mobile", clusterInfoResponse.Buckets[rt.Bucket().GetName()].BootstrapTarget) } // TestMetadataMigrationRESTStartRejectedWithoutOptIn verifies a manual start via the admin API is diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index 919499dd18..b2c981530b 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -448,6 +448,29 @@ func (rt *RestTester) waitForResyncDCPStatus(status db.BackgroundProcessState, d return resyncStatus } +func (rt *RestTester) WaitForMetadataMigrationStatus(status db.BackgroundProcessState) db.MigrationManagerResponse { + return rt.waitForMetadataMigrationStatus(status, "{{.db}}") +} + +func (rt *RestTester) waitForMetadataMigrationStatus(status db.BackgroundProcessState, dbName string) db.MigrationManagerResponse { + timeout := 10 * time.Second + pollInterval := 10 * time.Millisecond + if !base.UnitTestUrlIsWalrus() || base.IsRaceDetectorEnabled(rt.TB()) || os.Getenv("CI") != "" { + timeout = 60 * time.Second + pollInterval = 500 * time.Millisecond + } + + var migrationStatus db.MigrationManagerResponse + require.EventuallyWithT(rt.TB(), func(c *assert.CollectT) { + response := rt.SendAdminRequest("GET", "/"+dbName+"/_metadata_migration", "") + RequireStatus(rt.TB(), response, http.StatusOK) + require.NoError(rt.TB(), json.Unmarshal(response.BodyBytes(), &migrationStatus)) + + assert.Equal(c, status, migrationStatus.State) + }, timeout, pollInterval) + return migrationStatus +} + // UpdatePersistedBucketName will update the persisted config bucket name to name specified in parameters func (rt *RestTester) UpdatePersistedBucketName(dbConfig *DatabaseConfig, newBucketName *string) (*DatabaseConfig, error) { updatedDbConfig := DatabaseConfig{}