diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index f114f16a25..55011f38a8 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,7 +1,7 @@
{
"name": "Go",
// TODO: bump when available, see: https://github.com/devcontainers/images/tree/main/src/go
- "image": "mcr.microsoft.com/devcontainers/go:1.25-bookworm",
+ "image": "mcr.microsoft.com/devcontainers/go:1.26-bookworm",
"features": {
"ghcr.io/devcontainers/features/docker-in-docker:3.0.1": {
"moby": true,
diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml
index 30f7feb036..ec793a3f45 100644
--- a/.github/workflows/helm-release.yml
+++ b/.github/workflows/helm-release.yml
@@ -37,16 +37,16 @@ jobs:
env:
GITHUB_TOKEN: "${{ secrets.GHA_TOKEN }}"
- olm:
- name: Helm OLM
- runs-on: ubuntu-latest
- steps:
- - name: Checkout
- uses: actions/checkout@v6
- with:
- fetch-depth: 0
+ # olm:
+ # name: Helm OLM
+ # runs-on: ubuntu-latest
+ # steps:
+ # - name: Checkout
+ # uses: actions/checkout@v6
+ # with:
+ # fetch-depth: 0
- - name: Dispatch helm OLM workflow
- run: gh workflow run bundle.yaml --repo mariadb-operator/mariadb-operator-helm -f version=$(make helm-version)
- env:
- GITHUB_TOKEN: "${{ secrets.GHA_TOKEN }}"
+ # - name: Dispatch helm OLM workflow
+ # run: gh workflow run bundle.yaml --repo mariadb-operator/mariadb-operator-helm -f version=$(make helm-version)
+ # env:
+ # GITHUB_TOKEN: "${{ secrets.GHA_TOKEN }}"
diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml
index a346002764..6a6ebe3827 100644
--- a/.github/workflows/helm.yml
+++ b/.github/workflows/helm.yml
@@ -3,7 +3,7 @@ name: Helm
on:
push:
branches:
- - main
+ - "band/*"
paths:
- "deploy/charts/**"
pull_request:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b50e0fabba..7ea069324e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -3,7 +3,7 @@ name: Release
on:
push:
tags:
- - "[0-9]+.[0-9]+.[0-9]+"
+ - "[0-9]+.[0-9]+.[0-9]+-bandwidth.[0-9]+"
env:
GORELEASER_VERSION: "v2.2.0"
@@ -136,7 +136,7 @@ jobs:
with:
fetch-depth: 0
token: "${{ secrets.GHA_TOKEN }}"
- ref: "${{ github.event.repository.default_branch }}"
+ ref: "band/${{ needs.args.outputs.VERSION }}"
- name: Configure Git
run: |
diff --git a/Makefile b/Makefile
index 8c4627535e..7fb85444d7 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@ endif
SHELL = /usr/bin/env bash -o pipefail
.SHELLFLAGS = -ec
-VERSION ?= 26.6.0
+VERSION ?= 26.6.0-bandwidth.1
# mariadb-operator
IMG_NAME ?= ghcr.io/mariadb-operator/mariadb-operator
diff --git a/api/v1alpha1/backup_types.go b/api/v1alpha1/backup_types.go
index ef740eecb9..f7be36374e 100644
--- a/api/v1alpha1/backup_types.go
+++ b/api/v1alpha1/backup_types.go
@@ -51,9 +51,16 @@ type BackupSpec struct {
// +operator-sdk:csv:customresourcedefinitions:type=spec
MaxRetention metav1.Duration `json:"maxRetention,omitempty" webhook:"inmutableinit"`
// Databases defines the logical databases to be backed up. If not provided, all databases are backed up.
+ // Mutually exclusive with Tables.
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec
Databases []string `json:"databases,omitempty"`
+ // Tables defines specific tables to be backed up, in "database.table" format. Entries may span
+ // multiple databases; when they do, --ignore-table flags are built at runtime by querying
+ // information_schema so that the dump remains a single consistent transaction. Mutually exclusive with Databases.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ Tables []string `json:"tables,omitempty"`
// IgnoreGlobalPriv indicates to ignore the mysql.global_priv in backups.
// If not provided, it will default to true when the referred MariaDB instance has Galera enabled and otherwise to false.
// See: https://github.com/mariadb-operator/mariadb-operator/issues/556
@@ -119,6 +126,11 @@ func (b *Backup) IsComplete() bool {
return meta.IsStatusConditionTrue(b.Status.Conditions, ConditionTypeComplete)
}
+func (b *Backup) IsFailed() bool {
+ condition := meta.FindStatusCondition(b.Status.Conditions, ConditionTypeComplete)
+ return condition != nil && condition.Status == metav1.ConditionTrue && condition.Reason == ConditionReasonJobFailed
+}
+
func (b *Backup) Validate() error {
if b.Spec.Schedule != nil {
if err := b.Spec.Schedule.Validate(); err != nil {
diff --git a/api/v1alpha1/base_types.go b/api/v1alpha1/base_types.go
index fd4cc8e467..d58e6c8fb9 100644
--- a/api/v1alpha1/base_types.go
+++ b/api/v1alpha1/base_types.go
@@ -915,8 +915,11 @@ type Schedule struct {
}
func (s *Schedule) Validate() error {
- _, err := CronParser.Parse(s.Cron)
- return err
+ if s.Cron != "" {
+ _, err := CronParser.Parse(s.Cron)
+ return err
+ }
+ return nil
}
// CronJobTemplate defines parameters for configuring CronJob objects.
diff --git a/api/v1alpha1/condition_types.go b/api/v1alpha1/condition_types.go
index e5b201c259..4122a84c03 100644
--- a/api/v1alpha1/condition_types.go
+++ b/api/v1alpha1/condition_types.go
@@ -26,40 +26,46 @@ const (
// ConditionTypeReplicationConfigured indicates that replication has been successfully configured.
ConditionTypeReplicationConfigured string = "ReplicationConfigured"
- ConditionReasonStatefulSetNotReady string = "StatefulSetNotReady"
- ConditionReasonStatefulSetReady string = "StatefulSetReady"
- ConditionReasonRestoreBackup string = "RestoreBackup"
- ConditionReasonRestorePhysicalBackup string = "RestorePhysicalBackup"
- ConditionReasonArchiveBinlogs string = "ArchiveBinlogs"
- ConditionReasonArchiveBinlogsError string = "ArchiveBinlogsError"
- ConditionReasonReplayBinlogs string = "ReplayBinlogs"
- ConditionReasonReplayBinlogsError string = "ReplayBinlogsError"
- ConditionReasonReplayBinlogsSkipped string = "ReplayBinlogsSkipped"
- ConditionReasonSwitchPrimary string = "SwitchPrimary"
- ConditionReasonGaleraReady string = "GaleraReady"
- ConditionReasonGaleraNotReady string = "GaleraNotReady"
- ConditionReasonGaleraConfigured string = "GaleraConfigured"
- ConditionReasonGaleraInitialized string = "GaleraInitialized"
- ConditionReasonGaleraInitializing string = "GaleraInitializing"
- ConditionReasonResizingStorage string = "ResizingStorage"
- ConditionReasonWaitStorageResize string = "WaitStorageResize"
- ConditionReasonStorageResized string = "StorageResized"
- ConditionReasonInitializing string = "Initializing"
- ConditionReasonInitialized string = "Initialized"
- ConditionReasonInitError string = "InitError"
- ConditionReasonScalingOut string = "ScalingOut"
- ConditionReasonScaledOut string = "ScaledOut"
- ConditionReasonScaleOutError string = "ScaleOutError"
- ConditionReasonReplicaRecovering string = "ReplicaRecovering"
- ConditionReasonReplicaRecovered string = "ReplicaRecovered"
- ConditionReasonReplicaRecoverError string = "ReplicaRecoverError"
- ConditionReasonReplicationConfigured string = "ReplicationConfigured"
- ConditionReasonPendingUpdate string = "PendingUpdate"
- ConditionReasonUpdating string = "Updating"
- ConditionReasonUpdated string = "Updated"
- ConditionReasonSuspended string = "Suspended"
- ConditionReasonMaintenance string = "Maintenance"
- ConditionReasonCordoned string = "Cordoned"
+ ConditionTypeExternalReplInitialized string = "ExternalReplInitialized"
+
+ ConditionReasonStatefulSetNotReady string = "StatefulSetNotReady"
+ ConditionReasonStatefulSetReady string = "StatefulSetReady"
+ ConditionReasonRestoreBackup string = "RestoreBackup"
+ ConditionReasonRestorePhysicalBackup string = "RestorePhysicalBackup"
+ ConditionReasonArchiveBinlogs string = "ArchiveBinlogs"
+ ConditionReasonArchiveBinlogsError string = "ArchiveBinlogsError"
+ ConditionReasonReplayBinlogs string = "ReplayBinlogs"
+ ConditionReasonReplayBinlogsError string = "ReplayBinlogsError"
+ ConditionReasonReplayBinlogsSkipped string = "ReplayBinlogsSkipped"
+ ConditionReasonSwitchPrimary string = "SwitchPrimary"
+ ConditionReasonGaleraReady string = "GaleraReady"
+ ConditionReasonGaleraNotReady string = "GaleraNotReady"
+ ConditionReasonGaleraConfigured string = "GaleraConfigured"
+ ConditionReasonGaleraInitialized string = "GaleraInitialized"
+ ConditionReasonGaleraInitializing string = "GaleraInitializing"
+ ConditionReasonResizingStorage string = "ResizingStorage"
+ ConditionReasonWaitStorageResize string = "WaitStorageResize"
+ ConditionReasonStorageResized string = "StorageResized"
+ ConditionReasonInitializing string = "Initializing"
+ ConditionReasonInitialized string = "Initialized"
+ ConditionReasonInitError string = "InitError"
+ ConditionReasonScalingOut string = "ScalingOut"
+ ConditionReasonScaledOut string = "ScaledOut"
+ ConditionReasonScaleOutError string = "ScaleOutError"
+ ConditionReasonReplicaRecovering string = "ReplicaRecovering"
+ ConditionReasonReplicaRecovered string = "ReplicaRecovered"
+ ConditionReasonReplicaRecoverError string = "ReplicaRecoverError"
+ ConditionReasonReplicationConfigured string = "ReplicationConfigured"
+ ConditionReasonPendingUpdate string = "PendingUpdate"
+ ConditionReasonUpdating string = "Updating"
+ ConditionReasonUpdated string = "Updated"
+ ConditionReasonSuspended string = "Suspended"
+ ConditionReasonMaintenance string = "Maintenance"
+ ConditionReasonCordoned string = "Cordoned"
+ ConditionReasonExternalReplInitError string = "ExternalReplInitError"
+ ConditionReasonExternalReplInitialized string = "ExternalReplInitialized"
+ ConditionReasonExternalReplInitializing string = "ExternalReplInitializing"
+ ConditionReasonPendingExternalReplInitialization string = "PendingExternalReplInitialization"
ConditionReasonMaxScaleNotReady string = "MaxScaleNotReady"
ConditionReasonMaxScaleReady string = "MaxScaleReady"
diff --git a/api/v1alpha1/external_mariadb_keys.go b/api/v1alpha1/external_mariadb_keys.go
index 894c0466f8..fc8b85cd2e 100644
--- a/api/v1alpha1/external_mariadb_keys.go
+++ b/api/v1alpha1/external_mariadb_keys.go
@@ -8,6 +8,14 @@ import (
"k8s.io/utils/ptr"
)
+// InternalServiceKey defines the key for the internal headless Service
+func (m *ExternalMariaDB) InternalServiceKey() types.NamespacedName {
+ return types.NamespacedName{
+ Name: "",
+ Namespace: m.Namespace,
+ }
+}
+
// TLSCABundleSecretKeyRef defines the key selector for the TLS Secret trust bundle
func (m *ExternalMariaDB) TLSCABundleSecretKeyRef() SecretKeySelector {
if m.Spec.TLS.ServerCASecretRef == nil {
diff --git a/api/v1alpha1/external_mariadb_types.go b/api/v1alpha1/external_mariadb_types.go
index 0a89cf8a0b..40a0f7afe4 100644
--- a/api/v1alpha1/external_mariadb_types.go
+++ b/api/v1alpha1/external_mariadb_types.go
@@ -53,6 +53,10 @@ type ExternalMariaDBSpec struct {
// +kubebuilder:default=3306
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number","urn:alm:descriptor:com.tectonic.ui:advanced"}
Port int32 `json:"port,omitempty"`
+ // Binlog proxy router port of the external MariaDB. Useful when the external MariaDB is behind a Maxscale and using the Binlogrouter to expose the binlog stream.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number","urn:alm:descriptor:com.tectonic.ui:advanced"}
+ BinlogProxyPort *int32 `json:"binlogPort,omitempty"`
// Username is the username to connect to the external MariaDB.
// +kubebuilder:validation:Required
// +operator-sdk:csv:customresourcedefinitions:type=spec
@@ -100,6 +104,11 @@ func (m *ExternalMariaDB) IsGaleraEnabled() bool {
return m.Status.IsGaleraEnabled
}
+// Replication with defaulting accessor
+func (m *ExternalMariaDB) Replication() Replication {
+ return Replication{}
+}
+
// +kubebuilder:object:root=true
// +kubebuilder:resource:shortName=emdb
// +kubebuilder:subresource:status
@@ -127,6 +136,11 @@ func (m *ExternalMariaDB) SetDefaults(env *environment.OperatorEnv) error {
return nil
}
+// Get MariaDB Object Meta
+func (m *ExternalMariaDB) GetObjectMeta() *metav1.ObjectMeta {
+ return &m.ObjectMeta
+}
+
// IsReady indicates whether the External MariaDB instance is ready
func (m *ExternalMariaDB) IsReady() bool {
return meta.IsStatusConditionTrue(m.Status.Conditions, ConditionTypeReady)
@@ -181,16 +195,31 @@ func (m *ExternalMariaDB) GetHost() string {
return m.Spec.Host
}
+// Get specific MariaDB Pod hostname
+func (m *ExternalMariaDB) GetPodHost(podIndex int) string {
+ return ""
+}
+
// Get MariaDB port
func (m *ExternalMariaDB) GetPort() int32 {
return m.Spec.Port
}
+// Get MariaDB binlog proxy router port
+func (m *ExternalMariaDB) GetBinlogProxyPort() *int32 {
+ return m.Spec.BinlogProxyPort
+}
+
// Get MariaDB replicas
func (m *ExternalMariaDB) GetReplicas() int32 {
return 0 // ExternalMariaDB does not make use of this
}
+// IsHAEnabled indicates whether the MariaDB instance has HA enabled (Always false for external MariaDB)
+func (m *ExternalMariaDB) IsHAEnabled() bool {
+ return false
+}
+
// Get MariaDB Superuser name
func (m *ExternalMariaDB) GetSUName() string {
return ptr.Deref(m.Spec.Username, "root")
diff --git a/api/v1alpha1/mariadb_keys.go b/api/v1alpha1/mariadb_keys.go
index bfe9e3a2fa..98cdbd61a9 100644
--- a/api/v1alpha1/mariadb_keys.go
+++ b/api/v1alpha1/mariadb_keys.go
@@ -185,7 +185,14 @@ func (m *MariaDB) BootstrapFromStagingPVCKey() types.NamespacedName {
// PITRJobKey defines the key for the PITR job used to replay the binary logs.
func (m *MariaDB) PITRJobKey() types.NamespacedName {
return types.NamespacedName{
- Name: fmt.Sprintf("%s-pitr", m.Name),
+ Name: fmt.Sprintf("%s-pitr", m.Name),
+ }
+}
+
+// RestoreKey defines the key for the Restore resource used to bootstrap.
+func (m *MariaDB) RestoreKeyInPod(podIndex int) types.NamespacedName {
+ return types.NamespacedName{
+ Name: fmt.Sprintf("%s-restore-%d", m.Name, podIndex),
Namespace: m.Namespace,
}
}
diff --git a/api/v1alpha1/mariadb_replication_types.go b/api/v1alpha1/mariadb_replication_types.go
index c6c5be811f..7778d9ba3f 100644
--- a/api/v1alpha1/mariadb_replication_types.go
+++ b/api/v1alpha1/mariadb_replication_types.go
@@ -8,6 +8,7 @@ import (
"github.com/mariadb-operator/mariadb-operator/v26/pkg/docker"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/environment"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/hash"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
@@ -120,6 +121,13 @@ type ReplicaBootstrapFrom struct {
// +kubebuilder:validation:Required
// +operator-sdk:csv:customresourcedefinitions:type=spec
PhysicalBackupTemplateRef LocalObjectReference `json:"physicalBackupTemplateRef"`
+ // LogicalBackupTemplateRef is a reference to a Backup object that will be used as template to create the logical Backup
+ // taken from the external MariaDB during external replication initialization and recovery. The template's Spec is copied
+ // over (resources, pod template, etc.) and the controller overrides the fields that are managed automatically
+ // (MariaDBRef, Storage, Args, Tables, Compression, MaxRetention).
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ LogicalBackupTemplateRef *LocalObjectReference `json:"logicalBackupTemplateRef,omitempty"`
// RestoreJob defines additional properties for the Job used to perform the restoration.
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
@@ -171,6 +179,21 @@ type ReplicaReplication struct {
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
MaxLagSeconds *int `json:"maxLagSeconds,omitempty"`
+ // IgnoreMaxLagSeconds is to ignore the lag behind primary checks.
+ // It's useful on situations when is preferred to keep sending read queries on a delayed (or with connection issues)
+ // replica than stopping sending traffic. It could be useful when replicating from a external MariaDB when
+ // connection issues with primary could happen.
+ // If not provided, it defaults to false.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
+ IgnoreMaxLagSeconds *bool `json:"ignoreMaxLagSeconds,omitempty"`
+ // IgnoreReplicationLivenessProbes is to ignore liveness replication checks.
+ // It's useful on situations when is preferred to keep sending read queries on a broken replicas
+ // replica than stopping sending traffic.
+ // If not provided, it defaults to false.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
+ IgnoreReplicationLivenessProbes *bool `json:"ignoreReplicationLivenessProbes,omitempty"`
// SyncTimeout defines the timeout for the synchronization phase during switchover and failover operations.
// During switchover, all replicas must be synced with the current primary before promoting the new primary.
// During failover, the new primary must be synced before being promoted as primary. This implies processing all the events in the relay log.
@@ -194,6 +217,52 @@ type ReplicaReplication struct {
ReplicaRecovery *ReplicaRecovery `json:"recovery,omitempty"`
}
+// ReplicaFromExternal is the replication configuration from external servers.
+type ReplicaFromExternal struct {
+
+ // MariaDBRef is a reference to a MariaDB object.
+ // +kubebuilder:validation:Required
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ MariaDBRef MariaDBRef `json:"mariaDbRef" webhook:"inmutable"`
+ // Gtid indicates which Global Transaction ID should be used when connecting a replica to the master.
+ // See: https://mariadb.com/kb/en/gtid/#using-current_pos-vs-slave_pos.
+ // +optional
+ // +kubebuilder:validation:Enum=CurrentPos;SlavePos
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ Gtid *Gtid `json:"gtid,omitempty"`
+ // ConnectionTimeout to be used when the replica connects to the primary.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ ConnectionTimeout *metav1.Duration `json:"connectionTimeout,omitempty"`
+ // ConnectionRetries to be used when the replica connects to the primary.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
+ ConnectionRetries *int `json:"connectionRetries,omitempty"`
+ // HealthCheckInterval to be used when the replica connects to the primary.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"}
+ HealthCheckInterval *metav1.Duration `json:"healthCheckInterval,omitempty"`
+ // ServerIdOffset to be used on the replicas. Each replica gets server_id = podIndex + offset.
+ // If not set, the operator auto-discovers a non-colliding offset by querying the external MariaDB
+ // for the server ids already in use, leaving room above them for scale out and other clusters. The
+ // discovered value is persisted to status.externalReplication.serverIdOffset and computed only once.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ ServerIdOffset *int `json:"serverIdOffset,omitempty"`
+ // FilteredReplicaTables is an optional list of tables in "database.table" format to replicate.
+ // When set, the logical backup will only include these tables and the replication will be
+ // configured with replicate_do_table for each entry. GTID strict mode is automatically
+ // disabled when this field is set, as partial replication is incompatible with it.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ FilteredReplicaTables []string `json:"filteredReplicaTables,omitempty"`
+}
+
+// HasFilteredTables returns true when at least one filtered table is defined.
+func (r *ReplicaFromExternal) HasFilteredTables() bool {
+ return len(r.FilteredReplicaTables) > 0
+}
+
// SetDefaults fills the current ReplicaReplication object with DefaultReplicationSpec.
// This enables having minimal ReplicaReplication objects and provides sensible defaults.
func (r *ReplicaReplication) SetDefaults(mdb *MariaDB) {
@@ -275,6 +344,10 @@ type ReplicationSpec struct {
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec
SemiSyncAckTimeout *metav1.Duration `json:"semiSyncAckTimeout,omitempty"`
+ // ReplicaFromExternal specifies whether the replica should be created from an external MariaDB instance.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
+ ReplicaFromExternal *ReplicaFromExternal `json:"replicaFromExternal,omitempty"`
// SemiSyncWaitPoint determines whether the transaction should wait for an ACK after having synced the binlog (AfterSync)
// or after having committed to the storage engine (AfterCommit, the default).
// It requires semi-synchronous replication to be enabled.
@@ -301,6 +374,10 @@ type ReplicationSpec struct {
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec
StandaloneProbes *bool `json:"standaloneProbes,omitempty"`
+ // MultiCluster Connection name
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ MultiClusterReplicaConnectionName *string `json:"multiClusterReplicaConnectionName,omitempty"`
}
// IsGtidStrictModeEnabled determines whether GTID strict mode is enabled.
@@ -313,6 +390,57 @@ func (r *Replication) IsSemiSyncEnabled() bool {
return ptr.Deref(r.SemiSyncEnabled, true)
}
+// FillWithDefaults fills the current ReplicationSpec object with DefaultReplicationSpec.
+// This enables having minimal ReplicationSpec objects and provides sensible defaults.
+func (r *ReplicationSpec) FillWithDefaults() {
+ if r.ReplicaFromExternal != nil {
+ r.ReplicaFromExternal.FillWithDefaults()
+ }
+}
+
+// FillWithDefaults fills the current ReplicationSpec object with DefaultReplicationSpec.
+// This enables having minimal ReplicationSpec objects and provides sensible defaults.
+func (r *ReplicaFromExternal) FillWithDefaults() {
+ if r.HealthCheckInterval == nil {
+ r.HealthCheckInterval = &metav1.Duration{
+ Duration: 15 * time.Second,
+ }
+ }
+ // ServerIdOffset is intentionally not defaulted: a nil value signals that the operator should
+ // auto-discover a non-colliding offset by querying the external MariaDB (see MariaDB.ExternalReplServerIdOffset).
+}
+
+// IsExternalReplication returns true is external replication is defined
+func (r *ReplicationSpec) IsExternalReplication() bool {
+ return r.ReplicaFromExternal != nil
+}
+
+// Return the MariaDB ref to the external primary MariaDB
+func (r *ReplicationSpec) GetExternalReplicationRef() ObjectReference {
+ if r.IsExternalReplication() {
+ return r.ReplicaFromExternal.MariaDBRef.ObjectReference
+ }
+ return ObjectReference{}
+}
+
+var (
+ tenSeconds = metav1.Duration{Duration: 10 * time.Second}
+
+ // DefaultReplicationSpec provides sensible defaults for the ReplicationSpec.
+ DefaultReplicationSpec = ReplicationSpec{
+ Primary: PrimaryReplication{
+ PodIndex: ptr.To(0),
+ AutoFailover: ptr.To(true),
+ AutoFailoverDelay: ptr.To(metav1.Duration{}),
+ },
+ Replica: ReplicaReplication{
+ Gtid: ptr.To(GtidCurrentPos),
+ SyncTimeout: ptr.To(tenSeconds),
+ },
+ SyncBinlog: ptr.To(1),
+ }
+)
+
// Validate determines whether replication config is valid.
func (r *Replication) Validate() error {
if r.IsSemiSyncEnabled() {
@@ -330,8 +458,21 @@ func (r *Replication) SetDefaults(mdb *MariaDB, env *environment.OperatorEnv) er
r.Primary.SetDefaults()
r.Replica.SetDefaults(mdb)
+ // Enable ReplicaRecovery by default if it is on external replication
+ if r.IsExternalReplication() && r.Replica.ReplicaRecovery == nil {
+ recovery := ReplicaRecovery{
+ Enabled: true,
+ }
+ r.Replica.ReplicaRecovery = &recovery
+ }
+
if r.GtidStrictMode == nil {
- r.GtidStrictMode = ptr.To(true)
+ // Filtered replica is incompatible with GTID strict mode; disable it automatically.
+ if r.IsExternalReplication() && r.ReplicaFromExternal.HasFilteredTables() {
+ r.GtidStrictMode = ptr.To(false)
+ } else {
+ r.GtidStrictMode = ptr.To(true)
+ }
}
if r.SemiSyncEnabled == nil {
r.SemiSyncEnabled = ptr.To(true)
@@ -414,6 +555,32 @@ func (m *MariaDB) IsRecoveringReplicas() bool {
return meta.IsStatusConditionFalse(m.Status.Conditions, ConditionTypeReplicaRecovered)
}
+// IsExternalReplInitialized indicates that the external replication init Job has successfully completed.
+func (m *MariaDB) IsExternalReplInitialized() bool {
+ return meta.IsStatusConditionTrue(m.Status.Conditions, ConditionTypeExternalReplInitialized)
+}
+
+// IsExternalReplInitialing indicates that the external replication initialization is in progress.
+func (m *MariaDB) IsExternalReplInitialing() bool {
+ return meta.IsStatusConditionFalse(m.Status.Conditions, ConditionTypeExternalReplInitialized)
+}
+
+// ExternalReplLogicalBackupName returns the name of the logical Backup object used during external replication init.
+func (m *MariaDB) ExternalReplLogicalBackupName() string {
+ ext := m.Replication().ReplicaFromExternal
+ emdbName := ext.MariaDBRef.Name
+ if !ext.HasFilteredTables() {
+ return emdbName
+ }
+
+ suffix := hash.Hash(m.Name)[:8]
+ prefix := emdbName + "-"
+ if len(prefix)+len(suffix) > 253 {
+ prefix = prefix[:253-len(suffix)]
+ }
+ return prefix + suffix
+}
+
// ReplicaRecoveryError indicates that the MariaDB instance has a replica recovery error.
func (m *MariaDB) ReplicaRecoveryError() error {
c := meta.FindStatusCondition(m.Status.Conditions, ConditionTypeReplicaRecovered)
@@ -426,6 +593,18 @@ func (m *MariaDB) ReplicaRecoveryError() error {
return nil
}
+// ExternalReplInitError indicates that the MariaDB instance has an external replication initialization error.
+func (m *MariaDB) ExternalReplInitError() error {
+ c := meta.FindStatusCondition(m.Status.Conditions, ConditionTypeExternalReplInitialized)
+ if c == nil {
+ return nil
+ }
+ if c.Status == metav1.ConditionFalse && c.Reason == ConditionReasonExternalReplInitError {
+ return errors.New(c.Message)
+ }
+ return nil
+}
+
// SetReplicaToRecover sets the replica to be recovered
func (m *MariaDB) SetReplicaToRecover(replica *string) {
if m.Status.Replication == nil {
@@ -568,6 +747,35 @@ type ReplicationStatus struct {
GtidStrictModePaused *bool `json:"gtidStrictModePaused,omitempty"`
}
+// ExternalReplicationStatus is the status of external replication.
+type ExternalReplicationStatus struct {
+ // ServerIdOffset is the server_id offset auto-discovered by querying the external MariaDB for the
+ // server ids already in use. It is computed only once and persisted here so it stays stable and
+ // does not need to be queried again. It is not set when a manual serverIdOffset is configured.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=status
+ ServerIdOffset *int `json:"serverIdOffset,omitempty"`
+}
+
+// ExternalReplServerIdOffset returns the effective server_id offset for external replication:
+// the manual spec value when set, otherwise the auto-discovered value persisted in status, otherwise nil.
+func (m *MariaDB) ExternalReplServerIdOffset() *int {
+ if !m.IsReplicationEnabled() {
+ return nil
+ }
+ replication := m.Replication()
+ if !replication.IsExternalReplication() {
+ return nil
+ }
+ if replication.ReplicaFromExternal.ServerIdOffset != nil {
+ return replication.ReplicaFromExternal.ServerIdOffset
+ }
+ if m.Status.ExternalReplication != nil {
+ return m.Status.ExternalReplication.ServerIdOffset
+ }
+ return nil
+}
+
// UseStandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
func (m *MariaDB) UseStandaloneProbes() bool {
replication := ptr.Deref(m.Spec.Replication, Replication{})
diff --git a/api/v1alpha1/mariadb_types.go b/api/v1alpha1/mariadb_types.go
index 933c4ea497..3fe0784648 100644
--- a/api/v1alpha1/mariadb_types.go
+++ b/api/v1alpha1/mariadb_types.go
@@ -802,6 +802,10 @@ type MariaDBSpec struct {
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
Maintenance *MariaDBMaintenance `json:"maintenance,omitempty"`
+ // MultiCluster Connection name
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ MultiClusterReplicaConnectionName *string `json:"multiClusterReplicaConnectionName,omitempty"`
}
// MariaDBTLSStatus aggregates the status of the certificates used by the MariaDB instance.
@@ -903,6 +907,10 @@ type MariaDBStatus struct {
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=status
RootPasswordHash *string `json:"rootPasswordHash,omitempty"`
+ // ExternalReplication is the status of external replication.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=status
+ ExternalReplication *ExternalReplicationStatus `json:"externalReplication,omitempty"`
}
// SetCondition sets a status condition to MariaDB
@@ -941,7 +949,7 @@ type MariaDB struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
- // +kubebuilder:validation:XValidation:rule="!has(self.galera) || !self.galera.enabled || (self.replicas % 2 == 1 || self.replicasAllowEvenNumber)", message="An odd number of MariaDB instances (mariadb.spec.replicas) is required to avoid split brain situations for Galera. Use 'mariadb.spec.replicasAllowEvenNumber: true' to disable this validation."
+ // +kubebuilder:validation:XValidation:rule="!has(self.galera) || !self.galera.enabled || (self.replicas % 2 == 1 || self.replicasAllowEvenNumber ) || ( has(self.replication) && has(self.replication.replicaFromExternal) )", message="An odd number of MariaDB instances (mariadb.spec.replicas) is required to avoid split brain situations for Galera. Use 'mariadb.spec.replicasAllowEvenNumber: true' to disable this validation."
Spec MariaDBSpec `json:"spec"`
Status MariaDBStatus `json:"status,omitempty"`
}
@@ -1004,6 +1012,7 @@ func (m *MariaDB) SetDefaults(env *environment.OperatorEnv) error {
return fmt.Errorf("error setting replication defaults: %v", err)
}
}
+
if m.Spec.BootstrapFrom != nil {
m.Spec.BootstrapFrom.SetDefaults(m)
}
@@ -1018,6 +1027,14 @@ func (m *MariaDB) SetDefaults(env *environment.OperatorEnv) error {
return nil
}
+func (m *MariaDB) Replication() Replication {
+ if m.Spec.Replication == nil {
+ m.Spec.Replication = &Replication{}
+ }
+ m.Spec.Replication.FillWithDefaults()
+ return *m.Spec.Replication
+}
+
// IsGaleraEnabled indicates whether the MariaDB instance has Galera enabled
func (m *MariaDB) IsGaleraEnabled() bool {
return ptr.Deref(m.Spec.Galera, Galera{}).Enabled
@@ -1325,7 +1342,7 @@ func (m *MariaDB) GetImage(env *environment.OperatorEnv) string {
// Get MariaDB hostname
func (m *MariaDB) GetHost() string {
- if m.IsHAEnabled() {
+ if m.IsHAEnabled() && m.Replication().ReplicaFromExternal == nil {
return statefulset.ServiceFQDNWithService(
m.ObjectMeta,
m.PrimaryServiceKey().Name,
@@ -1334,6 +1351,20 @@ func (m *MariaDB) GetHost() string {
return statefulset.ServiceFQDN(m.ObjectMeta)
}
+// Get specific MariaDB Pod hostname
+func (m *MariaDB) GetPodHost(podIndex int) string {
+ return statefulset.PodFQDNWithService(
+ m.ObjectMeta,
+ podIndex,
+ m.InternalServiceKey().Name,
+ )
+}
+
+// Get MariaDB Object Meta
+func (m *MariaDB) GetObjectMeta() *metav1.ObjectMeta {
+ return &m.ObjectMeta
+}
+
// Get MariaDB port
func (m *MariaDB) GetPort() int32 {
return m.Spec.Port
diff --git a/api/v1alpha1/mariadb_types_test.go b/api/v1alpha1/mariadb_types_test.go
index cc667b00f9..3cbe103fb4 100644
--- a/api/v1alpha1/mariadb_types_test.go
+++ b/api/v1alpha1/mariadb_types_test.go
@@ -2430,6 +2430,62 @@ var _ = Describe("MariaDB types", func() {
),
)
})
+
+ Context("When defaulting a ReplicaFromExternal object", func() {
+ It("should not default serverIdOffset (nil means auto-discover)", func() {
+ ext := &ReplicaFromExternal{}
+ ext.FillWithDefaults()
+ Expect(ext.ServerIdOffset).To(BeNil())
+ Expect(ext.HealthCheckInterval).NotTo(BeNil())
+ })
+
+ It("should preserve an explicit serverIdOffset", func() {
+ ext := &ReplicaFromExternal{
+ ServerIdOffset: ptr.To(30),
+ }
+ ext.FillWithDefaults()
+ Expect(ext.ServerIdOffset).To(Equal(ptr.To(30)))
+ })
+ })
+
+ Context("When resolving the external replication serverId offset", func() {
+ newExternalReplMariaDB := func(manual *int, status *int) *MariaDB {
+ mdb := &MariaDB{
+ Spec: MariaDBSpec{
+ Replication: &Replication{
+ Enabled: true,
+ ReplicationSpec: ReplicationSpec{
+ ReplicaFromExternal: &ReplicaFromExternal{
+ ServerIdOffset: manual,
+ },
+ },
+ },
+ },
+ }
+ if status != nil {
+ mdb.Status.ExternalReplication = &ExternalReplicationStatus{
+ ServerIdOffset: status,
+ }
+ }
+ return mdb
+ }
+
+ DescribeTable(
+ "Should resolve the effective offset",
+ func(mdb *MariaDB, expected *int) {
+ Expect(mdb.ExternalReplServerIdOffset()).To(Equal(expected))
+ },
+ Entry("replication not enabled", &MariaDB{}, nil),
+ Entry(
+ "replication enabled but not external",
+ &MariaDB{Spec: MariaDBSpec{Replication: &Replication{Enabled: true}}},
+ nil,
+ ),
+ Entry("manual offset wins over status", newExternalReplMariaDB(ptr.To(30), ptr.To(150)), ptr.To(30)),
+ Entry("discovered status offset when no manual offset", newExternalReplMariaDB(nil, ptr.To(150)), ptr.To(150)),
+ Entry("nil when neither manual nor status offset", newExternalReplMariaDB(nil, nil), nil),
+ )
+ })
})
var _ = Describe("MariaDBVolume conversion", func() {
diff --git a/api/v1alpha1/restore_types.go b/api/v1alpha1/restore_types.go
index 619199202b..f04ba17071 100644
--- a/api/v1alpha1/restore_types.go
+++ b/api/v1alpha1/restore_types.go
@@ -90,6 +90,10 @@ type RestoreSpec struct {
// +kubebuilder:validation:Required
// +operator-sdk:csv:customresourcedefinitions:type=spec
MariaDBRef MariaDBRef `json:"mariaDbRef" webhook:"inmutable"`
+ // PodIndex is the StatefulSet index of pod to restore. Used to bootstrap nodes on external replication.
+ // +optional
+ // +operator-sdk:csv:customresourcedefinitions:type=spec
+ PodIndex *int `json:"podIndex" webhook:"inmutable"`
// Database defines the logical database to be restored. If not provided, all databases available in the backup are restored.
// IMPORTANT: The database must previously exist.
// +optional
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index dea8699152..ca4abcd7f5 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -224,6 +224,11 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
+ if in.Tables != nil {
+ in, out := &in.Tables, &out.Tables
+ *out = make([]string, len(*in))
+ copy(*out, *in)
+ }
if in.IgnoreGlobalPriv != nil {
in, out := &in.IgnoreGlobalPriv, &out.IgnoreGlobalPriv
*out = new(bool)
@@ -1193,6 +1198,11 @@ func (in *ExternalMariaDBSpec) DeepCopyInto(out *ExternalMariaDBSpec) {
*out = new(Metadata)
(*in).DeepCopyInto(*out)
}
+ if in.BinlogProxyPort != nil {
+ in, out := &in.BinlogProxyPort, &out.BinlogProxyPort
+ *out = new(int32)
+ **out = **in
+ }
if in.Username != nil {
in, out := &in.Username, &out.Username
*out = new(string)
@@ -1247,6 +1257,26 @@ func (in *ExternalMariaDBStatus) DeepCopy() *ExternalMariaDBStatus {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ExternalReplicationStatus) DeepCopyInto(out *ExternalReplicationStatus) {
+ *out = *in
+ if in.ServerIdOffset != nil {
+ in, out := &in.ServerIdOffset, &out.ServerIdOffset
+ *out = new(int)
+ **out = **in
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalReplicationStatus.
+func (in *ExternalReplicationStatus) DeepCopy() *ExternalReplicationStatus {
+ if in == nil {
+ return nil
+ }
+ out := new(ExternalReplicationStatus)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ExternalTLS) DeepCopyInto(out *ExternalTLS) {
*out = *in
@@ -2407,6 +2437,11 @@ func (in *MariaDBSpec) DeepCopyInto(out *MariaDBSpec) {
*out = new(MariaDBMaintenance)
**out = **in
}
+ if in.MultiClusterReplicaConnectionName != nil {
+ in, out := &in.MultiClusterReplicaConnectionName, &out.MultiClusterReplicaConnectionName
+ *out = new(string)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MariaDBSpec.
@@ -2478,6 +2513,11 @@ func (in *MariaDBStatus) DeepCopyInto(out *MariaDBStatus) {
*out = new(string)
**out = **in
}
+ if in.ExternalReplication != nil {
+ in, out := &in.ExternalReplication, &out.ExternalReplication
+ *out = new(ExternalReplicationStatus)
+ (*in).DeepCopyInto(*out)
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MariaDBStatus.
@@ -4219,6 +4259,11 @@ func (in *ProbeHandler) DeepCopy() *ProbeHandler {
func (in *ReplicaBootstrapFrom) DeepCopyInto(out *ReplicaBootstrapFrom) {
*out = *in
out.PhysicalBackupTemplateRef = in.PhysicalBackupTemplateRef
+ if in.LogicalBackupTemplateRef != nil {
+ in, out := &in.LogicalBackupTemplateRef, &out.LogicalBackupTemplateRef
+ *out = new(LocalObjectReference)
+ **out = **in
+ }
if in.RestoreJob != nil {
in, out := &in.RestoreJob, &out.RestoreJob
*out = new(Job)
@@ -4236,6 +4281,52 @@ func (in *ReplicaBootstrapFrom) DeepCopy() *ReplicaBootstrapFrom {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ReplicaFromExternal) DeepCopyInto(out *ReplicaFromExternal) {
+ *out = *in
+ out.MariaDBRef = in.MariaDBRef
+ if in.Gtid != nil {
+ in, out := &in.Gtid, &out.Gtid
+ *out = new(Gtid)
+ **out = **in
+ }
+ if in.ConnectionTimeout != nil {
+ in, out := &in.ConnectionTimeout, &out.ConnectionTimeout
+ *out = new(v1.Duration)
+ **out = **in
+ }
+ if in.ConnectionRetries != nil {
+ in, out := &in.ConnectionRetries, &out.ConnectionRetries
+ *out = new(int)
+ **out = **in
+ }
+ if in.HealthCheckInterval != nil {
+ in, out := &in.HealthCheckInterval, &out.HealthCheckInterval
+ *out = new(v1.Duration)
+ **out = **in
+ }
+ if in.ServerIdOffset != nil {
+ in, out := &in.ServerIdOffset, &out.ServerIdOffset
+ *out = new(int)
+ **out = **in
+ }
+ if in.FilteredReplicaTables != nil {
+ in, out := &in.FilteredReplicaTables, &out.FilteredReplicaTables
+ *out = make([]string, len(*in))
+ copy(*out, *in)
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaFromExternal.
+func (in *ReplicaFromExternal) DeepCopy() *ReplicaFromExternal {
+ if in == nil {
+ return nil
+ }
+ out := new(ReplicaFromExternal)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ReplicaRecovery) DeepCopyInto(out *ReplicaRecovery) {
*out = *in
@@ -4279,6 +4370,16 @@ func (in *ReplicaReplication) DeepCopyInto(out *ReplicaReplication) {
*out = new(int)
**out = **in
}
+ if in.IgnoreMaxLagSeconds != nil {
+ in, out := &in.IgnoreMaxLagSeconds, &out.IgnoreMaxLagSeconds
+ *out = new(bool)
+ **out = **in
+ }
+ if in.IgnoreReplicationLivenessProbes != nil {
+ in, out := &in.IgnoreReplicationLivenessProbes, &out.IgnoreReplicationLivenessProbes
+ *out = new(bool)
+ **out = **in
+ }
if in.SyncTimeout != nil {
in, out := &in.SyncTimeout, &out.SyncTimeout
*out = new(v1.Duration)
@@ -4434,6 +4535,11 @@ func (in *ReplicationSpec) DeepCopyInto(out *ReplicationSpec) {
*out = new(v1.Duration)
**out = **in
}
+ if in.ReplicaFromExternal != nil {
+ in, out := &in.ReplicaFromExternal, &out.ReplicaFromExternal
+ *out = new(ReplicaFromExternal)
+ (*in).DeepCopyInto(*out)
+ }
if in.SemiSyncWaitPoint != nil {
in, out := &in.SemiSyncWaitPoint, &out.SemiSyncWaitPoint
*out = new(WaitPoint)
@@ -4451,6 +4557,11 @@ func (in *ReplicationSpec) DeepCopyInto(out *ReplicationSpec) {
*out = new(bool)
**out = **in
}
+ if in.MultiClusterReplicaConnectionName != nil {
+ in, out := &in.MultiClusterReplicaConnectionName, &out.MultiClusterReplicaConnectionName
+ *out = new(string)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationSpec.
@@ -4636,6 +4747,11 @@ func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) {
in.JobPodTemplate.DeepCopyInto(&out.JobPodTemplate)
in.RestoreSource.DeepCopyInto(&out.RestoreSource)
out.MariaDBRef = in.MariaDBRef
+ if in.PodIndex != nil {
+ in, out := &in.PodIndex, &out.PodIndex
+ *out = new(int)
+ **out = **in
+ }
if in.InheritMetadata != nil {
in, out := &in.InheritMetadata, &out.InheritMetadata
*out = new(Metadata)
diff --git a/cmd/backup/restore.go b/cmd/backup/restore.go
index ab52e11f2b..9ebda6d104 100644
--- a/cmd/backup/restore.go
+++ b/cmd/backup/restore.go
@@ -12,11 +12,16 @@ import (
"github.com/spf13/cobra"
)
-var targetTimeRaw string
+var (
+ targetTimeRaw string
+ targetTimeAgeThresholdRaw string
+)
func init() {
restoreCommand.Flags().StringVar(&targetTimeRaw, "target-time", "",
"RFC3339 (1970-01-01T00:00:00Z) date and time that defines the backup target time.")
+ restoreCommand.Flags().StringVar(&targetTimeAgeThresholdRaw, "target-time-age-threshold", "",
+ "RFC3339 (1970-01-01T00:00:00Z) date and time that defines the target time age threshold.")
}
var restoreCommand = &cobra.Command{
@@ -57,17 +62,29 @@ var restoreCommand = &cobra.Command{
}
logger.Info("obtained target time", "time", targetTime.String())
+ targetTimeAgeThreshold, err := getTargetTimeAgeThreshold()
+ if err != nil {
+ logger.Error(err, "error getting target time age threshold")
+ os.Exit(1)
+ }
+ if targetTimeAgeThreshold != nil {
+ logger.Info("obtained target time age threshold", "threshold", targetTimeAgeThreshold.String())
+ } else {
+ logger.Info("no target time age threshold provided")
+ }
+
backupFileNames, err := backupStorage.List(ctx)
if err != nil {
logger.Error(err, "error listing backup files")
os.Exit(1)
}
-
- backupTargetFile, err := backupProcessor.GetBackupTargetFile(backupFileNames, targetTime, logger.WithName("target-recovery-time"))
+ backupTargetFile, err := backupProcessor.GetBackupTargetFile(backupFileNames, targetTime,
+ targetTimeAgeThreshold, logger.WithName("target-recovery-time"))
if err != nil {
logger.Error(err, "error reading getting target backup")
os.Exit(1)
}
+
logger.Info("obtained target backup", "file", backupTargetFile)
logger.Info("pulling target backup", "file", backupTargetFile, "prefix", s3Prefix)
@@ -129,6 +146,17 @@ func getTargetTime() (time.Time, error) {
return backup.ParseBackupDate(targetTimeRaw)
}
+func getTargetTimeAgeThreshold() (*time.Time, error) {
+ if targetTimeAgeThresholdRaw == "" {
+ return nil, nil
+ }
+ t, err := backup.ParseBackupDate(targetTimeAgeThresholdRaw)
+ if err != nil {
+ return nil, fmt.Errorf("error parsing target time age threshold: %v", err)
+ }
+ return &t, nil
+}
+
func writeTargetFile(backupTargetFile string) error {
return os.WriteFile(targetFilePath, []byte(backupTargetFile), 0777)
}
diff --git a/config/crd/bases/k8s.mariadb.com_backups.yaml b/config/crd/bases/k8s.mariadb.com_backups.yaml
index b3a1d0ee90..34bc55aa8b 100644
--- a/config/crd/bases/k8s.mariadb.com_backups.yaml
+++ b/config/crd/bases/k8s.mariadb.com_backups.yaml
@@ -301,8 +301,9 @@ spec:
- gzip
type: string
databases:
- description: Databases defines the logical databases to be backed
- up. If not provided, all databases are backed up.
+ description: |-
+ Databases defines the logical databases to be backed up. If not provided, all databases are backed up.
+ Mutually exclusive with Tables.
items:
type: string
type: array
@@ -1032,6 +1033,14 @@ spec:
format: int32
minimum: 0
type: integer
+ tables:
+ description: |-
+ Tables defines specific tables to be backed up, in "database.table" format. Entries may span
+ multiple databases; when they do, --ignore-table flags are built at runtime by querying
+ information_schema so that the dump remains a single consistent transaction. Mutually exclusive with Databases.
+ items:
+ type: string
+ type: array
timeZone:
description: TimeZone defines the timezone associated with the cron
expression.
diff --git a/config/crd/bases/k8s.mariadb.com_externalmariadbs.yaml b/config/crd/bases/k8s.mariadb.com_externalmariadbs.yaml
index 589c49317d..306c1df1b7 100644
--- a/config/crd/bases/k8s.mariadb.com_externalmariadbs.yaml
+++ b/config/crd/bases/k8s.mariadb.com_externalmariadbs.yaml
@@ -53,6 +53,12 @@ spec:
description: ExternalMariaDBSpec defines the desired state of an External
MariaDB
properties:
+ binlogPort:
+ description: Binlog proxy router port of the external MariaDB. Useful
+ when the external MariaDB is behind a Maxscale and using the Binlogrouter
+ to expose the binlog stream.
+ format: int32
+ type: integer
connection:
description: Connection defines a template to configure a Connection
for the external MariaDB.
diff --git a/config/crd/bases/k8s.mariadb.com_mariadbs.yaml b/config/crd/bases/k8s.mariadb.com_mariadbs.yaml
index 94f7847ccf..12d1bd8215 100644
--- a/config/crd/bases/k8s.mariadb.com_mariadbs.yaml
+++ b/config/crd/bases/k8s.mariadb.com_mariadbs.yaml
@@ -3518,6 +3518,9 @@ spec:
to a member in the 'members' field, containing its full specification.
type: string
type: object
+ multiClusterReplicaConnectionName:
+ description: MultiCluster Connection name
+ type: string
myCnf:
description: |-
MyCnf allows to specify the my.cnf file mounted by Mariadb.
@@ -4929,6 +4932,9 @@ spec:
required:
- image
type: object
+ multiClusterReplicaConnectionName:
+ description: MultiCluster Connection name
+ type: string
primary:
description: Primary is the replication configuration for the
primary node.
@@ -4959,6 +4965,17 @@ spec:
This will be used as part of the scaling out and recovery operations, when new replicas are created.
If not provided, scale out and recovery operations will return an error.
properties:
+ logicalBackupTemplateRef:
+ description: |-
+ LogicalBackupTemplateRef is a reference to a Backup object that will be used as template to create the logical Backup
+ taken from the external MariaDB during external replication initialization and recovery. The template's Spec is copied
+ over (resources, pod template, etc.) and the controller overrides the fields that are managed automatically
+ (MariaDBRef, Storage, Args, Tables, Compression, MaxRetention).
+ properties:
+ name:
+ default: ""
+ type: string
+ type: object
physicalBackupTemplateRef:
description: |-
PhysicalBackupTemplateRef is a reference to a PhysicalBackup object that will be used as template to create a new PhysicalBackup object
@@ -5326,6 +5343,21 @@ spec:
- CurrentPos
- SlavePos
type: string
+ ignoreMaxLagSeconds:
+ description: |-
+ IgnoreMaxLagSeconds is to ignore the lag behind primary checks.
+ It's useful on situations when is preferred to keep sending read queries on a delayed (or with connection issues)
+ replica than stopping sending traffic. It could be useful when replicating from a external MariaDB when
+ connection issues with primary could happen.
+ If not provided, it defaults to false.
+ type: boolean
+ ignoreReplicationLivenessProbes:
+ description: |-
+ IgnoreReplicationLivenessProbes is to ignore liveness replication checks.
+ It's useful on situations when is preferred to keep sending read queries on a broken replicas
+ replica than stopping sending traffic.
+ If not provided, it defaults to false.
+ type: boolean
maxLagSeconds:
description: |-
MaxLagSeconds is the maximum number of seconds that replicas are allowed to lag behind the primary.
@@ -5385,6 +5417,65 @@ spec:
See: https://mariadb.com/docs/server/reference/sql-functions/secondary-functions/miscellaneous-functions/master_gtid_wait
type: string
type: object
+ replicaFromExternal:
+ description: ReplicaFromExternal specifies whether the replica
+ should be created from an external MariaDB instance.
+ properties:
+ connectionRetries:
+ description: ConnectionRetries to be used when the replica
+ connects to the primary.
+ type: integer
+ connectionTimeout:
+ description: ConnectionTimeout to be used when the replica
+ connects to the primary.
+ type: string
+ filteredReplicaTables:
+ description: |-
+ FilteredReplicaTables is an optional list of tables in "database.table" format to replicate.
+ When set, the logical backup will only include these tables and the replication will be
+ configured with replicate_do_table for each entry. GTID strict mode is automatically
+ disabled when this field is set, as partial replication is incompatible with it.
+ items:
+ type: string
+ type: array
+ gtid:
+ description: |-
+ Gtid indicates which Global Transaction ID should be used when connecting a replica to the master.
+ See: https://mariadb.com/kb/en/gtid/#using-current_pos-vs-slave_pos.
+ enum:
+ - CurrentPos
+ - SlavePos
+ type: string
+ healthCheckInterval:
+ description: HealthCheckInterval to be used when the replica
+ connects to the primary.
+ type: string
+ mariaDbRef:
+ description: MariaDBRef is a reference to a MariaDB object.
+ properties:
+ kind:
+ description: Kind of the referent.
+ type: string
+ name:
+ type: string
+ namespace:
+ type: string
+ waitForIt:
+ default: true
+ description: WaitForIt indicates whether the controller
+ using this reference should wait for MariaDB to be ready.
+ type: boolean
+ type: object
+ serverIdOffset:
+ description: |-
+ ServerIdOffset to be used on the replicas. Each replica gets server_id = podIndex + offset.
+ If not set, the operator auto-discovers a non-colliding offset by querying the external MariaDB
+ for the server ids already in use, leaving room above them for scale out and other clusters. The
+ discovered value is persisted to status.externalReplication.serverIdOffset and computed only once.
+ type: integer
+ required:
+ - mariaDbRef
+ type: object
semiSyncAckTimeout:
description: |-
SemiSyncAckTimeout for the replica to acknowledge transactions to the primary.
@@ -6583,7 +6674,8 @@ spec:
is required to avoid split brain situations for Galera. Use ''mariadb.spec.replicasAllowEvenNumber:
true'' to disable this validation.'
rule: '!has(self.galera) || !self.galera.enabled || (self.replicas %
- 2 == 1 || self.replicasAllowEvenNumber)'
+ 2 == 1 || self.replicasAllowEvenNumber ) || ( has(self.replication)
+ && has(self.replication.replicaFromExternal) )'
status:
description: MariaDBStatus defines the observed state of MariaDB
properties:
@@ -6665,6 +6757,16 @@ spec:
from spec.image. This can happen if the image uses a digest (e.g. sha256) instead
of a version tag.
type: string
+ externalReplication:
+ description: ExternalReplication is the status of external replication.
+ properties:
+ serverIdOffset:
+ description: |-
+ ServerIdOffset is the server_id offset auto-discovered by querying the external MariaDB for the
+ server ids already in use. It is computed only once and persisted here so it stays stable and
+ does not need to be queried again. It is not set when a manual serverIdOffset is configured.
+ type: integer
+ type: object
galeraRecovery:
description: GaleraRecovery is the Galera recovery current state.
properties:
diff --git a/config/crd/bases/k8s.mariadb.com_restores.yaml b/config/crd/bases/k8s.mariadb.com_restores.yaml
index e043a9408f..a9bce467b5 100644
--- a/config/crd/bases/k8s.mariadb.com_restores.yaml
+++ b/config/crd/bases/k8s.mariadb.com_restores.yaml
@@ -367,6 +367,10 @@ spec:
type: string
description: NodeSelector to be used in the Pod.
type: object
+ podIndex:
+ description: PodIndex is the StatefulSet index of pod to restore.
+ Used to bootstrap nodes on external replication.
+ type: integer
podMetadata:
description: PodMetadata defines extra metadata for the Pod.
properties:
diff --git a/deploy/charts/mariadb-operator-crds/templates/crds.yaml b/deploy/charts/mariadb-operator-crds/templates/crds.yaml
index b4342a333f..eb85bb7186 100644
--- a/deploy/charts/mariadb-operator-crds/templates/crds.yaml
+++ b/deploy/charts/mariadb-operator-crds/templates/crds.yaml
@@ -300,8 +300,9 @@ spec:
- gzip
type: string
databases:
- description: Databases defines the logical databases to be backed
- up. If not provided, all databases are backed up.
+ description: |-
+ Databases defines the logical databases to be backed up. If not provided, all databases are backed up.
+ Mutually exclusive with Tables.
items:
type: string
type: array
@@ -1031,6 +1032,14 @@ spec:
format: int32
minimum: 0
type: integer
+ tables:
+ description: |-
+ Tables defines specific tables to be backed up, in "database.table" format. Entries may span
+ multiple databases; when they do, --ignore-table flags are built at runtime by querying
+ information_schema so that the dump remains a single consistent transaction. Mutually exclusive with Databases.
+ items:
+ type: string
+ type: array
timeZone:
description: TimeZone defines the timezone associated with the cron
expression.
@@ -1631,6 +1640,12 @@ spec:
description: ExternalMariaDBSpec defines the desired state of an External
MariaDB
properties:
+ binlogPort:
+ description: Binlog proxy router port of the external MariaDB. Useful
+ when the external MariaDB is behind a Maxscale and using the Binlogrouter
+ to expose the binlog stream.
+ format: int32
+ type: integer
connection:
description: Connection defines a template to configure a Connection
for the external MariaDB.
@@ -5684,6 +5699,9 @@ spec:
to a member in the 'members' field, containing its full specification.
type: string
type: object
+ multiClusterReplicaConnectionName:
+ description: MultiCluster Connection name
+ type: string
myCnf:
description: |-
MyCnf allows to specify the my.cnf file mounted by Mariadb.
@@ -7095,6 +7113,9 @@ spec:
required:
- image
type: object
+ multiClusterReplicaConnectionName:
+ description: MultiCluster Connection name
+ type: string
primary:
description: Primary is the replication configuration for the
primary node.
@@ -7125,6 +7146,17 @@ spec:
This will be used as part of the scaling out and recovery operations, when new replicas are created.
If not provided, scale out and recovery operations will return an error.
properties:
+ logicalBackupTemplateRef:
+ description: |-
+ LogicalBackupTemplateRef is a reference to a Backup object that will be used as template to create the logical Backup
+ taken from the external MariaDB during external replication initialization and recovery. The template's Spec is copied
+ over (resources, pod template, etc.) and the controller overrides the fields that are managed automatically
+ (MariaDBRef, Storage, Args, Tables, Compression, MaxRetention).
+ properties:
+ name:
+ default: ""
+ type: string
+ type: object
physicalBackupTemplateRef:
description: |-
PhysicalBackupTemplateRef is a reference to a PhysicalBackup object that will be used as template to create a new PhysicalBackup object
@@ -7492,6 +7524,21 @@ spec:
- CurrentPos
- SlavePos
type: string
+ ignoreMaxLagSeconds:
+ description: |-
+ IgnoreMaxLagSeconds is to ignore the lag behind primary checks.
+ It's useful on situations when is preferred to keep sending read queries on a delayed (or with connection issues)
+ replica than stopping sending traffic. It could be useful when replicating from a external MariaDB when
+ connection issues with primary could happen.
+ If not provided, it defaults to false.
+ type: boolean
+ ignoreReplicationLivenessProbes:
+ description: |-
+ IgnoreReplicationLivenessProbes is to ignore liveness replication checks.
+ It's useful on situations when is preferred to keep sending read queries on a broken replicas
+ replica than stopping sending traffic.
+ If not provided, it defaults to false.
+ type: boolean
maxLagSeconds:
description: |-
MaxLagSeconds is the maximum number of seconds that replicas are allowed to lag behind the primary.
@@ -7551,6 +7598,65 @@ spec:
See: https://mariadb.com/docs/server/reference/sql-functions/secondary-functions/miscellaneous-functions/master_gtid_wait
type: string
type: object
+ replicaFromExternal:
+ description: ReplicaFromExternal specifies whether the replica
+ should be created from an external MariaDB instance.
+ properties:
+ connectionRetries:
+ description: ConnectionRetries to be used when the replica
+ connects to the primary.
+ type: integer
+ connectionTimeout:
+ description: ConnectionTimeout to be used when the replica
+ connects to the primary.
+ type: string
+ filteredReplicaTables:
+ description: |-
+ FilteredReplicaTables is an optional list of tables in "database.table" format to replicate.
+ When set, the logical backup will only include these tables and the replication will be
+ configured with replicate_do_table for each entry. GTID strict mode is automatically
+ disabled when this field is set, as partial replication is incompatible with it.
+ items:
+ type: string
+ type: array
+ gtid:
+ description: |-
+ Gtid indicates which Global Transaction ID should be used when connecting a replica to the master.
+ See: https://mariadb.com/kb/en/gtid/#using-current_pos-vs-slave_pos.
+ enum:
+ - CurrentPos
+ - SlavePos
+ type: string
+ healthCheckInterval:
+ description: HealthCheckInterval to be used when the replica
+ connects to the primary.
+ type: string
+ mariaDbRef:
+ description: MariaDBRef is a reference to a MariaDB object.
+ properties:
+ kind:
+ description: Kind of the referent.
+ type: string
+ name:
+ type: string
+ namespace:
+ type: string
+ waitForIt:
+ default: true
+ description: WaitForIt indicates whether the controller
+ using this reference should wait for MariaDB to be ready.
+ type: boolean
+ type: object
+ serverIdOffset:
+ description: |-
+ ServerIdOffset to be used on the replicas. Each replica gets server_id = podIndex + offset.
+ If not set, the operator auto-discovers a non-colliding offset by querying the external MariaDB
+ for the server ids already in use, leaving room above them for scale out and other clusters. The
+ discovered value is persisted to status.externalReplication.serverIdOffset and computed only once.
+ type: integer
+ required:
+ - mariaDbRef
+ type: object
semiSyncAckTimeout:
description: |-
SemiSyncAckTimeout for the replica to acknowledge transactions to the primary.
@@ -8749,7 +8855,8 @@ spec:
is required to avoid split brain situations for Galera. Use ''mariadb.spec.replicasAllowEvenNumber:
true'' to disable this validation.'
rule: '!has(self.galera) || !self.galera.enabled || (self.replicas %
- 2 == 1 || self.replicasAllowEvenNumber)'
+ 2 == 1 || self.replicasAllowEvenNumber ) || ( has(self.replication)
+ && has(self.replication.replicaFromExternal) )'
status:
description: MariaDBStatus defines the observed state of MariaDB
properties:
@@ -8831,6 +8938,16 @@ spec:
from spec.image. This can happen if the image uses a digest (e.g. sha256) instead
of a version tag.
type: string
+ externalReplication:
+ description: ExternalReplication is the status of external replication.
+ properties:
+ serverIdOffset:
+ description: |-
+ ServerIdOffset is the server_id offset auto-discovered by querying the external MariaDB for the
+ server ids already in use. It is computed only once and persisted here so it stays stable and
+ does not need to be queried again. It is not set when a manual serverIdOffset is configured.
+ type: integer
+ type: object
galeraRecovery:
description: GaleraRecovery is the Galera recovery current state.
properties:
@@ -13443,6 +13560,10 @@ spec:
type: string
description: NodeSelector to be used in the Pod.
type: object
+ podIndex:
+ description: PodIndex is the StatefulSet index of pod to restore.
+ Used to bootstrap nodes on external replication.
+ type: integer
podMetadata:
description: PodMetadata defines extra metadata for the Pod.
properties:
diff --git a/deploy/crds/crds.yaml b/deploy/crds/crds.yaml
index b4342a333f..eb85bb7186 100644
--- a/deploy/crds/crds.yaml
+++ b/deploy/crds/crds.yaml
@@ -300,8 +300,9 @@ spec:
- gzip
type: string
databases:
- description: Databases defines the logical databases to be backed
- up. If not provided, all databases are backed up.
+ description: |-
+ Databases defines the logical databases to be backed up. If not provided, all databases are backed up.
+ Mutually exclusive with Tables.
items:
type: string
type: array
@@ -1031,6 +1032,14 @@ spec:
format: int32
minimum: 0
type: integer
+ tables:
+ description: |-
+ Tables defines specific tables to be backed up, in "database.table" format. Entries may span
+ multiple databases; when they do, --ignore-table flags are built at runtime by querying
+ information_schema so that the dump remains a single consistent transaction. Mutually exclusive with Databases.
+ items:
+ type: string
+ type: array
timeZone:
description: TimeZone defines the timezone associated with the cron
expression.
@@ -1631,6 +1640,12 @@ spec:
description: ExternalMariaDBSpec defines the desired state of an External
MariaDB
properties:
+ binlogPort:
+ description: Binlog proxy router port of the external MariaDB. Useful
+ when the external MariaDB is behind a Maxscale and using the Binlogrouter
+ to expose the binlog stream.
+ format: int32
+ type: integer
connection:
description: Connection defines a template to configure a Connection
for the external MariaDB.
@@ -5684,6 +5699,9 @@ spec:
to a member in the 'members' field, containing its full specification.
type: string
type: object
+ multiClusterReplicaConnectionName:
+ description: MultiCluster Connection name
+ type: string
myCnf:
description: |-
MyCnf allows to specify the my.cnf file mounted by Mariadb.
@@ -7095,6 +7113,9 @@ spec:
required:
- image
type: object
+ multiClusterReplicaConnectionName:
+ description: MultiCluster Connection name
+ type: string
primary:
description: Primary is the replication configuration for the
primary node.
@@ -7125,6 +7146,17 @@ spec:
This will be used as part of the scaling out and recovery operations, when new replicas are created.
If not provided, scale out and recovery operations will return an error.
properties:
+ logicalBackupTemplateRef:
+ description: |-
+ LogicalBackupTemplateRef is a reference to a Backup object that will be used as template to create the logical Backup
+ taken from the external MariaDB during external replication initialization and recovery. The template's Spec is copied
+ over (resources, pod template, etc.) and the controller overrides the fields that are managed automatically
+ (MariaDBRef, Storage, Args, Tables, Compression, MaxRetention).
+ properties:
+ name:
+ default: ""
+ type: string
+ type: object
physicalBackupTemplateRef:
description: |-
PhysicalBackupTemplateRef is a reference to a PhysicalBackup object that will be used as template to create a new PhysicalBackup object
@@ -7492,6 +7524,21 @@ spec:
- CurrentPos
- SlavePos
type: string
+ ignoreMaxLagSeconds:
+ description: |-
+ IgnoreMaxLagSeconds is to ignore the lag behind primary checks.
+ It's useful on situations when is preferred to keep sending read queries on a delayed (or with connection issues)
+ replica than stopping sending traffic. It could be useful when replicating from a external MariaDB when
+ connection issues with primary could happen.
+ If not provided, it defaults to false.
+ type: boolean
+ ignoreReplicationLivenessProbes:
+ description: |-
+ IgnoreReplicationLivenessProbes is to ignore liveness replication checks.
+ It's useful on situations when is preferred to keep sending read queries on a broken replicas
+ replica than stopping sending traffic.
+ If not provided, it defaults to false.
+ type: boolean
maxLagSeconds:
description: |-
MaxLagSeconds is the maximum number of seconds that replicas are allowed to lag behind the primary.
@@ -7551,6 +7598,65 @@ spec:
See: https://mariadb.com/docs/server/reference/sql-functions/secondary-functions/miscellaneous-functions/master_gtid_wait
type: string
type: object
+ replicaFromExternal:
+ description: ReplicaFromExternal specifies whether the replica
+ should be created from an external MariaDB instance.
+ properties:
+ connectionRetries:
+ description: ConnectionRetries to be used when the replica
+ connects to the primary.
+ type: integer
+ connectionTimeout:
+ description: ConnectionTimeout to be used when the replica
+ connects to the primary.
+ type: string
+ filteredReplicaTables:
+ description: |-
+ FilteredReplicaTables is an optional list of tables in "database.table" format to replicate.
+ When set, the logical backup will only include these tables and the replication will be
+ configured with replicate_do_table for each entry. GTID strict mode is automatically
+ disabled when this field is set, as partial replication is incompatible with it.
+ items:
+ type: string
+ type: array
+ gtid:
+ description: |-
+ Gtid indicates which Global Transaction ID should be used when connecting a replica to the master.
+ See: https://mariadb.com/kb/en/gtid/#using-current_pos-vs-slave_pos.
+ enum:
+ - CurrentPos
+ - SlavePos
+ type: string
+ healthCheckInterval:
+ description: HealthCheckInterval to be used when the replica
+ connects to the primary.
+ type: string
+ mariaDbRef:
+ description: MariaDBRef is a reference to a MariaDB object.
+ properties:
+ kind:
+ description: Kind of the referent.
+ type: string
+ name:
+ type: string
+ namespace:
+ type: string
+ waitForIt:
+ default: true
+ description: WaitForIt indicates whether the controller
+ using this reference should wait for MariaDB to be ready.
+ type: boolean
+ type: object
+ serverIdOffset:
+ description: |-
+ ServerIdOffset to be used on the replicas. Each replica gets server_id = podIndex + offset.
+ If not set, the operator auto-discovers a non-colliding offset by querying the external MariaDB
+ for the server ids already in use, leaving room above them for scale out and other clusters. The
+ discovered value is persisted to status.externalReplication.serverIdOffset and computed only once.
+ type: integer
+ required:
+ - mariaDbRef
+ type: object
semiSyncAckTimeout:
description: |-
SemiSyncAckTimeout for the replica to acknowledge transactions to the primary.
@@ -8749,7 +8855,8 @@ spec:
is required to avoid split brain situations for Galera. Use ''mariadb.spec.replicasAllowEvenNumber:
true'' to disable this validation.'
rule: '!has(self.galera) || !self.galera.enabled || (self.replicas %
- 2 == 1 || self.replicasAllowEvenNumber)'
+ 2 == 1 || self.replicasAllowEvenNumber ) || ( has(self.replication)
+ && has(self.replication.replicaFromExternal) )'
status:
description: MariaDBStatus defines the observed state of MariaDB
properties:
@@ -8831,6 +8938,16 @@ spec:
from spec.image. This can happen if the image uses a digest (e.g. sha256) instead
of a version tag.
type: string
+ externalReplication:
+ description: ExternalReplication is the status of external replication.
+ properties:
+ serverIdOffset:
+ description: |-
+ ServerIdOffset is the server_id offset auto-discovered by querying the external MariaDB for the
+ server ids already in use. It is computed only once and persisted here so it stays stable and
+ does not need to be queried again. It is not set when a manual serverIdOffset is configured.
+ type: integer
+ type: object
galeraRecovery:
description: GaleraRecovery is the Galera recovery current state.
properties:
@@ -13443,6 +13560,10 @@ spec:
type: string
description: NodeSelector to be used in the Pod.
type: object
+ podIndex:
+ description: PodIndex is the StatefulSet index of pod to restore.
+ Used to bootstrap nodes on external replication.
+ type: integer
podMetadata:
description: PodMetadata defines extra metadata for the Pod.
properties:
diff --git a/docs/README.md b/docs/README.md
index 90a550225d..1b29d06cd6 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -34,6 +34,7 @@
- [Metrics](./metrics.md)
- [SQL resources](./sql_resources.md)
- [External MariaDB](./external_mariadb.md)
+- [External replication](./external_replication.md)
- [Metadata](./metadata.md)
- [Suspend reconciliation](./suspend.md)
- [Maintenance](./maintenance.md)
diff --git a/docs/api_reference.md b/docs/api_reference.md
index 474e21af1e..6f2f826be2 100644
--- a/docs/api_reference.md
+++ b/docs/api_reference.md
@@ -198,7 +198,8 @@ _Appears in:_
| `storage` _[BackupStorage](#backupstorage)_ | Storage defines the final storage for backups. | | Required: \{\}
|
| `schedule` _[Schedule](#schedule)_ | Schedule defines when the Backup will be taken. | | |
| `maxRetention` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.36/#duration-v1-meta)_ | MaxRetention defines the retention policy for backups. Old backups will be cleaned up by the Backup Job.
It defaults to 30 days. | | |
-| `databases` _string array_ | Databases defines the logical databases to be backed up. If not provided, all databases are backed up. | | |
+| `databases` _string array_ | Databases defines the logical databases to be backed up. If not provided, all databases are backed up.
Mutually exclusive with Tables. | | |
+| `tables` _string array_ | Tables defines specific tables to be backed up, in "database.table" format. Entries may span
multiple databases; when they do, --ignore-table flags are built at runtime by querying
information_schema so that the dump remains a single consistent transaction. Mutually exclusive with Databases. | | |
| `ignoreGlobalPriv` _boolean_ | IgnoreGlobalPriv indicates to ignore the mysql.global_priv in backups.
If not provided, it will default to true when the referred MariaDB instance has Galera enabled and otherwise to false.
See: https://github.com/mariadb-operator/mariadb-operator/issues/556 | | |
| `logLevel` _string_ | LogLevel to be used in the Backup Job. It defaults to 'info'. | info | Enum: [debug info warn error dpanic panic fatal]
|
| `backoffLimit` _integer_ | BackoffLimit defines the maximum number of attempts to successfully take a Backup. | | |
@@ -774,6 +775,7 @@ _Appears in:_
| `inheritMetadata` _[Metadata](#metadata)_ | InheritMetadata defines the metadata to be inherited by children resources. | | |
| `host` _string_ | Hostname of the external MariaDB. | | Required: \{\}
|
| `port` _integer_ | Port of the external MariaDB. | 3306 | |
+| `binlogPort` _integer_ | Binlog proxy router port of the external MariaDB. Useful when the external MariaDB is behind a Maxscale and using the Binlogrouter to expose the binlog stream. | | |
| `username` _string_ | Username is the username to connect to the external MariaDB. | | Required: \{\}
|
| `passwordSecretKeyRef` _[SecretKeySelector](#secretkeyselector)_ | PasswordSecretKeyRef is a reference to the password to connect to the external MariaDB. | | |
| `tls` _[ExternalTLS](#externaltls)_ | TLS defines the PKI to be used with the external MariaDB. | | |
@@ -1024,6 +1026,7 @@ See: https://mariadb.com/kb/en/gtid/#using-current_pos-vs-slave_pos.
_Appears in:_
+- [ReplicaFromExternal](#replicafromexternal)
- [ReplicaReplication](#replicareplication)
| Field | Description |
@@ -1406,6 +1409,7 @@ _Appears in:_
- [GrantSpec](#grantspec)
- [MaxScaleSpec](#maxscalespec)
- [PhysicalBackupSpec](#physicalbackupspec)
+- [ReplicaFromExternal](#replicafromexternal)
- [RestoreSpec](#restorespec)
- [SqlJobSpec](#sqljobspec)
- [UserSpec](#userspec)
@@ -1493,6 +1497,7 @@ _Appears in:_
| `secondaryService` _[ServiceTemplate](#servicetemplate)_ | SecondaryService defines a template to configure the secondary Service object.
The network traffic of this Service will be routed to the secondary Pods. | | |
| `secondaryConnection` _[ConnectionTemplate](#connectiontemplate)_ | SecondaryConnection defines a template to configure the secondary Connection object.
This Connection provides the initial User access to the initial Database.
It will make use of the SecondaryService to route network traffic to the secondary Pods. | | |
| `maintenance` _[MariaDBMaintenance](#mariadbmaintenance)_ | Maintenance defines different capabilities of the operator to allow for maintenance to be performed on the DB.
Not to be confused with `suspend`, maintenance does not interfere with the normal reconciliation of the operator. | | |
+| `multiClusterReplicaConnectionName` _string_ | MultiCluster Connection name | | |
#### MariaDBVolume
@@ -2614,9 +2619,33 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `physicalBackupTemplateRef` _[LocalObjectReference](#localobjectreference)_ | PhysicalBackupTemplateRef is a reference to a PhysicalBackup object that will be used as template to create a new PhysicalBackup object
used synchronize the data from an up to date replica to the new replica to be bootstrapped. | | Required: \{\}
|
+| `logicalBackupTemplateRef` _[LocalObjectReference](#localobjectreference)_ | LogicalBackupTemplateRef is a reference to a Backup object that will be used as template to create the logical Backup
taken from the external MariaDB during external replication initialization and recovery. The template's Spec is copied
over (resources, pod template, etc.) and the controller overrides the fields that are managed automatically
(MariaDBRef, Storage, Args, Tables, Compression, MaxRetention). | | |
| `restoreJob` _[Job](#job)_ | RestoreJob defines additional properties for the Job used to perform the restoration. | | |
+#### ReplicaFromExternal
+
+
+
+ReplicaFromExternal is the replication configuration from external servers.
+
+
+
+_Appears in:_
+- [Replication](#replication)
+- [ReplicationSpec](#replicationspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `mariaDbRef` _[MariaDBRef](#mariadbref)_ | MariaDBRef is a reference to a MariaDB object. | | Required: \{\}
|
+| `gtid` _[Gtid](#gtid)_ | Gtid indicates which Global Transaction ID should be used when connecting a replica to the master.
See: https://mariadb.com/kb/en/gtid/#using-current_pos-vs-slave_pos. | | Enum: [CurrentPos SlavePos]
|
+| `connectionTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.36/#duration-v1-meta)_ | ConnectionTimeout to be used when the replica connects to the primary. | | |
+| `connectionRetries` _integer_ | ConnectionRetries to be used when the replica connects to the primary. | | |
+| `healthCheckInterval` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.36/#duration-v1-meta)_ | HealthCheckInterval to be used when the replica connects to the primary. | | |
+| `serverIdOffset` _integer_ | ServerIdOffset to be used on the replicas. Each replica gets server_id = podIndex + offset.
If not set, the operator auto-discovers a non-colliding offset by querying the external MariaDB
for the server ids already in use, leaving room above them for scale out and other clusters. The
discovered value is persisted to status.externalReplication.serverIdOffset and computed only once. | | |
+| `filteredReplicaTables` _string array_ | FilteredReplicaTables is an optional list of tables in "database.table" format to replicate.
When set, the logical backup will only include these tables and the replication will be
configured with replicate_do_table for each entry. GTID strict mode is automatically
disabled when this field is set, as partial replication is incompatible with it. | | |
+
+
#### ReplicaRecovery
@@ -2652,6 +2681,8 @@ _Appears in:_
| `gtid` _[Gtid](#gtid)_ | Gtid indicates which Global Transaction ID (GTID) position mode should be used when connecting a replica to the master.
By default, CurrentPos is used.
See: https://mariadb.com/docs/server/reference/sql-statements/administrative-sql-statements/replication-statements/change-master-to#master_use_gtid. | | Enum: [CurrentPos SlavePos]
|
| `connectionRetrySeconds` _integer_ | ConnectionRetrySeconds is the number of seconds that the replica will wait between connection retries.
See: https://mariadb.com/docs/server/reference/sql-statements/administrative-sql-statements/replication-statements/change-master-to#master_connect_retry. | | |
| `maxLagSeconds` _integer_ | MaxLagSeconds is the maximum number of seconds that replicas are allowed to lag behind the primary.
If a replica exceeds this threshold, it is marked as not ready and read queries will no longer be forwarded to it.
If not provided, it defaults to 0, which means that replicas are not allowed to lag behind the primary (recommended).
Lagged replicas will not be taken into account as candidates for the new primary during failover,
and they will block other operations, such as switchover and upgrade.
This field is not taken into account by MaxScale, you can define the maximum lag as router parameters.
See: https://mariadb.com/docs/maxscale/reference/maxscale-routers/maxscale-readwritesplit#max_replication_lag. | | |
+| `ignoreMaxLagSeconds` _boolean_ | IgnoreMaxLagSeconds is to ignore the lag behind primary checks.
It's useful on situations when is preferred to keep sending read queries on a delayed (or with connection issues)
replica than stopping sending traffic. It could be useful when replicating from a external MariaDB when
connection issues with primary could happen.
If not provided, it defaults to false. | | |
+| `ignoreReplicationLivenessProbes` _boolean_ | IgnoreReplicationLivenessProbes is to ignore liveness replication checks.
It's useful on situations when is preferred to keep sending read queries on a broken replicas
replica than stopping sending traffic.
If not provided, it defaults to false. | | |
| `syncTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.36/#duration-v1-meta)_ | SyncTimeout defines the timeout for the synchronization phase during switchover and failover operations.
During switchover, all replicas must be synced with the current primary before promoting the new primary.
During failover, the new primary must be synced before being promoted as primary. This implies processing all the events in the relay log.
When the timeout is reached, the operator restarts the operation from the beginning.
It defaults to 10s.
See: https://mariadb.com/docs/server/reference/sql-functions/secondary-functions/miscellaneous-functions/master_gtid_wait | | |
| `bootstrapFrom` _[ReplicaBootstrapFrom](#replicabootstrapfrom)_ | ReplicaBootstrapFrom defines the data sources used to bootstrap new replicas.
This will be used as part of the scaling out and recovery operations, when new replicas are created.
If not provided, scale out and recovery operations will return an error. | | |
| `recovery` _[ReplicaRecovery](#replicarecovery)_ | ReplicaRecovery defines how the replicas should be recovered after they enter an error state.
This process deletes data from faulty replicas and recreates them using the source defined in the bootstrapFrom field.
It is disabled by default, and it requires the bootstrapFrom field to be set. | | |
@@ -2679,11 +2710,13 @@ _Appears in:_
| `serverIdStartIndex` _integer_ | ServerIDStartIndex sets the start index of the MariaDB nodes. Each subsequent replica will increment this by 1.
It is immutable.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/replication-and-binary-log-system-variables#server_id | | |
| `semiSyncEnabled` _boolean_ | SemiSyncEnabled determines whether semi-synchronous replication is enabled.
Semi-synchronous replication requires that at least one replica should have sent an ACK to the primary node
before committing the transaction back to the client.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/semisynchronous-replication
It is enabled by default | | |
| `semiSyncAckTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.36/#duration-v1-meta)_ | SemiSyncAckTimeout for the replica to acknowledge transactions to the primary.
It requires semi-synchronous replication to be enabled.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/semisynchronous-replication#rpl_semi_sync_master_timeout | | |
+| `replicaFromExternal` _[ReplicaFromExternal](#replicafromexternal)_ | ReplicaFromExternal specifies whether the replica should be created from an external MariaDB instance. | | |
| `semiSyncWaitPoint` _[WaitPoint](#waitpoint)_ | SemiSyncWaitPoint determines whether the transaction should wait for an ACK after having synced the binlog (AfterSync)
or after having committed to the storage engine (AfterCommit, the default).
It requires semi-synchronous replication to be enabled.
See: https://mariadb.com/kb/en/semisynchronous-replication/#rpl_semi_sync_master_wait_point. | | Enum: [AfterSync AfterCommit]
|
| `syncBinlog` _integer_ | SyncBinlog indicates after how many events the binary log is synchronized to the disk.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/replication-and-binary-log-system-variables#sync_binlog | | |
| `initContainer` _[InitContainer](#initcontainer)_ | InitContainer is an init container that runs in the MariaDB Pod and co-operates with mariadb-operator. | | |
| `agent` _[Agent](#agent)_ | Agent is a sidecar agent that runs in the MariaDB Pod and co-operates with mariadb-operator. | | |
| `standaloneProbes` _boolean_ | StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
It is disabled by default | | |
+| `multiClusterReplicaConnectionName` _string_ | MultiCluster Connection name | | |
| `enabled` _boolean_ | Enabled is a flag to enable replication. | | |
@@ -2709,11 +2742,13 @@ _Appears in:_
| `serverIdStartIndex` _integer_ | ServerIDStartIndex sets the start index of the MariaDB nodes. Each subsequent replica will increment this by 1.
It is immutable.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/replication-and-binary-log-system-variables#server_id | | |
| `semiSyncEnabled` _boolean_ | SemiSyncEnabled determines whether semi-synchronous replication is enabled.
Semi-synchronous replication requires that at least one replica should have sent an ACK to the primary node
before committing the transaction back to the client.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/semisynchronous-replication
It is enabled by default | | |
| `semiSyncAckTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.36/#duration-v1-meta)_ | SemiSyncAckTimeout for the replica to acknowledge transactions to the primary.
It requires semi-synchronous replication to be enabled.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/semisynchronous-replication#rpl_semi_sync_master_timeout | | |
+| `replicaFromExternal` _[ReplicaFromExternal](#replicafromexternal)_ | ReplicaFromExternal specifies whether the replica should be created from an external MariaDB instance. | | |
| `semiSyncWaitPoint` _[WaitPoint](#waitpoint)_ | SemiSyncWaitPoint determines whether the transaction should wait for an ACK after having synced the binlog (AfterSync)
or after having committed to the storage engine (AfterCommit, the default).
It requires semi-synchronous replication to be enabled.
See: https://mariadb.com/kb/en/semisynchronous-replication/#rpl_semi_sync_master_wait_point. | | Enum: [AfterSync AfterCommit]
|
| `syncBinlog` _integer_ | SyncBinlog indicates after how many events the binary log is synchronized to the disk.
See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/replication-and-binary-log-system-variables#sync_binlog | | |
| `initContainer` _[InitContainer](#initcontainer)_ | InitContainer is an init container that runs in the MariaDB Pod and co-operates with mariadb-operator. | | |
| `agent` _[Agent](#agent)_ | Agent is a sidecar agent that runs in the MariaDB Pod and co-operates with mariadb-operator. | | |
| `standaloneProbes` _boolean_ | StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
It is disabled by default | | |
+| `multiClusterReplicaConnectionName` _string_ | MultiCluster Connection name | | |
#### ResourceRequirements
@@ -2811,6 +2846,7 @@ _Appears in:_
| `targetRecoveryTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.36/#time-v1-meta)_ | TargetRecoveryTime is a RFC3339 (1970-01-01T00:00:00Z) date and time that defines the point in time recovery objective.
It is used to determine the closest restoration source in time. | | |
| `stagingStorage` _[StagingStorage](#stagingstorage)_ | StagingStorage defines the temporary storage used to keep external backups (i.e. S3) while they are being processed.
It defaults to an emptyDir volume, meaning that the backups will be temporarily stored in the node where the Restore Job is scheduled. | | |
| `mariaDbRef` _[MariaDBRef](#mariadbref)_ | MariaDBRef is a reference to a MariaDB object. | | Required: \{\}
|
+| `podIndex` _integer_ | PodIndex is the StatefulSet index of pod to restore. Used to bootstrap nodes on external replication. | | |
| `database` _string_ | Database defines the logical database to be restored. If not provided, all databases available in the backup are restored.
IMPORTANT: The database must previously exist. | | |
| `logLevel` _string_ | LogLevel to be used n the Backup Job. It defaults to 'info'. | info | Enum: [debug info warn error dpanic panic fatal]
|
| `backoffLimit` _integer_ | BackoffLimit defines the maximum number of attempts to successfully perform a Backup. | 5 | |
diff --git a/docs/development.md b/docs/development.md
index 0165803c33..31b95014bc 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -162,6 +162,8 @@ make test
```bash
make cluster
make install
+make install-csi-hostpath
+make install-trust-manager
make install-minio
make net
make test-int
diff --git a/docs/docker.md b/docs/docker.md
index c605afb429..909c1063f8 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -36,7 +36,7 @@
| MariaDB Operator |
- ghcr.io/mariadb-operator/mariadb-operator:26.6.0 |
+ ghcr.io/mariadb-operator/mariadb-operator:26.6.0-bandwidth.1 |
amd64 arm64 |
diff --git a/docs/external_mariadb.md b/docs/external_mariadb.md
index fc1fc82559..346e3d4569 100644
--- a/docs/external_mariadb.md
+++ b/docs/external_mariadb.md
@@ -5,7 +5,9 @@
## Table of contents
- [`ExternalMariaDB` configuration](#externalmariadb-configuration)
+- [Binlog proxy port](#binlog-proxy-port)
- [Supported objects](#supported-objects)
+- [External replication](#external-replication)
## `ExternalMariaDB` configuration
@@ -86,6 +88,25 @@ As a result, you will be able to specify the `ExternalMariaDB` as a reference in
As part of the `ExternalMariaDB` reconciliation, a `Connection` will be created whenever the `connection` template is specified. This could be handy to track the external connection status and declaratively create a connection string in a `Secret` to be consumed by applications to connect to the external `MariaDB`.
+## Binlog proxy port
+
+When the external MariaDB is exposed behind a [MaxScale](https://mariadb.com/docs/maxscale/) using the [Binlogrouter](https://mariadb.com/docs/maxscale/maxscale-archive/mariadb-maxscale-25-08/maxscale-25-08-routers/mariadb-maxscale-25-08-maxscale-25-08-binlogrouter) to expose the binlog stream, you can set the `binlogPort` field. This is only relevant for [external replication](#external-replication): replicas will use `binlogPort` to stream the binary logs, while `port` is still used for the rest of the operations.
+
+```yaml
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: ExternalMariaDB
+metadata:
+ name: external-mariadb
+spec:
+ host: maxscale.example.com
+ port: 3306
+ binlogPort: 4000
+ username: root
+ passwordSecretKeyRef:
+ name: mariadb
+ key: password
+```
+
## Supported objects
Currently, the `ExternalMariaDB` resource is supported by the following objects:
@@ -118,4 +139,8 @@ spec:
retryInterval: 30s
```
-When the previous example gets reconciled, an user will be created in the referred external MariaDB instance.
\ No newline at end of file
+When the previous example gets reconciled, an user will be created in the referred external MariaDB instance.
+
+## External replication
+
+Besides managing resources, an `ExternalMariaDB` can also be used as a replication source to create a cluster of replicas running inside Kubernetes. See [External replication](./external_replication.md) for more details.
\ No newline at end of file
diff --git a/docs/external_replication.md b/docs/external_replication.md
new file mode 100644
index 0000000000..f585d32e08
--- /dev/null
+++ b/docs/external_replication.md
@@ -0,0 +1,293 @@
+# External replication
+
+`mariadb-operator` supports replication from an external MariaDB instance i.e. running outside of the Kubernetes cluster where the operator runs. This feature allows us to create a cluster of replicas of an external MariaDB.
+
+## Table of contents
+
+- [`ExternalMariaDB` configuration](#externalmariadb-configuration)
+- [`MariaDB` configuration](#mariadb-configuration)
+- [Bootstrapping and recovery sources](#bootstrapping-and-recovery-sources)
+- [Replication initialization](#replication-initialization)
+- [Scaling out](#scaling-out)
+- [Replica recovery](#replica-recovery)
+ - [Errors that trigger a recovery](#errors-that-trigger-a-recovery)
+ - [How recovery works](#how-recovery-works)
+ - [Timeouts](#timeouts)
+- [Automatic serverId offset](#automatic-serverid-offset)
+- [Backup validity and retention](#backup-validity-and-retention)
+- [Filtered replication](#filtered-replication)
+ - [How the filtered backup is taken](#how-the-filtered-backup-is-taken)
+- [Replicating through a MaxScale Binlogrouter](#replicating-through-a-maxscale-binlogrouter)
+- [Services considerations](#services-considerations)
+
+
+## `ExternalMariaDB` configuration
+
+To setup the external replication first we need to add our source MariaDB as an `ExternalMariaDB`:
+```yaml
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: ExternalMariaDB
+metadata:
+ name: external-mariadb
+spec:
+ host: mariadb.example.com
+ port: 3306
+ username: root
+ passwordSecretKeyRef:
+ name: mariadb
+ key: password
+ connection:
+ secretName: external-mariadb
+ healthCheck:
+ interval: 5s
+```
+
+See [External MariaDB](./external_mariadb.md) for the full `ExternalMariaDB` reference, including TLS configuration.
+
+## `MariaDB` configuration
+
+With the `ExternalMariaDB` created, we just need to define a regular `MariaDB` object with replication enabled and use the `replicaFromExternal` property to point it to our external database:
+
+```yaml
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: MariaDB
+metadata:
+ name: external-replicas
+spec:
+ storage:
+ size: 10Gi
+ replicas: 3
+ replication:
+ enabled: true
+ replica:
+ bootstrapFrom:
+ physicalBackupTemplateRef:
+ name: physicalbackup-external
+ replicaFromExternal:
+ mariaDbRef:
+ name: external-mariadb
+ kind: ExternalMariaDB
+ serverIdOffset: 30
+ service:
+ type: ClusterIP
+ primaryService:
+ type: ClusterIP
+ secondaryService:
+ type: ClusterIP
+```
+
+When applied it will create 3 new replicas from the external database. The operator will create a logical backup of the external MariaDB, restore it on each Pod and configure the replication.
+
+The `replicaFromExternal` field supports the following options:
+
+| Field | Description | Default |
+| ----- | ----------- | ------- |
+| `mariaDbRef` | Reference to the `ExternalMariaDB` object that acts as the replication source. Immutable. | - |
+| `serverIdOffset` | `serverId` offset used on the replicas (each replica gets `server_id = podIndex + offset`), to avoid conflicting with other replicas or with the source server. When unset, the operator auto-discovers a non-colliding offset (see [Automatic serverId offset](#automatic-serverid-offset)). | auto-discovered |
+| `gtid` | Global Transaction ID position mode used when connecting a replica to the source (`CurrentPos` or `SlavePos`). | `CurrentPos` |
+| `connectionTimeout` | Timeout used when the replica connects to the source. | - |
+| `connectionRetries` | Number of connection retries when the replica connects to the source. | - |
+| `healthCheckInterval` | Interval used to health-check the connection to the source. | `15s` |
+| `filteredReplicaTables` | Optional list of `database.table` entries to replicate. See [Filtered replication](#filtered-replication). | - |
+
+## Automatic serverId offset
+
+Every replica needs a `server_id` that is unique across everything connected to the source: its own replicas, the source itself, and any other cluster replicating from the same source. Each replica Pod is assigned `server_id = podIndex + offset`.
+
+When `serverIdOffset` is **not set**, the operator discovers a non-colliding offset automatically:
+
+1. It queries the source for the `server_id`s already in use — the source's own `server_id` plus the `server_id` of every replica currently registered against it (`SHOW SLAVE HOSTS`). If a `binlogPort` is configured on the `ExternalMariaDB`, the query is issued against that port, so the MaxScale Binlogrouter reports the replicas registered across **all** clusters, not just this one.
+2. It picks `offset = max(in-use serverId) + 100`, leaving room above the highest existing `server_id` for this cluster to scale out and for other clusters to claim their own blocks.
+3. The result is persisted to `status.externalReplication.serverIdOffset` and **computed only once** — it is never re-queried, so it stays stable and does not cause Pod restarts.
+
+The discovery runs before the `StatefulSet` is created; while the source is not reachable the reconciliation waits (external replication is not possible until the source is reachable anyway), so Pods are always created with the final `server_id`.
+
+Setting `serverIdOffset` explicitly disables the discovery and uses the provided value verbatim (`status.externalReplication` is left unset). This keeps existing clusters unaffected.
+
+## Bootstrapping and recovery sources
+
+New replicas (created during the initial provisioning, when scaling out, or during recovery) are bootstrapped from the data sources defined in `replica.bootstrapFrom`:
+
+```yaml
+spec:
+ replication:
+ enabled: true
+ replica:
+ bootstrapFrom:
+ physicalBackupTemplateRef:
+ name: physicalbackup-external
+ logicalBackupTemplateRef:
+ name: backup-external
+ replicaFromExternal:
+ mariaDbRef:
+ name: external-mariadb
+ kind: ExternalMariaDB
+```
+
+* `physicalBackupTemplateRef` (**required**): reference to a `PhysicalBackup` object used as a template to create a new `PhysicalBackup`. This is used to synchronize the data from an **up to date replica** to a new replica being bootstrapped during scale out and recovery. Its `target` should typically be set to `PreferReplica` so the backup is taken from a healthy replica when one is available (see [Replica recovery](#replica-recovery)).
+* `logicalBackupTemplateRef` (optional): reference to a `Backup` object used as a template for the logical `Backup` taken **from the external MariaDB** during initialization and as a fallback during recovery. The template's `Spec` is copied over (resources, Pod template, etc.) and the operator overrides the fields it manages automatically (`mariaDbRef`, `storage`, `args`, `tables`, `compression`, `maxRetention`). If not provided, the operator uses sensible defaults for the logical backup.
+
+Because `bootstrapFrom` is consumed by the scale out and recovery flows, and **replica recovery is enabled by default for external replication**, `bootstrapFrom` is effectively required: if it is not provided, scale out and recovery operations return an error. Recovery can be tuned (or disabled) through `replica.recovery`:
+
+```yaml
+spec:
+ replication:
+ replica:
+ recovery:
+ enabled: true
+ errorDurationThreshold: 5m
+```
+
+When replicating from an external server, connection issues with the source are more likely. The following `replica` options are useful to avoid dropping read traffic in those situations:
+
+* `ignoreMaxLagSeconds`: keep forwarding read queries to a replica even if it lags behind the source.
+* `ignoreReplicationLivenessProbes`: keep forwarding read queries to a replica even if the replication liveness checks fail.
+
+## Replication initialization
+
+When a new external-replica `MariaDB` cluster is created, the operator runs a **replication initialization** before configuring replication. This happens once per cluster (tracked by the `ExternalReplInitialized` status condition):
+
+1. **Take a logical backup from the external MariaDB.** The operator creates a logical `Backup` object pointing to the `ExternalMariaDB`. The dump is taken with `--master-data=1 --gtid --single-transaction` so it captures a consistent GTID position, and the `mysql.global_priv` table is excluded so users/grants are not imported (manage them with `User`/`Grant` objects instead). If a `logicalBackupTemplateRef` is configured, its spec is used as the base for this `Backup`. When [filtered replication](#filtered-replication) is enabled, the dump only includes the listed tables.
+2. **Restore on every Pod.** The operator restores that single logical backup into each Pod of the cluster (primary and replicas), resetting the binary logs (`RESET MASTER`) on each one beforehand. Initialization waits until all Pods have a completed `Restore`, then cleans the `Restore` objects up.
+3. **Configure replication.** Each Pod is pointed at the external MariaDB (using `binlogPort` if set, see [below](#replicating-through-a-maxscale-binlogrouter)) and replication is started from the GTID captured in the backup.
+
+The logical backup is reused across Pods and across restarts as long as it is still valid (see [Backup validity and retention](#backup-validity-and-retention)); the operator only takes a new one when none exists or the existing one has expired.
+
+## Scaling out
+
+Increasing `spec.replicas` triggers a **scale out**. New replicas are always bootstrapped from a **physical backup** (it is faster than a logical restore and keeps the existing replicas as the source instead of hitting the external server):
+
+1. The operator ensures a `PhysicalBackup` exists for the scale out, created from `replica.bootstrapFrom.physicalBackupTemplateRef`.
+2. PVCs are provisioned for the new Pods (from a `VolumeSnapshot` if the template uses one, otherwise via a restore Job).
+3. The physical backup is restored into the new Pods through rolling init Jobs, replication is configured against the external MariaDB and the new replicas join the cluster.
+
+Scale out only starts when all current replicas are ready, and it can be rolled back at any point by setting `spec.replicas` back to the previous value.
+
+## Replica recovery
+
+Replicas can break for reasons that the operator cannot fix just by restarting replication (corrupted data, conflicting GTIDs, missing binlogs on the source, etc.). When recovery is enabled, the operator rebuilds the affected replica from scratch, one replica at a time to avoid service disruption.
+
+### Errors that trigger a recovery
+
+The operator inspects the last I/O and SQL error reported by each replica and classifies it:
+
+* **Recoverable errors — recovery is triggered immediately:**
+ * I/O errors: `1236` (fatal error reading binlog from master), `1945`, `1955` (requested GTID not in master's binlog), `1947` (GTID conflicts with a more recent one in the binlog), `1951` (master is missing the requested GTID).
+ * SQL errors: `1062` (duplicate entry), `1032` (can't find record), `1034` (incorrect key file), `1049` (unknown database), `1146` (table doesn't exist).
+* **Non-recoverable errors — recovery is never triggered** (these are transient or operational and rebuilding the replica would not help): `2003` (can't connect), `2013` (lost connection to master), `1158` (error reading communication packets), `2026` (TLS handshake failed), `1045` (access denied), `1130` (host not allowed), `1129` (host blocked), `1040` (too many connections).
+* **Any other error** (not in either list): recovery is triggered only if the error persists for longer than `replica.recovery.errorDurationThreshold` (default `5m`). This avoids reacting to short-lived transient failures.
+
+### How recovery works
+
+For each replica that needs to be recovered, the operator tries a **physical backup first** and falls back to a **logical backup** from the external source if a physical backup cannot be produced:
+
+1. **Ensure a physical backup is available.** The operator looks for an existing recovery `PhysicalBackup`; if it does not exist it creates one from `physicalBackupTemplateRef` with an immediate schedule.
+2. **Pick a source node.** With `target: PreferReplica`, the backup is taken from a **healthy replica**: a secondary Pod that is `Ready`, has both the I/O and SQL replication threads running, and has been stable (no error transition) for at least **120 seconds**. This stability window avoids picking a replica that is only transiently healthy. If a healthy replica is found, the physical backup runs against it.
+3. **Fall back to a logical backup.** For external replication, if **no healthy replica is available**, no backup Job gets launched. After **240 seconds** without a launched Job, the operator gives up on the physical backup, takes (or reuses) a logical backup **from the external MariaDB**, and restores that into the broken replica instead. The stale physical backup is then cleaned up.
+4. **Rebuild the replica.** The faulty Pod and its PVC are deleted and recreated from the chosen backup (data directory is wiped first). Replication is then reconfigured against the external MariaDB.
+5. **Confirm recovery.** The operator waits for the Pod to become `Ready` and for the replica to report no I/O or SQL errors before marking the replica as recovered and cleaning up the recovery Jobs/backups.
+
+> If `bootstrapFrom` is not set, recovery cannot proceed and the operator emits a warning event and sets a recovery error on the `MariaDB` status.
+
+### Timeouts
+
+| Timeout | Value | Meaning |
+| ------- | ----- | ------- |
+| Replica stability window | `120s` (fixed) | A replica must be healthy and free of error transitions for this long before it is eligible as a physical backup source. |
+| Physical backup Job launch timeout | `240s` (fixed) | If no backup Job is launched within this window (e.g. because no healthy replica is available), recovery falls back to a logical backup from the external source. |
+| `errorDurationThreshold` | `5m` (configurable via `replica.recovery.errorDurationThreshold`) | How long an unclassified error must persist before triggering a recovery. Recoverable error codes trigger recovery immediately, ignoring this threshold. |
+
+## Backup validity and retention
+
+Both the logical and the physical backups used for external replication are validated against the **binary log retention of the source** before being reused, so the operator never restores from a backup whose binlogs are already gone (which would make replication impossible):
+
+* The retention period is read from the source's `binlog_expire_logs_seconds` system variable (for servers older than `10.6.1`, the day-based value is converted to seconds).
+* **Logical backup:** considered expired when its age (time since creation) exceeds the source's binlog retention. Expired logical backups are deleted and a fresh one is taken.
+* **Physical backup:** considered expired when its last schedule time is older than `now - binlogRetention`. Expired physical backups are destroyed and re-created. If the operator cannot read `binlog_expire_logs_seconds`, it conservatively forces a new backup.
+* The `maxRetention` of the generated backups is aligned with this same binlog retention period.
+
+Additional notes:
+
+* The backup storage size is the same as the storage size defined for the replicas.
+* The backups do not include users (`mysql.global_priv` is excluded) to avoid privilege issues and conflicts, mainly with the `root` user. Use the regular `User` and `Grant` objects to manage users and privileges on the replicas.
+
+## Filtered replication
+
+It is possible to replicate only a subset of tables from the external MariaDB by listing them in `replicaFromExternal.filteredReplicaTables`, using the `database.table` format. This enables replicating tables across multiple schemas:
+
+```yaml
+spec:
+ replication:
+ enabled: true
+ replica:
+ bootstrapFrom:
+ physicalBackupTemplateRef:
+ name: physicalbackup-external
+ replicaFromExternal:
+ mariaDbRef:
+ name: external-mariadb
+ kind: ExternalMariaDB
+ serverIdOffset: 30
+ filteredReplicaTables:
+ - db1.table1
+ - db1.table2
+```
+
+When `filteredReplicaTables` is set:
+
+* The logical backup taken from the external MariaDB only includes the listed tables (see [below](#how-the-filtered-backup-is-taken)).
+* Replication is configured with a `replicate_do_table` entry for each table.
+* GTID strict mode is automatically disabled, as partial replication is incompatible with it. This is equivalent to setting `replication.gtidStrictMode: false` and can still be overridden explicitly.
+
+### How the filtered backup is taken
+
+The logical backup is generated with `mariadb-dump`, on top of the common flags (`--single-transaction --events --routines`, `--master-data=1 --gtid` to capture the GTID position, and `--ignore-table=mysql.global_priv` so users/grants are not exported). The way the tables are selected depends on whether all the entries in `filteredReplicaTables` belong to the **same schema** or are spread across **multiple schemas**:
+
+**Single schema** (all tables share the same database, e.g. `db1.table1`, `db1.table2`):
+
+The operator dumps the database and explicitly lists the tables to include:
+
+```bash
+mariadb-dump ... --databases db1 --tables table1 table2
+```
+
+`--databases` (rather than the plain positional form) is used so the dump emits the database-context statements needed for a clean restore, and `--tables` restricts the dump to the listed tables. On restore, the target database is set upfront (`--database db1`).
+
+**Multiple schemas** (tables span more than one database, e.g. `db1.table1`, `db2.table2`):
+
+`mariadb-dump` cannot mix `--tables` with several databases, so the operator instead dumps all the involved schemas and **excludes everything that is not in the list** using dynamically built `--ignore-table` flags:
+
+```bash
+# the ignore flags are computed at backup time...
+mapfile -t MARIADB_IGNORE_ARGS < <(mariadb ... -BNe "")
+# ...and appended to the dump command
+mariadb-dump ... --databases db1 db2 "${MARIADB_IGNORE_ARGS[@]}"
+```
+
+* The exclusion list is computed by querying `information_schema.TABLES` for every `BASE TABLE` **and** `VIEW` in the target schemas that is **not** in `filteredReplicaTables`, producing one `--ignore-table=.` token per object. Views are excluded as well so the dump does not try to recreate views that reference tables left out of the backup.
+* The query result is read with `mapfile` so each `--ignore-table` token is passed as a single argument. This keeps identifiers that contain spaces (e.g. `` `db2`.`my table` ``) intact instead of being word-split by the shell.
+* No default database is set on restore; the dump carries the per-schema context, so the listed tables are restored into their respective databases.
+
+## Replicating through a MaxScale Binlogrouter
+
+If the external MariaDB is exposed behind a [MaxScale](https://mariadb.com/docs/maxscale/) using the [Binlogrouter](https://mariadb.com/docs/maxscale/maxscale-archive/mariadb-maxscale-25-08/maxscale-25-08-routers/mariadb-maxscale-25-08-maxscale-25-08-binlogrouter) to expose the binlog stream, set `binlogPort` on the `ExternalMariaDB`. The replicas will use this port to stream the binary logs (i.e. as the `CHANGE MASTER` port), while still using `port` for the rest of the operations:
+
+```yaml
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: ExternalMariaDB
+metadata:
+ name: external-mariadb
+spec:
+ host: maxscale.example.com
+ port: 3306
+ binlogPort: 4000
+ username: root
+ passwordSecretKeyRef:
+ name: mariadb
+ key: password
+```
+
+## Services considerations
+* Service: sends connections to any Pod, regardless of its replication status.
+* PrimaryService: despite there being no real primary node on this setup (all Pods replicate from the external MariaDB), this Service is kept to provide a stable way to always send connections to the same Pod, as it could be required by some applications.
+* SecondaryService: sends connections to the replica Pods that are `Ready`. A Pod that is currently being rebuilt during [recovery](#replica-recovery) (replication role `Unknown`) is excluded from the Service endpoints. Beyond that, traffic follows the Pod readiness: a Pod whose replication is broken, or that lags behind the source beyond `maxLagSeconds`, fails its readiness probe and stops receiving connections — unless `ignoreReplicationLivenessProbes` / `ignoreMaxLagSeconds` are set, in which case it keeps receiving read traffic.
diff --git a/examples/manifests/mariadb_external_replication.yaml b/examples/manifests/mariadb_external_replication.yaml
new file mode 100644
index 0000000000..0d57056a43
--- /dev/null
+++ b/examples/manifests/mariadb_external_replication.yaml
@@ -0,0 +1,21 @@
+apiVersion: k8s.mariadb.com/v1alpha1
+kind: MariaDB
+metadata:
+ name: external-replicas
+spec:
+ storage:
+ size: 10Gi
+ replicas: 3
+ replication:
+ enabled: true
+ replicaFromExternal:
+ mariaDbRef:
+ name: external-mariadb
+ kind: ExternalMariaDB
+ serverIdOffset: 30
+ service:
+ type: ClusterIP
+ primaryService:
+ type: ClusterIP
+ secondaryService:
+ type: ClusterIP
\ No newline at end of file
diff --git a/examples/manifests/mariadb_replication.yaml b/examples/manifests/mariadb_replication.yaml
index fdca0f361b..89769eddce 100644
--- a/examples/manifests/mariadb_replication.yaml
+++ b/examples/manifests/mariadb_replication.yaml
@@ -66,10 +66,10 @@ spec:
# syncBinlog: 1
# Init container that cooperates with mariadb-operator.
# initContainer:
- # image: ghcr.io/mariadb-operator/mariadb-operator:26.6.0
+ # image: ghcr.io/mariadb-operator/mariadb-operator:26.6.0-bandwidth.1
# Agent sidecar that cooperates with mariadb-operator.
# agent:
- # image: ghcr.io/mariadb-operator/mariadb-operator:26.6.0
+ # image: ghcr.io/mariadb-operator/mariadb-operator:26.6.0-bandwidth.1
service:
type: LoadBalancer
diff --git a/go.mod b/go.mod
index ab92a84f83..2ed4bd060c 100644
--- a/go.mod
+++ b/go.mod
@@ -31,6 +31,7 @@ require (
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
go.uber.org/zap v1.27.1
+ golang.org/x/mod v0.35.0
golang.org/x/sync v0.20.0
k8s.io/api v0.36.1
k8s.io/apimachinery v0.36.1
@@ -122,6 +123,7 @@ require (
github.com/gonvenience/ytbx v1.4.4 // indirect
github.com/google/gnostic-models v0.7.1 // indirect
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect
+ github.com/gorilla/mux v1.8.1 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
github.com/gruntwork-io/go-commons v0.8.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -184,7 +186,6 @@ require (
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.51.0 // indirect
- golang.org/x/mod v0.35.0 // indirect
golang.org/x/net v0.54.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.44.0 // indirect
diff --git a/go.sum b/go.sum
index bf3aac7652..c3260b4f3f 100644
--- a/go.sum
+++ b/go.sum
@@ -227,8 +227,8 @@ github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/v
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRaxEM6G0ro=
diff --git a/hack/manifests/metallb/mariadb-repl-service.yaml b/hack/manifests/metallb/mariadb-repl-service.yaml
index 5f90cab4e6..d505c66df5 100644
--- a/hack/manifests/metallb/mariadb-repl-service.yaml
+++ b/hack/manifests/metallb/mariadb-repl-service.yaml
@@ -108,4 +108,341 @@ spec:
app.kubernetes.io/name: mariadb
statefulset.kubernetes.io/pod-name: mariadb-repl-3
publishNotReadyAddresses: true
- type: LoadBalancer
\ No newline at end of file
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-filtered-0-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.184
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-filtered
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-filtered-0
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-filtered-1-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.185
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-filtered
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-filtered-1
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-filtered-2-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.186
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-filtered
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-filtered-2
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-filtered-3-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.187
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-filtered
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-filtered-3
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-external-0-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.180
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-external
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-external-0
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-external-1-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.181
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-external
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-external-1
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-external-2-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.182
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-external
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-external-2
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-external-3-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.183
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-external
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-external-3
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-multi-schema-0-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.198
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-multi-schema
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-multi-schema-0
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-multi-schema-1-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.199
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-multi-schema
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-multi-schema-1
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-multi-schema-2-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.202
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-multi-schema
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-multi-schema-2
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mariadb-repl-ext-multi-schema-3-lb
+ namespace: default
+ annotations:
+ metallb.universe.tf/loadBalancerIPs: $CIDR_PREFIX.0.203
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mariadb-repl-ext-multi-schema
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mariadb-repl-ext-multi-schema-3
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+
diff --git a/hack/manifests/metallb/mdb-autodiscovery-service.yaml b/hack/manifests/metallb/mdb-autodiscovery-service.yaml
new file mode 100644
index 0000000000..0a5c12b394
--- /dev/null
+++ b/hack/manifests/metallb/mdb-autodiscovery-service.yaml
@@ -0,0 +1,56 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mdb-autodisc-master-0-lb
+ namespace: default
+ annotations:
+ metallb.io/loadBalancerIPs: $CIDR_PREFIX.0.207
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mdb-autodisc-master
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mdb-autodisc-master-0
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mdb-autodisc-master-1-lb
+ namespace: default
+ annotations:
+ metallb.io/loadBalancerIPs: $CIDR_PREFIX.0.208
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mdb-autodisc-master
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mdb-autodisc-master-1
+ publishNotReadyAddresses: true
+ type: LoadBalancer
diff --git a/hack/manifests/metallb/mdb-emulate-external-test.yaml b/hack/manifests/metallb/mdb-emulate-external-test.yaml
index 2250a35e64..fc09d4d77b 100644
--- a/hack/manifests/metallb/mdb-emulate-external-test.yaml
+++ b/hack/manifests/metallb/mdb-emulate-external-test.yaml
@@ -1,3 +1,4 @@
+---
apiVersion: v1
kind: Service
metadata:
@@ -11,8 +12,45 @@ spec:
port: 3306
protocol: TCP
targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
+ selector:
+ app.kubernetes.io/instance: mdb-emulate-external-test
+ app.kubernetes.io/name: mariadb
+ statefulset.kubernetes.io/pod-name: mdb-emulate-external-test-0
+ publishNotReadyAddresses: true
+ type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mdb-emulate-external-test-1-lb
+ namespace: default
+ annotations:
+ metallb.io/loadBalancerIPs: $CIDR_PREFIX.0.204
+spec:
+ ports:
+ - name: mariadb
+ port: 3306
+ protocol: TCP
+ targetPort: 3306
+ - name: agent
+ port: 5555
+ protocol: TCP
+ targetPort: 5555
+ - name: agent-probe
+ port: 5566
+ protocol: TCP
+ targetPort: 5566
selector:
app.kubernetes.io/instance: mdb-emulate-external-test
app.kubernetes.io/name: mariadb
- statefulset.kubernetes.io/pod-name: mdb-emulate-external-test-0
- type: LoadBalancer
\ No newline at end of file
+ statefulset.kubernetes.io/pod-name: mdb-emulate-external-test-1
+ publishNotReadyAddresses: true
+ type: LoadBalancer
diff --git a/internal/controller/backup_controller.go b/internal/controller/backup_controller.go
index e84fbd57b6..1c25949545 100644
--- a/internal/controller/backup_controller.go
+++ b/internal/controller/backup_controller.go
@@ -16,6 +16,7 @@ import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
@@ -74,6 +75,22 @@ func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, fmt.Errorf("error reconciling ServiceAccount: %v", err)
}
+ // Mirror the PhysicalBackup reconciler: when the schedule is suspended the Backup acts as a template
+ // (e.g. for ReplicaBootstrapFrom.LogicalBackupTemplateRef) and no Job/CronJob should be reconciled.
+ if backup.Spec.Schedule != nil && backup.Spec.Schedule.Suspend {
+ if err := r.patchStatus(ctx, &backup, func(c condition.Conditioner) {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeComplete,
+ Status: metav1.ConditionFalse,
+ Reason: mariadbv1alpha1.ConditionReasonJobSuspended,
+ Message: "Suspended",
+ })
+ }); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error patching Backup status: %v", err)
+ }
+ return ctrl.Result{}, nil
+ }
+
var batchErr *multierror.Error
err = r.BatchReconciler.Reconcile(ctx, &backup, mariaDb)
batchErr = multierror.Append(batchErr, err)
diff --git a/internal/controller/grant_controller_test.go b/internal/controller/grant_controller_test.go
index b247864367..998fa0050d 100644
--- a/internal/controller/grant_controller_test.go
+++ b/internal/controller/grant_controller_test.go
@@ -4,9 +4,14 @@ import (
"fmt"
"time"
+ "strconv"
+
mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/sql"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
+ discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -764,3 +769,156 @@ var _ = Describe("Grant on an external MariaDB", func() {
}, testTimeout, testInterval).Should(BeTrue())
})
})
+
+var _ = Describe("Grant on a MariaDB replicating from external MariaDB", func() {
+ var (
+ // key = testMdbERkey
+ mdb = &mariadbv1alpha1.MariaDB{}
+ )
+
+ It("should grant privileges for all tables and databases", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, testMdbERkey, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ var endpoints discoveryv1.EndpointSlice
+
+ By("Expecting to create secondary Endpoints: " + strconv.Itoa(int(mdb.Spec.Replicas)))
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+ count := 0
+ for _, address := range endpoints.Endpoints {
+ if *address.Conditions.Ready {
+ count++
+ }
+ }
+ return count == int(mdb.Spec.Replicas)
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Creating a User")
+ userKey := types.NamespacedName{
+ Name: "grant-user-all-erep-test",
+ Namespace: testNamespace,
+ }
+ user := mariadbv1alpha1.User{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: userKey.Name,
+ Namespace: userKey.Namespace,
+ },
+ Spec: mariadbv1alpha1.UserSpec{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testMdbERkey.Name,
+ },
+ WaitForIt: true,
+ },
+ PasswordSecretKeyRef: &mariadbv1alpha1.SecretKeySelector{
+ LocalObjectReference: mariadbv1alpha1.LocalObjectReference{
+ Name: testPwdKey.Name,
+ },
+ Key: testPwdSecretKey,
+ },
+ MaxUserConnections: 20,
+ },
+ }
+ Expect(k8sClient.Create(testCtx, &user)).To(Succeed())
+ DeferCleanup(func() {
+ Expect(k8sClient.Delete(testCtx, &user)).To(Succeed())
+ })
+
+ By("Expecting User to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, userKey, &user); err != nil {
+ return false
+ }
+ return user.IsReady()
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Creating a Grant")
+ grantKey := types.NamespacedName{
+ Name: "grant-select-insert-update-erep-test",
+ Namespace: testNamespace,
+ }
+ grant := mariadbv1alpha1.Grant{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: grantKey.Name,
+ Namespace: grantKey.Namespace,
+ },
+ Spec: mariadbv1alpha1.GrantSpec{
+ SQLTemplate: mariadbv1alpha1.SQLTemplate{
+ RetryInterval: &metav1.Duration{Duration: 1 * time.Second},
+ },
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testMdbERkey.Name,
+ },
+ WaitForIt: true,
+ },
+ Privileges: []string{
+ "SELECT",
+ "INSERT",
+ "UPDATE",
+ },
+ Database: "*",
+ Table: "*",
+ Username: userKey.Name,
+ GrantOption: true,
+ },
+ }
+ Expect(k8sClient.Create(testCtx, &grant)).To(Succeed())
+ DeferCleanup(func() {
+ Expect(k8sClient.Delete(testCtx, &grant)).To(Succeed())
+ })
+
+ By("Expecting Grant to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, grantKey, &grant); err != nil {
+ return false
+ }
+ return grant.IsReady()
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting Grant to eventually have finalizer")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, grantKey, &grant); err != nil {
+ return false
+ }
+ return controllerutil.ContainsFinalizer(&grant, grantFinalizerName)
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ replicas := int(mdb.Spec.Replicas)
+ refResolver := refresolver.New(k8sClient)
+
+ By("Expecting to get password from secret")
+ password, err := refResolver.SecretKeyRef(testCtx, testPasswordSecretRef, mdb.GetNamespace())
+ Expect(err).To(Succeed())
+
+ for i := 0; i < replicas; i++ {
+
+ client, err := sql.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i,
+ sql.WithUsername(user.Name),
+ sql.WithPassword(password))
+
+ By("Expecting to get SqlClient from Pod " + strconv.Itoa(i))
+ Expect(err).To(Succeed())
+
+ By("Expecting to GRANT exists on Pod" + strconv.Itoa(i) + " eventually")
+ Eventually(func() bool {
+ exists, err := client.GrantExists(testCtx, grant.Spec.Privileges, grant.Spec.Database, grant.Spec.Table, grant.AccountName())
+ if err != nil {
+ fmt.Fprintf(GinkgoWriter, "Error: %v\n", err)
+ return false
+ }
+ return exists
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ Expect(client.Exec(testCtx, "SELECT 1")).To(Succeed())
+ }
+ })
+
+})
diff --git a/internal/controller/mariadb_controller.go b/internal/controller/mariadb_controller.go
index 5f49a95376..cc7252843c 100644
--- a/internal/controller/mariadb_controller.go
+++ b/internal/controller/mariadb_controller.go
@@ -169,6 +169,10 @@ func (r *MariaDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
Name: "Scale out",
Reconcile: r.reconcileScaleOut,
},
+ {
+ Name: "Service",
+ Reconcile: r.reconcileService,
+ },
{
Name: "Replica recovery",
Reconcile: r.reconcileReplicaRecovery,
@@ -185,9 +189,10 @@ func (r *MariaDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
Name: "PodDisruptionBudget",
Reconcile: r.reconcilePodDisruptionBudget,
},
+
{
- Name: "Service",
- Reconcile: r.reconcileService,
+ Name: "External Repl Init",
+ Reconcile: r.reconcileExternalReplInit,
},
{
Name: "Replication",
@@ -282,6 +287,10 @@ func requeueResult(ctx context.Context, mdb *mariadbv1alpha1.MariaDB) (ctrl.Resu
log.FromContext(ctx).V(1).Info("Maintenance mode enabled. Requeuing MariaDB...")
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
+ if mdb.Replication().ReplicaFromExternal != nil {
+ log.FromContext(ctx).V(1).Info("Requeuing MariaDB")
+ return ctrl.Result{RequeueAfter: mdb.Replication().ReplicaFromExternal.HealthCheckInterval.Duration}, nil // ensure replicas status are updated
+ }
if mdb.IsTLSEnabled() {
log.FromContext(ctx).V(1).Info("Requeuing MariaDB")
@@ -416,6 +425,7 @@ func (r *MariaDBReconciler) reconcileRBAC(ctx context.Context, mariadb *mariadbv
func (r *MariaDBReconciler) reconcileStatefulSet(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB) (ctrl.Result, error) {
key := client.ObjectKeyFromObject(mariadb)
+
updateAnnotations, err := r.getUpdateAnnotations(ctx, mariadb)
if err != nil {
return ctrl.Result{}, fmt.Errorf("error getting Pod annotations: %v", err)
@@ -573,8 +583,8 @@ func (r *MariaDBReconciler) reconcileRestore(ctx context.Context, mdb *mariadbv1
}); err != nil {
return ctrl.Result{}, fmt.Errorf("error patching status: %v", err)
}
-
- restore, err := r.Builder.BuildRestore(mdb, mdb.RestoreKey())
+ restoreOpts := builder.LogicalRestoreOpts{}
+ restore, err := r.Builder.BuildRestore(mdb, mdb.RestoreKey(), restoreOpts)
if err != nil {
return ctrl.Result{}, fmt.Errorf("error building restore: %v", err)
}
@@ -1055,7 +1065,7 @@ func (r *MariaDBReconciler) getTargetVolumeSnapshot(ctx context.Context, backup
recoveryTime := ptr.Deref(targetRecoveryTime, metav1.Time{Time: time.Now()})
logger := log.FromContext(ctx).WithName("snapshot")
- targetSnapshot, err := r.BackupProcessor.GetBackupTargetFile(snapshotNames, recoveryTime.Time, logger)
+ targetSnapshot, err := r.BackupProcessor.GetBackupTargetFile(snapshotNames, recoveryTime.Time, nil, logger)
if err != nil {
return "", fmt.Errorf("error getting target VolumeSnapshot: %v", err)
}
diff --git a/internal/controller/mariadb_controller_external_repl_init.go b/internal/controller/mariadb_controller_external_repl_init.go
new file mode 100644
index 0000000000..35d47daebc
--- /dev/null
+++ b/internal/controller/mariadb_controller_external_repl_init.go
@@ -0,0 +1,414 @@
+package controller
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+ "time"
+
+ "github.com/go-logr/logr"
+ mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/builder"
+ condition "github.com/mariadb-operator/mariadb-operator/v26/pkg/condition"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/controller/replication"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/sql"
+ stsobj "github.com/mariadb-operator/mariadb-operator/v26/pkg/statefulset"
+ "golang.org/x/mod/semver"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/apimachinery/pkg/types"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// reconcileExternalReplInit handles the initialization of external replication for a MariaDB cluster.
+// It ensures that a backup of the external MariaDB is taken and restored to the replica pods
+// before marking the external replication as initialized.
+func (r *MariaDBReconciler) reconcileExternalReplInit(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB) (ctrl.Result, error) {
+ logger := log.FromContext(ctx).WithName("external-repl-init")
+
+ if !mariadb.IsReplicationEnabled() {
+ logger.Info("replication is not enabled")
+ return ctrl.Result{}, nil
+ }
+ replication := mariadb.Replication()
+
+ if !replication.IsExternalReplication() {
+ logger.Info("replication is enabled but is not external")
+ return ctrl.Result{}, nil
+ }
+
+ if mariadb.HasConfiguredReplication() {
+ logger.Info("replication is enabled and external but replication it is already configured")
+ return ctrl.Result{}, nil
+ }
+
+ if mariadb.IsExternalReplInitialized() {
+ logger.Info("external replication already initialized")
+ return ctrl.Result{}, nil
+ }
+
+ logger.Info("reconciling external replication init")
+ if err := r.patchStatus(ctx, mariadb, func(status *mariadbv1alpha1.MariaDBStatus) error {
+ condition.SetExternalReplInitializing(status)
+ return nil
+ }); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error patching MariaDB status: %v", err)
+ }
+
+ // Ensure External MariaDB is ready before proceeding with the backup and restore
+
+ emdb, err := r.RefResolver.ExternalMariaDB(ctx, &replication.ReplicaFromExternal.MariaDBRef.ObjectReference, mariadb.Namespace)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting external MariaDB object: %v", err)
+ }
+
+ if !emdb.IsReady() {
+ logger.Info("external MariaDB is not ready")
+ return ctrl.Result{RequeueAfter: time.Minute * 1}, nil
+ }
+
+ logger.Info("reconciling logical backup")
+ if result, err := r.reconcileLogicalBackup(ctx, mariadb, replication, logger); err != nil || !result.IsZero() {
+ return result, err
+ }
+
+ logger.Info("reconciling logical restore on each pod")
+ total_pods := 0
+ total_restored_pods := 0
+ for _, i := range r.replicationPodIndexes(mariadb) {
+ total_pods++
+ if _, err := r.reconcileRestoreInPod(ctx, mariadb, i, logger, false); err == nil {
+ total_restored_pods++
+ }
+ }
+
+ if total_pods != total_restored_pods {
+ logger.Info("restore in pod in-progress")
+ return ctrl.Result{RequeueAfter: time.Minute * 1}, nil
+ }
+
+ //cleanup the restore
+ logger.Info("cleaning up the restore on each pod")
+ for _, i := range r.replicationPodIndexes(mariadb) {
+ _ = r.cleanupRestoreInPod(ctx, mariadb, i, logger)
+ }
+
+ logger.Info("reconciling restore finished")
+
+ logger.Info("setting ExternalReplInitialized status to true")
+ if err := r.patchStatus(ctx, mariadb, func(status *mariadbv1alpha1.MariaDBStatus) error {
+ condition.SetExternalReplInitialized(status)
+ return nil
+ }); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error patching MariaDB status: %v", err)
+ }
+
+ logger.Info("reconciling external replication init finished")
+ return ctrl.Result{}, nil
+}
+
+// reconcileRestoreInPod ensures that the given replica pod index is restored from the backup taken from the external MariaDB.
+func (r *MariaDBReconciler) reconcileRestoreInPod(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
+ replicaPodIndex int, logger logr.Logger, removeCurrentPod bool) (ctrl.Result, error) {
+
+ logger.Info("reconciling restore in pod", "pod", replicaPodIndex)
+
+ replClientSet, err := replication.NewReplicationClientSet(mariadb, r.RefResolver)
+ if err != nil {
+ logger.Error(err, "error getting replica clientset", "err", err, "pod", replicaPodIndex)
+ return ctrl.Result{RequeueAfter: 5 * time.Second}, err
+ }
+
+ client, err := replClientSet.ClientForIndex(ctx, replicaPodIndex)
+ if err != nil {
+ logger.Error(err, "error getting replica client", "err", err, "pod", replicaPodIndex)
+ return ctrl.Result{RequeueAfter: 5 * time.Second}, err
+ }
+ defer client.Close()
+
+ if err := client.ResetMaster(ctx); err != nil {
+ logger.Error(err, "error resetting master")
+ return ctrl.Result{}, fmt.Errorf("error resetting master: %v", err)
+ }
+
+ var existingRestore mariadbv1alpha1.Restore
+ err = r.Get(ctx, mariadb.RestoreKeyInPod(replicaPodIndex), &existingRestore)
+
+ if err == nil && !existingRestore.IsComplete() {
+ logger.Info("restore exists, but not complete", "pod", replicaPodIndex)
+ return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("restore is not complete")
+ }
+
+ podKey := types.NamespacedName{
+ Name: stsobj.PodName(*mariadb.GetObjectMeta(), replicaPodIndex),
+ Namespace: mariadb.Namespace,
+ }
+
+ if !existingRestore.IsComplete() {
+ // Restore/Bootstrap node from backup
+ logger.Info("restore does not exists, create a new restore", "pod", replicaPodIndex)
+
+ if removeCurrentPod {
+ logger.Info("Recreating Pod")
+
+ pvcKey := mariadb.PVCKey(builder.StorageVolume, replicaPodIndex)
+ var pvc corev1.PersistentVolumeClaim
+ if err := r.Get(ctx, pvcKey, &pvc); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting pvc from Pod '%v': %v", replicaPodIndex, err)
+ }
+ if err := r.Delete(ctx, &pvc); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error deleting pvc from Pod '%v': %v", replicaPodIndex, err)
+ }
+
+ var existingPod corev1.Pod
+ if err := r.Get(ctx, podKey, &existingPod); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting Pod '%v': %v", replicaPodIndex, err)
+ }
+ if err := r.Delete(ctx, &existingPod); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error deleting Pod '%v': %v", replicaPodIndex, err)
+ }
+ }
+ err := newRestore(mariadb, *r, ctx, replicaPodIndex)
+ return ctrl.Result{}, fmt.Errorf("new restore attempt%v", err)
+ }
+
+ if err != nil {
+ logger.Error(err, "error creating new restore", "pod", replicaPodIndex)
+ return ctrl.Result{}, fmt.Errorf("error creating new restore: %v", err)
+ }
+
+ logger.Info("restore complete", "pod", replicaPodIndex)
+
+ return ctrl.Result{}, nil
+}
+
+// cleanupRestoreInPod deletes the Restore object for the given replica pod index.
+// This is used to clean up the Restore object after the restore is complete.
+func (r *MariaDBReconciler) cleanupRestoreInPod(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
+ replicaPodIndex int, logger logr.Logger) error {
+
+ logger.Info("cleaning up restore in pod", "pod", replicaPodIndex)
+
+ var existingRestore mariadbv1alpha1.Restore
+ err := r.Get(ctx, mariadb.RestoreKeyInPod(replicaPodIndex), &existingRestore)
+
+ if err == nil {
+ if err := r.Delete(ctx, &existingRestore); err != nil {
+ logger.Info("failed to delete restore", "pod", replicaPodIndex)
+ return fmt.Errorf("error deleting Restore: %v", err)
+ }
+ }
+ return nil
+}
+
+// handleInitialBackup ensures that a valid backup exists for the external MariaDB. If a backup does not exist, it creates a new one.
+func (r *MariaDBReconciler) reconcileLogicalBackup(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
+ replication mariadbv1alpha1.Replication, logger logr.Logger) (ctrl.Result, error) {
+ logger.Info("Reconciling initial logical backup for external replication")
+
+ emdb, err := r.RefResolver.ExternalMariaDB(ctx, &replication.ReplicaFromExternal.MariaDBRef.ObjectReference, mariadb.Namespace)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting external MariaDB object: %v", err)
+ }
+ key := types.NamespacedName{
+ Name: mariadb.ExternalReplLogicalBackupName(),
+ Namespace: emdb.Namespace,
+ }
+
+ logger.Info("Checking if viable backup already exists")
+ var isBackupInvalid = false
+ var binlogExpireLogsDuration time.Duration
+ var existingBackup mariadbv1alpha1.Backup
+
+ logger.Info("Getting the binlog_expire_logs_seconds on the external MariaDB")
+ if binlogExpireLogsDuration, err = getBinlogExpireLogsDuration(emdb, ctx, r.RefResolver, logger); err != nil {
+ return ctrl.Result{}, fmt.Errorf("unable to get binlog_expire_logs_seconds: %v", err)
+ }
+
+ logger.Info("Trying to get the current backup")
+ err = r.Get(ctx, key, &existingBackup)
+
+ if err == nil {
+ logger.Info("Backup exists, check if it is expired. binlogExpireLogsDuration", "duration", binlogExpireLogsDuration.String())
+ isBackupInvalid = removeBackupIfExpired(existingBackup, ctx, binlogExpireLogsDuration, *r, logger)
+ }
+
+ // Create a new backup if required
+ if err != nil || isBackupInvalid {
+ logger.Info("Take a new backup")
+ template, err := r.getLogicalBackupTemplate(ctx, mariadb, replication)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting logical backup template: %v", err)
+ }
+ backup_error := newBackup(emdb, *r, ctx, binlogExpireLogsDuration, mariadb.GetImagePullSecrets(), mariadb.Spec.Storage.Size,
+ key, replication.ReplicaFromExternal.FilteredReplicaTables, template)
+ return ctrl.Result{RequeueAfter: time.Minute * 1}, backup_error
+ }
+
+ if !existingBackup.IsComplete() {
+ logger.Info("Backup is running")
+ return ctrl.Result{RequeueAfter: time.Minute * 1}, nil
+ }
+
+ if existingBackup.IsFailed() {
+ logger.Info("Backup has failed, deleting and retrying")
+ if err := r.Delete(ctx, &existingBackup); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error deleting failed Backup: %v", err)
+ }
+ return ctrl.Result{}, fmt.Errorf("backup failed, retrying")
+ }
+
+ return ctrl.Result{}, nil
+}
+
+// replicationPodIndexes returns the list of pod indexes that are part of the replication setup.
+func (r *MariaDBReconciler) replicationPodIndexes(mariadb *mariadbv1alpha1.MariaDB) []int {
+ podIndexes := []int{
+ *mariadb.Status.CurrentPrimaryPodIndex,
+ }
+ for i := 0; i < int(mariadb.Spec.Replicas); i++ {
+ if i != *mariadb.Status.CurrentPrimaryPodIndex {
+ podIndexes = append(podIndexes, i)
+ }
+ }
+ return podIndexes
+}
+
+// removeBackupIfExpired checks if the existing backup is expired based on the binlog_expire_logs_seconds value
+// from the external MariaDB.
+// If the backup is expired, it deletes the backup and returns true. Otherwise, it returns false.
+func removeBackupIfExpired(existingBackup mariadbv1alpha1.Backup, ctx context.Context,
+ binlogExpireLogsDuration time.Duration, r MariaDBReconciler, logger logr.Logger) bool {
+ if time.Since(existingBackup.CreationTimestamp.Time) > binlogExpireLogsDuration {
+ logger.Info("Backup is expired, deleting it", "backup", existingBackup.Name)
+ if err := r.Delete(ctx, &existingBackup); err == nil {
+ return true
+ }
+ }
+ return false
+}
+
+// newBackup creates a Backup object to take a backup of the external MariaDB.
+// The backup will be used to restore the replica pods.
+func newBackup(emdb *mariadbv1alpha1.ExternalMariaDB, r MariaDBReconciler, ctx context.Context,
+ binlogExpireLogsDuration time.Duration, imagePullSecrets []mariadbv1alpha1.LocalObjectReference,
+ size *resource.Quantity, key types.NamespacedName, filteredTables []string,
+ template *mariadbv1alpha1.Backup) error {
+
+ args := []string{
+ "--master-data=1",
+ "--gtid",
+ "--verbose",
+ "--single-transaction",
+ "--ignore-table=mysql.global_priv",
+ }
+
+ backupOps := builder.BackupOpts{
+ Metadata: []*mariadbv1alpha1.Metadata{emdb.Spec.InheritMetadata},
+ Key: key,
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: emdb.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ },
+ Args: args,
+ Tables: filteredTables,
+ Compression: mariadbv1alpha1.CompressGzip,
+ Storage: mariadbv1alpha1.BackupStorage{
+ PersistentVolumeClaim: &mariadbv1alpha1.PersistentVolumeClaimSpec{
+ AccessModes: []corev1.PersistentVolumeAccessMode{
+ corev1.ReadWriteOnce,
+ },
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: corev1.ResourceList{
+ "storage": *size,
+ },
+ },
+ },
+ },
+ MaxRetention: binlogExpireLogsDuration,
+ ImagePullSecrets: imagePullSecrets,
+ Template: template,
+ }
+
+ backup, err := r.Builder.BuildBackup(backupOps, emdb)
+ if err != nil {
+ return fmt.Errorf("error building Backup object: %v", err)
+ }
+ if err := r.Create(ctx, backup); err != nil {
+ return fmt.Errorf("error creating base Backup: %v", err)
+ }
+ return nil
+}
+
+// getLogicalBackupTemplate loads the optional Backup template referenced from
+// replica.bootstrapFrom.logicalBackupTemplateRef. Returns nil when the user has not configured a template.
+func (r *MariaDBReconciler) getLogicalBackupTemplate(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
+ replication mariadbv1alpha1.Replication) (*mariadbv1alpha1.Backup, error) {
+ if replication.Replica.ReplicaBootstrapFrom == nil ||
+ replication.Replica.ReplicaBootstrapFrom.LogicalBackupTemplateRef == nil {
+ return nil, nil
+ }
+ tplKey := types.NamespacedName{
+ Name: replication.Replica.ReplicaBootstrapFrom.LogicalBackupTemplateRef.Name,
+ Namespace: mariadb.Namespace,
+ }
+ var tpl mariadbv1alpha1.Backup
+ if err := r.Get(ctx, tplKey, &tpl); err != nil {
+ return nil, fmt.Errorf("error getting Backup template '%s': %v", tplKey.Name, err)
+ }
+ return &tpl, nil
+}
+
+// getBinlogExpireLogsDuration gets the binlog_expire_logs_seconds value from
+// the external MariaDB and returns it as a time.Duration.
+func getBinlogExpireLogsDuration(emdb *mariadbv1alpha1.ExternalMariaDB, ctx context.Context,
+ refResolver *refresolver.RefResolver, logger logr.Logger) (time.Duration, error) {
+ var external_client *sql.Client
+ var err error
+ if external_client, err = sql.NewClientWithMariaDB(ctx, emdb, refResolver); err != nil {
+ return time.Duration(0), fmt.Errorf("error getting external MariaDB client: %v", err)
+ }
+ defer external_client.Close()
+
+ var binlogExpireLogsSecondsStr string
+ var binlogExpireLogsSeconds int
+
+ if semver.Compare("v"+emdb.Status.Version, "v10.6.1") >= 0 {
+ logger.Info("Using binlog_expire_logs_seconds", "version", emdb.Status.Version)
+ binlogExpireLogsSecondsStr, err = external_client.SystemVariable(ctx, "binlog_expire_logs_seconds")
+ if err != nil {
+ return time.Duration(0), fmt.Errorf("unable to get binlog_expire_logs_seconds: %v", err)
+ }
+ binlogExpireLogsSeconds, _ = strconv.Atoi(binlogExpireLogsSecondsStr)
+ } else {
+ logger.Info("Using expire_logs_days", "version", emdb.Status.Version)
+ binlogExpireLogsDaysStr, err := external_client.SystemVariable(ctx, "expire_logs_days")
+ if err != nil {
+ return time.Duration(0), fmt.Errorf("unable to get expire_logs_days: %v", err)
+ }
+ binlogExpireLogsDays, _ := strconv.Atoi(binlogExpireLogsDaysStr)
+ binlogExpireLogsSeconds = binlogExpireLogsDays * 86400
+ }
+ logger.Info("binlog expire logs duration", "seconds", binlogExpireLogsSeconds)
+ return time.Duration(binlogExpireLogsSeconds) * time.Second, nil
+}
+
+// newRestore creates a Restore object for the given replica pod index. The Restore will be responsible for restoring
+// the backup taken from the external MariaDB to the replica pod.
+func newRestore(mariadb *mariadbv1alpha1.MariaDB, r MariaDBReconciler, ctx context.Context, replicaPodIndex int) error {
+ restoreOpts := builder.LogicalRestoreOpts{
+ PodIndex: &replicaPodIndex,
+ }
+ restore, err := r.Builder.BuildRestore(mariadb, mariadb.RestoreKeyInPod(replicaPodIndex), restoreOpts)
+ if err != nil {
+ return fmt.Errorf("error building Restore object: %v", err)
+ }
+ if err := r.Create(ctx, restore); err != nil {
+ return fmt.Errorf("error creating Restore object: %v", err)
+ }
+ return nil
+}
diff --git a/internal/controller/mariadb_controller_external_repl_init_test.go b/internal/controller/mariadb_controller_external_repl_init_test.go
new file mode 100644
index 0000000000..1eec26fdea
--- /dev/null
+++ b/internal/controller/mariadb_controller_external_repl_init_test.go
@@ -0,0 +1,1214 @@
+package controller
+
+import (
+ "fmt"
+ "slices"
+ "strconv"
+ "time"
+
+ mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/builder"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
+ sqlClient "github.com/mariadb-operator/mariadb-operator/v26/pkg/sql"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/statefulset"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ discoveryv1 "k8s.io/api/discovery/v1"
+ policyv1 "k8s.io/api/policy/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/utils/ptr"
+)
+
+var _ = Describe("MariaDB replication from external server", Ordered, func() {
+
+ var (
+ key = testMdbERkey
+ pbRecoveryKey = testMdbPbRecoveryERkey
+ mdb = &mariadbv1alpha1.MariaDB{}
+ )
+
+ It("should reconcile", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, testMdbERkey, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting to create a Service")
+ var svc corev1.Service
+ Expect(k8sClient.Get(testCtx, key, &svc)).To(Succeed())
+
+ By("Expecting to create a primary Service")
+ Expect(k8sClient.Get(testCtx, mdb.PrimaryServiceKey(), &svc)).To(Succeed())
+ Expect(svc.Spec.Selector["statefulset.kubernetes.io/pod-name"]).To(Equal(statefulset.PodName(mdb.ObjectMeta, 0)))
+
+ By("Expecting to create a secondary Service")
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &svc)).To(Succeed())
+
+ By("Expecting role label to be set to primary")
+ Eventually(func() bool {
+ currentPrimary := *mdb.Status.CurrentPrimary
+ primaryPodKey := types.NamespacedName{
+ Name: currentPrimary,
+ Namespace: mdb.Namespace,
+ }
+ var primaryPod corev1.Pod
+ if err := k8sClient.Get(testCtx, primaryPodKey, &primaryPod); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ return primaryPod.Labels["k8s.mariadb.com/role"] == "primary"
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting Connection to be ready eventually")
+ Eventually(func() bool {
+ var conn mariadbv1alpha1.Connection
+ if err := k8sClient.Get(testCtx, key, &conn); err != nil {
+ return false
+ }
+ return conn.IsReady()
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting primary Connection to be ready eventually")
+ Eventually(func() bool {
+ var conn mariadbv1alpha1.Connection
+ if err := k8sClient.Get(testCtx, mdb.PrimaryConnectioneKey(), &conn); err != nil {
+ return false
+ }
+ return conn.IsReady()
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting secondary Connection to be ready eventually")
+ Eventually(func() bool {
+ var conn mariadbv1alpha1.Connection
+ if err := k8sClient.Get(testCtx, mdb.SecondaryConnectioneKey(), &conn); err != nil {
+ return false
+ }
+ return conn.IsReady()
+ }, testTimeout, testInterval).Should(BeTrue())
+ var endpoints discoveryv1.EndpointSlice
+
+ By("Expecting to create secondary Endpoints: " + strconv.Itoa(int(mdb.Spec.Replicas)))
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+ count := 0
+ for _, address := range endpoints.Endpoints {
+ if *address.Conditions.Ready {
+ count++
+ }
+ }
+ return count == int(mdb.Spec.Replicas)
+
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting to create a PodDisruptionBudget")
+ var pdb policyv1.PodDisruptionBudget
+ Expect(k8sClient.Get(testCtx, key, &pdb)).To(Succeed())
+
+ By("Expecting the logical backup to inherit resources from the template")
+ refResolver := refresolver.New(k8sClient)
+ emdb, err := refResolver.ExternalMariaDB(testCtx, &mdb.Replication().ReplicaFromExternal.MariaDBRef.ObjectReference, testNamespace)
+ Expect(err).To(Succeed())
+ var logicalBackup mariadbv1alpha1.Backup
+ Expect(k8sClient.Get(testCtx, types.NamespacedName{
+ Name: mdb.ExternalReplLogicalBackupName(),
+ Namespace: emdb.Namespace,
+ }, &logicalBackup)).To(Succeed())
+ Expect(logicalBackup.Spec.Resources).NotTo(BeNil())
+ Expect(logicalBackup.Spec.Resources.Limits.Cpu().String()).To(Equal("300m"))
+ Expect(logicalBackup.Spec.Resources.Limits.Memory().String()).To(Equal("512Mi"))
+ Expect(logicalBackup.Spec.Resources.Requests.Cpu().String()).To(Equal("100m"))
+ Expect(logicalBackup.Spec.Resources.Requests.Memory().String()).To(Equal("128Mi"))
+
+ By("Expecting each Restore to inherit resources from replica.bootstrapFrom.restoreJob")
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ var restore mariadbv1alpha1.Restore
+ err := k8sClient.Get(testCtx, mdb.RestoreKeyInPod(i), &restore)
+ if apierrors.IsNotFound(err) {
+ continue
+ }
+ Expect(err).To(Succeed())
+ Expect(restore.Spec.Resources).NotTo(BeNil())
+ Expect(restore.Spec.Resources.Limits.Cpu().String()).To(Equal("300m"))
+ Expect(restore.Spec.Resources.Limits.Memory().String()).To(Equal("512Mi"))
+ Expect(restore.Spec.Resources.Requests.Cpu().String()).To(Equal("100m"))
+ Expect(restore.Spec.Resources.Requests.Memory().String()).To(Equal("128Mi"))
+ }
+ })
+
+ It("should recover if replication is broken", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady() && mdb.IsExternalReplInitialized()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting to get SqlClient from Pod 2")
+ refResolver := refresolver.New(k8sClient)
+ var client *sqlClient.Client
+ var err error
+ podIndex := 2
+ client, err = sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, podIndex)
+ Expect(err).To(Succeed())
+ defer client.Close()
+
+ By("Expecting to break replication on Pod 2")
+ Expect(
+ client.Exec(testCtx, "STOP SLAVE;"),
+ client.Exec(testCtx, "RESET MASTER;"),
+ client.Exec(testCtx, "RESET SLAVE;"),
+ client.Exec(testCtx, "SET GLOBAL gtid_slave_pos='0-1-0';"),
+ client.Exec(testCtx, "START SLAVE;"),
+ ).To(Succeed())
+
+ By("Expecting replication to be ready eventually on Pod " + strconv.Itoa(podIndex))
+ Eventually(func() bool {
+ isReplicaHealthy, _ := client.IsReplicationHealthy(testCtx)
+ return isReplicaHealthy
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting replication status to get back to slave Pod " + strconv.Itoa(podIndex))
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ return (mdb.Status.Replication.Roles)[statefulset.PodName(mdb.ObjectMeta, podIndex)] == mariadbv1alpha1.ReplicationRoleReplica
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting MariaDB status to get back to running and Ready")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ condition := meta.FindStatusCondition(mdb.Status.Conditions, mariadbv1alpha1.ConditionTypeReady)
+ return condition != nil && condition.Status == metav1.ConditionTrue
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ var endpoints discoveryv1.EndpointSlice
+ By("Expecting Pod " + strconv.Itoa(podIndex) + " to present on the secondary endpoints")
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+
+ podKey := types.NamespacedName{
+ Name: statefulset.PodName(mdb.ObjectMeta, podIndex),
+ Namespace: testNamespace,
+ }
+ var pod corev1.Pod
+ Expect(k8sClient.Get(testCtx, podKey, &pod)).To(Succeed())
+
+ for _, address := range endpoints.Endpoints {
+ if address.Addresses[0] == pod.Status.PodIP && *address.Conditions.Ready {
+ return true
+ }
+ }
+ return false
+ }, testTimeout, testInterval).Should(BeTrue())
+ })
+
+ It("should recover in case of missing GTID replication error (1236)", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting to get SqlClient from Pod 2")
+ refResolver := refresolver.New(k8sClient)
+ var client *sqlClient.Client
+ var err error
+ podIndex := 2
+ client, err = sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, podIndex)
+ Expect(err).To(Succeed())
+ defer client.Close()
+
+ By("Suspend MariaDB")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ mdb.Spec.Suspend = true
+
+ return k8sClient.Update(testCtx, mdb) == nil
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting MariaDB to eventually be suspended")
+ expectMariadbFn(testCtx, k8sClient, key, func(mdb *mariadbv1alpha1.MariaDB) bool {
+ condition := meta.FindStatusCondition(mdb.Status.Conditions, mariadbv1alpha1.ConditionTypeReady)
+ if condition == nil {
+ return false
+ }
+ return condition.Status == metav1.ConditionFalse && condition.Reason == mariadbv1alpha1.ConditionReasonSuspended
+ })
+
+ By("Expecting to stop replication on Pod 2")
+ Expect(client.Exec(testCtx, "STOP SLAVE")).To(Succeed(), client.Exec(testCtx, "SET GLOBAL gtid_slave_pos = '0-9999-9999'"))
+
+ By("Expecting to set Invalid GTID position on Pod 2")
+ Expect(client.Exec(testCtx, "SET GLOBAL gtid_slave_pos = '0-9999-9999'")).To(Succeed())
+
+ By("Expecting to start replication on Pod 2")
+ Expect(client.Exec(testCtx, "START SLAVE")).To(Succeed())
+
+ By("Expecting replication error 1236 on Pod " + strconv.Itoa(podIndex))
+ Eventually(func() bool {
+ rStatus, err := client.GetReplicationStatus(testCtx)
+ if err != nil {
+ return false
+ }
+ return rStatus.LastIOErrno.Int32 == 1236
+
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Resume MariaDB")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ mdb.Spec.Suspend = false
+
+ return k8sClient.Update(testCtx, mdb) == nil
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting no replication error 1236 on Pod " + strconv.Itoa(podIndex))
+ Eventually(func() bool {
+ rStatus, err := client.GetReplicationStatus(testCtx)
+ if err != nil {
+ return false
+ }
+ return rStatus.LastIOErrno.Int32 != 1236
+
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting replication status to get back to slave Pod " + strconv.Itoa(podIndex))
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ return (mdb.Status.Replication.Roles)[statefulset.PodName(mdb.ObjectMeta, podIndex)] == mariadbv1alpha1.ReplicationRoleReplica
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ var endpoints discoveryv1.EndpointSlice
+ By("Expecting Pod " + strconv.Itoa(podIndex) + " to present on the secondary endpoints")
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+
+ podKey := types.NamespacedName{
+ Name: statefulset.PodName(mdb.ObjectMeta, podIndex),
+ Namespace: testNamespace,
+ }
+ var pod corev1.Pod
+ Expect(k8sClient.Get(testCtx, podKey, &pod)).To(Succeed())
+
+ for _, address := range endpoints.Endpoints {
+ if address.Addresses[0] == pod.Status.PodIP && *address.Conditions.Ready {
+ return true
+ }
+ }
+ return false
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ })
+
+ It("should reuse physical backup if it still valid", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Get current backup age
+ By("Expecting to get current external MariadDB object")
+ refResolver := refresolver.New(k8sClient)
+ emdb, err := refResolver.ExternalMariaDB(testCtx, &mdb.Replication().ReplicaFromExternal.MariaDBRef.ObjectReference, testNamespace)
+ Expect(err).To(Succeed())
+
+ emdbKey := types.NamespacedName{
+ Name: emdb.Name,
+ Namespace: emdb.Namespace,
+ }
+ var existingLogicalBackup mariadbv1alpha1.Backup
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, emdbKey, &existingLogicalBackup)
+ Expect(err).To(Succeed())
+ firstLogicalBackupCreationTimestamp := existingLogicalBackup.CreationTimestamp.Time
+
+ var existingPhysicalBackup mariadbv1alpha1.PhysicalBackup
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, pbRecoveryKey, &existingPhysicalBackup)
+ Expect(err).To(Succeed())
+ firstPhysicalBackupCreationTimestamp := existingPhysicalBackup.CreationTimestamp.Time
+
+ // Insert data on the external master to be sure that replica is not up to date with the master
+ By("Expecting to get SqlClient from the external MariaDB")
+ client, err := sqlClient.NewClientWithMariaDB(testCtx, emdb, refResolver)
+ Expect(err).To(Succeed())
+ defer client.Close()
+
+ By("Expecting to insert data on the external master")
+ Expect(
+ client.Exec(testCtx, "CREATE DATABASE IF NOT EXISTS test;"),
+ client.Exec(testCtx, "USE test;"),
+ client.Exec(testCtx, "CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY);"),
+ client.Exec(testCtx, "INSERT INTO t VALUES (1);"),
+ client.Exec(testCtx, "USE inttest;"),
+ client.Exec(testCtx, "INSERT INTO t VALUES (1);"),
+ ).To(Succeed())
+ podIndex := 1
+ testDeletePod(mdb, podIndex, true)
+
+ // Expect to get in recovering state eventually
+ By("Expecting MariaDB to be in recovering state eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+
+ return mdb.IsRecoveringReplicas()
+
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Expect to get back to ready state eventually
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+
+ // return (mdb.Status.Replication.Roles)[statefulset.PodName(mdb.ObjectMeta, podIndex)] == mariadbv1alpha1.ReplicationRoleReplica
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Get current Physical backup age
+ By("Expecting to get physical backup object")
+ err = k8sClient.Get(testCtx, pbRecoveryKey, &existingPhysicalBackup)
+ Expect(err).To(Succeed())
+ secondPhysicalBackupCreationTimestamp := existingPhysicalBackup.CreationTimestamp.Time
+
+ // Physical backup should not be updated as it's still valid
+ By("Expecting to have same CreationTimestamp on physical backup Object before and after the Pod recreation")
+ Expect(firstPhysicalBackupCreationTimestamp).To(Equal(secondPhysicalBackupCreationTimestamp))
+
+ // Get current backup age
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, emdbKey, &existingLogicalBackup)
+ Expect(err).To(Succeed())
+ secondLogicalBackupCreationTimestamp := existingLogicalBackup.CreationTimestamp.Time
+
+ // Last age should be older than first
+ By("Expecting to have same CreationTimestamp on backup Object before and after the Pod recreation")
+ Expect(firstLogicalBackupCreationTimestamp).To(Equal(secondLogicalBackupCreationTimestamp))
+
+ })
+
+ It("should invalidate physical backup if older than the master binlog retention period", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Get current backup age
+ By("Expecting to get current external MariadDB object")
+ refResolver := refresolver.New(k8sClient)
+ emdb, err := refResolver.ExternalMariaDB(testCtx, &mdb.Replication().ReplicaFromExternal.MariaDBRef.ObjectReference, testNamespace)
+ Expect(err).To(Succeed())
+
+ logicalBackupKey := types.NamespacedName{
+ Name: emdb.Name,
+ Namespace: emdb.Namespace,
+ }
+
+ var existingLogicalBackup mariadbv1alpha1.Backup
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, logicalBackupKey, &existingLogicalBackup)
+ Expect(err).To(Succeed())
+ firstLogicalBackupCreationTimestamp := existingLogicalBackup.CreationTimestamp.Time
+
+ var existingPhysicalBackup mariadbv1alpha1.PhysicalBackup
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, pbRecoveryKey, &existingPhysicalBackup)
+ Expect(err).To(Succeed())
+ firstPhysicalBackupCreationTimestamp := existingPhysicalBackup.CreationTimestamp.Time
+
+ podIndex := 2
+
+ // Change binlog_expire_logs_seconds to 10 on the master server
+ By("Expecting to get SqlClient from the external MariaDB")
+ client, err := sqlClient.NewClientWithMariaDB(testCtx, emdb, refResolver)
+ Expect(err).To(Succeed())
+ defer client.Close()
+
+ By("Expecting to set binlog_expire_logs_seconds to 10 on the master server")
+ Expect(client.SetSystemVariable(testCtx, "binlog_expire_logs_seconds", "10")).To(Succeed())
+
+ testDeletePod(mdb, podIndex, true)
+
+ By("Expecting MariaDB to be in recovering state eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+
+ return mdb.IsRecoveringReplicas()
+
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Revert binlog_expire_logs_seconds to 30 days on the master server
+ By("Expecting to set expire_logs_days to 30 on the master server")
+ Expect(client.SetSystemVariable(testCtx, "expire_logs_days", "30")).To(Succeed())
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+
+ // return (mdb.Status.Replication.Roles)[statefulset.PodName(mdb.ObjectMeta, podIndex)] == mariadbv1alpha1.ReplicationRoleReplica
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Get current Physical backup age
+ By("Expecting to get physical backup object")
+ err = k8sClient.Get(testCtx, pbRecoveryKey, &existingPhysicalBackup)
+ Expect(err).To(Succeed())
+ secondPhysicalBackupCreationTimestamp := existingPhysicalBackup.CreationTimestamp.Time
+
+ // Physical backup should be updated as it's older than the master binlog retention period
+ By("Expecting to have different CreationTimestamp on physical backup Object before and after the Pod recreation")
+ Expect(firstPhysicalBackupCreationTimestamp).ShouldNot(Equal(secondPhysicalBackupCreationTimestamp))
+
+ // Get current backup age
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, logicalBackupKey, &existingLogicalBackup)
+ Expect(err).To(Succeed())
+ secondLogicalBackupCreationTimestamp := existingLogicalBackup.CreationTimestamp.Time
+
+ // Logical backup should should not be touched as the cluster still has valid replicas for a physical backup
+ By("Expecting to have same CreationTimestamp on backup Object before and after the Pod recreation")
+ Expect(firstLogicalBackupCreationTimestamp).Should(Equal(secondLogicalBackupCreationTimestamp))
+
+ // Revert binlog_expire_logs_seconds to 30 days on the master server
+ By("Expecting to set expire_logs_days to 30 on the master server")
+ Expect(client.SetSystemVariable(testCtx, "expire_logs_days", "30")).To(Succeed())
+ })
+
+ It("should invalidate logical backup if older than the master binlog retention period and no phy backup is avail and no valid replicas",
+ func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Get current backup age
+ By("Expecting to get current external MariadDB object")
+ refResolver := refresolver.New(k8sClient)
+ emdb, err := refResolver.ExternalMariaDB(testCtx, &mdb.Replication().ReplicaFromExternal.MariaDBRef.ObjectReference, testNamespace)
+ Expect(err).To(Succeed())
+
+ logicalBackupKey := types.NamespacedName{
+ Name: emdb.Name,
+ Namespace: emdb.Namespace,
+ }
+
+ var existingLogicalBackup mariadbv1alpha1.Backup
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, logicalBackupKey, &existingLogicalBackup)
+ Expect(err).To(Succeed())
+ firstLogicalBackupCreationTimestamp := existingLogicalBackup.CreationTimestamp.Time
+
+ var existingPhysicalBackup mariadbv1alpha1.PhysicalBackup
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, pbRecoveryKey, &existingPhysicalBackup)
+ Expect(err).To(Succeed())
+ firstPhysicalBackupCreationTimestamp := existingPhysicalBackup.CreationTimestamp.Time
+
+ // Change binlog_expire_logs_seconds to 10 on the master server
+ By("Expecting to get SqlClient from the external MariaDB")
+ client, err := sqlClient.NewClientWithMariaDB(testCtx, emdb, refResolver)
+ Expect(err).To(Succeed())
+ defer client.Close()
+
+ By("Expecting to set binlog_expire_logs_seconds to 30 on the master server")
+ Expect(client.SetSystemVariable(testCtx, "binlog_expire_logs_seconds", "30")).To(Succeed())
+
+ // Delete physical backup to be sure that only logical backup is available
+ By("Expecting to delete physical backup")
+ Expect(k8sClient.Delete(testCtx, &existingPhysicalBackup)).To(Succeed())
+
+ // Delete all replicas to be sure that no valid replica exists for a physical backup
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ testDeletePod(mdb, i, true)
+ }
+
+ By("Expecting MariaDB to be in recovering state eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsRecoveringReplicas()
+
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting Logical backup to replaced eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, logicalBackupKey, &existingLogicalBackup); err != nil {
+ return false
+ }
+ secondLogicalBackupCreationTimestamp := existingLogicalBackup.CreationTimestamp.Time
+ return secondLogicalBackupCreationTimestamp.After(firstLogicalBackupCreationTimestamp)
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // Revert binlog_expire_logs_seconds to 30 days on the master server to avoid issues with other tests
+ Expect(client.SetSystemVariable(testCtx, "expire_logs_days", "30")).To(Succeed())
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+
+ // return (mdb.Status.Replication.Roles)[statefulset.PodName(mdb.ObjectMeta, podIndex)] == mariadbv1alpha1.ReplicationRoleReplica
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ // Get current Physical backup age
+ By("Expecting to get physical backup object")
+ err = k8sClient.Get(testCtx, pbRecoveryKey, &existingPhysicalBackup)
+ Expect(err).To(Succeed())
+ secondPhysicalBackupCreationTimestamp := existingPhysicalBackup.CreationTimestamp.Time
+
+ // Physical backup should be updated as it's older than the master binlog retention period
+ By("Expecting to have different CreationTimestamp on physical backup Object before and after the Pod recreation")
+ Expect(firstPhysicalBackupCreationTimestamp).ShouldNot(Equal(secondPhysicalBackupCreationTimestamp))
+
+ // Get current backup age
+ By("Expecting to get backup object")
+ err = k8sClient.Get(testCtx, logicalBackupKey, &existingLogicalBackup)
+ Expect(err).To(Succeed())
+ secondLogicalBackupCreationTimestamp := existingLogicalBackup.CreationTimestamp.Time
+
+ // Logical backup should be updated as it's older than the master binlog retention period and no physical backup
+ // is available and no valid replicas exist
+ By("Expecting to have different CreationTimestamp on backup Object before and after the Pod recreation")
+ Expect(firstLogicalBackupCreationTimestamp).ShouldNot(Equal(secondLogicalBackupCreationTimestamp))
+
+ // Revert binlog_expire_logs_seconds to 30 days on the master server
+ By("Expecting to set expire_logs_days to 30 on the master server")
+ Expect(client.SetSystemVariable(testCtx, "expire_logs_days", "30")).To(Succeed())
+ })
+
+ It("scale out replicas", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Increasing MariaDB replicas to 4")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ mdb.Spec.Replicas = 4
+
+ return k8sClient.Update(testCtx, mdb) == nil
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ var endpoints discoveryv1.EndpointSlice
+ By("Expecting to create secondary Endpoints: 4")
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+ count := 0
+ for _, address := range endpoints.Endpoints {
+ if *address.Conditions.Ready {
+ count++
+ }
+ }
+ return count == 4
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ })
+
+ It("scale in replicas", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Decreasing MariaDB replicas to 3")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ mdb.Spec.Replicas = 3
+
+ return k8sClient.Update(testCtx, mdb) == nil
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ var endpoints discoveryv1.EndpointSlice
+ By("Expecting secondary Endpoints: 3")
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+ count := 0
+ for _, address := range endpoints.Endpoints {
+ if *address.Conditions.Ready {
+ count++
+ }
+ }
+ return count == 3
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ })
+
+ It("use the server_id offset", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ offset := mdb.Replication().ReplicaFromExternal.ServerIdOffset
+ replicas := int(mdb.Spec.Replicas)
+ refResolver := refresolver.New(k8sClient)
+ for i := 0; i < replicas; i++ {
+
+ client, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i)
+ By("Expecting to get SqlClient from Pod " + strconv.Itoa(i))
+ Expect(err).To(Succeed())
+
+ server_id, err := client.SystemVariable(testCtx, "server_id")
+ By("Expecting to get server_id from Pod " + strconv.Itoa(i))
+ Expect(err).To(Succeed())
+
+ server_id_int, _ := strconv.Atoi(server_id)
+
+ By("Expecting server_id to be equal to podIndex + ServerIdOffset on Pod " + strconv.Itoa(i))
+ Expect(server_id_int).To(Equal(i + *offset))
+
+ }
+ })
+
+ It("should update", func() {
+ By("Updating MariaDB")
+ testMariadbUpdate(mdb)
+ })
+
+ It("should resize PVCs", func() {
+ By("Resizing MariaDB PVCs")
+ testMariadbVolumeResize(mdb, "400Mi")
+ })
+
+ It("should heal external master connection drift", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady() && mdb.IsExternalReplInitialized() && !mdb.IsRecoveringReplicas()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ By("Getting the desired external master host")
+ var emdb mariadbv1alpha1.ExternalMariaDB
+ Expect(k8sClient.Get(testCtx, testEMdbkey, &emdb)).To(Succeed())
+ desiredHost := emdb.GetHost()
+ Expect(desiredHost).NotTo(BeEmpty())
+
+ // RFC 5737 TEST-NET-1 address, guaranteed not to be the real external master.
+ const bogusHost = "192.0.2.123"
+
+ By("Pointing every replica at a bogus master to simulate connection drift")
+ refResolver := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+
+ var podClient *sqlClient.Client
+ Eventually(func() error {
+ var err error
+ podClient, err = sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i)
+ return err
+ }, testTimeout, testInterval).Should(Succeed())
+ defer podClient.Close()
+
+ Expect(podClient.StopAllSlaves(testCtx)).To(Succeed())
+ Expect(podClient.Exec(testCtx, fmt.Sprintf("CHANGE MASTER TO MASTER_HOST='%s';", bogusHost))).To(Succeed())
+
+ By(fmt.Sprintf("Verifying Pod %d master host has drifted", i))
+ status, err := podClient.QueryColumnMap(testCtx, "SHOW REPLICA STATUS")
+ Expect(err).To(Succeed())
+ Expect(status["Master_Host"]).To(Equal(bogusHost))
+ }
+
+ By("Expecting the operator to re-point every replica at the external master and resume replication")
+ refResolver2 := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver2, i)
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ Eventually(func(g Gomega) {
+ status, err := podClient.QueryColumnMap(testCtx, "SHOW REPLICA STATUS")
+ g.Expect(err).To(Succeed())
+ g.Expect(status["Master_Host"]).To(Equal(desiredHost))
+ g.Expect(status["Slave_IO_Running"]).To(Equal("Yes"))
+ g.Expect(status["Slave_SQL_Running"]).To(Equal("Yes"))
+ }, testHighTimeout, testInterval).Should(Succeed(),
+ fmt.Sprintf("Pod %d should be re-pointed at the external master", i))
+ }
+ })
+
+ It("should re-apply the replication password on an authentication error", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady() && mdb.IsExternalReplInitialized()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ // The master host and user are left untouched: only the password is broken. This exercises
+ // the authentication-error repair path specifically, since no host/port/user drift exists.
+ By("Breaking the replication credentials on every replica to trigger an authentication error")
+ refResolver := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i)
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ Expect(podClient.StopAllSlaves(testCtx)).To(Succeed())
+ Expect(podClient.Exec(testCtx, "CHANGE MASTER TO MASTER_PASSWORD='wrong-password';")).To(Succeed())
+ Expect(podClient.StartSlave(testCtx)).To(Succeed())
+ }
+
+ By("Expecting the operator to re-apply the credentials and restore healthy replication")
+ refResolver2 := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver2, i)
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ Eventually(func(g Gomega) {
+ status, err := podClient.QueryColumnMap(testCtx, "SHOW REPLICA STATUS")
+ g.Expect(err).To(Succeed())
+ g.Expect(status["Slave_IO_Running"]).To(Equal("Yes"))
+ g.Expect(status["Slave_SQL_Running"]).To(Equal("Yes"))
+ }, testHighTimeout, testInterval).Should(Succeed(),
+ fmt.Sprintf("Pod %d should resume replication after credential repair", i))
+ }
+ })
+
+})
+
+var _ = Describe("MariaDB replication from external server with server_id offset auto-discovery", Ordered, func() {
+ var (
+ // Two ExternalMariaDBs referencing the same HA testEmulateExternalMdb: one at its primary
+ // service (master endpoint) and one at its secondary service (slave endpoint). Both reuse the
+ // emulated-external credentials/TLS, so the operator can follow the secondary endpoint to the
+ // primary with the same connection settings.
+ emdbMasterKey = types.NamespacedName{Name: "emdb-autodisc-master", Namespace: testNamespace}
+ emdbSlaveKey = types.NamespacedName{Name: "emdb-autodisc-slave", Namespace: testNamespace}
+
+ // mdbFromMaster replicates from the primary endpoint and is fully bootstrapped, so its replica
+ // server_ids register on the external primary. mdbFromSlave then replicates from the secondary
+ // endpoint and must discover a higher, non-colliding offset by following to the same primary.
+ mdbFromMasterKey = types.NamespacedName{Name: "mdb-autodisc-master", Namespace: testNamespace}
+ mdbFromSlaveKey = types.NamespacedName{Name: "mdb-autodisc-slave", Namespace: testNamespace}
+
+ pbFromMasterKey = types.NamespacedName{Name: mdbFromMasterKey.Name + "-backup-template", Namespace: testNamespace}
+ pbFromSlaveKey = types.NamespacedName{Name: mdbFromSlaveKey.Name + "-backup-template", Namespace: testNamespace}
+
+ externalPrimaryHost = fmt.Sprintf("%s-primary.%s.svc.cluster.local", testEmulateExternalMdbkey.Name, testNamespace)
+ externalSecondaryHost = fmt.Sprintf("%s-secondary.%s.svc.cluster.local", testEmulateExternalMdbkey.Name, testNamespace)
+
+ // server_ids already in use on testEmulateExternalMdb (its own nodes), captured before any of
+ // the clusters under test connect. The discovered offsets must not collide with these.
+ externalServerIds []int
+ masterOffset int
+ )
+
+ BeforeAll(func() {
+ By("Capturing the server_ids already in use on the external primary")
+ var extMdb mariadbv1alpha1.MariaDB
+ Expect(k8sClient.Get(testCtx, testEmulateExternalMdbkey, &extMdb)).To(Succeed())
+ extClient, err := sqlClient.NewClientWithMariaDB(testCtx, &extMdb, testRefResolver)
+ Expect(err).To(Succeed())
+ defer extClient.Close()
+ externalServerIds, err = extClient.InUseServerIds(testCtx)
+ Expect(err).To(Succeed())
+ Expect(externalServerIds).NotTo(BeEmpty())
+
+ By("Creating the ExternalMariaDB pointing at the external primary service (master endpoint)")
+ Expect(k8sClient.Create(testCtx, buildAutodiscExternalMariaDB(emdbMasterKey, externalPrimaryHost))).To(Succeed())
+ expectExternalMariadbReady(testCtx, k8sClient, emdbMasterKey)
+
+ By("Creating the physical backup template and the master-endpoint cluster (serverIdOffset unset)")
+ Expect(k8sClient.Create(testCtx, buildAutodiscBackupTemplate(pbFromMasterKey, mdbFromMasterKey))).To(Succeed())
+ Expect(k8sClient.Create(testCtx, buildAutodiscReplica(mdbFromMasterKey, emdbMasterKey, pbFromMasterKey))).To(Succeed())
+
+ DeferCleanup(func() {
+ deleteMariadb(mdbFromMasterKey, false)
+ deleteMariadb(mdbFromSlaveKey, false)
+ deleteExternalMariadbIfExists(emdbMasterKey)
+ deleteExternalMariadbIfExists(emdbSlaveKey)
+ deletePhysicalBackupIfExists(pbFromMasterKey)
+ deletePhysicalBackupIfExists(pbFromSlaveKey)
+ })
+ })
+
+ It("should auto-discover the offset from a master endpoint", func() {
+ By("Expecting the discovered offset to be persisted to status")
+ var mdb mariadbv1alpha1.MariaDB
+ Eventually(func(g Gomega) {
+ g.Expect(k8sClient.Get(testCtx, mdbFromMasterKey, &mdb)).To(Succeed())
+ g.Expect(mdb.Status.ExternalReplication).NotTo(BeNil())
+ g.Expect(mdb.Status.ExternalReplication.ServerIdOffset).NotTo(BeNil())
+ }, testHighTimeout, testInterval).Should(Succeed())
+ masterOffset = *mdb.Status.ExternalReplication.ServerIdOffset
+
+ By("Expecting the offset to be the highest external server_id plus the gap")
+ Expect(masterOffset >= slices.Max(externalServerIds)+externalReplServerIdGap).To(
+ BeTrue(),
+ "discovered offset %d should be higher than the highest external server_id %d plus the gap %d",
+ masterOffset,
+ slices.Max(externalServerIds),
+ externalReplServerIdGap,
+ )
+
+ By("Expecting the spec serverIdOffset to remain unset (discovery lives in status)")
+ Expect(mdb.Replication().ReplicaFromExternal.ServerIdOffset).To(BeNil())
+ Expect(ptr.Deref(mdb.ExternalReplServerIdOffset(), 0)).To(Equal(masterOffset))
+
+ By("Expecting the StatefulSet to carry the discovered offset as the server_id offset env var")
+ Eventually(func(g Gomega) {
+ var sts appsv1.StatefulSet
+ g.Expect(k8sClient.Get(testCtx, mdbFromMasterKey, &sts)).To(Succeed())
+ value, ok := autodiscContainerEnv(&sts, builder.MariadbContainerName, "MARIADB_EXTERNAL_REPL_SERVER_ID_OFFSET")
+ g.Expect(ok).To(BeTrue())
+ g.Expect(value).To(Equal(strconv.Itoa(masterOffset)))
+ }, testHighTimeout, testInterval).Should(Succeed())
+ })
+
+ It("should apply the discovered offset to the replica server_ids", func() {
+ By("Expecting the master-endpoint cluster to be ready eventually")
+ var mdb mariadbv1alpha1.MariaDB
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, mdbFromMasterKey, &mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting server_id to equal podIndex + discovered offset on every replica")
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, &mdb, testRefResolver, i)
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ serverID, err := podClient.SystemVariable(testCtx, "server_id")
+ Expect(err).To(Succeed())
+ serverIDInt, err := strconv.Atoi(serverID)
+ Expect(err).To(Succeed())
+ Expect(serverIDInt).To(Equal(i + masterOffset))
+ }
+ })
+
+ It("should discover a higher, non-colliding offset from a slave endpoint by following it to the primary", func() {
+ By("Waiting until the master-endpoint cluster's replica server_ids are registered on the external primary")
+ var extMdb mariadbv1alpha1.MariaDB
+ Expect(k8sClient.Get(testCtx, testEmulateExternalMdbkey, &extMdb)).To(Succeed())
+ primaryClient, err := sqlClient.NewClientWithMariaDB(testCtx, &extMdb, testRefResolver)
+ Expect(err).To(Succeed())
+ defer primaryClient.Close()
+
+ masterCluster := &mariadbv1alpha1.MariaDB{}
+ Expect(k8sClient.Get(testCtx, mdbFromMasterKey, masterCluster)).To(Succeed())
+ masterIds := make([]int, 0, masterCluster.Spec.Replicas)
+ for i := 0; i < int(masterCluster.Spec.Replicas); i++ {
+ masterIds = append(masterIds, i+masterOffset)
+ }
+ Eventually(func(g Gomega) {
+ ids, err := primaryClient.InUseServerIds(testCtx)
+ g.Expect(err).To(Succeed())
+ g.Expect(ids).To(ContainElements(masterIds))
+ }, testVeryHighTimeout, testInterval).Should(Succeed())
+
+ By("Capturing the server_ids in use on the external primary before creating the slave-endpoint cluster")
+ primaryIds, err := primaryClient.InUseServerIds(testCtx)
+ Expect(err).To(Succeed())
+ expectedSlaveOffset := slices.Max(primaryIds) + externalReplServerIdGap
+
+ By("Creating the ExternalMariaDB pointing at the external secondary service (slave endpoint)")
+ Expect(k8sClient.Create(testCtx, buildAutodiscExternalMariaDB(emdbSlaveKey, externalSecondaryHost))).To(Succeed())
+ expectExternalMariadbReady(testCtx, k8sClient, emdbSlaveKey)
+
+ By("Creating the physical backup template and the slave-endpoint cluster (serverIdOffset unset)")
+ Expect(k8sClient.Create(testCtx, buildAutodiscBackupTemplate(pbFromSlaveKey, mdbFromSlaveKey))).To(Succeed())
+ Expect(k8sClient.Create(testCtx, buildAutodiscReplica(mdbFromSlaveKey, emdbSlaveKey, pbFromSlaveKey))).To(Succeed())
+
+ By("Expecting the slave-endpoint cluster to discover the offset derived from the primary")
+ var mdb mariadbv1alpha1.MariaDB
+ Eventually(func(g Gomega) {
+ g.Expect(k8sClient.Get(testCtx, mdbFromSlaveKey, &mdb)).To(Succeed())
+ g.Expect(mdb.Status.ExternalReplication).NotTo(BeNil())
+ g.Expect(mdb.Status.ExternalReplication.ServerIdOffset).NotTo(BeNil())
+ }, testHighTimeout, testInterval).Should(Succeed())
+ slaveOffset := *mdb.Status.ExternalReplication.ServerIdOffset
+ Expect(slaveOffset).To(Equal(expectedSlaveOffset))
+
+ By("Expecting the slave-endpoint offset to be higher than the master-endpoint one")
+ Expect(slaveOffset).To(BeNumerically(">", masterOffset))
+
+ By("Expecting the server_ids of both clusters not to collide with each other nor with the external servers")
+ slaveIds := make([]int, 0, mdb.Spec.Replicas)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ slaveIds = append(slaveIds, i+slaveOffset)
+ }
+ Expect(autodiscDisjoint(masterIds, slaveIds)).To(BeTrue(), "master and slave server_ids must not collide")
+ Expect(autodiscDisjoint(masterIds, externalServerIds)).To(BeTrue(), "master server_ids must not collide with the external servers")
+ Expect(autodiscDisjoint(slaveIds, externalServerIds)).To(BeTrue(), "slave server_ids must not collide with the external servers")
+
+ By("Expecting the StatefulSet to carry the discovered offset as the server_id offset env var")
+ Eventually(func(g Gomega) {
+ var sts appsv1.StatefulSet
+ g.Expect(k8sClient.Get(testCtx, mdbFromSlaveKey, &sts)).To(Succeed())
+ value, ok := autodiscContainerEnv(&sts, builder.MariadbContainerName, "MARIADB_EXTERNAL_REPL_SERVER_ID_OFFSET")
+ g.Expect(ok).To(BeTrue())
+ g.Expect(value).To(Equal(strconv.Itoa(slaveOffset)))
+ }, testHighTimeout, testInterval).Should(Succeed())
+ })
+})
+
+// buildAutodiscExternalMariaDB builds an ExternalMariaDB pointing at host, reusing the emulated
+// external credentials and TLS material. Both endpoints belong to the same testEmulateExternalMdb
+// cluster (one CA), so the operator can follow the secondary endpoint to the primary with the same
+// TLS settings.
+func buildAutodiscExternalMariaDB(key types.NamespacedName, host string) *mariadbv1alpha1.ExternalMariaDB {
+ return &mariadbv1alpha1.ExternalMariaDB{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: key.Name,
+ Namespace: key.Namespace,
+ },
+ Spec: mariadbv1alpha1.ExternalMariaDBSpec{
+ Host: host,
+ Username: ptr.To("root"),
+ PasswordSecretKeyRef: &mariadbv1alpha1.SecretKeySelector{
+ LocalObjectReference: mariadbv1alpha1.LocalObjectReference{
+ Name: testEmulatedExternalPwdKey.Name,
+ },
+ Key: testPwdSecretKey,
+ },
+ TLS: &mariadbv1alpha1.ExternalTLS{
+ TLS: mariadbv1alpha1.TLS{
+ Enabled: true,
+ Required: ptr.To(false),
+ ServerCASecretRef: &mariadbv1alpha1.LocalObjectReference{
+ Name: "mdb-emulate-external-test-ca",
+ },
+ ClientCertSecretRef: &mariadbv1alpha1.LocalObjectReference{
+ Name: "mdb-emulate-external-test-client-cert",
+ },
+ ServerCertSecretRef: &mariadbv1alpha1.LocalObjectReference{
+ Name: "mdb-emulate-external-test-server-cert",
+ },
+ },
+ },
+ },
+ }
+}
+
+// buildAutodiscBackupTemplate builds the physical backup template that a cluster under test bootstraps
+// from. External replication requires a bootstrap source; the offset is discovered in the status phase
+// well before the bootstrap runs.
+func buildAutodiscBackupTemplate(key, mdbKey types.NamespacedName) *mariadbv1alpha1.PhysicalBackup {
+ return &mariadbv1alpha1.PhysicalBackup{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: key.Name,
+ Namespace: key.Namespace,
+ },
+ Spec: mariadbv1alpha1.PhysicalBackupSpec{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: mdbKey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ WaitForIt: false,
+ },
+ Target: ptr.To(mariadbv1alpha1.PhysicalBackupTargetPreferReplica),
+ Schedule: &mariadbv1alpha1.PhysicalBackupSchedule{Suspend: true},
+ Compression: mariadbv1alpha1.CompressBzip2,
+ Storage: mariadbv1alpha1.PhysicalBackupStorage{
+ PersistentVolumeClaim: &mariadbv1alpha1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")},
+ },
+ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+ },
+ },
+ Timeout: &metav1.Duration{Duration: 1 * time.Hour},
+ PodAffinity: ptr.To(true),
+ JobContainerTemplate: mariadbv1alpha1.JobContainerTemplate{
+ Resources: &mariadbv1alpha1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("100m"),
+ corev1.ResourceMemory: resource.MustParse("128Mi"),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("300m"),
+ corev1.ResourceMemory: resource.MustParse("512Mi"),
+ },
+ },
+ },
+ },
+ }
+}
+
+// buildAutodiscReplica builds a MariaDB that replicates from the given external endpoint with
+// serverIdOffset intentionally unset, so the operator must auto-discover it.
+func buildAutodiscReplica(key, emdbKey, pbKey types.NamespacedName) *mariadbv1alpha1.MariaDB {
+ mdb := &mariadbv1alpha1.MariaDB{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: key.Name,
+ Namespace: key.Namespace,
+ },
+ Spec: mariadbv1alpha1.MariaDBSpec{
+ Username: &testUser,
+ PasswordSecretKeyRef: &mariadbv1alpha1.GeneratedSecretKeyRef{
+ SecretKeySelector: mariadbv1alpha1.SecretKeySelector{
+ LocalObjectReference: mariadbv1alpha1.LocalObjectReference{
+ Name: testPwdKey.Name,
+ },
+ Key: testPwdSecretKey,
+ },
+ },
+ Database: &testDatabase,
+ MyCnf: ptr.To(`[mariadb]
+bind-address=*
+default_storage_engine=InnoDB
+binlog_format=row
+innodb_autoinc_lock_mode=2
+max_allowed_packet=256M`),
+ Replication: &mariadbv1alpha1.Replication{
+ ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+ ReplicaFromExternal: &mariadbv1alpha1.ReplicaFromExternal{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: emdbKey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ },
+ // ServerIdOffset intentionally left unset to exercise auto-discovery.
+ },
+ Replica: mariadbv1alpha1.ReplicaReplication{
+ ReplicaBootstrapFrom: &mariadbv1alpha1.ReplicaBootstrapFrom{
+ PhysicalBackupTemplateRef: mariadbv1alpha1.LocalObjectReference{
+ Name: pbKey.Name,
+ },
+ },
+ IgnoreMaxLagSeconds: ptr.To(true),
+ IgnoreReplicationLivenessProbes: ptr.To(true),
+ },
+ },
+ Enabled: true,
+ },
+ Replicas: 2,
+ Storage: mariadbv1alpha1.Storage{
+ Size: ptr.To(resource.MustParse("300Mi")),
+ StorageClassName: "standard-resize",
+ },
+ TLS: &mariadbv1alpha1.TLS{
+ Enabled: true,
+ Required: ptr.To(true),
+ },
+ },
+ }
+ return applyMariadbTestConfig(mdb)
+}
+
+// autodiscContainerEnv returns the value of the named env var on the named container of the StatefulSet.
+func autodiscContainerEnv(sts *appsv1.StatefulSet, containerName, envName string) (string, bool) {
+ for _, c := range sts.Spec.Template.Spec.Containers {
+ if c.Name != containerName {
+ continue
+ }
+ for _, e := range c.Env {
+ if e.Name == envName {
+ return e.Value, true
+ }
+ }
+ }
+ return "", false
+}
+
+// autodiscDisjoint reports whether the two server_id sets have no element in common.
+func autodiscDisjoint(a, b []int) bool {
+ for _, id := range a {
+ if slices.Contains(b, id) {
+ return false
+ }
+ }
+ return true
+}
+
+// deleteExternalMariadbIfExists deletes an ExternalMariaDB, ignoring a not-found error.
+func deleteExternalMariadbIfExists(key types.NamespacedName) {
+ var emdb mariadbv1alpha1.ExternalMariaDB
+ if err := k8sClient.Get(testCtx, key, &emdb); err == nil {
+ Expect(k8sClient.Delete(testCtx, &emdb)).To(Succeed())
+ }
+}
+
+// deletePhysicalBackupIfExists deletes a PhysicalBackup, ignoring a not-found error.
+func deletePhysicalBackupIfExists(key types.NamespacedName) {
+ var pb mariadbv1alpha1.PhysicalBackup
+ if err := k8sClient.Get(testCtx, key, &pb); err == nil {
+ Expect(k8sClient.Delete(testCtx, &pb)).To(Succeed())
+ }
+}
diff --git a/internal/controller/mariadb_controller_init.go b/internal/controller/mariadb_controller_init.go
index bb1d5bdccf..6a4c6271f3 100644
--- a/internal/controller/mariadb_controller_init.go
+++ b/internal/controller/mariadb_controller_init.go
@@ -288,7 +288,7 @@ func (r *MariaDBReconciler) waitForReadyVolumeSnapshot(ctx context.Context, key
}
func (r *MariaDBReconciler) reconcileRollingInitJobs(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
- fromIndex int, logger logr.Logger, restoreOpts ...builder.RestoreOpt) (ctrl.Result, error) {
+ fromIndex int, logger logr.Logger, restoreOpts ...builder.PhysicalBackupRestoreOpt) (ctrl.Result, error) {
return r.forEachMariaDBPod(mariadb, fromIndex, func(podIndex int) (ctrl.Result, error) {
physicalBackupKey := mariadb.PhysicalBackupInitJobKey(podIndex)
@@ -320,7 +320,7 @@ func (r *MariaDBReconciler) reconcileRollingInitJobs(ctx context.Context, mariad
}
func (r *MariaDBReconciler) reconcileAndWaitForInitJob(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
- key types.NamespacedName, podIndex int, logger logr.Logger, restoreOpts ...builder.RestoreOpt) (ctrl.Result, error) {
+ key types.NamespacedName, podIndex int, logger logr.Logger, restoreOpts ...builder.PhysicalBackupRestoreOpt) (ctrl.Result, error) {
var job batchv1.Job
if err := r.Get(ctx, key, &job); err != nil {
if apierrors.IsNotFound(err) {
@@ -339,7 +339,7 @@ func (r *MariaDBReconciler) reconcileAndWaitForInitJob(ctx context.Context, mari
}
func (r *MariaDBReconciler) createInitJob(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
- key types.NamespacedName, podIndex int, restoreOpts ...builder.RestoreOpt) error {
+ key types.NamespacedName, podIndex int, restoreOpts ...builder.PhysicalBackupRestoreOpt) error {
job, err := r.Builder.BuildPhysicalBackupRestoreJob(
key,
mariadb,
diff --git a/internal/controller/mariadb_controller_replica_recovery.go b/internal/controller/mariadb_controller_replica_recovery.go
index 4bfa8b3bd4..83cc166de7 100644
--- a/internal/controller/mariadb_controller_replica_recovery.go
+++ b/internal/controller/mariadb_controller_replica_recovery.go
@@ -30,6 +30,73 @@ var recoverableIOErrorCodes = []int{
// Error 1236: Got fatal error from master when reading data from binary log.
// See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1200-to-1299/e1236
1236,
+ // Error 1945: Connecting slave requested to start from GTID, which is not in the master's binlog
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1900-to-1999/e1945
+ 1945,
+ // Error 1947: Specified GTID conflicts with the binary log which contains a more recent GTID
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1900-to-1999/e1947
+ 1947,
+ // Error 1951: The binlog on the master is missing the GTID requested by the slave
+ // https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1900-to-1999/e1951
+ 1951,
+ // Error 1955: Connecting slave requested to start from GTID which is not in the master's binlog
+ // https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1900-to-1999/e1955
+ 1955,
+}
+
+var recoverableSQLErrorCodes = []int{
+ // Error 1062: Duplicate entry for key.
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1000-to-1099/e1062
+ 1062,
+ // Error 1032: Can't find record in
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1000-to-1099/e1032
+ 1032,
+ // Error 1034: Incorrect key file for table; try to repair it
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1000-to-1099/e1034
+ 1034,
+ // Error 1049: Unknown database
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1000-to-1099/e1049
+ 1049,
+ // Error 1046: No database selected
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1000-to-1099/e1046
+ 1146,
+}
+
+var externalUserPrivilegesSkippableSQLErrorCodes = []int{
+ // Error 1133: Can't find any matching row in the user table
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1100-to-1199/e1133
+ 1133,
+ // Error 1269: Can't revoke all privileges for one or more of the requested users
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1200-to-1299/e1269
+ 1269,
+ // Error 1396: Operation failed for
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1300-to-1399/e1396
+ 1396,
+}
+
+// These errors will never trigger the recovery process
+var notRecoverableIOErrorCodes = []int{
+ // Error 2003: Can't connect to MariaDB server.
+ 2003,
+ // Error 2013: Lost connection to the master during a query (TCP timeout or network blip).
+ 2013,
+ // Error 1158: Got an error reading communication packets
+ // See: https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1100-to-1199/e1158
+ 1158,
+ // Error 2026: TLS/SSL handshake failed (usually expired certificates)
+ 2026,
+ // Error 1045: Access denied for user (using password)
+ // https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1000-to-1099/e1045
+ 1045,
+ // Error 1130: Host is not allowed to connect to this MariaDB server
+ // https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1100-to-1199/e1130
+ 1130,
+ // Error 1129: Host is blocked because of many connection errors
+ // https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1100-to-1199/e1129
+ 1129,
+ // Error 1040: Too many connections
+ // https://mariadb.com/docs/server/reference/error-codes/mariadb-error-codes-1000-to-1099/e1040
+ 1040,
}
func shouldReconcileReplicaRecovery(mdb *mariadbv1alpha1.MariaDB) bool {
@@ -48,31 +115,51 @@ func shouldReconcileReplicaRecovery(mdb *mariadbv1alpha1.MariaDB) bool {
}
func (r *MariaDBReconciler) reconcileReplicaRecovery(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB) (ctrl.Result, error) {
+
+ logger := log.FromContext(ctx).
+ WithName("replica-recovery")
+ logger.Info("ReconcileReplicaRecovery")
+
if !shouldReconcileReplicaRecovery(mariadb) {
+ logger.Info("Should not reconcile replica recovery")
return ctrl.Result{}, nil
}
if !mariadb.IsReplicaRecoveryEnabled() {
+ logger.Info("Replica recovery is not enabled")
if err := r.resetReplicaRecovery(ctx, mariadb); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
- logger := log.FromContext(ctx).
- WithName("replica-recovery")
if !mariadb.IsRecoveringReplicas() || mariadb.ReplicaRecoveryError() != nil {
+ isRecovering := mariadb.IsRecoveringReplicas()
+ recoveryError := mariadb.ReplicaRecoveryError()
+ logger.Info("Is not recovering replicas or there is an error, so we can't reconcile replica recovery",
+ "isRecovering", isRecovering, "recoveryError", recoveryError)
if result, err := r.reconcileReplicaRecoveryError(ctx, mariadb, logger); !result.IsZero() || err != nil {
+ logger.Info("reconcile replica recovery error failed", "result", result, "err", err)
return result, err
}
}
+
+ replication := mariadb.Replication()
+
+ if replication.IsExternalReplication() {
+ //Check for skippable SQL Errors (user and privileges relates), and skip the error
+ replicasToSkipReplicationError := getReplicasToSkipReplicationError(mariadb, logger)
+ if _, err := r.reconcileSkippableReplicationError(ctx, replicasToSkipReplicationError, mariadb, logger); err != nil {
+ logger.Info("ExternalReplication, logical restore error", "error", err)
+ return ctrl.Result{}, err
+ }
+ }
+
replicasToRecover := getReplicasToRecover(mariadb, logger)
logger = logger.
WithValues("replicas", replicasToRecover)
+ logger.Info("Replicas to recover", "total replicas", replicasToRecover)
if len(replicasToRecover) == 0 {
- if err := r.setReplicaRecoveredAndCleanup(ctx, mariadb); err != nil {
- return ctrl.Result{}, err
- }
return ctrl.Result{}, nil
}
@@ -86,8 +173,29 @@ func (r *MariaDBReconciler) reconcileReplicaRecovery(ctx context.Context, mariad
physicalBackupKey := mariadb.PhysicalBackupReplicaRecoveryKey()
if result, err := r.reconcileReplicaPhysicalBackup(ctx, physicalBackupKey, mariadb, logger); !result.IsZero() || err != nil {
+ if replication.IsExternalReplication() {
+
+ if err != nil && errors.Is(err, errPhysicalBackupJobLaunchTimeout) {
+ logger.Info("ExternalReplication, PhysicalBackup not able to launch jobs, trigger LogicalBackup", "error", err)
+ if result, err := r.reconcileLogicalBackup(ctx, mariadb, replication, logger); err != nil || !result.IsZero() {
+ return result, err
+ }
+ if _, err := r.reconcileLogicalBackupReplicaRecovery(ctx, replicasToRecover[0], mariadb, logger); err != nil {
+ logger.Info("ExternalReplication, logical restore error", "error", err)
+ return ctrl.Result{}, err
+ }
+
+ // Remove current physical backup as logical backup was required to avoid reaching the timeout again
+ _ = r.cleanupPhysicalBackup(ctx, mariadb.PhysicalBackupReplicaRecoveryKey())
+ logger.Info("ExternalReplication, logical restore finished - requeue in 1min")
+ return ctrl.Result{}, nil
+ }
+ logger.Info("ExternalReplication, logical restore not required")
+ return ctrl.Result{}, nil
+ }
return result, err
}
+ logger.Info("PhysicalBackup finished")
physicalBackup, err := r.getPhysicalBackup(ctx, physicalBackupKey, mariadb)
if err != nil {
return ctrl.Result{}, fmt.Errorf("error getting PhysicalBackup: %v", err)
@@ -129,6 +237,7 @@ func (r *MariaDBReconciler) reconcileReplicaRecoveryError(ctx context.Context, m
func (r *MariaDBReconciler) reconcileReplicasToRecover(ctx context.Context, replicas []string, mariadb *mariadbv1alpha1.MariaDB,
physicalBackup *mariadbv1alpha1.PhysicalBackup, snapshotKey *types.NamespacedName, logger logr.Logger) (ctrl.Result, error) {
+ logger.Info("Reconcile replicas to recover")
for _, replica := range replicas {
replicaLogger := logger.WithValues("replica", replica)
replicaLogger.V(1).Info("Recovering replica")
@@ -163,6 +272,11 @@ func (r *MariaDBReconciler) reconcileReplicasToRecover(ctx context.Context, repl
return ctrl.Result{}, fmt.Errorf("error ensuring replica %s recovered: %v", replica, err)
}
}
+ logger.Info("Replicas to recovered, cleaning up")
+ if err := r.setReplicaRecoveredAndCleanup(ctx, mariadb); err != nil {
+ return ctrl.Result{}, err
+ }
+
// Requeue to track replication status
return ctrl.Result{Requeue: true}, nil
}
@@ -176,11 +290,16 @@ func (r *MariaDBReconciler) reconcileJobReplicaRecovery(ctx context.Context, rep
if err := r.patchStatus(ctx, mariadb, func(status *mariadbv1alpha1.MariaDBStatus) error {
mariadb.SetReplicaToRecover(&replica)
+ mariadb.Status.Replication.Roles[replica] = mariadbv1alpha1.ReplicationRoleUnknown
return nil
}); err != nil {
return ctrl.Result{}, fmt.Errorf("error patching MariaDB status: %v", err)
}
+ if _, err := r.reconcileService(ctx, mariadb); err != nil {
+ return ctrl.Result{}, err
+ }
+
isPodInitializing, err := r.isPodInitializing(ctx, podKey)
if err != nil {
return ctrl.Result{}, fmt.Errorf("error checking Pod initializing: %v", err)
@@ -201,9 +320,11 @@ func (r *MariaDBReconciler) reconcileJobReplicaRecovery(ctx context.Context, rep
); !result.IsZero() || err != nil {
return result, err
}
+ // Wait for pod get ready
if err := r.patchStatus(ctx, mariadb, func(status *mariadbv1alpha1.MariaDBStatus) error {
mariadb.SetReplicaToRecover(nil)
+ mariadb.Status.Replication.Roles[replica] = mariadbv1alpha1.ReplicationRoleReplica
return nil
}); err != nil {
return ctrl.Result{}, fmt.Errorf("error patching MariaDB status: %v", err)
@@ -211,6 +332,26 @@ func (r *MariaDBReconciler) reconcileJobReplicaRecovery(ctx context.Context, rep
return ctrl.Result{}, nil
}
+func (r *MariaDBReconciler) reconcileLogicalBackupReplicaRecovery(ctx context.Context, replica string,
+ mariadb *mariadbv1alpha1.MariaDB, logger logr.Logger) (ctrl.Result, error) {
+
+ podIndex, err := stsobj.PodIndex(replica)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting replica pod index: %v", err)
+ }
+
+ if _, err := r.reconcileRestoreInPod(ctx, mariadb, *podIndex, logger, true); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error reconciling restore in Pod: %v", err)
+ }
+
+ logger.Info("cleaning up the restore pod")
+ _ = r.cleanupRestoreInPod(ctx, mariadb, *podIndex, logger)
+
+ logger.Info("reconciling external logical restore finished")
+
+ return ctrl.Result{}, nil
+}
+
func (r *MariaDBReconciler) reconcileSnapshotReplicaRecovery(ctx context.Context, replica string,
physicalBackup *mariadbv1alpha1.PhysicalBackup, mariadb *mariadbv1alpha1.MariaDB, snapshotKey *types.NamespacedName,
logger logr.Logger) (ctrl.Result, error) {
@@ -356,6 +497,38 @@ func (r *MariaDBReconciler) reconcileAndWaitForRecoveryJob(ctx context.Context,
replication := ptr.Deref(mariadb.Spec.Replication, mariadbv1alpha1.Replication{})
bootstrapFrom := ptr.Deref(replication.Replica.ReplicaBootstrapFrom, mariadbv1alpha1.ReplicaBootstrapFrom{})
+ if replication.IsExternalReplication() {
+ emdb, err := r.RefResolver.ExternalMariaDB(ctx, &replication.ReplicaFromExternal.MariaDBRef.ObjectReference, mariadb.Namespace)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting external MariaDB: %v", err)
+ }
+
+ var binlogExpireLogsDuration time.Duration
+
+ logger.Info("Getting the binlog_expire_logs_seconds on the external MariaDB")
+ if binlogExpireLogsDuration, err = getBinlogExpireLogsDuration(emdb, ctx, r.RefResolver, logger); err != nil {
+ return ctrl.Result{}, fmt.Errorf("unable to get binlog_expire_logs_seconds: %v", err)
+ }
+
+ ageThreshold := time.Now().Add(-binlogExpireLogsDuration)
+
+ return r.reconcileAndWaitForInitJob(
+ ctx,
+ mariadb,
+ mariadb.PhysicalBackupInitJobKey(*podIndex),
+ *podIndex,
+ logger,
+ builder.WithPhysicalBackup(
+ physicalBackup,
+ time.Now(),
+ bootstrapFrom.RestoreJob,
+ command.WithCleanupDataDir(true),
+ ),
+ builder.WithReplicaRecovery(&pod),
+ builder.WithAgeThreshold(&ageThreshold),
+ )
+ }
+
return r.reconcileAndWaitForInitJob(
ctx,
mariadb,
@@ -370,6 +543,7 @@ func (r *MariaDBReconciler) reconcileAndWaitForRecoveryJob(ctx context.Context,
),
builder.WithReplicaRecovery(&pod),
)
+
}
func (r *MariaDBReconciler) ensureReplicaRecovered(ctx context.Context, replica string, mariadb *mariadbv1alpha1.MariaDB,
@@ -394,6 +568,21 @@ func (r *MariaDBReconciler) ensureReplicaRecovered(ctx context.Context, replica
return fmt.Errorf("error getting replica status: %v", err)
}
+ //ensure pod is on ready state
+ pod := corev1.Pod{}
+ podKey := types.NamespacedName{
+ Name: replica,
+ Namespace: mariadb.Namespace,
+ }
+
+ if err := r.Get(ctx, podKey, &pod); err != nil {
+ return fmt.Errorf("error getting replica pod: %v", err)
+ }
+
+ if !podobj.PodReady(&pod) {
+ return errors.New("pod not ready")
+ }
+
if replStatus.LastIOErrno != nil && *replStatus.LastIOErrno == 0 &&
replStatus.LastSQLErrno != nil && *replStatus.LastSQLErrno == 0 {
logger.Info("Replica recovered")
@@ -415,9 +604,6 @@ func (r *MariaDBReconciler) setReplicaRecoveredAndCleanup(ctx context.Context, m
return fmt.Errorf("error patching MariaDB status: %v", err)
}
- if err := r.cleanupPhysicalBackup(ctx, mariadb.PhysicalBackupReplicaRecoveryKey()); err != nil {
- return err
- }
if err := r.cleanupInitJobs(ctx, mariadb); err != nil {
return err
}
@@ -439,10 +625,32 @@ func getReplicasToRecover(mdb *mariadbv1alpha1.MariaDB, logger logr.Logger) []st
replication := ptr.Deref(mdb.Status.Replication, mariadbv1alpha1.ReplicationStatus{})
var replicas []string
for replica, err := range replication.Replicas {
+ logger.Info("Check if it is a recoverable error", "replica", replica)
if isRecoverableError(
mdb,
err,
recoverableIOErrorCodes,
+ recoverableSQLErrorCodes,
+ logger.WithValues("replica", replica),
+ ) {
+ replicas = append(replicas, replica)
+ }
+ }
+ sort.Slice(replicas, func(i, j int) bool {
+ return replicas[i] < replicas[j]
+ })
+ return replicas
+}
+
+func getReplicasToSkipReplicationError(mdb *mariadbv1alpha1.MariaDB, logger logr.Logger) []string {
+ replication := ptr.Deref(mdb.Status.Replication, mariadbv1alpha1.ReplicationStatus{})
+ var replicas []string
+ for replica, err := range replication.Replicas {
+ logger.Info("Check if it is a skippable error", "replica", replica)
+ if isSkippableError(
+ mdb,
+ err,
+ externalUserPrivilegesSkippableSQLErrorCodes,
logger.WithValues("replica", replica),
) {
replicas = append(replicas, replica)
@@ -455,23 +663,42 @@ func getReplicasToRecover(mdb *mariadbv1alpha1.MariaDB, logger logr.Logger) []st
}
func isRecoverableError(mdb *mariadbv1alpha1.MariaDB, status mariadbv1alpha1.ReplicaStatus,
- recoverableIOErrorCodes []int, logger logr.Logger) bool {
+ recoverableIOErrorCodes []int, recoverableSQLErrorCodes []int, logger logr.Logger) bool {
for _, code := range recoverableIOErrorCodes {
if status.LastIOErrno != nil && *status.LastIOErrno == code {
logger.V(1).Info("Recoverable IO error code detected", "io-errno", *status.LastIOErrno)
+ logger.Info("Recoverable IO error code detected", "io-errno", *status.LastIOErrno)
+ return true
+ }
+ }
+ for _, code := range recoverableSQLErrorCodes {
+ if status.LastSQLErrno != nil && *status.LastSQLErrno == code {
+ logger.V(1).Info("Recoverable SQL error code detected", "sql-errno", *status.LastSQLErrno)
+ logger.Info("Recoverable SQL error code detected", "sql-errno", *status.LastSQLErrno)
return true
}
}
+
+ for _, code := range notRecoverableIOErrorCodes {
+ if status.LastIOErrno != nil && *status.LastIOErrno == code {
+ logger.V(1).Info("Not recoverable IO error code detected", "io-errno", *status.LastIOErrno)
+ logger.Info("Not recoverable IO error code detected", "io-errno", *status.LastIOErrno)
+ return false
+ }
+ }
+
lastIOErrno := ptr.Deref(status.LastIOErrno, 0)
lastSQLErrno := ptr.Deref(status.LastSQLErrno, 0)
if (lastIOErrno != 0 || lastSQLErrno != 0) && !status.LastErrorTransitionTime.IsZero() {
+ logger.Info("Non recoverable error", "lastIOErrno", lastIOErrno,
+ "lastSQLErrno", lastSQLErrno, "LastErrorTransitionTime", status.LastErrorTransitionTime)
replication := ptr.Deref(mdb.Spec.Replication, mariadbv1alpha1.Replication{})
recovery := ptr.Deref(replication.Replica.ReplicaRecovery, mariadbv1alpha1.ReplicaRecovery{})
errThreshold := ptr.Deref(recovery.ErrorDurationThreshold, metav1.Duration{Duration: 5 * time.Minute})
age := time.Since(status.LastErrorTransitionTime.Time)
- logger.V(1).Info(
+ logger.Info(
"Current error",
"io-errno", lastIOErrno,
"sql-errno", lastSQLErrno,
@@ -479,7 +706,7 @@ func isRecoverableError(mdb *mariadbv1alpha1.MariaDB, status mariadbv1alpha1.Rep
"threshold", errThreshold.Duration,
)
if age > errThreshold.Duration {
- logger.V(1).Info(
+ logger.Info(
"Error surpassed threshold",
"io-errno", lastIOErrno,
"sql-errno", lastSQLErrno,
@@ -491,3 +718,45 @@ func isRecoverableError(mdb *mariadbv1alpha1.MariaDB, status mariadbv1alpha1.Rep
}
return false
}
+
+func isSkippableError(mdb *mariadbv1alpha1.MariaDB, status mariadbv1alpha1.ReplicaStatus,
+ externalUserPrivilegesSkippableSQLErrorCodes []int, logger logr.Logger) bool {
+ for _, code := range externalUserPrivilegesSkippableSQLErrorCodes {
+ if status.LastSQLErrno != nil && *status.LastSQLErrno == code {
+ logger.V(1).Info("Skippable SQL error code detected", "sql-errno", *status.LastSQLErrno)
+ return true
+ }
+ }
+ return false
+}
+
+func (r *MariaDBReconciler) reconcileSkippableReplicationError(ctx context.Context, replicasToSkip []string,
+ mdb *mariadbv1alpha1.MariaDB, logger logr.Logger) (ctrl.Result, error) {
+
+ for _, replica := range replicasToSkip {
+ logger.V(1).Info("Skip SQL Replica Error on pod", "pod", replica)
+ podIndex, err := stsobj.PodIndex(replica)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting replica pod index: %v", err)
+ }
+ client, err := sql.NewInternalClientWithPodIndex(ctx, mdb, r.RefResolver, *podIndex)
+ if err != nil {
+ logger.V(1).Info("error getting replica client", "err", err, "pod", *podIndex)
+ return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+ }
+ defer client.Close()
+
+ if err := client.StopSlave(ctx); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error stopping slave: %v", err)
+ }
+
+ if err := client.SetSqlSlaveSkipCounter(ctx, 1); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error setting skip counter: %v", err)
+ }
+
+ if err := client.StartSlave(ctx); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error starting slave: %v", err)
+ }
+ }
+ return ctrl.Result{}, nil
+}
diff --git a/internal/controller/mariadb_controller_replica_recovery_test.go b/internal/controller/mariadb_controller_replica_recovery_test.go
index a970868d33..1a99fee9c6 100644
--- a/internal/controller/mariadb_controller_replica_recovery_test.go
+++ b/internal/controller/mariadb_controller_replica_recovery_test.go
@@ -23,7 +23,7 @@ var _ = Describe("isRecoverableError", func() {
DescribeTable("should evaluate recoverability",
func(buildReplicaStatus func() mariadbv1alpha1.ReplicaStatus, mdb *mariadbv1alpha1.MariaDB, expected bool) {
- res := isRecoverableError(mdb, buildReplicaStatus(), recoverableIOErrorCodes, logger)
+ res := isRecoverableError(mdb, buildReplicaStatus(), recoverableIOErrorCodes, recoverableSQLErrorCodes, logger)
Expect(res).To(Equal(expected))
},
Entry("recoverable IO code matches",
diff --git a/internal/controller/mariadb_controller_replication_test.go b/internal/controller/mariadb_controller_replication_test.go
index 7a34a6030b..1bf83412a0 100644
--- a/internal/controller/mariadb_controller_replication_test.go
+++ b/internal/controller/mariadb_controller_replication_test.go
@@ -1,11 +1,15 @@
package controller
import (
+ "fmt"
+ "strconv"
"time"
volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/metadata"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
+ sqlClient "github.com/mariadb-operator/mariadb-operator/v26/pkg/sql"
stsobj "github.com/mariadb-operator/mariadb-operator/v26/pkg/statefulset"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -14,6 +18,7 @@ import (
policyv1 "k8s.io/api/policy/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
@@ -48,7 +53,7 @@ var _ = Describe("MariaDB replication", Ordered, func() {
return false
}
return mdb.IsReady()
- }, testHighTimeout, testInterval).Should(BeTrue())
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
By("Expecting to create a Service")
var svc corev1.Service
@@ -323,6 +328,7 @@ var _ = Describe("MariaDB replication restore from backup", Ordered, func() {
return mdb.IsReady()
}, testHighTimeout, testInterval).Should(BeTrue())
+
})
DescribeTable(
@@ -626,3 +632,755 @@ var _ = Describe("MariaDB replication with password", Ordered, func() {
executeSqlInPodByIndex(&mdb, 0, "SELECT 1")
})
})
+var _ = Describe("MariaDB replication from external server with filtered tables", Ordered, func() {
+ const (
+ filteredDB = "filtereddb"
+ replicatedTable = "replicated_table"
+ excludedTable = "excluded_table"
+ otherDB = "otherdb"
+ otherTable = "other_table"
+ )
+
+ var (
+ key = testMdbERFilteredKey
+ mdb = &mariadbv1alpha1.MariaDB{}
+ )
+
+ BeforeAll(func() {
+ By("Getting the external MariaDB client")
+ var emdb mariadbv1alpha1.ExternalMariaDB
+ Expect(k8sClient.Get(testCtx, testEMdbkey, &emdb)).To(Succeed())
+ refResolver := refresolver.New(k8sClient)
+ externalClient, err := sqlClient.NewClientWithMariaDB(testCtx, &emdb, refResolver)
+ Expect(err).To(Succeed())
+ defer externalClient.Close()
+
+ By("Creating tables on the external server")
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", filteredDB))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)",
+ filteredDB, replicatedTable))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)",
+ filteredDB, excludedTable))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", otherDB))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)",
+ otherDB, otherTable))).To(Succeed())
+
+ By("Creating PhysicalBackup template for filtered external replication recovery")
+ backupTemplate := mariadbv1alpha1.PhysicalBackup{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: testPbTemplateERFilteredKey.Name,
+ Namespace: testPbTemplateERFilteredKey.Namespace,
+ },
+ Spec: mariadbv1alpha1.PhysicalBackupSpec{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testMdbERFilteredKey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ WaitForIt: false,
+ },
+ Target: ptr.To(mariadbv1alpha1.PhysicalBackupTargetPreferReplica),
+ Schedule: &mariadbv1alpha1.PhysicalBackupSchedule{
+ Suspend: true,
+ },
+ Compression: mariadbv1alpha1.CompressBzip2,
+ Storage: mariadbv1alpha1.PhysicalBackupStorage{
+ PersistentVolumeClaim: &mariadbv1alpha1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceStorage: resource.MustParse("1Gi"),
+ },
+ },
+ AccessModes: []corev1.PersistentVolumeAccessMode{
+ corev1.ReadWriteOnce,
+ },
+ },
+ },
+ Timeout: &metav1.Duration{Duration: 1 * time.Hour},
+ PodAffinity: ptr.To(true),
+ JobContainerTemplate: mariadbv1alpha1.JobContainerTemplate{
+ Resources: &mariadbv1alpha1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("100m"),
+ corev1.ResourceMemory: resource.MustParse("128Mi"),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("300m"),
+ corev1.ResourceMemory: resource.MustParse("512Mi"),
+ },
+ },
+ },
+ },
+ }
+ Expect(k8sClient.Create(testCtx, &backupTemplate)).To(Succeed())
+
+ mdbFiltered := &mariadbv1alpha1.MariaDB{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: key.Name,
+ Namespace: key.Namespace,
+ },
+ Spec: mariadbv1alpha1.MariaDBSpec{
+ Username: &testUser,
+ PasswordSecretKeyRef: &mariadbv1alpha1.GeneratedSecretKeyRef{
+ SecretKeySelector: mariadbv1alpha1.SecretKeySelector{
+ LocalObjectReference: mariadbv1alpha1.LocalObjectReference{
+ Name: testPwdKey.Name,
+ },
+ Key: testPwdSecretKey,
+ },
+ },
+ Database: &testDatabase,
+ MyCnf: ptr.To(`[mariadb]
+ bind-address=*
+ default_storage_engine=InnoDB
+ binlog_format=row
+ innodb_autoinc_lock_mode=2
+ max_allowed_packet=256M`),
+ Replication: &mariadbv1alpha1.Replication{
+ ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+ ReplicaFromExternal: &mariadbv1alpha1.ReplicaFromExternal{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testEMdbkey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ },
+ ServerIdOffset: ptr.To(70),
+ FilteredReplicaTables: []string{
+ fmt.Sprintf("%s.%s", filteredDB, replicatedTable),
+ },
+ },
+ Replica: mariadbv1alpha1.ReplicaReplication{
+ ReplicaBootstrapFrom: &mariadbv1alpha1.ReplicaBootstrapFrom{
+ PhysicalBackupTemplateRef: mariadbv1alpha1.LocalObjectReference{
+ Name: testPbTemplateERFilteredKey.Name,
+ },
+ },
+ IgnoreMaxLagSeconds: ptr.To(true),
+ IgnoreReplicationLivenessProbes: ptr.To(true),
+ },
+ },
+ Enabled: true,
+ },
+ Replicas: 2,
+ Storage: mariadbv1alpha1.Storage{
+ Size: ptr.To(resource.MustParse("300Mi")),
+ StorageClassName: "standard-resize",
+ ResizeInUseVolumes: ptr.To(true),
+ WaitForVolumeResize: ptr.To(true),
+ },
+ TLS: &mariadbv1alpha1.TLS{
+ Enabled: true,
+ Required: ptr.To(true),
+ },
+ Service: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.188",
+ },
+ },
+ },
+ Connection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-ext-filtered-conn"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ PrimaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.189",
+ },
+ },
+ },
+ PrimaryConnection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-ext-filtered-conn-primary"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ SecondaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.194",
+ },
+ },
+ },
+ SecondaryConnection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-ext-filtered-conn-secondary"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ UpdateStrategy: mariadbv1alpha1.UpdateStrategy{
+ Type: mariadbv1alpha1.ReplicasFirstPrimaryLastUpdateType,
+ },
+ },
+ }
+ applyMariadbTestConfig(mdbFiltered)
+ By("Creating MariaDB with filtered external replication")
+ Expect(k8sClient.Create(testCtx, mdbFiltered)).To(Succeed())
+
+ DeferCleanup(func() {
+ var pbTemplate mariadbv1alpha1.PhysicalBackup
+ if err := k8sClient.Get(testCtx, testPbTemplateERFilteredKey, &pbTemplate); err == nil {
+ Expect(k8sClient.Delete(testCtx, &pbTemplate)).To(Succeed())
+ }
+ var pbRecoveryPvc corev1.PersistentVolumeClaim
+ if err := k8sClient.Get(testCtx, testMdbPbRecoveryERFilteredKey, &pbRecoveryPvc); err == nil {
+ Expect(k8sClient.Delete(testCtx, &pbRecoveryPvc)).To(Succeed())
+ }
+ deleteMariadb(key, false)
+ })
+ })
+
+ It("should reconcile", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+ })
+
+ It("should only have the filtered table after initial restore", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ refResolver := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i)
+ By("Expecting to get SQL client for Pod " + strconv.Itoa(i))
+ if err != nil {
+ fmt.Fprintf(GinkgoWriter, "Not able get SQL for POD: %v \n", err)
+ }
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ By(fmt.Sprintf("Expecting Pod %d to have the replicated table", i))
+ exists, err := podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ filteredDB, replicatedTable,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeTrue())
+
+ By(fmt.Sprintf("Expecting Pod %d to NOT have the excluded table from the same database", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ filteredDB, excludedTable,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeFalse())
+
+ By(fmt.Sprintf("Expecting Pod %d to NOT have the other database table", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ otherDB, otherTable,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeFalse())
+ }
+ })
+
+ It("should replicate only changes to the filtered table", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ By("Getting the external MariaDB client")
+ var emdb mariadbv1alpha1.ExternalMariaDB
+ Expect(k8sClient.Get(testCtx, testEMdbkey, &emdb)).To(Succeed())
+ refResolver := refresolver.New(k8sClient)
+ externalClient, err := sqlClient.NewClientWithMariaDB(testCtx, &emdb, refResolver)
+ Expect(err).To(Succeed())
+ defer externalClient.Close()
+
+ By("Inserting a row into the replicated table on the external server")
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "INSERT IGNORE INTO `%s`.`%s` VALUES (42)", filteredDB, replicatedTable,
+ ))).To(Succeed())
+
+ By("Expecting the inserted row to appear on all replicas")
+ refResolver2 := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, pErr := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver2, i)
+ Expect(pErr).To(Succeed())
+ defer podClient.Close()
+
+ Eventually(func() bool {
+ exists, eErr := podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM `%s`.`%s` WHERE id = 42", filteredDB, replicatedTable,
+ ))
+ return eErr == nil && exists
+ }, testTimeout, testInterval).Should(BeTrue(),
+ fmt.Sprintf("Pod %d should have the replicated row", i))
+ }
+ })
+
+ It("should have GTID strict mode disabled", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ refResolver := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i)
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ By(fmt.Sprintf("Expecting GTID strict mode to be disabled on Pod %d", i))
+ val, err := podClient.SystemVariable(testCtx, "gtid_strict_mode")
+ Expect(err).To(Succeed())
+ fmt.Fprintf(GinkgoWriter, "gtid_strict_mode: %v \n", val)
+ Expect(val).To(Equal("0"))
+ }
+ })
+})
+
+var _ = Describe("MariaDB replication from external server with filtered tables from multiple schemas", Ordered, func() {
+ const (
+ schema1 = "multischema1db"
+ schema2 = "multischema2db"
+ otherSchema = "otherschemadb"
+ replicatedInSchema1 = "replicated_in_schema1"
+ replicatedInSchema2 = "replicated_in_schema2"
+ excludedInSchema1 = "excluded_in_schema1"
+ excludedInSchema2 = "excluded_in_schema2"
+ otherSchemaTable = "other_table"
+ viewOnExcluded1 = "view_on_excluded_schema1"
+ viewOnExcluded2 = "view on excluded schema2"
+ )
+
+ var (
+ key = testMdbERMultiSchemaKey
+ mdb = &mariadbv1alpha1.MariaDB{}
+ )
+
+ BeforeAll(func() {
+ By("Getting the external MariaDB client")
+ var emdb mariadbv1alpha1.ExternalMariaDB
+ Expect(k8sClient.Get(testCtx, testEMdbkey, &emdb)).To(Succeed())
+ refResolver := refresolver.New(k8sClient)
+ externalClient, err := sqlClient.NewClientWithMariaDB(testCtx, &emdb, refResolver)
+ Expect(err).To(Succeed())
+ defer externalClient.Close()
+
+ By("Creating tables on the external server across two schemas")
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", schema1))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)", schema1, replicatedInSchema1,
+ ))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)", schema1, excludedInSchema1,
+ ))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", schema2))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)", schema2, replicatedInSchema2,
+ ))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)", schema2, excludedInSchema2,
+ ))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", otherSchema))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "CREATE TABLE IF NOT EXISTS `%s`.`%s` (id INT PRIMARY KEY)", otherSchema, otherSchemaTable,
+ ))).To(Succeed())
+
+ By("Creating views that reference excluded tables (must be ignored by the dump)")
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "CREATE OR REPLACE VIEW `%s`.`%s` AS SELECT * FROM `%s`.`%s`",
+ schema1, viewOnExcluded1, schema1, excludedInSchema1,
+ ))).To(Succeed())
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "CREATE OR REPLACE VIEW `%s`.`%s` AS SELECT * FROM `%s`.`%s`",
+ schema2, viewOnExcluded2, schema2, excludedInSchema2,
+ ))).To(Succeed())
+
+ By("Creating PhysicalBackup template for multi-schema filtered external replication recovery")
+ backupTemplate := mariadbv1alpha1.PhysicalBackup{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: testPbTemplateERMultiSchemaKey.Name,
+ Namespace: testPbTemplateERMultiSchemaKey.Namespace,
+ },
+ Spec: mariadbv1alpha1.PhysicalBackupSpec{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testMdbERMultiSchemaKey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ WaitForIt: false,
+ },
+ Target: ptr.To(mariadbv1alpha1.PhysicalBackupTargetPreferReplica),
+ Schedule: &mariadbv1alpha1.PhysicalBackupSchedule{
+ Suspend: true,
+ },
+ Compression: mariadbv1alpha1.CompressBzip2,
+ Storage: mariadbv1alpha1.PhysicalBackupStorage{
+ PersistentVolumeClaim: &mariadbv1alpha1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceStorage: resource.MustParse("1Gi"),
+ },
+ },
+ AccessModes: []corev1.PersistentVolumeAccessMode{
+ corev1.ReadWriteOnce,
+ },
+ },
+ },
+ Timeout: &metav1.Duration{Duration: 1 * time.Hour},
+ PodAffinity: ptr.To(true),
+ JobContainerTemplate: mariadbv1alpha1.JobContainerTemplate{
+ Resources: &mariadbv1alpha1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("100m"),
+ corev1.ResourceMemory: resource.MustParse("128Mi"),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("300m"),
+ corev1.ResourceMemory: resource.MustParse("512Mi"),
+ },
+ },
+ },
+ },
+ }
+ Expect(k8sClient.Create(testCtx, &backupTemplate)).To(Succeed())
+
+ mdbMultiSchema := &mariadbv1alpha1.MariaDB{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: key.Name,
+ Namespace: key.Namespace,
+ },
+ Spec: mariadbv1alpha1.MariaDBSpec{
+ Username: &testUser,
+ PasswordSecretKeyRef: &mariadbv1alpha1.GeneratedSecretKeyRef{
+ SecretKeySelector: mariadbv1alpha1.SecretKeySelector{
+ LocalObjectReference: mariadbv1alpha1.LocalObjectReference{
+ Name: testPwdKey.Name,
+ },
+ Key: testPwdSecretKey,
+ },
+ },
+ Database: &testDatabase,
+ MyCnf: ptr.To(`[mariadb]
+ bind-address=*
+ default_storage_engine=InnoDB
+ binlog_format=row
+ innodb_autoinc_lock_mode=2
+ max_allowed_packet=256M`),
+ Replication: &mariadbv1alpha1.Replication{
+ ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+ ReplicaFromExternal: &mariadbv1alpha1.ReplicaFromExternal{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testEMdbkey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ },
+ ServerIdOffset: ptr.To(80),
+ FilteredReplicaTables: []string{
+ fmt.Sprintf("%s.%s", schema1, replicatedInSchema1),
+ fmt.Sprintf("%s.%s", schema2, replicatedInSchema2),
+ },
+ },
+ Replica: mariadbv1alpha1.ReplicaReplication{
+ ReplicaBootstrapFrom: &mariadbv1alpha1.ReplicaBootstrapFrom{
+ PhysicalBackupTemplateRef: mariadbv1alpha1.LocalObjectReference{
+ Name: testPbTemplateERMultiSchemaKey.Name,
+ },
+ },
+ IgnoreMaxLagSeconds: ptr.To(true),
+ IgnoreReplicationLivenessProbes: ptr.To(true),
+ },
+ },
+ Enabled: true,
+ },
+ Replicas: 2,
+ Storage: mariadbv1alpha1.Storage{
+ Size: ptr.To(resource.MustParse("300Mi")),
+ StorageClassName: "standard-resize",
+ ResizeInUseVolumes: ptr.To(true),
+ WaitForVolumeResize: ptr.To(true),
+ },
+ TLS: &mariadbv1alpha1.TLS{
+ Enabled: true,
+ Required: ptr.To(true),
+ },
+ Service: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.195",
+ },
+ },
+ },
+ Connection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-ext-multi-schema-conn"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ PrimaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.196",
+ },
+ },
+ },
+ PrimaryConnection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-ext-multi-schema-conn-primary"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ SecondaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.197",
+ },
+ },
+ },
+ SecondaryConnection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-ext-multi-schema-conn-secondary"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ UpdateStrategy: mariadbv1alpha1.UpdateStrategy{
+ Type: mariadbv1alpha1.ReplicasFirstPrimaryLastUpdateType,
+ },
+ },
+ }
+ applyMariadbTestConfig(mdbMultiSchema)
+ By("Creating MariaDB with multi-schema filtered external replication")
+ Expect(k8sClient.Create(testCtx, mdbMultiSchema)).To(Succeed())
+
+ DeferCleanup(func() {
+ var pbTemplate mariadbv1alpha1.PhysicalBackup
+ if err := k8sClient.Get(testCtx, testPbTemplateERMultiSchemaKey, &pbTemplate); err == nil {
+ Expect(k8sClient.Delete(testCtx, &pbTemplate)).To(Succeed())
+ }
+ var pbRecoveryPvc corev1.PersistentVolumeClaim
+ if err := k8sClient.Get(testCtx, testMdbPbRecoveryERMultiSchemaKey, &pbRecoveryPvc); err == nil {
+ Expect(k8sClient.Delete(testCtx, &pbRecoveryPvc)).To(Succeed())
+ }
+ deleteMariadb(key, false)
+ })
+ })
+
+ It("should reconcile", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+ })
+
+ It("should have only the filtered tables from each schema after initial restore", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ refResolver := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i)
+ By("Expecting to get SQL client for Pod " + strconv.Itoa(i))
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ By(fmt.Sprintf("Expecting Pod %d to have the replicated table from schema1", i))
+ exists, err := podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ schema1, replicatedInSchema1,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeTrue())
+
+ By(fmt.Sprintf("Expecting Pod %d to have the replicated table from schema2", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ schema2, replicatedInSchema2,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeTrue())
+
+ By(fmt.Sprintf("Expecting Pod %d to NOT have the excluded table from schema1", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ schema1, excludedInSchema1,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeFalse())
+
+ By(fmt.Sprintf("Expecting Pod %d to NOT have the excluded table from schema2", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ schema2, excludedInSchema2,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeFalse())
+
+ By(fmt.Sprintf("Expecting Pod %d to NOT have any table from the excluded schema", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ otherSchema, otherSchemaTable,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeFalse())
+
+ By(fmt.Sprintf("Expecting Pod %d to NOT have the view referencing the excluded schema1 table", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.views WHERE table_schema='%s' AND table_name='%s'",
+ schema1, viewOnExcluded1,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeFalse())
+
+ By(fmt.Sprintf("Expecting Pod %d to NOT have the view referencing the excluded schema2 table", i))
+ exists, err = podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.views WHERE table_schema='%s' AND table_name='%s'",
+ schema2, viewOnExcluded2,
+ ))
+ Expect(err).To(Succeed())
+ Expect(exists).To(BeFalse())
+ }
+ })
+
+ It("should replicate changes to the filtered table in each schema", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ By("Getting the external MariaDB client")
+ var emdb mariadbv1alpha1.ExternalMariaDB
+ Expect(k8sClient.Get(testCtx, testEMdbkey, &emdb)).To(Succeed())
+ refResolver := refresolver.New(k8sClient)
+ externalClient, err := sqlClient.NewClientWithMariaDB(testCtx, &emdb, refResolver)
+ Expect(err).To(Succeed())
+ defer externalClient.Close()
+
+ By("Inserting a row into the replicated table in schema1 on the external server")
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "INSERT IGNORE INTO `%s`.`%s` VALUES (1)", schema1, replicatedInSchema1,
+ ))).To(Succeed())
+
+ By("Inserting a row into the replicated table in schema2 on the external server")
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "INSERT IGNORE INTO `%s`.`%s` VALUES (2)", schema2, replicatedInSchema2,
+ ))).To(Succeed())
+
+ By("Expecting inserted rows from both schemas to appear on all replicas")
+ refResolver2 := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, pErr := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver2, i)
+ Expect(pErr).To(Succeed())
+ defer podClient.Close()
+
+ Eventually(func() bool {
+ exists, eErr := podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM `%s`.`%s` WHERE id = 1", schema1, replicatedInSchema1,
+ ))
+ return eErr == nil && exists
+ }, testTimeout, testInterval).Should(BeTrue(),
+ fmt.Sprintf("Pod %d should have the schema1 replicated row", i))
+
+ Eventually(func() bool {
+ exists, eErr := podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM `%s`.`%s` WHERE id = 2", schema2, replicatedInSchema2,
+ ))
+ return eErr == nil && exists
+ }, testTimeout, testInterval).Should(BeTrue(),
+ fmt.Sprintf("Pod %d should have the schema2 replicated row", i))
+ }
+
+ By("Expecting changes to the excluded table in schema1 NOT to be replicated")
+ Expect(externalClient.Exec(testCtx, fmt.Sprintf(
+ "INSERT IGNORE INTO `%s`.`%s` VALUES (99)", schema1, excludedInSchema1,
+ ))).To(Succeed())
+
+ refResolver3 := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, pErr := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver3, i)
+ Expect(pErr).To(Succeed())
+ defer podClient.Close()
+
+ Consistently(func() bool {
+ exists, eErr := podClient.Exists(testCtx, fmt.Sprintf(
+ "SELECT 1 FROM information_schema.tables WHERE table_schema='%s' AND table_name='%s'",
+ schema1, excludedInSchema1,
+ ))
+ return eErr == nil && !exists
+ }, 10*time.Second, testInterval).Should(BeTrue(),
+ fmt.Sprintf("Pod %d should never receive the excluded schema1 table", i))
+ }
+ })
+
+ It("should have GTID strict mode disabled", func() {
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, key, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testVeryHighTimeout, testInterval).Should(BeTrue())
+
+ refResolver := refresolver.New(k8sClient)
+ for i := 0; i < int(mdb.Spec.Replicas); i++ {
+ podClient, err := sqlClient.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i)
+ Expect(err).To(Succeed())
+ defer podClient.Close()
+
+ By(fmt.Sprintf("Expecting GTID strict mode to be disabled on Pod %d", i))
+ val, err := podClient.SystemVariable(testCtx, "gtid_strict_mode")
+ Expect(err).To(Succeed())
+ fmt.Fprintf(GinkgoWriter, "gtid_strict_mode: %v \n", val)
+ Expect(val).To(Equal("0"))
+ }
+ })
+})
diff --git a/internal/controller/mariadb_controller_scale_out.go b/internal/controller/mariadb_controller_scale_out.go
index 9f338826f7..7b3751e65d 100644
--- a/internal/controller/mariadb_controller_scale_out.go
+++ b/internal/controller/mariadb_controller_scale_out.go
@@ -5,12 +5,16 @@ import (
"errors"
"fmt"
"sort"
+ "strconv"
"time"
"github.com/go-logr/logr"
mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/builder"
condition "github.com/mariadb-operator/mariadb-operator/v26/pkg/condition"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/job"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/sql"
mdbsnapshot "github.com/mariadb-operator/mariadb-operator/v26/pkg/volumesnapshot"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
@@ -176,6 +180,7 @@ func (r *MariaDBReconciler) reconcileReplicaPhysicalBackup(ctx context.Context,
logger logr.Logger) (ctrl.Result, error) {
var physicalBackup mariadbv1alpha1.PhysicalBackup
if err := r.Get(ctx, key, &physicalBackup); err != nil {
+ logger.Info("Current PhysicalBackup err!=nil", "err", err)
if apierrors.IsNotFound(err) {
logger.Info("Creating PhysicalBackup", "name", key.Name)
if err := r.createReplicaPhysicalBackup(ctx, key, mariadb); err != nil {
@@ -184,13 +189,79 @@ func (r *MariaDBReconciler) reconcileReplicaPhysicalBackup(ctx context.Context,
}
return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
}
+
+ // If backup is already present but expired (backup age > master binlog_retention) we need to destroy it to force a new backup
+ var binlogExpireLogsDuration time.Duration
+ var binlogExpireErr error
+ var ageThreshold *time.Time
+ replication := mariadb.Replication()
+
+ if replication.IsExternalReplication() {
+ emdb, err := r.RefResolver.ExternalMariaDB(ctx, &replication.ReplicaFromExternal.MariaDBRef.ObjectReference, mariadb.Namespace)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error getting external MariaDB: %v", err)
+ }
+ logger.Info("Getting the binlog_expire_logs_seconds on the external MariaDB")
+ binlogExpireLogsDuration, binlogExpireErr = getBinlogExpireLogsDuration(emdb, ctx, r.RefResolver, logger)
+ } else {
+ logger.Info("Getting the binlog_expire_logs_seconds on primary MariaDB")
+ binlogExpireLogsDuration, binlogExpireErr = getInternalBinlogExpireLogsDuration(mariadb, ctx, r.RefResolver)
+ }
+
+ if binlogExpireErr == nil && binlogExpireLogsDuration != 0 {
+ ageThreshold = ptr.To(time.Now().Add(-binlogExpireLogsDuration))
+ } else if binlogExpireErr != nil {
+ // In case of failure to get the binlogExpireLogsDuration set ageThreshold do now to force a new backup
+ logger.Info("Unable to get binlog_expire_logs_seconds, setting age threshold to now to force new backup", "error", binlogExpireErr)
+ ageThreshold = ptr.To(time.Now())
+ }
+
if !physicalBackup.IsComplete() {
+ if replication.IsExternalReplication() {
+ backupStartTime := physicalBackup.CreationTimestamp.Time
+ jobWaitTimeout, _ := time.ParseDuration("240s")
+ jobs, err := job.ListJobs(ctx, r.Client, &physicalBackup)
+ if (err != nil || len(jobs.Items) == 0) && time.Since(backupStartTime) > jobWaitTimeout {
+ logger.Info("ExternalReplication, physical backup job launch wait timeout. Trigger logical backup")
+ return ctrl.Result{RequeueAfter: 1 * time.Second}, errPhysicalBackupJobLaunchTimeout
+ }
+ }
logger.V(1).Info("Replica PhysicalBackup job not completed. Requeuing")
return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
+ } else {
+ if ageThreshold != nil && physicalBackup.Status.LastScheduleTime.Time.Before(*ageThreshold) {
+ logger.Info("Existent backup is expired, destroying to create a new one")
+ if err := r.cleanupPhysicalBackup(ctx, mariadb.PhysicalBackupReplicaRecoveryKey()); err != nil {
+ return ctrl.Result{}, err
+ }
+ return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
+ }
}
+ logger.V(1).Info("Replica PhysicalBackup completed.")
return ctrl.Result{}, nil
}
+func getInternalBinlogExpireLogsDuration(mdb *mariadbv1alpha1.MariaDB, ctx context.Context,
+ refResolver *refresolver.RefResolver) (time.Duration, error) {
+ var external_client *sql.Client
+ var err error
+ if external_client, err = sql.NewClientWithMariaDB(ctx, mdb, refResolver); err != nil {
+ return time.Duration(0), fmt.Errorf("error getting external MariaDB client: %v", err)
+ }
+ defer external_client.Close()
+
+ var binlogExpireLogsSecondsStr string
+ var binlogExpireLogsSeconds int
+
+ binlogExpireLogsSecondsStr, err = external_client.SystemVariable(ctx, "binlog_expire_logs_seconds")
+ if err != nil {
+ return time.Duration(0), fmt.Errorf("unable to get binlog_expire_logs_seconds: %v", err)
+ }
+ binlogExpireLogsSeconds, _ = strconv.Atoi(binlogExpireLogsSecondsStr)
+
+ return time.Duration(binlogExpireLogsSeconds) * time.Second, nil
+}
+
func (r *MariaDBReconciler) createReplicaPhysicalBackup(ctx context.Context, key types.NamespacedName,
mariadb *mariadbv1alpha1.MariaDB) error {
replication := ptr.Deref(mariadb.Spec.Replication, mariadbv1alpha1.Replication{})
@@ -243,12 +314,16 @@ func (r *MariaDBReconciler) getVolumeSnapshotKey(ctx context.Context, mariadb *m
func (r *MariaDBReconciler) setScaledOutAndCleanup(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB,
logger logr.Logger) (ctrl.Result, error) {
+ logger.Info("Scale out and cleanup")
if !mariadb.IsScalingOut() {
+ logger.Info("Not scaling out")
return ctrl.Result{}, nil
}
physicalBackupKey := mariadb.PhysicalBackupScaleOutKey()
+ logger.Info("physical backup", "key", physicalBackupKey)
if mariadb.Status.ScaleOutInitialIndex != nil {
+ logger.Info("Scale out initial index", "index", *mariadb.Status.ScaleOutInitialIndex)
fromIndex := *mariadb.Status.ScaleOutInitialIndex
physicalBackup, err := r.getPhysicalBackup(ctx, physicalBackupKey, mariadb)
diff --git a/internal/controller/mariadb_controller_status.go b/internal/controller/mariadb_controller_status.go
index 3b7fc6e359..3ce3dc2679 100644
--- a/internal/controller/mariadb_controller_status.go
+++ b/internal/controller/mariadb_controller_status.go
@@ -4,6 +4,8 @@ import (
"context"
"errors"
"fmt"
+ "slices"
+ "time"
"github.com/go-logr/logr"
"github.com/hashicorp/go-multierror"
@@ -31,6 +33,14 @@ func (r *MariaDBReconciler) reconcileStatus(ctx context.Context, mdb *mariadbv1a
}
logger := log.FromContext(ctx).WithName("status").V(1)
+ logger.Info("Reconciling MariaDB Status", "MariaDB:", mdb.Name)
+
+ // Discover and persist the external replication server_id offset before the StatefulSet is built,
+ // so pods are created with a non-colliding server_id and never need a restart to adopt it.
+ if result, err := r.reconcileExternalReplServerId(ctx, mdb); !result.IsZero() || err != nil {
+ return result, err
+ }
+
var sts appsv1.StatefulSet
if err := r.Get(ctx, client.ObjectKeyFromObject(mdb), &sts); err != nil {
logger.Info("error getting StatefulSet", "err", err)
@@ -55,7 +65,15 @@ func (r *MariaDBReconciler) reconcileStatus(ctx context.Context, mdb *mariadbv1a
logger.Info("error getting TLS status", "err", err)
}
- return ctrl.Result{}, r.patchStatus(ctx, mdb, func(status *mariadbv1alpha1.MariaDBStatus) error {
+ return ctrl.Result{}, r.patchStatus(ctx, mdb,
+ r.statusPatcher(ctx, mdb, &sts, replRoles, replStatus, tlsStatus, mxsPrimaryPodIndex, mxsErr))
+}
+
+// statusPatcher builds the patcher that reconciles the MariaDB status from the state gathered by reconcileStatus.
+func (r *MariaDBReconciler) statusPatcher(ctx context.Context, mdb *mariadbv1alpha1.MariaDB, sts *appsv1.StatefulSet,
+ replRoles map[string]mariadbv1alpha1.ReplicationRole, replStatus map[string]mariadbv1alpha1.ReplicaStatus,
+ tlsStatus *mariadbv1alpha1.MariaDBTLSStatus, mxsPrimaryPodIndex *int, mxsErr error) func(*mariadbv1alpha1.MariaDBStatus) error {
+ return func(status *mariadbv1alpha1.MariaDBStatus) error {
status.DefaultVersion = r.Environment.MariadbDefaultVersion
status.Replicas = sts.Status.ReadyReplicas
defaultPrimary(mdb)
@@ -93,9 +111,101 @@ func (r *MariaDBReconciler) reconcileStatus(ctx context.Context, mdb *mariadbv1a
if err := r.setUpdatedCondition(ctx, mdb); err != nil {
log.FromContext(ctx).V(1).Info("error setting MariaDB updated condition", "err", err)
}
- condition.SetReadyWithMariaDB(&mdb.Status, &sts, mdb)
+ condition.SetReadyWithMariaDB(&mdb.Status, sts, mdb)
return nil
- })
+ }
+}
+
+// externalReplServerIdGap is the room left between the highest server_id already in use on the
+// external MariaDB and the offset assigned to this cluster. It gives headroom for this cluster to
+// scale out and for other clusters replicating from the same source to claim their own blocks.
+const externalReplServerIdGap = 100
+
+// reconcileExternalReplServerId auto-discovers a non-colliding server_id offset for external
+// replication and persists it to status. It is computed only once: when a manual serverIdOffset is
+// set, or once the offset is already persisted, it is a no-op. While the external MariaDB is not
+// reachable it requeues, which short-circuits the reconcile loop and prevents the StatefulSet from
+// being created before the offset is known (avoiding a rolling restart).
+func (r *MariaDBReconciler) reconcileExternalReplServerId(ctx context.Context,
+ mdb *mariadbv1alpha1.MariaDB) (ctrl.Result, error) {
+ if !mdb.IsReplicationEnabled() {
+ return ctrl.Result{}, nil
+ }
+ replication := mdb.Replication()
+ if !replication.IsExternalReplication() {
+ return ctrl.Result{}, nil
+ }
+ // Manual offset takes precedence and is left untouched.
+ if replication.ReplicaFromExternal.ServerIdOffset != nil {
+ return ctrl.Result{}, nil
+ }
+ // Compute-once: never re-query once persisted.
+ if mdb.Status.ExternalReplication != nil && mdb.Status.ExternalReplication.ServerIdOffset != nil {
+ return ctrl.Result{}, nil
+ }
+ logger := log.FromContext(ctx).WithName("external-repl-server-id")
+
+ emdb, err := r.RefResolver.ExternalMariaDB(ctx, &replication.ReplicaFromExternal.MariaDBRef.ObjectReference, mdb.Namespace)
+ if err != nil {
+ logger.Info("error getting external MariaDB, requeuing", "err", err)
+ return ctrl.Result{RequeueAfter: time.Minute}, nil
+ }
+ if !emdb.IsReady() {
+ logger.Info("external MariaDB is not ready, requeuing")
+ return ctrl.Result{RequeueAfter: time.Minute}, nil
+ }
+
+ // The external MariaDB endpoint (e.g. a VIP fronting a MaxScale readconnroute) may resolve to
+ // either the primary or a replica. Server ids must be enumerated from the primary, as only it sees
+ // every replica registered in the topology (SHOW SLAVE HOSTS). If we land on a replica, follow
+ // SHOW REPLICA STATUS to the primary and connect there directly, reusing the same credentials/TLS.
+ client, err := sql.NewClientWithMariaDB(ctx, emdb, r.RefResolver)
+ if err != nil {
+ logger.Info("error connecting to external MariaDB, requeuing", "err", err)
+ return ctrl.Result{RequeueAfter: time.Minute}, nil
+ }
+ defer client.Close()
+
+ masterHost, masterPort, isReplica, err := client.ReplicationMasterEndpoint(ctx)
+ if err != nil {
+ logger.Info("error resolving external primary, requeuing", "err", err)
+ return ctrl.Result{RequeueAfter: time.Minute}, nil
+ }
+
+ primaryClient := client
+ if isReplica {
+ logger.Info("external endpoint is a replica, connecting to its primary",
+ "master-host", masterHost, "master-port", masterPort)
+ masterClient, err := sql.NewClientWithMariaDB(ctx, emdb, r.RefResolver, sql.WithHost(masterHost), sql.WithPort(masterPort))
+ if err != nil {
+ logger.Info("error connecting to external primary, requeuing", "err", err)
+ return ctrl.Result{RequeueAfter: time.Minute}, nil
+ }
+ defer masterClient.Close()
+ primaryClient = masterClient
+ }
+
+ ids, err := primaryClient.InUseServerIds(ctx)
+ if err != nil {
+ logger.Info("error getting in-use server ids, requeuing", "err", err)
+ return ctrl.Result{RequeueAfter: time.Minute}, nil
+ }
+
+ offset := externalReplServerIdGap
+ if len(ids) > 0 {
+ offset = slices.Max(ids) + externalReplServerIdGap
+ }
+ logger.Info("discovered external replication server_id offset", "offset", offset, "in-use-server-ids", ids)
+
+ if err := r.patchStatus(ctx, mdb, func(status *mariadbv1alpha1.MariaDBStatus) error {
+ status.ExternalReplication = &mariadbv1alpha1.ExternalReplicationStatus{
+ ServerIdOffset: &offset,
+ }
+ return nil
+ }); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error patching MariaDB status: %v", err)
+ }
+ return ctrl.Result{}, nil
}
func shouldReconcileReplicationRoleForPod(mdb *mariadbv1alpha1.MariaDB, podIndex int) bool {
@@ -105,11 +215,17 @@ func shouldReconcileReplicationRoleForPod(mdb *mariadbv1alpha1.MariaDB, podIndex
func (r *MariaDBReconciler) getReplicationRoles(ctx context.Context,
mdb *mariadbv1alpha1.MariaDB) (map[string]mariadbv1alpha1.ReplicationRole, error) {
+ logger := log.FromContext(ctx)
+ logger.V(1).Info("Getting Replication Roles")
+ if !mdb.IsReplicationEnabled() && (mdb.Spec.MultiCluster == nil || !mdb.Spec.MultiCluster.Enabled) {
+ return nil, nil
+ }
+
clientSet := sql.NewClientSet(mdb, r.RefResolver)
defer clientSet.Close()
var replState map[string]mariadbv1alpha1.ReplicationRole
- logger := log.FromContext(ctx)
+
for i := 0; i < int(mdb.Spec.Replicas); i++ {
if !shouldReconcileReplicationRoleForPod(mdb, i) {
continue
@@ -192,6 +308,8 @@ func (r *MariaDBReconciler) isMultiClusterPrimaryReplica(ctx context.Context, md
func shouldReconcileReplicaStatusForPod(mdb *mariadbv1alpha1.MariaDB, podIndex int) bool {
isReplica := podIndex != *mdb.Status.CurrentPrimaryPodIndex
+ replication := mdb.Replication()
+ isExternalReplication := replication.IsExternalReplication()
if mdb.IsMultiClusterEnabled() {
if mdb.IsReplicationEnabled() {
return isReplica || mdb.IsMultiClusterPrimaryReplica(podIndex)
@@ -200,7 +318,7 @@ func shouldReconcileReplicaStatusForPod(mdb *mariadbv1alpha1.MariaDB, podIndex i
}
return false
}
- return isReplica
+ return isReplica || isExternalReplication
}
func (r *MariaDBReconciler) getReplicaStatus(ctx context.Context,
@@ -215,10 +333,15 @@ func (r *MariaDBReconciler) getReplicaStatus(ctx context.Context,
defer clientSet.Close()
var replicaStatus map[string]mariadbv1alpha1.ReplicaStatus
+ replication := mdb.Replication()
for i := 0; i < int(mdb.Spec.Replicas); i++ {
if !shouldReconcileReplicaStatusForPod(mdb, i) {
continue
}
+
+ if i == *mdb.Status.CurrentPrimaryPodIndex && !replication.IsExternalReplication() {
+ continue
+ }
pod := stspkg.PodName(mdb.ObjectMeta, i)
var currentReplicaStatus *mariadbv1alpha1.ReplicaStatus
@@ -244,7 +367,7 @@ func (r *MariaDBReconciler) getReplicaStatus(ctx context.Context,
var replOpts []sql.ReplicationOpt
if mdb.IsMultiClusterPrimaryReplica(i) {
- replOpts = append(replOpts, sql.WithConnectionName(replication.MultiClusterReplicaConnectionName))
+ replOpts = append(replOpts, sql.WithConnectionName(*replication.MultiClusterReplicaConnectionName))
}
newReplicaStatus, err := client.ReplicaStatus(ctx, logger, replOpts...)
if err != nil {
diff --git a/internal/controller/mariadb_controller_update.go b/internal/controller/mariadb_controller_update.go
index 878d41c628..029241f49a 100644
--- a/internal/controller/mariadb_controller_update.go
+++ b/internal/controller/mariadb_controller_update.go
@@ -400,7 +400,8 @@ func (r *MariaDBReconciler) triggerMaxScaleSwitchover(ctx context.Context, maria
}
func shouldTriggerSwitchover(mariadb *mariadbv1alpha1.MariaDB) bool {
- if mariadb.IsRestoringBackup() {
+ replication := mariadb.Replication()
+ if mariadb.IsMaxScaleEnabled() || mariadb.IsRestoringBackup() || replication.IsExternalReplication() {
return false
}
return mariadb.IsReplicationEnabled() && mariadb.HasConfiguredReplica()
diff --git a/internal/controller/physicalbackup_controller.go b/internal/controller/physicalbackup_controller.go
index 8068ee30b0..11c2c2a650 100644
--- a/internal/controller/physicalbackup_controller.go
+++ b/internal/controller/physicalbackup_controller.go
@@ -38,6 +38,7 @@ import (
)
var errPhysicalBackupNoTargetPodsAvailable = errors.New("no target Pods available")
+var errPhysicalBackupJobLaunchTimeout = errors.New("BackupJobLaunchTimeout")
// PhysicalBackupReconciler reconciles a PhysicalBackup object
type PhysicalBackupReconciler struct {
@@ -437,6 +438,11 @@ func physicalBackupTargetWithFuncs(ctx context.Context, backup *mariadbv1alpha1.
if err != nil && !errors.Is(err, errPhysicalBackupNoTargetPodsAvailable) {
return nil, fmt.Errorf("error getting replica target: %v", err)
}
+ replication := mariadb.Replication()
+ if replication.IsExternalReplication() && errors.Is(err, errPhysicalBackupNoTargetPodsAvailable) {
+ return nil, err
+ }
+
if podIndex != nil {
return podIndex, nil
}
diff --git a/internal/controller/physicalbackup_controller_job.go b/internal/controller/physicalbackup_controller_job.go
index f859b80279..81a9d94eeb 100644
--- a/internal/controller/physicalbackup_controller_job.go
+++ b/internal/controller/physicalbackup_controller_job.go
@@ -331,11 +331,13 @@ func (r *PhysicalBackupReconciler) reconcileStorage(ctx context.Context, backup
func (r *PhysicalBackupReconciler) createJob(ctx context.Context, backup *mariadbv1alpha1.PhysicalBackup, mariadb *mariadbv1alpha1.MariaDB,
now time.Time, schedule cron.Schedule, logger logr.Logger) (ctrl.Result, error) {
+
podIndex, err := r.physicalBackupTarget(ctx, backup, mariadb, logger)
+
if err != nil {
if errors.Is(err, errPhysicalBackupNoTargetPodsAvailable) {
logger.Info("No target Pods available. Requeuing...", "target", backup.Spec.Target)
- return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+ return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
return ctrl.Result{}, fmt.Errorf("error getting target Pod index: %v", err)
}
diff --git a/internal/controller/pod_replication_controller.go b/internal/controller/pod_replication_controller.go
index 6faa849cac..0d3dec4289 100644
--- a/internal/controller/pod_replication_controller.go
+++ b/internal/controller/pod_replication_controller.go
@@ -75,6 +75,7 @@ func (r *PodReplicationController) ReconcilePodNotReady(ctx context.Context, pod
if err != nil {
return fmt.Errorf("error getting Pod index: %v", err)
}
+
if *index != *mariadb.Status.CurrentPrimaryPodIndex {
return nil
}
@@ -140,7 +141,8 @@ func (r *PodReplicationController) ReconcilePodNotReady(ctx context.Context, pod
func shouldReconcile(mdb *mariadbv1alpha1.MariaDB) bool {
if mdb.IsMaxScaleEnabled() || mdb.IsSwitchingPrimary() || mdb.IsReplicationSwitchoverRequired() ||
- mdb.IsRestoringBackup() || mdb.IsResizingStorage() || mdb.IsSuspended() {
+ mdb.IsRestoringBackup() || mdb.IsResizingStorage() || mdb.IsSuspended() ||
+ mdb.Replication().ReplicaFromExternal != nil {
return false
}
primaryRepl := ptr.Deref(mdb.Spec.Replication, mariadbv1alpha1.Replication{}).Primary
diff --git a/internal/controller/sqljob_controller_test.go b/internal/controller/sqljob_controller_test.go
index 6f77dc6ed3..1a60c8c342 100644
--- a/internal/controller/sqljob_controller_test.go
+++ b/internal/controller/sqljob_controller_test.go
@@ -465,6 +465,12 @@ var _ = Describe("SqlJob on External MariaDB", func() {
})
By("Expecting SqlJob to report unready status")
+ Eventually(func(g Gomega) bool {
+ var updatedSqlJob mariadbv1alpha1.SqlJob
+ g.Expect(k8sClient.Get(testCtx, client.ObjectKeyFromObject(&sqlJob), &updatedSqlJob)).To(Succeed())
+ return !updatedSqlJob.IsComplete()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
Consistently(func() bool {
var updatedSqlJob mariadbv1alpha1.SqlJob
if err := k8sClient.Get(testCtx, client.ObjectKeyFromObject(&sqlJob), &updatedSqlJob); err != nil {
diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go
index ba50bbdcdb..f524502a52 100644
--- a/internal/controller/suite_test.go
+++ b/internal/controller/suite_test.go
@@ -58,7 +58,7 @@ var (
k8sClient client.Client
testRefResolver *refresolver.RefResolver
testCidrPrefix string
- testEmulateExternalMdbHost string = "mdb-emulate-external-test.default.svc.cluster.local"
+ testEmulateExternalMdbHost string = "mdb-emulate-external-test-primary.default.svc.cluster.local"
// This is to make sure that backups taken during the tests are matched
testTargetRecoveryTime = &metav1.Time{Time: time.Now().Add(100 * time.Hour)}
)
diff --git a/internal/controller/user_controller_test.go b/internal/controller/user_controller_test.go
index cc4f483f62..9dcd64a52d 100644
--- a/internal/controller/user_controller_test.go
+++ b/internal/controller/user_controller_test.go
@@ -1,11 +1,16 @@
package controller
import (
+ "strconv"
+
mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/metadata"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/sql"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
+ discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
@@ -40,7 +45,15 @@ var _ = Describe("User", Label("basic"), func() {
MaxUserConnections: 20,
},
}
- Expect(k8sClient.Create(testCtx, &user)).To(Succeed())
+
+ By("Expecting User to be created")
+ Eventually(func() bool {
+ if err := k8sClient.Create(testCtx, &user); err != nil {
+ return false
+ }
+ return true
+ }, testTimeout, testInterval).Should(BeTrue())
+
DeferCleanup(func() {
Expect(k8sClient.Delete(testCtx, &user)).To(Succeed())
})
@@ -551,7 +564,15 @@ var _ = Describe("User on a external MariaDB", func() {
MaxUserConnections: 20,
},
}
- Expect(k8sClient.Create(testCtx, &user)).To(Succeed())
+
+ By("Expecting User to be created")
+ Eventually(func() bool {
+ if err := k8sClient.Create(testCtx, &user); err != nil {
+ return false
+ }
+ return true
+ }, testTimeout, testInterval).Should(BeTrue())
+
DeferCleanup(func() {
Expect(k8sClient.Delete(testCtx, &user)).To(Succeed())
})
@@ -951,3 +972,116 @@ var _ = Describe("User on a external MariaDB", func() {
testConnection(userKey.Name, testPasswordSecretRef, testTLSClientCertRef, databaseKey.Name, true)
})
})
+
+var _ = Describe("User on MariaDB replicating from external server", Ordered, func() {
+
+ var (
+ // key = testMdbERkey
+ mdb = &mariadbv1alpha1.MariaDB{}
+ )
+ BeforeEach(func() {
+ By("Waiting for MariaDB to be ready")
+ expectMariadbReady(testCtx, k8sClient, testMdbERkey)
+ })
+
+ It("should reconcile", func() {
+
+ By("Expecting MariaDB to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, testMdbERkey, mdb); err != nil {
+ return false
+ }
+ return mdb.IsReady()
+ }, testHighTimeout, testInterval).Should(BeTrue())
+
+ var endpoints discoveryv1.EndpointSlice
+ By("Expecting to create secondary Endpoints: " + strconv.Itoa(int(mdb.Spec.Replicas)))
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+ count := 0
+ for _, address := range endpoints.Endpoints {
+ if *address.Conditions.Ready {
+ count++
+ }
+ }
+ return count == int(mdb.Spec.Replicas)
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ userKey := types.NamespacedName{
+ Name: "user-test",
+ Namespace: testNamespace,
+ }
+ user := mariadbv1alpha1.User{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: userKey.Name,
+ Namespace: userKey.Namespace,
+ },
+ Spec: mariadbv1alpha1.UserSpec{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testMdbERkey.Name,
+ },
+ WaitForIt: true,
+ },
+ PasswordSecretKeyRef: &testPasswordSecretRef,
+ MaxUserConnections: 20,
+ },
+ }
+ Expect(k8sClient.Create(testCtx, &user)).To(Succeed())
+ DeferCleanup(func() {
+ Expect(k8sClient.Delete(testCtx, &user)).To(Succeed())
+ })
+
+ By("Expecting User to be ready eventually")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, userKey, &user); err != nil {
+ return false
+ }
+ return user.IsReady()
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting User to eventually have finalizer")
+ Eventually(func() bool {
+ if err := k8sClient.Get(testCtx, userKey, &user); err != nil {
+ return false
+ }
+ return controllerutil.ContainsFinalizer(&user, userFinalizerName)
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting to secondary to have endpoints: " + strconv.Itoa(int(mdb.Spec.Replicas)))
+ Eventually(func() bool {
+ Expect(k8sClient.Get(testCtx, mdb.SecondaryServiceKey(), &endpoints)).To(Succeed())
+ count := 0
+ for _, address := range endpoints.Endpoints {
+ if *address.Conditions.Ready {
+ count++
+ }
+ }
+ return count == int(mdb.Spec.Replicas)
+ }, testTimeout, testInterval).Should(BeTrue())
+
+ By("Expecting credentials to be valid")
+ testConnection(user.Name, testPasswordSecretRef, testTLSClientCertRef, testDatabase, true)
+
+ // Check user is present on every all nodes
+ replicas := int(mdb.Spec.Replicas)
+ refResolver := refresolver.New(k8sClient)
+
+ By("Expecting to get password from secret")
+ password, err := refResolver.SecretKeyRef(testCtx, testPasswordSecretRef, mdb.GetNamespace())
+ Expect(err).To(Succeed())
+
+ for i := 0; i < replicas; i++ {
+
+ client, err := sql.NewInternalClientWithPodIndex(testCtx, mdb, refResolver, i,
+ sql.WithUsername(user.Name),
+ sql.WithPassword(password))
+
+ By("Expecting to get SqlClient from Pod " + strconv.Itoa(i))
+ Expect(err).To(Succeed())
+
+ By("Expecting to execute SELECT 1 from Pod " + strconv.Itoa(i))
+ Expect(client.Exec(testCtx, "SELECT 1")).To(Succeed())
+ }
+ })
+})
diff --git a/internal/controller/utils_test.go b/internal/controller/utils_test.go
index 04f3ed7ca5..09ae1dfae2 100644
--- a/internal/controller/utils_test.go
+++ b/internal/controller/utils_test.go
@@ -5,6 +5,7 @@ import (
"fmt"
"net"
"os"
+ "strconv"
"strings"
"time"
@@ -16,6 +17,7 @@ import (
"github.com/mariadb-operator/mariadb-operator/v26/pkg/environment"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/job"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/metadata"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/sql"
stsobj "github.com/mariadb-operator/mariadb-operator/v26/pkg/statefulset"
. "github.com/onsi/ginkgo/v2"
@@ -38,7 +40,7 @@ import (
var (
testVeryHighTimeout = 10 * time.Minute
testHighTimeout = 5 * time.Minute
- testTimeout = 2 * time.Minute
+ testTimeout = 3 * time.Minute
testInterval = 1 * time.Second
testNamespace = "default"
@@ -54,6 +56,57 @@ var (
Name: "emdb-test",
Namespace: testNamespace,
}
+
+ testMdbERkey = types.NamespacedName{
+ Name: "mariadb-repl-external",
+ Namespace: testNamespace,
+ }
+
+ testMdbPbRecoveryERkey = types.NamespacedName{
+ Name: testMdbERkey.Name + "-pb-recovery",
+ Namespace: testNamespace,
+ }
+
+ testPbTemplateERkey = types.NamespacedName{
+ Name: testMdbERkey.Name + "-backup-template",
+ Namespace: testNamespace,
+ }
+
+ testLogicalBackupTemplateERkey = types.NamespacedName{
+ Name: testMdbERkey.Name + "-logical-backup-template",
+ Namespace: testNamespace,
+ }
+
+ testMdbERFilteredKey = types.NamespacedName{
+ Name: "mariadb-repl-ext-filtered",
+ Namespace: testNamespace,
+ }
+
+ testMdbPbRecoveryERFilteredKey = types.NamespacedName{
+ Name: testMdbERFilteredKey.Name + "-pb-recovery",
+ Namespace: testNamespace,
+ }
+
+ testPbTemplateERFilteredKey = types.NamespacedName{
+ Name: testMdbERFilteredKey.Name + "-backup-template",
+ Namespace: testNamespace,
+ }
+
+ testMdbERMultiSchemaKey = types.NamespacedName{
+ Name: "mariadb-repl-ext-multi-schema",
+ Namespace: testNamespace,
+ }
+
+ testMdbPbRecoveryERMultiSchemaKey = types.NamespacedName{
+ Name: testMdbERMultiSchemaKey.Name + "-pb-recovery",
+ Namespace: testNamespace,
+ }
+
+ testPbTemplateERMultiSchemaKey = types.NamespacedName{
+ Name: testMdbERMultiSchemaKey.Name + "-backup-template",
+ Namespace: testNamespace,
+ }
+
testPwdKey = types.NamespacedName{
Name: "password",
Namespace: testNamespace,
@@ -321,9 +374,25 @@ max_allowed_packet=256M`),
bind-address=*
default_storage_engine=InnoDB
binlog_format=row
+log_bin=ON
innodb_autoinc_lock_mode=2
-max_allowed_packet=256M`),
+max_allowed_packet=256M
+binlog_expire_logs_seconds=300`),
Port: 3306,
+ // Native replication so the emulated external exposes a primary and a secondary service.
+ // This lets the server_id offset auto-discovery tests point one ExternalMariaDB at the
+ // primary (master endpoint) and another at the secondary (slave endpoint, followed to the
+ // primary). AutoFailover is disabled to keep pod-0 the stable primary during the tests.
+ Replication: &mariadbv1alpha1.Replication{
+ ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+ Primary: mariadbv1alpha1.PrimaryReplication{
+ PodIndex: ptr.To(0),
+ AutoFailover: ptr.To(false),
+ },
+ },
+ Enabled: true,
+ },
+ Replicas: 2,
Service: &mariadbv1alpha1.ServiceTemplate{
Type: corev1.ServiceTypeLoadBalancer,
Metadata: &mariadbv1alpha1.Metadata{
@@ -332,6 +401,22 @@ max_allowed_packet=256M`),
},
},
},
+ PrimaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.io/loadBalancerIPs": testCidrPrefix + ".0.205",
+ },
+ },
+ },
+ SecondaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.io/loadBalancerIPs": testCidrPrefix + ".0.206",
+ },
+ },
+ },
TLS: &mariadbv1alpha1.TLS{
Enabled: true,
Required: ptr.To(true),
@@ -344,6 +429,7 @@ max_allowed_packet=256M`),
applyMariadbTestConfig(&emulateExternalMdb)
Expect(k8sClient.Create(ctx, &emulateExternalMdb)).To(Succeed())
expectMariadbReady(ctx, k8sClient, testEmulateExternalMdbkey)
+ populateEmulatedExternalMariaDB(ctx, k8sClient, testEmulateExternalMdbkey, "0-1-1000")
emdb := mariadbv1alpha1.ExternalMariaDB{
ObjectMeta: metav1.ObjectMeta{
@@ -388,6 +474,244 @@ max_allowed_packet=256M`),
Expect(k8sClient.Create(ctx, &emdb)).To(Succeed())
expectExternalMariadbReady(ctx, k8sClient, testEMdbkey)
+ backupTemplate := mariadbv1alpha1.PhysicalBackup{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: testPbTemplateERkey.Name,
+ Namespace: testPbTemplateERkey.Namespace,
+ },
+ Spec: mariadbv1alpha1.PhysicalBackupSpec{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testMdbERkey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ WaitForIt: false,
+ },
+ Target: ptr.To(mariadbv1alpha1.PhysicalBackupTargetPreferReplica),
+ Schedule: &mariadbv1alpha1.PhysicalBackupSchedule{
+ Suspend: true,
+ },
+ Compression: mariadbv1alpha1.CompressBzip2,
+ Storage: mariadbv1alpha1.PhysicalBackupStorage{
+ PersistentVolumeClaim: &mariadbv1alpha1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceStorage: resource.MustParse("1Gi"),
+ },
+ },
+ AccessModes: []corev1.PersistentVolumeAccessMode{
+ corev1.ReadWriteOnce,
+ },
+ },
+ },
+ Timeout: &metav1.Duration{Duration: 1 * time.Hour},
+ PodAffinity: ptr.To(true),
+ JobContainerTemplate: mariadbv1alpha1.JobContainerTemplate{
+ Resources: &mariadbv1alpha1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("100m"),
+ corev1.ResourceMemory: resource.MustParse("128Mi"),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("300m"),
+ corev1.ResourceMemory: resource.MustParse("512Mi"),
+ },
+ },
+ },
+ },
+ }
+
+ By("Creating PhysicalBackup template for external replication recovery")
+ Expect(k8sClient.Create(ctx, &backupTemplate)).To(Succeed())
+
+ logicalBackupTemplate := mariadbv1alpha1.Backup{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: testLogicalBackupTemplateERkey.Name,
+ Namespace: testLogicalBackupTemplateERkey.Namespace,
+ },
+ Spec: mariadbv1alpha1.BackupSpec{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testEMdbkey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ WaitForIt: false,
+ },
+ Storage: mariadbv1alpha1.BackupStorage{
+ PersistentVolumeClaim: &mariadbv1alpha1.PersistentVolumeClaimSpec{
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceStorage: resource.MustParse("1Gi"),
+ },
+ },
+ AccessModes: []corev1.PersistentVolumeAccessMode{
+ corev1.ReadWriteOnce,
+ },
+ },
+ },
+ Schedule: &mariadbv1alpha1.Schedule{
+ Suspend: true,
+ },
+ JobContainerTemplate: mariadbv1alpha1.JobContainerTemplate{
+ Resources: &mariadbv1alpha1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("100m"),
+ corev1.ResourceMemory: resource.MustParse("128Mi"),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("300m"),
+ corev1.ResourceMemory: resource.MustParse("512Mi"),
+ },
+ },
+ },
+ },
+ }
+
+ By("Creating Backup template for external replication logical backup")
+ Expect(k8sClient.Create(ctx, &logicalBackupTemplate)).To(Succeed())
+
+ // var GtidSlavePos mariadbv1alpha1.Gtid = "SlavePos"
+
+ mdber := mariadbv1alpha1.MariaDB{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: testMdbERkey.Name,
+ Namespace: testMdbERkey.Namespace,
+ },
+ Spec: mariadbv1alpha1.MariaDBSpec{
+ Username: &testUser,
+ PasswordSecretKeyRef: &mariadbv1alpha1.GeneratedSecretKeyRef{
+ SecretKeySelector: mariadbv1alpha1.SecretKeySelector{
+ LocalObjectReference: mariadbv1alpha1.LocalObjectReference{
+ Name: testPwdKey.Name,
+ },
+ Key: testPwdSecretKey,
+ },
+ },
+ Database: &testDatabase,
+ MyCnf: ptr.To(`[mariadb]
+ bind-address=*
+ default_storage_engine=InnoDB
+ binlog_format=row
+ innodb_autoinc_lock_mode=2
+ max_allowed_packet=256M`,
+ ),
+ Replication: &mariadbv1alpha1.Replication{
+ ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+ ReplicaFromExternal: &mariadbv1alpha1.ReplicaFromExternal{
+ MariaDBRef: mariadbv1alpha1.MariaDBRef{
+ ObjectReference: mariadbv1alpha1.ObjectReference{
+ Name: testEMdbkey.Name,
+ },
+ Kind: mariadbv1alpha1.ExternalMariaDBKind,
+ },
+ ServerIdOffset: ptr.To(50),
+ // Gtid: ptr.To(GtidSlavePos),
+ },
+ Replica: mariadbv1alpha1.ReplicaReplication{
+ ReplicaBootstrapFrom: &mariadbv1alpha1.ReplicaBootstrapFrom{
+ PhysicalBackupTemplateRef: mariadbv1alpha1.LocalObjectReference{
+ Name: testPbTemplateERkey.Name,
+ },
+ LogicalBackupTemplateRef: &mariadbv1alpha1.LocalObjectReference{
+ Name: testLogicalBackupTemplateERkey.Name,
+ },
+ RestoreJob: &mariadbv1alpha1.Job{
+ Resources: &mariadbv1alpha1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("100m"),
+ corev1.ResourceMemory: resource.MustParse("128Mi"),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("300m"),
+ corev1.ResourceMemory: resource.MustParse("512Mi"),
+ },
+ },
+ },
+ },
+ // Gtid: ptr.To(GtidSlavePos),
+ IgnoreMaxLagSeconds: ptr.To(true),
+ IgnoreReplicationLivenessProbes: ptr.To(true),
+ },
+ },
+ Enabled: true,
+ },
+ Replicas: 3,
+ Storage: mariadbv1alpha1.Storage{
+ Size: ptr.To(resource.MustParse("300Mi")),
+ StorageClassName: "standard-resize",
+ ResizeInUseVolumes: ptr.To(true),
+ WaitForVolumeResize: ptr.To(true),
+ },
+
+ TLS: &mariadbv1alpha1.TLS{
+ Enabled: true,
+ Required: ptr.To(true),
+ },
+ Service: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.120",
+ },
+ },
+ },
+ Connection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-conn"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ PrimaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.193",
+ },
+ },
+ },
+ PrimaryConnection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-conn-primary"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ SecondaryService: &mariadbv1alpha1.ServiceTemplate{
+ Type: corev1.ServiceTypeLoadBalancer,
+ Metadata: &mariadbv1alpha1.Metadata{
+ Annotations: map[string]string{
+ "metallb.universe.tf/loadBalancerIPs": testCidrPrefix + ".0.192",
+ },
+ },
+ },
+ SecondaryConnection: &mariadbv1alpha1.ConnectionTemplate{
+ SecretName: func() *string {
+ s := "mdb-repl-conn-secondary"
+ return &s
+ }(),
+ SecretTemplate: &mariadbv1alpha1.SecretTemplate{
+ Key: &testConnSecretKey,
+ },
+ },
+ UpdateStrategy: mariadbv1alpha1.UpdateStrategy{
+ Type: mariadbv1alpha1.ReplicasFirstPrimaryLastUpdateType,
+ },
+ },
+ }
+ applyMariadbTestConfig(&mdber)
+
+ By("Waiting for external MariaDB to be ready")
+ expectExternalMariadbReady(testCtx, k8sClient, testEMdbkey)
+
+ By("Creating MariaDB with external replication")
+ Expect(k8sClient.Create(testCtx, &mdber)).To(Succeed())
+ expectMariadbReady(ctx, k8sClient, testMdbERkey)
+
}
func testCleanupInitialData(ctx context.Context) {
@@ -395,16 +719,33 @@ func testCleanupInitialData(ctx context.Context) {
var ssec corev1.Secret
var externalPassword corev1.Secret
var emdb mariadbv1alpha1.ExternalMariaDB
+ var pbTemplate mariadbv1alpha1.PhysicalBackup
+ var logicalBackupTemplate mariadbv1alpha1.Backup
+ var pbRecoveryPvc corev1.PersistentVolumeClaim
+ var logicalBackupPvc corev1.PersistentVolumeClaim
Expect(k8sClient.Get(ctx, testPwdKey, &password)).To(Succeed())
Expect(k8sClient.Delete(ctx, &password)).To(Succeed())
Expect(k8sClient.Get(ctx, testSSECKey, &ssec)).To(Succeed())
Expect(k8sClient.Delete(ctx, &ssec)).To(Succeed())
deleteMariadb(testMdbkey, false)
+ deleteMariadb(testMdbERkey, false)
Expect(k8sClient.Get(ctx, testEMdbkey, &emdb)).To(Succeed())
Expect(k8sClient.Delete(ctx, &emdb)).To(Succeed())
deleteMariadb(testEmulateExternalMdbkey, false)
Expect(k8sClient.Get(ctx, testEmulatedExternalPwdKey, &externalPassword)).To(Succeed())
Expect(k8sClient.Delete(ctx, &externalPassword)).To(Succeed())
+ Expect(k8sClient.Get(ctx, testPbTemplateERkey, &pbTemplate)).To(Succeed())
+ Expect(k8sClient.Delete(ctx, &pbTemplate)).To(Succeed())
+ if err := k8sClient.Get(ctx, testLogicalBackupTemplateERkey, &logicalBackupTemplate); err == nil {
+ Expect(k8sClient.Delete(ctx, &logicalBackupTemplate)).To(Succeed())
+ }
+
+ if err := k8sClient.Get(ctx, testMdbPbRecoveryERkey, &pbRecoveryPvc); err == nil {
+ Expect(k8sClient.Delete(ctx, &pbRecoveryPvc)).To(Succeed())
+ }
+ if err := k8sClient.Get(ctx, testEMdbkey, &logicalBackupPvc); err == nil {
+ Expect(k8sClient.Delete(ctx, &logicalBackupPvc)).To(Succeed())
+ }
}
func testMariadbUpdate(mdb *mariadbv1alpha1.MariaDB) {
@@ -814,12 +1155,37 @@ func testExternalConnection(username string, password mariadbv1alpha1.SecretKeyS
}, testTimeout, testInterval).Should(BeTrue())
}
+func testDeletePod(mdb *mariadbv1alpha1.MariaDB, podIndex int, deletePVC bool) {
+
+ // Delete PVC
+ if deletePVC {
+ PVCKey := types.NamespacedName{
+ Name: fmt.Sprintf("storage-%s-%d", mdb.Name, podIndex),
+ Namespace: testNamespace,
+ }
+ var existingPvc corev1.PersistentVolumeClaim
+ By("Expecting to get PVC from Pod" + strconv.Itoa(podIndex))
+ Expect(k8sClient.Get(testCtx, PVCKey, &existingPvc)).To(Succeed())
+ By("Expecting to delete PVC from Pod" + strconv.Itoa(podIndex))
+ Expect(k8sClient.Delete(testCtx, &existingPvc)).To(Succeed())
+ }
+ // Delete POD
+ podKey := types.NamespacedName{
+ Name: fmt.Sprintf("%s-%d", mdb.Name, podIndex),
+ Namespace: testNamespace,
+ }
+ var existingPod corev1.Pod
+ Expect(k8sClient.Get(testCtx, podKey, &existingPod)).To(Succeed())
+ Expect(k8sClient.Delete(testCtx, &existingPod)).To(Succeed())
+
+}
+
// See: https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
func applyMariadbTestConfig(mdb *mariadbv1alpha1.MariaDB) *mariadbv1alpha1.MariaDB {
mdb.Spec.Resources = &mariadbv1alpha1.ResourceRequirements{
Requests: corev1.ResourceList{
- "cpu": resource.MustParse("500m"),
- "memory": resource.MustParse("1Gi"),
+ "cpu": resource.MustParse("100m"),
+ "memory": resource.MustParse("512Mi"),
},
Limits: corev1.ResourceList{
"memory": resource.MustParse("1Gi"),
@@ -1179,6 +1545,38 @@ func expectMariadbReady(ctx context.Context, k8sClient client.Client, key types.
})
}
+func populateEmulatedExternalMariaDB(ctx context.Context, k8sClient client.Client, key types.NamespacedName, state string) {
+ Eventually(func(g Gomega) bool {
+ var mdb mariadbv1alpha1.MariaDB
+ g.Expect(k8sClient.Get(ctx, key, &mdb)).To(Succeed())
+ client, err := sql.NewClientWithMariaDB(ctx, &mdb, refresolver.New(k8sClient), sql.WithMultiStatements(true))
+ if err != nil {
+ return false
+ }
+ defer client.Close()
+ sqlCommand := `
+ SET SESSION gtid_seq_no = 1000;
+ BEGIN;
+ CREATE DATABASE inttest;
+ COMMIT;
+ USE inttest;
+ CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY);
+ `
+ g.Expect(
+ client.Exec(ctx, sqlCommand),
+ ).To(Succeed())
+ time.Sleep(2 * time.Second)
+ sqlCommand = `
+ FLUSH LOGS;
+ PURGE BINARY LOGS BEFORE DATE_ADD(NOW(), INTERVAL 1 MINUTE);
+ `
+ g.Expect(
+ client.Exec(ctx, sqlCommand),
+ ).To(Succeed())
+ return true
+ }, testHighTimeout, testInterval).Should(BeTrue())
+}
+
func expectExternalMariadbReady(ctx context.Context, k8sClient client.Client, key types.NamespacedName) {
By("Expecting MariaDB to be ready eventually")
expectExternalMariadbFn(ctx, k8sClient, key, func(mdb *mariadbv1alpha1.ExternalMariaDB) bool {
@@ -1254,6 +1652,26 @@ func deleteMariadb(key types.NamespacedName, assertPVCDeletion bool) {
return apierrors.IsNotFound(err)
}, testTimeout, testInterval).Should(BeTrue())
+ // The MariaDB object being gone does not mean its Pods are: they are garbage collected
+ // asynchronously and MariaDB shuts down gracefully. Leftover Pods still serve SQL and still have
+ // replication configured, so a test that immediately re-creates a MariaDB with the same name may
+ // observe them and trigger an automatic failover on the brand new object. Waiting here also lets
+ // the pvc-protection finalizer release the PVCs deleted below.
+ By("Expecting MariaDB Pods to be deleted")
+ Eventually(func(g Gomega) bool {
+ var podList corev1.PodList
+ listOpts := []client.ListOption{
+ client.MatchingLabels(
+ labels.NewLabelsBuilder().
+ WithMariaDBSelectorLabels(&mdb).
+ Build(),
+ ),
+ client.InNamespace(mdb.Namespace),
+ }
+ g.Expect(k8sClient.List(testCtx, &podList, listOpts...)).To(Succeed())
+ return len(podList.Items) == 0
+ }, testTimeout, testInterval).Should(BeTrue())
+
By("Deleting PVCs")
opts := []client.DeleteAllOfOption{
client.MatchingLabels(
diff --git a/make/dev.mk b/make/dev.mk
index 51a2572fde..becb4ddeba 100644
--- a/make/dev.mk
+++ b/make/dev.mk
@@ -20,7 +20,7 @@ ENV ?= \
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)
TEST_ARGS ?=
-TEST_TIMEOUT ?= 1h10m
+TEST_TIMEOUT ?= 1h30m
TEST ?= $(ENV) $(GINKGO) --coverprofile=cover.out --timeout $(TEST_TIMEOUT) $(TEST_ARGS)
GOCOVERDIR ?= .
diff --git a/make/net.mk b/make/net.mk
index 07f3c87f62..d3d7a28e31 100644
--- a/make/net.mk
+++ b/make/net.mk
@@ -24,6 +24,9 @@ host-mdb-test: ## Add MariaDB test hosts to /etc/hosts.
host-mdb-emulated-external-test: ## Add MariaDB test hosts to /etc/hosts.
@./hack/add_host.sh 0 47 mdb-emulate-external-test.default.svc.cluster.local
@./hack/add_host.sh 0 48 mdb-emulate-external-test-0.mdb-emulate-external-test-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 204 mdb-emulate-external-test-1.mdb-emulate-external-test-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 205 mdb-emulate-external-test-primary.default.svc.cluster.local
+ @./hack/add_host.sh 0 206 mdb-emulate-external-test-secondary.default.svc.cluster.local
.PHONY: host-mxs-test
host-mxs-test: ## Add MaxScale test hosts to /etc/hosts.
@@ -40,10 +43,39 @@ host-mariadb-repl: ## Add mariadb repl hosts to /etc/hosts.
@./hack/add_host.sh 0 111 mariadb-repl-1.mariadb-repl-internal.default.svc.cluster.local
@./hack/add_host.sh 0 112 mariadb-repl-2.mariadb-repl-internal.default.svc.cluster.local
@./hack/add_host.sh 0 113 mariadb-repl-3.mariadb-repl-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 180 mariadb-repl-external-0.mariadb-repl-external-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 181 mariadb-repl-external-1.mariadb-repl-external-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 182 mariadb-repl-external-2.mariadb-repl-external-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 183 mariadb-repl-external-3.mariadb-repl-external-internal.default.svc.cluster.local
@./hack/add_host.sh 0 120 mariadb-repl.default.svc.cluster.local
@./hack/add_host.sh 0 130 mariadb-repl-primary.default.svc.cluster.local
@./hack/add_host.sh 0 131 mariadb-repl-secondary.default.svc.cluster.local
+.PHONY: host-mariadb-repl-ext-filtered
+host-mariadb-repl-ext-filtered: ## Add mariadb-repl-ext-filtered hosts to /etc/hosts.
+ @./hack/add_host.sh 0 184 mariadb-repl-ext-filtered-0.mariadb-repl-ext-filtered-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 185 mariadb-repl-ext-filtered-1.mariadb-repl-ext-filtered-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 186 mariadb-repl-ext-filtered-2.mariadb-repl-ext-filtered-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 187 mariadb-repl-ext-filtered-3.mariadb-repl-ext-filtered-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 188 mariadb-repl-ext-filtered.default.svc.cluster.local
+ @./hack/add_host.sh 0 189 mariadb-repl-ext-filtered-primary.default.svc.cluster.local
+ @./hack/add_host.sh 0 194 mariadb-repl-ext-filtered-secondary.default.svc.cluster.local
+
+.PHONY: host-mariadb-repl-ext-multi-schema
+host-mariadb-repl-ext-multi-schema: ## Add mariadb-repl-ext-multi-schema hosts to /etc/hosts.
+ @./hack/add_host.sh 0 195 mariadb-repl-ext-multi-schema.default.svc.cluster.local
+ @./hack/add_host.sh 0 196 mariadb-repl-ext-multi-schema-primary.default.svc.cluster.local
+ @./hack/add_host.sh 0 197 mariadb-repl-ext-multi-schema-secondary.default.svc.cluster.local
+ @./hack/add_host.sh 0 198 mariadb-repl-ext-multi-schema-0.mariadb-repl-ext-multi-schema-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 199 mariadb-repl-ext-multi-schema-1.mariadb-repl-ext-multi-schema-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 202 mariadb-repl-ext-multi-schema-2.mariadb-repl-ext-multi-schema-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 203 mariadb-repl-ext-multi-schema-3.mariadb-repl-ext-multi-schema-internal.default.svc.cluster.local
+
+.PHONY: host-mariadb-repl-ext-autodiscovery
+host-mariadb-repl-ext-autodiscovery: ## Add mariadb-repl-ext server_id offset auto-discovery hosts to /etc/hosts.
+ @./hack/add_host.sh 0 207 mdb-autodisc-master-0.mdb-autodisc-master-internal.default.svc.cluster.local
+ @./hack/add_host.sh 0 208 mdb-autodisc-master-1.mdb-autodisc-master-internal.default.svc.cluster.local
+
.PHONY: host-mariadb-galera
host-mariadb-galera: ## Add mariadb galera hosts to /etc/hosts.
@./hack/add_host.sh 0 140 mariadb-galera-0.mariadb-galera-internal.default.svc.cluster.local
@@ -129,7 +161,7 @@ host-multi-cluster-mxs: ## Add multi-cluster maxscale hosts to /etc/hosts.
@./hack/add_host.sh 1 27 maxscale-eu-central-1.maxscale-eu-central-internal.default.svc.cluster.local
.PHONY: host
-host: host-mariadb host-mdb-test host-mdb-emulated-external-test host-mxs-test host-mariadb-repl host-mariadb-galera host-mariadb-galera-test host-monitoring host-minio host-maxscale-repl host-maxscale-galera host-maxscale-gui host-multi-cluster host-multi-cluster-mxs host-azurite ## Configure hosts for local development.
+host: host-mariadb host-mdb-test host-mdb-emulated-external-test host-mxs-test host-mariadb-repl host-mariadb-repl-ext-filtered host-mariadb-repl-ext-multi-schema host-mariadb-repl-ext-autodiscovery host-mariadb-galera host-mariadb-galera-test host-monitoring host-minio host-maxscale-repl host-maxscale-galera host-maxscale-gui host-multi-cluster host-multi-cluster-mxs host-azurite ## Configure hosts for local development.
.PHONY: net
net: install-metallb host ## Configure networking for local development.
diff --git a/pkg/agent/handler/replication/probe.go b/pkg/agent/handler/replication/probe.go
index 54fd07e0f5..a3a5f33dc0 100644
--- a/pkg/agent/handler/replication/probe.go
+++ b/pkg/agent/handler/replication/probe.go
@@ -63,6 +63,16 @@ func (p *ReplicationProbe) Liveness(w http.ResponseWriter, r *http.Request) {
return
}
if isReplica {
+
+ k8sCtx, k8sCancel := context.WithTimeout(context.Background(), requestTimeout)
+ defer k8sCancel()
+
+ if p.getIgnoreReplicationLivenessProbes(k8sCtx) {
+ p.readinessLogger.V(1).Info("Ignoring liveness replication probe")
+ p.responseWriter.WriteOK(w, nil)
+ return
+ }
+
status, err := sqlClient.ReplicaStatus(sqlCtx, p.livenessLogger)
if err != nil {
p.livenessLogger.Error(err, "error getting replica status")
@@ -137,6 +147,13 @@ func (p *ReplicationProbe) Readiness(w http.ResponseWriter, r *http.Request) {
p.responseWriter.WriteErrorf(w, "error getting replica status: %v", err)
return
}
+
+ if p.getIgnoreMaxLagSeconds(k8sCtx) {
+ p.readinessLogger.V(1).Info("Ignoring max lag seconds check based on replica status")
+ p.responseWriter.WriteOK(w, nil)
+ return
+ }
+
if status.SecondsBehindMaster == nil {
p.readinessLogger.Error(nil, "could not determine replica lag")
p.responseWriter.WriteError(w, "could not determine replica lag")
@@ -185,3 +202,25 @@ func (p *ReplicationProbe) getMaxLagSeconds(ctx context.Context) int {
replica := replication.Replica
return ptr.Deref(replica.MaxLagSeconds, 0)
}
+
+func (p *ReplicationProbe) getIgnoreMaxLagSeconds(ctx context.Context) bool {
+ var mdb mariadbv1alpha1.MariaDB
+ if err := p.k8sClient.Get(ctx, p.mariadbKey, &mdb); err != nil {
+ p.readinessLogger.Error(err, "error getting MariaDB. Using default ignore max lag seconds value")
+ return false
+ }
+ replication := ptr.Deref(mdb.Spec.Replication, mariadbv1alpha1.Replication{})
+ replica := replication.Replica
+ return ptr.Deref(replica.IgnoreMaxLagSeconds, false)
+}
+
+func (p *ReplicationProbe) getIgnoreReplicationLivenessProbes(ctx context.Context) bool {
+ var mdb mariadbv1alpha1.MariaDB
+ if err := p.k8sClient.Get(ctx, p.mariadbKey, &mdb); err != nil {
+ p.readinessLogger.Error(err, "error getting MariaDB. Using default behavior (Replication probe checks enabled)")
+ return false
+ }
+ replication := ptr.Deref(mdb.Spec.Replication, mariadbv1alpha1.Replication{})
+ replica := replication.Replica
+ return ptr.Deref(replica.IgnoreReplicationLivenessProbes, false)
+}
diff --git a/pkg/backup/processor.go b/pkg/backup/processor.go
index 814f5f9b74..fdb3fb2d90 100644
--- a/pkg/backup/processor.go
+++ b/pkg/backup/processor.go
@@ -14,9 +14,10 @@ import (
)
type BackupProcessor interface {
- GetBackupTargetFile(backupFileNames []string, targetRecoveryTime time.Time, logger logr.Logger) (string, error)
+ GetBackupTargetFile(backupFileNames []string, targetRecoveryTime time.Time,
+ targetTimeAgeThreshold *time.Time, logger logr.Logger) (string, error)
GetOldBackupFiles(backupFileNames []string, maxRetention time.Duration, logger logr.Logger) []string
- IsValidBackupFile(fileName string) bool
+ IsValidBackupFile(fileName string, logger logr.Logger) bool
ParseCompressionAlgorithm(fileName string) (mariadbv1alpha1.CompressAlgorithm, error)
GetUncompressedBackupFile(compressedBackupFile string) (string, error)
parseDateInBackupFile(fileName string) (time.Time, error)
@@ -37,7 +38,7 @@ func NewLogicalBackupProcessor() BackupProcessor {
// GetBackupTargetFile returns the backup file whose timestamp is closest to, but not after, the target recovery time.
func (p *LogicalBackupProcessor) GetBackupTargetFile(backupFileNames []string, targetRecoveryTime time.Time,
- backupLogger logr.Logger) (string, error) {
+ targetTimeAgeThreshold *time.Time, backupLogger logr.Logger) (string, error) {
logger := backupLogger.WithValues(
"target-time", targetRecoveryTime.Format(time.RFC3339),
)
@@ -91,17 +92,20 @@ func (p *LogicalBackupProcessor) GetOldBackupFiles(backupFileNames []string, max
}
// IsValidBackupFile determines whether a backup file name is valid.
-func (p *LogicalBackupProcessor) IsValidBackupFile(fileName string) bool {
- // Must start with "backup." and contain ".sql" either as suffix (uncompressed)
- // or before the compression extension (e.g. ".sql.gz", ".sql.bz2").
+func (p *LogicalBackupProcessor) IsValidBackupFile(fileName string, logger logr.Logger) bool {
if !strings.HasPrefix(fileName, "backup.") || !strings.Contains(fileName, ".sql") {
+ logger.Info("File has no backup. prefix or .sql suffix or .sql.gz or .sql.bz2", "file", fileName)
return false
}
_, err := p.ParseCompressionAlgorithm(fileName)
if err != nil {
+ logger.Error(err, "Error parsing compression algorithm", "file", fileName)
return false
}
_, err = p.parseDateInBackupFile(fileName)
+ if err != nil {
+ logger.Error(err, "Error parsing date in backup file", "file", fileName)
+ }
return err == nil
}
@@ -196,7 +200,7 @@ func NewPhysicalBackupProcessor(opts ...PhysicalBackupProcsssorOpt) BackupProces
// GetBackupTargetFile returns the backup file whose timestamp is closest to, but not after, the target recovery time.
func (p *PhysicalBackupProcessor) GetBackupTargetFile(backupFileNames []string, targetRecoveryTime time.Time,
- backupLogger logr.Logger) (string, error) {
+ targetTimeAgeThreshold *time.Time, backupLogger logr.Logger) (string, error) {
logger := backupLogger.WithValues(
"target-time", targetRecoveryTime.Format(time.RFC3339),
)
@@ -229,6 +233,19 @@ func (p *PhysicalBackupProcessor) GetBackupTargetFile(backupFileNames []string,
sort.Slice(backupDiffs, func(i, j int) bool {
return backupDiffs[i].diff < backupDiffs[j].diff
})
+
+ if targetTimeAgeThreshold != nil {
+ targetFileTime, err := p.parseDateInBackupFile(backupDiffs[0].fileName)
+ if err != nil {
+ logger.Error(err, "error parsing backup date in target file", "file", backupDiffs[0].fileName)
+ return "", fmt.Errorf("error parsing backup date in target file: %v", err)
+ }
+ if targetFileTime.Before(*targetTimeAgeThreshold) {
+ return "", fmt.Errorf("the closest backup file '%s' is too old according to the provided target time age threshold",
+ backupDiffs[0].fileName)
+ }
+ }
+
return backupDiffs[0].fileName, nil
}
@@ -250,7 +267,7 @@ func (p *PhysicalBackupProcessor) GetOldBackupFiles(backupFileNames []string, ma
}
// IsValidBackupFile determines whether a backup file name is valid.
-func (p *PhysicalBackupProcessor) IsValidBackupFile(fileName string) bool {
+func (p *PhysicalBackupProcessor) IsValidBackupFile(fileName string, logger logr.Logger) bool {
if validationFn := p.isValidBackupFileFn; validationFn != nil {
return validationFn(fileName)
}
diff --git a/pkg/backup/processor_test.go b/pkg/backup/processor_test.go
index c296f89012..ba1bc56c63 100644
--- a/pkg/backup/processor_test.go
+++ b/pkg/backup/processor_test.go
@@ -142,7 +142,7 @@ func TestLogicalGetTargetFile(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- file, err := p.GetBackupTargetFile(tt.backupFiles, tt.targetRecovery, logger)
+ file, err := p.GetBackupTargetFile(tt.backupFiles, tt.targetRecovery, nil, logger)
if err != nil && !tt.wantErr {
t.Fatalf("unexpected error getting target recovery file: %v", err)
}
@@ -348,7 +348,7 @@ func TestLogicalIsValidBackupFile(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- valid := p.IsValidBackupFile(tt.backupFile)
+ valid := p.IsValidBackupFile(tt.backupFile, logger)
if tt.wantValid != valid {
t.Fatalf("unexpected backup file validity, expected: %v got: %v", tt.wantValid, valid)
}
@@ -630,7 +630,7 @@ func TestPhysicalGetTargetFile(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- file, err := p.GetBackupTargetFile(tt.backupFiles, tt.targetRecovery, logger)
+ file, err := p.GetBackupTargetFile(tt.backupFiles, tt.targetRecovery, nil, logger)
if err != nil && !tt.wantErr {
t.Fatalf("unexpected error: %v", err)
}
@@ -859,7 +859,7 @@ func TestPhysicalIsValidBackupFile(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- valid := p.IsValidBackupFile(tt.backupFile)
+ valid := p.IsValidBackupFile(tt.backupFile, logger)
if tt.wantValid != valid {
t.Fatalf("unexpected backup file validity, expected: %v got: %v", tt.wantValid, valid)
}
@@ -1116,7 +1116,7 @@ func TestSnapshotGetTargetFile(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- file, err := p.GetBackupTargetFile(tt.backupFiles, tt.targetRecovery, logger)
+ file, err := p.GetBackupTargetFile(tt.backupFiles, tt.targetRecovery, nil, logger)
if err != nil && !tt.wantErr {
t.Fatalf("unexpected error: %v", err)
}
@@ -1302,7 +1302,7 @@ func TestSnapshotIsValidBackupFile(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- valid := p.IsValidBackupFile(tt.backupFile)
+ valid := p.IsValidBackupFile(tt.backupFile, logger)
if tt.wantValid != valid {
t.Fatalf("unexpected backup file validity, expected: %v got: %v", tt.wantValid, valid)
}
diff --git a/pkg/backup/storage.go b/pkg/backup/storage.go
index 4743c4e440..2114879d34 100644
--- a/pkg/backup/storage.go
+++ b/pkg/backup/storage.go
@@ -34,14 +34,20 @@ func NewFileSystemBackupStorage(basePath string, processor BackupProcessor, logg
}
func (f *FileSystemBackupStorage) List(ctx context.Context) ([]string, error) {
+ f.logger.Info("List files in file system storage", "path", f.basePath)
entries, err := os.ReadDir(f.basePath)
+
if err != nil {
return nil, err
}
+ f.logger.Info("List files in file system storage", "files", entries)
+
var fileNames []string
for _, e := range entries {
fileName := e.Name()
+ f.logger.Info("List files in file system storage", "filename", fileName)
if f.shouldProcessBackupFile(fileName, f.logger) {
+ f.logger.Info("List files in file system storage", "append", fileName)
fileNames = append(fileNames, fileName)
}
}
@@ -61,11 +67,11 @@ func (f *FileSystemBackupStorage) Delete(ctx context.Context, fileName string) e
}
func (f *FileSystemBackupStorage) shouldProcessBackupFile(fileName string, logger logr.Logger) bool {
- logger.V(1).Info("processing backup file", "file", fileName)
- if f.processor.IsValidBackupFile(fileName) {
+ logger.Info("processing backup file", "file", fileName)
+ if f.processor.IsValidBackupFile(fileName, logger) {
return true
}
- logger.V(1).Info("ignoring file", "file", fileName)
+ logger.Info("ignoring file", "file", fileName)
return false
}
@@ -118,7 +124,7 @@ func (s *BlobBackupStorage) Pull(ctx context.Context, fileName string) error {
func (s *BlobBackupStorage) shouldProcessBackupFile(fileName string, logger logr.Logger) bool {
logger.V(1).Info("processing backup file", "file", fileName)
- if s.processor.IsValidBackupFile(s.client.UnprefixedFilename(fileName)) {
+ if s.processor.IsValidBackupFile(s.client.UnprefixedFilename(fileName), logger) {
return true
}
logger.V(1).Info("ignoring file", "file", fileName)
diff --git a/pkg/builder/backup_builder.go b/pkg/builder/backup_builder.go
new file mode 100644
index 0000000000..cf74bfb5fc
--- /dev/null
+++ b/pkg/builder/backup_builder.go
@@ -0,0 +1,78 @@
+package builder
+
+import (
+ "fmt"
+ "time"
+
+ mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
+ metadata "github.com/mariadb-operator/mariadb-operator/v26/pkg/builder/metadata"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+type BackupOpts struct {
+ Metadata []*mariadbv1alpha1.Metadata
+ Key types.NamespacedName
+ MariaDBRef mariadbv1alpha1.MariaDBRef
+ Compression mariadbv1alpha1.CompressAlgorithm
+ Storage mariadbv1alpha1.BackupStorage
+ Args []string
+ Tables []string
+ Resources mariadbv1alpha1.ResourceRequirements
+ Affinity mariadbv1alpha1.AffinityConfig
+ // MaxRetention metav1.Duration
+ MaxRetention time.Duration
+ ImagePullSecrets []mariadbv1alpha1.LocalObjectReference
+ // Template is an optional Backup whose Spec is used as the base for the new Backup. The fields managed
+ // by the controller (Storage, MariaDBRef, Compression, Args, Tables, MaxRetention, ImagePullSecrets)
+ // are overridden by the values in BackupOpts. The remaining fields (resources, pod template, etc.) are
+ // preserved from the template, which lets callers customize the backup Pod via a templated Backup object.
+ Template *mariadbv1alpha1.Backup
+}
+
+func (b *Builder) BuildBackup(opts BackupOpts, owner metav1.Object) (*mariadbv1alpha1.Backup, error) {
+ objMetaBuilder :=
+ metadata.NewMetadataBuilder(opts.Key)
+ for _, meta := range opts.Metadata {
+ objMetaBuilder = objMetaBuilder.WithMetadata(meta)
+ }
+ objMeta := objMetaBuilder.Build()
+
+ var spec mariadbv1alpha1.BackupSpec
+ if opts.Template != nil {
+ spec = *opts.Template.Spec.DeepCopy()
+ }
+ spec.Storage = opts.Storage
+ spec.MariaDBRef = opts.MariaDBRef
+ spec.Compression = opts.Compression
+ spec.Tables = opts.Tables
+ spec.MaxRetention = metav1.Duration{Duration: opts.MaxRetention}
+ spec.Args = opts.Args
+ // The operator-managed Backup runs as a one-shot Job. A suspended Schedule on the template is what
+ // makes the template object skip Job/CronJob reconciliation; clearing it here ensures the resulting
+ // Backup is reconciled as a regular Job.
+ spec.Schedule = nil
+ if spec.Resources == nil {
+ spec.Resources = &opts.Resources
+ }
+ if spec.Affinity == nil {
+ spec.Affinity = &opts.Affinity
+ }
+ if len(opts.ImagePullSecrets) > 0 {
+ spec.ImagePullSecrets = opts.ImagePullSecrets
+ }
+
+ backup := &mariadbv1alpha1.Backup{
+ ObjectMeta: objMeta,
+ Spec: spec,
+ }
+
+ if owner != nil {
+ if err := controllerutil.SetControllerReference(owner, backup, b.scheme); err != nil {
+ return nil, fmt.Errorf("error setting controller reference to Backup: %v", err)
+ }
+ }
+
+ return backup, nil
+}
diff --git a/pkg/builder/batch_builder.go b/pkg/builder/batch_builder.go
index b03698211f..857bc9b2e5 100644
--- a/pkg/builder/batch_builder.go
+++ b/pkg/builder/batch_builder.go
@@ -454,31 +454,32 @@ func (b *Builder) BuildRestoreJob(key types.NamespacedName, restore *mariadbv1al
return job, nil
}
-type RestoreOpts struct {
- StartGtid *replication.Gtid
- TargetRecoveryTime *time.Time
- Volume *mariadbv1alpha1.StorageVolumeSource
- S3 *mariadbv1alpha1.S3
- ABS *mariadbv1alpha1.AzureBlob
- RestoreJob *mariadbv1alpha1.Job
- RestoreCommandOpts []command.MariaDBBackupRestoreOpt
- MariaDBLabels *bool
- Affinity *bool
- NodeSelector map[string]string
- LogLevel string
+type PhysicalBackupRestoreOpts struct {
+ StartGtid *replication.Gtid
+ TargetRecoveryTime *time.Time
+ TargetRecoveryTimeAgeThreshold *time.Time
+ Volume *mariadbv1alpha1.StorageVolumeSource
+ S3 *mariadbv1alpha1.S3
+ ABS *mariadbv1alpha1.AzureBlob
+ RestoreJob *mariadbv1alpha1.Job
+ RestoreCommandOpts []command.MariaDBBackupRestoreOpt
+ MariaDBLabels *bool
+ Affinity *bool
+ NodeSelector map[string]string
+ LogLevel string
}
-type RestoreOpt func(*RestoreOpts) error
-
-func WithStartGtid(gtid *replication.Gtid) RestoreOpt {
- return func(opts *RestoreOpts) error {
+func WithStartGtid(gtid *replication.Gtid) PhysicalBackupRestoreOpt {
+ return func(opts *PhysicalBackupRestoreOpts) error {
opts.StartGtid = gtid
return nil
}
}
-func WithBootstrapFrom(bootstrapFrom *mariadbv1alpha1.BootstrapFrom) RestoreOpt {
- return func(opts *RestoreOpts) error {
+type PhysicalBackupRestoreOpt func(*PhysicalBackupRestoreOpts) error
+
+func WithBootstrapFrom(bootstrapFrom *mariadbv1alpha1.BootstrapFrom) PhysicalBackupRestoreOpt {
+ return func(opts *PhysicalBackupRestoreOpts) error {
opts.TargetRecoveryTime = ptr.To(bootstrapFrom.TargetRecoveryTimeOrDefault())
opts.Volume = bootstrapFrom.Volume
opts.S3 = bootstrapFrom.S3
@@ -490,8 +491,8 @@ func WithBootstrapFrom(bootstrapFrom *mariadbv1alpha1.BootstrapFrom) RestoreOpt
}
func WithPhysicalBackup(pb *mariadbv1alpha1.PhysicalBackup, targetRecoveryTime time.Time,
- restoreJob *mariadbv1alpha1.Job, restoreCommandOpts ...command.MariaDBBackupRestoreOpt) RestoreOpt {
- return func(opts *RestoreOpts) error {
+ restoreJob *mariadbv1alpha1.Job, restoreCommandOpts ...command.MariaDBBackupRestoreOpt) PhysicalBackupRestoreOpt {
+ return func(opts *PhysicalBackupRestoreOpts) error {
volume, err := pb.Volume()
if err != nil {
return err
@@ -506,8 +507,8 @@ func WithPhysicalBackup(pb *mariadbv1alpha1.PhysicalBackup, targetRecoveryTime t
}
}
-func WithReplicaRecovery(podToRecover *corev1.Pod) RestoreOpt {
- return func(opts *RestoreOpts) error {
+func WithReplicaRecovery(podToRecover *corev1.Pod) PhysicalBackupRestoreOpt {
+ return func(opts *PhysicalBackupRestoreOpts) error {
// By default, both MariaDB Pod and the init Job will have the same labels and affinity rules.
// MariaDB Pod will be running, removing the labels and affinity will allow the recovery Job to be scheduled.
opts.MariaDBLabels = ptr.To(false)
@@ -521,9 +522,19 @@ func WithReplicaRecovery(podToRecover *corev1.Pod) RestoreOpt {
}
}
+func WithAgeThreshold(ageThreshold *time.Time) PhysicalBackupRestoreOpt {
+ return func(opts *PhysicalBackupRestoreOpts) error {
+ // By default, the restore job will be executed using just the target recovery time.
+ // If an age threshold is provided, the job will fail if the most recent backup time is too old.
+ opts.TargetRecoveryTimeAgeThreshold = ageThreshold
+
+ return nil
+ }
+}
+
func (b *Builder) BuildPhysicalBackupRestoreJob(key types.NamespacedName, mariadb *mariadbv1alpha1.MariaDB,
- podIndex *int, restoreOpts ...RestoreOpt) (*batchv1.Job, error) {
- opts := RestoreOpts{}
+ podIndex *int, restoreOpts ...PhysicalBackupRestoreOpt) (*batchv1.Job, error) {
+ opts := PhysicalBackupRestoreOpts{}
for _, setOpt := range restoreOpts {
if err := setOpt(&opts); err != nil {
return nil, fmt.Errorf("error setting restore option: %v", err)
@@ -569,6 +580,11 @@ func (b *Builder) BuildPhysicalBackupRestoreJob(key types.NamespacedName, mariad
command.WithOmitCredentials(true),
command.WithExtraOpts(restoreJob.Args),
}
+
+ if opts.TargetRecoveryTimeAgeThreshold != nil {
+ cmdOpts = append(cmdOpts, command.WithBackupTargetTimeAgeThreshold(opts.TargetRecoveryTimeAgeThreshold))
+ }
+
cmdOpts = append(cmdOpts, s3Opts(opts.S3)...)
cmdOpts = append(cmdOpts, absOpts(opts.ABS)...)
@@ -664,8 +680,8 @@ func (b *Builder) BuildPhysicalBackupRestoreJob(key types.NamespacedName, mariad
}
func (b *Builder) BuildPITRJob(key types.NamespacedName, pitr *mariadbv1alpha1.PointInTimeRecovery,
- mariadb *mariadbv1alpha1.MariaDB, restoreOpts ...RestoreOpt) (*batchv1.Job, error) {
- opts := RestoreOpts{}
+ mariadb *mariadbv1alpha1.MariaDB, restoreOpts ...PhysicalBackupRestoreOpt) (*batchv1.Job, error) {
+ opts := PhysicalBackupRestoreOpts{}
for _, setOpt := range restoreOpts {
if err := setOpt(&opts); err != nil {
return nil, fmt.Errorf("error setting restore option: %v", err)
diff --git a/pkg/builder/batch_builder_test.go b/pkg/builder/batch_builder_test.go
index 2de7179cf7..83e6e689f9 100644
--- a/pkg/builder/batch_builder_test.go
+++ b/pkg/builder/batch_builder_test.go
@@ -3404,7 +3404,7 @@ func TestBuildPITRJob(t *testing.T) {
name string
pitr *mariadbv1alpha1.PointInTimeRecovery
mariadb *mariadbv1alpha1.MariaDB
- restoreOpts []RestoreOpt
+ restoreOpts []PhysicalBackupRestoreOpt
wantErr bool
wantJob bool
wantJobMeta *mariadbv1alpha1.Metadata
@@ -3415,7 +3415,7 @@ func TestBuildPITRJob(t *testing.T) {
name: "PITR job missing startGtid",
pitr: pitr,
mariadb: &mariadbv1alpha1.MariaDB{},
- restoreOpts: []RestoreOpt{
+ restoreOpts: []PhysicalBackupRestoreOpt{
WithBootstrapFrom(&mariadbv1alpha1.BootstrapFrom{
TargetRecoveryTime: targetRecoveryTime,
Volume: &mariadbv1alpha1.StorageVolumeSource{
@@ -3430,7 +3430,7 @@ func TestBuildPITRJob(t *testing.T) {
name: "PITR job missing targetRecoveryTime",
pitr: pitr,
mariadb: &mariadbv1alpha1.MariaDB{},
- restoreOpts: []RestoreOpt{
+ restoreOpts: []PhysicalBackupRestoreOpt{
WithStartGtid(mustParseGtid(t, "0-10-1")),
},
wantErr: true,
@@ -3440,7 +3440,7 @@ func TestBuildPITRJob(t *testing.T) {
name: "PITR job missing volume",
pitr: pitr,
mariadb: &mariadbv1alpha1.MariaDB{},
- restoreOpts: []RestoreOpt{
+ restoreOpts: []PhysicalBackupRestoreOpt{
WithStartGtid(mustParseGtid(t, "0-10-1")),
WithBootstrapFrom(&mariadbv1alpha1.BootstrapFrom{
TargetRecoveryTime: &metav1.Time{Time: time.Now()},
@@ -3453,7 +3453,7 @@ func TestBuildPITRJob(t *testing.T) {
name: "PITR job missing volume",
pitr: pitr,
mariadb: &mariadbv1alpha1.MariaDB{},
- restoreOpts: []RestoreOpt{
+ restoreOpts: []PhysicalBackupRestoreOpt{
WithStartGtid(mustParseGtid(t, "0-10-1")),
WithBootstrapFrom(&mariadbv1alpha1.BootstrapFrom{
TargetRecoveryTime: &metav1.Time{Time: time.Now()},
@@ -3466,7 +3466,7 @@ func TestBuildPITRJob(t *testing.T) {
name: "Valid PITR job ",
pitr: pitr,
mariadb: &mariadbv1alpha1.MariaDB{},
- restoreOpts: []RestoreOpt{
+ restoreOpts: []PhysicalBackupRestoreOpt{
WithStartGtid(startGtid),
WithBootstrapFrom(&mariadbv1alpha1.BootstrapFrom{
TargetRecoveryTime: targetRecoveryTime,
@@ -3497,7 +3497,7 @@ func TestBuildPITRJob(t *testing.T) {
},
},
},
- restoreOpts: []RestoreOpt{
+ restoreOpts: []PhysicalBackupRestoreOpt{
WithStartGtid(startGtid),
WithBootstrapFrom(&mariadbv1alpha1.BootstrapFrom{
TargetRecoveryTime: targetRecoveryTime,
@@ -3532,7 +3532,7 @@ func TestBuildPITRJob(t *testing.T) {
name: "Valid PITR job with affinity",
pitr: pitr,
mariadb: &mariadbv1alpha1.MariaDB{},
- restoreOpts: []RestoreOpt{
+ restoreOpts: []PhysicalBackupRestoreOpt{
WithStartGtid(startGtid),
WithBootstrapFrom(&mariadbv1alpha1.BootstrapFrom{
TargetRecoveryTime: targetRecoveryTime,
diff --git a/pkg/builder/container_builder.go b/pkg/builder/container_builder.go
index 36249d213c..696e957513 100644
--- a/pkg/builder/container_builder.go
+++ b/pkg/builder/container_builder.go
@@ -7,6 +7,7 @@ import (
"path/filepath"
"reflect"
"strconv"
+ "strings"
mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
agentresources "github.com/mariadb-operator/mariadb-operator/v26/pkg/agent/resources"
@@ -511,9 +512,9 @@ func mariadbEnv(mariadb *mariadbv1alpha1.MariaDB) ([]corev1.EnvVar, error) {
}
if mariadb.IsReplicationEnabled() {
- replEnv, err := mariadbReplEnv(mariadb)
+ replEnv, err := replicationEnv(mariadb)
if err != nil {
- return nil, fmt.Errorf("error getting MariaDB replication environment: %v", err)
+ return nil, err
}
env = append(env, replEnv...)
}
@@ -556,6 +557,31 @@ func mariadbEnv(mariadb *mariadbv1alpha1.MariaDB) ([]corev1.EnvVar, error) {
return env, nil
}
+// replicationEnv builds the replication-related environment variables for a MariaDB
+// with replication enabled.
+func replicationEnv(mariadb *mariadbv1alpha1.MariaDB) ([]corev1.EnvVar, error) {
+ replEnv, err := mariadbReplEnv(mariadb)
+ if err != nil {
+ return nil, fmt.Errorf("error getting MariaDB replication environment: %v", err)
+ }
+
+ var env []corev1.EnvVar
+ if mariadb.Replication().ReplicaFromExternal != nil {
+ env = append(env, externalReplEnvVars(mariadb)...)
+ }
+
+ replication := ptr.Deref(mariadb.Spec.Replication, mariadbv1alpha1.Replication{})
+
+ if replication.SyncBinlog != nil {
+ env = append(env, corev1.EnvVar{
+ Name: "MARIADB_REPL_SYNC_BINLOG",
+ Value: fmt.Sprintf("%d", *replication.SyncBinlog),
+ })
+ }
+ env = append(env, replEnv...)
+ return env, nil
+}
+
func mariadbReplEnv(mariadb *mariadbv1alpha1.MariaDB) ([]corev1.EnvVar, error) {
if !mariadb.IsReplicationEnabled() {
return nil, nil
@@ -617,6 +643,25 @@ func mariadbReplEnv(mariadb *mariadbv1alpha1.MariaDB) ([]corev1.EnvVar, error) {
return env, nil
}
+func externalReplEnvVars(mariadb *mariadbv1alpha1.MariaDB) []corev1.EnvVar {
+ ext := mariadb.Replication().ReplicaFromExternal
+ // Effective offset: manual spec value if set, otherwise the auto-discovered value persisted in
+ // status. It falls back to 0 as a safeguard; in practice the status discovery persists the offset
+ // before the StatefulSet is ever built, so a real value is always present here.
+ offset := ptr.Deref(mariadb.ExternalReplServerIdOffset(), 0)
+ env := []corev1.EnvVar{
+ {Name: "MARIADB_EXTERNAL_REPL_ENABLED", Value: fmt.Sprint(true)},
+ {Name: "MARIADB_EXTERNAL_REPL_SERVER_ID_OFFSET", Value: fmt.Sprint(offset)},
+ }
+ if ext.HasFilteredTables() {
+ env = append(env, corev1.EnvVar{
+ Name: "MARIADB_EXTERNAL_REPL_FILTERED_TABLES",
+ Value: strings.Join(ext.FilteredReplicaTables, ","),
+ })
+ }
+ return env
+}
+
func s3Env(s3 *mariadbv1alpha1.S3) []corev1.EnvVar {
if s3 == nil {
return nil
diff --git a/pkg/builder/restore_builder.go b/pkg/builder/restore_builder.go
index 11ee64721d..0a661a9266 100644
--- a/pkg/builder/restore_builder.go
+++ b/pkg/builder/restore_builder.go
@@ -2,6 +2,7 @@ package builder
import (
"fmt"
+ "strings"
mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
metadata "github.com/mariadb-operator/mariadb-operator/v26/pkg/builder/metadata"
@@ -10,7 +11,14 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)
-func (b *Builder) BuildRestore(mariadb *mariadbv1alpha1.MariaDB, key types.NamespacedName) (*mariadbv1alpha1.Restore, error) {
+type LogicalRestoreOpts struct {
+ PodIndex *int
+}
+
+// type CertOpt func(*CertOpts)
+
+func (b *Builder) BuildRestore(mariadb *mariadbv1alpha1.MariaDB, key types.NamespacedName,
+ opts LogicalRestoreOpts) (*mariadbv1alpha1.Restore, error) {
objMeta :=
metadata.NewMetadataBuilder(key).
WithMetadata(mariadb.Spec.InheritMetadata).
@@ -18,6 +26,15 @@ func (b *Builder) BuildRestore(mariadb *mariadbv1alpha1.MariaDB, key types.Names
bootstrapFrom := ptr.Deref(mariadb.Spec.BootstrapFrom, mariadbv1alpha1.BootstrapFrom{})
restoreJob := ptr.Deref(bootstrapFrom.RestoreJob, mariadbv1alpha1.Job{})
+ // External replication doesn't use mariadb.Spec.BootstrapFrom; the restore Job template lives under
+ // replication.replica.bootstrapFrom.restoreJob instead. Pull from there so resources/tolerations/etc.
+ // get applied to the per-replica restore Pods.
+ if mariadb.Replication().ReplicaFromExternal != nil {
+ if rbf := mariadb.Replication().Replica.ReplicaBootstrapFrom; rbf != nil && rbf.RestoreJob != nil {
+ restoreJob = *rbf.RestoreJob
+ }
+ }
+
podTpl := mariadbv1alpha1.JobPodTemplate{}
podTpl.FromPodTemplate(mariadb.Spec.MariaDBPodTemplate.DeepCopy())
podTpl.Affinity = restoreJob.Affinity
@@ -40,9 +57,20 @@ func (b *Builder) BuildRestore(mariadb *mariadbv1alpha1.MariaDB, key types.Names
containerTpl.Resources = restoreJob.Resources
containerTpl.Args = restoreJob.Args
- restoreSource, err := bootstrapFrom.RestoreSource()
- if err != nil {
- return nil, fmt.Errorf("error getting restore source: %v", err)
+ var restoreSource *mariadbv1alpha1.RestoreSource
+ var err error
+ if mariadb.Replication().ReplicaFromExternal == nil {
+ restoreSource, err = bootstrapFrom.RestoreSource()
+
+ if err != nil {
+ return nil, fmt.Errorf("error getting restore source: %v", err)
+ }
+ } else {
+ restoreSource = &mariadbv1alpha1.RestoreSource{
+ BackupRef: &mariadbv1alpha1.LocalObjectReference{
+ Name: mariadb.ExternalReplLogicalBackupName(),
+ },
+ }
}
restore := &mariadbv1alpha1.Restore{
@@ -59,9 +87,29 @@ func (b *Builder) BuildRestore(mariadb *mariadbv1alpha1.MariaDB, key types.Names
},
},
}
+
+ ext := mariadb.Replication().ReplicaFromExternal
+ if ext != nil && len(ext.FilteredReplicaTables) > 0 {
+ // Only set a default database when all filtered tables share a single schema.
+ // Multi-schema dumps include per-schema USE statements, so no default is needed.
+ schemas := make(map[string]struct{})
+ for _, t := range ext.FilteredReplicaTables {
+ if s, _, found := strings.Cut(t, "."); found {
+ schemas[s] = struct{}{}
+ }
+ }
+ if len(schemas) == 1 {
+ if db, _, found := strings.Cut(ext.FilteredReplicaTables[0], "."); found {
+ restore.Spec.Database = db
+ }
+ }
+ }
if restoreJob.Metadata != nil {
restore.Spec.InheritMetadata = restoreJob.Metadata
}
+ if opts.PodIndex != nil {
+ restore.Spec.PodIndex = opts.PodIndex
+ }
if err := controllerutil.SetControllerReference(mariadb, restore, b.scheme); err != nil {
return nil, fmt.Errorf("error setting controller reference to restore Job: %v", err)
diff --git a/pkg/builder/restore_builder_test.go b/pkg/builder/restore_builder_test.go
index 37aa5a80c0..12a5147784 100644
--- a/pkg/builder/restore_builder_test.go
+++ b/pkg/builder/restore_builder_test.go
@@ -21,6 +21,7 @@ func TestRestoreMeta(t *testing.T) {
mariadb *mariadbv1alpha1.MariaDB
wantRestoreMeta *mariadbv1alpha1.Metadata
wantPodMeta *mariadbv1alpha1.Metadata
+ restoreOpts LogicalRestoreOpts
}{
{
name: "no meta",
@@ -33,6 +34,7 @@ func TestRestoreMeta(t *testing.T) {
Labels: map[string]string{},
Annotations: map[string]string{},
},
+ restoreOpts: LogicalRestoreOpts{},
},
{
name: "inherit meta",
@@ -64,6 +66,7 @@ func TestRestoreMeta(t *testing.T) {
"database.myorg.io": "mariadb",
},
},
+ restoreOpts: LogicalRestoreOpts{},
},
{
name: "pod meta",
@@ -95,6 +98,7 @@ func TestRestoreMeta(t *testing.T) {
"database.myorg.io": "job",
},
},
+ restoreOpts: LogicalRestoreOpts{},
},
{
name: "all",
@@ -138,12 +142,13 @@ func TestRestoreMeta(t *testing.T) {
"database.myorg.io": "job",
},
},
+ restoreOpts: LogicalRestoreOpts{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- restore, err := builder.BuildRestore(tt.mariadb, key)
+ restore, err := builder.BuildRestore(tt.mariadb, key, tt.restoreOpts)
if err != nil {
t.Fatalf("unexpected error building Restore: %v", err)
}
@@ -187,8 +192,8 @@ func TestBuildRestore(t *testing.T) {
key := types.NamespacedName{
Name: "test-restore",
}
-
- restore, err := builder.BuildRestore(mariadb, key)
+ restoreOpts := LogicalRestoreOpts{}
+ restore, err := builder.BuildRestore(mariadb, key, restoreOpts)
if err != nil {
t.Errorf("unexpected error building Restore: %v", err)
}
diff --git a/pkg/command/backup.go b/pkg/command/backup.go
index bcfa8c1534..21ba730802 100644
--- a/pkg/command/backup.go
+++ b/pkg/command/backup.go
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"path/filepath"
+ "sort"
"strings"
"time"
@@ -20,20 +21,21 @@ import (
type BackupOpts struct {
CommandOpts
- Path string
- TargetFilePath string
- BackupFullDirPath string
- BackupContentType mariadbv1alpha1.BackupContentType
- PhysicalBackupMeta bool
- PhysicalBackupKey *types.NamespacedName
- OmitCredentials bool
- CleanupTargetFile bool
- MaxRetentionDuration time.Duration
- StartGtid *replication.Gtid
- TargetTime time.Time
- Compression mariadbv1alpha1.CompressAlgorithm
- LogLevel string
- ExtraOpts []string
+ Path string
+ TargetFilePath string
+ BackupFullDirPath string
+ BackupContentType mariadbv1alpha1.BackupContentType
+ PhysicalBackupMeta bool
+ PhysicalBackupKey *types.NamespacedName
+ OmitCredentials bool
+ CleanupTargetFile bool
+ MaxRetentionDuration time.Duration
+ StartGtid *replication.Gtid
+ TargetTime time.Time
+ TargetTimeAgeThreshold *time.Time
+ Compression mariadbv1alpha1.CompressAlgorithm
+ LogLevel string
+ ExtraOpts []string
S3 bool
S3Bucket string
@@ -110,6 +112,12 @@ func WithCompression(c mariadbv1alpha1.CompressAlgorithm) BackupOpt {
}
}
+func WithBackupTargetTimeAgeThreshold(threshold *time.Time) BackupOpt {
+ return func(bo *BackupOpts) {
+ bo.TargetTimeAgeThreshold = threshold
+ }
+}
+
func WithS3(bucket, endpoint, region, prefix string) BackupOpt {
return func(bo *BackupOpts) {
bo.S3 = true
@@ -224,9 +232,11 @@ func (b *BackupCommand) MariadbDump(backup *mariadbv1alpha1.Backup,
if err != nil {
return nil, fmt.Errorf("error getting connection flags: %v", err)
}
- dumpArgs := strings.Join(b.mariadbDumpArgs(backup, mariadb), " ")
+ args := strings.Join(b.mariadbDumpArgs(backup, mariadb), " ")
+ tablesBySchema := groupTablesBySchema(backup.Spec.Tables)
+ isMultiSchema := len(tablesBySchema) > 1
- args := []string{
+ cmds := []string{
"set -euo pipefail",
"echo 💾 Exporting env",
fmt.Sprintf(
@@ -241,6 +251,25 @@ func (b *BackupCommand) MariadbDump(backup *mariadbv1alpha1.Backup,
"printf \"${BACKUP_FILE}\" > %s",
b.TargetFilePath,
),
+ }
+
+ dumpArgs := args
+ if isMultiSchema {
+ // mapfile reads each row as a separate array element so identifiers with spaces
+ // (e.g. `lerg`.`LERG 6 ATC`) survive intact; "${MARIADB_IGNORE_ARGS[@]}" expands
+ // each element as one shell word rather than word-splitting on whitespace.
+ cmds = append(cmds,
+ "echo 💾 Building ignore-table flags",
+ fmt.Sprintf(
+ `mapfile -t MARIADB_IGNORE_ARGS < <(mariadb %s -BNe "%s")`,
+ connFlags,
+ buildIgnoreTableQuery(tablesBySchema),
+ ),
+ )
+ dumpArgs = args + ` "${MARIADB_IGNORE_ARGS[@]}"`
+ }
+
+ cmds = append(cmds,
fmt.Sprintf(
"echo 💾 Taking backup: %s",
b.getTargetFilePath(),
@@ -251,8 +280,8 @@ func (b *BackupCommand) MariadbDump(backup *mariadbv1alpha1.Backup,
dumpArgs,
b.getTargetFilePath(),
),
- }
- return NewBashCommand(args), nil
+ )
+ return NewBashCommand(cmds), nil
}
func (b *BackupCommand) MariadbBackup(mariadb *mariadbv1alpha1.MariaDB, backupFilePath string,
@@ -374,6 +403,14 @@ func (b *BackupCommand) MariadbOperatorRestore() (*Command, error) {
"--backup-content-type",
string(b.BackupContentType),
}
+
+ if b.TargetTimeAgeThreshold != nil {
+ args = append(args, []string{
+ "--target-time-age-threshold",
+ backuppkg.FormatBackupDate(*b.TargetTimeAgeThreshold),
+ }...)
+ }
+
if b.LogLevel != "" {
args = append(args, []string{
"--log-level",
@@ -390,7 +427,14 @@ func (b *BackupCommand) MariadbOperatorRestore() (*Command, error) {
func (b *BackupCommand) MariadbRestore(restore *mariadbv1alpha1.Restore,
mariadb interfaces.MariaDBObject) (*Command, error) {
- connFlags, err := ConnectionFlags(&b.CommandOpts, mariadb)
+
+ var err error
+ var connFlags string
+ if restore.Spec.PodIndex != nil {
+ connFlags, err = PodConnectionFlags(&b.CommandOpts, mariadb, *restore.Spec.PodIndex)
+ } else {
+ connFlags, err = ConnectionFlags(&b.CommandOpts, mariadb)
+ }
if err != nil {
return nil, fmt.Errorf("error getting connection flags: %v", err)
}
@@ -402,13 +446,20 @@ func (b *BackupCommand) MariadbRestore(restore *mariadbv1alpha1.Restore,
"echo 💾 Restoring backup: %s",
b.getTargetFilePath(),
),
- fmt.Sprintf(
- "mariadb %s %s < %s",
- connFlags,
- args,
- b.getTargetFilePath(),
- ),
}
+ if restore.Spec.Database != "" {
+ cmds = append(cmds, fmt.Sprintf(
+ "mariadb %s -e 'CREATE DATABASE IF NOT EXISTS `%s`;'",
+ connFlags,
+ restore.Spec.Database,
+ ))
+ }
+ cmds = append(cmds, fmt.Sprintf(
+ "mariadb %s %s < %s",
+ connFlags,
+ args,
+ b.getTargetFilePath(),
+ ))
return NewBashCommand(cmds), nil
}
@@ -565,6 +616,24 @@ func (b *BackupCommand) mariadbDumpArgs(backup *mariadbv1alpha1.Backup, mariadb
if hasDatabases {
dumpOpts = ds.Remove(dumpOpts, hasDatabasesOpt)
}
+ } else if len(backup.Spec.Tables) > 0 {
+ tablesBySchema := groupTablesBySchema(backup.Spec.Tables)
+ if len(tablesBySchema) > 1 {
+ // Multi-schema: list all target databases; per-table filtering is applied at
+ // runtime via --ignore-table flags built by querying information_schema.
+ schemas := make([]string, 0, len(tablesBySchema))
+ for s := range tablesBySchema {
+ schemas = append(schemas, s)
+ }
+ sort.Strings(schemas)
+ args = append(args, "--databases")
+ args = append(args, schemas...)
+ } else {
+ // Single schema: --databases db --tables tbl1 tbl2 includes CREATE DATABASE / USE
+ // statements (the plain positional form does not), required for a clean restore.
+ db, _ := tableSelectionArgs(backup.Spec.Tables)
+ args = append(args, "--databases", db)
+ }
} else if !hasDatabases {
args = append(args, "--all-databases")
}
@@ -586,7 +655,83 @@ func (b *BackupCommand) mariadbDumpArgs(backup *mariadbv1alpha1.Backup, mariadb
args = append(args, b.tlsArgs(mariadb)...)
}
- return ds.UniqueArgs(ds.Merge(args, dumpOpts)...)
+ result := ds.UniqueArgs(ds.Merge(args, dumpOpts)...)
+
+ // --tables must come after all other flags; it overrides --databases to limit
+ // which tables are dumped while still emitting the database context statements.
+ // Not used for multi-schema: ignore-table flags are injected at runtime instead.
+ if len(backup.Spec.Tables) > 0 && len(groupTablesBySchema(backup.Spec.Tables)) == 1 {
+ _, tables := tableSelectionArgs(backup.Spec.Tables)
+ result = append(result, "--tables")
+ result = append(result, tables...)
+ }
+
+ return result
+}
+
+// groupTablesBySchema groups "db.table" entries into a map of schema → []table.
+func groupTablesBySchema(tables []string) map[string][]string {
+ result := make(map[string][]string)
+ for _, t := range tables {
+ schema, table, found := strings.Cut(t, ".")
+ if !found {
+ continue
+ }
+ result[schema] = append(result[schema], table)
+ }
+ return result
+}
+
+// buildIgnoreTableQuery returns a SQL query that emits one "--ignore-table=schema.table"
+// token per row for every BASE TABLE or VIEW in the given schemas that is NOT in
+// tablesBySchema. Views are included so the dump doesn't try to recreate views that
+// reference tables excluded from the filtered backup.
+func buildIgnoreTableQuery(tablesBySchema map[string][]string) string {
+ schemas := make([]string, 0, len(tablesBySchema))
+ for s := range tablesBySchema {
+ schemas = append(schemas, s)
+ }
+ sort.Strings(schemas)
+
+ quotedSchemas := make([]string, len(schemas))
+ for i, s := range schemas {
+ quotedSchemas[i] = "'" + s + "'"
+ }
+
+ var pairs []string
+ for _, s := range schemas {
+ for _, t := range tablesBySchema[s] {
+ pairs = append(pairs, fmt.Sprintf("('%s','%s')", s, t))
+ }
+ }
+
+ return fmt.Sprintf(
+ "SELECT CONCAT('--ignore-table=', TABLE_SCHEMA, '.', TABLE_NAME)"+
+ " FROM information_schema.TABLES"+
+ " WHERE TABLE_SCHEMA IN (%s)"+
+ " AND TABLE_TYPE IN ('BASE TABLE','VIEW')"+
+ " AND (TABLE_SCHEMA, TABLE_NAME) NOT IN (%s)",
+ strings.Join(quotedSchemas, ","),
+ strings.Join(pairs, ","),
+ )
+}
+
+// tableSelectionArgs parses "db.table" entries into a database name and table list.
+// All entries are expected to share the same database.
+func tableSelectionArgs(tables []string) (string, []string) {
+ var db string
+ var tableNames []string
+ for _, t := range tables {
+ d, tbl, found := strings.Cut(t, ".")
+ if !found {
+ continue
+ }
+ if db == "" {
+ db = d
+ }
+ tableNames = append(tableNames, tbl)
+ }
+ return db, tableNames
}
func (b *BackupCommand) mariadbBinlogArgs(mariadb *mariadbv1alpha1.MariaDB) ([]string, error) {
@@ -650,11 +795,20 @@ func (b *BackupCommand) mariadbBackupArgs(mariadb *mariadbv1alpha1.MariaDB, targ
return ds.UniqueArgs(ds.Merge(args, backupOpts)...)
}
-func (b *BackupCommand) mariadbRestoreArgs(restore *mariadbv1alpha1.Restore, mariadb interfaces.TLSProvider) []string {
+func (b *BackupCommand) mariadbRestoreArgs(restore *mariadbv1alpha1.Restore, mariadb interfaces.MariaDBObject) []string {
args := b.mariadbArgs(mariadb)
if restore.Spec.Database != "" {
- args = append(args, fmt.Sprintf("--one-database %s", restore.Spec.Database))
+ repl := mariadb.Replication()
+ isFilteredReplication := repl.ReplicaFromExternal != nil &&
+ len(repl.ReplicaFromExternal.FilteredReplicaTables) > 0
+ if isFilteredReplication {
+ // Filtered-table dumps have no USE statements; --database sets the connection
+ // default database upfront so every statement runs in the right context.
+ args = append(args, fmt.Sprintf("--database %s", restore.Spec.Database))
+ } else {
+ args = append(args, fmt.Sprintf("--one-database %s", restore.Spec.Database))
+ }
}
return ds.UniqueArgs(args...)
diff --git a/pkg/command/backup_test.go b/pkg/command/backup_test.go
index d947340056..e2930317b8 100644
--- a/pkg/command/backup_test.go
+++ b/pkg/command/backup_test.go
@@ -353,6 +353,52 @@ func TestMariadbDumpArgs(t *testing.T) {
"--add-drop-table",
},
},
+ {
+ name: "single-schema tables",
+ backupCmd: &BackupCommand{},
+ backup: &mariadbv1alpha1.Backup{
+ Spec: mariadbv1alpha1.BackupSpec{
+ Tables: []string{
+ "mydb.tbl1",
+ "mydb.tbl2",
+ },
+ },
+ },
+ mariadb: &mariadbv1alpha1.MariaDB{},
+ wantArgs: []string{
+ "--single-transaction",
+ "--events",
+ "--routines",
+ "--databases",
+ "mydb",
+ "--tables",
+ "tbl1",
+ "tbl2",
+ },
+ },
+ {
+ name: "multi-schema tables",
+ backupCmd: &BackupCommand{},
+ backup: &mariadbv1alpha1.Backup{
+ Spec: mariadbv1alpha1.BackupSpec{
+ Tables: []string{
+ "schema1.tbl1",
+ "schema1.tbl2",
+ "schema2.tbl3",
+ },
+ },
+ },
+ mariadb: &mariadbv1alpha1.MariaDB{},
+ // --tables is omitted; --ignore-table flags are built at runtime.
+ wantArgs: []string{
+ "--single-transaction",
+ "--events",
+ "--routines",
+ "--databases",
+ "schema1",
+ "schema2",
+ },
+ },
}
for _, tt := range tests {
diff --git a/pkg/command/command.go b/pkg/command/command.go
index a95338cffd..59f1889517 100644
--- a/pkg/command/command.go
+++ b/pkg/command/command.go
@@ -78,3 +78,24 @@ func host(mariadb interfaces.Connector, opts *ConnectionFlagsOpts) string {
}
return mariadb.GetHost()
}
+func PodConnectionFlags(co *CommandOpts, mariadb interfaces.Connector, podIndex int) (string, error) {
+
+ if co.UserEnv == "" {
+ return "", errors.New("UserEnv must be set")
+ }
+ if co.PasswordEnv == "" {
+ return "", errors.New("PasswordEnv must be set")
+ }
+
+ flags := fmt.Sprintf(
+ "--user=${%s} --password=${%s} --host=%s --port=%d",
+ co.UserEnv,
+ co.PasswordEnv,
+ mariadb.GetPodHost(podIndex),
+ mariadb.GetPort(),
+ )
+ if co.Database != nil {
+ flags += fmt.Sprintf(" --database=%s", *co.Database)
+ }
+ return flags, nil
+}
diff --git a/pkg/condition/condition.go b/pkg/condition/condition.go
index c909d056e7..c458b144c7 100644
--- a/pkg/condition/condition.go
+++ b/pkg/condition/condition.go
@@ -83,7 +83,7 @@ func (p *Complete) PatcherFailed(msg string) Patcher {
func (p *Complete) PatcherWithCronJob(ctx context.Context, err error, key types.NamespacedName) (Patcher, error) {
if err != nil {
return func(c Conditioner) {
- SetCompleteFailedWithMessage(c, "Error creating CronJob")
+ SetCompleteFailedWithMessage(c, fmt.Sprintf("Error creating CronJob %v", err))
}, nil
}
diff --git a/pkg/condition/external_repl_initialized.go b/pkg/condition/external_repl_initialized.go
new file mode 100644
index 0000000000..725d4261a9
--- /dev/null
+++ b/pkg/condition/external_repl_initialized.go
@@ -0,0 +1,24 @@
+package conditions
+
+import (
+ mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func SetExternalReplInitialized(c Conditioner) {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeExternalReplInitialized,
+ Status: metav1.ConditionTrue,
+ Reason: mariadbv1alpha1.ConditionReasonExternalReplInitialized,
+ Message: "External replication initialized",
+ })
+}
+
+func SetExternalReplInitializing(c Conditioner) {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeExternalReplInitialized,
+ Status: metav1.ConditionFalse,
+ Reason: mariadbv1alpha1.ConditionReasonExternalReplInitialized,
+ Message: "External replication initializing",
+ })
+}
diff --git a/pkg/condition/ready.go b/pkg/condition/ready.go
index dc27ab911c..848161daa2 100644
--- a/pkg/condition/ready.go
+++ b/pkg/condition/ready.go
@@ -74,6 +74,67 @@ func SetReadyWithStatefulSet(c Conditioner, sts *appsv1.StatefulSet) {
}
func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1alpha1.MariaDB) {
+ if setReadyWithMariaDBPhase(c, mdb) {
+ return
+ }
+ if mdb.IsUpdating() {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeReady,
+ Status: metav1.ConditionFalse,
+ Reason: mariadbv1alpha1.ConditionReasonUpdating,
+ Message: "Updating",
+ })
+ return
+ }
+ if sts.Status.Replicas == 0 || sts.Status.ReadyReplicas != sts.Status.Replicas {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeReady,
+ Status: metav1.ConditionFalse,
+ Reason: mariadbv1alpha1.ConditionReasonStatefulSetNotReady,
+ Message: "Not ready",
+ })
+ return
+ }
+
+ if mdb.HasPendingUpdate() {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeReady,
+ Status: metav1.ConditionTrue,
+ Reason: mariadbv1alpha1.ConditionReasonPendingUpdate,
+ Message: "Pending update",
+ })
+ return
+ }
+
+ if mdb.IsMaintenanceModeEnabled() {
+ SetReadyWithMaintenance(c, mdb)
+ return
+ }
+ replication := mdb.Replication()
+ // Pending External Replication initialization
+ if replication.IsExternalReplication() && !mdb.IsExternalReplInitialing() && !mdb.IsExternalReplInitialized() {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeReady,
+ Status: metav1.ConditionFalse,
+ Reason: mariadbv1alpha1.ConditionReasonPendingExternalReplInitialization,
+ Message: "Pending external replication initialization",
+ })
+ return
+ }
+
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeReady,
+ Status: metav1.ConditionTrue,
+ Reason: mariadbv1alpha1.ConditionReasonStatefulSetReady,
+ Message: "Running",
+ })
+}
+
+// setReadyWithMariaDBPhase handles the in-progress lifecycle phases of a MariaDB
+// (initializing, point-in-time recovery, binlog replay, scaling out, replica recovery
+// and external replication initialization). It returns true when a Ready condition was
+// set, signaling the caller to stop further processing.
+func setReadyWithMariaDBPhase(c Conditioner, mdb *mariadbv1alpha1.MariaDB) bool {
if mdb.IsInitializing() || (mdb.IsGaleraEnabled() && mdb.IsGaleraInitializing()) {
if err := mdb.InitError(); err != nil {
c.SetCondition(metav1.Condition{
@@ -82,7 +143,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonInitError,
Message: err.Error(),
})
- return
+ return true
}
c.SetCondition(metav1.Condition{
Type: mariadbv1alpha1.ConditionTypeReady,
@@ -90,7 +151,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonInitializing,
Message: "Initializing",
})
- return
+ return true
}
if mdb.IsPointInTimeRecoveryEnabled() {
if err := mdb.ArchiveBinlogsError(); err != nil {
@@ -100,7 +161,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonArchiveBinlogsError,
Message: err.Error(),
})
- return
+ return true
}
}
if mdb.IsReplayingBinlogs() {
@@ -111,7 +172,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonReplayBinlogsError,
Message: err.Error(),
})
- return
+ return true
}
c.SetCondition(metav1.Condition{
Type: mariadbv1alpha1.ConditionTypeReady,
@@ -119,7 +180,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonReplayBinlogs,
Message: "Replaying binlogs",
})
- return
+ return true
}
if mdb.IsScalingOut() {
if err := mdb.ScalingOutError(); err != nil {
@@ -129,7 +190,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonScaleOutError,
Message: err.Error(),
})
- return
+ return true
}
c.SetCondition(metav1.Condition{
Type: mariadbv1alpha1.ConditionTypeReady,
@@ -137,7 +198,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonScalingOut,
Message: "Scaling out",
})
- return
+ return true
}
if mdb.IsRecoveringReplicas() {
if err := mdb.ReplicaRecoveryError(); err != nil {
@@ -147,7 +208,7 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonReplicaRecoverError,
Message: err.Error(),
})
- return
+ return true
}
c.SetCondition(metav1.Condition{
Type: mariadbv1alpha1.ConditionTypeReady,
@@ -155,48 +216,27 @@ func SetReadyWithMariaDB(c Conditioner, sts *appsv1.StatefulSet, mdb *mariadbv1a
Reason: mariadbv1alpha1.ConditionReasonReplicaRecovering,
Message: "Recovering replicas",
})
- return
+ return true
}
- if mdb.IsUpdating() {
+ if mdb.IsExternalReplInitialing() {
+ if err := mdb.ExternalReplInitError(); err != nil {
+ c.SetCondition(metav1.Condition{
+ Type: mariadbv1alpha1.ConditionTypeReady,
+ Status: metav1.ConditionFalse,
+ Reason: mariadbv1alpha1.ConditionReasonExternalReplInitError,
+ Message: err.Error(),
+ })
+ return true
+ }
c.SetCondition(metav1.Condition{
Type: mariadbv1alpha1.ConditionTypeReady,
Status: metav1.ConditionFalse,
- Reason: mariadbv1alpha1.ConditionReasonUpdating,
- Message: "Updating",
+ Reason: mariadbv1alpha1.ConditionReasonExternalReplInitializing,
+ Message: "Initializing external replication",
})
- return
+ return true
}
- if sts.Status.Replicas == 0 || sts.Status.ReadyReplicas != sts.Status.Replicas {
- c.SetCondition(metav1.Condition{
- Type: mariadbv1alpha1.ConditionTypeReady,
- Status: metav1.ConditionFalse,
- Reason: mariadbv1alpha1.ConditionReasonStatefulSetNotReady,
- Message: "Not ready",
- })
- return
- }
-
- if mdb.HasPendingUpdate() {
- c.SetCondition(metav1.Condition{
- Type: mariadbv1alpha1.ConditionTypeReady,
- Status: metav1.ConditionTrue,
- Reason: mariadbv1alpha1.ConditionReasonPendingUpdate,
- Message: "Pending update",
- })
- return
- }
-
- if mdb.IsMaintenanceModeEnabled() {
- SetReadyWithMaintenance(c, mdb)
- return
- }
-
- c.SetCondition(metav1.Condition{
- Type: mariadbv1alpha1.ConditionTypeReady,
- Status: metav1.ConditionTrue,
- Reason: mariadbv1alpha1.ConditionReasonStatefulSetReady,
- Message: "Running",
- })
+ return false
}
func SetReadyWithInitJob(c Conditioner, job *batchv1.Job) {
diff --git a/pkg/controller/endpoints/controller.go b/pkg/controller/endpoints/controller.go
index 70016e9a59..c3766c31cd 100644
--- a/pkg/controller/endpoints/controller.go
+++ b/pkg/controller/endpoints/controller.go
@@ -106,7 +106,11 @@ func (r *EndpointsReconciler) endpointSlice(ctx context.Context, key types.Names
}
endpoints := []discoveryv1.Endpoint{}
+
for _, pod := range pods {
+ if mariadb.Status.Replication != nil && mariadb.Status.Replication.Roles[pod.Name] == mariadbv1alpha1.ReplicationRoleUnknown {
+ continue
+ }
endpoint, err := buildEndpoint(&pod)
if err != nil {
logger.Info("error building Endpoint", "err", err)
diff --git a/pkg/controller/galera/init.go b/pkg/controller/galera/init.go
index a89ce98393..404c04b520 100644
--- a/pkg/controller/galera/init.go
+++ b/pkg/controller/galera/init.go
@@ -34,8 +34,19 @@ func (r *GaleraReconciler) ReconcileInit(ctx context.Context, mariadb *mariadbv1
if err != nil {
return ctrl.Result{}, fmt.Errorf("error listing PVCs: %v", err)
}
- if len(pvcs) > 0 {
- for _, p := range pvcs {
+ // Ignore PVCs that are being deleted. A terminating PVC (e.g. left over from a previous MariaDB with the
+ // same name that is still being cleaned up) remains in the Bound phase until its finalizers are released,
+ // but it does not mean the cluster has already been initialized. Treating it as such skips the init Job and
+ // sends a fresh cluster straight into recovery over empty data directories, which can never produce a
+ // bootstrap source and deadlocks.
+ storagePVCs := make([]corev1.PersistentVolumeClaim, 0, len(pvcs))
+ for _, p := range pvcs {
+ if p.DeletionTimestamp == nil {
+ storagePVCs = append(storagePVCs, p)
+ }
+ }
+ if len(storagePVCs) > 0 {
+ for _, p := range storagePVCs {
if p.Status.Phase != corev1.ClaimBound {
r.recorder.Eventf(mariadb, nil, corev1.EventTypeWarning, mariadbv1alpha1.ReasonGaleraPVCNotBound,
mariadbv1alpha1.ActionReconciling, "Unable to init Galera cluster: PVC \"%s\" in non Bound phase", p.Name)
diff --git a/pkg/controller/replication/config.go b/pkg/controller/replication/config.go
index a61c05259f..b9d195bddd 100644
--- a/pkg/controller/replication/config.go
+++ b/pkg/controller/replication/config.go
@@ -2,16 +2,43 @@ package replication
import (
"bytes"
+ "context"
"errors"
"fmt"
+
+ // "html/template"
"strconv"
"text/template"
+ mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/builder"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/controller/secret"
env "github.com/mariadb-operator/mariadb-operator/v26/pkg/environment"
+ "github.com/mariadb-operator/mariadb-operator/v26/pkg/refresolver"
"github.com/mariadb-operator/mariadb-operator/v26/pkg/statefulset"
+ "sigs.k8s.io/controller-runtime/pkg/client"
)
+type ReplicationConfigClient struct {
+ client.Client
+ builder *builder.Builder
+ refResolver *refresolver.RefResolver
+ secretReconciler *secret.SecretReconciler
+}
+
+func NewReplicationConfigClient(client client.Client, builder *builder.Builder,
+ secretReconciler *secret.SecretReconciler) *ReplicationConfigClient {
+ return &ReplicationConfigClient{
+ Client: client,
+ builder: builder,
+ refResolver: refresolver.New(client),
+ secretReconciler: secretReconciler,
+ }
+}
+
func NewReplicationConfig(env *env.PodEnvironment) ([]byte, error) {
+ var sId int
+
replEnabled, err := env.IsReplEnabled()
if err != nil {
return nil, fmt.Errorf("error checking if replication is enabled: %v", err)
@@ -35,19 +62,39 @@ func NewReplicationConfig(env *env.PodEnvironment) ([]byte, error) {
if err != nil {
return nil, fmt.Errorf("error getting semi-sync master timeout: %v", err)
}
- serverIDStartIndex, err := serverIDStartIndex(env.MariaDBReplServerIDStartIndex)
+ externalReplEnabled, err := env.IsExternalReplEnabled()
if err != nil {
- return nil, fmt.Errorf("error getting server ID start index: %v", err)
+ return nil, fmt.Errorf("error checking if external replication is enabled: %v", err)
}
- serverID, err := serverId(env.PodName, serverIDStartIndex)
+
+ externalReplServerIdOffset, err := env.ExternalReplServerIdOffset()
if err != nil {
- return nil, fmt.Errorf("error getting server ID: %v", err)
+ return nil, fmt.Errorf("error get serverId offset for external replication: %v", err)
}
+
+ if externalReplEnabled && externalReplServerIdOffset != nil {
+ sId, err = offsetServerId(env.PodName, *externalReplServerIdOffset)
+ if err != nil {
+ return nil, fmt.Errorf("error getting server_id with offset server ID: %v", err)
+ }
+ } else {
+ serverIDStartIndex, err := serverIDStartIndex(env.MariaDBReplServerIDStartIndex)
+ if err != nil {
+ return nil, fmt.Errorf("error getting server ID start index: %v", err)
+ }
+ sId, err = serverId(env.PodName, serverIDStartIndex)
+ if err != nil {
+ return nil, fmt.Errorf("error getting server ID: %v", err)
+ }
+ }
+
syncBinlog, err := env.ReplSyncBinlog()
if err != nil {
return nil, fmt.Errorf("error getting master sync binlog: %v", err)
}
+ filteredTables := env.ExternalReplFilteredTables()
+
// To facilitate switchover/failover and avoid clashing with MaxScale, this configuration allows any Pod to act either as a primary or a replica.
// See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/semisynchronous-replication#enabling-semisynchronous-replication
tpl := createTpl("replication", `[mariadb]
@@ -73,6 +120,9 @@ server_id={{ .ServerID }}
{{- with .SyncBinlog }}
sync_binlog={{ . }}
{{- end }}
+{{- range .ReplicateDoTables }}
+replicate_do_table={{ . }}
+{{- end }}
`)
buf := new(bytes.Buffer)
err = tpl.Execute(buf, struct {
@@ -84,6 +134,7 @@ sync_binlog={{ . }}
SemiSyncMasterWaitPoint string
SyncBinlog *int
ServerID int
+ ReplicateDoTables []string
}{
LogName: env.MariadbName,
GtidStrictMode: gtidStrictMode,
@@ -91,8 +142,9 @@ sync_binlog={{ . }}
SemiSyncEnabled: semiSyncEnabled,
SemiSyncMasterTimeout: semiSyncMasterTimeout,
SemiSyncMasterWaitPoint: env.MariaDBReplSemiSyncMasterWaitPoint,
- ServerID: serverID,
+ ServerID: sId,
SyncBinlog: syncBinlog,
+ ReplicateDoTables: filteredTables,
})
if err != nil {
return nil, err
@@ -130,6 +182,35 @@ func serverId(podName string, startIndex int) (int, error) {
return startIndex + *podIndex, nil
}
+func externalReplPasswordRef(mariadb *mariadbv1alpha1.MariaDB, r *refresolver.RefResolver,
+ ctx context.Context) (mariadbv1alpha1.SecretKeySelector, error) {
+ replication := mariadb.Replication()
+ // if mariadb.Replication().Enabled && mariadb.Replication().Replica.ReplPasswordSecretKeyRef != nil {
+ // return mariadb.Replication().Replica.ReplPasswordSecretKeyRef.SecretKeySelector, nil
+ // }
+ if replication.IsExternalReplication() {
+ emdbRef := replication.GetExternalReplicationRef()
+ emdb, err := r.ExternalMariaDB(ctx, &emdbRef, mariadb.Namespace)
+ if err == nil {
+ return *emdb.GetSUCredential(), nil
+ }
+ }
+ return mariadbv1alpha1.SecretKeySelector{
+ LocalObjectReference: mariadbv1alpha1.LocalObjectReference{
+ Name: "",
+ },
+ Key: "",
+ }, fmt.Errorf("not able to get PasswordRef for external replication")
+}
+
+func offsetServerId(podName string, offset int) (int, error) {
+ podIndex, err := statefulset.PodIndex(podName)
+ if err != nil {
+ return 0, fmt.Errorf("error getting Pod index: %v", err)
+ }
+ return *podIndex + offset, nil
+}
+
func formatAccountName(username, host string) string {
return fmt.Sprintf("'%s'@'%s'", username, host)
}
diff --git a/pkg/controller/replication/controller.go b/pkg/controller/replication/controller.go
index f5173448ac..5b8fe8a323 100644
--- a/pkg/controller/replication/controller.go
+++ b/pkg/controller/replication/controller.go
@@ -3,6 +3,8 @@ package replication
import (
"context"
"fmt"
+ "strconv"
+ "strings"
"time"
"github.com/go-logr/logr"
@@ -152,6 +154,7 @@ func (r *ReplicationReconciler) reconcileReplication(ctx context.Context, req *R
if result, err := r.shouldReconcileReplication(ctx, req, logger); !result.IsZero() || err != nil {
return result, err
}
+
for _, i := range r.replicationPodIndexes(req) {
if result, err := r.ReconcileReplicationInPod(ctx, req, i, logger); !result.IsZero() || err != nil {
return result, err
@@ -169,12 +172,22 @@ func (r *ReplicationReconciler) reconcileReplication(ctx context.Context, req *R
func (r *ReplicationReconciler) shouldReconcileReplication(ctx context.Context, req *ReconcileRequest,
logger logr.Logger) (ctrl.Result, error) {
- if req.mariadb.Status.CurrentPrimaryPodIndex == nil {
+ replication := req.mariadb.Replication()
+ isExternalReplication := replication.IsExternalReplication()
+
+ if req.mariadb.Status.CurrentPrimaryPodIndex == nil && !isExternalReplication {
return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
}
+
+ if isExternalReplication && !req.mariadb.IsExternalReplInitialized() {
+ logger.Info("external replication no initialized, trying again in 5s")
+ return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+ }
+
if req.mariadb.IsSwitchingPrimary() {
return ctrl.Result{}, nil
}
+
if req.mariadb.IsMaxScaleEnabled() {
mxs, err := r.refResolver.MaxScale(ctx, req.mariadb.Spec.MaxScaleRef, req.mariadb.Namespace)
if err != nil {
@@ -235,8 +248,10 @@ func (r *ReplicationReconciler) ReconcileReplicationInPod(ctx context.Context, r
replRoles := replStatus.Roles
pod := statefulset.PodName(req.mariadb.ObjectMeta, podIndex)
topology := r.topologyManager.TopologyForMariaDB(req.mariadb, logger.WithValues("pod", pod))
+ replication := req.mariadb.Replication()
+ isExternalReplication := replication.IsExternalReplication()
- if primaryPodIndex == podIndex {
+ if primaryPodIndex == podIndex && !isExternalReplication {
if shouldSkipPrimaryReconciliation(req.mariadb, replRoles, pod, logger) {
return ctrl.Result{}, nil
}
@@ -251,10 +266,35 @@ func (r *ReplicationReconciler) ReconcileReplicationInPod(ctx context.Context, r
}
return ctrl.Result{}, nil
}
-
if !opts.forceReplicaConfiguration {
role, ok := replRoles[pod]
if ok && role == mariadbv1alpha1.ReplicationRoleReplica {
+
+ // If not external or is in recovery, we can skip configuration drift checks
+ if !isExternalReplication || req.mariadb.IsRecoveringReplicas() {
+ return ctrl.Result{}, nil
+ }
+ // For external replication the master connection details live in the ExternalMariaDB
+ // resource and may change over time. Detect drift and re-point the replica without
+ // resetting the master/GTID position.
+ client, err := req.replClientSet.clientForIndex(ctx, podIndex)
+ if err != nil {
+ logger.V(1).Info("error getting replica client", "err", err, "pod", pod)
+ return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+ }
+ defer client.Close()
+ // convert topology to externalReplicationTopology to access external-replication specific methods
+ externalReplicationTopology, ok := topology.(*externalReplicationTopology)
+ if !ok {
+ logger.Error(nil, "error converting topology to externalReplicationTopology", "pod", pod)
+ return ctrl.Result{}, fmt.Errorf("error converting topology to externalReplicationTopology: %v", err)
+ }
+
+ if _, err := r.ReconcileExternalReplicaDrift(ctx, externalReplicationTopology,
+ req.mariadb, client, primaryPodIndex, logger); err != nil {
+ logger.Error(err, "error reconciling external replica drift", "pod", pod)
+ return ctrl.Result{}, fmt.Errorf("error reconciling external replica drift: %v", err)
+ }
return ctrl.Result{}, nil
}
}
@@ -264,12 +304,16 @@ func (r *ReplicationReconciler) ReconcileReplicationInPod(ctx context.Context, r
logger.V(1).Info("error getting replica client", "err", err, "pod", pod)
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
+ defer client.Close()
+ logger.Info("Configuring replica", "pod", pod)
replicaOpts, err := r.getReplicaOpts(ctx, req, pod, podIndex, logger, reconcilePodOpts...)
if err != nil {
+ logger.Error(err, "error getting replica opts", "error", err, "pod", pod)
return ctrl.Result{}, fmt.Errorf("error getting replica opts: %v", err)
}
if err := topology.ConfigureReplica(ctx, client, primaryPodIndex, replicaOpts...); err != nil {
+ logger.Error(err, "error configuring replica")
return ctrl.Result{}, fmt.Errorf("error configuring replica: %v", err)
}
return ctrl.Result{}, nil
@@ -336,6 +380,71 @@ func (r *ReplicationReconciler) patchStatus(ctx context.Context, mariadb *mariad
return r.Status().Patch(ctx, mariadb, patch)
}
+// ReconcileExternalReplicaDrift re-points a replica that is already configured for external
+// replication at the current ExternalMariaDB connection details when it has drifted.
+//
+// It repairs in two cases:
+// - The configured master host, port or user no longer matches the ExternalMariaDB endpoint.
+// - The replica IO thread is failing with an authentication error. The configured password
+// cannot be read back, so re-issuing CHANGE MASTER (which always re-sends the current secret
+// value) is how a rotated replication password gets applied.
+//
+// Unlike ConfigureReplica, this performs a minimal, non-destructive repair: it does NOT reset the
+// master nor require a GTID position. It only stops the slave threads (when they are running),
+// issues CHANGE MASTER with the updated connection details (keeping MASTER_USE_GTID=current_pos)
+// and starts the slave again. It returns true when a repair was performed.
+func (r *ReplicationReconciler) ReconcileExternalReplicaDrift(ctx context.Context, topology *externalReplicationTopology,
+ mariadb *mariadbv1alpha1.MariaDB, client *sql.Client, primaryPodIndex int, logger logr.Logger) (bool, error) {
+ desiredHost, desiredPort, desiredUser, err := r.externalMasterEndpoint(ctx, mariadb)
+ if err != nil {
+ return false, fmt.Errorf("error getting external master endpoint: %v", err)
+ }
+
+ // Read SHOW REPLICA STATUS as a column map rather than via a positional scan so the check is
+ // resilient to column ordering changes across MariaDB versions.
+ status, err := client.QueryColumnMap(ctx, "SHOW REPLICA STATUS")
+ if err != nil {
+ return false, fmt.Errorf("error getting replica status: %v", err)
+ }
+ currentHost := status["Master_Host"]
+ currentPort := status["Master_Port"]
+ currentUser := status["Master_User"]
+ ioRunning := status["Slave_IO_Running"]
+ sqlRunning := status["Slave_SQL_Running"]
+
+ endpointDrift := currentHost != desiredHost ||
+ currentPort != strconv.Itoa(int(desiredPort)) ||
+ currentUser != desiredUser
+ authError := isReplicaAuthError(ioRunning, status["Last_IO_Errno"], status["Last_IO_Error"])
+
+ if !endpointDrift && !authError {
+ return false, nil
+ }
+
+ if endpointDrift {
+ logger.Info("external replica master drift detected, repairing",
+ "current-host", currentHost, "current-port", currentPort, "current-user", currentUser,
+ "desired-host", desiredHost, "desired-port", desiredPort, "desired-user", desiredUser)
+ }
+ if authError {
+ logger.Info("external replica authentication error detected, re-applying credentials",
+ "last-io-errno", status["Last_IO_Errno"], "last-io-error", status["Last_IO_Error"])
+ }
+
+ if ioRunning != "No" || sqlRunning != "No" {
+ if err := client.StopAllSlaves(ctx); err != nil {
+ return false, fmt.Errorf("error stopping slaves: %v", err)
+ }
+ }
+ if err := topology.changeMaster(ctx, client, primaryPodIndex); err != nil {
+ return false, fmt.Errorf("error changing master: %v", err)
+ }
+ if err := client.StartSlave(ctx); err != nil {
+ return false, fmt.Errorf("error starting slave: %v", err)
+ }
+ return true, nil
+}
+
func shouldSkipPrimaryReconciliation(mariadb *mariadbv1alpha1.MariaDB, replRoles map[string]mariadbv1alpha1.ReplicationRole,
pod string, logger logr.Logger) bool {
role, ok := replRoles[pod]
@@ -348,3 +457,43 @@ func shouldSkipPrimaryReconciliation(mariadb *mariadbv1alpha1.MariaDB, replRoles
}
return role == mariadbv1alpha1.ReplicationRolePrimary
}
+
+// isReplicaAuthError reports whether the replica IO thread is failing to authenticate against
+// the master. The password configured on the replica cannot be read back, so an access-denied
+// error is our only signal that a rotated password needs to be re-applied.
+func isReplicaAuthError(ioRunning, lastIOErrno, lastIOError string) bool {
+ if ioRunning == "Yes" {
+ return false
+ }
+ if errno, err := strconv.Atoi(lastIOErrno); err == nil {
+ if _, ok := replicaAuthErrnos[int32(errno)]; ok {
+ return true
+ }
+ }
+ return strings.Contains(strings.ToLower(lastIOError), "access denied")
+}
+
+// externalMasterEndpoint resolves the master connection details (host, port and user) that an
+// external replica should currently be replicating from, reading them from the referenced
+// ExternalMariaDB resource.
+func (r *ReplicationReconciler) externalMasterEndpoint(ctx context.Context,
+ mariadb *mariadbv1alpha1.MariaDB) (host string, port int32, user string, err error) {
+ replication := mariadb.Replication()
+ emdbRef := replication.GetExternalReplicationRef()
+ emdb, err := r.refResolver.ExternalMariaDB(ctx, &emdbRef, mariadb.Namespace)
+ if err != nil {
+ return "", 0, "", fmt.Errorf("error getting ExternalMariaDB: %v", err)
+ }
+ port = emdb.GetPort()
+ if emdb.GetBinlogProxyPort() != nil {
+ port = *emdb.GetBinlogProxyPort()
+ }
+ return emdb.GetHost(), port, emdb.GetSUName(), nil
+}
+
+// replicaAuthErrnos are the IO thread error codes MariaDB reports when the replica cannot
+// authenticate against the master, e.g. after the replication password has been rotated.
+var replicaAuthErrnos = map[int32]struct{}{
+ 1045: {}, // ER_ACCESS_DENIED_ERROR
+ 1698: {}, // ER_ACCESS_DENIED_NO_PASSWORD_ERROR
+}
diff --git a/pkg/controller/replication/switchover.go b/pkg/controller/replication/switchover.go
index ae5f8a0188..1f986433b2 100644
--- a/pkg/controller/replication/switchover.go
+++ b/pkg/controller/replication/switchover.go
@@ -31,10 +31,11 @@ func isSwitchoverStale(mdb *mariadbv1alpha1.MariaDB) bool {
}
func shouldReconcileSwitchover(mdb *mariadbv1alpha1.MariaDB) bool {
+ replication := mdb.Replication()
if mdb.IsMaxScaleEnabled() || mdb.IsRestoringBackup() || mdb.IsResizingStorage() {
return false
}
- if !mdb.HasConfiguredReplica() {
+ if !mdb.HasConfiguredReplica() || replication.IsExternalReplication() {
return false
}
return mdb.IsReplicationSwitchoverRequired()
@@ -367,6 +368,7 @@ func (r *ReplicationReconciler) connectReplicasToNewPrimary(ctx context.Context,
}
topology := r.topologyManager.TopologyForMariaDB(req.mariadb, logger.WithValues("replica", i))
+ logger.V(1).Info("Connecting replica to new primary", "replica", i)
if err := topology.ConfigureReplica(ctx, replClient, newPrimary, replicaOpts...); err != nil {
return fmt.Errorf("error configuring replica '%d': %v", i, err)
}
diff --git a/pkg/controller/replication/topology.go b/pkg/controller/replication/topology.go
index 8c5a478110..af1024490d 100644
--- a/pkg/controller/replication/topology.go
+++ b/pkg/controller/replication/topology.go
@@ -93,6 +93,18 @@ func (t *TopologyManager) TopologyForMariaDB(mariadb *mariadbv1alpha1.MariaDB, l
)
}
+ replication := mariadb.Replication()
+ if replication.Enabled && replication.IsExternalReplication() {
+ externalReplicationLogger := logger.WithName("external-replication")
+ externalReplicationLogger.V(1).Info("Configuring external replication topology")
+ return newExternalReplicationTopology(
+ mariadb,
+ t.Client,
+ t.refResolver,
+ externalReplicationLogger,
+ )
+ }
+
singleClusterLogger := logger.WithName("single-cluster")
singleClusterLogger.V(1).Info("Configuring single-cluster topology")
@@ -440,3 +452,103 @@ func (m *multiClusterTopology) configureSecondaryReplica(ctx context.Context, cl
}
return nil
}
+
+type externalReplicationTopology struct {
+ client.Client
+ mariadb *mariadbv1alpha1.MariaDB
+ refResolver *refresolver.RefResolver
+ logger logr.Logger
+}
+
+func newExternalReplicationTopology(mariadb *mariadbv1alpha1.MariaDB, client client.Client,
+ refResolver *refresolver.RefResolver, logger logr.Logger) *externalReplicationTopology {
+ return &externalReplicationTopology{
+ Client: client,
+ mariadb: mariadb,
+ refResolver: refResolver,
+ logger: logger,
+ }
+}
+
+func (r *externalReplicationTopology) ConfigureReplica(ctx context.Context, client *sql.Client,
+ primaryPodIndex int, replicaOpts ...ConfigureReplicaOpt) error {
+
+ opts := ConfigureReplicaOpts{}
+ for _, setOpt := range replicaOpts {
+ setOpt(&opts)
+ }
+
+ if err := client.ResetMaster(ctx); err != nil {
+ return fmt.Errorf("error resetting master: %v", err)
+ }
+ if err := client.StopAllSlaves(ctx); err != nil {
+ return fmt.Errorf("error stopping slaves: %v", err)
+ }
+ if opts.GtidSlavePos != nil {
+ if err := client.SetGtidSlavePos(ctx, *opts.GtidSlavePos); err != nil {
+ return fmt.Errorf("error setting slave position \"%s\": %v", *opts.GtidSlavePos, err)
+ }
+ } else if opts.ResetGtidSlavePos {
+ if err := client.ResetGtidSlavePos(ctx); err != nil {
+ return fmt.Errorf("error resetting slave position: %v", err)
+ }
+ }
+ if err := client.EnableReadOnly(ctx); err != nil {
+ return fmt.Errorf("error enabling read_only: %v", err)
+ }
+
+ if err := r.changeMaster(ctx, client, primaryPodIndex, opts.ChangeMasterOpts...); err != nil {
+ return fmt.Errorf("error changing master: %v", err)
+ }
+ if err := client.StartSlave(ctx); err != nil {
+ return fmt.Errorf("error starting slave: %v", err)
+ }
+ return nil
+}
+
+func (r *externalReplicationTopology) ConfigurePrimary(ctx context.Context, client *sql.Client) error {
+ // noop: in external replication topology, the operator does not manage the primary, so no configuration is needed
+ return nil
+}
+
+func (r *externalReplicationTopology) changeMaster(ctx context.Context, client *sql.Client,
+ primaryPodIndex int, opts ...sql.ChangeMasterOpt) error {
+ replication := ptr.Deref(r.mariadb.Spec.Replication, mariadbv1alpha1.Replication{})
+
+ if replication.Replica.ReplPasswordSecretKeyRef == nil {
+ return errors.New("'spec.replication.replica.replPasswordSecretKeyRef` must not be nil'")
+ }
+
+ var changeMasterOpts []sql.ChangeMasterOpt
+ var emdb *mariadbv1alpha1.ExternalMariaDB
+
+ replPasswordRef, err := externalReplPasswordRef(r.mariadb, r.refResolver, ctx)
+ if err != nil {
+ return fmt.Errorf("error getting ExternalMariaDB password Ref: %v", err)
+ }
+ password, err := r.refResolver.SecretKeyRef(ctx, replPasswordRef, r.mariadb.Namespace)
+ if err != nil {
+ return fmt.Errorf("error getting ExternalMariaDB password replication secret: %v", err)
+ }
+ emdbRef := replication.GetExternalReplicationRef()
+ emdb, err = r.refResolver.ExternalMariaDB(ctx, &emdbRef, r.mariadb.Namespace)
+ if err != nil {
+ return fmt.Errorf("error getting ExternalMariaDB: %v", err)
+ }
+ changeMasterOpts = []sql.ChangeMasterOpt{
+ sql.WithChangeMasterHost(
+ emdb.GetHost(),
+ ),
+ sql.WithChangeMasterCredentials(emdb.GetSUName(), password),
+ }
+ if emdb.GetBinlogProxyPort() != nil {
+ changeMasterOpts = append(changeMasterOpts, sql.WithChangeMasterPort(*emdb.GetBinlogProxyPort()))
+ } else {
+ changeMasterOpts = append(changeMasterOpts, sql.WithChangeMasterPort(emdb.GetPort()))
+ }
+
+ if err := client.ChangeMaster(ctx, changeMasterOpts...); err != nil {
+ return fmt.Errorf("error changing master: %v", err)
+ }
+ return nil
+}
diff --git a/pkg/controller/sql/controller.go b/pkg/controller/sql/controller.go
index cd62faf1fe..9511347bba 100644
--- a/pkg/controller/sql/controller.go
+++ b/pkg/controller/sql/controller.go
@@ -107,32 +107,58 @@ func (r *SqlReconciler) Reconcile(ctx context.Context, resource Resource) (ctrl.
return result, errBundle.ErrorOrNil()
}
- if mdb, ok := mariadb.(*mariadbv1alpha1.MariaDB); ok {
- if mdb.HasPendingBinlogReplay() {
- if err := r.WrappedReconciler.PatchStatus(ctx, r.ConditionReady.PatcherFailed("MariaDB has pending binlog replay")); err != nil {
- return ctrl.Result{}, fmt.Errorf("error patching status: %v", err)
+ var errBundle *multierror.Error
+ if (mariadb.IsHAEnabled() && mariadb.Replication().ReplicaFromExternal == nil) || !mariadb.IsHAEnabled() {
+
+ if mdb, ok := mariadb.(*mariadbv1alpha1.MariaDB); ok {
+ if mdb.HasPendingBinlogReplay() {
+ if err := r.WrappedReconciler.PatchStatus(ctx, r.ConditionReady.PatcherFailed("MariaDB has pending binlog replay")); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error patching status: %v", err)
+ }
+ return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
- return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
- }
- // TODO: connection pooling. See https://github.com/mariadb-operator/mariadb-operator/issues/7.
- mdbClient, err := sqlClient.NewClientWithMariaDB(ctx, mariadb, r.RefResolver)
- if err != nil {
- var errBundle *multierror.Error
- errBundle = multierror.Append(errBundle, err)
+ // TODO: connection pooling. See https://github.com/mariadb-operator/mariadb-operator/issues/7.
+ mdbClient, err := sqlClient.NewClientWithMariaDB(ctx, mariadb, r.RefResolver)
+ if err != nil {
+ var errBundle *multierror.Error
+ errBundle = multierror.Append(errBundle, err)
- msg := fmt.Sprintf("Error connecting to MariaDB: %v", err)
- err = r.WrappedReconciler.PatchStatus(ctx, r.ConditionReady.PatcherFailed(msg))
+ msg := fmt.Sprintf("Error connecting to MariaDB: %v", err)
+ err = r.WrappedReconciler.PatchStatus(ctx, r.ConditionReady.PatcherFailed(msg))
+ errBundle = multierror.Append(errBundle, err)
+
+ return r.retryResult(ctx, resource, errBundle)
+ }
+ defer mdbClient.Close()
+ err = r.WrappedReconciler.Reconcile(ctx, mdbClient)
errBundle = multierror.Append(errBundle, err)
+ } else {
+ for i := 0; i < int(mariadb.GetReplicas()); i++ {
+
+ mdbInternalClient, err := sqlClient.NewInternalClientWithPodIndex(ctx, mariadb, r.RefResolver, i, sqlClient.WithParams(
+ map[string]string{
+ "SQL_LOG_BIN": "OFF",
+ },
+ ))
+ if err != nil {
+ var errBundle *multierror.Error
+ errBundle = multierror.Append(errBundle, err)
+
+ msg := fmt.Sprintf("Error connecting to MariaDB: %v", err)
+ err = r.WrappedReconciler.PatchStatus(ctx, r.ConditionReady.PatcherFailed(msg))
+ errBundle = multierror.Append(errBundle, err)
+
+ return r.retryResult(ctx, resource, errBundle)
+ }
+ defer mdbInternalClient.Close()
+ err = r.WrappedReconciler.Reconcile(ctx, mdbInternalClient)
- return r.retryResult(ctx, resource, errBundle)
- }
- defer mdbClient.Close()
+ errBundle = multierror.Append(errBundle, err)
+ }
- err = r.WrappedReconciler.Reconcile(ctx, mdbClient)
- var errBundle *multierror.Error
- errBundle = multierror.Append(errBundle, err)
+ }
if err := errBundle.ErrorOrNil(); err != nil {
msg := fmt.Sprintf("Error creating %s: %v", resource.GetName(), err)
diff --git a/pkg/controller/sql/finalizer.go b/pkg/controller/sql/finalizer.go
index b9733dd1bb..29a788cc4e 100644
--- a/pkg/controller/sql/finalizer.go
+++ b/pkg/controller/sql/finalizer.go
@@ -77,21 +77,44 @@ func (tf *SqlFinalizer) Finalize(ctx context.Context, resource Resource) (ctrl.R
if result, err := waitForMariaDB(ctx, tf.Client, mariadb, tf.LogSql); !result.IsZero() || err != nil {
return result, err
}
+ if (mariadb.IsHAEnabled() && mariadb.Replication().ReplicaFromExternal == nil) || !mariadb.IsHAEnabled() {
+ // TODO: connection pooling. See https://github.com/mariadb-operator/mariadb-operator/issues/7.
+ mdbClient, err := sqlClient.NewClientWithMariaDB(ctx, mariadb, tf.RefResolver)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error connecting to MariaDB: %v", err)
+ }
+ defer mdbClient.Close()
- // TODO: connection pooling. See https://github.com/mariadb-operator/mariadb-operator/issues/7.
- mdbClient, err := sqlClient.NewClientWithMariaDB(ctx, mariadb, tf.RefResolver)
- if err != nil {
- return ctrl.Result{}, fmt.Errorf("error connecting to MariaDB: %v", err)
- }
- defer mdbClient.Close()
+ cleanupPolicy := ptr.Deref(resource.CleanupPolicy(), mariadbv1alpha1.CleanupPolicyDelete)
+ if cleanupPolicy == mariadbv1alpha1.CleanupPolicyDelete {
+ log.FromContext(ctx).Info("Cleaning up SQL resource")
- cleanupPolicy := ptr.Deref(resource.CleanupPolicy(), mariadbv1alpha1.CleanupPolicyDelete)
- if cleanupPolicy == mariadbv1alpha1.CleanupPolicyDelete {
- log.FromContext(ctx).Info("Cleaning up SQL resource")
+ if err := tf.WrappedFinalizer.Reconcile(ctx, mdbClient); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error reconciling in TemplateFinalizer: %v", err)
+ }
+ }
+ } else {
+ for i := 0; i < int(mariadb.GetReplicas()); i++ {
+ mdbInternalClient, err := sqlClient.NewInternalClientWithPodIndex(ctx, mariadb, tf.RefResolver, i, sqlClient.WithParams(
+ map[string]string{
+ "SQL_LOG_BIN": "OFF",
+ },
+ ))
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("error connecting to MariaDB: %v", err)
+ }
+ defer mdbInternalClient.Close()
+
+ cleanupPolicy := ptr.Deref(resource.CleanupPolicy(), mariadbv1alpha1.CleanupPolicyDelete)
+ if cleanupPolicy == mariadbv1alpha1.CleanupPolicyDelete {
+ log.FromContext(ctx).Info("Cleaning up SQL resource")
- if err := tf.WrappedFinalizer.Reconcile(ctx, mdbClient); err != nil {
- return ctrl.Result{}, fmt.Errorf("error reconciling in TemplateFinalizer: %v", err)
+ if err := tf.WrappedFinalizer.Reconcile(ctx, mdbInternalClient); err != nil {
+ return ctrl.Result{}, fmt.Errorf("error reconciling in TemplateFinalizer: %v", err)
+ }
+ }
}
+
}
if err := tf.WrappedFinalizer.RemoveFinalizer(ctx); err != nil {
diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go
index 5292da251c..f19b322cba 100644
--- a/pkg/environment/environment.go
+++ b/pkg/environment/environment.go
@@ -69,6 +69,9 @@ type PodEnvironment struct {
MariadbPort string `env:"MYSQL_TCP_PORT,required"`
MariaDBReplEnabled string `env:"MARIADB_REPL_ENABLED"`
+ MariaDBExternalReplEnabled string `env:"MARIADB_EXTERNAL_REPL_ENABLED"`
+ MariaDBExternalReplServerIdOffset string `env:"MARIADB_EXTERNAL_REPL_SERVER_ID_OFFSET"`
+ MariaDBExternalReplFilteredTables string `env:"MARIADB_EXTERNAL_REPL_FILTERED_TABLES"`
MariaDBReplGtidStrictMode string `env:"MARIADB_REPL_GTID_STRICT_MODE"`
MariaDBReplGtidDomainID string `env:"MARIADB_REPL_GTID_DOMAIN_ID"`
MariaDBReplServerIDStartIndex string `env:"MARIADB_REPL_SERVER_ID_START_INDEX"`
@@ -165,6 +168,41 @@ func (e *PodEnvironment) ReplSyncBinlog() (*int, error) {
return &timeout, nil
}
+func (e *PodEnvironment) IsExternalReplEnabled() (bool, error) {
+ replEnabled, err := e.IsReplEnabled()
+ if err != nil {
+ return false, err
+ }
+ if !replEnabled {
+ return false, errors.New("replication must be enabled")
+ }
+ if e.MariaDBExternalReplEnabled == "" {
+ return false, nil
+ }
+ return strconv.ParseBool(e.MariaDBExternalReplEnabled)
+}
+
+func (e *PodEnvironment) ExternalReplServerIdOffset() (*int, error) {
+ extReplEnabled, err := e.IsExternalReplEnabled()
+ if err != nil {
+ return nil, err
+ }
+ if !extReplEnabled {
+ return nil, nil
+ }
+ offset, err := strconv.Atoi(e.MariaDBExternalReplServerIdOffset)
+ return &offset, err
+}
+
+// ExternalReplFilteredTables returns the list of "database.table" entries to replicate,
+// or nil when filtered replication is not configured.
+func (e *PodEnvironment) ExternalReplFilteredTables() []string {
+ if e.MariaDBExternalReplFilteredTables == "" {
+ return nil
+ }
+ return strings.Split(e.MariaDBExternalReplFilteredTables, ",")
+}
+
func GetPodEnv(ctx context.Context) (*PodEnvironment, error) {
var env PodEnvironment
if err := envconfig.Process(ctx, &env); err != nil {
diff --git a/pkg/health/health.go b/pkg/health/health.go
index 5e275d0079..a9e6b03e4a 100644
--- a/pkg/health/health.go
+++ b/pkg/health/health.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"sort"
+ "time"
mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
labels "github.com/mariadb-operator/mariadb-operator/v26/pkg/builder/labels"
@@ -145,9 +146,25 @@ func secondaryPodHealthyIndex(ctx context.Context, client ctrlclient.Client, mar
if err != nil {
return nil, fmt.Errorf("error getting index for Pod '%s': %v", p.Name, err)
}
- if isHealthy(&p) {
- return index, nil
+
+ if mariadb.Status.Replication != nil && mariadb.Status.Replication.Replicas != nil {
+ IOStatusRunning := mariadb.Status.Replication.Replicas[p.Name].SlaveIORunning
+ SQLStatusRunning := mariadb.Status.Replication.Replicas[p.Name].SlaveSQLRunning
+ LastErrorTransitionTime := mariadb.Status.Replication.Replicas[p.Name].LastErrorTransitionTime
+
+ /*To avoid issues with false transients we'll only consider a replica healthy after 3 minutes of stability*/
+ stableMinDuration, _ := time.ParseDuration("120s")
+ // stableMinDuration, _ := time.ParseDuration("0s")
+
+ if isHealthy(&p) && *IOStatusRunning && *SQLStatusRunning && time.Since(LastErrorTransitionTime.Time) > stableMinDuration {
+ return index, nil
+ }
+ } else {
+ if isHealthy(&p) {
+ return index, nil
+ }
}
+
}
return nil, ErrNoHealthyInstancesAvailable
}
diff --git a/pkg/interfaces/interfaces.go b/pkg/interfaces/interfaces.go
index 680ccd829c..5f2f4e4ffb 100644
--- a/pkg/interfaces/interfaces.go
+++ b/pkg/interfaces/interfaces.go
@@ -8,6 +8,7 @@ import (
"github.com/mariadb-operator/mariadb-operator/v26/pkg/environment"
corev1 "k8s.io/api/core/v1"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
clientpkg "sigs.k8s.io/controller-runtime/pkg/client"
@@ -29,10 +30,17 @@ type TLSProvider interface {
type Replicator interface {
GetReplicas() int32
+ IsHAEnabled() bool
+ Replication() mariadbv1alpha1.Replication
+}
+
+type ServiceAwareInterface interface {
+ InternalServiceKey() types.NamespacedName
}
type Connector interface {
GetHost() string
+ GetPodHost(podIndex int) string
GetPort() int32
GetSUName() string
GetSUCredential() *mariadbv1alpha1.SecretKeySelector
@@ -49,8 +57,9 @@ type MariaDBObject interface {
GaleraProvider
Imager
Replicator
+ ServiceAwareInterface
TLSProvider
-
+ GetObjectMeta() *v1.ObjectMeta
IsReady() bool
}
diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go
index b581ef9871..9b6984ce1e 100644
--- a/pkg/pod/pod.go
+++ b/pkg/pod/pod.go
@@ -89,7 +89,7 @@ func ListMariaDBSecondaryPods(ctx context.Context, client ctrlclient.Client,
if err != nil {
return nil, fmt.Errorf("error getting Pod '%s' index: %v", p.Name, err)
}
- if *podIndex == *mariadb.Status.CurrentPrimaryPodIndex {
+ if *podIndex == *mariadb.Status.CurrentPrimaryPodIndex && mariadb.Replication().ReplicaFromExternal == nil {
continue
}
secondaryPods = append(secondaryPods, p)
diff --git a/pkg/sql/sql.go b/pkg/sql/sql.go
index e6bd625504..582d474f39 100644
--- a/pkg/sql/sql.go
+++ b/pkg/sql/sql.go
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"os"
+ "slices"
"strconv"
"strings"
"text/template"
@@ -48,10 +49,26 @@ type Opts struct {
TLSClientPrivateKey []byte
CustomTLSCAName string
+ MultiStatements bool
+
Params map[string]string
Timeout *time.Duration
}
+type ReplicaStatus struct {
+ MasterHost sql.NullString `mysql:"Master_Host"`
+ MasterUser sql.NullString `mysql:"Master_User"`
+ MasterPort sql.NullInt32 `mysql:"Master_Port"`
+ SlaveIORunning string `mysql:"Slave_IO_Running"`
+ SlaveSQLRunning string `mysql:"Slave_SQL_Running"`
+ LastError sql.NullString `mysql:"Last_Error"`
+ LastIOError sql.NullString `mysql:"Last_IO_Error"`
+ LastIOErrno sql.NullInt32 `mysql:"Last_IO_Errno"`
+ LastSQLError sql.NullString `mysql:"Last_SQL_Error"`
+ LastSQLErrno sql.NullInt32 `mysql:"Last_SQL_Errno"`
+ SecondsBehindMaster sql.NullInt64 `mysql:"Seconds_Behind_Master"`
+}
+
type Opt func(*Opts)
func WithUsername(username string) Opt {
@@ -127,6 +144,12 @@ func WithTimeout(d time.Duration) Opt {
}
}
+func WithMultiStatements(multiStatements bool) Opt {
+ return func(o *Opts) {
+ o.MultiStatements = multiStatements
+ }
+}
+
type Client struct {
db *sql.DB
}
@@ -208,12 +231,12 @@ func NewClientWithMariaDB(ctx context.Context, mariadb interfaces.MariaDBObject,
return NewClient(opts...)
}
-func NewInternalClientWithPodIndex(ctx context.Context, mariadb *mariadbv1alpha1.MariaDB, refResolver *refresolver.RefResolver,
+func NewInternalClientWithPodIndex(ctx context.Context, mariadb interfaces.MariaDBObject, refResolver *refresolver.RefResolver,
podIndex int, clientOpts ...Opt) (*Client, error) {
opts := []Opt{
WithHost(
statefulset.PodFQDNWithService(
- mariadb.ObjectMeta,
+ *mariadb.GetObjectMeta(),
podIndex,
mariadb.InternalServiceKey().Name,
),
@@ -276,6 +299,9 @@ func BuildDSN(opts Opts) (string, error) {
if opts.Params != nil {
config.Params = opts.Params
}
+ if opts.MultiStatements {
+ config.MultiStatements = opts.MultiStatements
+ }
if (opts.MariadbName != "" || opts.MaxscaleName != "" || opts.ExternalMariadbName != "") && opts.Namespace != "" && opts.TLSCACert != nil {
configName, err := configureTLS(opts)
if err != nil {
@@ -572,6 +598,53 @@ func (c *Client) UserExists(ctx context.Context, username, host string) (bool, e
return count > 0, nil
}
+func (c *Client) GrantExists(ctx context.Context,
+ privileges []string,
+ database string,
+ table string,
+ accountName string,
+ opts ...GrantOption) (bool, error) {
+
+ var privilege string
+ var current_privileges []string
+ var rows *sql.Rows
+ var err error
+
+ if table != "*" && database != "*" {
+ rows, err = c.db.QueryContext(ctx,
+ "SELECT PRIVILEGE_TYPE FROM information_schema.TABLE_PRIVILEGES where TABLE_NAME=? and TABLE_SCHEMA=? and GRANTEE=?",
+ table, database, accountName)
+ } else if database != "*" {
+ rows, err = c.db.QueryContext(ctx,
+ "SELECT PRIVILEGE_TYPE FROM information_schema.SCHEMA_PRIVILEGES where TABLE_SCHEMA=? and GRANTEE=?",
+ database,
+ accountName)
+ } else {
+ rows, err = c.db.QueryContext(ctx, "SELECT PRIVILEGE_TYPE FROM information_schema.USER_PRIVILEGES where GRANTEE=?", accountName)
+ }
+
+ if err != nil {
+ return false, fmt.Errorf("failing getting privileges %v", err)
+ }
+
+ for rows.Next() {
+ err := rows.Scan(&privilege)
+ if err != nil {
+ return false, fmt.Errorf("failing getting privileges %v", err)
+ }
+ current_privileges = append(current_privileges, privilege)
+ }
+
+ for _, priv := range privileges {
+ if !slices.Contains(current_privileges, priv) {
+ return false, nil
+ }
+ }
+
+ return true, nil
+
+}
+
type grantOpts struct {
grantOption bool
}
@@ -1012,6 +1085,62 @@ func (c Client) HasConnectedReplicas(ctx context.Context) (bool, error) {
return false, nil
}
+// ReplicationMasterEndpoint returns the host and port of the primary this server replicates from,
+// as reported by SHOW REPLICA STATUS. It returns ok=false when the server is not a replica (i.e. it
+// is itself a primary or a standalone server), which is signaled by an empty result set.
+func (c *Client) ReplicationMasterEndpoint(ctx context.Context) (host string, port int32, ok bool, err error) {
+ rows, err := c.QueryColumnMaps(ctx, "SHOW REPLICA STATUS")
+ if err != nil {
+ return "", 0, false, err
+ }
+ if len(rows) == 0 {
+ return "", 0, false, nil
+ }
+ masterHost := rows[0]["Master_Host"]
+ if masterHost == "" {
+ return "", 0, false, nil
+ }
+ masterPort, err := strconv.Atoi(rows[0]["Master_Port"])
+ if err != nil {
+ return "", 0, false, fmt.Errorf("error parsing Master_Port %q: %v", rows[0]["Master_Port"], err)
+ }
+ return masterHost, int32(masterPort), true, nil
+}
+
+// InUseServerIds returns the server ids in use in the topology as seen from this server: its own
+// server_id plus the server_id of every replica registered against it (SHOW SLAVE HOSTS). It must be
+// run against the primary to observe the full set of replicas.
+func (c *Client) InUseServerIds(ctx context.Context) ([]int, error) {
+ ids := make([]int, 0)
+
+ ownServerID, err := c.SystemVariable(ctx, "server_id")
+ if err != nil {
+ return nil, fmt.Errorf("error getting server_id: %v", err)
+ }
+ if id, err := strconv.Atoi(ownServerID); err == nil {
+ ids = append(ids, id)
+ }
+
+ rows, err := c.QueryColumnMaps(ctx, "SHOW SLAVE HOSTS")
+ if err != nil {
+ return nil, fmt.Errorf("error listing slave hosts: %v", err)
+ }
+ for _, row := range rows {
+ id, err := strconv.Atoi(row["Server_id"])
+ if err != nil {
+ continue
+ }
+ if !slices.Contains(ids, id) {
+ ids = append(ids, id)
+ }
+ }
+ return ids, nil
+}
+
+func (c *Client) SetSqlSlaveSkipCounter(ctx context.Context, count int) error {
+ return c.Exec(ctx, fmt.Sprintf("SET GLOBAL sql_slave_skip_counter = %d;", count))
+}
+
type ChangeMasterOpts struct {
ReplicationOpts
Host string
@@ -1080,7 +1209,10 @@ func (c *Client) ChangeMaster(ctx context.Context, changeMasterOpts ...ChangeMas
if err != nil {
return fmt.Errorf("error building CHANGE MASTER query: %v", err)
}
- return c.Exec(ctx, query)
+ if c.Exec(ctx, query) != nil {
+ return fmt.Errorf("error exec CHANGE MASTER query: %v", query)
+ }
+ return nil
}
func buildChangeMasterQuery(changeMasterOpts ...ChangeMasterOpt) (string, error) {
@@ -1089,7 +1221,7 @@ func buildChangeMasterQuery(changeMasterOpts ...ChangeMasterOpt) (string, error)
ConnectionName: "", // default connection name in MariaDB server, used in single-cluster topology
},
Port: 3306,
- Gtid: "CurrentPos",
+ Gtid: "current_pos",
}
for _, setOpt := range changeMasterOpts {
setOpt(&opts)
@@ -1111,6 +1243,8 @@ MASTER_SSL_CERT='{{ .SSLCertPath }}',
MASTER_SSL_KEY='{{ .SSLKeyPath }}',
MASTER_SSL_CA='{{ .SSLCAPath }}',
MASTER_SSL_VERIFY_SERVER_CERT=1,
+{{- else }}
+MASTER_SSL_VERIFY_SERVER_CERT=0,
{{- end }}
MASTER_HOST='{{ .Host }}',
MASTER_PORT={{ .Port }},
@@ -1129,6 +1263,132 @@ MASTER_USE_GTID={{ .Gtid }};
return buf.String(), nil
}
+func (c *Client) IsReplicationConfigured(ctx context.Context) (bool, string) {
+ sql := "SHOW REPLICA STATUS"
+ rows, err := c.db.QueryContext(ctx, sql)
+ if err != nil {
+ return false, err.Error()
+ }
+
+ count := 0
+ for rows.Next() {
+ count++
+ }
+
+ if count == 0 {
+ return false, ""
+ }
+
+ return true, ""
+}
+
+func (c *Client) GetReplicationStatus(ctx context.Context) (ReplicaStatus, error) {
+ query := "SHOW REPLICA STATUS"
+ row := c.db.QueryRowContext(ctx, query)
+
+ var status ReplicaStatus
+
+ // Scan the results into the struct fields. The order of fields must
+ // match the order of columns in the query result.
+ // You must list ALL columns, even if you don't use them.
+ // This is a limitation of SHOW REPLICA STATUS and rows.Scan().
+ var ignored any // Used to scan columns you don't care about.
+
+ err := row.Scan(
+ &ignored, // Slave_IO_State
+ &status.MasterHost,
+ &status.MasterUser,
+ &status.MasterPort,
+ &ignored, // Connect_Retry
+ &ignored, // Master_Log_File
+ &ignored, // Read_Master_Log_Pos
+ &ignored, // Relay_Log_File
+ &ignored, // Relay_Log_Pos
+ &ignored, // Relay_Master_Log_File
+ &status.SlaveIORunning,
+ &status.SlaveSQLRunning,
+ &ignored, // Replicate_Do_DB
+ &ignored, // Replicate_Ignore_DB
+ &ignored, // Replicate_Do_Table
+ &ignored, // Replicate_Ignore_Table
+ &ignored, // Replicate_Wild_Do_Table
+ &ignored, // Replicate_Wild_Ignore_Table
+ &ignored, // Last_Errno
+ &status.LastError, // Note: Last_Error in output, maps to LastSQLError in struct
+ &ignored, // Skip_Counter
+ &ignored, // Exec_Master_Log_Pos
+ &ignored, // Relay_Log_Space
+ &ignored, // Until_Condition
+ &ignored, // Until_Log_File
+ &ignored, // Until_Log_Pos
+ &ignored, // Master_SSL_Allowed
+ &ignored, // Master_SSL_CA_File
+ &ignored, // Master_SSL_CA_Path
+ &ignored, // Master_SSL_Cert
+ &ignored, // Master_SSL_Cipher
+ &ignored, // Master_SSL_Key
+ &status.SecondsBehindMaster,
+ &ignored, // Master_SSL_Verify_Server_Cert
+ &status.LastIOErrno, // Last_IO_Errno
+ &status.LastIOError, // Note: Last_IO_Error in output, maps to LastIOError in struct
+ &status.LastSQLErrno, // Last_SQL_Errno
+ &status.LastSQLError, // Last_SQL_Error
+ &ignored, // Replicate_Ignore_Server_Ids
+ &ignored, // Master_Server_Id
+ &ignored, // Master_SSL_Crl
+ &ignored, // Master_SSL_Crlpath
+ &ignored, // Using_Gtid
+ &ignored, // Gtid_IO_Pos
+ &ignored, // Replicate_Do_Domain_Ids
+ &ignored, // Replicate_Ignore_Domain_Ids
+ &ignored, // Parallel_Mode
+ &ignored, // SQL_Delay
+ &ignored, // SQL_Remaining_Delay
+ &ignored, // Slave_SQL_Running_State
+ &ignored, // Slave_DDL_Groups
+ &ignored, // Slave_Non_Transactional_Groups
+ &ignored, // Slave_Transactional_Groups
+ &ignored, // Replicate_Rewrite_DB
+ )
+
+ if err != nil {
+ // If no rows were returned, it means replication is not configured.
+ if err == sql.ErrNoRows {
+ return status, fmt.Errorf("replication is not configured: %w", err) // Return nil to indicate not configured.
+ }
+ // Otherwise, it's a real database error.
+ return status, fmt.Errorf("failed to scan replica status: %w", err)
+ }
+
+ return status, nil
+
+}
+
+func (c *Client) IsReplicationHealthy(ctx context.Context) (bool, error) {
+ status, error := c.GetReplicationStatus(ctx)
+
+ if error != nil {
+ return false, error
+ }
+
+ if status.SlaveIORunning == "Yes" && status.SlaveSQLRunning == "Yes" {
+ return true, nil
+ }
+
+ if (status.SlaveIORunning == "Preparing" || status.SlaveIORunning == "Connecting") &&
+ status.LastIOErrno.Int32 == 0 {
+ return true, nil
+ }
+
+ return false, nil
+
+}
+
+func (c *Client) ResetSlavePos(ctx context.Context) error {
+ sql := fmt.Sprintf("SET @@global.%s='';", "gtid_slave_pos")
+ return c.Exec(ctx, sql)
+}
+
const statusVariableSql = "SELECT variable_value FROM information_schema.global_status WHERE variable_name=?;"
func (c *Client) StatusVariable(ctx context.Context, variable string) (string, error) {
diff --git a/pkg/sql/sql_test.go b/pkg/sql/sql_test.go
index c13bb6dfda..88f9e9078a 100644
--- a/pkg/sql/sql_test.go
+++ b/pkg/sql/sql_test.go
@@ -42,6 +42,7 @@ func TestBuildChangeMasterQuery(t *testing.T) {
WithChangeMasterGtid("CurrentPos"),
},
wantQuery: `CHANGE MASTER TO
+MASTER_SSL_VERIFY_SERVER_CERT=0,
MASTER_HOST='127.0.0.1',
MASTER_PORT=3306,
MASTER_USER='repl',
@@ -94,6 +95,7 @@ MASTER_USE_GTID=CurrentPos;
WithChangeMasterRetries(10),
},
wantQuery: `CHANGE MASTER TO
+MASTER_SSL_VERIFY_SERVER_CERT=0,
MASTER_HOST='127.0.0.1',
MASTER_PORT=3306,
MASTER_USER='repl',
@@ -113,6 +115,7 @@ MASTER_USE_GTID=CurrentPos;
WithChangeMasterGtid("CurrentPos"),
},
wantQuery: `CHANGE MASTER 'replica' TO
+MASTER_SSL_VERIFY_SERVER_CERT=0,
MASTER_HOST='127.0.0.1',
MASTER_PORT=3306,
MASTER_USER='repl',