diff --git a/api/v1beta1/ansibletest_types.go b/api/v1beta1/ansibletest_types.go index 26143cd4..c24b5fa8 100644 --- a/api/v1beta1/ansibletest_types.go +++ b/api/v1beta1/ansibletest_types.go @@ -220,3 +220,8 @@ func (instance AnsibleTest) RbacNamespace() string { func (instance AnsibleTest) RbacResourceName() string { return instance.Name } + +// GetConditions - return the conditions from the status +func (instance *AnsibleTest) GetConditions() *condition.Conditions { + return &instance.Status.Conditions +} diff --git a/api/v1beta1/horizontest_types.go b/api/v1beta1/horizontest_types.go index 15a463ee..7be96d67 100644 --- a/api/v1beta1/horizontest_types.go +++ b/api/v1beta1/horizontest_types.go @@ -191,3 +191,8 @@ func (instance HorizonTest) RbacNamespace() string { func (instance HorizonTest) RbacResourceName() string { return instance.Name } + +// GetConditions - return the conditions from the status +func (instance *HorizonTest) GetConditions() *condition.Conditions { + return &instance.Status.Conditions +} diff --git a/api/v1beta1/tempest_types.go b/api/v1beta1/tempest_types.go index c77cf840..2dc36dc1 100644 --- a/api/v1beta1/tempest_types.go +++ b/api/v1beta1/tempest_types.go @@ -522,3 +522,8 @@ func (instance Tempest) RbacNamespace() string { func (instance Tempest) RbacResourceName() string { return instance.Name } + +// GetConditions - return the conditions from the status +func (instance *Tempest) GetConditions() *condition.Conditions { + return &instance.Status.Conditions +} diff --git a/api/v1beta1/tobiko_types.go b/api/v1beta1/tobiko_types.go index ceeae290..4b1aaf39 100644 --- a/api/v1beta1/tobiko_types.go +++ b/api/v1beta1/tobiko_types.go @@ -246,3 +246,8 @@ func (instance Tobiko) RbacNamespace() string { func (instance Tobiko) RbacResourceName() string { return instance.Name } + +// GetConditions - return the conditions from the status +func (instance *Tobiko) GetConditions() *condition.Conditions { + return &instance.Status.Conditions +} diff --git a/internal/controller/ansibletest_controller.go b/internal/controller/ansibletest_controller.go index 1269bf2d..3b440742 100644 --- a/internal/controller/ansibletest_controller.go +++ b/internal/controller/ansibletest_controller.go @@ -19,18 +19,13 @@ package controller import ( "context" - "fmt" - "strconv" "github.com/go-logr/logr" - "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/env" - "github.com/openstack-k8s-operators/lib-common/modules/common/helper" testv1beta1 "github.com/openstack-k8s-operators/test-operator/api/v1beta1" "github.com/openstack-k8s-operators/test-operator/internal/ansibletest" corev1 "k8s.io/api/core/v1" - k8s_errors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -56,222 +51,84 @@ func (r *AnsibleTestReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;create;update;watch;patch;delete // Reconcile - AnsibleTest -func (r *AnsibleTestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { - Log := r.GetLogger(ctx) - - // Fetch the ansible instance +func (r *AnsibleTestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { instance := &testv1beta1.AnsibleTest{} - err := r.Client.Get(ctx, req.NamespacedName, instance) - if err != nil { - if k8s_errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - // Create a helper - helper, err := helper.NewHelper( - instance, - r.Client, - r.Kclient, - r.Scheme, - r.Log, - ) - if err != nil { - return ctrl.Result{}, err - } - - // initialize status - isNewInstance := instance.Status.Conditions == nil - if isNewInstance { - instance.Status.Conditions = condition.Conditions{} - } - - // Save a copy of the conditions so that we can restore the LastTransitionTime - // when a condition's state doesn't change. - savedConditions := instance.Status.Conditions.DeepCopy() - - // Always patch the instance status when exiting this function so we - // can persist any changes. - defer func() { - // Don't update the status, if reconciler Panics - if r := recover(); r != nil { - Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) - panic(r) - } - condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) - if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { - instance.Status.Conditions.Set( - instance.Status.Conditions.Mirror(condition.ReadyCondition)) - } - err := helper.PatchInstance(ctx, instance) - if err != nil { - _err = err - return - } - }() - - if isNewInstance { - // Initialize conditions used later as Status=Unknown - cl := condition.CreateList( - condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), - condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), - condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), - ) - instance.Status.Conditions.Init(&cl) - // Register overall status immediately to have an early feedback - // e.g. in the cli - return ctrl.Result{}, nil + config := FrameworkConfig[*testv1beta1.AnsibleTest]{ + ServiceName: ansibletest.ServiceName, + NeedsNetworkAttachments: false, + NeedsConfigMaps: false, + NeedsFinalizer: false, + SupportsWorkflow: true, + + BuildPod: func(ctx context.Context, instance *testv1beta1.AnsibleTest, labels, annotations map[string]string, workflowStepNum int, pvcIndex int) (*corev1.Pod, error) { + return r.buildAnsibleTestPod(ctx, instance, labels, annotations, workflowStepNum, pvcIndex) + }, + + GetInitialConditions: func() []*condition.Condition { + return []*condition.Condition{ + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + } + }, + + ValidateInputs: func(ctx context.Context, instance *testv1beta1.AnsibleTest) error { + return r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret) + }, + + GetSpec: func(instance *testv1beta1.AnsibleTest) interface{} { + return &instance.Spec + }, + + GetWorkflowStep: func(instance *testv1beta1.AnsibleTest, step int) interface{} { + return instance.Spec.Workflow[step] + }, + + GetWorkflowLength: func(instance *testv1beta1.AnsibleTest) int { + return len(instance.Spec.Workflow) + }, + + GetStorageClass: func(instance *testv1beta1.AnsibleTest) string { + return instance.Spec.StorageClass + }, + + SetObservedGeneration: func(instance *testv1beta1.AnsibleTest) { + instance.Status.ObservedGeneration = instance.Generation + }, } - instance.Status.ObservedGeneration = instance.Generation - - workflowLength := len(instance.Spec.Workflow) - nextAction, nextWorkflowStep, err := r.NextAction(ctx, instance, workflowLength) - if nextWorkflowStep < workflowLength { - MergeSections(&instance.Spec, instance.Spec.Workflow[nextWorkflowStep]) - } - - switch nextAction { - case Failure: - return ctrl.Result{}, err - - case Wait: - Log.Info(InfoWaitingOnPod) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, nil - - case EndTesting: - // All pods created by the instance were completed. Release the lock - // so that other instances can spawn their pods. - if lockReleased, err := r.ReleaseLock(ctx, instance); !lockReleased { - Log.Info(fmt.Sprintf(InfoCanNotReleaseLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - instance.Status.Conditions.MarkTrue( - condition.DeploymentReadyCondition, - condition.DeploymentReadyMessage) - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - Log.Info(InfoTestingCompleted) - return ctrl.Result{}, nil - - case CreateFirstPod: - lockAcquired, err := r.AcquireLock(ctx, instance, helper, false) - if !lockAcquired { - Log.Info(fmt.Sprintf(InfoCanNotAcquireLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - Log.Info(fmt.Sprintf(InfoCreatingFirstPod, nextWorkflowStep)) - - case CreateNextPod: - // Confirm that we still hold the lock. This is useful to check if for - // example somebody / something deleted the lock and it got claimed by - // another instance. This is considered to be an error state. - lockAcquired, err := r.AcquireLock(ctx, instance, helper, false) - if !lockAcquired { - Log.Error(err, ErrConfirmLockOwnership, testOperatorLockName) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - Log.Info(fmt.Sprintf(InfoCreatingNextPod, nextWorkflowStep)) - - default: - return ctrl.Result{}, ErrReceivedUnexpectedAction - } - - serviceLabels := map[string]string{ - common.AppSelector: ansibletest.ServiceName, - workflowStepLabel: strconv.Itoa(nextWorkflowStep), - instanceNameLabel: instance.Name, - operatorNameLabel: "test-operator", - } - - err = r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - - // Create PersistentVolumeClaim - ctrlResult, err := r.EnsureLogsPVCExists( - ctx, - instance, - helper, - serviceLabels, - instance.Spec.StorageClass, - 0, - ) - if err != nil { - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil - } - // Create PersistentVolumeClaim - end + return CommonReconcile(ctx, &r.Reconciler, req, instance, config, r.GetLogger(ctx)) +} - // Create a new pod +func (r *AnsibleTestReconciler) buildAnsibleTestPod( + ctx context.Context, + instance *testv1beta1.AnsibleTest, + labels, _ map[string]string, + workflowStepNum int, + pvcIndex int, +) (*corev1.Pod, error) { mountCerts := r.CheckSecretExists(ctx, instance, "combined-ca-bundle") - podName := r.GetPodName(instance, nextWorkflowStep) envVars := r.PrepareAnsibleEnv(instance) - logsPVCName := r.GetPVCLogsName(instance, 0) + + podName := r.GetPodName(instance, workflowStepNum) + logsPVCName := r.GetPVCLogsName(instance, pvcIndex) + containerImage, err := r.GetContainerImage(ctx, instance) if err != nil { - return ctrl.Result{}, err + return nil, err } - podDef := ansibletest.Pod( + return ansibletest.Pod( instance, - serviceLabels, + labels, podName, logsPVCName, mountCerts, envVars, - nextWorkflowStep, + workflowStepNum, containerImage, - ) - - ctrlResult, err = r.CreatePod(ctx, *helper, podDef) - if err != nil { - // Creation of the ansibleTests pod was not successful. - // Release the lock and allow other controllers to spawn - // a pod. - if lockReleased, err := r.ReleaseLock(ctx, instance); !lockReleased { - return ctrl.Result{}, err - } - - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DeploymentReadyErrorMessage, - err.Error())) - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DeploymentReadyRunningMessage)) - return ctrlResult, nil - } - // Create a new pod - end - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - Log.Info("Reconciled Service successfully") - return ctrl.Result{}, nil + ), nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/common_controller.go b/internal/controller/common_controller.go new file mode 100644 index 00000000..d5ea08f1 --- /dev/null +++ b/internal/controller/common_controller.go @@ -0,0 +1,386 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "strconv" + + "github.com/go-logr/logr" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// FrameworkInstance defines the interface that all test framework CRs must implement +type FrameworkInstance interface { + client.Object + GetConditions() *condition.Conditions +} + +// FrameworkConfig defines framework-specific configuration and behavior +type FrameworkConfig[T FrameworkInstance] struct { + // ServiceName for labeling (e.g., "tempest", "tobiko") + ServiceName string + + // NeedsNetworkAttachments indicates if NADs should be handled + NeedsNetworkAttachments bool + + // NeedsConfigMaps indicates if ServiceConfigReadyCondition is needed + NeedsConfigMaps bool + + // NeedsFinalizer indicates if the controller needs finalizer handling + NeedsFinalizer bool + + // SupportsWorkflow indicates if the controller supports workflow feature + SupportsWorkflow bool + + // GenerateServiceConfigMaps creates framework-specific config maps + GenerateServiceConfigMaps func(ctx context.Context, helper *helper.Helper, instance T, workflowStepNum int) error + + // BuildPod creates the framework-specific pod definition + BuildPod func(ctx context.Context, instance T, labels, annotations map[string]string, workflowStepNum int, pvcIndex int) (*corev1.Pod, error) + + // GetInitialConditions returns the condition list for a new instance + GetInitialConditions func() []*condition.Condition + + // ValidateInputs validates framework-specific inputs + ValidateInputs func(ctx context.Context, instance T) error + + // Field accessors + GetParallel func(instance T) bool + GetStorageClass func(instance T) string + GetNetworkAttachments func(instance T) []string + GetNetworkAttachmentStatus func(instance T) *map[string][]string + SetObservedGeneration func(instance T) + + GetSpec func(instance T) interface{} + GetWorkflowStep func(instance T, step int) interface{} + GetWorkflowLength func(instance T) int +} + +// CommonReconcile executes the standard reconciliation workflow using generics +func CommonReconcile[T FrameworkInstance]( + ctx context.Context, + r *Reconciler, + req ctrl.Request, + instance T, + config FrameworkConfig[T], + Log logr.Logger, +) (result ctrl.Result, _err error) { + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Create a helper + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + r.Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // Get conditions from instance + conditions := instance.GetConditions() + + // Initialize status + isNewInstance := len(*conditions) == 0 + if isNewInstance { + *conditions = condition.Conditions{} + } + + // Save a copy of the conditions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we + // can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(conditions, savedConditions) + if conditions.IsUnknown(condition.ReadyCondition) { + conditions.Set(conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + if isNewInstance { + // Initialize conditions used later as Status=Unknown + cl := condition.CreateList(config.GetInitialConditions()...) + conditions.Init(&cl) + + // Register overall status immediately to have an early feedback + // e.g. in the cli + return ctrl.Result{}, nil + } + + // Set observed generation + if config.SetObservedGeneration != nil { + config.SetObservedGeneration(instance) + } + + // If we're not deleting this and the service object doesn't have our + // finalizer, add it. + if config.NeedsFinalizer && instance.GetDeletionTimestamp().IsZero() && + controllerutil.AddFinalizer(instance, helper.GetFinalizer()) { + return ctrl.Result{}, nil + } + + if config.NeedsNetworkAttachments { + networkStatus := config.GetNetworkAttachmentStatus(instance) + if *networkStatus == nil { + *networkStatus = map[string][]string{} + } + } + + // Handle service delete + if config.NeedsFinalizer && !instance.GetDeletionTimestamp().IsZero() { + Log.Info("Reconciling Service delete") + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info("Reconciled Service delete successfully") + return ctrl.Result{}, nil + } + + workflowLength := 0 + if config.SupportsWorkflow { + workflowLength = config.GetWorkflowLength(instance) + } + + nextAction, workflowStepNum, err := r.NextAction(ctx, instance, workflowLength) + + if config.SupportsWorkflow && workflowStepNum < workflowLength { + spec := config.GetSpec(instance) + workflowStepData := config.GetWorkflowStep(instance, workflowStepNum) + MergeSections(spec, workflowStepData) + } + + switch nextAction { + case Failure: + return ctrl.Result{}, err + + case Wait: + Log.Info(InfoWaitingOnPod) + return ctrl.Result{RequeueAfter: RequeueAfterValue}, nil + + case EndTesting: + // All pods created by the instance were completed. Release the lock + // so that other instances can spawn their pods. + if lockReleased, err := r.ReleaseLock(ctx, instance); !lockReleased { + Log.Info(fmt.Sprintf(InfoCanNotReleaseLock, testOperatorLockName)) + return ctrl.Result{RequeueAfter: RequeueAfterValue}, err + } + + conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + + if conditions.AllSubConditionIsTrue() { + conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + + Log.Info(InfoTestingCompleted) + return ctrl.Result{}, nil + + case CreateFirstPod: + lockAcquired, err := r.AcquireLock(ctx, instance, helper, config.GetParallel(instance)) + if !lockAcquired { + Log.Info(fmt.Sprintf(InfoCanNotAcquireLock, testOperatorLockName)) + return ctrl.Result{RequeueAfter: RequeueAfterValue}, err + } + + Log.Info(fmt.Sprintf(InfoCreatingFirstPod, workflowStepNum)) + + case CreateNextPod: + // Confirm that we still hold the lock. This is useful to check if for + // example somebody / something deleted the lock and it got claimed by + // another instance. This is considered to be an error state. + lockAcquired, err := r.AcquireLock(ctx, instance, helper, config.GetParallel(instance)) + if !lockAcquired { + Log.Error(err, fmt.Sprintf(ErrConfirmLockOwnership, testOperatorLockName)) + return ctrl.Result{RequeueAfter: RequeueAfterValue}, err + } + + Log.Info(fmt.Sprintf(InfoCreatingNextPod, workflowStepNum)) + + default: + return ctrl.Result{}, ErrReceivedUnexpectedAction + } + + serviceLabels := map[string]string{ + common.AppSelector: config.ServiceName, + workflowStepLabel: strconv.Itoa(workflowStepNum), + instanceNameLabel: instance.GetName(), + operatorNameLabel: "test-operator", + } + + // Get parallel execution for reasources that support it + parallel := false + if config.GetParallel != nil { + parallel = config.GetParallel(instance) + } + + pvcIndex := 0 + // Create multiple PVCs for parallel execution + if parallel && config.SupportsWorkflow && workflowStepNum < workflowLength { + pvcIndex = workflowStepNum + } + + if config.ValidateInputs != nil { + if err := config.ValidateInputs(ctx, instance); err != nil { + conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityError, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{RequeueAfter: RequeueAfterValue}, err + } + conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + } + + // Create PersistentVolumeClaim + ctrlResult, err := r.EnsureLogsPVCExists( + ctx, + instance, + helper, + serviceLabels, + config.GetStorageClass(instance), + pvcIndex, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + // Create PersistentVolumeClaim - end + + // Generate ConfigMaps if needed + if config.NeedsConfigMaps { + err = config.GenerateServiceConfigMaps(ctx, helper, instance, workflowStepNum) + if err != nil { + conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + } + // Generate ConfigMaps - end + + // Ensure NetworkAttachments if needed + var serviceAnnotations map[string]string + if config.NeedsNetworkAttachments { + annotations, ctrlResult, err := r.EnsureNetworkAttachments( + ctx, + Log, + helper, + config.GetNetworkAttachments(instance), + instance.GetNamespace(), + conditions, + ) + if err != nil || (ctrlResult != ctrl.Result{}) { + return ctrlResult, err + } + serviceAnnotations = annotations + } + + // Build pod + podDef, err := config.BuildPod( + ctx, + instance, + serviceLabels, + serviceAnnotations, + workflowStepNum, + pvcIndex, + ) + if err != nil { + return ctrl.Result{}, err + } + + // Create a new pod + ctrlResult, err = r.CreatePod(ctx, *helper, podDef) + if err != nil { + // Release the lock and allow other controllers to spawn + // a pod. + if lockReleased, lockErr := r.ReleaseLock(ctx, instance); lockReleased { + return ctrl.Result{RequeueAfter: RequeueAfterValue}, lockErr + } + + conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrlResult, nil + } + // Create a new pod - end + + // Verify NetworkAttachments if needed + if config.NeedsNetworkAttachments { + ctrlResult, err = r.VerifyNetworkAttachments( + ctx, + helper, + instance, + config.GetNetworkAttachments(instance), + serviceLabels, + workflowStepNum, + conditions, + config.GetNetworkAttachmentStatus(instance), + ) + if err != nil || (ctrlResult != ctrl.Result{}) { + return ctrlResult, err + } + } + + // Mark ready if all conditions are true + if conditions.AllSubConditionIsTrue() { + conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + + return ctrl.Result{}, nil +} diff --git a/internal/controller/horizontest_controller.go b/internal/controller/horizontest_controller.go index 75b2afe4..93febeaa 100644 --- a/internal/controller/horizontest_controller.go +++ b/internal/controller/horizontest_controller.go @@ -18,17 +18,14 @@ package controller import ( "context" - "fmt" "github.com/go-logr/logr" - "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" testv1beta1 "github.com/openstack-k8s-operators/test-operator/api/v1beta1" "github.com/openstack-k8s-operators/test-operator/internal/horizontest" corev1 "k8s.io/api/core/v1" - k8s_errors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -54,245 +51,108 @@ func (r *HorizonTestReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;create;update;watch;patch;delete // Reconcile - HorizonTest -func (r *HorizonTestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { - Log := r.GetLogger(ctx) +func (r *HorizonTestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { instance := &testv1beta1.HorizonTest{} - err := r.Client.Get(ctx, req.NamespacedName, instance) - if err != nil { - if k8s_errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - helper, err := helper.NewHelper( - instance, - r.Client, - r.Kclient, - r.Scheme, - r.Log, - ) - if err != nil { - return ctrl.Result{}, err - } - // initialize status - isNewInstance := instance.Status.Conditions == nil - if isNewInstance { - instance.Status.Conditions = condition.Conditions{} - } - - // Save a copy of the conditions so that we can restore the LastTransitionTime - // when a condition's state doesn't change. - savedConditions := instance.Status.Conditions.DeepCopy() - - // Always patch the instance status when exiting this function so we - // can persist any changes. - defer func() { - // Don't update the status, if reconciler Panics - if r := recover(); r != nil { - Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) - panic(r) - } - condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) - if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { - instance.Status.Conditions.Set( - instance.Status.Conditions.Mirror(condition.ReadyCondition)) - } - err := helper.PatchInstance(ctx, instance) - if err != nil { - _err = err - return - } - }() - - if isNewInstance { - // Initialize conditions used later as Status=Unknown - cl := condition.CreateList( - condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), - condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), - condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), - ) - instance.Status.Conditions.Init(&cl) - - // Register overall status immediately to have an early feedback - // e.g. in the cli - return ctrl.Result{}, nil - } - instance.Status.ObservedGeneration = instance.Generation - - workflowLength := 0 - nextAction, nextWorkflowStep, err := r.NextAction(ctx, instance, workflowLength) - - switch nextAction { - case Failure: - return ctrl.Result{}, err - - case Wait: - Log.Info(InfoWaitingOnPod) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, nil - - case EndTesting: - // All pods created by the instance were completed. Release the lock - // so that other instances can spawn their pods. - if lockReleased, err := r.ReleaseLock(ctx, instance); !lockReleased { - Log.Info(fmt.Sprintf(InfoCanNotReleaseLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - instance.Status.Conditions.MarkTrue( - condition.DeploymentReadyCondition, - condition.DeploymentReadyMessage) - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - Log.Info(InfoTestingCompleted) - return ctrl.Result{}, nil - - case CreateFirstPod: - lockAcquired, err := r.AcquireLock(ctx, instance, helper, instance.Spec.Parallel) - if !lockAcquired { - Log.Info(fmt.Sprintf(InfoCanNotAcquireLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - Log.Info(fmt.Sprintf(InfoCreatingFirstPod, nextWorkflowStep)) - - case CreateNextPod: - // Confirm that we still hold the lock. This is useful to check if for - // example somebody / something deleted the lock and it got claimed by - // another instance. This is considered to be an error state. - lockAcquired, err := r.AcquireLock(ctx, instance, helper, instance.Spec.Parallel) - if !lockAcquired { - Log.Error(err, ErrConfirmLockOwnership, testOperatorLockName) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - Log.Info(fmt.Sprintf(InfoCreatingNextPod, nextWorkflowStep)) - - default: - return ctrl.Result{}, ErrReceivedUnexpectedAction + config := FrameworkConfig[*testv1beta1.HorizonTest]{ + ServiceName: horizontest.ServiceName, + NeedsNetworkAttachments: false, + NeedsConfigMaps: true, + NeedsFinalizer: false, + SupportsWorkflow: false, + + GenerateServiceConfigMaps: func(ctx context.Context, helper *helper.Helper, instance *testv1beta1.HorizonTest, _ int) error { + return r.generateServiceConfigMaps(ctx, helper, instance) + }, + + BuildPod: func(ctx context.Context, instance *testv1beta1.HorizonTest, labels, annotations map[string]string, workflowStepNum int, pvcIndex int) (*corev1.Pod, error) { + return r.buildHorizonTestPod(ctx, instance, labels, annotations, workflowStepNum, pvcIndex) + }, + + GetInitialConditions: func() []*condition.Condition { + return []*condition.Condition{ + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + } + }, + + ValidateInputs: func(ctx context.Context, instance *testv1beta1.HorizonTest) error { + if err := r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret); err != nil { + return err + } + return r.ValidateSecretWithKeys(ctx, instance, instance.Spec.KubeconfigSecretName, []string{}) + }, + + GetSpec: func(instance *testv1beta1.HorizonTest) interface{} { + return &instance.Spec + }, + + GetParallel: func(instance *testv1beta1.HorizonTest) bool { + return instance.Spec.Parallel + }, + + GetStorageClass: func(instance *testv1beta1.HorizonTest) string { + return instance.Spec.StorageClass + }, + + SetObservedGeneration: func(instance *testv1beta1.HorizonTest) { + instance.Status.ObservedGeneration = instance.Generation + }, } - serviceLabels := map[string]string{ - common.AppSelector: horizontest.ServiceName, - instanceNameLabel: instance.Name, - operatorNameLabel: "test-operator", - - // NOTE(lpiwowar): This is a workaround since the Horizontest CR does not support - // workflows. However, the label might be required by automation that - // consumes the test-operator (e.g., ci-framework). - workflowStepLabel: "0", - } + return CommonReconcile(ctx, &r.Reconciler, req, instance, config, r.GetLogger(ctx)) +} - err = r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err +func (r *HorizonTestReconciler) generateServiceConfigMaps( + ctx context.Context, + h *helper.Helper, + instance *testv1beta1.HorizonTest, +) error { + labels := map[string]string{ + operatorNameLabel: "test-operator", + instanceNameLabel: instance.Name, } - yamlResult, err := EnsureCloudsConfigMapExists( + _, err := EnsureCloudsConfigMapExists( ctx, instance, - helper, - serviceLabels, + h, + labels, instance.Spec.OpenStackConfigMap, ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return yamlResult, err - } - - err = r.ValidateSecretWithKeys(ctx, instance, instance.Spec.KubeconfigSecretName, []string{}) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err - } - mountKubeconfig := len(instance.Spec.KubeconfigSecretName) != 0 - - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - - // Create PersistentVolumeClaim - ctrlResult, err := r.EnsureLogsPVCExists( - ctx, - instance, - helper, - serviceLabels, - instance.Spec.StorageClass, - 0, - ) - if err != nil { - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil - } - // Create PersistentVolumeClaim - end + return err +} - // Create Pod +func (r *HorizonTestReconciler) buildHorizonTestPod( + ctx context.Context, + instance *testv1beta1.HorizonTest, + labels, _ map[string]string, + workflowStepNum int, + pvcIndex int, +) (*corev1.Pod, error) { mountCerts := r.CheckSecretExists(ctx, instance, "combined-ca-bundle") + mountKubeconfig := len(instance.Spec.KubeconfigSecretName) != 0 - // Prepare HorizonTest env vars envVars := r.PrepareHorizonTestEnvVars(instance) - podName := r.GetPodName(instance, 0) - logsPVCName := r.GetPVCLogsName(instance, 0) + podName := r.GetPodName(instance, workflowStepNum) + logsPVCName := r.GetPVCLogsName(instance, pvcIndex) + containerImage, err := r.GetContainerImage(ctx, instance) if err != nil { - return ctrl.Result{}, err + return nil, err } - podDef := horizontest.Pod( + return horizontest.Pod( instance, - serviceLabels, + labels, podName, logsPVCName, mountCerts, mountKubeconfig, envVars, containerImage, - ) - - ctrlResult, err = r.CreatePod(ctx, *helper, podDef) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DeploymentReadyErrorMessage, - err.Error())) - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DeploymentReadyRunningMessage)) - return ctrlResult, nil - } - // create Pod - end - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - Log.Info("Reconciled Service successfully") - return ctrl.Result{}, nil + ), nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/tempest_controller.go b/internal/controller/tempest_controller.go index 0ff8f91b..ebc2e494 100644 --- a/internal/controller/tempest_controller.go +++ b/internal/controller/tempest_controller.go @@ -18,11 +18,9 @@ package controller import ( "context" - "fmt" "strconv" "github.com/go-logr/logr" - "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -31,9 +29,7 @@ import ( testv1beta1 "github.com/openstack-k8s-operators/test-operator/api/v1beta1" "github.com/openstack-k8s-operators/test-operator/internal/tempest" corev1 "k8s.io/api/core/v1" - k8s_errors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -58,318 +54,110 @@ func (r *TempestReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;create;update;watch;patch;delete // Reconcile - Tempest -func (r *TempestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { - Log := r.GetLogger(ctx) - - // Fetch the Tempest instance +func (r *TempestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { instance := &testv1beta1.Tempest{} - err := r.Client.Get(ctx, req.NamespacedName, instance) - if err != nil { - if k8s_errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - // Create a helper - helper, err := helper.NewHelper( - instance, - r.Client, - r.Kclient, - r.Scheme, - r.Log, - ) - if err != nil { - return ctrl.Result{}, err - } + config := FrameworkConfig[*testv1beta1.Tempest]{ + ServiceName: tempest.ServiceName, + NeedsNetworkAttachments: true, + NeedsConfigMaps: true, + NeedsFinalizer: true, + SupportsWorkflow: true, - // initialize status - isNewInstance := instance.Status.Conditions == nil - if isNewInstance { - instance.Status.Conditions = condition.Conditions{} - } + GenerateServiceConfigMaps: func(ctx context.Context, helper *helper.Helper, instance *testv1beta1.Tempest, workflowStep int) error { + return r.generateServiceConfigMaps(ctx, helper, instance, workflowStep) + }, - // Save a copy of the conditions so that we can restore the LastTransitionTime - // when a condition's state doesn't change. - savedConditions := instance.Status.Conditions.DeepCopy() - - // Always patch the instance status when exiting this function so we - // can persist any changes. - defer func() { - // Don't update the status, if reconciler Panics - if r := recover(); r != nil { - Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) - panic(r) - } - condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) - if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { - instance.Status.Conditions.Set( - instance.Status.Conditions.Mirror(condition.ReadyCondition)) - } - err := helper.PatchInstance(ctx, instance) - if err != nil { - _err = err - return - } - }() - - if isNewInstance { - // Initialize conditions used later as Status=Unknown - cl := condition.CreateList( - condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), - condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), - condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), - condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), - condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), - ) - instance.Status.Conditions.Init(&cl) - - // Register overall status immediately to have an early feedback - // e.g. in the cli - return ctrl.Result{}, nil - } - instance.Status.ObservedGeneration = instance.Generation + BuildPod: func(ctx context.Context, instance *testv1beta1.Tempest, labels, annotations map[string]string, workflowStepNum int, pvcIndex int) (*corev1.Pod, error) { + return r.buildTempestPod(ctx, instance, labels, annotations, workflowStepNum, pvcIndex) + }, - // If we're not deleting this and the service object doesn't have our - // finalizer, add it. - if instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer()) { - return ctrl.Result{}, nil - } + GetInitialConditions: func() []*condition.Condition { + return []*condition.Condition{ + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + } + }, - if instance.Status.NetworkAttachments == nil { - instance.Status.NetworkAttachments = map[string][]string{} - } + ValidateInputs: func(ctx context.Context, instance *testv1beta1.Tempest) error { + if err := r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret); err != nil { + return err + } + return r.ValidateSecretWithKeys(ctx, instance, instance.Spec.SSHKeySecretName, []string{}) + }, - // Handle service delete - if !instance.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, instance, helper) - } + GetSpec: func(instance *testv1beta1.Tempest) interface{} { + return &instance.Spec + }, - workflowLength := len(instance.Spec.Workflow) - nextAction, nextWorkflowStep, err := r.NextAction(ctx, instance, workflowLength) - if nextWorkflowStep < workflowLength { - MergeSections(&instance.Spec, instance.Spec.Workflow[nextWorkflowStep]) - } + GetWorkflowStep: func(instance *testv1beta1.Tempest, step int) interface{} { + return instance.Spec.Workflow[step] + }, - switch nextAction { - case Failure: - return ctrl.Result{}, err - - case Wait: - Log.Info(InfoWaitingOnPod) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, nil - - case EndTesting: - // All pods created by the instance were completed. Release the lock - // so that other instances can spawn their pods. - if lockReleased, err := r.ReleaseLock(ctx, instance); !lockReleased { - Log.Info(fmt.Sprintf(InfoCanNotReleaseLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - instance.Status.Conditions.MarkTrue( - condition.DeploymentReadyCondition, - condition.DeploymentReadyMessage) - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - Log.Info(InfoTestingCompleted) - return ctrl.Result{}, nil - - case CreateFirstPod: - lockAcquired, err := r.AcquireLock(ctx, instance, helper, instance.Spec.Parallel) - if !lockAcquired { - Log.Info(fmt.Sprintf(InfoCanNotAcquireLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - Log.Info(fmt.Sprintf(InfoCreatingFirstPod, nextWorkflowStep)) - - case CreateNextPod: - // Confirm that we still hold the lock. This is useful to check if for - // example somebody / something deleted the lock and it got claimed by - // another instance. This is considered to be an error state. - lockAcquired, err := r.AcquireLock(ctx, instance, helper, instance.Spec.Parallel) - if !lockAcquired { - Log.Error(err, ErrConfirmLockOwnership, testOperatorLockName) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - Log.Info(fmt.Sprintf(InfoCreatingNextPod, nextWorkflowStep)) - - default: - return ctrl.Result{}, ErrReceivedUnexpectedAction - } + GetWorkflowLength: func(instance *testv1beta1.Tempest) int { + return len(instance.Spec.Workflow) + }, - serviceLabels := map[string]string{ - common.AppSelector: tempest.ServiceName, - workflowStepLabel: strconv.Itoa(nextWorkflowStep), - instanceNameLabel: instance.Name, - operatorNameLabel: "test-operator", - } + GetParallel: func(instance *testv1beta1.Tempest) bool { + return instance.Spec.Parallel + }, - workflowStepNum := 0 - // Create multiple PVCs for parallel execution - if instance.Spec.Parallel && nextWorkflowStep < len(instance.Spec.Workflow) { - workflowStepNum = nextWorkflowStep - } + GetStorageClass: func(instance *testv1beta1.Tempest) string { + return instance.Spec.StorageClass + }, - err = r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } + GetNetworkAttachments: func(instance *testv1beta1.Tempest) []string { + return instance.Spec.NetworkAttachments + }, - err = r.ValidateSecretWithKeys(ctx, instance, instance.Spec.SSHKeySecretName, []string{}) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err + GetNetworkAttachmentStatus: func(instance *testv1beta1.Tempest) *map[string][]string { + return &instance.Status.NetworkAttachments + }, + + SetObservedGeneration: func(instance *testv1beta1.Tempest) { + instance.Status.ObservedGeneration = instance.Generation + }, } - mountSSHKey := len(instance.Spec.SSHKeySecretName) != 0 - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + return CommonReconcile(ctx, &r.Reconciler, req, instance, config, r.GetLogger(ctx)) +} - // Create PersistentVolumeClaim - ctrlResult, err := r.EnsureLogsPVCExists( - ctx, - instance, - helper, - serviceLabels, - instance.Spec.StorageClass, - workflowStepNum, - ) +func (r *TempestReconciler) buildTempestPod( + ctx context.Context, + instance *testv1beta1.Tempest, + labels, annotations map[string]string, + workflowStepNum int, + pvcIndex int, +) (*corev1.Pod, error) { + mountSSHKey := len(instance.Spec.SSHKeySecretName) != 0 + mountCerts := r.CheckSecretExists(ctx, instance, "combined-ca-bundle") - if err != nil { - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil - } - // Create PersistentVolumeClaim - end + customDataConfigMapName := GetCustomDataConfigMapName(instance, workflowStepNum) + envVarsConfigMapName := GetEnvVarsConfigMapName(instance, workflowStepNum) - // Generate ConfigMaps - err = r.generateServiceConfigMaps(ctx, helper, instance, nextWorkflowStep) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ServiceConfigReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.ServiceConfigReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err - } - instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - // Generate ConfigMaps - end - - serviceAnnotations, ctrlResult, err := r.EnsureNetworkAttachments( - ctx, - Log, - helper, - instance.Spec.NetworkAttachments, - instance.Namespace, - &instance.Status.Conditions, - ) - if err != nil || (ctrlResult != ctrl.Result{}) { - return ctrlResult, err - } + podName := r.GetPodName(instance, workflowStepNum) + logsPVCName := r.GetPVCLogsName(instance, pvcIndex) - // Create a new pod - mountCerts := r.CheckSecretExists(ctx, instance, "combined-ca-bundle") - customDataConfigMapName := GetCustomDataConfigMapName(instance, nextWorkflowStep) - EnvVarsConfigMapName := GetEnvVarsConfigMapName(instance, nextWorkflowStep) - podName := r.GetPodName(instance, nextWorkflowStep) - logsPVCName := r.GetPVCLogsName(instance, workflowStepNum) containerImage, err := r.GetContainerImage(ctx, instance) if err != nil { - return ctrl.Result{}, err + return nil, err } - podDef := tempest.Pod( + return tempest.Pod( instance, - serviceLabels, - serviceAnnotations, + labels, + annotations, podName, - EnvVarsConfigMapName, + envVarsConfigMapName, customDataConfigMapName, logsPVCName, mountCerts, mountSSHKey, containerImage, - ) - - ctrlResult, err = r.CreatePod(ctx, *helper, podDef) - if err != nil { - // Creation of the tempest pod was not successful. - // Release the lock and allow other controllers to spawn - // a pod. - if lockReleased, lockErr := r.ReleaseLock(ctx, instance); lockReleased { - return ctrl.Result{RequeueAfter: RequeueAfterValue}, lockErr - } - - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DeploymentReadyErrorMessage, - err.Error())) - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DeploymentReadyRunningMessage)) - return ctrlResult, nil - } - // Create a new pod - end - - // NetworkAttachments - ctrlResult, err = r.VerifyNetworkAttachments( - ctx, - helper, - instance, - instance.Spec.NetworkAttachments, - serviceLabels, - nextWorkflowStep, - &instance.Status.Conditions, - &instance.Status.NetworkAttachments, - ) - if err != nil || (ctrlResult != ctrl.Result{}) { - return ctrlResult, err - } - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - return ctrl.Result{}, nil -} - -func (r *TempestReconciler) reconcileDelete( - ctx context.Context, - instance *testv1beta1.Tempest, - helper *helper.Helper, -) (ctrl.Result, error) { - Log := r.GetLogger(ctx) - Log.Info("Reconciling Service delete") - - // remove the finalizer - controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) - - Log.Info("Reconciled Service delete successfully") - - return ctrl.Result{}, nil + ), nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/tobiko_controller.go b/internal/controller/tobiko_controller.go index 0126949f..06ee04e6 100644 --- a/internal/controller/tobiko_controller.go +++ b/internal/controller/tobiko_controller.go @@ -18,11 +18,9 @@ package controller import ( "context" - "fmt" "strconv" "github.com/go-logr/logr" - "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/env" @@ -31,7 +29,6 @@ import ( testv1beta1 "github.com/openstack-k8s-operators/test-operator/api/v1beta1" "github.com/openstack-k8s-operators/test-operator/internal/tobiko" corev1 "k8s.io/api/core/v1" - k8s_errors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -57,259 +54,101 @@ func (r *TobikoReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;create;update;watch;patch;delete // Reconcile - Tobiko -func (r *TobikoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { - Log := r.GetLogger(ctx) - +func (r *TobikoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { instance := &testv1beta1.Tobiko{} - err := r.Client.Get(ctx, req.NamespacedName, instance) - if err != nil { - if k8s_errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - helper, err := helper.NewHelper( - instance, - r.Client, - r.Kclient, - r.Scheme, - r.Log, - ) - if err != nil { - return ctrl.Result{}, err - } - // initialize status - isNewInstance := instance.Status.Conditions == nil - if isNewInstance { - instance.Status.Conditions = condition.Conditions{} - } + config := FrameworkConfig[*testv1beta1.Tobiko]{ + ServiceName: tobiko.ServiceName, + NeedsNetworkAttachments: true, + NeedsConfigMaps: true, + NeedsFinalizer: false, + SupportsWorkflow: true, - // Save a copy of the conditions so that we can restore the LastTransitionTime - // when a condition's state doesn't change. - savedConditions := instance.Status.Conditions.DeepCopy() - - // Always patch the instance status when exiting this function so we - // can persist any changes. - defer func() { - // Don't update the status, if reconciler Panics - if r := recover(); r != nil { - Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) - panic(r) - } - condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) - if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { - instance.Status.Conditions.Set( - instance.Status.Conditions.Mirror(condition.ReadyCondition)) - } - err := helper.PatchInstance(ctx, instance) - if err != nil { - _err = err - return - } - }() - - if isNewInstance { - // Initialize conditions used later as Status=Unknown - cl := condition.CreateList( - condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), - condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), - condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), - condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), - ) - instance.Status.Conditions.Init(&cl) - - // Register overall status immediately to have an early feedback - // e.g. in the cli - return ctrl.Result{}, nil - } - instance.Status.ObservedGeneration = instance.Generation - - if instance.Status.NetworkAttachments == nil { - instance.Status.NetworkAttachments = map[string][]string{} - } + GenerateServiceConfigMaps: func(ctx context.Context, helper *helper.Helper, instance *testv1beta1.Tobiko, _ int) error { + return r.generateServiceConfigMaps(ctx, helper, instance) + }, - workflowLength := len(instance.Spec.Workflow) - nextAction, nextWorkflowStep, err := r.NextAction(ctx, instance, workflowLength) - if nextWorkflowStep < workflowLength { - MergeSections(&instance.Spec, instance.Spec.Workflow[nextWorkflowStep]) - } + BuildPod: func(ctx context.Context, instance *testv1beta1.Tobiko, labels, annotations map[string]string, workflowStepNum int, pvcIndex int) (*corev1.Pod, error) { + return r.buildTobikoPod(ctx, instance, labels, annotations, workflowStepNum, pvcIndex) + }, - switch nextAction { - case Failure: - return ctrl.Result{}, err - - case Wait: - Log.Info(InfoWaitingOnPod) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, nil - - case EndTesting: - // All pods created by the instance were completed. Release the lock - // so that other instances can spawn their pods. - if lockReleased, err := r.ReleaseLock(ctx, instance); !lockReleased { - Log.Info(fmt.Sprintf(InfoCanNotReleaseLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - instance.Status.Conditions.MarkTrue( - condition.DeploymentReadyCondition, - condition.DeploymentReadyMessage) - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - Log.Info(InfoTestingCompleted) - return ctrl.Result{}, nil - - case CreateFirstPod: - lockAcquired, err := r.AcquireLock(ctx, instance, helper, instance.Spec.Parallel) - if !lockAcquired { - Log.Info(fmt.Sprintf(InfoCanNotAcquireLock, testOperatorLockName)) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - Log.Info(fmt.Sprintf(InfoCreatingFirstPod, nextWorkflowStep)) - - case CreateNextPod: - // Confirm that we still hold the lock. This needs to be checked in order - // to prevent situation when somebody / something deleted the lock and it - // got claimed by another instance. - lockAcquired, err := r.AcquireLock(ctx, instance, helper, instance.Spec.Parallel) - if !lockAcquired { - Log.Error(err, ErrConfirmLockOwnership, testOperatorLockName) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } - - Log.Info(fmt.Sprintf(InfoCreatingNextPod, nextWorkflowStep)) - - default: - return ctrl.Result{}, ErrReceivedUnexpectedAction - } + GetInitialConditions: func() []*condition.Condition { + return []*condition.Condition{ + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + } + }, - serviceLabels := map[string]string{ - common.AppSelector: tobiko.ServiceName, - workflowStepLabel: strconv.Itoa(nextWorkflowStep), - instanceNameLabel: instance.Name, - operatorNameLabel: "test-operator", - } + ValidateInputs: func(ctx context.Context, instance *testv1beta1.Tobiko) error { + if err := r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret); err != nil { + return err + } + return r.ValidateSecretWithKeys(ctx, instance, instance.Spec.KubeconfigSecretName, []string{}) + }, - err = r.ValidateOpenstackInputs(ctx, instance, instance.Spec.OpenStackConfigMap, instance.Spec.OpenStackConfigSecret) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityError, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{RequeueAfter: RequeueAfterValue}, err - } + GetSpec: func(instance *testv1beta1.Tobiko) interface{} { + return &instance.Spec + }, - yamlResult, err := EnsureCloudsConfigMapExists( - ctx, - instance, - helper, - serviceLabels, - instance.Spec.OpenStackConfigMap, - ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return yamlResult, err - } + GetWorkflowStep: func(instance *testv1beta1.Tobiko, step int) interface{} { + return instance.Spec.Workflow[step] + }, - err = r.ValidateSecretWithKeys(ctx, instance, instance.Spec.KubeconfigSecretName, []string{}) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err - } - mountKubeconfig := len(instance.Spec.KubeconfigSecretName) != 0 + GetWorkflowLength: func(instance *testv1beta1.Tobiko) int { + return len(instance.Spec.Workflow) + }, - instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + GetParallel: func(instance *testv1beta1.Tobiko) bool { + return instance.Spec.Parallel + }, - workflowStepNum := 0 + GetStorageClass: func(instance *testv1beta1.Tobiko) string { + return instance.Spec.StorageClass + }, - // Create multiple PVCs for parallel execution - if instance.Spec.Parallel && nextWorkflowStep < len(instance.Spec.Workflow) { - workflowStepNum = nextWorkflowStep - } + GetNetworkAttachments: func(instance *testv1beta1.Tobiko) []string { + return instance.Spec.NetworkAttachments + }, - // Create PersistentVolumeClaim - ctrlResult, err := r.EnsureLogsPVCExists( - ctx, - instance, - helper, - serviceLabels, - instance.Spec.StorageClass, - workflowStepNum, - ) - if err != nil { - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil - } - // Create PersistentVolumeClaim - end + GetNetworkAttachmentStatus: func(instance *testv1beta1.Tobiko) *map[string][]string { + return &instance.Status.NetworkAttachments + }, - serviceAnnotations, ctrlResult, err := r.EnsureNetworkAttachments( - ctx, - Log, - helper, - instance.Spec.NetworkAttachments, - instance.Namespace, - &instance.Status.Conditions, - ) - if err != nil || (ctrlResult != ctrl.Result{}) { - return ctrlResult, err + SetObservedGeneration: func(instance *testv1beta1.Tobiko) { + instance.Status.ObservedGeneration = instance.Generation + }, } - // NetworkAttachments - ctrlResult, err = r.VerifyNetworkAttachments( - ctx, - helper, - instance, - instance.Spec.NetworkAttachments, - serviceLabels, - nextWorkflowStep, - &instance.Status.Conditions, - &instance.Status.NetworkAttachments, - ) - if err != nil || (ctrlResult != ctrl.Result{}) { - return ctrlResult, err - } + return CommonReconcile(ctx, &r.Reconciler, req, instance, config, r.GetLogger(ctx)) +} - // Create Pod +func (r *TobikoReconciler) buildTobikoPod( + ctx context.Context, + instance *testv1beta1.Tobiko, + labels, annotations map[string]string, + workflowStepNum int, + pvcIndex int, +) (*corev1.Pod, error) { mountCerts := r.CheckSecretExists(ctx, instance, "combined-ca-bundle") - - mountKeys := false - if (len(instance.Spec.PublicKey) == 0) || (len(instance.Spec.PrivateKey) == 0) { - Log.Info("Both values privateKey and publicKey need to be specified. Keys not mounted.") - } else { - mountKeys = true - } + mountKubeconfig := len(instance.Spec.KubeconfigSecretName) != 0 + mountKeys := len(instance.Spec.PublicKey) > 0 && len(instance.Spec.PrivateKey) > 0 // Prepare Tobiko env vars - envVars := r.PrepareTobikoEnvVars(ctx, serviceLabels, instance, helper, nextWorkflowStep) - podName := r.GetPodName(instance, nextWorkflowStep) - logsPVCName := r.GetPVCLogsName(instance, workflowStepNum) + envVars := r.PrepareTobikoEnvVars(instance, workflowStepNum) + podName := r.GetPodName(instance, workflowStepNum) + logsPVCName := r.GetPVCLogsName(instance, pvcIndex) + containerImage, err := r.GetContainerImage(ctx, instance) if err != nil { - return ctrl.Result{}, err + return nil, err } - podDef := tobiko.Pod( + return tobiko.Pod( instance, - serviceLabels, - serviceAnnotations, + labels, + annotations, podName, logsPVCName, mountCerts, @@ -317,33 +156,7 @@ func (r *TobikoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res mountKubeconfig, envVars, containerImage, - ) - - ctrlResult, err = r.CreatePod(ctx, *helper, podDef) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DeploymentReadyErrorMessage, - err.Error())) - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DeploymentReadyRunningMessage)) - return ctrlResult, nil - } - // create Pod - end - - if instance.Status.Conditions.AllSubConditionIsTrue() { - instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) - } - - Log.Info("Reconciled Service successfully") - return ctrl.Result{}, nil + ), nil } // SetupWithManager sets up the controller with the Manager. @@ -356,12 +169,67 @@ func (r *TobikoReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } +func (r *TobikoReconciler) generateServiceConfigMaps( + ctx context.Context, + h *helper.Helper, + instance *testv1beta1.Tobiko, +) error { + labels := map[string]string{ + operatorNameLabel: "test-operator", + instanceNameLabel: instance.Name, + } + + _, err := EnsureCloudsConfigMapExists( + ctx, + instance, + h, + labels, + instance.Spec.OpenStackConfigMap, + ) + if err != nil { + return err + } + + // Prepare custom data + customData := make(map[string]string) + customData["tobiko.conf"] = instance.Spec.Config + + privateKeyData := make(map[string]string) + privateKeyData["id_ecdsa"] = instance.Spec.PrivateKey + + publicKeyData := make(map[string]string) + publicKeyData["id_ecdsa.pub"] = instance.Spec.PublicKey + + cms := []util.Template{ + { + Name: instance.Name + "tobiko-config", + Namespace: instance.Namespace, + InstanceType: instance.Kind, + Labels: labels, + CustomData: customData, + }, + { + Name: instance.Name + "tobiko-private-key", + Namespace: instance.Namespace, + InstanceType: instance.Kind, + Labels: labels, + CustomData: privateKeyData, + }, + { + Name: instance.Name + "tobiko-public-key", + Namespace: instance.Namespace, + InstanceType: instance.Kind, + Labels: labels, + CustomData: publicKeyData, + }, + } + + return configmap.EnsureConfigMaps(ctx, h, instance, cms, nil) +} + // PrepareTobikoEnvVars prepares environment variables for a single workflow step func (r *TobikoReconciler) PrepareTobikoEnvVars( - ctx context.Context, - labels map[string]string, instance *testv1beta1.Tobiko, - helper *helper.Helper, workflowStepNum int, ) map[string]env.Setter { // Prepare env vars @@ -402,44 +270,5 @@ func (r *TobikoReconciler) PrepareTobikoEnvVars( }) } - // Prepare custom data - customData := make(map[string]string) - customData["tobiko.conf"] = instance.Spec.Config - - privateKeyData := make(map[string]string) - privateKeyData["id_ecdsa"] = instance.Spec.PrivateKey - - publicKeyData := make(map[string]string) - publicKeyData["id_ecdsa.pub"] = instance.Spec.PublicKey - - cms := []util.Template{ - { - Name: instance.Name + "tobiko-config", - Namespace: instance.Namespace, - InstanceType: instance.Kind, - Labels: labels, - CustomData: customData, - }, - { - Name: instance.Name + "tobiko-private-key", - Namespace: instance.Namespace, - InstanceType: instance.Kind, - Labels: labels, - CustomData: privateKeyData, - }, - { - Name: instance.Name + "tobiko-public-key", - Namespace: instance.Namespace, - InstanceType: instance.Kind, - Labels: labels, - CustomData: publicKeyData, - }, - } - - err := configmap.EnsureConfigMaps(ctx, helper, instance, cms, nil) - if err != nil { - return map[string]env.Setter{} - } - return envVars }