Skip to content

Commit a0d7f94

Browse files
JianLi-RHJianLi-RH
committed
Automate OCP-42543: should not install resources annotated with release.openshift.io/delete=true
1 parent 557510e commit a0d7f94

4 files changed

Lines changed: 319 additions & 0 deletions

File tree

‎.openshift-tests-extension/openshift_payload_cluster-version-operator.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,19 @@
3838
"source": "openshift:payload:cluster-version-operator",
3939
"lifecycle": "blocking",
4040
"environmentSelector": {}
41+
},
42+
{
43+
"name": "[Jira:\"Cluster Version Operator\"] cluster-version-operator should not install resources annotated with release.openshift.io/delete=true",
44+
"labels": {
45+
"42543": {},
46+
"Conformance": {},
47+
"High": {}
48+
},
49+
"resources": {
50+
"isolation": {}
51+
},
52+
"source": "openshift:payload:cluster-version-operator",
53+
"lifecycle": "blocking",
54+
"environmentSelector": {}
4155
}
4256
]

‎test/cvo/cvo.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package cvo
22

33
import (
4+
"bytes"
45
"context"
6+
"fmt"
7+
"os"
8+
"path/filepath"
9+
"time"
510

611
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
712
"k8s.io/client-go/kubernetes"
@@ -13,6 +18,7 @@ import (
1318
"github.com/openshift/cluster-version-operator/test/oc"
1419
ocapi "github.com/openshift/cluster-version-operator/test/oc/api"
1520
"github.com/openshift/cluster-version-operator/test/util"
21+
"github.com/openshift/library-go/pkg/manifest"
1622
)
1723

1824
var logger = g.GinkgoLogr.WithName("cluster-version-operator-tests")
@@ -84,4 +90,58 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
8490
sccAnnotation := cvoPod.Annotations["openshift.io/scc"]
8591
o.Expect(sccAnnotation).To(o.Equal("hostaccess"), "Expected the annotation 'openshift.io/scc annotation' on pod %s to have the value 'hostaccess', but got %s", cvoPod.Name, sccAnnotation)
8692
})
93+
94+
g.It(`should not install resources annotated with release.openshift.io/delete=true`, g.Label("Conformance", "High", "42543"), func() {
95+
ctx := context.Background()
96+
err := util.SkipIfHypershift(ctx, restCfg)
97+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to determine if cluster is HyperShift")
98+
err = util.SkipIfMicroshift(ctx, restCfg)
99+
o.Expect(err).NotTo(o.HaveOccurred(), "Failed to determine if cluster is MicroShift")
100+
101+
// Initialize the ocapi.OC instance
102+
g.By("Setting up oc")
103+
ocClient, err := oc.NewOC(ocapi.Options{Logger: logger, Timeout: 90 * time.Second})
104+
o.Expect(err).NotTo(o.HaveOccurred())
105+
106+
g.By("Extracting manifests in the release")
107+
annotation := "release.openshift.io/delete"
108+
tempDir, err := os.MkdirTemp("", "OTA-42543-manifest-")
109+
o.Expect(err).NotTo(o.HaveOccurred(), "create temp manifest dir failed")
110+
manifestDir := ocapi.ReleaseExtractOptions{To: tempDir}
111+
logger.Info(fmt.Sprintf("Extract manifests to: %s", manifestDir.To))
112+
defer func() { _ = os.RemoveAll(manifestDir.To) }()
113+
err = ocClient.AdmReleaseExtract(manifestDir)
114+
o.Expect(err).NotTo(o.HaveOccurred(), "extracting manifests failed")
115+
116+
files, err := os.ReadDir(manifestDir.To)
117+
o.Expect(err).NotTo(o.HaveOccurred())
118+
g.By(fmt.Sprintf("Checking if getting manifests with %s on the cluster led to not-found error", annotation))
119+
for _, manifestFile := range files {
120+
// if strings.Contains(strings.ToLower(manifestFile.Name()), "cleanup") {
121+
// logger.Info(fmt.Sprintf("skipping file %s because it matches cleanup filter", manifestFile.Name()))
122+
// continue
123+
// }
124+
125+
filePath := filepath.Join(manifestDir.To, manifestFile.Name())
126+
raw, err := os.ReadFile(filePath)
127+
if err != nil {
128+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to read manifest file")
129+
}
130+
manifests, err := manifest.ParseManifests(bytes.NewReader(raw))
131+
if err != nil {
132+
// files like release-metadata are not manifest file, so skip them
133+
logger.Error(err, "failed to parse manifest file: "+filePath)
134+
}
135+
136+
for _, ms := range manifests {
137+
ann := ms.Obj.GetAnnotations()
138+
if ann == nil || ann[annotation] != "true" {
139+
continue
140+
}
141+
manifestFilePath := filepath.Join(manifestDir.To, manifestFile.Name())
142+
err := util.GetManifestExpectNotFoundError(ms)
143+
o.Expect(err).To(o.HaveOccurred(), "The deleted manifest should not be installed, but actually installed:"+manifestFilePath+", manifest name: "+ms.Obj.GetName())
144+
}
145+
}
146+
})
87147
})

‎test/util/connection.go‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package util
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"os"
7+
8+
"k8s.io/client-go/rest"
9+
"k8s.io/client-go/tools/clientcmd"
10+
11+
imageclientv1 "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1"
12+
securityclientv1 "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1"
13+
operatorsclientv1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1"
14+
apiextclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
15+
admissionregistrationclientv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1"
16+
appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
17+
batchclientv1 "k8s.io/client-go/kubernetes/typed/batch/v1"
18+
coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
19+
rbacclientv1 "k8s.io/client-go/kubernetes/typed/rbac/v1"
20+
)
21+
22+
// getKubeConfig get KUBECONFIG file from environment variable
23+
func getKubeConfig() (*rest.Config, error) {
24+
configPath, present := os.LookupEnv("KUBECONFIG")
25+
if !present {
26+
return nil, errors.New("the environment variable KUBECONFIG must be set")
27+
}
28+
config, err := clientcmd.BuildConfigFromFlags("", configPath)
29+
return config, err
30+
}
31+
32+
func getImageClient() (*imageclientv1.ImageV1Client, error) {
33+
config, err := getKubeConfig()
34+
if err != nil {
35+
return nil, fmt.Errorf("unable to load build config: %w", err)
36+
}
37+
return imageclientv1.NewForConfigOrDie(config), nil
38+
}
39+
40+
func getSecretClient() (*securityclientv1.SecurityV1Client, error) {
41+
config, err := getKubeConfig()
42+
if err != nil {
43+
return nil, fmt.Errorf("unable to load build config: %w", err)
44+
}
45+
return securityclientv1.NewForConfigOrDie(config), nil
46+
}
47+
48+
func getOperatorClient() (*operatorsclientv1.OperatorsV1Client, error) {
49+
config, err := getKubeConfig()
50+
if err != nil {
51+
return nil, fmt.Errorf("unable to load build config: %w", err)
52+
}
53+
return operatorsclientv1.NewForConfigOrDie(config), nil
54+
}
55+
56+
func getAdmissionRegistrationClient() (*admissionregistrationclientv1.AdmissionregistrationV1Client, error) {
57+
config, err := getKubeConfig()
58+
if err != nil {
59+
return nil, fmt.Errorf("unable to load build config: %w", err)
60+
}
61+
return admissionregistrationclientv1.NewForConfigOrDie(config), nil
62+
}
63+
64+
func getAppsClient() (*appsclientv1.AppsV1Client, error) {
65+
config, err := getKubeConfig()
66+
if err != nil {
67+
return nil, fmt.Errorf("unable to load build config: %w", err)
68+
}
69+
return appsclientv1.NewForConfigOrDie(config), nil
70+
}
71+
72+
func getBatchClient() (*batchclientv1.BatchV1Client, error) {
73+
config, err := getKubeConfig()
74+
if err != nil {
75+
return nil, fmt.Errorf("unable to load build config: %w", err)
76+
}
77+
return batchclientv1.NewForConfigOrDie(config), nil
78+
}
79+
80+
func getCoreV1Client() (*coreclientv1.CoreV1Client, error) {
81+
config, err := getKubeConfig()
82+
if err != nil {
83+
return nil, fmt.Errorf("unable to load build config: %w", err)
84+
}
85+
return coreclientv1.NewForConfigOrDie(config), nil
86+
}
87+
88+
func getRbacV1Client() (*rbacclientv1.RbacV1Client, error) {
89+
config, err := getKubeConfig()
90+
if err != nil {
91+
return nil, fmt.Errorf("unable to load build config: %w", err)
92+
}
93+
return rbacclientv1.NewForConfigOrDie(config), nil
94+
}
95+
96+
func getApiextV1Client() (*apiextclientv1.ApiextensionsV1Client, error) {
97+
config, err := getKubeConfig()
98+
if err != nil {
99+
return nil, fmt.Errorf("unable to load build config: %w", err)
100+
}
101+
return apiextclientv1.NewForConfigOrDie(config), nil
102+
}

‎test/util/util.go‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,20 @@ package util
22

33
import (
44
"context"
5+
"fmt"
56
"time"
67

8+
imagev1 "github.com/openshift/api/image/v1"
9+
securityv1 "github.com/openshift/api/security/v1"
10+
"github.com/openshift/cluster-version-operator/lib/resourceread"
11+
"github.com/openshift/library-go/pkg/manifest"
12+
operatorsv1 "github.com/operator-framework/api/pkg/operators/v1"
13+
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
14+
appsv1 "k8s.io/api/apps/v1"
15+
batchv1 "k8s.io/api/batch/v1"
16+
rbacv1 "k8s.io/api/rbac/v1"
17+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
18+
719
g "github.com/onsi/ginkgo/v2"
820
configv1 "github.com/openshift/api/config/v1"
921
clientconfigv1 "github.com/openshift/client-go/config/clientset/versioned"
@@ -16,6 +28,137 @@ import (
1628
"k8s.io/client-go/tools/clientcmd"
1729
)
1830

31+
// GetManifestExpectNotFoundError get manifest to check if it is installed
32+
func GetManifestExpectNotFoundError(ms manifest.Manifest) error {
33+
obj, err := resourceread.Read(ms.Raw)
34+
if err != nil {
35+
return err
36+
}
37+
switch typedObject := obj.(type) {
38+
case *imagev1.ImageStream:
39+
client, err := getImageClient()
40+
if err != nil {
41+
return err
42+
}
43+
_, err = client.ImageStreams(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
44+
return err
45+
case *securityv1.SecurityContextConstraints:
46+
client, err := getSecretClient()
47+
if err != nil {
48+
return err
49+
}
50+
_, err = client.SecurityContextConstraints().Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
51+
return err
52+
case *operatorsv1.OperatorGroup:
53+
client, err := getOperatorClient()
54+
if err != nil {
55+
return err
56+
}
57+
_, err = client.OperatorGroups(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
58+
return err
59+
case *admissionregistrationv1.ValidatingWebhookConfiguration:
60+
client, err := getAdmissionRegistrationClient()
61+
if err != nil {
62+
return err
63+
}
64+
_, err = client.ValidatingWebhookConfigurations().Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
65+
return err
66+
case *appsv1.DaemonSet:
67+
client, err := getAppsClient()
68+
if err != nil {
69+
return err
70+
}
71+
_, err = client.DaemonSets(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
72+
return err
73+
case *appsv1.Deployment:
74+
client, err := getAppsClient()
75+
if err != nil {
76+
return err
77+
}
78+
_, err = client.Deployments(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
79+
return err
80+
case *batchv1.CronJob:
81+
client, err := getBatchClient()
82+
if err != nil {
83+
return err
84+
}
85+
_, err = client.CronJobs(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
86+
return err
87+
case *batchv1.Job:
88+
client, err := getBatchClient()
89+
if err != nil {
90+
return err
91+
}
92+
_, err = client.Jobs(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
93+
return err
94+
case *corev1.ConfigMap:
95+
client, err := getCoreV1Client()
96+
if err != nil {
97+
return err
98+
}
99+
_, err = client.ConfigMaps(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
100+
return err
101+
case *corev1.Namespace:
102+
client, err := getCoreV1Client()
103+
if err != nil {
104+
return err
105+
}
106+
_, err = client.Namespaces().Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
107+
return err
108+
case *corev1.Service:
109+
client, err := getCoreV1Client()
110+
if err != nil {
111+
return err
112+
}
113+
_, err = client.Services(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
114+
return err
115+
case *corev1.ServiceAccount:
116+
client, err := getCoreV1Client()
117+
if err != nil {
118+
return err
119+
}
120+
_, err = client.ServiceAccounts(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
121+
return err
122+
case *rbacv1.ClusterRole:
123+
client, err := getRbacV1Client()
124+
if err != nil {
125+
return err
126+
}
127+
_, err = client.ClusterRoles().Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
128+
return err
129+
case *rbacv1.ClusterRoleBinding:
130+
client, err := getRbacV1Client()
131+
if err != nil {
132+
return err
133+
}
134+
_, err = client.ClusterRoleBindings().Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
135+
return err
136+
case *rbacv1.Role:
137+
client, err := getRbacV1Client()
138+
if err != nil {
139+
return err
140+
}
141+
_, err = client.Roles(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
142+
return err
143+
case *rbacv1.RoleBinding:
144+
client, err := getRbacV1Client()
145+
if err != nil {
146+
return err
147+
}
148+
_, err = client.RoleBindings(typedObject.Namespace).Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
149+
return err
150+
case *apiextensionsv1.CustomResourceDefinition:
151+
client, err := getApiextV1Client()
152+
if err != nil {
153+
return err
154+
}
155+
_, err = client.CustomResourceDefinitions().Get(context.TODO(), typedObject.Name, metav1.GetOptions{})
156+
return err
157+
default:
158+
return fmt.Errorf("unrecognized manifest type: %T", obj)
159+
}
160+
}
161+
19162
// IsHypershift checks if running on a HyperShift hosted cluster
20163
// Refer to https://github.com/openshift/origin/blob/31704414237b8bd5c66ad247c105c94abc9470b1/test/extended/util/framework.go#L2301
21164
func IsHypershift(ctx context.Context, restConfig *rest.Config) (bool, error) {

0 commit comments

Comments
 (0)