CI: check if PowerVS instances are cleaned up in workspace before creating a cluster

test/e2e/e2e_test.go

@@ -33,10 +33,19 @@ import (
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	"sigs.k8s.io/cluster-api/util"
 
+	"sigs.k8s.io/cluster-api-provider-ibmcloud/test/helpers"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
+const (
+	// powervsRemediationFlavor represents the flavor of PowerVS cluster creation being tested.
+	powervsRemediationFlavor = "powervs-md-remediation"
+	kubernetesVersion        = "KUBERNETES_VERSION"
+	serviceInstanceID        = "IBMPOWERVS_SERVICE_INSTANCE_ID"
+)
+
 var _ = Describe("Workload cluster creation", func() {
 	var (
 		ctx                 = context.TODO()
@@ -55,7 +64,7 @@ var _ = Describe("Workload cluster creation", func() {
 		Expect(bootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName)
 		Expect(os.MkdirAll(artifactFolder, 0750)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName)
 
-		Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
+		Expect(e2eConfig.Variables).To(HaveKey(kubernetesVersion))
 
 		clusterName = fmt.Sprintf("capibm-e2e-%s", util.RandomString(6))
 
@@ -70,6 +79,14 @@ var _ = Describe("Workload cluster creation", func() {
 		// Path to the CNI file is defined in the config
 		Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath), "Missing %s variable in the config", capi_e2e.CNIPath)
 		cniPath = e2eConfig.GetVariable(capi_e2e.CNIPath)
+
+		// When e2e falvour is powervs-md-remediation, check if virtual server instances are cleaned up
+		// before proceeding with cluster creation.
+		if flavor == powervsRemediationFlavor {
+			Expect(e2eConfig.Variables).To(HaveKey(serviceInstanceID))
+			err := helpers.VerifyServerInstancesDeletion(e2eConfig.GetVariable(serviceInstanceID))
+			Expect(err).To(BeNil())
+		}
 	})
 
 	AfterEach(func() {
@@ -100,7 +117,7 @@ var _ = Describe("Workload cluster creation", func() {
 					Flavor:                   flavor,
 					Namespace:                namespace.Name,
 					ClusterName:              clusterName,
-					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
+					KubernetesVersion:        e2eConfig.GetVariable(kubernetesVersion),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(1)),
 				},
@@ -121,7 +138,7 @@ var _ = Describe("Workload cluster creation", func() {
 					Flavor:                   flavor,
 					Namespace:                namespace.Name,
 					ClusterName:              clusterName,
-					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
+					KubernetesVersion:        e2eConfig.GetVariable(kubernetesVersion),
 					ControlPlaneMachineCount: ptr.To(int64(1)),
 					WorkerMachineCount:       ptr.To(int64(3)),
 				},
@@ -146,7 +163,7 @@ var _ = Describe("Workload cluster creation", func() {
 					Flavor:                   flavor,
 					Namespace:                namespace.Name,
 					ClusterName:              clusterName,
-					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
+					KubernetesVersion:        e2eConfig.GetVariable(kubernetesVersion),
 					ControlPlaneMachineCount: ptr.To(int64(3)),
 					WorkerMachineCount:       ptr.To(int64(1)),
 				},

test/e2e/suite_test.go

@@ -43,12 +43,6 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-const (
-	KubernetesVersion = "KUBERNETES_VERSION"
-	CNIPath           = "CNI"
-	CNIResources      = "CNI_RESOURCES"
-)
-
 // Test suite flags.
 var (
 	// configPath is the path to the e2e config file.

test/helpers/powervs.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helpers
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/IBM-Cloud/power-go-client/clients/instance"
+	"github.com/IBM-Cloud/power-go-client/ibmpisession"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/authenticator"
+	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/utils"
+)
+
+var (
+	pollingInterval                = time.Second * 30
+	powerVSInstanceDeletionTimeout = time.Minute * 10
+)
+
+const (
+	powervsInstanceDoesNotExist  = "pvm-instance does not exist"
+	powervsInstanceNotFound      = "could not be found"
+	powervsInstanceStateDeleting = "deleting"
+)
+
+// VerifyServerInstancesDeletion checks if the virtual server instances
+// are deleted in a given PowerVS workspace.
+func VerifyServerInstancesDeletion(serviceInstanceID string) error {
+	pclient, err := getPowerVSInstanceClient(serviceInstanceID)
+	if err != nil {
+		return err
+	}
+
+	instances, err := pclient.GetAll()
+	if err != nil {
+		return err
+	}
+
+	for _, ins := range instances.PvmInstances {
+		err = wait.PollUntilContextTimeout(context.Background(), pollingInterval, powerVSInstanceDeletionTimeout, false, func(_ context.Context) (done bool, err error) {
+			instance, err := pclient.Get(*ins.PvmInstanceID)
+			if err != nil {
+				if strings.Contains(err.Error(), powervsInstanceNotFound) || strings.Contains(err.Error(), powervsInstanceDoesNotExist) {
+					return true, nil
+				}
+				return false, err
+			}
+
+			if instance.TaskState == powervsInstanceStateDeleting {
+				return false, nil
+			}
+			return false, nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getPowerVSInstanceClient(serviceInstanceID string) (*instance.IBMPIInstanceClient, error) {
+	auth, err := authenticator.GetAuthenticator()
+	if err != nil {
+		return nil, err
+	}
+
+	zone := os.Getenv("IBMPOWERVS_ZONE")
+	if zone == "" {
+		return nil, fmt.Errorf("IBMPOWERVS_ZONE is not set")
+	}
+
+	account, err := utils.GetAccount(auth)
+	if err != nil {
+		return nil, err
+	}
+
+	piOptions := ibmpisession.IBMPIOptions{
+		Authenticator: auth,
+		UserAccount:   account,
+		Zone:          zone,
+		Debug:         true,
+	}
+
+	session, err := ibmpisession.NewIBMPISession(&piOptions)
+	if err != nil {
+		return nil, err
+	}
+	return instance.NewIBMPIInstanceClient(context.Background(), session, serviceInstanceID), nil
+}