WLDI- Test case to add or remove zones, with a CSI restart and WCP service restart in between

tests/e2e/mgmt_wrkld_domain_isolation.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -33,6 +34,7 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/kubernetes/test/e2e/framework"
 	fdep "k8s.io/kubernetes/test/e2e/framework/deployment"
+	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
 	fpod "k8s.io/kubernetes/test/e2e/framework/pod"
 	fpv "k8s.io/kubernetes/test/e2e/framework/pv"
 	fss "k8s.io/kubernetes/test/e2e/framework/statefulset"
@@ -617,4 +619,144 @@ var _ bool = ginkgo.Describe("[domain-isolation] Management-Workload-Domain-Isol
 			allowedTopologies)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	})
+
+	/*
+		Testcase-5
+		Deploy statefulsets with 3 replica on namespace-1 in the supervisor cluster.
+		Use vsan-shared policy with CSI and WCP restart in between.
+		Steps:
+		1. Create a wcp namespace and tagged it to zone-1, zone-2.
+		2. Read a shared storage policy which is tagged to wcp namespace created in step #1 using Immediate Binding mode.
+		3. Create statefulset with replica count 3.
+		4. Wait for PVC and PV to reach Bound state.
+		5. Verify PVC has csi.vsphere.volume-accessible-topology annotation with all zones
+		6. Verify PV has node affinity rule for all zones
+		7. Verify statefulset pod is in up and running state.
+		8. Veirfy Pod node annoation.
+		9. Add zone-3 and zone-4 to the WCP namespace and restart the WCP service at the same time.
+		10. Perform a scaling operation on the StatefulSet, increasing the replica count to 6.
+		11. Wait for the scaling operation to complete successfully.
+		12. Mark zones 1 and 2 for removal from the WCP namespace and restart the CSI while the zone removal is in progress.
+		13. Perform a ScaleUp/ScaleDown operation on the StatefulSet.
+		14. Verify that the scaling operation is completed successfully.
+		15. Verify the StatefulSet PVC annotations and affinity details for the PV.
+		16. Verify the StatefulSet Pod node annotation.
+		17. Verify CNS volume metadata for the Pods and PVCs created.
+		18. Perform cleanup: Delete Statefulset
+		19. Perform cleanup: Delete PVC
+	*/
+
+	ginkgo.It("Add or remove zones, with a CSI restart and WCP service restart in between.", func() {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// statefulset replica count
+		replicas = 3
+
+		//zones used in this test
+		zone1 := topologyAffinityDetails[topologyCategories[0]][0]
+		zone2 := topologyAffinityDetails[topologyCategories[0]][1]
+		zone3 := topologyAffinityDetails[topologyCategories[0]][2]
+		zone4 := topologyAffinityDetails[topologyCategories[0]][3]
+
+		//vc ip
+		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
+
+		// expected status code while add/removing the zones from NS
+		expectedStatusCodes := []int{500, 204}
+
+		// reading shared storage policy
+		storagePolicyName = GetAndExpectStringEnvVar(envSharedStoragePolicyName)
+		storageProfileId = e2eVSphere.GetSpbmPolicyID(storagePolicyName)
+
+		ginkgo.By("Create a WCP namespace tagged to zone-1 & zone-2")
+		// here fetching zone:zone-3 from topologyAffinityDetails
+		namespace = createTestWcpNsWithZones(vcRestSessionId, storageProfileId, getSvcId(vcRestSessionId),
+			[]string{zone1, zone2})
+		defer func() {
+			delTestWcpNs(vcRestSessionId, namespace)
+			gomega.Expect(waitForNamespaceToGetDeleted(ctx, client, namespace, poll, pollTimeout)).To(gomega.Succeed())
+		}()
+
+		ginkgo.By("Read shared storage policy tagged to wcp namespace")
+		storageclass, err := client.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{})
+		if !apierrors.IsNotFound(err) {
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}
+
+		ginkgo.By("Creating service")
+		service := CreateService(namespace, client)
+		defer func() {
+			deleteService(namespace, client, service)
+		}()
+
+		ginkgo.By("Creating statefulset")
+		statefulset := createCustomisedStatefulSets(ctx, client, namespace, true, replicas, false, nil,
+			false, true, "", "", storageclass, storageclass.Name)
+		defer func() {
+			fss.DeleteAllStatefulSets(ctx, client, namespace)
+		}()
+
+		ginkgo.By("Verify svc pv affinity, pvc annotation and pod node affinity")
+		err = verifyPvcAnnotationPvAffinityPodAnnotationInSvc(ctx, client, statefulset, nil, nil, namespace,
+			allowedTopologies)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Add zone-3 and zone-4 to the WCP namespace and restart the WCP service at the same time")
+		var wg sync.WaitGroup
+		wg.Add(3)
+		go addZoneToWcpNsParallel(vcRestSessionId, namespace,
+			zone3, expectedStatusCodes, &wg)
+		go addZoneToWcpNsParallel(vcRestSessionId, namespace,
+			zone4, expectedStatusCodes, &wg)
+		go restartWcp(ctx, vcAddress, &wg)
+		wg.Wait()
+
+		ginkgo.By("Check if namespace has new zones added")
+		output, _, _ := e2ekubectl.RunKubectlWithFullOutput(namespace, "get", "zones")
+		framework.Logf("Check bool %v", !strings.Contains(output, zone3))
+		if !strings.Contains(output, zone3) {
+			framework.Logf("Adding zone-3 to NS might have failed due to WCP restart, adding it again")
+			err = addZoneToWcpNs(vcRestSessionId, namespace, zone3)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}
+		if !strings.Contains(output, zone4) {
+			framework.Logf("Adding zone-4 to NS might have failed due to WCP restart, adding it again")
+			err = addZoneToWcpNs(vcRestSessionId, namespace, zone4)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}
+
+		ginkgo.By("Perform a scaling operation on the StatefulSet, increasing the replica count to 6.")
+		err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client,
+			6, 0, statefulset, true, namespace, allowedTopologies, true, false, false)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Mark zone-1 and zone-2 for removal from wcp namespace and restart the CSI driver at the same time")
+		// Get CSI NS name and replica count
+		csiNamespace := GetAndExpectStringEnvVar(envCSINamespace)
+		csiDeployment, err := client.AppsV1().Deployments(csiNamespace).Get(
+			ctx, vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		csiReplicas := *csiDeployment.Spec.Replicas
+
+		wg.Add(3)
+		go markZoneForRemovalFromWcpNsWithStatusCode(vcRestSessionId, namespace,
+			zone1, expectedStatusCodes, &wg)
+		go markZoneForRemovalFromWcpNsWithStatusCode(vcRestSessionId, namespace,
+			zone2, expectedStatusCodes, &wg)
+		restartstatus, err := restartCSIDriverParallel(ctx, client, csiNamespace, csiReplicas, &wg)
+		gomega.Expect(restartstatus).To(gomega.BeTrue(), "csi driver restart not successful")
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		wg.Wait()
+
+		ginkgo.By("Perform a scaling operation on the StatefulSet, decreasing the replica count to 4.")
+		err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client,
+			4, 0, statefulset, true, namespace, allowedTopologies, true, false, false)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Verify svc pv affinity, pvc annotation and pod node affinity")
+		err = verifyPvcAnnotationPvAffinityPodAnnotationInSvc(ctx, client, statefulset, nil, nil, namespace,
+			allowedTopologies)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	})
 })

tests/e2e/mgmt_wrkld_domain_isolation_utils.go

@@ -24,8 +24,10 @@ import (
 	"math/rand"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -34,6 +36,7 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 	fdep "k8s.io/kubernetes/test/e2e/framework/deployment"
 	fnodes "k8s.io/kubernetes/test/e2e/framework/node"
+	fssh "k8s.io/kubernetes/test/e2e/framework/ssh"
 )
 
 /*
@@ -280,3 +283,99 @@ func invokeVCRestAPIPatchRequest(vcRestSessionId string, url string, reqBody str
 
 	return resp, statusCode
 }
+
+/*
+Add zone to namespace with WaitGroup
+*/
+func addZoneToWcpNsParallel(vcRestSessionId string,
+	namespace string,
+	zoneName string,
+	expectedStatusCode []int,
+	wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	vcIp := e2eVSphere.Config.Global.VCenterHostname
+	AddZoneToNs := "https://" + vcIp + "/api/vcenter/namespaces/instances/" + namespace
+
+	// Create the request body with zone name inside a zones array
+	reqBody := fmt.Sprintf(`{
+        "zones": [{"name": "%s"}]
+    }`, zoneName)
+
+	// Print the request body for debugging
+	fmt.Println(reqBody)
+
+	// Make the API request
+	_, statusCode := invokeVCRestAPIPatchRequest(vcRestSessionId, AddZoneToNs, reqBody)
+
+	if !isAvailable(expectedStatusCode, statusCode) {
+		framework.Logf("failed to add zone %s to NS %s, received status code: %d", zoneName, namespace, statusCode)
+	}
+}
+
+/*
+Restart WCP with WaitGroup
+*/
+func restartWcp(ctx context.Context, vcAddress string, wg *sync.WaitGroup) {
+	defer wg.Done()
+	sshCmd := "vmon-cli -r wcp"
+	framework.Logf("Restarting WCP on vCenter host %v", vcAddress)
+	result, err := fssh.SSH(ctx, sshCmd, vcAddress, framework.TestContext.Provider)
+	fssh.LogResult(result)
+	if err == nil && result.Code == 0 {
+		vcVersion = strings.TrimSpace(result.Stdout)
+	} else {
+		ginkgo.By(fmt.Sprintf("couldn't execute command: %s on vCenter host: %v", sshCmd, err))
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	}
+}
+
+/*
+Mark zone for removal with expected success/failure statuscode.
+*/
+func markZoneForRemovalFromWcpNsWithStatusCode(vcRestSessionId string,
+	namespace string,
+	zone string,
+	expectedStatusCode []int,
+	wg *sync.WaitGroup) {
+	defer wg.Done()
+	vcIp := e2eVSphere.Config.Global.VCenterHostname
+	deleteZoneFromNs := "https://" + vcIp + "/api/vcenter/namespaces/instances/" + namespace + "/zones/" + zone
+	fmt.Println(deleteZoneFromNs)
+	_, statusCode := invokeVCRestAPIDeleteRequest(vcRestSessionId, deleteZoneFromNs)
+	if !isAvailable(expectedStatusCode, statusCode) {
+		framework.Logf("failed to remove zone %s from namespace %s, received status code: %d", zone, namespace, statusCode)
+	}
+}
+
+/*
+Check if given integer list has a value
+*/
+func isAvailable(alpha []int, str int) bool {
+	// iterate using the for loop
+	for i := 0; i < len(alpha); i++ {
+		// check
+		if alpha[i] == str {
+			// return true
+			return true
+		}
+	}
+	return false
+}
+
+/*
+Restart CSI driver with WaitGroup
+*/
+func restartCSIDriverParallel(ctx context.Context, client clientset.Interface, namespace string,
+	csiReplicas int32, wg *sync.WaitGroup) (bool, error) {
+	defer wg.Done()
+	isServiceStopped, err := stopCSIPods(ctx, client, namespace)
+	if err != nil {
+		return isServiceStopped, err
+	}
+	isServiceStarted, err := startCSIPods(ctx, client, csiReplicas, namespace)
+	if err != nil {
+		return isServiceStarted, err
+	}
+	return true, nil
+}