Skip to content

Commit f666450

Browse files
Aishwarya-HebbarAishwarya-Hebbar
authored
vsan stretch automation for TKG, WCP and VMService VMs (#3020)
1 parent 1fe34da commit f666450

22 files changed

+4752
-3170
lines changed

‎go.sum

+1-1
Original file line numberDiff line numberDiff line change
@@ -963,4 +963,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s
963963
sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
964964
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
965965
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
966-
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
966+
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

‎tests/e2e/crypto.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ var _ = ginkgo.Describe("[csi-supervisor] [encryption] Block volume encryption",
121121
vmImageName := GetAndExpectStringEnvVar(envVmsvcVmImageName)
122122
framework.Logf("Waiting for virtual machine image list to be available in namespace '%s' for image '%s'",
123123
namespace, vmImageName)
124-
vmi = waitNGetVmiForImageName(ctx, vmopClient, namespace, vmImageName)
124+
vmi = waitNGetVmiForImageName(ctx, vmopClient, vmImageName)
125125
gomega.Expect(vmi).NotTo(gomega.BeEmpty())
126126
})
127127

‎tests/e2e/crypto_snapshot.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ var _ = ginkgo.Describe("[csi-supervisor] [encryption] Block volume snapshot enc
135135
vmImageName := GetAndExpectStringEnvVar(envVmsvcVmImageName)
136136
framework.Logf("Waiting for virtual machine image list to be available in namespace '%s' for image '%s'",
137137
namespace, vmImageName)
138-
vmi = waitNGetVmiForImageName(ctx, vmopClient, namespace, vmImageName)
138+
vmi = waitNGetVmiForImageName(ctx, vmopClient, vmImageName)
139139
gomega.Expect(vmi).NotTo(gomega.BeEmpty())
140140
})
141141

‎tests/e2e/csi_static_provisioning_basic.go

-1
Original file line numberDiff line numberDiff line change
@@ -2482,5 +2482,4 @@ var _ = ginkgo.Describe("Basic Static Provisioning", func() {
24822482
gomega.Expect(err).NotTo(gomega.HaveOccurred())
24832483

24842484
})
2485-
24862485
})

‎tests/e2e/e2e_common.go

+36-32
Original file line numberDiff line numberDiff line change
@@ -288,38 +288,42 @@ customPort -> label include the testcases running on vCenter custom port <VC:444
288288
deprecated ->label include the testcases which are no longer in execution
289289
*/
290290
const (
291-
flaky = "flaky"
292-
disruptive = "disruptive"
293-
wcp = "wcp"
294-
tkg = "tkg"
295-
vanilla = "vanilla"
296-
preferential = "preferential"
297-
vsphereConfigSecret = "vsphereConfigSecret"
298-
snapshot = "snapshot"
299-
stable = "stable"
300-
newTest = "newTest"
301-
multiVc = "multiVc"
302-
block = "block"
303-
file = "file"
304-
core = "core"
305-
hci = "hci"
306-
p0 = "p0"
307-
p1 = "p1"
308-
p2 = "p2"
309-
vsanStretch = "vsanStretch"
310-
longRunning = "longRunning"
311-
deprecated = "deprecated"
312-
vmc = "vmc"
313-
tkgsHA = "tkgsHA"
314-
thickThin = "thickThin"
315-
customPort = "customPort"
316-
windows = "windows"
317-
semiAutomated = "semiAutomated"
318-
level2 = "level2"
319-
level5 = "level5"
320-
negative = "negative"
321-
listVolume = "listVolume"
322-
multiSvc = "multiSvc"
291+
flaky = "flaky"
292+
disruptive = "disruptive"
293+
wcp = "wcp"
294+
tkg = "tkg"
295+
vanilla = "vanilla"
296+
preferential = "preferential"
297+
vsphereConfigSecret = "vsphereConfigSecret"
298+
snapshot = "snapshot"
299+
stable = "stable"
300+
newTest = "newTest"
301+
multiVc = "multiVc"
302+
block = "block"
303+
file = "file"
304+
core = "core"
305+
hci = "hci"
306+
p0 = "p0"
307+
p1 = "p1"
308+
p2 = "p2"
309+
vsanStretch = "vsanStretch"
310+
longRunning = "longRunning"
311+
deprecated = "deprecated"
312+
vmc = "vmc"
313+
tkgsHA = "tkgsHA"
314+
thickThin = "thickThin"
315+
customPort = "customPort"
316+
windows = "windows"
317+
semiAutomated = "semiAutomated"
318+
level2 = "level2"
319+
level5 = "level5"
320+
negative = "negative"
321+
listVolume = "listVolume"
322+
multiSvc = "multiSvc"
323+
primaryCentric = "primaryCentric"
324+
controlPlaneOnPrimary = "controlPlaneOnPrimary"
325+
distributed = "distributed"
326+
vmsvc = "vmsvc"
323327
)
324328

325329
// The following variables are required to know cluster type to run common e2e

‎tests/e2e/hci.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ var _ bool = ginkgo.Describe("hci", func() {
462462
}()
463463

464464
ginkgo.By("Wait for 5-10 mins, verify that the k8s-worker is restarted and brought up on another host")
465-
wait4AllK8sNodesToBeUp(ctx, client, nodes)
465+
wait4AllK8sNodesToBeUp(nodes)
466466
gomega.Expect(waitForAllNodes2BeReady(ctx, client)).To(gomega.Succeed())
467467

468468
ginkgo.By("Scale up statefulset2 and scale down statefulset1 to 2 replicas")

‎tests/e2e/hci_mesh_rwx_disruptive.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
677677

678678
ginkgo.By("PSOD all host in remote cluster4 and when psod is triggered, create new set of rwx pvc")
679679
for i := 0; i < len(hostListCluster4); i++ {
680-
err = psodHost(hostListCluster4[i])
680+
err = psodHost(hostListCluster4[i], "")
681681
gomega.Expect(err).NotTo(gomega.HaveOccurred())
682682

683683
if i == 0 {
@@ -729,7 +729,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
729729
ginkgo.By("PSOD again all host in remote cluster4 and perform scaleup " +
730730
"operation on deployment and statefulset")
731731
for i := 0; i < len(hostListCluster4); i++ {
732-
err = psodHost(hostListCluster4[i])
732+
err = psodHost(hostListCluster4[i], "")
733733
gomega.Expect(err).NotTo(gomega.HaveOccurred())
734734

735735
if i == 0 {
@@ -1873,7 +1873,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
18731873

18741874
ginkgo.By("PSOD all host in local cluster2 and when psod is triggered, create new set of rwx pvc")
18751875
for i := 0; i < len(hostListCluster2); i++ {
1876-
err = psodHost(hostListCluster2[i])
1876+
err = psodHost(hostListCluster2[i], "")
18771877
gomega.Expect(err).NotTo(gomega.HaveOccurred())
18781878

18791879
if i == 0 {
@@ -1929,7 +1929,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
19291929
ginkgo.By("PSOD all host in local cluster3 and perform scaleup " +
19301930
"operation on deployment and statefulset")
19311931
for i := 0; i < len(hostListCluster3); i++ {
1932-
err = psodHost(hostListCluster3[i])
1932+
err = psodHost(hostListCluster3[i], "")
19331933
gomega.Expect(err).NotTo(gomega.HaveOccurred())
19341934

19351935
if i == 0 {

‎tests/e2e/multi_vc_operation_storm.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ var _ = ginkgo.Describe("[multivc-operationstorm] MultiVc-OperationStorm", func(
287287
}
288288

289289
ginkgo.By("Wait for k8s cluster to be healthy")
290-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
290+
wait4AllK8sNodesToBeUp(nodeList)
291291
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
292292
gomega.Expect(err).NotTo(gomega.HaveOccurred())
293293
}()
@@ -349,7 +349,7 @@ var _ = ginkgo.Describe("[multivc-operationstorm] MultiVc-OperationStorm", func(
349349

350350
// Wait for k8s cluster to be healthy
351351
ginkgo.By("Wait for k8s cluster to be healthy")
352-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
352+
wait4AllK8sNodesToBeUp(nodeList)
353353
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
354354
gomega.Expect(err).NotTo(gomega.HaveOccurred())
355355

‎tests/e2e/multi_vc_site_down.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -234,14 +234,14 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
234234
}
235235

236236
ginkgo.By("Wait for k8s cluster to be healthy")
237-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
237+
wait4AllK8sNodesToBeUp(nodeList)
238238
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
239239
gomega.Expect(err).NotTo(gomega.HaveOccurred())
240240

241241
}()
242242

243243
ginkgo.By("Wait for k8s cluster to be healthy")
244-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
244+
wait4AllK8sNodesToBeUp(nodeList)
245245
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
246246
gomega.Expect(err).NotTo(gomega.HaveOccurred())
247247

@@ -383,14 +383,14 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
383383
}
384384

385385
ginkgo.By("Wait for k8s cluster to be healthy")
386-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
386+
wait4AllK8sNodesToBeUp(nodeList)
387387
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
388388
gomega.Expect(err).NotTo(gomega.HaveOccurred())
389389

390390
}()
391391

392392
ginkgo.By("Wait for k8s cluster to be healthy")
393-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
393+
wait4AllK8sNodesToBeUp(nodeList)
394394
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
395395
gomega.Expect(err).NotTo(gomega.HaveOccurred())
396396

@@ -493,7 +493,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
493493
}
494494

495495
ginkgo.By("Wait for k8s cluster to be healthy")
496-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
496+
wait4AllK8sNodesToBeUp(nodeList)
497497
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
498498
gomega.Expect(err).NotTo(gomega.HaveOccurred())
499499

@@ -505,7 +505,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
505505
}
506506

507507
ginkgo.By("Wait for k8s cluster to be healthy")
508-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
508+
wait4AllK8sNodesToBeUp(nodeList)
509509
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
510510
gomega.Expect(err).NotTo(gomega.HaveOccurred())
511511

@@ -632,7 +632,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
632632
}
633633

634634
ginkgo.By("Wait for k8s cluster to be healthy")
635-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
635+
wait4AllK8sNodesToBeUp(nodeList)
636636
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
637637
gomega.Expect(err).NotTo(gomega.HaveOccurred())
638638

@@ -644,7 +644,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
644644
}
645645

646646
ginkgo.By("Wait for k8s cluster to be healthy")
647-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
647+
wait4AllK8sNodesToBeUp(nodeList)
648648
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
649649
gomega.Expect(err).NotTo(gomega.HaveOccurred())
650650

‎tests/e2e/no_hci_mesh_rwx_multivc_disruptive.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
292292
}
293293

294294
ginkgo.By("Wait for k8s cluster to be healthy")
295-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
295+
wait4AllK8sNodesToBeUp(nodeList)
296296
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
297297
gomega.Expect(err).NotTo(gomega.HaveOccurred())
298298

@@ -517,7 +517,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
517517
}
518518

519519
ginkgo.By("Wait for k8s cluster to be healthy")
520-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
520+
wait4AllK8sNodesToBeUp(nodeList)
521521
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
522522
gomega.Expect(err).NotTo(gomega.HaveOccurred())
523523

@@ -787,7 +787,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
787787
}
788788

789789
ginkgo.By("Wait for k8s cluster to be healthy")
790-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
790+
wait4AllK8sNodesToBeUp(nodeList)
791791
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
792792
gomega.Expect(err).NotTo(gomega.HaveOccurred())
793793
}()
@@ -824,7 +824,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
824824
}
825825

826826
ginkgo.By("Wait for k8s cluster to be healthy")
827-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
827+
wait4AllK8sNodesToBeUp(nodeList)
828828
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
829829
gomega.Expect(err).NotTo(gomega.HaveOccurred())
830830

‎tests/e2e/no_hci_mesh_rwx_singlevc_topology_disruptive.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ var _ = ginkgo.Describe("[rwx-nohci-singlevc-disruptive] RWX-Topology-NoHciMesh-
752752

753753
ginkgo.By("PSOD all host")
754754
for i := 0; i < len(hostList); i++ {
755-
err = psodHost(hostList[i])
755+
err = psodHost(hostList[i], "")
756756
gomega.Expect(err).NotTo(gomega.HaveOccurred())
757757

758758
if i == 2 {

‎tests/e2e/preferential_topology_disruptive.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ var _ = ginkgo.Describe("[preferential-disruptive] Preferential-Topology-Disrupt
365365
}()
366366

367367
ginkgo.By("Wait for k8s cluster to be healthy")
368-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
368+
wait4AllK8sNodesToBeUp(nodeList)
369369
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
370370
gomega.Expect(err).NotTo(gomega.HaveOccurred())
371371

@@ -429,7 +429,7 @@ var _ = ginkgo.Describe("[preferential-disruptive] Preferential-Topology-Disrupt
429429
}
430430

431431
ginkgo.By("Wait for k8s cluster to be healthy")
432-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
432+
wait4AllK8sNodesToBeUp(nodeList)
433433
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
434434
gomega.Expect(err).NotTo(gomega.HaveOccurred())
435435

‎tests/e2e/rwx_topology_utils.go

+12-6
Original file line numberDiff line numberDiff line change
@@ -886,7 +886,7 @@ func verifyK8sNodeStatusAfterSiteRecovery(client clientset.Interface, ctx contex
886886
time.Sleep(pollTimeoutShort)
887887

888888
ginkgo.By("Wait for k8s cluster to be healthy")
889-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
889+
wait4AllK8sNodesToBeUp(nodeList)
890890
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
891891
if err != nil {
892892
return fmt.Errorf("error waiting for all nodes to be ready: %w", err)
@@ -895,18 +895,24 @@ func verifyK8sNodeStatusAfterSiteRecovery(client clientset.Interface, ctx contex
895895
}
896896

897897
/* This util will perform psod operation on a host */
898-
func psodHost(hostIP string) error {
898+
func psodHost(hostIP string, psodTimeOut string) error {
899899
ginkgo.By("PSOD")
900-
sshCmd := fmt.Sprintf("vsish -e set /config/Misc/intOpts/BlueScreenTimeout %s", psodTime)
901-
op, err := runCommandOnESX("root", hostIP, sshCmd)
900+
var timeout string
901+
if psodTimeOut != "" {
902+
timeout = psodTimeOut
903+
} else {
904+
timeout = psodTime
905+
}
906+
sshCmd := fmt.Sprintf("vsish -e set /config/Misc/intOpts/BlueScreenTimeout %s", timeout)
907+
op, err := runCommandOnESX(rootUser, hostIP, sshCmd)
902908
framework.Logf(op)
903909
if err != nil {
904910
return fmt.Errorf("failed to set BlueScreenTimeout: %w", err)
905911
}
906912

907913
ginkgo.By("Injecting PSOD")
908-
psodCmd := "vsish -e set /reliability/crashMe/Panic 1"
909-
op, err = runCommandOnESX("root", hostIP, psodCmd)
914+
psodCmd := "vsish -e set /reliability/crashMe/Panic 1; exit"
915+
op, err = runCommandOnESX(rootUser, hostIP, psodCmd)
910916
framework.Logf(op)
911917
if err != nil {
912918
return fmt.Errorf("failed to inject PSOD: %w", err)

‎tests/e2e/snapshot_vmservice_vm.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ var _ bool = ginkgo.Describe("[snapshot-vmsvc] Snapshot VM Service VM", func() {
145145
vmImageName := GetAndExpectStringEnvVar(envVmsvcVmImageName)
146146
framework.Logf("Waiting for virtual machine image list to be available in namespace '%s' for image '%s'",
147147
namespace, vmImageName)
148-
vmi = waitNGetVmiForImageName(ctx, vmopC, namespace, vmImageName)
148+
vmi = waitNGetVmiForImageName(ctx, vmopC, vmImageName)
149149
gomega.Expect(vmi).NotTo(gomega.BeEmpty())
150150

151151
// Get snapshot client using the rest config

‎tests/e2e/topology_operation_strom_cases.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
223223
}()
224224

225225
ginkgo.By("Wait for k8s cluster to be healthy")
226-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
226+
wait4AllK8sNodesToBeUp(nodeList)
227227
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
228228
gomega.Expect(err).NotTo(gomega.HaveOccurred())
229229

@@ -261,7 +261,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
261261
}
262262

263263
ginkgo.By("Wait for k8s cluster to be healthy")
264-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
264+
wait4AllK8sNodesToBeUp(nodeList)
265265
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
266266
gomega.Expect(err).NotTo(gomega.HaveOccurred())
267267

@@ -287,7 +287,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
287287
}()
288288

289289
ginkgo.By("Wait for k8s cluster to be healthy")
290-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
290+
wait4AllK8sNodesToBeUp(nodeList)
291291
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
292292
gomega.Expect(err).NotTo(gomega.HaveOccurred())
293293

@@ -326,7 +326,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
326326
}
327327

328328
ginkgo.By("Wait for k8s cluster to be healthy")
329-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
329+
wait4AllK8sNodesToBeUp(nodeList)
330330
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
331331
gomega.Expect(err).NotTo(gomega.HaveOccurred())
332332

@@ -557,7 +557,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
557557
}
558558

559559
ginkgo.By("Wait for k8s cluster to be healthy")
560-
wait4AllK8sNodesToBeUp(ctx, client, nodeList)
560+
wait4AllK8sNodesToBeUp(nodeList)
561561
err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
562562
gomega.Expect(err).NotTo(gomega.HaveOccurred())
563563

0 commit comments

Comments
 (0)