Skip to content

Commit 8b0c1bc

Browse files
lubronzhanlubronzhan
committed
Fix the integration test logic when there are multiple test competing with each other
Signed-off-by: Lubron Zhan <[email protected]>
1 parent cadc346 commit 8b0c1bc

File tree

2 files changed

+106
-82
lines changed

2 files changed

+106
-82
lines changed

‎controllers/akodeploymentconfig/akodeploymentconfig_controller_intg_test.go

+103-80
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
corev1 "k8s.io/api/core/v1"
1616
apierrors "k8s.io/apimachinery/pkg/api/errors"
1717
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
18+
utilerrors "k8s.io/apimachinery/pkg/util/errors"
1819
"k8s.io/utils/pointer"
1920
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2021
"sigs.k8s.io/cluster-api/util/conditions"
@@ -26,6 +27,7 @@ import (
2627
"github.com/vmware-tanzu/load-balancer-operator-for-kubernetes/pkg/ako"
2728
ako_operator "github.com/vmware-tanzu/load-balancer-operator-for-kubernetes/pkg/ako-operator"
2829
"github.com/vmware-tanzu/load-balancer-operator-for-kubernetes/pkg/test/builder"
30+
"github.com/vmware-tanzu/load-balancer-operator-for-kubernetes/pkg/test/util"
2931
testutil "github.com/vmware-tanzu/load-balancer-operator-for-kubernetes/pkg/test/util"
3032
)
3133

@@ -90,11 +92,28 @@ func intgTestAkoDeploymentConfigController() {
9092
Expect(err).To(BeNil())
9193
}
9294
}
93-
deleteObjects := func(objs ...client.Object) {
94-
for _, o := range objs {
95-
// ignore error
96-
_ = ctx.Client.Delete(ctx.Context, o)
95+
96+
ensureObjectsDeleted := func(obj ...client.Object) error {
97+
errs := []error{}
98+
for _, o := range obj {
99+
if err := ctx.Client.Get(ctx, client.ObjectKeyFromObject(o), o); err != nil {
100+
if apierrors.IsNotFound(err) {
101+
return nil
102+
}
103+
errs = append(errs, err)
104+
}
105+
106+
if err := ctx.Client.Delete(ctx, o); err != nil {
107+
if !apierrors.IsNotFound(err) {
108+
errs = append(errs, err)
109+
}
110+
}
111+
}
112+
113+
if len(errs) > 0 {
114+
return utilerrors.NewAggregate(errs)
97115
}
116+
return nil
98117
}
99118
getCluster := func(obj *clusterv1.Cluster, name, namespace string) error {
100119
err := ctx.Client.Get(ctx.Context, client.ObjectKey{
@@ -182,7 +201,19 @@ func intgTestAkoDeploymentConfigController() {
182201
}
183202
_, ok := obj.Labels[label]
184203
return expect == ok
185-
}, "30s", "5s").Should(BeTrue())
204+
}).Should(BeTrue())
205+
}
206+
207+
ensureClusterAviLabelValueMatchExpectation := func(key client.ObjectKey, label, value string, expect bool) {
208+
Eventually(func() bool {
209+
obj := &clusterv1.Cluster{}
210+
err := ctx.Client.Get(ctx.Context, key, obj)
211+
if err != nil {
212+
return false
213+
}
214+
v := obj.Labels[label]
215+
return (v == value) == expect
216+
}).Should(BeTrue())
186217
}
187218

188219
ensureSubnetMatchExpectation := func(newIPAddrEnd string, expect bool) {
@@ -306,25 +337,15 @@ func intgTestAkoDeploymentConfigController() {
306337

307338
})
308339
AfterEach(func() {
309-
latestCluster := &clusterv1.Cluster{}
310-
if err := getCluster(latestCluster, cluster.Name, cluster.Namespace); err == nil {
311-
latestCluster.Finalizers = nil
312-
deleteObjects(latestCluster)
313-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
340+
latestCluster := &clusterv1.Cluster{
341+
ObjectMeta: metav1.ObjectMeta{
314342
Name: cluster.Name,
315343
Namespace: cluster.Namespace,
316-
}, &clusterv1.Cluster{}, false)
344+
},
317345
}
318-
deleteObjects(cluster)
319-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
320-
Name: cluster.Name,
321-
Namespace: cluster.Namespace,
322-
}, &clusterv1.Cluster{}, false)
323-
deleteObjects(akoDeploymentConfig)
324-
deleteObjects(controllerCredentials, controllerCA)
325-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
326-
Name: akoDeploymentConfig.Name,
327-
}, &akoov1alpha1.AKODeploymentConfig{}, false)
346+
Eventually(func() error {
347+
return ensureObjectsDeleted(latestCluster, akoDeploymentConfig, controllerCredentials, controllerCA)
348+
}, "30s", "5s").Should(BeNil())
328349
err := os.Setenv(ako_operator.IsControlPlaneHAProvider, "False")
329350
Expect(err).ShouldNot(HaveOccurred())
330351
})
@@ -369,27 +390,22 @@ func intgTestAkoDeploymentConfigController() {
369390
BeforeEach(func() {
370391
createObjects(akoDeploymentConfig, cluster, controllerCredentials, controllerCA)
371392
conditions.MarkTrue(cluster, clusterv1.ReadyCondition)
372-
err = ctx.Client.Status().Update(ctx, cluster)
373-
Expect(err).To(BeNil())
393+
Eventually(func() error {
394+
err = ctx.Client.Status().Update(ctx, cluster)
395+
return err
396+
}).Should(BeNil())
374397
_ = kcfg.CreateSecret(ctx, ctx.Client, cluster)
375398
})
376399
AfterEach(func() {
377-
latestCluster := &clusterv1.Cluster{}
378-
if err := getCluster(latestCluster, cluster.Name, cluster.Namespace); err == nil {
379-
latestCluster.Finalizers = nil
380-
deleteObjects(latestCluster)
381-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
400+
latestCluster := &clusterv1.Cluster{
401+
ObjectMeta: metav1.ObjectMeta{
382402
Name: cluster.Name,
383403
Namespace: cluster.Namespace,
384-
}, &clusterv1.Cluster{}, false)
404+
},
385405
}
386-
387-
deleteObjects(akoDeploymentConfig)
388-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
389-
Name: akoDeploymentConfig.Name,
390-
}, &akoov1alpha1.AKODeploymentConfig{}, false)
391-
392-
deleteObjects(controllerCredentials, controllerCA)
406+
Eventually(func() error {
407+
return ensureObjectsDeleted(latestCluster, akoDeploymentConfig, controllerCredentials, controllerCA)
408+
}, "30s", "5s").Should(BeNil())
393409
})
394410
When("there is no matching cluster", func() {
395411
It("cluster should not have ClusterFinalizer", func() {
@@ -618,18 +634,15 @@ func intgTestAkoDeploymentConfigController() {
618634
When("the cluster is being deleted ", func() {
619635
When("the cluster is ready", func() {
620636
BeforeEach(func() {
621-
latestCluster := &clusterv1.Cluster{}
622-
err := getCluster(latestCluster, cluster.Name, cluster.Namespace)
623-
Expect(err).To(BeNil())
624-
conditions.MarkTrue(latestCluster, akoov1alpha1.AviResourceCleanupSucceededCondition)
625-
err = ctx.Client.Status().Update(ctx, latestCluster)
626-
Expect(err).To(BeNil())
627-
deleteObjects(latestCluster)
628-
629-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
630-
Name: cluster.Name,
631-
Namespace: cluster.Namespace,
632-
}, &clusterv1.Cluster{}, false)
637+
latestCluster := &clusterv1.Cluster{
638+
ObjectMeta: metav1.ObjectMeta{
639+
Name: cluster.Name,
640+
Namespace: cluster.Namespace,
641+
},
642+
}
643+
Eventually(func() error {
644+
return ensureObjectsDeleted(latestCluster)
645+
}).Should(BeNil())
633646
})
634647

635648
//Reconcile -> reconcileNormal -> r.userReconciler.ReconcileAviUserDelete
@@ -647,13 +660,13 @@ func intgTestAkoDeploymentConfigController() {
647660
Expect(err).To(BeNil())
648661
conditions.MarkFalse(obj, clusterv1.ReadyCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "")
649662
conditions.MarkTrue(obj, akoov1alpha1.AviResourceCleanupSucceededCondition)
650-
err = ctx.Client.Status().Update(ctx, obj)
651-
Expect(err).To(BeNil())
652-
deleteObjects(obj)
653-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
654-
Name: obj.Name,
655-
Namespace: obj.Namespace,
656-
}, &clusterv1.Cluster{}, false)
663+
Eventually(func() error {
664+
err = ctx.Client.Status().Update(ctx, obj)
665+
return err
666+
}).Should(BeNil())
667+
Eventually(func() error {
668+
return ensureObjectsDeleted(obj)
669+
}).Should(BeNil())
657670
})
658671

659672
//Reconcile -> reconcileNormal -> r.userReconciler.ReconcileAviUserDelete
@@ -671,10 +684,9 @@ func intgTestAkoDeploymentConfigController() {
671684
// Reconcile -> reconcileDelete
672685
When("AKODeploymentConfig is being deleted", func() {
673686
BeforeEach(func() {
674-
deleteObjects(akoDeploymentConfig)
675-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
676-
Name: akoDeploymentConfig.Name,
677-
}, &akoov1alpha1.AKODeploymentConfig{}, false)
687+
Eventually(func() error {
688+
return ensureObjectsDeleted(akoDeploymentConfig)
689+
}).Should(BeNil())
678690
})
679691

680692
// Reconcile -> reconcileDelete -> phases.ReconcilePhases(normal)
@@ -704,11 +716,9 @@ func intgTestAkoDeploymentConfigController() {
704716

705717
When("the cluster is being deleted ", func() {
706718
BeforeEach(func() {
707-
deleteObjects(cluster)
708-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
709-
Name: cluster.Name,
710-
Namespace: cluster.Namespace,
711-
}, &clusterv1.Cluster{}, false)
719+
Eventually(func() error {
720+
return ensureObjectsDeleted(cluster)
721+
}).Should(BeNil())
712722
})
713723

714724
//Reconcile -> reconcileDelete -> r.reconcileClustersDelete -> r.clusterReconciler.ReconcileAddonSecretDelete
@@ -722,7 +732,7 @@ func intgTestAkoDeploymentConfigController() {
722732
})
723733
})
724734

725-
// Tests for adding & removing the networking.tkg.tanzu.vmware.com/avi-skip-default-adc labels
735+
// Tests when there are multpile ADC selecting the same cluster.
726736
// When there is matching cluster for ADC -> and when there is another ADC install-ako-for-all
727737
defaultAkoDeploymentConfig := staticDefaultAkoDeploymentConfig.DeepCopy()
728738
defaultAkoDeploymentConfigWithNonEmptyClusterSelector := staticDefaultAkoDeploymentConfig.DeepCopy()
@@ -732,14 +742,16 @@ func intgTestAkoDeploymentConfigController() {
732742

733743
defaultADCTestCaseInputs := []DefaultADCTestCaseInput{
734744
{
735-
Name: "there is default ADC install-ako-for-all",
736-
DefaultADC: defaultAkoDeploymentConfig,
745+
Name: "there is default ADC install-ako-for-all",
746+
DefaultADC: defaultAkoDeploymentConfig,
747+
HasSelector: false,
737748
},
738749
{
739750
// This test case covers the bug https://github.com/vmware-tanzu/load-balancer-operator-for-kubernetes/pull/81
740751
// The bug was triggerred when the default workload ADC install-ako-for-all has non-empty cluster selector
741-
Name: "there is default ADC with non-empty clusterSelector",
742-
DefaultADC: defaultAkoDeploymentConfigWithNonEmptyClusterSelector,
752+
Name: "there is default ADC with non-empty clusterSelector",
753+
DefaultADC: defaultAkoDeploymentConfigWithNonEmptyClusterSelector,
754+
HasSelector: true,
743755
},
744756
}
745757

@@ -755,17 +767,20 @@ func intgTestAkoDeploymentConfigController() {
755767
})
756768

757769
AfterEach(func() {
758-
deleteObjects(defaultADC)
759-
ensureRuntimeObjectMatchExpectation(client.ObjectKey{
760-
Name: akoov1alpha1.WorkloadClusterAkoDeploymentConfig,
761-
}, &akoov1alpha1.AKODeploymentConfig{}, false)
770+
Eventually(func() error {
771+
return ensureObjectsDeleted(defaultADC)
772+
}).Should(BeNil())
773+
ensureClusterAviLabelMatchExpectation(client.ObjectKey{
774+
Name: cluster.Name,
775+
Namespace: cluster.Namespace,
776+
}, akoov1alpha1.AviClusterLabel, false)
762777
})
763778

764779
It("is selected by a customized ADC", func() {
765-
ensureClusterAviLabelMatchExpectation(client.ObjectKey{
780+
ensureClusterAviLabelValueMatchExpectation(client.ObjectKey{
766781
Name: cluster.Name,
767782
Namespace: cluster.Namespace,
768-
}, akoov1alpha1.AviClusterLabel, true)
783+
}, akoov1alpha1.AviClusterLabel, util.CustomADCName, true)
769784
})
770785

771786
When("no longer selected by a customized ADC", func() {
@@ -781,11 +796,18 @@ func intgTestAkoDeploymentConfigController() {
781796
}, "test", false)
782797
})
783798

784-
It("should drop the AviClusterLabel)", func() {
785-
ensureClusterAviLabelMatchExpectation(client.ObjectKey{
799+
It("picks up the default ADC if there is no selector on default ADC", func() {
800+
ensureClusterAviLabelValueMatchExpectation(client.ObjectKey{
786801
Name: cluster.Name,
787802
Namespace: cluster.Namespace,
788-
}, akoov1alpha1.AviClusterLabel, false)
803+
}, akoov1alpha1.AviClusterLabel, akoov1alpha1.WorkloadClusterAkoDeploymentConfig, !tc.HasSelector)
804+
})
805+
806+
It("should drop the AviClusterLabel for custom ADC", func() {
807+
ensureClusterAviLabelValueMatchExpectation(client.ObjectKey{
808+
Name: cluster.Name,
809+
Namespace: cluster.Namespace,
810+
}, akoov1alpha1.AviClusterLabel, util.CustomADCName, false)
789811
})
790812
})
791813
})
@@ -795,6 +817,7 @@ func intgTestAkoDeploymentConfigController() {
795817
}
796818

797819
type DefaultADCTestCaseInput struct {
798-
Name string // test case name
799-
DefaultADC *akoov1alpha1.AKODeploymentConfig // default ADC input
820+
Name string // test case name
821+
DefaultADC *akoov1alpha1.AKODeploymentConfig // default ADC input
822+
HasSelector bool // whether the default ADC has a cluster selector
800823
}

‎pkg/test/util/data.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ import (
1515

1616
const (
1717
// CharSet defines the alphanumeric set for random string generation.
18-
CharSet = "0123456789abcdefghijklmnopqrstuvwxyz"
18+
CharSet = "0123456789abcdefghijklmnopqrstuvwxyz"
19+
CustomADCName = "ako-deployment-config"
1920
)
2021

2122
var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec
@@ -96,7 +97,7 @@ func GetCustomizedADC(labels map[string]string) *akoov1alpha1.AKODeploymentConfi
9697
}
9798
return &akoov1alpha1.AKODeploymentConfig{
9899
ObjectMeta: metav1.ObjectMeta{
99-
Name: "ako-deployment-config",
100+
Name: CustomADCName,
100101
},
101102
Spec: *spec,
102103
}

0 commit comments

Comments
 (0)