Skip to content

Commit ccaea78

Browse files
chrischdichrischdi
authored
🌱 e2e: create ExtensionConfig including name in settings and create one… (#11956)
* e2e: create ExtensionConfig including name in settings and create one cm per ExtensionConfig * e2e: limit scale-test's ExtensionConfig to only spec related namespaces * fixes * review fixes
1 parent a093b06 commit ccaea78

File tree

5 files changed

+100
-50
lines changed

5 files changed

+100
-50
lines changed

‎test/e2e/cluster_upgrade_runtimesdk.go

+31-26
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ import (
5151

5252
var hookResponsesConfigMapNameSuffix = "test-extension-hookresponses"
5353

54-
func hookResponsesConfigMapName(clusterName string) string {
55-
return fmt.Sprintf("%s-%s", clusterName, hookResponsesConfigMapNameSuffix)
54+
func hookResponsesConfigMapName(clusterName, extensionConfigName string) string {
55+
return fmt.Sprintf("%s-%s-%s", clusterName, extensionConfigName, hookResponsesConfigMapNameSuffix)
5656
}
5757

5858
// ClusterUpgradeWithRuntimeSDKSpecInput is the input for clusterUpgradeWithRuntimeSDKSpec.
@@ -229,6 +229,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
229229
beforeClusterCreateTestHandler(ctx,
230230
input.BootstrapClusterProxy.GetClient(),
231231
clusterRef,
232+
input.ExtensionConfigName,
232233
input.E2EConfig.GetIntervals(specName, "wait-cluster"))
233234
},
234235
PostMachinesProvisioned: func() {
@@ -295,17 +296,20 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
295296
beforeClusterUpgradeTestHandler(ctx,
296297
input.BootstrapClusterProxy.GetClient(),
297298
clusterRef,
299+
input.ExtensionConfigName,
298300
input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeTo),
299301
input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"))
300302
},
301303
PreWaitForWorkersToBeUpgraded: func() {
302304
machineSetPreflightChecksTestHandler(ctx,
303305
input.BootstrapClusterProxy.GetClient(),
304-
clusterRef)
306+
clusterRef,
307+
input.ExtensionConfigName)
305308

306309
afterControlPlaneUpgradeTestHandler(ctx,
307310
input.BootstrapClusterProxy.GetClient(),
308311
clusterRef,
312+
input.ExtensionConfigName,
309313
input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeTo),
310314
input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"))
311315
},
@@ -329,11 +333,11 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
329333
By("Dumping resources and deleting the workload cluster; deletion waits for BeforeClusterDeleteHook to gate the operation")
330334
dumpAndDeleteCluster(ctx, input.BootstrapClusterProxy, namespace.Name, clusterName, input.ArtifactFolder)
331335

332-
beforeClusterDeleteHandler(ctx, input.BootstrapClusterProxy.GetClient(), clusterRef, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster"))
336+
beforeClusterDeleteHandler(ctx, input.BootstrapClusterProxy.GetClient(), clusterRef, input.ExtensionConfigName, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster"))
333337

334338
By("Checking all lifecycle hooks have been called")
335339
// Assert that each hook has been called and returned "Success" during the test.
336-
Expect(checkLifecycleHookResponses(ctx, input.BootstrapClusterProxy.GetClient(), clusterRef, map[string]string{
340+
Expect(checkLifecycleHookResponses(ctx, input.BootstrapClusterProxy.GetClient(), clusterRef, input.ExtensionConfigName, map[string]string{
337341
"BeforeClusterCreate": "Status: Success, RetryAfterSeconds: 0",
338342
"BeforeClusterUpgrade": "Status: Success, RetryAfterSeconds: 0",
339343
"BeforeClusterDelete": "Status: Success, RetryAfterSeconds: 0",
@@ -388,11 +392,11 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
388392
// should be blocked by the AfterControlPlaneUpgrade hook.
389393
// Test the MachineSet preflight checks by scaling up the MachineDeployment. The creation on the new Machine
390394
// should be blocked because the preflight checks should not pass (kubeadm version skew preflight check should fail).
391-
func machineSetPreflightChecksTestHandler(ctx context.Context, c client.Client, clusterRef types.NamespacedName) {
395+
func machineSetPreflightChecksTestHandler(ctx context.Context, c client.Client, clusterRef types.NamespacedName, extensionConfigName string) {
392396
// Verify that the hook is called and the topology reconciliation is blocked.
393397
hookName := "AfterControlPlaneUpgrade"
394398
Eventually(func() error {
395-
if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, clusterRef, []string{hookName}); err != nil {
399+
if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, clusterRef, extensionConfigName, []string{hookName}); err != nil {
396400
return err
397401
}
398402

@@ -505,6 +509,7 @@ func extensionConfig(name, extensionServiceNamespace, extensionServiceName strin
505509
},
506510
},
507511
Settings: map[string]string{
512+
"extensionConfigName": name,
508513
"defaultAllHandlersToBlocking": strconv.FormatBool(defaultAllHandlersToBlocking),
509514
},
510515
},
@@ -526,12 +531,12 @@ func extensionConfig(name, extensionServiceNamespace, extensionServiceName strin
526531

527532
// Check that each hook in hooks has been called at least once by checking if its actualResponseStatus is in the hook response configmap.
528533
// If the provided hooks have both keys and values check that the values match those in the hook response configmap.
529-
func checkLifecycleHookResponses(ctx context.Context, c client.Client, cluster types.NamespacedName, expectedHookResponses map[string]string) error {
530-
responseData := getLifecycleHookResponsesFromConfigMap(ctx, c, cluster)
534+
func checkLifecycleHookResponses(ctx context.Context, c client.Client, cluster types.NamespacedName, extensionConfigName string, expectedHookResponses map[string]string) error {
535+
responseData := getLifecycleHookResponsesFromConfigMap(ctx, c, cluster, extensionConfigName)
531536
for hookName, expectedResponse := range expectedHookResponses {
532537
actualResponse, ok := responseData[hookName+"-actualResponseStatus"]
533538
if !ok {
534-
return errors.Errorf("hook %s call not recorded in configMap %s", hookName, klog.KRef(cluster.Namespace, hookResponsesConfigMapName(cluster.Name)))
539+
return errors.Errorf("hook %s call not recorded in configMap %s", hookName, klog.KRef(cluster.Namespace, hookResponsesConfigMapName(cluster.Name, extensionConfigName)))
535540
}
536541
if expectedResponse != "" && expectedResponse != actualResponse {
537542
return errors.Errorf("hook %s was expected to be %s in configMap got %s", hookName, expectedResponse, actualResponse)
@@ -541,29 +546,29 @@ func checkLifecycleHookResponses(ctx context.Context, c client.Client, cluster t
541546
}
542547

543548
// Check that each hook in expectedHooks has been called at least once by checking if its actualResponseStatus is in the hook response configmap.
544-
func checkLifecycleHooksCalledAtLeastOnce(ctx context.Context, c client.Client, cluster types.NamespacedName, expectedHooks []string) error {
545-
responseData := getLifecycleHookResponsesFromConfigMap(ctx, c, cluster)
549+
func checkLifecycleHooksCalledAtLeastOnce(ctx context.Context, c client.Client, cluster types.NamespacedName, extensionConfigName string, expectedHooks []string) error {
550+
responseData := getLifecycleHookResponsesFromConfigMap(ctx, c, cluster, extensionConfigName)
546551
for _, hookName := range expectedHooks {
547552
if _, ok := responseData[hookName+"-actualResponseStatus"]; !ok {
548-
return errors.Errorf("hook %s call not recorded in configMap %s", hookName, klog.KRef(cluster.Namespace, hookResponsesConfigMapName(cluster.Name)))
553+
return errors.Errorf("hook %s call not recorded in configMap %s", hookName, klog.KRef(cluster.Namespace, hookResponsesConfigMapName(cluster.Name, extensionConfigName)))
549554
}
550555
}
551556
return nil
552557
}
553558

554-
func getLifecycleHookResponsesFromConfigMap(ctx context.Context, c client.Client, cluster types.NamespacedName) map[string]string {
559+
func getLifecycleHookResponsesFromConfigMap(ctx context.Context, c client.Client, cluster types.NamespacedName, extensionConfigName string) map[string]string {
555560
configMap := &corev1.ConfigMap{}
556561
Eventually(func() error {
557-
return c.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: hookResponsesConfigMapName(cluster.Name)}, configMap)
562+
return c.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: hookResponsesConfigMapName(cluster.Name, extensionConfigName)}, configMap)
558563
}).Should(Succeed(), "Failed to get the hook response configmap")
559564
return configMap.Data
560565
}
561566

562567
// beforeClusterCreateTestHandler calls runtimeHookTestHandler with a blockedCondition function which returns false if
563568
// the Cluster has entered ClusterPhaseProvisioned.
564-
func beforeClusterCreateTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, intervals []interface{}) {
569+
func beforeClusterCreateTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, extensionConfigName string, intervals []interface{}) {
565570
hookName := "BeforeClusterCreate"
566-
runtimeHookTestHandler(ctx, c, cluster, hookName, true, func() bool {
571+
runtimeHookTestHandler(ctx, c, cluster, hookName, extensionConfigName, true, func() bool {
567572
blocked := true
568573
// This hook should block the Cluster from entering the "Provisioned" state.
569574
cluster := framework.GetClusterByName(ctx,
@@ -638,9 +643,9 @@ func beforeClusterUpgradeAnnotationIsBlocking(ctx context.Context, c client.Clie
638643

639644
// beforeClusterUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if
640645
// any of the machines in the control plane has been updated to the target Kubernetes version.
641-
func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, toVersion string, intervals []interface{}) {
646+
func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, extensionConfigName string, toVersion string, intervals []interface{}) {
642647
hookName := "BeforeClusterUpgrade"
643-
runtimeHookTestHandler(ctx, c, cluster, hookName, true, func() bool {
648+
runtimeHookTestHandler(ctx, c, cluster, hookName, extensionConfigName, true, func() bool {
644649
var blocked = true
645650

646651
controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx,
@@ -656,9 +661,9 @@ func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, clust
656661

657662
// afterControlPlaneUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if any
658663
// MachineDeployment in the Cluster has upgraded to the target Kubernetes version.
659-
func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, version string, intervals []interface{}) {
664+
func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, extensionConfigName string, version string, intervals []interface{}) {
660665
hookName := "AfterControlPlaneUpgrade"
661-
runtimeHookTestHandler(ctx, c, cluster, hookName, true, func() bool {
666+
runtimeHookTestHandler(ctx, c, cluster, hookName, extensionConfigName, true, func() bool {
662667
var blocked = true
663668

664669
mds := framework.GetMachineDeploymentsByCluster(ctx,
@@ -675,9 +680,9 @@ func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, c
675680

676681
// beforeClusterDeleteHandler calls runtimeHookTestHandler with a blocking function which returns false if the Cluster
677682
// can not be found in the API server.
678-
func beforeClusterDeleteHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, intervals []interface{}) {
683+
func beforeClusterDeleteHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, extensionConfigName string, intervals []interface{}) {
679684
hookName := "BeforeClusterDelete"
680-
runtimeHookTestHandler(ctx, c, cluster, hookName, false, func() bool {
685+
runtimeHookTestHandler(ctx, c, cluster, hookName, extensionConfigName, false, func() bool {
681686
var blocked = true
682687

683688
// If the Cluster is not found it has been deleted and the hook is unblocked.
@@ -696,12 +701,12 @@ func beforeClusterDeleteHandler(ctx context.Context, c client.Client, cluster ty
696701
//
697702
// Note: runtimeHookTestHandler assumes that the hook passed to it is currently returning a blocking response.
698703
// Updating the response to be non-blocking happens inline in the function.
699-
func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, hookName string, withTopologyReconciledCondition bool, blockingCondition func() bool, intervals []interface{}) {
704+
func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster types.NamespacedName, hookName, extensionConfigName string, withTopologyReconciledCondition bool, blockingCondition func() bool, intervals []interface{}) {
700705
log.Logf("Blocking with %s hook for 60 seconds after the hook has been called for the first time", hookName)
701706

702707
// Check that the LifecycleHook has been called at least once and - when required - that the TopologyReconciled condition is a Failure.
703708
Eventually(func() error {
704-
if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, []string{hookName}); err != nil {
709+
if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, extensionConfigName, []string{hookName}); err != nil {
705710
return err
706711
}
707712

@@ -726,7 +731,7 @@ func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster types.
726731
// Patch the ConfigMap to set the hook response to "Success".
727732
Byf("Setting %s response to Status:Success to unblock the reconciliation", hookName)
728733

729-
configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: hookResponsesConfigMapName(cluster.Name), Namespace: cluster.Namespace}}
734+
configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: hookResponsesConfigMapName(cluster.Name, extensionConfigName), Namespace: cluster.Namespace}}
730735
Eventually(func() error {
731736
return c.Get(ctx, util.ObjectKey(configMap), configMap)
732737
}).Should(Succeed(), "Failed to get ConfigMap %s", klog.KObj(configMap))

‎test/e2e/scale.go

+18-3
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ func ScaleSpec(ctx context.Context, inputGetter func() ScaleSpecInput) {
219219
ClientSet: input.BootstrapClusterProxy.GetClientSet(),
220220
Name: specName,
221221
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
222+
Labels: map[string]string{"e2e-test": specName},
222223
IgnoreAlreadyExists: true,
223224
})
224225

@@ -287,8 +288,21 @@ func ScaleSpec(ctx context.Context, inputGetter func() ScaleSpecInput) {
287288
if !deployClusterInSeparateNamespaces {
288289
namespaces = append(namespaces, namespace.Name)
289290
}
291+
extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...)
292+
if deployClusterInSeparateNamespaces {
293+
extensionConfig.Spec.NamespaceSelector = &metav1.LabelSelector{
294+
// Note: we are limiting the test extension to be used by the namespace where the test is run.
295+
MatchExpressions: []metav1.LabelSelectorRequirement{
296+
{
297+
Key: "e2e-test",
298+
Operator: metav1.LabelSelectorOpIn,
299+
Values: []string{specName},
300+
},
301+
},
302+
}
303+
}
290304
Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
291-
extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...))).
305+
extensionConfig)).
292306
To(Succeed(), "Failed to create the ExtensionConfig")
293307
}
294308

@@ -390,7 +404,7 @@ func ScaleSpec(ctx context.Context, inputGetter func() ScaleSpecInput) {
390404
createClusterWorker(ctx, input.BootstrapClusterProxy, inputChan, resultChan, wg, namespace.Name,
391405
deployClusterInSeparateNamespaces, useCrossNamespaceClusterClass,
392406
baseClusterClassYAML, baseClusterTemplateYAML, creator, input.PostScaleClusterNamespaceCreated,
393-
additionalClusterClassCount, input.ClusterClassName)
407+
additionalClusterClassCount, input.ClusterClassName, specName)
394408
},
395409
})
396410
if err != nil {
@@ -646,7 +660,7 @@ func getClusterCreateFn(clusterProxy framework.ClusterProxy) clusterCreator {
646660

647661
type PostScaleClusterNamespaceCreated func(clusterProxy framework.ClusterProxy, clusterNamespace string, clusterName string, clusterClassYAML []byte, clusterTemplateYAML []byte) ([]byte, []byte)
648662

649-
func createClusterWorker(ctx context.Context, clusterProxy framework.ClusterProxy, inputChan <-chan string, resultChan chan<- workResult, wg *sync.WaitGroup, defaultNamespace string, deployClusterInSeparateNamespaces, enableCrossNamespaceClusterClass bool, baseClusterClassYAML, baseClusterTemplateYAML []byte, create clusterCreator, postScaleClusterNamespaceCreated PostScaleClusterNamespaceCreated, additionalClusterClasses int64, clusterClassName string) {
663+
func createClusterWorker(ctx context.Context, clusterProxy framework.ClusterProxy, inputChan <-chan string, resultChan chan<- workResult, wg *sync.WaitGroup, defaultNamespace string, deployClusterInSeparateNamespaces, enableCrossNamespaceClusterClass bool, baseClusterClassYAML, baseClusterTemplateYAML []byte, create clusterCreator, postScaleClusterNamespaceCreated PostScaleClusterNamespaceCreated, additionalClusterClasses int64, clusterClassName, specName string) {
650664
defer wg.Done()
651665

652666
for {
@@ -688,6 +702,7 @@ func createClusterWorker(ctx context.Context, clusterProxy framework.ClusterProx
688702
Creator: clusterProxy.GetClient(),
689703
Name: namespaceName,
690704
IgnoreAlreadyExists: true,
705+
Labels: map[string]string{"e2e-test": specName},
691706
}, "40s", "10s")
692707
}
693708

‎test/extension/config/tilt/extensionconfig.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ metadata:
55
runtime.cluster.x-k8s.io/inject-ca-from-secret: test-extension-system/test-extension-webhook-service-cert
66
name: test-extension
77
spec:
8+
settings:
9+
extensionConfigName: test-extension
810
clientConfig:
911
service:
1012
name: test-extension-webhook-service

0 commit comments

Comments
 (0)