Skip to content

Commit a0aeaa7

Browse files
chrischdichrischdi
committed
add WorkerAntiAffinity feature gate and implement validating webhook
1 parent b92c1c3 commit a0aeaa7

File tree

6 files changed

+300
-1
lines changed

6 files changed

+300
-1
lines changed

‎config/manager/manager.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ spec:
2121
- "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
2222
- "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
2323
- --v=4
24-
- "--feature-gates=NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false}"
24+
- "--feature-gates=NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},WorkerAntiAffinity=${EXP_WORKER_ANTI_AFFINITY:=false}"
2525
image: controller:latest
2626
imagePullPolicy: IfNotPresent
2727
name: manager

‎config/supervisor/webhook/manifests.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,27 @@ kind: ValidatingWebhookConfiguration
3131
metadata:
3232
name: validating-webhook-configuration
3333
webhooks:
34+
- admissionReviewVersions:
35+
- v1beta1
36+
clientConfig:
37+
service:
38+
name: webhook-service
39+
namespace: system
40+
path: /validate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspherecluster
41+
failurePolicy: Fail
42+
matchPolicy: Equivalent
43+
name: validation.vspherecluster.vmware.infrastructure.cluster.x-k8s.io
44+
rules:
45+
- apiGroups:
46+
- vmware.infrastructure.cluster.x-k8s.io
47+
apiVersions:
48+
- v1beta1
49+
operations:
50+
- CREATE
51+
- UPDATE
52+
resources:
53+
- vsphereclusters
54+
sideEffects: None
3455
- admissionReviewVersions:
3556
- v1beta1
3657
clientConfig:

‎feature/feature.go

+12
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,15 @@ const (
4444
//
4545
// alpha: v1.10
4646
PriorityQueue featuregate.Feature = "PriorityQueue"
47+
48+
// WorkerAntiAffinity allows configuring how soft anti-affinity should be done for worker nodes.[]
49+
// If disabled it disallows:
50+
// * mutating `VSphereCluster.spec.placement.workerAntiAffinity.mode`.
51+
// * Setting `MachineDeployment` as value for `VSphereCluster.spec.placement.workerAntiAffinity.mode` on creation.
52+
// Note: the feature requires a version of vm-operator which allows mutation of `VirtualMachineSetResourcePolicy's`.
53+
//
54+
// alpha: v1.13
55+
WorkerAntiAffinity featuregate.Feature = "WorkerAntiAffinity"
4756
)
4857

4958
func init() {
@@ -57,4 +66,7 @@ var defaultCAPVFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
5766
NodeAntiAffinity: {Default: false, PreRelease: featuregate.Alpha},
5867
NamespaceScopedZones: {Default: false, PreRelease: featuregate.Alpha},
5968
PriorityQueue: {Default: false, PreRelease: featuregate.Alpha},
69+
70+
// Feature gates specific to supervisor mode:
71+
WorkerAntiAffinity: {Default: false, PreRelease: featuregate.Alpha},
6072
}

‎internal/webhooks/vmware/vspherecluster.go

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package vmware is the package for webhooks of vmware resources.
18+
package vmware
19+
20+
import (
21+
"context"
22+
"fmt"
23+
24+
apierrors "k8s.io/apimachinery/pkg/api/errors"
25+
"k8s.io/apimachinery/pkg/runtime"
26+
"k8s.io/apimachinery/pkg/util/validation/field"
27+
ctrl "sigs.k8s.io/controller-runtime"
28+
"sigs.k8s.io/controller-runtime/pkg/webhook"
29+
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
30+
31+
vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
32+
"sigs.k8s.io/cluster-api-provider-vsphere/feature"
33+
"sigs.k8s.io/cluster-api-provider-vsphere/internal/webhooks"
34+
)
35+
36+
// +kubebuilder:webhook:verbs=create;update,path=/validate-vmware-infrastructure-cluster-x-k8s-io-v1beta1-vspherecluster,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters,versions=v1beta1,name=validation.vspherecluster.vmware.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1beta1
37+
38+
// VSphereClusterWebhook implements a validation and defaulting webhook for VSphereCluster.
39+
type VSphereClusterWebhook struct{}
40+
41+
var _ webhook.CustomValidator = &VSphereClusterWebhook{}
42+
43+
func (webhook *VSphereClusterWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error {
44+
return ctrl.NewWebhookManagedBy(mgr).
45+
For(&vmwarev1.VSphereCluster{}).
46+
WithValidator(webhook).
47+
Complete()
48+
}
49+
50+
// ValidateCreate implements webhook.Validator so a webhook will be registered for the type.
51+
func (webhook *VSphereClusterWebhook) ValidateCreate(_ context.Context, newRaw runtime.Object) (admission.Warnings, error) {
52+
var allErrs field.ErrorList
53+
54+
newTyped, ok := newRaw.(*vmwarev1.VSphereCluster)
55+
if !ok {
56+
return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a VSphereCluster but got a %T", newRaw))
57+
}
58+
59+
newSpec := newTyped.Spec
60+
61+
if !feature.Gates.Enabled(feature.WorkerAntiAffinity) {
62+
// Cluster mode is not allowed without WorkerAntiAffinity being enabled.
63+
if newSpec.Placement != nil && newSpec.Placement.WorkerAntiAffinity != nil && newSpec.Placement.WorkerAntiAffinity.Mode == vmwarev1.VSphereClusterWorkerAntiAffinityModeMachineDeployment {
64+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "placement", "workerAntiAffinity", "mode"), "cannot be set to Cluster with feature-gate WorkerAntiAffinity being disabled"))
65+
}
66+
}
67+
68+
return nil, webhooks.AggregateObjErrors(newTyped.GroupVersionKind().GroupKind(), newTyped.Name, allErrs)
69+
}
70+
71+
// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
72+
func (webhook *VSphereClusterWebhook) ValidateUpdate(_ context.Context, _ runtime.Object, newRaw runtime.Object) (admission.Warnings, error) {
73+
var allErrs field.ErrorList
74+
75+
newTyped, ok := newRaw.(*vmwarev1.VSphereCluster)
76+
if !ok {
77+
return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a VSphereCluster but got a %T", newRaw))
78+
}
79+
80+
newSpec := newTyped.Spec
81+
82+
if !feature.Gates.Enabled(feature.WorkerAntiAffinity) {
83+
// Cluster mode is not allowed without WorkerAntiAffinity being enabled.
84+
if newSpec.Placement != nil && newSpec.Placement.WorkerAntiAffinity != nil && newSpec.Placement.WorkerAntiAffinity.Mode == vmwarev1.VSphereClusterWorkerAntiAffinityModeMachineDeployment {
85+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "placement", "workerAntiAffinity", "mode"), "cannot be set to Cluster with feature-gate WorkerAntiAffinity being disabled"))
86+
}
87+
}
88+
89+
return nil, webhooks.AggregateObjErrors(newTyped.GroupVersionKind().GroupKind(), newTyped.Name, allErrs)
90+
}
91+
92+
// ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
93+
func (webhook *VSphereClusterWebhook) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) {
94+
return nil, nil
95+
}

‎internal/webhooks/vmware/vspherecluster_test.go

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package vmware
18+
19+
import (
20+
"context"
21+
"testing"
22+
23+
. "github.com/onsi/gomega"
24+
utilfeature "k8s.io/component-base/featuregate/testing"
25+
26+
vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
27+
"sigs.k8s.io/cluster-api-provider-vsphere/feature"
28+
)
29+
30+
func TestVSphereCluster_ValidateCreate(t *testing.T) {
31+
tests := []struct {
32+
name string
33+
vsphereCluster *vmwarev1.VSphereCluster
34+
workerAntiAffinity bool
35+
wantErr bool
36+
}{
37+
{
38+
name: "Allow Cluster (WorkerAntiAffinity=false)",
39+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
40+
workerAntiAffinity: false,
41+
wantErr: false,
42+
},
43+
{
44+
name: "Allow Cluster (WorkerAntiAffinity=true)",
45+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
46+
workerAntiAffinity: true,
47+
wantErr: false,
48+
},
49+
{
50+
name: "Allow None (WorkerAntiAffinity=false)",
51+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
52+
workerAntiAffinity: false,
53+
wantErr: false,
54+
},
55+
{
56+
name: "Allow None (WorkerAntiAffinity=true)",
57+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
58+
workerAntiAffinity: true,
59+
wantErr: false,
60+
},
61+
{
62+
name: "Deny MachineDeployment (WorkerAntiAffinity=false)",
63+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeMachineDeployment),
64+
workerAntiAffinity: false,
65+
wantErr: true,
66+
},
67+
{
68+
name: "Allow MachineDeployment (WorkerAntiAffinity=true)",
69+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeMachineDeployment),
70+
workerAntiAffinity: true,
71+
wantErr: false,
72+
},
73+
}
74+
for _, tt := range tests {
75+
t.Run(tt.name, func(t *testing.T) {
76+
g := NewWithT(t)
77+
78+
utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.WorkerAntiAffinity, tt.workerAntiAffinity)
79+
80+
webhook := &VSphereClusterWebhook{}
81+
_, err := webhook.ValidateCreate(context.Background(), tt.vsphereCluster)
82+
if tt.wantErr {
83+
g.Expect(err).To(HaveOccurred())
84+
} else {
85+
g.Expect(err).NotTo(HaveOccurred())
86+
}
87+
})
88+
}
89+
}
90+
func TestVSphereCluster_ValidateUpdate(t *testing.T) {
91+
tests := []struct {
92+
name string
93+
oldVSphereCluster *vmwarev1.VSphereCluster
94+
vsphereCluster *vmwarev1.VSphereCluster
95+
workerAntiAffinity bool
96+
wantErr bool
97+
}{
98+
{
99+
name: "noop (WorkerAntiAffinity=false)",
100+
oldVSphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
101+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
102+
workerAntiAffinity: false,
103+
wantErr: false,
104+
},
105+
{
106+
name: "noop (WorkerAntiAffinity=true)",
107+
oldVSphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
108+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
109+
workerAntiAffinity: true,
110+
wantErr: false,
111+
},
112+
{
113+
name: "Allow Cluster to None (WorkerAntiAffinity=false)",
114+
oldVSphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
115+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeNone),
116+
workerAntiAffinity: false,
117+
wantErr: false,
118+
},
119+
{
120+
name: "Allow Cluster to None (WorkerAntiAffinity=true)",
121+
oldVSphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
122+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeNone),
123+
workerAntiAffinity: true,
124+
wantErr: false,
125+
},
126+
{
127+
name: "Disallow Cluster to MachineDeployment (WorkerAntiAffinity=false)",
128+
oldVSphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
129+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeMachineDeployment),
130+
workerAntiAffinity: false,
131+
wantErr: true,
132+
},
133+
{
134+
name: "Allow Cluster to MachineDeployment (WorkerAntiAffinity=true)",
135+
oldVSphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeCluster),
136+
vsphereCluster: createVSphereCluster(vmwarev1.VSphereClusterWorkerAntiAffinityModeMachineDeployment),
137+
workerAntiAffinity: true,
138+
wantErr: false,
139+
},
140+
}
141+
for _, tt := range tests {
142+
t.Run(tt.name, func(t *testing.T) {
143+
g := NewWithT(t)
144+
145+
utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.WorkerAntiAffinity, tt.workerAntiAffinity)
146+
147+
webhook := &VSphereClusterWebhook{}
148+
_, err := webhook.ValidateUpdate(context.Background(), tt.oldVSphereCluster, tt.vsphereCluster)
149+
if tt.wantErr {
150+
g.Expect(err).To(HaveOccurred())
151+
} else {
152+
g.Expect(err).NotTo(HaveOccurred())
153+
}
154+
})
155+
}
156+
}
157+
158+
func createVSphereCluster(mode vmwarev1.VSphereClusterWorkerAntiAffinityMode) *vmwarev1.VSphereCluster {
159+
vSphereCluster := &vmwarev1.VSphereCluster{}
160+
if mode != "" {
161+
vSphereCluster.Spec.Placement = &vmwarev1.VSphereClusterPlacement{
162+
WorkerAntiAffinity: &vmwarev1.VSphereClusterWorkerAntiAffinity{
163+
Mode: mode,
164+
},
165+
}
166+
}
167+
return vSphereCluster
168+
}

‎main.go

+3
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,9 @@ func setupVAPIControllers(ctx context.Context, controllerCtx *capvcontext.Contro
404404
}
405405

406406
func setupSupervisorControllers(ctx context.Context, controllerCtx *capvcontext.ControllerManagerContext, mgr ctrlmgr.Manager, clusterCache clustercache.ClusterCache) error {
407+
if err := (&vmwarewebhooks.VSphereClusterWebhook{}).SetupWebhookWithManager(mgr); err != nil {
408+
return err
409+
}
407410
if err := (&vmwarewebhooks.VSphereMachineTemplateWebhook{}).SetupWebhookWithManager(mgr); err != nil {
408411
return err
409412
}

0 commit comments

Comments
 (0)