@@ -18,15 +18,22 @@ package vmoperator
1818
1919import (
2020 "context"
21+ "fmt"
22+ "sort"
2123
2224 "github.com/pkg/errors"
2325 vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
2426 apierrors "k8s.io/apimachinery/pkg/api/errors"
2527 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+ "k8s.io/apimachinery/pkg/util/sets"
29+ "k8s.io/klog/v2"
30+ clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
31+ "sigs.k8s.io/cluster-api/util/patch"
2632 "sigs.k8s.io/controller-runtime/pkg/client"
2733 ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2834
29- "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/vmware"
35+ vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
36+ "sigs.k8s.io/cluster-api-provider-vsphere/feature"
3037)
3138
3239// RPService represents the ability to reconcile a VirtualMachineSetResourcePolicy via vmoperator.
@@ -36,73 +43,175 @@ type RPService struct {
3643
3744// ReconcileResourcePolicy ensures that a VirtualMachineSetResourcePolicy exists for the cluster
3845// Returns the name of a policy if it exists, otherwise returns an error.
39- func (s * RPService ) ReconcileResourcePolicy (ctx context.Context , clusterCtx * vmware. ClusterContext ) (string , error ) {
40- resourcePolicy , err := s . getVirtualMachineSetResourcePolicy (ctx , clusterCtx )
46+ func (s * RPService ) ReconcileResourcePolicy (ctx context.Context , cluster * clusterv1. Cluster , vSphereCluster * vmwarev1. VSphereCluster ) (string , error ) {
47+ clusterModuleGroups , err := getTargetClusterModuleGroups (ctx , s . Client , cluster , vSphereCluster )
4148 if err != nil {
49+ return "" , err
50+ }
51+
52+ resourcePolicyName := cluster .Name
53+ resourcePolicy := & vmoprv1.VirtualMachineSetResourcePolicy {}
54+
55+ if err := s .Client .Get (ctx , client.ObjectKey {Namespace : cluster .Namespace , Name : resourcePolicyName }, resourcePolicy ); err != nil {
4256 if ! apierrors .IsNotFound (err ) {
43- return "" , errors .Errorf ( "unexpected error in getting the Resource policy: %+v" , err )
57+ return "" , errors .Wrap ( err , "failed to get existing VirtualMachineSetResourcePolicy" )
4458 }
45- resourcePolicy , err = s .createVirtualMachineSetResourcePolicy (ctx , clusterCtx )
46- if err != nil {
47- return "" , errors .Errorf ("failed to create Resource Policy: %+v" , err )
59+
60+ resourcePolicy = & vmoprv1.VirtualMachineSetResourcePolicy {
61+ ObjectMeta : metav1.ObjectMeta {
62+ Namespace : cluster .Namespace ,
63+ Name : resourcePolicyName ,
64+ },
65+ }
66+
67+ if err := s .mutateResourcePolicy (resourcePolicy , clusterModuleGroups , cluster , vSphereCluster , true ); err != nil {
68+ return "" , errors .Wrap (err , "failed to mutate VirtualMachineSetResourcePolicy" )
4869 }
70+
71+ if err := s .Client .Create (ctx , resourcePolicy ); err != nil {
72+ return "" , errors .Wrap (err , "failed to create VirtualMachineSetResourcePolicy" )
73+ }
74+
75+ return resourcePolicyName , nil
76+ }
77+
78+ // Ensure .spec.clusterModuleGroups is up to date.
79+ helper , err := patch .NewHelper (resourcePolicy , s .Client )
80+ if err != nil {
81+ return "" , err
82+ }
83+
84+ if err := s .mutateResourcePolicy (resourcePolicy , clusterModuleGroups , cluster , vSphereCluster , false ); err != nil {
85+ return "" , errors .Wrap (err , "failed to mutate VirtualMachineSetResourcePolicy" )
86+ }
87+
88+ resourcePolicy .Spec .ClusterModuleGroups = clusterModuleGroups
89+ if err := helper .Patch (ctx , resourcePolicy ); err != nil {
90+ return "" , err
4991 }
5092
51- return resourcePolicy . Name , nil
93+ return resourcePolicyName , nil
5294}
5395
54- func (s * RPService ) newVirtualMachineSetResourcePolicy (clusterCtx * vmware.ClusterContext ) * vmoprv1.VirtualMachineSetResourcePolicy {
55- return & vmoprv1.VirtualMachineSetResourcePolicy {
56- ObjectMeta : metav1.ObjectMeta {
57- Namespace : clusterCtx .Cluster .Namespace ,
58- Name : clusterCtx .Cluster .Name ,
59- },
96+ func (s * RPService ) mutateResourcePolicy (resourcePolicy * vmoprv1.VirtualMachineSetResourcePolicy , clusterModuleGroups []string , cluster * clusterv1.Cluster , vSphereCluster * vmwarev1.VSphereCluster , isCreate bool ) error {
97+ // Always ensure the owner reference
98+ if err := ctrlutil .SetOwnerReference (vSphereCluster , resourcePolicy , s .Client .Scheme ()); err != nil {
99+ return errors .Wrapf (err , "failed to set owner reference for virtualMachineSetResourcePolicy %s for cluster %s" , klog .KObj (resourcePolicy ), klog .KObj (vSphereCluster ))
100+ }
101+
102+ // Always ensure the clusterModuleGroups are up-to-date.
103+ resourcePolicy .Spec .ClusterModuleGroups = clusterModuleGroups
104+
105+ // On create: Also set resourcePool and folder
106+ if isCreate {
107+ resourcePolicy .Spec .Folder = cluster .Name
108+ resourcePolicy .Spec .ResourcePool = vmoprv1.ResourcePoolSpec {
109+ Name : cluster .Name ,
110+ }
60111 }
112+
113+ return nil
61114}
62115
63- func ( s * RPService ) getVirtualMachineSetResourcePolicy (ctx context.Context , clusterCtx * vmware. ClusterContext ) (* vmoprv1.VirtualMachineSetResourcePolicy , error ) {
116+ func getVirtualMachineSetResourcePolicy (ctx context.Context , ctrlClient client. Client , cluster * clusterv1. Cluster ) (* vmoprv1.VirtualMachineSetResourcePolicy , error ) {
64117 vmResourcePolicy := & vmoprv1.VirtualMachineSetResourcePolicy {}
65118 vmResourcePolicyName := client.ObjectKey {
66- Namespace : clusterCtx . Cluster .Namespace ,
67- Name : clusterCtx . Cluster .Name ,
119+ Namespace : cluster .Namespace ,
120+ Name : cluster .Name ,
68121 }
69- err := s .Client .Get (ctx , vmResourcePolicyName , vmResourcePolicy )
70- return vmResourcePolicy , err
122+ if err := ctrlClient .Get (ctx , vmResourcePolicyName , vmResourcePolicy ); err != nil {
123+ return nil , err
124+ }
125+
126+ return vmResourcePolicy , nil
71127}
72128
73- func (s * RPService ) createVirtualMachineSetResourcePolicy (ctx context.Context , clusterCtx * vmware.ClusterContext ) (* vmoprv1.VirtualMachineSetResourcePolicy , error ) {
74- vmResourcePolicy := s .newVirtualMachineSetResourcePolicy (clusterCtx )
129+ func getFallbackWorkerClusterModuleGroupName (clusterName string ) string {
130+ return fmt .Sprintf ("%s-workers-0" , clusterName )
131+ }
75132
76- _ , err := ctrlutil .CreateOrPatch (ctx , s .Client , vmResourcePolicy , func () error {
77- vmResourcePolicy .Spec = vmoprv1.VirtualMachineSetResourcePolicySpec {
78- ResourcePool : vmoprv1.ResourcePoolSpec {
79- Name : clusterCtx .Cluster .Name ,
80- },
81- Folder : clusterCtx .Cluster .Name ,
82- ClusterModuleGroups : []string {
83- ControlPlaneVMClusterModuleGroupName ,
84- getMachineDeploymentNameForCluster (clusterCtx .Cluster ),
85- },
86- }
87- // Ensure that the VirtualMachineSetResourcePolicy is owned by the VSphereCluster
88- if err := ctrlutil .SetOwnerReference (
89- clusterCtx .VSphereCluster ,
90- vmResourcePolicy ,
91- s .Client .Scheme (),
92- ); err != nil {
93- return errors .Wrapf (
94- err ,
95- "error setting %s/%s as owner of %s/%s" ,
96- clusterCtx .VSphereCluster .Namespace ,
97- clusterCtx .VSphereCluster .Name ,
98- vmResourcePolicy .Namespace ,
99- vmResourcePolicy .Name ,
100- )
133+ func getWorkerAntiAffinityMode (vSphereCluster * vmwarev1.VSphereCluster ) vmwarev1.VSphereClusterWorkerAntiAffinityMode {
134+ if vSphereCluster .Spec .Placement == nil || vSphereCluster .Spec .Placement .WorkerAntiAffinity == nil {
135+ return vmwarev1 .VSphereClusterWorkerAntiAffinityModeCluster
136+ }
137+
138+ return vSphereCluster .Spec .Placement .WorkerAntiAffinity .Mode
139+ }
140+
141+ func getTargetClusterModuleGroups (ctx context.Context , ctrlClient client.Client , cluster * clusterv1.Cluster , vSphereCluster * vmwarev1.VSphereCluster ) ([]string , error ) {
142+ if ! feature .Gates .Enabled (feature .WorkerAntiAffinity ) {
143+ // Fallback to old behaviour
144+ return []string {
145+ ControlPlaneVMClusterModuleGroupName ,
146+ getFallbackWorkerClusterModuleGroupName (cluster .Name ),
147+ }, nil
148+ }
149+ // Always add a cluster module for control plane machines.
150+ modules := []string {
151+ ControlPlaneVMClusterModuleGroupName ,
152+ }
153+
154+ switch mode := getWorkerAntiAffinityMode (vSphereCluster ); mode {
155+ case vmwarev1 .VSphereClusterWorkerAntiAffinityModeNone :
156+ // Only configure a cluster module for control-plane nodes
157+ case vmwarev1 .VSphereClusterWorkerAntiAffinityModeCluster :
158+ // Add an additional cluster module for workers when using Cluster mode.
159+ modules = append (modules , ClusterWorkerVMClusterModuleGroupName )
160+ case vmwarev1 .VSphereClusterWorkerAntiAffinityModeMachineDeployment :
161+ // Add an additional cluster module for each MachineDeployment workers when using MachineDeployment mode.
162+ machineDeploymentNames , err := getMachineDeploymentNamesForCluster (ctx , ctrlClient , cluster )
163+ if err != nil {
164+ return nil , err
101165 }
102- return nil
103- })
166+
167+ modules = append (modules , machineDeploymentNames ... )
168+ default :
169+ return nil , errors .Errorf ("unknown mode %q configured for WorkerAntiAffinity" , mode )
170+ }
171+
172+ // Add cluster modules from existing VirtualMachines and deduplicate with the target ones.
173+ existingModules , err := getVirtualMachineClusterModulesForCluster (ctx , ctrlClient , cluster )
104174 if err != nil {
105175 return nil , err
106176 }
107- return vmResourcePolicy , nil
177+ modules = existingModules .Insert (modules ... ).UnsortedList ()
178+
179+ // Sort elements to have deterministic output.
180+ sort .Strings (modules )
181+
182+ return modules , nil
183+ }
184+
185+ func getVirtualMachineClusterModulesForCluster (ctx context.Context , ctrlClient client.Client , cluster * clusterv1.Cluster ) (sets.Set [string ], error ) {
186+ labels := map [string ]string {clusterv1 .ClusterNameLabel : cluster .GetName ()}
187+ virtualMachineList := & vmoprv1.VirtualMachineList {}
188+ if err := ctrlClient .List (
189+ ctx , virtualMachineList ,
190+ client .InNamespace (cluster .GetNamespace ()),
191+ client .MatchingLabels (labels )); err != nil {
192+ return nil , errors .Wrapf (err , "failed to list MachineDeployment objects" )
193+ }
194+
195+ clusterModules := sets.Set [string ]{}
196+ for _ , virtualMachine := range virtualMachineList .Items {
197+ if clusterModule , ok := virtualMachine .Annotations [ClusterModuleNameAnnotationKey ]; ok {
198+ clusterModules = clusterModules .Insert (clusterModule )
199+ }
200+ }
201+ return clusterModules , nil
202+ }
203+
204+ func checkClusterModuleGroup (ctx context.Context , ctrlClient client.Client , cluster * clusterv1.Cluster , clusterModuleGroupName string ) error {
205+ resourcePolicy , err := getVirtualMachineSetResourcePolicy (ctx , ctrlClient , cluster )
206+ if err != nil {
207+ return err
208+ }
209+
210+ for _ , cm := range resourcePolicy .Status .ClusterModules {
211+ if cm .GroupName == clusterModuleGroupName {
212+ return nil
213+ }
214+ }
215+
216+ return errors .Errorf ("VirtualMachineSetResourcePolicy's .status.clusterModules does not yet contain group %q" , clusterModuleGroupName )
108217}
0 commit comments