Skip to content

Commit 6adac7a

Browse files
chrischdichrischdi
committed
supervisor: create ClusterModule per MachineDeployment and re-reconcile VirtualMachineResourceSetPolicy to update VirtualMachineSetResourcePolicy
1 parent da51433 commit 6adac7a

File tree

3 files changed

+93
-20
lines changed

3 files changed

+93
-20
lines changed

‎pkg/services/vmoperator/resource_policy.go

+57-12
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@ package vmoperator
1818

1919
import (
2020
"context"
21+
"sort"
2122

2223
"github.com/pkg/errors"
2324
vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
2425
apierrors "k8s.io/apimachinery/pkg/api/errors"
2526
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
28+
"sigs.k8s.io/cluster-api/util/patch"
2629
"sigs.k8s.io/controller-runtime/pkg/client"
2730
ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2831

@@ -37,15 +40,31 @@ type RPService struct {
3740
// ReconcileResourcePolicy ensures that a VirtualMachineSetResourcePolicy exists for the cluster
3841
// Returns the name of a policy if it exists, otherwise returns an error.
3942
func (s *RPService) ReconcileResourcePolicy(ctx context.Context, clusterCtx *vmware.ClusterContext) (string, error) {
40-
resourcePolicy, err := s.getVirtualMachineSetResourcePolicy(ctx, clusterCtx)
43+
clusterModuleGroups, err := getClusterModuleGroups(ctx, s.Client, clusterCtx.Cluster)
44+
if err != nil {
45+
return "", err
46+
}
47+
48+
resourcePolicy, err := getVirtualMachineSetResourcePolicy(ctx, s.Client, clusterCtx.Cluster)
4149
if err != nil {
4250
if !apierrors.IsNotFound(err) {
4351
return "", errors.Errorf("unexpected error in getting the Resource policy: %+v", err)
4452
}
45-
resourcePolicy, err = s.createVirtualMachineSetResourcePolicy(ctx, clusterCtx)
53+
resourcePolicy, err = s.createVirtualMachineSetResourcePolicy(ctx, clusterCtx, clusterModuleGroups)
4654
if err != nil {
4755
return "", errors.Errorf("failed to create Resource Policy: %+v", err)
4856
}
57+
return resourcePolicy.Name, nil
58+
}
59+
60+
// Ensure .spec.clusterModuleGroups is up to date.
61+
helper, err := patch.NewHelper(resourcePolicy, s.Client)
62+
if err != nil {
63+
return "", err
64+
}
65+
resourcePolicy.Spec.ClusterModuleGroups = clusterModuleGroups
66+
if err := helper.Patch(ctx, resourcePolicy); err != nil {
67+
return "", err
4968
}
5069

5170
return resourcePolicy.Name, nil
@@ -60,29 +79,26 @@ func (s *RPService) newVirtualMachineSetResourcePolicy(clusterCtx *vmware.Cluste
6079
}
6180
}
6281

63-
func (s *RPService) getVirtualMachineSetResourcePolicy(ctx context.Context, clusterCtx *vmware.ClusterContext) (*vmoprv1.VirtualMachineSetResourcePolicy, error) {
82+
func getVirtualMachineSetResourcePolicy(ctx context.Context, ctrlClient client.Client, cluster *clusterv1.Cluster) (*vmoprv1.VirtualMachineSetResourcePolicy, error) {
6483
vmResourcePolicy := &vmoprv1.VirtualMachineSetResourcePolicy{}
6584
vmResourcePolicyName := client.ObjectKey{
66-
Namespace: clusterCtx.Cluster.Namespace,
67-
Name: clusterCtx.Cluster.Name,
85+
Namespace: cluster.Namespace,
86+
Name: cluster.Name,
6887
}
69-
err := s.Client.Get(ctx, vmResourcePolicyName, vmResourcePolicy)
88+
err := ctrlClient.Get(ctx, vmResourcePolicyName, vmResourcePolicy)
7089
return vmResourcePolicy, err
7190
}
7291

73-
func (s *RPService) createVirtualMachineSetResourcePolicy(ctx context.Context, clusterCtx *vmware.ClusterContext) (*vmoprv1.VirtualMachineSetResourcePolicy, error) {
92+
func (s *RPService) createVirtualMachineSetResourcePolicy(ctx context.Context, clusterCtx *vmware.ClusterContext, clusterModuleGroups []string) (*vmoprv1.VirtualMachineSetResourcePolicy, error) {
7493
vmResourcePolicy := s.newVirtualMachineSetResourcePolicy(clusterCtx)
7594

7695
_, err := ctrlutil.CreateOrPatch(ctx, s.Client, vmResourcePolicy, func() error {
7796
vmResourcePolicy.Spec = vmoprv1.VirtualMachineSetResourcePolicySpec{
7897
ResourcePool: vmoprv1.ResourcePoolSpec{
7998
Name: clusterCtx.Cluster.Name,
8099
},
81-
Folder: clusterCtx.Cluster.Name,
82-
ClusterModuleGroups: []string{
83-
ControlPlaneVMClusterModuleGroupName,
84-
getMachineDeploymentNameForCluster(clusterCtx.Cluster),
85-
},
100+
Folder: clusterCtx.Cluster.Name,
101+
ClusterModuleGroups: clusterModuleGroups,
86102
}
87103
// Ensure that the VirtualMachineSetResourcePolicy is owned by the VSphereCluster
88104
if err := ctrlutil.SetOwnerReference(
@@ -106,3 +122,32 @@ func (s *RPService) createVirtualMachineSetResourcePolicy(ctx context.Context, c
106122
}
107123
return vmResourcePolicy, nil
108124
}
125+
126+
func getClusterModuleGroups(ctx context.Context, ctrlClient client.Client, cluster *clusterv1.Cluster) ([]string, error) {
127+
machineDeploymentNames, err := getMachineDeploymentNamesForCluster(ctx, ctrlClient, cluster)
128+
if err != nil {
129+
return nil, err
130+
}
131+
132+
clusterModuleGroups := append([]string{ControlPlaneVMClusterModuleGroupName}, machineDeploymentNames...)
133+
134+
// sort elements to have deterministic output.
135+
sort.Strings(clusterModuleGroups)
136+
137+
return clusterModuleGroups, nil
138+
}
139+
140+
func checkClusterModuleGroup(ctx context.Context, ctrlClient client.Client, cluster *clusterv1.Cluster, clusterModuleGroupName string) error {
141+
resourcePolicy, err := getVirtualMachineSetResourcePolicy(ctx, ctrlClient, cluster)
142+
if err != nil {
143+
return err
144+
}
145+
146+
for _, cm := range resourcePolicy.Status.ClusterModules {
147+
if cm.GroupName == clusterModuleGroupName {
148+
return nil
149+
}
150+
}
151+
152+
return errors.Errorf("VirtualMachineSetResourcePolicy's .status.clusterModules does not yet contain %s", clusterModuleGroupName)
153+
}

‎pkg/services/vmoperator/resource_policy_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func TestRPService(t *testing.T) {
4444
g.Expect(err).NotTo(HaveOccurred())
4545
g.Expect(name).To(Equal(clusterName))
4646

47-
resourcePolicy, err := rpService.getVirtualMachineSetResourcePolicy(ctx, clusterCtx)
47+
resourcePolicy, err := getVirtualMachineSetResourcePolicy(ctx, controllerCtx.Client, clusterCtx.Cluster)
4848
g.Expect(err).NotTo(HaveOccurred())
4949
g.Expect(resourcePolicy.Spec.ResourcePool.Name).To(Equal(clusterName))
5050
g.Expect(resourcePolicy.Spec.Folder).To(Equal(clusterName))

‎pkg/services/vmoperator/vmopmachine.go

+35-7
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,9 @@ func (v *VmopMachineService) reconcileVMOperatorVM(ctx context.Context, supervis
475475
// Assign the VM's labels.
476476
vmOperatorVM.Labels = getVMLabels(supervisorMachineCtx, vmOperatorVM.Labels)
477477

478-
addResourcePolicyAnnotations(supervisorMachineCtx, vmOperatorVM)
478+
if err := addResourcePolicyAnnotations(ctx, v.Client, supervisorMachineCtx, vmOperatorVM); err != nil {
479+
return err
480+
}
479481

480482
if err := v.addVolumes(ctx, supervisorMachineCtx, vmOperatorVM); err != nil {
481483
return err
@@ -580,7 +582,7 @@ func (v *VmopMachineService) getVirtualMachinesInCluster(ctx context.Context, su
580582

581583
// Helper function to add annotations to indicate which tag vm-operator should add as well as which clusterModule VM
582584
// should be associated.
583-
func addResourcePolicyAnnotations(supervisorMachineCtx *vmware.SupervisorMachineContext, vm *vmoprv1.VirtualMachine) {
585+
func addResourcePolicyAnnotations(ctx context.Context, ctrlClient client.Client, supervisorMachineCtx *vmware.SupervisorMachineContext, vm *vmoprv1.VirtualMachine) error {
584586
annotations := vm.ObjectMeta.GetAnnotations()
585587
if annotations == nil {
586588
annotations = make(map[string]string)
@@ -591,10 +593,17 @@ func addResourcePolicyAnnotations(supervisorMachineCtx *vmware.SupervisorMachine
591593
annotations[ClusterModuleNameAnnotationKey] = ControlPlaneVMClusterModuleGroupName
592594
} else {
593595
annotations[ProviderTagsAnnotationKey] = WorkerVMVMAntiAffinityTagValue
594-
annotations[ClusterModuleNameAnnotationKey] = getMachineDeploymentNameForCluster(supervisorMachineCtx.Cluster)
596+
clusterModuleName := getMachineDeploymentNameForMachine(supervisorMachineCtx.Machine)
597+
598+
if err := checkClusterModuleGroup(ctx, ctrlClient, supervisorMachineCtx.Cluster, clusterModuleName); err != nil {
599+
return err
600+
}
601+
602+
annotations[ClusterModuleNameAnnotationKey] = clusterModuleName
595603
}
596604

597605
vm.ObjectMeta.SetAnnotations(annotations)
606+
return nil
598607
}
599608

600609
func volumeName(machine *vmwarev1.VSphereMachine, volume vmwarev1.VSphereMachineVolume) string {
@@ -742,8 +751,27 @@ func getTopologyLabels(supervisorMachineCtx *vmware.SupervisorMachineContext) ma
742751
return nil
743752
}
744753

745-
// getMachineDeploymentName returns the MachineDeployment name for a Cluster.
746-
// This is also the name used by VSphereMachineTemplate and KubeadmConfigTemplate.
747-
func getMachineDeploymentNameForCluster(cluster *clusterv1.Cluster) string {
748-
return fmt.Sprintf("%s-workers-0", cluster.Name)
754+
func getMachineDeploymentNameForMachine(machine *clusterv1.Machine) string {
755+
if mdName, ok := machine.Labels[clusterv1.MachineDeploymentNameLabel]; ok {
756+
return mdName
757+
}
758+
return ""
759+
}
760+
761+
func getMachineDeploymentNamesForCluster(ctx context.Context, ctrlClient client.Client, cluster *clusterv1.Cluster) ([]string, error) {
762+
mdNames := []string{}
763+
labels := map[string]string{clusterv1.ClusterNameLabel: cluster.GetName()}
764+
mdList := &clusterv1.MachineDeploymentList{}
765+
if err := ctrlClient.List(
766+
ctx, mdList,
767+
client.InNamespace(cluster.GetNamespace()),
768+
client.MatchingLabels(labels)); err != nil {
769+
return nil, errors.Wrapf(err, "failed to list MachineDeployment objects")
770+
}
771+
for _, md := range mdList.Items {
772+
if md.DeletionTimestamp.IsZero() {
773+
mdNames = append(mdNames, md.Name)
774+
}
775+
}
776+
return mdNames, nil
749777
}

0 commit comments

Comments
 (0)