Skip to content

Commit 382c908

Browse files
rejoshedrejoshed
authored
Merge pull request #76 from aws/feature/subdomain-user-operations
Subdomain User Operations
2 parents 0507507 + 51b0661 commit 382c908

18 files changed

+435
-370
lines changed

‎controllers/cloudstackaffinitygroup_controller.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ import (
3737
type CloudStackAGReconciliationRunner struct {
3838
csCtrlrUtils.ReconciliationRunner
3939
ReconciliationSubject *infrav1.CloudStackAffinityGroup
40-
CSUser cloud.Client
4140
}
4241

4342
// CloudStackAGReconciler is the base reconciler to adapt to k8s.
@@ -65,7 +64,7 @@ func (reconciler *CloudStackAffinityGroupReconciler) Reconcile(ctx context.Conte
6564
func (r *CloudStackAGReconciliationRunner) Reconcile() (ctrl.Result, error) {
6665
controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.AffinityGroupFinalizer)
6766
affinityGroup := &cloud.AffinityGroup{Name: r.ReconciliationSubject.Spec.Name, Type: "host affinity"}
68-
if err := r.CSClient.GetOrCreateAffinityGroup(affinityGroup); err != nil {
67+
if err := r.CSUser.GetOrCreateAffinityGroup(affinityGroup); err != nil {
6968
return ctrl.Result{}, err
7069
}
7170
r.ReconciliationSubject.Spec.ID = affinityGroup.ID
@@ -75,11 +74,11 @@ func (r *CloudStackAGReconciliationRunner) Reconcile() (ctrl.Result, error) {
7574

7675
func (r *CloudStackAGReconciliationRunner) ReconcileDelete() (ctrl.Result, error) {
7776
group := &cloud.AffinityGroup{Name: r.ReconciliationSubject.Name}
78-
_ = r.CSClient.FetchAffinityGroup(group)
77+
_ = r.CSUser.FetchAffinityGroup(group)
7978
if group.ID == "" { // Affinity group not found, must have been deleted.
8079
return ctrl.Result{}, nil
8180
}
82-
if err := r.CSClient.DeleteAffinityGroup(group); err != nil {
81+
if err := r.CSUser.DeleteAffinityGroup(group); err != nil {
8382
return ctrl.Result{}, err
8483
}
8584
controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.AffinityGroupFinalizer)

‎controllers/cloudstackcluster_controller.go

+5-20
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import (
3030

3131
infrav1 "github.com/aws/cluster-api-provider-cloudstack/api/v1beta1"
3232
csCtrlrUtils "github.com/aws/cluster-api-provider-cloudstack/controllers/utils"
33-
"github.com/aws/cluster-api-provider-cloudstack/pkg/cloud"
3433
"github.com/pkg/errors"
3534
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3635
"sigs.k8s.io/cluster-api/util"
@@ -53,7 +52,6 @@ type CloudStackClusterReconciliationRunner struct {
5352
csCtrlrUtils.ReconciliationRunner
5453
Zones *infrav1.CloudStackZoneList
5554
ReconciliationSubject *infrav1.CloudStackCluster
56-
CSUser cloud.Client
5755
}
5856

5957
// CloudStackClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster.
@@ -94,24 +92,14 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re
9492
r.GetZones(r.Zones),
9593
r.VerifyZoneCRDs,
9694
r.SetFailureDomains,
97-
r.ResolveClusterDetails)
95+
r.SetReady)
9896
}
9997

100-
// ResolveClusterDetails fetches cluster specific details like domain and account IDs.
101-
func (r *CloudStackClusterReconciliationRunner) ResolveClusterDetails() (ctrl.Result, error) {
102-
// Ensure that CAPI won't prematurely delete this CloudStackCluster.
98+
// SetReady adds a finalizer and sets the cluster status to ready.
99+
func (r *CloudStackClusterReconciliationRunner) SetReady() (ctrl.Result, error) {
103100
controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
104-
105-
// Create and or fetch cluster components.
106-
err := r.CSClient.GetOrCreateCluster(r.ReconciliationSubject)
107-
if err == nil {
108-
r.Log.Info("Fetched cluster info successfully.")
109-
r.Log.V(1).Info("Post fetch cluster status.", "clusterStatus", r.ReconciliationSubject.Status)
110-
111-
// Set cluster to ready to indicate readiness to CAPI.
112-
r.ReconciliationSubject.Status.Ready = true
113-
}
114-
return ctrl.Result{}, err
101+
r.ReconciliationSubject.Status.Ready = true
102+
return ctrl.Result{}, nil
115103
}
116104

117105
// CheckZoneDetails verifies the Zone CRDs found match against those requested.
@@ -152,9 +140,6 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result,
152140
}
153141
return r.RequeueWithMessage("Child Zones still present, requeueing.")
154142
}
155-
if err := r.CSClient.DisposeClusterResources(r.ReconciliationSubject); err != nil {
156-
return ctrl.Result{}, err
157-
}
158143
controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
159144
return ctrl.Result{}, nil
160145
}

‎controllers/cloudstackisolatednetwork_controller.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ type CloudStackIsoNetReconciliationRunner struct {
4444
csCtrlrUtils.ReconciliationRunner
4545
Zone *infrav1.CloudStackZone
4646
ReconciliationSubject *infrav1.CloudStackIsolatedNetwork
47-
CSUser cloud.Client
4847
}
4948

5049
// Initialize a new CloudStackIsoNet reconciliation runner with concrete types and initialized member fields.
@@ -80,11 +79,11 @@ func (r *CloudStackIsoNetReconciliationRunner) Reconcile() (retRes ctrl.Result,
8079
if err != nil {
8180
return r.ReturnWrappedError(retErr, "setting up CloudStackCluster patcher:")
8281
}
83-
if err := r.CSClient.GetOrCreateIsolatedNetwork(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
82+
if err := r.CSUser.GetOrCreateIsolatedNetwork(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
8483
return ctrl.Result{}, err
8584
}
8685
// Tag the created network.
87-
if err := r.CSClient.AddClusterTag(cloud.ResourceTypeNetwork, r.ReconciliationSubject.Spec.ID, r.CSCluster); err != nil {
86+
if err := r.CSUser.AddClusterTag(cloud.ResourceTypeNetwork, r.ReconciliationSubject.Spec.ID, r.CSCluster); err != nil {
8887
return ctrl.Result{}, errors.Wrapf(err, "tagging network with id %s:", r.ReconciliationSubject.Spec.ID)
8988
}
9089
if err := csClusterPatcher.Patch(r.RequestCtx, r.CSCluster); err != nil {
@@ -97,7 +96,7 @@ func (r *CloudStackIsoNetReconciliationRunner) Reconcile() (retRes ctrl.Result,
9796

9897
func (r *CloudStackIsoNetReconciliationRunner) ReconcileDelete() (retRes ctrl.Result, retErr error) {
9998
r.Log.Info("Deleting IsolatedNetwork.")
100-
if err := r.CSClient.DisposeIsoNetResources(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
99+
if err := r.CSUser.DisposeIsoNetResources(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
101100
if !strings.Contains(strings.ToLower(err.Error()), "no match found") {
102101
return ctrl.Result{}, err
103102
}

‎controllers/cloudstackmachine_controller.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ type CloudStackMachineReconciliationRunner struct {
5353
ReconciliationSubject *infrav1.CloudStackMachine
5454
CAPIMachine *capiv1.Machine
5555
StateChecker *infrav1.CloudStackMachineStateChecker
56-
CSUser cloud.Client
5756
Zones *infrav1.CloudStackZoneList
5857
FailureDomain *infrav1.CloudStackZone
5958
IsoNet *infrav1.CloudStackIsolatedNetwork
@@ -122,7 +121,7 @@ func (r *CloudStackMachineReconciliationRunner) ConsiderAffinity() (ctrl.Result,
122121
return res, err
123122
}
124123
if !r.AffinityGroup.Status.Ready {
125-
return r.RequeueWithMessage("Required afinity group not ready.")
124+
return r.RequeueWithMessage("Required affinity group not ready.")
126125
}
127126
return ctrl.Result{}, nil
128127
}
@@ -191,7 +190,7 @@ func (r *CloudStackMachineReconciliationRunner) GetOrCreateVMInstance() (retRes
191190
return ctrl.Result{}, errors.New("bootstrap secret data not yet set")
192191
}
193192

194-
err := r.CSClient.GetOrCreateVMInstance(r.ReconciliationSubject, r.CAPIMachine, r.CSCluster, &machineZone, r.AffinityGroup, string(data))
193+
err := r.CSUser.GetOrCreateVMInstance(r.ReconciliationSubject, r.CAPIMachine, r.CSCluster, &machineZone, r.AffinityGroup, string(data))
195194

196195
if err == nil && !controllerutil.ContainsFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer) { // Fetched or Created?
197196
r.Log.Info("CloudStack instance Created", "instanceStatus", r.ReconciliationSubject.Status)
@@ -225,7 +224,7 @@ func (r *CloudStackMachineReconciliationRunner) AddToLBIfNeeded() (retRes ctrl.R
225224
if r.IsoNet.Spec.Name == "" {
226225
return r.RequeueWithMessage("Could not get required Isolated Network for VM, requeueing.")
227226
}
228-
err := r.CSClient.AssignVMToLoadBalancerRule(r.IsoNet, *r.ReconciliationSubject.Spec.InstanceID)
227+
err := r.CSUser.AssignVMToLoadBalancerRule(r.IsoNet, *r.ReconciliationSubject.Spec.InstanceID)
229228
if err != nil {
230229
return ctrl.Result{}, err
231230
}
@@ -251,6 +250,9 @@ func (r *CloudStackMachineReconciliationRunner) GetOrCreateMachineStateChecker()
251250

252251
func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.Result, reterr error) {
253252
r.Log.Info("Deleting instance", "instance-id", r.ReconciliationSubject.Spec.InstanceID)
253+
// Use CSClient instead of CSUser here to expunge as admin.
254+
// The CloudStack-Go API does not return an error, but the VM won't delete with Expunge set if requested by
255+
// non-domain admin user.
254256
if err := r.CSClient.DestroyVMInstance(r.ReconciliationSubject); err != nil {
255257
if err.Error() == "VM deletion in progress" {
256258
r.Log.Info(err.Error())

‎controllers/cloudstackzone_controller.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import (
2424

2525
infrav1 "github.com/aws/cluster-api-provider-cloudstack/api/v1beta1"
2626
csCtrlrUtils "github.com/aws/cluster-api-provider-cloudstack/controllers/utils"
27-
"github.com/aws/cluster-api-provider-cloudstack/pkg/cloud"
2827
"github.com/pkg/errors"
2928
)
3029

@@ -37,7 +36,6 @@ type CloudStackZoneReconciliationRunner struct {
3736
csCtrlrUtils.ReconciliationRunner
3837
Zones *infrav1.CloudStackZoneList
3938
ReconciliationSubject *infrav1.CloudStackZone
40-
CSUser cloud.Client
4139
IsoNet *infrav1.CloudStackIsolatedNetwork
4240
}
4341

@@ -83,10 +81,10 @@ func (r *CloudStackZoneReconciliationRunner) Reconcile() (retRes ctrl.Result, re
8381

8482
r.Log.V(1).Info("Reconciling CloudStackZone.", "zoneSpec", r.ReconciliationSubject.Spec)
8583
// Start by purely data fetching information about the zone and specified network.
86-
if err := r.CSClient.ResolveZone(r.ReconciliationSubject); err != nil {
84+
if err := r.CSUser.ResolveZone(r.ReconciliationSubject); err != nil {
8785
return ctrl.Result{}, errors.Wrap(err, "resolving CloudStack zone information:")
8886
}
89-
if err := r.CSClient.ResolveNetworkForZone(r.ReconciliationSubject); err != nil &&
87+
if err := r.CSUser.ResolveNetworkForZone(r.ReconciliationSubject); err != nil &&
9088
!csCtrlrUtils.ContainsNoMatchSubstring(err) {
9189
return ctrl.Result{}, errors.Wrap(err, "resolving Cloudstack network information:")
9290
}

‎controllers/utils/affinity_group.go

+11-5
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
infrav1 "github.com/aws/cluster-api-provider-cloudstack/api/v1beta1"
2424
"github.com/pkg/errors"
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2526
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2627
ctrl "sigs.k8s.io/controller-runtime"
2728
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -40,25 +41,30 @@ func (r *ReconciliationRunner) GetOrCreateAffinityGroup(name string, affinityTyp
4041
return ctrl.Result{}, nil
4142
} // Didn't find a group, so create instead.
4243

44+
// Set affinity group type.
4345
if affinityType == infrav1.ProAffinity {
4446
ag.Spec.Type = "host affinity"
4547
} else if affinityType == infrav1.AntiAffinity {
4648
ag.Spec.Type = "host anti-affinity"
4749
} else {
4850
return ctrl.Result{}, errors.Errorf("unrecognized affinity type %s", affinityType)
4951
}
52+
53+
// Setup basic metadata.
54+
ag.Name = name
5055
ag.Spec.Name = name
56+
ag.ObjectMeta = r.NewChildObjectMeta(lowerName)
5157

58+
// Replace owner reference with controller of CAPI and CloudStack machines.
5259
for _, ref := range r.ReconciliationSubject.GetOwnerReferences() {
53-
if ref.Kind == "EtcdadmCluster" || ref.Kind == "KubeadmControlPlane" || ref.Kind == "MachineSet" {
54-
ag.OwnerReferences = append(ag.OwnerReferences, ref)
60+
if strings.EqualFold(ref.Kind, "EtcdadmCluster") ||
61+
strings.EqualFold(ref.Kind, "KubeadmControlPlane") ||
62+
strings.EqualFold(ref.Kind, "MachineSet") {
63+
ag.OwnerReferences = []metav1.OwnerReference{ref}
5564
break
5665
}
5766
}
5867

59-
ag.Name = name
60-
ag.Spec.Name = name
61-
ag.ObjectMeta = r.NewChildObjectMeta(lowerName)
6268
if err := r.K8sClient.Create(r.RequestCtx, ag); err != nil && !ContainsAlreadyExistsSubstring(err) {
6369
return r.ReturnWrappedError(err, "creating affinity group CRD:")
6470
}

‎controllers/utils/base_reconciler.go

+26
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ type ReconciliationRunner struct {
6868
returnEarly bool // A signal that the reconcile should return early.
6969
ReconcileDelete CloudStackReconcilerMethod
7070
Reconcile CloudStackReconcilerMethod
71+
CSUser cloud.Client
7172
}
7273

7374
type ConcreteRunner interface {
@@ -219,6 +220,29 @@ func (r *ReconciliationRunner) CheckOwnedCRDsForReadiness(gvks ...schema.GroupVe
219220
}
220221
}
221222

223+
// SetCSUser sets the CSUser client to any user that can operate in specified domain and account if specified.
224+
func (r *ReconciliationRunner) SetCSUser() (ctrl.Result, error) {
225+
r.CSUser = r.CSClient
226+
if r.CSCluster.Spec.Account != "" {
227+
user := &cloud.User{}
228+
user.Account.Domain.Path = r.CSCluster.Spec.Domain
229+
user.Account.Name = r.CSCluster.Spec.Account
230+
if found, err := r.CSClient.GetUserWithKeys(user); err != nil {
231+
return ctrl.Result{}, err
232+
} else if !found {
233+
return ctrl.Result{}, errors.Errorf("could not find sufficient user (with API keys) in domain/account %s/%s",
234+
r.CSCluster.Spec.Domain, r.CSCluster.Spec.Account)
235+
}
236+
cfg := cloud.Config{APIKey: user.APIKey, SecretKey: user.SecretKey}
237+
client, err := r.CSClient.NewClientFromSpec(cfg)
238+
if err != nil {
239+
return ctrl.Result{}, err
240+
}
241+
r.CSUser = client
242+
}
243+
return ctrl.Result{}, nil
244+
}
245+
222246
// RequeueIfCloudStackClusterNotReady requeues the reconciliation request if the CloudStackCluster is not ready.
223247
func (r *ReconciliationRunner) RequeueIfCloudStackClusterNotReady() (ctrl.Result, error) {
224248
if !r.CSCluster.Status.Ready {
@@ -303,6 +327,8 @@ func (r *ReconciliationRunner) RunBaseReconciliationStages() (res ctrl.Result, r
303327
r.GetReconciliationSubject,
304328
r.GetCAPICluster,
305329
r.GetCSCluster,
330+
r.RequeueIfMissingBaseCRs,
331+
r.SetCSUser,
306332
r.IfDeletionTimestampIsZero(r.Reconcile),
307333
r.Else(r.ReconcileDelete),
308334
)

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/aws/cluster-api-provider-cloudstack
33
go 1.16
44

55
require (
6-
github.com/apache/cloudstack-go/v2 v2.11.1-0.20211020121644-369057554f66
6+
github.com/apache/cloudstack-go/v2 v2.13.0
77
github.com/go-logr/logr v0.4.0
88
github.com/golang/mock v1.6.0
99
github.com/hashicorp/go-multierror v1.1.1

‎go.sum

+2-3
Original file line numberDiff line numberDiff line change
@@ -76,16 +76,15 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
7676
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
7777
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
7878
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
79-
github.com/apache/cloudstack-go/v2 v2.11.1-0.20211020121644-369057554f66 h1:oy3ZgpLnqxczrbO48a3ajp2sEmRELLWcFH7foIACrkU=
80-
github.com/apache/cloudstack-go/v2 v2.11.1-0.20211020121644-369057554f66/go.mod h1:2mPSVR6kkM3u1i9L3mmR+Du1f+QBwFKYOSI2XQEyhj4=
79+
github.com/apache/cloudstack-go/v2 v2.13.0 h1:t0uj7QxQpnzD/LSTP6a4w2NTuZXisxIM/mIDNkF44lc=
80+
github.com/apache/cloudstack-go/v2 v2.13.0/go.mod h1:aosD8Svfu5nhH5Sp4zcsVV1hT5UGt3mTgRXM8YqTKe0=
8181
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
8282
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
8383
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
8484
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
8585
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
8686
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
8787
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
88-
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
8988
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
9089
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
9190
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=

‎pkg/cloud/affinity_groups_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,6 @@ var _ = Describe("AffinityGroup Unit Tests", func() {
8383
Ω(client.GetOrCreateAffinityGroup(dummies.AffinityGroup)).Should(Succeed())
8484
})
8585
It("Associates an affinity group.", func() {
86-
if err := client.GetOrCreateCluster(dummies.CSCluster); err != nil {
87-
Skip("Could not flesh out Cluster." + err.Error())
88-
}
8986
if err := client.GetOrCreateVMInstance(
9087
dummies.CSMachine1, dummies.CAPIMachine, dummies.CSCluster, dummies.CSZone1, dummies.CSAffinityGroup, "",
9188
); err != nil {

0 commit comments

Comments
 (0)