Skip to content

Commit 04c9bf5

Browse files
kishorjkishorj
authored
Add support for NLB instance mode (#1832)
* add support for NLB instance mode * decouple health check default values from builder task
1 parent 88b90c0 commit 04c9bf5

File tree

8 files changed

+1156
-64
lines changed

8 files changed

+1156
-64
lines changed

‎controllers/service/eventhandlers/service_events.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,12 @@ import (
88
"k8s.io/client-go/tools/record"
99
"k8s.io/client-go/util/workqueue"
1010
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
11+
svcpkg "sigs.k8s.io/aws-load-balancer-controller/pkg/service"
1112
"sigs.k8s.io/controller-runtime/pkg/event"
1213
"sigs.k8s.io/controller-runtime/pkg/handler"
1314
"sigs.k8s.io/controller-runtime/pkg/reconcile"
1415
)
1516

16-
const loadBalancerTypeNLBIP = "nlb-ip"
17-
1817
// NewEnqueueRequestForServiceEvent constructs new enqueueRequestsForServiceEvent.
1918
func NewEnqueueRequestForServiceEvent(eventRecorder record.EventRecorder, annotationParser annotations.Parser, logger logr.Logger) *enqueueRequestsForServiceEvent {
2019
return &enqueueRequestsForServiceEvent{
@@ -61,7 +60,13 @@ func (h *enqueueRequestsForServiceEvent) Generic(e event.GenericEvent, queue wor
6160
func (h *enqueueRequestsForServiceEvent) isServiceSupported(service *corev1.Service) bool {
6261
lbType := ""
6362
_ = h.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixLoadBalancerType, &lbType, service.Annotations)
64-
if lbType == loadBalancerTypeNLBIP {
63+
if lbType == svcpkg.LoadBalancerTypeNLBIP {
64+
return true
65+
}
66+
var lbTargetType string
67+
_ = h.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixTargetType, &lbTargetType, service.Annotations)
68+
if lbType == svcpkg.LoadBalancerTypeExternal && (lbTargetType == svcpkg.LoadBalancerTargetTypeNLBIP ||
69+
lbTargetType == svcpkg.LoadBalancerTargetTypeNLBInstance) {
6570
return true
6671
}
6772
return false

‎pkg/annotations/constants.go

+2
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,6 @@ const (
7171
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
7272
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
7373
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
74+
SvcLBSuffixTargetType = "aws-load-balancer-target-type"
75+
SvcLBSuffixTargetNodeLabels = "aws-load-balancer-target-node-labels"
7476
)

‎pkg/deploy/elbv2/target_group_binding_manager.go

+1
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ func buildK8sTargetGroupBindingSpec(ctx context.Context, resTGB *elbv2model.Targ
185185
}
186186
k8sTGBSpec.Networking = &k8sTGBNetworking
187187
}
188+
k8sTGBSpec.NodeSelector = resTGB.Spec.Template.Spec.NodeSelector
188189
return k8sTGBSpec, nil
189190
}
190191

‎pkg/model/elbv2/target_group_binding.go

+4
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,10 @@ type TargetGroupBindingSpec struct {
9595
// networking provides the networking setup for ELBV2 LoadBalancer to access targets in TargetGroup.
9696
// +optional
9797
Networking *TargetGroupBindingNetworking `json:"networking,omitempty"`
98+
99+
// node selector for instance type target groups to only register certain nodes
100+
// +optional
101+
NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"`
98102
}
99103

100104
// Template for TargetGroupBinding Custom Resource.

‎pkg/service/model_build_target_group.go

+135-52
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,11 @@ func (t *defaultModelBuildTask) buildTargetGroup(ctx context.Context, port corev
3232
if targetGroup, exists := t.tgByResID[tgResourceID]; exists {
3333
return targetGroup, nil
3434
}
35-
healthCheckConfig, err := t.buildTargetGroupHealthCheckConfig(ctx)
35+
targetType, err := t.buildTargetType(ctx)
3636
if err != nil {
3737
return nil, err
3838
}
39-
targetType, err := t.buildTargetType(ctx)
39+
healthCheckConfig, err := t.buildTargetGroupHealthCheckConfig(ctx, targetType)
4040
if err != nil {
4141
return nil, err
4242
}
@@ -53,8 +53,11 @@ func (t *defaultModelBuildTask) buildTargetGroup(ctx context.Context, port corev
5353
return nil, err
5454
}
5555
targetGroup := elbv2model.NewTargetGroup(t.stack, tgResourceID, tgSpec)
56+
_, err = t.buildTargetGroupBinding(ctx, targetGroup, preserveClientIP, port, healthCheckConfig)
57+
if err != nil {
58+
return nil, err
59+
}
5660
t.tgByResID[tgResourceID] = targetGroup
57-
_ = t.buildTargetGroupBinding(ctx, targetGroup, preserveClientIP, port, healthCheckConfig)
5861
return targetGroup, nil
5962
}
6063

@@ -77,28 +80,70 @@ func (t *defaultModelBuildTask) buildTargetGroupSpec(ctx context.Context, tgProt
7780
}, nil
7881
}
7982

80-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfig(ctx context.Context) (*elbv2model.TargetGroupHealthCheckConfig, error) {
81-
healthCheckProtocol, err := t.buildTargetGroupHealthCheckProtocol(ctx)
83+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfig(ctx context.Context, targetType elbv2model.TargetType) (*elbv2model.TargetGroupHealthCheckConfig, error) {
84+
if targetType == elbv2model.TargetTypeInstance && t.service.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeLocal {
85+
return t.buildTargetGroupHealthCheckConfigForInstanceModeLocal(ctx)
86+
}
87+
return t.buildTargetGroupHealthCheckConfigDefault(ctx)
88+
}
89+
90+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfigDefault(ctx context.Context) (*elbv2model.TargetGroupHealthCheckConfig, error) {
91+
healthCheckProtocol, err := t.buildTargetGroupHealthCheckProtocol(ctx, t.defaultHealthCheckProtocol)
8292
if err != nil {
8393
return nil, err
8494
}
8595
var healthCheckPathPtr *string
8696
if healthCheckProtocol != elbv2model.ProtocolTCP {
87-
healthCheckPathPtr = t.buildTargetGroupHealthCheckPath(ctx)
97+
healthCheckPathPtr = t.buildTargetGroupHealthCheckPath(ctx, t.defaultHealthCheckPath)
8898
}
89-
healthCheckPort, err := t.buildTargetGroupHealthCheckPort(ctx)
99+
healthCheckPort, err := t.buildTargetGroupHealthCheckPort(ctx, t.defaultHealthCheckPort)
90100
if err != nil {
91101
return nil, err
92102
}
93-
intervalSeconds, err := t.buildTargetGroupHealthCheckIntervalSeconds(ctx)
103+
intervalSeconds, err := t.buildTargetGroupHealthCheckIntervalSeconds(ctx, t.defaultHealthCheckInterval)
94104
if err != nil {
95105
return nil, err
96106
}
97-
healthyThresholdCount, err := t.buildTargetGroupHealthCheckHealthyThresholdCount(ctx)
107+
healthyThresholdCount, err := t.buildTargetGroupHealthCheckHealthyThresholdCount(ctx, t.defaultHealthCheckHealthyThreshold)
98108
if err != nil {
99109
return nil, err
100110
}
101-
unhealthyThresholdCount, err := t.buildTargetGroupHealthCheckUnhealthyThresholdCount(ctx)
111+
unhealthyThresholdCount, err := t.buildTargetGroupHealthCheckUnhealthyThresholdCount(ctx, t.defaultHealthCheckUnhealthyThreshold)
112+
if err != nil {
113+
return nil, err
114+
}
115+
return &elbv2model.TargetGroupHealthCheckConfig{
116+
Port: &healthCheckPort,
117+
Protocol: &healthCheckProtocol,
118+
Path: healthCheckPathPtr,
119+
IntervalSeconds: &intervalSeconds,
120+
HealthyThresholdCount: &healthyThresholdCount,
121+
UnhealthyThresholdCount: &unhealthyThresholdCount,
122+
}, nil
123+
}
124+
125+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfigForInstanceModeLocal(ctx context.Context) (*elbv2model.TargetGroupHealthCheckConfig, error) {
126+
healthCheckProtocol, err := t.buildTargetGroupHealthCheckProtocol(ctx, t.defaultHealthCheckProtocolForInstanceModeLocal)
127+
if err != nil {
128+
return nil, err
129+
}
130+
var healthCheckPathPtr *string
131+
if healthCheckProtocol != elbv2model.ProtocolTCP {
132+
healthCheckPathPtr = t.buildTargetGroupHealthCheckPath(ctx, t.defaultHealthCheckPathForInstanceModeLocal)
133+
}
134+
healthCheckPort, err := t.buildTargetGroupHealthCheckPort(ctx, t.defaultHealthCheckPortForInstanceModeLocal)
135+
if err != nil {
136+
return nil, err
137+
}
138+
intervalSeconds, err := t.buildTargetGroupHealthCheckIntervalSeconds(ctx, t.defaultHealthCheckIntervalForInstanceModeLocal)
139+
if err != nil {
140+
return nil, err
141+
}
142+
healthyThresholdCount, err := t.buildTargetGroupHealthCheckHealthyThresholdCount(ctx, t.defaultHealthCheckHealthyThresholdForInstanceModeLocal)
143+
if err != nil {
144+
return nil, err
145+
}
146+
unhealthyThresholdCount, err := t.buildTargetGroupHealthCheckUnhealthyThresholdCount(ctx, t.defaultHealthCheckUnhealthyThresholdForInstanceModeLocal)
102147
if err != nil {
103148
return nil, err
104149
}
@@ -193,21 +238,37 @@ func (t *defaultModelBuildTask) buildPreserveClientIPFlag(_ context.Context, tar
193238
return false, nil
194239
}
195240

196-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPort(_ context.Context) (intstr.IntOrString, error) {
197-
rawHealthCheckPort := t.defaultHealthCheckPort
241+
// buildTargetGroupPort constructs the TargetGroup's port.
242+
// Note: TargetGroup's port is not in the data path as we always register targets with port specified.
243+
// so this settings don't really matter to our controller, and we do our best to use the most appropriate port as targetGroup's port to avoid UX confusing.
244+
func (t *defaultModelBuildTask) buildTargetGroupPort(_ context.Context, targetType elbv2model.TargetType, svcPort corev1.ServicePort) int64 {
245+
if targetType == elbv2model.TargetTypeInstance {
246+
return int64(svcPort.NodePort)
247+
}
248+
if svcPort.TargetPort.Type == intstr.Int {
249+
return int64(svcPort.TargetPort.IntValue())
250+
}
251+
252+
// when a literal targetPort is used, we just use a fixed 1 here as this setting is not in the data path.
253+
// also, under extreme edge case, it can actually be different ports for different pods.
254+
return 1
255+
}
256+
257+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPort(_ context.Context, defaultHealthCheckPort string) (intstr.IntOrString, error) {
258+
rawHealthCheckPort := defaultHealthCheckPort
198259
t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixHCPort, &rawHealthCheckPort, t.service.Annotations)
199-
if rawHealthCheckPort == t.defaultHealthCheckPort {
260+
if rawHealthCheckPort == healthCheckPortTrafficPort {
200261
return intstr.FromString(rawHealthCheckPort), nil
201262
}
202-
var portVal int64
203-
if _, err := t.annotationParser.ParseInt64Annotation(annotations.SvcLBSuffixHCPort, &portVal, t.service.Annotations); err != nil {
204-
return intstr.IntOrString{}, err
263+
portVal, err := strconv.ParseInt(rawHealthCheckPort, 10, 64)
264+
if err != nil {
265+
return intstr.IntOrString{}, errors.Errorf("health check port \"%v\" not supported", rawHealthCheckPort)
205266
}
206267
return intstr.FromInt(int(portVal)), nil
207268
}
208269

209-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckProtocol(_ context.Context) (elbv2model.Protocol, error) {
210-
rawHealthCheckProtocol := string(t.defaultHealthCheckProtocol)
270+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckProtocol(_ context.Context, defaultHealthCheckProtocol elbv2model.Protocol) (elbv2model.Protocol, error) {
271+
rawHealthCheckProtocol := string(defaultHealthCheckProtocol)
211272
t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixHCProtocol, &rawHealthCheckProtocol, t.service.Annotations)
212273
switch strings.ToUpper(rawHealthCheckProtocol) {
213274
case string(elbv2model.ProtocolTCP):
@@ -221,62 +282,56 @@ func (t *defaultModelBuildTask) buildTargetGroupHealthCheckProtocol(_ context.Co
221282
}
222283
}
223284

224-
// buildTargetGroupPort constructs the TargetGroup's port.
225-
// Note: TargetGroup's port is not in the data path as we always register targets with port specified.
226-
// so this settings don't really matter to our controller, and we do our best to use the most appropriate port as targetGroup's port to avoid UX confusing.
227-
func (t *defaultModelBuildTask) buildTargetGroupPort(_ context.Context, targetType elbv2model.TargetType, svcPort corev1.ServicePort) int64 {
228-
if targetType == elbv2model.TargetTypeInstance {
229-
return int64(svcPort.NodePort)
230-
}
231-
if svcPort.TargetPort.Type == intstr.Int {
232-
return int64(svcPort.TargetPort.IntValue())
233-
}
234-
235-
// when a literal targetPort is used, we just use a fixed 1 here as this setting is not in the data path.
236-
// also, under extreme edge case, it can actually be different ports for different pods.
237-
return 1
238-
}
239-
240-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPath(_ context.Context) *string {
241-
healthCheckPath := t.defaultHealthCheckPath
285+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPath(_ context.Context, defaultHealthCheckPath string) *string {
286+
healthCheckPath := defaultHealthCheckPath
242287
t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixHCPath, &healthCheckPath, t.service.Annotations)
243288
return &healthCheckPath
244289
}
245290

246-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckIntervalSeconds(_ context.Context) (int64, error) {
247-
intervalSeconds := t.defaultHealthCheckInterval
291+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckIntervalSeconds(_ context.Context, defaultHealthCheckInterval int64) (int64, error) {
292+
intervalSeconds := defaultHealthCheckInterval
248293
if _, err := t.annotationParser.ParseInt64Annotation(annotations.SvcLBSuffixHCInterval, &intervalSeconds, t.service.Annotations); err != nil {
249294
return 0, err
250295
}
251296
return intervalSeconds, nil
252297
}
253298

254-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckTimeoutSeconds(_ context.Context) (int64, error) {
255-
timeoutSeconds := t.defaultHealthCheckTimeout
299+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckTimeoutSeconds(_ context.Context, defaultHealthCheckTimeout int64) (int64, error) {
300+
timeoutSeconds := defaultHealthCheckTimeout
256301
if _, err := t.annotationParser.ParseInt64Annotation(annotations.SvcLBSuffixHCTimeout, &timeoutSeconds, t.service.Annotations); err != nil {
257302
return 0, err
258303
}
259304
return timeoutSeconds, nil
260305
}
261306

262-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckHealthyThresholdCount(_ context.Context) (int64, error) {
263-
healthyThresholdCount := t.defaultHealthCheckHealthyThreshold
307+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckHealthyThresholdCount(_ context.Context, defaultHealthCheckHealthyThreshold int64) (int64, error) {
308+
healthyThresholdCount := defaultHealthCheckHealthyThreshold
264309
if _, err := t.annotationParser.ParseInt64Annotation(annotations.SvcLBSuffixHCHealthyThreshold, &healthyThresholdCount, t.service.Annotations); err != nil {
265310
return 0, err
266311
}
267312
return healthyThresholdCount, nil
268313
}
269314

270-
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckUnhealthyThresholdCount(_ context.Context) (int64, error) {
271-
unhealthyThresholdCount := t.defaultHealthCheckUnhealthyThreshold
315+
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckUnhealthyThresholdCount(_ context.Context, defaultHealthCheckUnhealthyThreshold int64) (int64, error) {
316+
unhealthyThresholdCount := defaultHealthCheckUnhealthyThreshold
272317
if _, err := t.annotationParser.ParseInt64Annotation(annotations.SvcLBSuffixHCUnhealthyThreshold, &unhealthyThresholdCount, t.service.Annotations); err != nil {
273318
return 0, err
274319
}
275320
return unhealthyThresholdCount, nil
276321
}
277322

278323
func (t *defaultModelBuildTask) buildTargetType(_ context.Context) (elbv2model.TargetType, error) {
279-
return elbv2model.TargetTypeIP, nil
324+
var lbType string
325+
_ = t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixLoadBalancerType, &lbType, t.service.Annotations)
326+
var lbTargetType string
327+
_ = t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixTargetType, &lbTargetType, t.service.Annotations)
328+
if lbType == LoadBalancerTargetTypeNLBIP || (lbType == LoadBalancerTypeExternal && lbTargetType == LoadBalancerTargetTypeNLBIP) {
329+
return elbv2model.TargetTypeIP, nil
330+
}
331+
if lbType == LoadBalancerTypeExternal && lbTargetType == LoadBalancerTargetTypeNLBInstance {
332+
return elbv2model.TargetTypeInstance, nil
333+
}
334+
return "", errors.Errorf("unsupported target type \"%v\" for load balancer type \"%v\"", lbTargetType, lbType)
280335
}
281336

282337
func (t *defaultModelBuildTask) buildTargetGroupResourceID(svcKey types.NamespacedName, port intstr.IntOrString) string {
@@ -288,15 +343,26 @@ func (t *defaultModelBuildTask) buildTargetGroupTags(ctx context.Context) (map[s
288343
}
289344

290345
func (t *defaultModelBuildTask) buildTargetGroupBinding(ctx context.Context, targetGroup *elbv2model.TargetGroup, preserveClientIP bool,
291-
port corev1.ServicePort, hc *elbv2model.TargetGroupHealthCheckConfig) *elbv2model.TargetGroupBindingResource {
292-
tgbSpec := t.buildTargetGroupBindingSpec(ctx, targetGroup, preserveClientIP, port, hc)
293-
return elbv2model.NewTargetGroupBindingResource(t.stack, targetGroup.ID(), tgbSpec)
346+
port corev1.ServicePort, hc *elbv2model.TargetGroupHealthCheckConfig) (*elbv2model.TargetGroupBindingResource, error) {
347+
tgbSpec, err := t.buildTargetGroupBindingSpec(ctx, targetGroup, preserveClientIP, port, hc)
348+
if err != nil {
349+
return nil, err
350+
}
351+
return elbv2model.NewTargetGroupBindingResource(t.stack, targetGroup.ID(), tgbSpec), nil
294352
}
295353

296354
func (t *defaultModelBuildTask) buildTargetGroupBindingSpec(ctx context.Context, targetGroup *elbv2model.TargetGroup, preserveClientIP bool,
297-
port corev1.ServicePort, hc *elbv2model.TargetGroupHealthCheckConfig) elbv2model.TargetGroupBindingResourceSpec {
298-
tgbNetworking := t.buildTargetGroupBindingNetworking(ctx, port.TargetPort, preserveClientIP, *hc.Port, port.Protocol)
355+
port corev1.ServicePort, hc *elbv2model.TargetGroupHealthCheckConfig) (elbv2model.TargetGroupBindingResourceSpec, error) {
356+
nodeSelector, err := t.buildTargetGroupBindingNodeSelector(ctx, targetGroup.Spec.TargetType)
357+
if err != nil {
358+
return elbv2model.TargetGroupBindingResourceSpec{}, err
359+
}
360+
targetPort := port.TargetPort
299361
targetType := elbv2api.TargetType(targetGroup.Spec.TargetType)
362+
if targetType == elbv2api.TargetTypeInstance {
363+
targetPort = intstr.FromInt(int(port.NodePort))
364+
}
365+
tgbNetworking := t.buildTargetGroupBindingNetworking(ctx, targetPort, preserveClientIP, *hc.Port, port.Protocol)
300366
return elbv2model.TargetGroupBindingResourceSpec{
301367
Template: elbv2model.TargetGroupBindingTemplate{
302368
ObjectMeta: metav1.ObjectMeta{
@@ -310,10 +376,11 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingSpec(ctx context.Context,
310376
Name: t.service.Name,
311377
Port: intstr.FromInt(int(port.Port)),
312378
},
313-
Networking: tgbNetworking,
379+
Networking: tgbNetworking,
380+
NodeSelector: nodeSelector,
314381
},
315382
},
316-
}
383+
}, nil
317384
}
318385

319386
func (t *defaultModelBuildTask) buildPeersFromSourceRanges(_ context.Context) []elbv2model.NetworkingPeer {
@@ -388,3 +455,19 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNetworking(ctx context.Co
388455
}
389456
return tgbNetworking
390457
}
458+
459+
func (t *defaultModelBuildTask) buildTargetGroupBindingNodeSelector(_ context.Context, targetType elbv2model.TargetType) (*metav1.LabelSelector, error) {
460+
if targetType != elbv2model.TargetTypeInstance {
461+
return nil, nil
462+
}
463+
var targetNodeLabels map[string]string
464+
if _, err := t.annotationParser.ParseStringMapAnnotation(annotations.SvcLBSuffixTargetNodeLabels, &targetNodeLabels, t.service.Annotations); err != nil {
465+
return nil, err
466+
}
467+
if len(targetNodeLabels) == 0 {
468+
return nil, nil
469+
}
470+
return &metav1.LabelSelector{
471+
MatchLabels: targetNodeLabels,
472+
}, nil
473+
}

0 commit comments

Comments
 (0)