Skip to content

Commit 616e39b

Browse files
shraddhabangwweiwei-li
shraddhabang
and
wweiwei-li
authored
Add support for capacity unit reservation for load balancers (#3950)
* Add support for capacity unit reservation for load balancers * Add support for capacity unit reservation for load balancers * enable http and https listener attributes (#3948) --------- Co-authored-by: wweiwei-li <[email protected]>
1 parent 6a2dfee commit 616e39b

37 files changed

+1675
-133
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ site
2424
*.swo
2525
*~
2626
*.bak
27+
scripts/aws_sdk_model_override/*

‎apis/elbv2/v1beta1/ingressclassparams_types.go

+10
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,12 @@ type Listener struct {
101101
ListenerAttributes []Attribute `json:"listenerAttributes,omitempty"`
102102
}
103103

104+
// Information about a load balancer capacity reservation.
105+
type MinimumLoadBalancerCapacity struct {
106+
// The Capacity Units Value.
107+
CapacityUnits int32 `json:"capacityUnits"`
108+
}
109+
104110
// IngressClassParamsSpec defines the desired state of IngressClassParams
105111
type IngressClassParamsSpec struct {
106112
// CertificateArn specifies the ARN of the certificates for all Ingresses that belong to IngressClass with this IngressClassParams.
@@ -146,6 +152,10 @@ type IngressClassParamsSpec struct {
146152
// Listeners define a list of listeners with their protocol, port and attributes.
147153
// +optional
148154
Listeners []Listener `json:"listeners,omitempty"`
155+
156+
// MinimumLoadBalancerCapacity define the capacity reservation for LoadBalancers for all Ingress that belong to IngressClass with this IngressClassParams.
157+
// +optional
158+
MinimumLoadBalancerCapacity *MinimumLoadBalancerCapacity `json:"minimumLoadBalancerCapacity,omitempty"`
149159
}
150160

151161
// +kubebuilder:object:root=true

‎apis/elbv2/v1beta1/zz_generated.deepcopy.go

+20
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,18 @@ spec:
133133
- value
134134
type: object
135135
type: array
136+
minimumLoadBalancerCapacity:
137+
description: MinimumLoadBalancerCapacity define the capacity reservation
138+
for LoadBalancers for all Ingress that belong to IngressClass with
139+
this IngressClassParams.
140+
properties:
141+
capacityUnits:
142+
description: The Capacity Units Value.
143+
format: int32
144+
type: integer
145+
required:
146+
- capacityUnits
147+
type: object
136148
namespaceSelector:
137149
description: |-
138150
NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams.

‎controllers/ingress/group_controller.go

+4
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,10 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
170170
r.logger.Info("successfully built model", "model", stackJSON)
171171

172172
if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
173+
var requeueNeededAfter *runtime.RequeueNeededAfter
174+
if errors.As(err, &requeueNeededAfter) {
175+
return nil, nil, err
176+
}
173177
r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
174178
return nil, nil, err
175179
}

‎controllers/service/service_controller.go

+4
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,10 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service)
124124

125125
func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error {
126126
if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
127+
var requeueNeededAfter *runtime.RequeueNeededAfter
128+
if errors.As(err, &requeueNeededAfter) {
129+
return err
130+
}
127131
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
128132
return err
129133
}

‎docs/deploy/configurations.md

+5
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne
105105
| [sync-period](#sync-period) | duration | 10h0m0s | Period at which the controller forces the repopulation of its local object stores |
106106
| targetgroupbinding-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for targetGroupBinding |
107107
| targetgroupbinding-max-exponential-backoff-delay | duration | 16m40s | Maximum duration of exponential backoff for targetGroupBinding reconcile failures |
108+
| [lb-stabilization-monitor-interval](#lb-stabilization-monitor-interval) | duration | 2m | Interval at which the controller monitors the state of load balancer after creation
108109
| tolerate-non-existent-backend-service | boolean | true | Whether to allow rules which refer to backend services that do not exist (When enabled, it will return 503 error if backend service not exist) |
109110
| tolerate-non-existent-backend-action | boolean | true | Whether to allow rules which refer to backend actions that do not exist (When enabled, it will return 503 error if backend action not exist) |
110111
| watch-namespace | string | | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. |
@@ -138,6 +139,9 @@ Once disabled:
138139

139140
As best practice, we do not recommend users to manually modify the resources managed by the controller. And users should not depend on the controller auto-reconciliation to revert the manual modification, or to mitigate any security risks.
140141

142+
### lb-stabilization-monitor-interval
143+
`--lb-stabilization-monitor-interval` defines a fixed interval for the controller to monitor the state of load balancer after the creation for stabilization, default to 2m. It monitors the load balancer state so that once it becomes active it can make the required updates like capacity reservation for the active load balancer. It calls DescribeLoadBalancer API at a fixed interval to monitor the state. Please be mindful that lower value will result into frequent calls which may incur unnecessary AWS API usage.
144+
141145
### waf-addons
142146
By default, the controller assumes sole ownership of the WAF addons associated to the provisioned ALBs, via the flag `--enable-waf` and `--enable-wafv2`.
143147
And the users should disable them accordingly if they want a third party like AWS Firewall Manager to associate or remove the WAF-ACL of the ALBs.
@@ -178,3 +182,4 @@ There are a set of key=value pairs that describe AWS load balancer controller fe
178182
| NLBHealthCheckAdvancedConfiguration | string | true | Enable or disable advanced health check configuration for NLB, for example health check timeout |
179183
| ALBSingleSubnet | string | false | If enabled, controller will allow using only 1 subnet for provisioning ALB, which need to get whitelisted by ELB in advance |
180184
| NLBSecurityGroup | string | true | Enable or disable all NLB security groups actions including frontend sg creation, backend sg creation, and backend sg modifications |
185+
| LBCapacityReservation | string | true | Enable or disable the capacity reservation feature on ALB and NLB

‎docs/guide/ingress/annotations.md

+21
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ You can add annotations to kubernetes Ingress and Service objects to customize t
6161
| [alb.ingress.kubernetes.io/mutual-authentication](#mutual-authentication) | json |N/A| Ingress | Exclusive |
6262
| [alb.ingress.kubernetes.io/multi-cluster-target-group](#multi-cluster-target-group) | boolean |N/A| Ingress, Service | N/A |
6363
| [alb.ingress.kubernetes.io/listener-attributes.${Protocol}-${Port}](#listener-attributes) | stringMap |N/A| Ingress |Merge|
64+
| [alb.ingress.kubernetes.io/minimum-load-balancer-capacity](#load-balancer-capacity-reservation) | stringMap |N/A| Ingress | Exclusive |
6465

6566
## IngressGroup
6667
IngressGroup feature enables you to group multiple Ingress resources together.
@@ -931,6 +932,26 @@ In addition, you can use annotations to specify additional tags
931932
alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
932933
```
933934
935+
## Capacity Unit Reservation
936+
Load balancer capacity unit reservation can be configured via following annotations:
937+
938+
- <a name="load-balancer-capacity-reservation">`alb.ingress.kubernetes.io/minimum-load-balancer-capacity`</a> specifies the
939+
[Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/capacity-unit-reservation.html) to be configured.
940+
941+
!!!example
942+
- set the capacity unit reservation to 1000
943+
```
944+
alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=1000
945+
```
946+
- reset the capacity unit reservation
947+
```
948+
alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=0
949+
```
950+
951+
!!!note "Notes"
952+
- If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above.
953+
- If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true```
954+
934955
## Addons
935956
936957
- <a name="waf-acl-id">`alb.ingress.kubernetes.io/waf-acl-id`</a> specifies the identifier for the Amazon WAF Classic web ACL.

‎docs/guide/ingress/ingress_class.md

+19
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,16 @@ You can use IngressClassParams to enforce settings for a set of Ingresses.
140140
spec:
141141
certificateArn: ['arn:aws:acm:us-east-1:123456789:certificate/test-arn-1','arn:aws:acm:us-east-1:123456789:certificate/test-arn-2']
142142
```
143+
- with minimumLoadBalancerCapacity.capacityUnits
144+
```
145+
apiVersion: elbv2.k8s.aws/v1beta1
146+
kind: IngressClassParams
147+
metadata:
148+
name: class2048-config
149+
spec:
150+
minimumLoadBalancerCapacity:
151+
capacityUnits: 1000
152+
```
143153

144154
### IngressClassParams specification
145155

@@ -233,3 +243,12 @@ Cluster administrators can use `loadBalancerAttributes` field to specify the [Lo
233243

234244
1. If `loadBalancerAttributes` is set, the attributes defined will be applied to the load balancer that belong to this IngressClass. If you specify invalid keys or values for the load balancer attributes, the controller will fail to reconcile ingresses belonging to the particular ingress class.
235245
2. If `loadBalancerAttributes` un-specified, Ingresses with this IngressClass can continue to use `alb.ingress.kubernetes.io/load-balancer-attributes` annotation to specify the load balancer attributes.
246+
247+
#### spec.minimumLoadBalancerCapacity
248+
249+
Cluster administrators can use the optional `minimumLoadBalancerCapacity` field to specify the capacity reservation for the load balancers that belong to this IngressClass.
250+
They may specify `capacityUnits`. If the field is specified, LBC will ignore the `alb.ingress.kubernetes.io/minimum-load-balancer-capacity annotation` annotation.
251+
252+
##### spec.minimumLoadBalancerCapacity.capacityUnits
253+
254+
If `capacityUnits` is specified, it must be to valid positive value greater than 0. If set to 0, the LBC will reset the capacity reservation for the load balancer.

‎docs/guide/service/annotations.md

+20
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
| [service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group](#multi-cluster-target-group) | boolean | false | If specified, the controller will only operate on targets that exist within the cluster, ignoring targets from other sources. |
5757
| [service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat](#enable-prefix-for-ipv6-source-nat) | string | off | Optional annotation. dualstack lb only. Allowed values - on and off |
5858
| [service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes](#source-nat-ipv6-prefixes) | stringList | | Optional annotation. dualstack lb only. This annotation is only applicable when user has to set the service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat to "on". Length must match the number of subnets |
59+
| [service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity](#load-balancer-capacity-reservation) | stringMap | |
5960

6061
## Traffic Routing
6162
Traffic Routing can be controlled with following annotations:
@@ -579,6 +580,25 @@ Load balancer access can be controlled via following annotations:
579580
service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic: "off"
580581
```
581582
583+
## Capacity Unit Reservation
584+
Load balancer capacity unit reservation can be configured via following annotations:
585+
586+
- <a name="load-balancer-capacity-reservation">`service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity`</a> specifies the
587+
[Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/capacity-unit-reservation.html) to be configured.
588+
589+
!!!example
590+
- set the capacity unit reservation to 1000
591+
```
592+
service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=3000
593+
```
594+
- reset the capacity unit reservation
595+
```
596+
service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=0
597+
```
598+
599+
!!!note "Notes"
600+
- If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above.
601+
- If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true```
582602
583603
## Legacy Cloud Provider
584604
The AWS Load Balancer Controller manages Kubernetes Services in a compatible way with the AWS cloud provider's legacy service controller.

‎docs/install/iam_policy.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
"elasticloadbalancing:DescribeTargetHealth",
4242
"elasticloadbalancing:DescribeTags",
4343
"elasticloadbalancing:DescribeTrustStores",
44-
"elasticloadbalancing:DescribeListenerAttributes"
44+
"elasticloadbalancing:DescribeListenerAttributes",
45+
"elasticloadbalancing:DescribeCapacityReservation"
4546
],
4647
"Resource": "*"
4748
},
@@ -191,7 +192,8 @@
191192
"elasticloadbalancing:ModifyTargetGroup",
192193
"elasticloadbalancing:ModifyTargetGroupAttributes",
193194
"elasticloadbalancing:DeleteTargetGroup",
194-
"elasticloadbalancing:ModifyListenerAttributes"
195+
"elasticloadbalancing:ModifyListenerAttributes",
196+
"elasticloadbalancing:ModifyCapacityReservation"
195197
],
196198
"Resource": "*",
197199
"Condition": {

‎docs/install/iam_policy_cn.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
"elasticloadbalancing:DescribeTargetHealth",
4242
"elasticloadbalancing:DescribeTags",
4343
"elasticloadbalancing:DescribeTrustStores",
44-
"elasticloadbalancing:DescribeListenerAttributes"
44+
"elasticloadbalancing:DescribeListenerAttributes",
45+
"elasticloadbalancing:DescribeCapacityReservation"
4546
],
4647
"Resource": "*"
4748
},
@@ -213,7 +214,8 @@
213214
"elasticloadbalancing:ModifyTargetGroup",
214215
"elasticloadbalancing:ModifyTargetGroupAttributes",
215216
"elasticloadbalancing:DeleteTargetGroup",
216-
"elasticloadbalancing:ModifyListenerAttributes"
217+
"elasticloadbalancing:ModifyListenerAttributes",
218+
"elasticloadbalancing:ModifyCapacityReservation"
217219
],
218220
"Resource": "*",
219221
"Condition": {

‎docs/install/iam_policy_us-gov.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
"elasticloadbalancing:DescribeTargetHealth",
4242
"elasticloadbalancing:DescribeTags",
4343
"elasticloadbalancing:DescribeTrustStores",
44-
"elasticloadbalancing:DescribeListenerAttributes"
44+
"elasticloadbalancing:DescribeListenerAttributes",
45+
"elasticloadbalancing:DescribeCapacityReservation"
4546
],
4647
"Resource": "*"
4748
},
@@ -213,7 +214,8 @@
213214
"elasticloadbalancing:ModifyTargetGroup",
214215
"elasticloadbalancing:ModifyTargetGroupAttributes",
215216
"elasticloadbalancing:DeleteTargetGroup",
216-
"elasticloadbalancing:ModifyListenerAttributes"
217+
"elasticloadbalancing:ModifyListenerAttributes",
218+
"elasticloadbalancing:ModifyCapacityReservation"
217219
],
218220
"Resource": "*",
219221
"Condition": {

0 commit comments

Comments
 (0)