Skip to content

Commit 369bdac

Browse files
marcosdiezmarcosdiez
authored
CI-14222_manipulate_target_groups_from_different_aws_accounts (#3691)
1 parent 2ff2e59 commit 369bdac

23 files changed

+474
-136
lines changed

‎apis/elbv2/v1beta1/targetgroupbinding_types.go

+8
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,14 @@ type TargetGroupBindingSpec struct {
157157
// VpcID is the VPC of the TargetGroup. If unspecified, it will be automatically inferred.
158158
// +optional
159159
VpcID string `json:"vpcID,omitempty"`
160+
161+
// IAM Role ARN to assume when calling AWS APIs. Useful if the target group is in a different AWS account
162+
// +optional
163+
IamRoleArnToAssume string `json:"-"` // `json:"iamRoleArnToAssume,omitempty"`
164+
165+
// IAM Role ARN to assume when calling AWS APIs. Needed to assume a role in another account and prevent the confused deputy problem. https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html
166+
// +optional
167+
AssumeRoleExternalId string `json:"-"` // `json:"assumeRoleExternalId,omitempty"`
160168
}
161169

162170
// TargetGroupBindingStatus defines the observed state of TargetGroupBinding

‎controllers/ingress/group_controller.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1"
1717
"sigs.k8s.io/aws-load-balancer-controller/controllers/ingress/eventhandlers"
1818
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
19-
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
19+
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
2020
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
2121
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
2222
elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
@@ -44,7 +44,7 @@ const (
4444
)
4545

4646
// NewGroupReconciler constructs new GroupReconciler
47-
func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
47+
func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
4848
finalizerManager k8s.FinalizerManager, networkingSGManager networkingpkg.SecurityGroupManager,
4949
networkingSGReconciler networkingpkg.SecurityGroupReconciler, subnetsResolver networkingpkg.SubnetsResolver,
5050
elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig, backendSGProvider networkingpkg.BackendSGProvider,

‎controllers/service/service_controller.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"k8s.io/client-go/tools/record"
1313
"sigs.k8s.io/aws-load-balancer-controller/controllers/service/eventhandlers"
1414
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
15-
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
15+
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
1616
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
1717
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
1818
elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
@@ -35,7 +35,7 @@ const (
3535
controllerName = "service"
3636
)
3737

38-
func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
38+
func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
3939
finalizerManager k8s.FinalizerManager, networkingSGManager networking.SecurityGroupManager,
4040
networkingSGReconciler networking.SecurityGroupReconciler, subnetsResolver networking.SubnetsResolver,
4141
vpcInfoProvider networking.VPCInfoProvider, elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig,

‎docs/deploy/installation.md

+59-4
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ The LBC is supported by AWS. Some clusters may be using the legacy "in-tree" fun
77
!!!question "Existing AWS ALB Ingress Controller users"
88
The AWS ALB Ingress controller must be uninstalled before installing the AWS Load Balancer Controller.
99
Please follow our [migration guide](upgrade/migrate_v1_v2.md) to do a migration.
10-
10+
1111
!!!warning "When using AWS Load Balancer Controller v2.5+"
12-
The AWS LBC provides a mutating webhook for service resources to set the `spec.loadBalancerClass` field for service of type LoadBalancer on create.
13-
This makes the AWS LBC the **default controller for service** of type LoadBalancer. You can disable this feature and revert to set Cloud Controller Manager (in-tree controller) as the default by setting the helm chart value **enableServiceMutatorWebhook to false** with `--set enableServiceMutatorWebhook=false` .
12+
The AWS LBC provides a mutating webhook for service resources to set the `spec.loadBalancerClass` field for service of type LoadBalancer on create.
13+
This makes the AWS LBC the **default controller for service** of type LoadBalancer. You can disable this feature and revert to set Cloud Controller Manager (in-tree controller) as the default by setting the helm chart value **enableServiceMutatorWebhook to false** with `--set enableServiceMutatorWebhook=false` .
1414
You will no longer be able to provision new Classic Load Balancer (CLB) from your kubernetes service unless you disable this feature. Existing CLB will continue to work fine.
1515

1616
## Supported Kubernetes versions
@@ -30,7 +30,7 @@ The LBC is supported by AWS. Some clusters may be using the legacy "in-tree" fun
3030
Isolated clusters are clusters without internet access, and instead reply on VPC endpoints for all required connects.
3131
When installing the AWS LBC in isolated clusters, you need to disable shield, waf and wafv2 via controller flags `--enable-shield=false, --enable-waf=false, --enable-wafv2=false`
3232
### Using the Amazon EC2 instance metadata server version 2 (IMDSv2)
33-
We recommend blocking the access to instance metadata by requiring the instance to use [IMDSv2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) only. For more information, please refer to the AWS guidance [here](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node). If you are using the IMDSv2, set the hop limit to 2 or higher in order to allow the LBC to perform the metadata introspection.
33+
We recommend blocking the access to instance metadata by requiring the instance to use [IMDSv2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) only. For more information, please refer to the AWS guidance [here](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node). If you are using the IMDSv2, set the hop limit to 2 or higher in order to allow the LBC to perform the metadata introspection.
3434

3535
You can set the IMDSv2 as follows:
3636
```
@@ -127,6 +127,10 @@ If you're not setting up IAM roles for service accounts, apply the IAM policies
127127
curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy.json
128128
```
129129
130+
## Special IAM cases
131+
132+
### You only want the LBC to add and remove IPs to already existing target groups:
133+
130134
The following IAM permissions subset is for those using `TargetGroupBinding` only and don't plan to use the LBC to manage security group rules:
131135
132136
```
@@ -152,6 +156,57 @@ The following IAM permissions subset is for those using `TargetGroupBinding` onl
152156
}
153157
```
154158
159+
### You only want the LBC to add and remove IPs to already existing target groups, also in other accounts, assuming roles
160+
161+
On the other hand, if you plan to use the LBC to manage also target groups in different accounts, you will need to add `"sts:AssumeRole"` to your list of permissions, in other words:
162+
163+
```
164+
{
165+
"Statement": [
166+
{
167+
"Action": [
168+
"ec2:DescribeVpcs",
169+
"ec2:DescribeSecurityGroups",
170+
"ec2:DescribeInstances",
171+
"elasticloadbalancing:DescribeTargetGroups",
172+
"elasticloadbalancing:DescribeTargetHealth",
173+
"elasticloadbalancing:ModifyTargetGroup",
174+
"elasticloadbalancing:ModifyTargetGroupAttributes",
175+
"elasticloadbalancing:RegisterTargets",
176+
"elasticloadbalancing:DeregisterTargets",
177+
"sts:AssumeRole"
178+
],
179+
"Effect": "Allow",
180+
"Resource": "*"
181+
}
182+
],
183+
"Version": "2012-10-17"
184+
}
185+
```
186+
187+
The assumed roles will need the exactly the same permissions, without `"sts:AssumeRole"`. The assumed role will need a to allow to be assumed by the main role, something like this:
188+
189+
```
190+
{
191+
"Version": "2012-10-17",
192+
"Statement": [
193+
{
194+
"Sid": "",
195+
"Effect": "Allow",
196+
"Principal": {
197+
"AWS": "arn:aws:iam::999999999999999:user/test-alb-controller"
198+
},
199+
"Action": "sts:AssumeRole",
200+
"Condition": {
201+
"StringEquals": {
202+
"sts:ExternalId": "very-secret-string"
203+
}
204+
}
205+
}
206+
]
207+
}
208+
```
209+
155210
## Network configuration
156211
157212
Review the [worker nodes security group](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) docs. Your node security group must permit incoming traffic on TCP port 9443 from the Kubernetes control plane. This is needed for webhook access.

‎docs/guide/targetgroupbinding/spec.md

+20-1
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,29 @@ Kubernetes meta/v1.ObjectMeta
5252
</em>
5353
</td>
5454
<td>
55-
Refer to the Kubernetes API documentation for the fields of the
55+
<table>
56+
<tr><td><code>annotations</code></td><td>
57+
58+
<table>
59+
<tr><td><code>alb.ingress.kubernetes.io/IamRoleArnToAssume</code><br><i>string</i></td>
60+
<td><i>(Optional)</i> In case the target group is in a differet AWS account, you put here the role that needs to be assumed in order to manipulate the target group.
61+
</td></tr>
62+
<tr><td><code>alb.ingress.kubernetes.io/AssumeRoleExternalId</code><br><i>string</i></td>
63+
<td><i>(Optional)</i> The external ID for the assume role operation. Optional, but recommended. It helps you to prevent the <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html" target="_blank">confused deputy problem</a>.
64+
</td></tr>
65+
</table>
66+
67+
<tr><td colspan=2>
68+
Refer to the Kubernetes API documentation for the other fields of the
5669
<code>metadata</code> field.
70+
</td></tr>
71+
</table></td></tr>
72+
73+
74+
5775
</td>
5876
</tr>
77+
5978
<tr>
6079
<td>
6180
<code>spec</code></br>

‎docs/guide/targetgroupbinding/targetgroupbinding.md

+23
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,29 @@ spec:
109109
...
110110
```
111111

112+
### AssumeRole
113+
114+
Sometimes the AWS LoadBalancer controller needs to manipulate target groups from different AWS accounts.
115+
The way to do that is assuming a role from such account. There are annotations that can help you with that:
116+
117+
* `alb.ingress.kubernetes.io/IamRoleArnToAssume`: the ARN that you need to assume
118+
* `alb.ingress.kubernetes.io/AssumeRoleExternalId`: the external ID for the assume role operation. Optional, but recommended. It helps you to prevent the confused deputy problem ( https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html )
119+
120+
121+
## Sample YAML
122+
123+
```yaml
124+
apiVersion: elbv2.k8s.aws/v1beta1
125+
kind: TargetGroupBinding
126+
metadata:
127+
name: my-tgb
128+
annotations:
129+
alb.ingress.kubernetes.io/IamRoleArnToAssume: "arn:aws:iam::999999999999:role/alb-controller-policy-to-assume"
130+
alb.ingress.kubernetes.io/AssumeRoleExternalId: "some-magic-string"
131+
spec:
132+
...
133+
```
134+
112135
## MultiCluster Target Group
113136
TargetGroupBinding CRD supports sharing the same target group ARN among multiple clusters. Setting this flag will ensure the controller only operates on targets within the cluster.
114137

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ require (
6060
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
6161
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
6262
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
63+
github.com/aws/aws-sdk-go-v2/service/iam v1.36.3 // indirect
6364
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
6465
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
6566
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect

‎pkg/aws/cloud.go

+94-37
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,22 @@ package aws
33
import (
44
"context"
55
"fmt"
6+
"log"
7+
"net"
8+
"os"
9+
"strings"
10+
611
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
712
"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
813
"github.com/aws/aws-sdk-go-v2/aws/retry"
914
"github.com/aws/aws-sdk-go-v2/config"
15+
"github.com/aws/aws-sdk-go-v2/credentials"
1016
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
17+
"github.com/aws/aws-sdk-go-v2/service/sts"
18+
1119
smithymiddleware "github.com/aws/smithy-go/middleware"
12-
"net"
13-
"os"
1420
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle"
1521
"sigs.k8s.io/aws-load-balancer-controller/pkg/version"
16-
"strings"
1722

1823
"github.com/aws/aws-sdk-go-v2/aws"
1924
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
@@ -29,37 +34,8 @@ import (
2934

3035
const userAgent = "elbv2.k8s.aws"
3136

32-
type Cloud interface {
33-
// EC2 provides API to AWS EC2
34-
EC2() services.EC2
35-
36-
// ELBV2 provides API to AWS ELBV2
37-
ELBV2() services.ELBV2
38-
39-
// ACM provides API to AWS ACM
40-
ACM() services.ACM
41-
42-
// WAFv2 provides API to AWS WAFv2
43-
WAFv2() services.WAFv2
44-
45-
// WAFRegional provides API to AWS WAFRegional
46-
WAFRegional() services.WAFRegional
47-
48-
// Shield provides API to AWS Shield
49-
Shield() services.Shield
50-
51-
// RGT provides API to AWS RGT
52-
RGT() services.RGT
53-
54-
// Region for the kubernetes cluster
55-
Region() string
56-
57-
// VpcID for the LoadBalancer resources.
58-
VpcID() string
59-
}
60-
6137
// NewCloud constructs new Cloud implementation.
62-
func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (Cloud, error) {
38+
func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (services.Cloud, error) {
6339
hasIPv4 := true
6440
addrs, err := net.InterfaceAddrs()
6541
if err == nil {
@@ -138,17 +114,26 @@ func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger l
138114
if err != nil {
139115
return nil, errors.Wrap(err, "failed to get VPC ID")
140116
}
117+
141118
cfg.VpcID = vpcID
142-
return &defaultCloud{
119+
120+
thisObj := &defaultCloud{
143121
cfg: cfg,
144122
ec2: ec2Service,
145-
elbv2: services.NewELBV2(awsClientsProvider),
146123
acm: services.NewACM(awsClientsProvider),
147124
wafv2: services.NewWAFv2(awsClientsProvider),
148125
wafRegional: services.NewWAFRegional(awsClientsProvider, cfg.Region),
149126
shield: services.NewShield(awsClientsProvider),
150127
rgt: services.NewRGT(awsClientsProvider),
151-
}, nil
128+
129+
assumeRoleElbV2: make(map[string]services.ELBV2),
130+
awsClientsProvider: awsClientsProvider,
131+
logger: logger,
132+
}
133+
134+
thisObj.elbv2 = services.NewELBV2(awsClientsProvider, thisObj)
135+
136+
return thisObj, nil
152137
}
153138

154139
func getVpcID(cfg CloudConfig, ec2Service services.EC2, ec2Metadata services.EC2Metadata, logger logr.Logger) (string, error) {
@@ -222,7 +207,7 @@ func inferVPCIDFromTags(ec2Service services.EC2, VpcNameTagKey string, VpcNameTa
222207
return *vpcs[0].VpcId, nil
223208
}
224209

225-
var _ Cloud = &defaultCloud{}
210+
var _ services.Cloud = &defaultCloud{}
226211

227212
type defaultCloud struct {
228213
cfg CloudConfig
@@ -234,6 +219,78 @@ type defaultCloud struct {
234219
wafRegional services.WAFRegional
235220
shield services.Shield
236221
rgt services.RGT
222+
223+
assumeRoleElbV2 map[string]services.ELBV2
224+
awsClientsProvider provider.AWSClientsProvider
225+
logger logr.Logger
226+
}
227+
228+
// returns ELBV2 client for the given assumeRoleArn, or the default ELBV2 client if assumeRoleArn is empty
229+
func (c *defaultCloud) GetAssumedRoleELBV2(ctx context.Context, assumeRoleArn string, externalId string) services.ELBV2 {
230+
231+
if assumeRoleArn == "" {
232+
return c.elbv2
233+
}
234+
235+
assumedRoleELBV2, exists := c.assumeRoleElbV2[assumeRoleArn]
236+
if exists {
237+
return assumedRoleELBV2
238+
}
239+
c.logger.Info("awsCloud", "method", "GetAssumedRoleELBV2", "AssumeRoleArn", assumeRoleArn, "externalId", externalId)
240+
241+
////////////////
242+
existingAwsConfig, _ := c.awsClientsProvider.GetAWSConfig(ctx, "GetAWSConfigForIAMRoleImpersonation")
243+
244+
sourceAccount := sts.NewFromConfig(*existingAwsConfig)
245+
response, err := sourceAccount.AssumeRole(ctx, &sts.AssumeRoleInput{
246+
RoleArn: aws.String(assumeRoleArn),
247+
RoleSessionName: aws.String("aws-load-balancer-controller"),
248+
ExternalId: aws.String(externalId),
249+
})
250+
if err != nil {
251+
log.Fatalf("Unable to assume target role, %v. Attempting to use default client", err)
252+
return c.elbv2
253+
}
254+
assumedRoleCreds := response.Credentials
255+
newCreds := credentials.NewStaticCredentialsProvider(*assumedRoleCreds.AccessKeyId, *assumedRoleCreds.SecretAccessKey, *assumedRoleCreds.SessionToken)
256+
newAwsConfig, err := config.LoadDefaultConfig(ctx, config.WithRegion(c.cfg.Region), config.WithCredentialsProvider(newCreds))
257+
if err != nil {
258+
log.Fatalf("Unable to load static credentials for service client config, %v. Attempting to use default client", err)
259+
return c.elbv2
260+
}
261+
262+
existingAwsConfig.Credentials = newAwsConfig.Credentials // response.Credentials
263+
264+
// // var assumedRoleCreds *stsTypes.Credentials = response.Credentials
265+
266+
// // Create config with target service client, using assumed role
267+
// cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(*assumedRoleCreds.AccessKeyId, *assumedRoleCreds.SecretAccessKey, *assumedRoleCreds.SessionToken)))
268+
// if err != nil {
269+
// log.Fatalf("unable to load static credentials for service client config, %v", err)
270+
// }
271+
272+
// ////////////////
273+
// appCreds := stscreds.NewAssumeRoleProvider(client, assumeRoleArn)
274+
// value, err := appCreds.Retrieve(context.TODO())
275+
// if err != nil {
276+
// // handle error
277+
// }
278+
// /////////
279+
280+
// ///////////// OLD
281+
// creds := stscreds.NewCredentials(c.session, assumeRoleArn, func(p *stscreds.AssumeRoleProvider) {
282+
// p.ExternalID = &externalId
283+
// })
284+
// //////////////
285+
286+
// c.awsConfig.Credentials = creds
287+
// // newObj := services.NewELBV2(c.session, c, c.awsCFG)
288+
// newObj := services.NewELBV2(*c.awsConfig, c.endpointsResolver, c)
289+
290+
newObj := services.NewELBV2(c.awsClientsProvider, c)
291+
c.assumeRoleElbV2[assumeRoleArn] = newObj
292+
293+
return newObj
237294
}
238295

239296
func (c *defaultCloud) EC2() services.EC2 {

0 commit comments

Comments
 (0)