Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit cd20ff1

Browse files
cjschaefcjschaef
committedMar 14, 2024·
VPC: Add infrastructure creation proposal
Add a new proposal with details and images for creating and deleting VPC resources for the IBMVPCCluster definition.
1 parent 1cc2bc3 commit cd20ff1

4 files changed

+402
-0
lines changed
 

‎docs/images/vpc-cluster-components.png

79 KB
Loading

‎docs/images/vpc-cluster-create-workflow.png

58.2 KB
Loading

‎docs/images/vpc-cluster-delete-workflow.png

48.7 KB
Loading

‎docs/proposal/20240308-vpc-infra-creation.md

+402
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,402 @@
1+
# Dynamically create infrastructure required for VPC cluster
2+
3+
## Motivation
4+
Currently, cluster api will provision a series of VPC resources as
5+
infrastructure for a cluster. However, the current process results in
6+
restrictions in terms of availability, security, and other important
7+
options (lack of Security Group support, single zone management, etc.).
8+
The alternative is using existing VPC resources, which require manual
9+
creation and configuration, although this support can still be restricted
10+
by the current implementations (single zone or subnet).
11+
12+
Instead, support to grant cluster api the capabilities to deploy a wider
13+
variety of VPC resources, with additional configurations would significantly
14+
reduce the need to create resources as prerequisites, or be limited by the
15+
current available configurations.
16+
17+
18+
## Goal
19+
1. Dynamically create the required cloud resources as a part of cluster
20+
creation, using a broader set of VPC resources and configurations.
21+
22+
23+
## Proposal
24+
25+
26+
### Cluster API VPC cluster components
27+
28+
![vpc-cluster-components.png](../images/vpc-cluster-components.png)
29+
30+
31+
### Proposed API changes
32+
33+
34+
#### VPCCluster improvements
35+
```go
36+
// IBMVPCClusterSpec defines the desired state of IBMVPCCluster.
37+
type IBMVPCClusterSpec struct {
38+
// Important: Run "make" to regenerate code after modifying this file
39+
40+
// controlPlaneEndpoint represents the endpoint used to communicate with the control plane.
41+
// +optional
42+
ControlPlaneEndpoint capiv1beta1.APIEndpoint `json:"controlPlaneEndpoint"`
43+
44+
// ControlPlaneLoadBalancer is optional configuration for customizing control plane behavior.
45+
// +optional
46+
// Deprecated: use LoadBalancers instead.
47+
ControlPlaneLoadBalancer *VPCLoadBalancerSpec `json:"controlPlaneLoadBalancer,omitempty"`
48+
49+
// cosInstance is the IBM COS instance to use for cluster resources.
50+
COSInstance *CosInstance `json:"cosInstance,omitempty"`
51+
52+
// loadBalancers is a set of IBM Cloud VPC Load Balancers for managing traffic to the cluster.
53+
LoadBalancers []VPCLoadBalancerSpec `json:"loadBalancers,omitempty"`
54+
55+
// networkSpec represents the VPC network to use for the cluster.
56+
NetworkSpec *VPCNetworkSpec `json:"networkSpec,omitempty"`
57+
58+
// region defines the IBM Cloud Region the cluster resources will be deployed in.
59+
Region string `json:"region"`
60+
61+
// resourceGroup defines the IBM Cloud resource group where the cluster resources should be created.
62+
ResourceGroup string `json:"resourceGroup"`
63+
64+
// The Name of VPC.
65+
// Deprecated: use NetworkSpec instead.
66+
VPC string `json:"vpc,omitempty"`
67+
68+
// The Name of availability zone.
69+
// Deprecated: use NetworkSpec instead.
70+
Zone string `json:"zone,omitempty"`
71+
}
72+
73+
// VPCNetworkSpec defines the desired state of the network resources for the cluster.
74+
type VPCNetworkSpec struct {
75+
// computeSubnetsSpec is a set of Subnet's which define the Compute subnets.
76+
ComputeSubnetsSpec []Subnet `json:"computeSubnetsSpec,omitempty"`
77+
78+
// controlPlaneSubnetsSpec is a set of Subnet's which define the Control Plane subnets.
79+
ControlPlaneSubnetsSpec []Subnet `json:"controlPlaneSubentsSpec,omitempty"`
80+
81+
// resourceGroup is the name of the Resource Group containing all of the newtork resources.
82+
// This can be different than the Resource Group containing the remaining cluster resources.
83+
ResourceGroup *string `json:"resourceGroup,omitempty"`
84+
85+
// securityGroups is a set of SecurityGroup's which define the VPC Security Groups that manage traffic within and out of the VPC.
86+
SecurityGroups []SecurityGroup `json:"securityGroups,omitempty"`
87+
88+
// vpc defines the IBM Cloud VPC.
89+
VPC *VPCResourceReference `json:"vpc,omitempty"`
90+
}
91+
92+
// IBMVPCClusterStatus defines the observed state of IBMVPCCluster.
93+
type IBMVPCClusterStatus struct {
94+
// Important: Run "make" to regenerate code after modifying this file
95+
96+
// Conditions defines current service state of the load balancer.
97+
// +optional
98+
Conditions capiv1beta1.Conditions `json:"conditions,omitempty"`
99+
100+
// ControlPlaneLoadBalancerState is the status of the load balancer.
101+
// Deprecated: rely on NetworkStatus instead.
102+
// +optional
103+
ControlPlaneLoadBalancerState *VPCLoadBalancerState `json:"controlPlaneLoadBalancerState,omitempty"`
104+
105+
// COSInstance is the reference to the IBM Cloud COS Instance used for the cluster.
106+
COSInstance *VPCResourceReference `json:"cosInstance,omitempty"`
107+
108+
// networkStatus is the status of the VPC network in its entirety resources.
109+
NetworkStatus *VPCNetworkStatus `json:"networkStatus,omitempty"`
110+
111+
// ready is true when the provider resource is ready.
112+
// +kubebuilder:default=false
113+
Ready bool `json:"ready"`
114+
115+
// resourceGroup is the reference to the IBM Cloud VPC resource group under which the resources will be created.
116+
ResourceGroup *ResourceReference `json:"resourceGroupID,omitempty"`
117+
118+
// Deprecated: rely on NetworkStatus instead.
119+
Subnet Subnet `json:"subnet,omitempty"`
120+
121+
// Deprecated: rely on NetworkStatus instead.
122+
VPC VPC `json:"vpc,omitempty"`
123+
124+
// Deprecated: rely on ControlPlaneEndpoint
125+
VPCEndpoint VPCEndpoint `json:"vpcEndpoint,omitempty"`
126+
}
127+
128+
// VPCNetworkStatus provides details on the status of VPC network resources.
129+
type VPCNetworkStatus struct {
130+
// computeSubnets references the VPC Subnets for the cluster's Data Plane.
131+
// +optional
132+
ComputeSubnets []*VPCResourceReference `json:"computeSubnets,omitempty"`
133+
134+
// controlPlaneSubnets references the VPC Subnets for the cluster's Control Plane.
135+
// +optional
136+
ControlPlaneSubnets []*VPCResourceReference `json:"controlPlaneSubnets,omitempty"`
137+
138+
// loadBalancers references the VPC Load Balancer's for the cluster.
139+
// +optional
140+
LoadBalancers []VPCLoadBalancerStatus `json:"loadBalancers,omitempty"`
141+
142+
// publicGateways references the VPC Public Gateways for the cluster.
143+
// +optional
144+
PublicGateways []*VPCResourceReference `json:"publicGateways,omitempty"`
145+
146+
// securityGroups references the VPC Security Groups for the cluster.
147+
// +optional
148+
SecurityGroups []*VPCResourceReference `json:"securityGroups,omitempty"`
149+
150+
// vpc references the IBM Cloud VPC.
151+
// +optional
152+
VPC *VPCResourceReference `json:"vpc,omitempty"`
153+
}
154+
155+
// VPCResourceReference identifies a resource by id and type.
156+
type VPCResourceReference struct {
157+
// controllerCreated indicates whether the resource is created by the CAPI controller.
158+
ControllerCreated bool `json:"controllerCreated,omitempty"`
159+
160+
// crn defines the IBM Cloud CRN of the resource.
161+
CRN string `json"crn,omitempty"`
162+
163+
// type defines the type of IBM Cloud resource.
164+
Type VPCResourceType `json:type"`
165+
}
166+
167+
type VPCResourceType string
168+
169+
const (
170+
VPCResourceTypeCOS VPCResourceType = VPCResourceType("cos")
171+
VPCResourceTypePublicGateway VPCResourceType = VPCResourceType("publicgateway")
172+
VPCResourceTypeSecurityGroup VPCResourceType = VPCResourceType("securitygroup")
173+
VPCResourceTypeSecurityGroupRule VPCResourceType = VPCResourceType("securitygrouprule")
174+
VPCResourceTypeSubnet VPCResourceType = VPCResourceType("subnet")
175+
VPCResourceTypeVPC VPCResourceType = VPCResourceType("vpc")
176+
)
177+
```
178+
179+
180+
#### Share existing common resource definitions
181+
Rather than duplicate code and also make references to existing resources
182+
complex, we also propose relocating existing resource references into a
183+
common file. This prevents the duplication of code in PowerVS and VPC
184+
definitions, but also removes the direct dependence on PowerVS definitions,
185+
placing them into a common location (preventing unexpected dependencies).
186+
187+
Relocate the following resource definitions in IBMPowerVSCluster
188+
```go
189+
// VPCResourceReference is a reference to a specific VPC resource by ID or Name
190+
// Only one of ID or Name may be specified. Specifying more than one will result in
191+
// a validation error.
192+
type VPCResourceReference struct {
193+
// ID of resource
194+
// +kubebuilder:validation:MinLength=1
195+
// +optional
196+
ID *string `json:"id,omitempty"`
197+
198+
// Name of resource
199+
// +kubebuilder:validation:MinLength=1
200+
// +optional
201+
Name *string `json:"name,omitempty"`
202+
203+
// IBM Cloud VPC region
204+
Region *string `json:"region,omitempty"`
205+
}
206+
207+
// CosInstance represents IBM Cloud COS instance.
208+
type CosInstance struct {
209+
// PresignedURLDuration defines the duration for which presigned URLs are valid.
210+
//
211+
// This is used to generate presigned URLs for S3 Bucket objects, which are used by
212+
// control-plane and worker nodes to fetch bootstrap data.
213+
//
214+
// When enabled, the IAM instance profiles specified are not used.
215+
// +optional
216+
PresignedURLDuration *metav1.Duration `json:"presignedURLDuration,omitempty"`
217+
218+
// Name defines name of IBM cloud COS instance to be created.
219+
// +kubebuilder:validation:MinLength:=3
220+
// +kubebuilder:validation:MaxLength:=63
221+
// +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`
222+
Name string `json:"name,omitempty"`
223+
224+
// bucketName is IBM cloud COS bucket name
225+
BucketName string `json:"bucketName,omitempty"`
226+
227+
// bucketRegion is IBM cloud COS bucket region
228+
BucketRegion string `json:"bucketRegion,omitempty"`
229+
}
230+
```
231+
232+
To a shared location, such as [types.go](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/api/v1beta2/types.go).
233+
234+
We also are adding additional resources and definitions that could be shared,
235+
such as the following:
236+
```go
237+
// PortRange represents a range of ports, minimum to maximum.
238+
type PortRange struct {
239+
// maximumPort is the inclusive upper range of ports.
240+
MaximumPort int `json:"maximumPort,omitempty"`
241+
242+
// minimumPort is the inclusive lower range of ports.
243+
MinimumPort int `json:"minimumPort,omitempty"`
244+
}
245+
246+
// SecurityGroup defines a VPC Security Group that should exist or be created within the specified VPC, with the specified Security Group Rules.
247+
type SecurityGroup struct {
248+
// id of the Security Group.
249+
ID string
250+
251+
// name of the Security Group.
252+
Name string
253+
254+
// resourceGroup of the Security Group.
255+
ResourceGroup string
256+
257+
// rules are the Security Group Rules for the Security Group.
258+
Rules []*SecurityGroupRule
259+
260+
// tags are tags to add to the Security Group.
261+
Tags []string
262+
263+
// vpc is the IBM Cloud VPC for the Security Group.
264+
VPC *VPCResourceReference
265+
}
266+
267+
// SecurityGroupRule defines a VPC Security Group Rule for a specified Security Group.
268+
type SecurityGroupRule struct {
269+
// action defines whether to allow or deny traffic defined by the Security Group Rule.
270+
Action SecurityGroupRuleAction
271+
272+
// destination is a SecurityGroupRulePrototype which defines the destination of outbound traffic for the Security Group Rule.
273+
// Only used when Direction is SecurityGroupRuleDirectionOutbound.
274+
// +optional
275+
Destination *SecurityGroupRulePrototype
276+
277+
// direction defines whether the traffic is inbound or outbound for the Security Group Rule.
278+
Direction SecurityGroupRuleDirection
279+
280+
// securityGroupID is the ID of the Security Group for the Security Group Rule.
281+
SecurityGroupID *string
282+
283+
// source is a SecurityGroupRulePrototype which defines the source of inbound traffic for the Security Group Rule.
284+
// Only used when Direction is SecurityGroupRuleDirectionInbound.
285+
Source *SecurityGroupRulePrototype
286+
}
287+
288+
// SecurityGroupRuleRemote defines a VPC Security Group Rule's remote details.
289+
// The type of remote defines the additional remote details where are used for defining the remote.
290+
type SecurityGroupRuleRemote struct {
291+
// cidrSubnetName is the name of the VPC Subnet to retrieve the CIDR from, to use for the remote's destination/source.
292+
// Only used when RemoteType is SecurityGroupRuleRemoteTypeCIDR.
293+
// +optional
294+
CIDRSubnetName *string `json:"cidrSubnetName,omitempty"`
295+
296+
// ip is the IP to use for the remote's destination/source.
297+
// Only used when RemoteType is SecurityGroupRuleRemoteTypeIP.
298+
// +optional
299+
IP *string `json:ip,omitempty"`
300+
301+
// remoteType defines the type of filter to define for the remote's destination/source.
302+
// +required
303+
RemoteType SecurityGroupRuleRemoteType `json:"remoteType"`
304+
305+
// securityGroupName is the name of the VPC Security Group to use for the remote's destination/source.
306+
// Only used when RemoteType is SecurityGroupRuleRemoteTypeSG
307+
// +optional
308+
SecurityGroupName *string `json:"securityGroupName,omitempty"`
309+
}
310+
311+
// SecurityGroupRulePrototype defines a VPC Security Group Rule's traffic specifics for a series of remotes (destinations or sources).
312+
type SecurityGroupRulePrototype struct {
313+
// icmpCode is the ICMP code for the Rule.
314+
// Only used when Protocol is SecurityGroupProtocolICMP.
315+
// +optional
316+
ICMPCode *string `json:"icmpCode,omitempty"`
317+
318+
// icmpType is the ICMP type for the Rule.
319+
// Only used when Protocol is SecurityGroupProtocolICMP.
320+
// +optional
321+
ICMPType *string `json:"icmpType,omitempty"`
322+
323+
// portRange is a PortRange allowed for the Rule's remote.
324+
// +optional
325+
PortRange *PortRange `json:"portRange,omitempty"`
326+
327+
// protocol defines the traffic protocol used for the Security Group Rule.
328+
// +required
329+
Protocol SecurityGroupRuleProtocol `json:"protocol"`
330+
331+
// remotes is a set of SecurityGroupRuleRemote's that define the traffic allowed by the Rule's remote.
332+
// Specifying multiple SecurityGroupRuleRemote's creates a unique Security Group Rule with the shared Protocol, PortRange, etc.
333+
// This allows for easier management of Security Group Rule's for sets of CIDR's, IP's, etc.
334+
Remotes []SecurityGroupRuleRemote `json:"remotes"`
335+
}
336+
```
337+
338+
#### Shared enumerations
339+
Since IBM Cloud API's do not define enumerations for resource options, we have
340+
added some enumerations to enforce values and for validation.
341+
```go
342+
type SecurityGroupRuleAction string
343+
344+
const (
345+
SecurityGroupRuleActionAllow SecurityGroupRuleAction = vpcv1.NetworkACLRuleActionAllowConst
346+
SecurityGroupRuleActionDeny SecurityGroupRuleAction = vpcv1.NetworkACLRuleActionDenyConst
347+
)
348+
349+
type SecurityGroupRuleDirection string
350+
351+
const (
352+
SecurityGroupRuleDirectionInbound SecurityGroupRuleDirection = vpcv1.NetworkACLRuleDirectionInboundConst
353+
SecurityGroupRuleDirectionOutbound SecurityGroupRuleDirection = vpcv1.NetworkACLRuleDirectionOutboundConst
354+
)
355+
356+
type SecurityGroupRuleProtocol string
357+
358+
const (
359+
SecurityGroupRuleProtocolAll SecurityGroupRuleProtocal = vpcv1.NetworkACLRuleProtocolAllConst
360+
SecurityGroupRuleProtocolICMP SecurityGroupRuleProtocal = vpcv1.NetworkACLRuleProtocolIcmpConst
361+
SecurityGroupRuleProtocolTCP SecurityGroupRuleProtocal = vpcv1.NetworkACLRuleProtocolTCPConst
362+
SecurityGroupRuleProtocolUDP SecurityGroupRuleProtocal = vpcv1.NetworkACLRuleProtocolUDPConst
363+
)
364+
365+
type SecurityGroupRuleRemoteType string
366+
367+
const (
368+
SecurityGroupRuleRemoteTypeAny SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("any")
369+
SecurityGroupRuleRemoteTypeCIDR SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("cidr")
370+
SecurityGroupRuleRemoteTypeIP SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("ip")
371+
SecurityGroupRuleRemoteTypeSG SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("sg")
372+
)
373+
```
374+
375+
376+
### Following resources can be created
377+
1. [VPC](https://cloud.ibm.com/docs/vpc?topic=vpc-about-vpc)
378+
2. [VPC Subnets](https://cloud.ibm.com/docs/vpc?topic=vpc-about-networking-for-vpc)
379+
3. [VPC Public Gatways](https://cloud.ibm.com/docs/vpc?topic=vpc-about-networking-for-vpc#external-connectivity)
380+
4. [VPC Security Groups](https://cloud.ibm.com/docs/vpc?topic=vpc-using-security-groups) & [VPC Security Group Rules](https://cloud.ibm.com/docs/vpc?topic=vpc-using-security-groups#defining-security-group-rules)
381+
5. [VPC Load Balancers](https://cloud.ibm.com/docs/vpc?topic=vpc-nlb-vs-elb)
382+
383+
384+
### Cluster creation workflow
385+
The user is expected to set the annotation
386+
`vpc.cluster.x-k8s.io/create-infra:true` on the IBMVPCCluster object to
387+
make use of this feature. If not set, the cluster creation will proceed using
388+
the existing process.
389+
390+
THe user can specify existing resources in Spec, and the specified controller
391+
will attempt to reuse those resources instead of creating new ones.
392+
393+
When the resource is not set in the Spec or the provided resource name does
394+
not exist in cloud, the controller will create a new resource.
395+
396+
![vpc-cluster-create-workflow.png](../images/vpc-cluster-create-workflow.png)
397+
398+
399+
### Cluster Deletion workflow
400+
The controller will only delete the resources which are created by it.
401+
402+
![vpc-cluster-delete-workflow.png](../images/vpc-cluster-delete-workflow.png)

0 commit comments

Comments
 (0)
Please sign in to comment.