From e9693c6384bee53fe773c9e9f2038945fa7bae1d Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Tue, 4 Feb 2025 10:19:32 +0200 Subject: [PATCH 1/9] add formation flow for clb lb rules --- pkg/provider/azure_loadbalancer.go | 30 ++++++++++++++++++++++++++++++ pkg/provider/config/azure.go | 4 ++++ 2 files changed, 34 insertions(+) diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index 4891292434..b95fa97a65 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -2800,6 +2800,36 @@ func (az *Cloud) getExpectedLBRules( isIPv6 bool, ) ([]*armnetwork.Probe, []*armnetwork.LoadBalancingRule, error) { var expectedRules []*armnetwork.LoadBalancingRule + // If we are using Pod IP in the LB backend, we skip health probes, disable floating IP and use port.TargetPort. + if az.IsLBBackendPoolTypePodIP() { + // Check for multi-IP families in the service spec (not allowed for CLB). + if len(service.Spec.IPFamilies) > 1 { + return nil, nil, fmt.Errorf("dual-stack services are not supported when LB backend pool type is PodIP") + } + // Build rules for each service port but with floatingIP = false, no health probes. + for _, port := range service.Spec.Ports { + ruleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port, isIPv6) + transportProto, _, _, err := getProtocolsFromKubernetesProtocol(port.Protocol) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse transport protocol: %w", err) + } + props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, port, transportProto) + if err != nil { + return nil, nil, fmt.Errorf("error generating LB rule: %w", err) + } + // Turn off floating IP and skip health probe attachments. + props.EnableFloatingIP = ptr.To(false) + props.Probe = nil + props.BackendPort = ptr.To(int32(port.TargetPort.IntValue())) + expectedRules = append(expectedRules, &armnetwork.LoadBalancingRule{ + Name: &ruleName, + Properties: props, + }) + } + return nil, expectedRules, nil + } + + // Otherwise, the existing flow remains unchanged var expectedProbes []*armnetwork.Probe // support podPresence health check when External Traffic Policy is local diff --git a/pkg/provider/config/azure.go b/pkg/provider/config/azure.go index 8cdf73f0b6..d6ba5ba2f3 100644 --- a/pkg/provider/config/azure.go +++ b/pkg/provider/config/azure.go @@ -183,6 +183,10 @@ func (az *Config) IsLBBackendPoolTypeNodeIP() bool { return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypeNodeIP) } +func (az *Config) IsLBBackendPoolTypePodIP() bool { + return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePodIP) +} + func (az *Config) GetPutVMSSVMBatchSize() int { return az.PutVMSSVMBatchSize } From 2a45c63f7111ffbc426966bd33a4d8d2b38be031 Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Tue, 4 Feb 2025 17:04:40 +0200 Subject: [PATCH 2/9] comment typo --- pkg/provider/azure_loadbalancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index b95fa97a65..3bc5087f66 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -2802,7 +2802,7 @@ func (az *Cloud) getExpectedLBRules( var expectedRules []*armnetwork.LoadBalancingRule // If we are using Pod IP in the LB backend, we skip health probes, disable floating IP and use port.TargetPort. if az.IsLBBackendPoolTypePodIP() { - // Check for multi-IP families in the service spec (not allowed for CLB). + // Check for multi-IP families in the service spec (not allowed for BackendPool of type PodIP). if len(service.Spec.IPFamilies) > 1 { return nil, nil, fmt.Errorf("dual-stack services are not supported when LB backend pool type is PodIP") } From e1eb35d5d4cb6c2302dc91af18dd8dbb1f0b6e74 Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Fri, 7 Feb 2025 16:39:31 +0200 Subject: [PATCH 3/9] named targetPort is not supported within CLB context --- pkg/provider/azure_loadbalancer.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index 3bc5087f66..1e8aff0c53 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -37,6 +37,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/intstr" cloudprovider "k8s.io/cloud-provider" servicehelpers "k8s.io/cloud-provider/service/helpers" "k8s.io/klog/v2" @@ -2808,6 +2809,9 @@ func (az *Cloud) getExpectedLBRules( } // Build rules for each service port but with floatingIP = false, no health probes. for _, port := range service.Spec.Ports { + if port.TargetPort.Type == intstr.String { + return nil, nil, fmt.Errorf("named targetPort is not supported when LB backend pool type is PodIP: %s", port.TargetPort.StrVal) + } ruleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port, isIPv6) transportProto, _, _, err := getProtocolsFromKubernetesProtocol(port.Protocol) if err != nil { @@ -2820,7 +2824,7 @@ func (az *Cloud) getExpectedLBRules( // Turn off floating IP and skip health probe attachments. props.EnableFloatingIP = ptr.To(false) props.Probe = nil - props.BackendPort = ptr.To(int32(port.TargetPort.IntValue())) + props.BackendPort = &port.TargetPort.IntVal expectedRules = append(expectedRules, &armnetwork.LoadBalancingRule{ Name: &ruleName, Properties: props, From 29bb3c820a1227907e92cfbcc1110e1dfd4e4a65 Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Mon, 10 Feb 2025 12:16:11 +0200 Subject: [PATCH 4/9] add unit tests --- pkg/consts/consts.go | 3 +- pkg/provider/azure.go | 4 +- pkg/provider/azure_loadbalancer_test.go | 45 ++++++++++++++++++ pkg/provider/azure_test.go | 62 +++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 6f75dc435e..9a3efb7ab4 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -382,8 +382,7 @@ const ( // LoadBalancerBackendPoolConfigurationTypeNodeIP is the lb backend pool config type node ip LoadBalancerBackendPoolConfigurationTypeNodeIP = "nodeIP" // LoadBalancerBackendPoolConfigurationTypePODIP is the lb backend pool config type pod ip - // TODO (nilo19): support pod IP in the future - LoadBalancerBackendPoolConfigurationTypePODIP = "podIP" + LoadBalancerBackendPoolConfigurationTypePodIP = "podIP" ) // error messages diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go index 54a30874f9..c211ef6909 100644 --- a/pkg/provider/azure.go +++ b/pkg/provider/azure.go @@ -294,13 +294,13 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *config.C if config.LoadBalancerBackendPoolConfigurationType == "" || // TODO(nilo19): support pod IP mode in the future - strings.EqualFold(config.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePODIP) { + strings.EqualFold(config.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePodIP) { config.LoadBalancerBackendPoolConfigurationType = consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration } else { supportedLoadBalancerBackendPoolConfigurationTypes := utilsets.NewString( strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration), strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypeNodeIP), - strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypePODIP)) + strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypePodIP)) if !supportedLoadBalancerBackendPoolConfigurationTypes.Has(strings.ToLower(config.LoadBalancerBackendPoolConfigurationType)) { return fmt.Errorf("loadBalancerBackendPoolConfigurationType %s is not supported, supported values are %v", config.LoadBalancerBackendPoolConfigurationType, supportedLoadBalancerBackendPoolConfigurationTypes.UnsortedList()) } diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go index 4cb722cc11..b9bdfff029 100644 --- a/pkg/provider/azure_loadbalancer_test.go +++ b/pkg/provider/azure_loadbalancer_test.go @@ -3027,9 +3027,43 @@ func TestReconcileLoadBalancerRuleCommon(t *testing.T) { expectedProbes: probes, expectedRules: rules1DualStack, }) + + testCases = append(testCases, []struct { + desc string + service v1.Service + loadBalancerSKU string + probeProtocol string + probePath string + expectedProbes map[bool][]*armnetwork.Probe + expectedRules map[bool][]*armnetwork.LoadBalancingRule + expectedErr bool + }{ + { + desc: "LB backend pool of type PodIP - getExpectedLBRules should return error when the service is dual-stack", + service: getTestServiceDualStack("test", v1.ProtocolTCP, nil, 80), + loadBalancerSKU: "standardV2", + expectedErr: true, + }, + { + desc: "LB backend pool of type PodIP - getExpectedLBRules should return error when the service port is named", + service: getTestServiceWithNamedTargetPorts("test", v1.ProtocolTCP, nil, false, 8080, "http-web-svc"), + loadBalancerSKU: "standardV2", + expectedErr: true, + }, + { + desc: "LB backend pool of type PodIP - getExpectedLBRules should return expected rules and probes", + service: getTestServiceWithIntTargetPorts("test1", v1.ProtocolTCP, nil, false, 8080, 1234), + loadBalancerSKU: "standardV2", + expectedRules: getTestRuleCLB(false, 8080, 1234, false), + expectedProbes: nil, + }, + }...) for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { az := GetTestCloud(ctrl) + if test.loadBalancerSKU == "standardV2" { + az.LoadBalancerBackendPoolConfigurationType = consts.LoadBalancerBackendPoolConfigurationTypePodIP + } az.Config.LoadBalancerSKU = test.loadBalancerSKU service := test.service firstPort := service.Spec.Ports[0] @@ -3130,6 +3164,17 @@ func getTCPResetTestRules(enableTCPReset bool) map[bool][]*armnetwork.LoadBalanc } } +func getTestRuleCLB(enableTCPReset bool, servicePort int32, targetPort int32, isIPv6 bool) map[bool][]*armnetwork.LoadBalancingRule { + rule := getTestRule(enableTCPReset, servicePort, isIPv6) + rule.Properties.EnableFloatingIP = to.Ptr(false) + rule.Properties.Probe = nil + rule.Properties.BackendPort = &targetPort + + return map[bool][]*armnetwork.LoadBalancingRule{ + false: {rule}, + } +} + // getTestRule returns a rule for dualStack. func getTestRule(enableTCPReset bool, port int32, isIPv6 bool) *armnetwork.LoadBalancingRule { suffix := "" diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go index ec6926cd00..c7cf3f3aae 100644 --- a/pkg/provider/azure_test.go +++ b/pkg/provider/azure_test.go @@ -36,6 +36,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" cloudprovider "k8s.io/cloud-provider" @@ -1259,6 +1260,67 @@ func getTestServiceCommon(identifier string, proto v1.Protocol, annotations map[ return svc } +func getTestServiceWithNamedTargetPorts(identifier string, proto v1.Protocol, annotations map[string]string, isIPv6 bool, servicePort int32, namedTargetPorts ...string) v1.Service { + targetPorts := []intstr.IntOrString{} + for _, port := range namedTargetPorts { + targetPorts = append(targetPorts, intstr.FromString(port)) + } + svc := getTestServiceWithTargetPortsCommon(identifier, proto, annotations, servicePort, targetPorts...) + svc.Spec.ClusterIP = "10.0.0.2" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv4Protocol} + if isIPv6 { + svc.Spec.ClusterIP = "fd00::1907" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol} + } + + return svc +} + +func getTestServiceWithIntTargetPorts(identifier string, proto v1.Protocol, annotations map[string]string, isIPv6 bool, servicePort int32, intTargetPorts ...int32) v1.Service { + targetPorts := []intstr.IntOrString{} + for _, port := range intTargetPorts { + targetPorts = append(targetPorts, intstr.FromInt(int(port))) + } + svc := getTestServiceWithTargetPortsCommon(identifier, proto, annotations, servicePort, targetPorts...) + svc.Spec.ClusterIP = "10.0.0.2" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv4Protocol} + if isIPv6 { + svc.Spec.ClusterIP = "fd00::1907" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol} + } + + return svc +} + +func getTestServiceWithTargetPortsCommon(identifier string, proto v1.Protocol, annotations map[string]string, servicePort int32, targetPorts ...intstr.IntOrString) v1.Service { + ports := []v1.ServicePort{} + for _, port := range targetPorts { + ports = append(ports, v1.ServicePort{ + Name: "target-port", + Protocol: proto, + TargetPort: port, + Port: servicePort, + }) + } + + svc := v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + Ports: ports, + }, + } + svc.Name = identifier + svc.Namespace = "default" + svc.UID = types.UID(identifier) + if annotations == nil { + svc.Annotations = make(map[string]string) + } else { + svc.Annotations = annotations + } + + return svc +} + func getInternalTestService(identifier string, requestedPorts ...int32) v1.Service { return getTestServiceWithAnnotation(identifier, map[string]string{consts.ServiceAnnotationLoadBalancerInternal: consts.TrueAnnotationValue}, false, requestedPorts...) } From f52242e32c048c25c674a7089c422e1d164d21ab Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Tue, 4 Feb 2025 10:19:32 +0200 Subject: [PATCH 5/9] add formation flow for clb lb rules --- pkg/provider/azure_loadbalancer.go | 30 ++++++++++++++++++++++++++++++ pkg/provider/config/azure.go | 4 ++++ 2 files changed, 34 insertions(+) diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index 7af2c1291a..8dd47caf0c 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -2815,6 +2815,36 @@ func (az *Cloud) getExpectedLBRules( isIPv6 bool, ) ([]*armnetwork.Probe, []*armnetwork.LoadBalancingRule, error) { var expectedRules []*armnetwork.LoadBalancingRule + // If we are using Pod IP in the LB backend, we skip health probes, disable floating IP and use port.TargetPort. + if az.IsLBBackendPoolTypePodIP() { + // Check for multi-IP families in the service spec (not allowed for CLB). + if len(service.Spec.IPFamilies) > 1 { + return nil, nil, fmt.Errorf("dual-stack services are not supported when LB backend pool type is PodIP") + } + // Build rules for each service port but with floatingIP = false, no health probes. + for _, port := range service.Spec.Ports { + ruleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port, isIPv6) + transportProto, _, _, err := getProtocolsFromKubernetesProtocol(port.Protocol) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse transport protocol: %w", err) + } + props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, port, transportProto) + if err != nil { + return nil, nil, fmt.Errorf("error generating LB rule: %w", err) + } + // Turn off floating IP and skip health probe attachments. + props.EnableFloatingIP = ptr.To(false) + props.Probe = nil + props.BackendPort = ptr.To(int32(port.TargetPort.IntValue())) + expectedRules = append(expectedRules, &armnetwork.LoadBalancingRule{ + Name: &ruleName, + Properties: props, + }) + } + return nil, expectedRules, nil + } + + // Otherwise, the existing flow remains unchanged var expectedProbes []*armnetwork.Probe // support podPresence health check when External Traffic Policy is local diff --git a/pkg/provider/config/azure.go b/pkg/provider/config/azure.go index 9d8801b9c3..d8dea6dee3 100644 --- a/pkg/provider/config/azure.go +++ b/pkg/provider/config/azure.go @@ -180,6 +180,10 @@ func (az *Config) IsLBBackendPoolTypeNodeIP() bool { return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypeNodeIP) } +func (az *Config) IsLBBackendPoolTypePodIP() bool { + return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePodIP) +} + func (az *Config) GetPutVMSSVMBatchSize() int { return az.PutVMSSVMBatchSize } From f51f58c41c247935cb3166dcbef42e71b4580278 Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Tue, 4 Feb 2025 17:04:40 +0200 Subject: [PATCH 6/9] comment typo --- pkg/provider/azure_loadbalancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index 8dd47caf0c..f12b27bb50 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -2817,7 +2817,7 @@ func (az *Cloud) getExpectedLBRules( var expectedRules []*armnetwork.LoadBalancingRule // If we are using Pod IP in the LB backend, we skip health probes, disable floating IP and use port.TargetPort. if az.IsLBBackendPoolTypePodIP() { - // Check for multi-IP families in the service spec (not allowed for CLB). + // Check for multi-IP families in the service spec (not allowed for BackendPool of type PodIP). if len(service.Spec.IPFamilies) > 1 { return nil, nil, fmt.Errorf("dual-stack services are not supported when LB backend pool type is PodIP") } From 1027221ca082d31404a2e72a93571520623b71dd Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Fri, 7 Feb 2025 16:39:31 +0200 Subject: [PATCH 7/9] named targetPort is not supported within CLB context --- pkg/provider/azure_loadbalancer.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index f12b27bb50..a755f56729 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -37,6 +37,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/intstr" cloudprovider "k8s.io/cloud-provider" servicehelpers "k8s.io/cloud-provider/service/helpers" "k8s.io/klog/v2" @@ -2823,6 +2824,9 @@ func (az *Cloud) getExpectedLBRules( } // Build rules for each service port but with floatingIP = false, no health probes. for _, port := range service.Spec.Ports { + if port.TargetPort.Type == intstr.String { + return nil, nil, fmt.Errorf("named targetPort is not supported when LB backend pool type is PodIP: %s", port.TargetPort.StrVal) + } ruleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port, isIPv6) transportProto, _, _, err := getProtocolsFromKubernetesProtocol(port.Protocol) if err != nil { @@ -2835,7 +2839,7 @@ func (az *Cloud) getExpectedLBRules( // Turn off floating IP and skip health probe attachments. props.EnableFloatingIP = ptr.To(false) props.Probe = nil - props.BackendPort = ptr.To(int32(port.TargetPort.IntValue())) + props.BackendPort = &port.TargetPort.IntVal expectedRules = append(expectedRules, &armnetwork.LoadBalancingRule{ Name: &ruleName, Properties: props, From 8f489e76082d21f6d440868dd8224a9582c109a8 Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Mon, 10 Feb 2025 12:16:11 +0200 Subject: [PATCH 8/9] add unit tests --- pkg/consts/consts.go | 3 +- pkg/provider/azure.go | 4 +- pkg/provider/azure_loadbalancer_test.go | 45 ++++++++++++++++++ pkg/provider/azure_test.go | 62 +++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 6f75dc435e..9a3efb7ab4 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -382,8 +382,7 @@ const ( // LoadBalancerBackendPoolConfigurationTypeNodeIP is the lb backend pool config type node ip LoadBalancerBackendPoolConfigurationTypeNodeIP = "nodeIP" // LoadBalancerBackendPoolConfigurationTypePODIP is the lb backend pool config type pod ip - // TODO (nilo19): support pod IP in the future - LoadBalancerBackendPoolConfigurationTypePODIP = "podIP" + LoadBalancerBackendPoolConfigurationTypePodIP = "podIP" ) // error messages diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go index 54a30874f9..c211ef6909 100644 --- a/pkg/provider/azure.go +++ b/pkg/provider/azure.go @@ -294,13 +294,13 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *config.C if config.LoadBalancerBackendPoolConfigurationType == "" || // TODO(nilo19): support pod IP mode in the future - strings.EqualFold(config.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePODIP) { + strings.EqualFold(config.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePodIP) { config.LoadBalancerBackendPoolConfigurationType = consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration } else { supportedLoadBalancerBackendPoolConfigurationTypes := utilsets.NewString( strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration), strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypeNodeIP), - strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypePODIP)) + strings.ToLower(consts.LoadBalancerBackendPoolConfigurationTypePodIP)) if !supportedLoadBalancerBackendPoolConfigurationTypes.Has(strings.ToLower(config.LoadBalancerBackendPoolConfigurationType)) { return fmt.Errorf("loadBalancerBackendPoolConfigurationType %s is not supported, supported values are %v", config.LoadBalancerBackendPoolConfigurationType, supportedLoadBalancerBackendPoolConfigurationTypes.UnsortedList()) } diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go index 700ec921df..ab72e6bafa 100644 --- a/pkg/provider/azure_loadbalancer_test.go +++ b/pkg/provider/azure_loadbalancer_test.go @@ -3027,9 +3027,43 @@ func TestReconcileLoadBalancerRuleCommon(t *testing.T) { expectedProbes: probes, expectedRules: rules1DualStack, }) + + testCases = append(testCases, []struct { + desc string + service v1.Service + loadBalancerSKU string + probeProtocol string + probePath string + expectedProbes map[bool][]*armnetwork.Probe + expectedRules map[bool][]*armnetwork.LoadBalancingRule + expectedErr bool + }{ + { + desc: "LB backend pool of type PodIP - getExpectedLBRules should return error when the service is dual-stack", + service: getTestServiceDualStack("test", v1.ProtocolTCP, nil, 80), + loadBalancerSKU: "standardV2", + expectedErr: true, + }, + { + desc: "LB backend pool of type PodIP - getExpectedLBRules should return error when the service port is named", + service: getTestServiceWithNamedTargetPorts("test", v1.ProtocolTCP, nil, false, 8080, "http-web-svc"), + loadBalancerSKU: "standardV2", + expectedErr: true, + }, + { + desc: "LB backend pool of type PodIP - getExpectedLBRules should return expected rules and probes", + service: getTestServiceWithIntTargetPorts("test1", v1.ProtocolTCP, nil, false, 8080, 1234), + loadBalancerSKU: "standardV2", + expectedRules: getTestRuleCLB(false, 8080, 1234, false), + expectedProbes: nil, + }, + }...) for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { az := GetTestCloud(ctrl) + if test.loadBalancerSKU == "standardV2" { + az.LoadBalancerBackendPoolConfigurationType = consts.LoadBalancerBackendPoolConfigurationTypePodIP + } az.Config.LoadBalancerSKU = test.loadBalancerSKU service := test.service firstPort := service.Spec.Ports[0] @@ -3130,6 +3164,17 @@ func getTCPResetTestRules(enableTCPReset bool) map[bool][]*armnetwork.LoadBalanc } } +func getTestRuleCLB(enableTCPReset bool, servicePort int32, targetPort int32, isIPv6 bool) map[bool][]*armnetwork.LoadBalancingRule { + rule := getTestRule(enableTCPReset, servicePort, isIPv6) + rule.Properties.EnableFloatingIP = to.Ptr(false) + rule.Properties.Probe = nil + rule.Properties.BackendPort = &targetPort + + return map[bool][]*armnetwork.LoadBalancingRule{ + false: {rule}, + } +} + // getTestRule returns a rule for dualStack. func getTestRule(enableTCPReset bool, port int32, isIPv6 bool) *armnetwork.LoadBalancingRule { suffix := "" diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go index ec6926cd00..c7cf3f3aae 100644 --- a/pkg/provider/azure_test.go +++ b/pkg/provider/azure_test.go @@ -36,6 +36,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" cloudprovider "k8s.io/cloud-provider" @@ -1259,6 +1260,67 @@ func getTestServiceCommon(identifier string, proto v1.Protocol, annotations map[ return svc } +func getTestServiceWithNamedTargetPorts(identifier string, proto v1.Protocol, annotations map[string]string, isIPv6 bool, servicePort int32, namedTargetPorts ...string) v1.Service { + targetPorts := []intstr.IntOrString{} + for _, port := range namedTargetPorts { + targetPorts = append(targetPorts, intstr.FromString(port)) + } + svc := getTestServiceWithTargetPortsCommon(identifier, proto, annotations, servicePort, targetPorts...) + svc.Spec.ClusterIP = "10.0.0.2" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv4Protocol} + if isIPv6 { + svc.Spec.ClusterIP = "fd00::1907" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol} + } + + return svc +} + +func getTestServiceWithIntTargetPorts(identifier string, proto v1.Protocol, annotations map[string]string, isIPv6 bool, servicePort int32, intTargetPorts ...int32) v1.Service { + targetPorts := []intstr.IntOrString{} + for _, port := range intTargetPorts { + targetPorts = append(targetPorts, intstr.FromInt(int(port))) + } + svc := getTestServiceWithTargetPortsCommon(identifier, proto, annotations, servicePort, targetPorts...) + svc.Spec.ClusterIP = "10.0.0.2" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv4Protocol} + if isIPv6 { + svc.Spec.ClusterIP = "fd00::1907" + svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol} + } + + return svc +} + +func getTestServiceWithTargetPortsCommon(identifier string, proto v1.Protocol, annotations map[string]string, servicePort int32, targetPorts ...intstr.IntOrString) v1.Service { + ports := []v1.ServicePort{} + for _, port := range targetPorts { + ports = append(ports, v1.ServicePort{ + Name: "target-port", + Protocol: proto, + TargetPort: port, + Port: servicePort, + }) + } + + svc := v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + Ports: ports, + }, + } + svc.Name = identifier + svc.Namespace = "default" + svc.UID = types.UID(identifier) + if annotations == nil { + svc.Annotations = make(map[string]string) + } else { + svc.Annotations = annotations + } + + return svc +} + func getInternalTestService(identifier string, requestedPorts ...int32) v1.Service { return getTestServiceWithAnnotation(identifier, map[string]string{consts.ServiceAnnotationLoadBalancerInternal: consts.TrueAnnotationValue}, false, requestedPorts...) } From 81d5568246362c795b86dd06e04be2893197bfb5 Mon Sep 17 00:00:00 2001 From: georgeedward2000 Date: Fri, 21 Feb 2025 13:29:30 +0200 Subject: [PATCH 9/9] extract function + check standardV2 sku --- pkg/consts/consts.go | 2 + pkg/provider/azure_loadbalancer.go | 60 +++++++++++++++++------------- pkg/provider/config/azure.go | 6 ++- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 9a3efb7ab4..04ee3f6c30 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -236,6 +236,8 @@ const ( LoadBalancerSKUBasic = "basic" // LoadBalancerSKUStandard is the load balancer standard SKU LoadBalancerSKUStandard = "standard" + // LoadBalancerSKUStandardV2 is the load balancer standardV2 SKU + LoadBalancerSKUStandardV2 = "standardV2" // ServiceAnnotationLoadBalancerInternal is the annotation used on the service ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/azure-load-balancer-internal" diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index a755f56729..fecba1a890 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -2818,32 +2818,9 @@ func (az *Cloud) getExpectedLBRules( var expectedRules []*armnetwork.LoadBalancingRule // If we are using Pod IP in the LB backend, we skip health probes, disable floating IP and use port.TargetPort. if az.IsLBBackendPoolTypePodIP() { - // Check for multi-IP families in the service spec (not allowed for BackendPool of type PodIP). - if len(service.Spec.IPFamilies) > 1 { - return nil, nil, fmt.Errorf("dual-stack services are not supported when LB backend pool type is PodIP") - } - // Build rules for each service port but with floatingIP = false, no health probes. - for _, port := range service.Spec.Ports { - if port.TargetPort.Type == intstr.String { - return nil, nil, fmt.Errorf("named targetPort is not supported when LB backend pool type is PodIP: %s", port.TargetPort.StrVal) - } - ruleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port, isIPv6) - transportProto, _, _, err := getProtocolsFromKubernetesProtocol(port.Protocol) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse transport protocol: %w", err) - } - props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, port, transportProto) - if err != nil { - return nil, nil, fmt.Errorf("error generating LB rule: %w", err) - } - // Turn off floating IP and skip health probe attachments. - props.EnableFloatingIP = ptr.To(false) - props.Probe = nil - props.BackendPort = &port.TargetPort.IntVal - expectedRules = append(expectedRules, &armnetwork.LoadBalancingRule{ - Name: &ruleName, - Properties: props, - }) + expectedRules, err := getExpectedLoadBalancingRuleforBackendPoolTypePodIP(service, az, isIPv6, lbFrontendIPConfigID, lbBackendPoolID, expectedRules) + if err != nil { + return nil, nil, err } return nil, expectedRules, nil } @@ -2997,6 +2974,37 @@ func (az *Cloud) getExpectedLBRules( return expectedProbes, expectedRules, nil } +func getExpectedLoadBalancingRuleforBackendPoolTypePodIP(service *v1.Service, az *Cloud, isIPv6 bool, lbFrontendIPConfigID string, lbBackendPoolID string, expectedRules []*armnetwork.LoadBalancingRule) ([]*armnetwork.LoadBalancingRule, error) { + // Check for multi-IP families in the service spec (not allowed for BackendPool of type PodIP). + if len(service.Spec.IPFamilies) > 1 { + return nil, fmt.Errorf("dual-stack services are not supported when LB backend pool type is PodIP") + } + // Build rules for each service port but with floatingIP = false, no health probes. + for _, port := range service.Spec.Ports { + if port.TargetPort.Type == intstr.String { + return nil, fmt.Errorf("named targetPort is not supported when LB backend pool type is PodIP: %s", port.TargetPort.StrVal) + } + ruleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port, isIPv6) + transportProto, _, _, err := getProtocolsFromKubernetesProtocol(port.Protocol) + if err != nil { + return nil, fmt.Errorf("failed to parse transport protocol: %w", err) + } + props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, port, transportProto) + if err != nil { + return nil, fmt.Errorf("error generating LB rule: %w", err) + } + // Turn off floating IP and skip health probe attachments. + props.EnableFloatingIP = ptr.To(false) + props.Probe = nil + props.BackendPort = &port.TargetPort.IntVal + expectedRules = append(expectedRules, &armnetwork.LoadBalancingRule{ + Name: &ruleName, + Properties: props, + }) + } + return expectedRules, nil +} + // getDefaultLoadBalancingRulePropertiesFormat returns the loadbalancing rule for one port func (az *Cloud) getExpectedLoadBalancingRulePropertiesForPort( service *v1.Service, diff --git a/pkg/provider/config/azure.go b/pkg/provider/config/azure.go index d8dea6dee3..fdae1ca392 100644 --- a/pkg/provider/config/azure.go +++ b/pkg/provider/config/azure.go @@ -181,7 +181,7 @@ func (az *Config) IsLBBackendPoolTypeNodeIP() bool { } func (az *Config) IsLBBackendPoolTypePodIP() bool { - return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePodIP) + return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypePodIP) && az.UseStandardV2LoadBalancer() } func (az *Config) GetPutVMSSVMBatchSize() int { @@ -192,6 +192,10 @@ func (az *Config) UseStandardLoadBalancer() bool { return strings.EqualFold(az.LoadBalancerSKU, consts.LoadBalancerSKUStandard) } +func (az *Config) UseStandardV2LoadBalancer() bool { + return strings.EqualFold(az.LoadBalancerSKU, consts.LoadBalancerSKUStandardV2) +} + func (az *Config) ExcludeMasterNodesFromStandardLB() bool { return az.ExcludeMasterFromStandardLB != nil && *az.ExcludeMasterFromStandardLB }