From 1d81dc6cc25a06cb5812649d65c25a69dbc26084 Mon Sep 17 00:00:00 2001 From: Denis Romanenko Date: Fri, 19 Apr 2024 17:04:35 +0300 Subject: [PATCH] add node annotation Signed-off-by: Denis Romanenko --- README.md | 4 ++- pkg/cloudprovider/yandex/load_balancer.go | 26 ++++++++++++------- .../yandex/load_balancer_tg_controller.go | 16 ++++++++---- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 964b6d3..6fd2f44 100644 --- a/README.md +++ b/README.md @@ -129,10 +129,12 @@ Due to API limitations, only one subnet from each zone must be present in each N * `yandex.cpi.flant.com/listener-subnet-id` – default SubnetID to use for Listeners in created NetworkLoadBalancers. NetworkLoadBalancers will be INTERNAL. * `yandex.cpi.flant.com/listener-address-ipv4` – select pre-defined IPv4 address. Works both on internal and external NetworkLoadBalancers. * `yandex.cpi.flant.com/loadbalancer-external` – override `YANDEX_CLOUD_DEFAULT_LB_LISTENER_SUBNET_ID` per-service. +* `yandex.cpi.flant.com/target-group-name-prefix` - set target group for LB to target group with name `yandex.cpi.flant.com/target-group-name-prefix` annotation value + yandex cluster name + `YANDEX_CLOUD_DEFAULT_LB_TARGET_GROUP_NETWORK_ID`. ##### Node annotations -* `yandex.cpi.flant.com/target-group` - set node to the non-default target group add this annotation to the node. Yandex CCM creates new target groups with name `yandex.cpi.flant.com/target-group` annotation value + network id of instance interfaces. +* `yandex.cpi.flant.com/target-group-name-prefix` - set node to the non-default target group add this annotation to the node. Yandex CCM creates new target groups with name `yandex.cpi.flant.com/target-group-name-prefix` annotation value + yandex cluster name + network id of instance interfaces. + ## Warning 1. If masters are created with their own target groups, then you need to attach the `node.kubernetes.io/exclude-from-external-load-balancers: ""` label on them so that the controller does not try to add the master to a new target group for balancers diff --git a/pkg/cloudprovider/yandex/load_balancer.go b/pkg/cloudprovider/yandex/load_balancer.go index a8b61be..7b09147 100644 --- a/pkg/cloudprovider/yandex/load_balancer.go +++ b/pkg/cloudprovider/yandex/load_balancer.go @@ -13,10 +13,12 @@ import ( ) const ( - targetGroupNetworkIdAnnotation = "yandex.cpi.flant.com/target-group-network-id" - externalLoadBalancerAnnotation = "yandex.cpi.flant.com/loadbalancer-external" - listenerSubnetIdAnnotation = "yandex.cpi.flant.com/listener-subnet-id" - listenerAddressIPv4 = "yandex.cpi.flant.com/listener-address-ipv4" + // node annotation to put node to the specific target group + customTargetGroupNamePrefixAnnotation = "yandex.cpi.flant.com/target-group-name-prefix" + targetGroupNetworkIdAnnotation = "yandex.cpi.flant.com/target-group-network-id" + externalLoadBalancerAnnotation = "yandex.cpi.flant.com/loadbalancer-external" + listenerSubnetIdAnnotation = "yandex.cpi.flant.com/listener-subnet-id" + listenerAddressIPv4 = "yandex.cpi.flant.com/listener-address-ipv4" nodesHealthCheckPath = "/healthz" // NOTE: Please keep the following port in sync with ProxyHealthzPort in pkg/cluster/ports/ports.go @@ -178,7 +180,8 @@ func (yc *Cloud) ensureLB(ctx context.Context, service *v1.Service, nodes []*v1. }, } - tgName := yc.config.ClusterName + lbParams.targetGroupNetworkID + tgName := lbParams.targetGroupNamePrefix + yc.config.ClusterName + lbParams.targetGroupNetworkID + tg, err := yc.yandexService.LbSvc.GetTgByName(ctx, tgName) if err != nil { return nil, err @@ -201,10 +204,11 @@ func (yc *Cloud) ensureLB(ctx context.Context, service *v1.Service, nodes []*v1. } type loadBalancerParameters struct { - targetGroupNetworkID string - listenerSubnetID string - listenerAddressIPv4 string - internal bool + targetGroupNetworkID string + targetGroupNamePrefix string + listenerSubnetID string + listenerAddressIPv4 string + internal bool } func (yc *Cloud) getLoadBalancerParameters(svc *v1.Service) (lbParams loadBalancerParameters) { @@ -227,5 +231,9 @@ func (yc *Cloud) getLoadBalancerParameters(svc *v1.Service) (lbParams loadBalanc lbParams.listenerAddressIPv4 = value } + if value, ok := svc.ObjectMeta.Annotations[customTargetGroupNamePrefixAnnotation]; ok { + lbParams.targetGroupNamePrefix = value + } + return } diff --git a/pkg/cloudprovider/yandex/load_balancer_tg_controller.go b/pkg/cloudprovider/yandex/load_balancer_tg_controller.go index b9c8a30..54dca0c 100644 --- a/pkg/cloudprovider/yandex/load_balancer_tg_controller.go +++ b/pkg/cloudprovider/yandex/load_balancer_tg_controller.go @@ -24,9 +24,6 @@ import ( "k8s.io/apimachinery/pkg/labels" ) -// node annotation to put node to the specific target group -const targetGroupNodeAnnotation = "yandex.cpi.flant.com/target-group" - type NodeTargetGroupSyncer struct { // TODO: refactor cloud out of here cloud *Cloud @@ -158,8 +155,8 @@ func (ntgs *NodeTargetGroupSyncer) constructNetworkIdToTargetMap(ctx context.Con } key := ntgs.cloud.config.ClusterName + subnetInfo.NetworkId - if v, ok := instance.Node.Annotations[targetGroupNodeAnnotation]; ok { - key = v + subnetInfo.NetworkId + if v, ok := instance.Node.Annotations[customTargetGroupNamePrefixAnnotation]; ok { + key = truncateAnnotationValue(v) + key } mapping[key] = append(mapping[subnetInfo.NetworkId], &loadbalancer.Target{ SubnetId: iface.SubnetId, @@ -174,3 +171,12 @@ func (ntgs *NodeTargetGroupSyncer) constructNetworkIdToTargetMap(ctx context.Con return mapping, nil } + +func truncateAnnotationValue(value string) string { + // maximum length of annotation values should not exceed 63 - length of cluster uuid(26 symbols) - length of network id(21) + if len(value) > 36 { + log.Printf("annotation '%s' length should be less than 36 characters, truncate it", value) + value = value[:36] + } + return value +}