feat(occm/lb): octavia prometheus listener's annotation

docs/openstack-cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md

@@ -202,6 +202,22 @@ Request Body:
 
   Defines the health monitor retry count for the loadbalancer pool members to be marked down.
 
+- `loadbalancer.openstack.org/metrics-enable`
+
+  If 'true', enable the Prometheus listener on the loadbalancer. (default: 'false')
+
+  The Kubernetes service must be the [owner of the LoadBalancer](#sharing-load-balancer-with-multiple-services)
+
+  Not supported when `lb-provider=ovn` is configured in openstack-cloud-controller-manager.
+
+- `loadbalancer.openstack.org/metrics-port`
+
+  Defines the Prometheus listener's port. If `metric-enable` is 'true', the annotation is automatically added to the service. Default: `9100`
+
+- `loadbalancer.openstack.org/metrics-allow-cidrs`
+
+  Defines the Prometheus listener's allowed cirds. __Warning__: [security recommendations](#metric-listener-allowed-cird-security-recommendation). Default: none 
+
 - `loadbalancer.openstack.org/flavor-id`
 
   The id of the flavor that is used for creating the loadbalancer.
@@ -240,6 +256,10 @@ Request Body:
   This annotation is automatically added and it contains the floating ip address of the load balancer service.
   When using `loadbalancer.openstack.org/hostname` annotation it is the only place to see the real address of the load balancer.
 
+- `loadbalancer.openstack.org/load-balancer-vip-address`
+
+  This annotation is automatically added and it contains the Octavia's Virtual-IP (VIP).
+
 - `loadbalancer.openstack.org/node-selector`
 
   A set of key=value annotations used to filter nodes for targeting by the load balancer. When defined, only nodes that match all the specified key=value annotations will be targeted. If an annotation includes only a key without a value, the filter will check only for the existence of the key on the node. If the value is not set, the `node-selector` value defined in the OCCM configuration is applied.
@@ -636,3 +656,64 @@ is not yet supported by OCCM.
 Internally, OCCM would automatically look for IPv4 or IPv6 subnet to allocate the load balancer
 address from based on the service's address family preference. If the subnet with preferred
 address family is not available, load balancer can not be created.
+
+### Metric endpoint configuration
+
+Since Octavia v2.25, Octavia proposes to expose an HTTP Prometheus endpoint. Using the annotation `loadbalancer.openstack.org/metrics-enable`, you will be able to configure this endpoint on the LoadBalancer:
+
+```yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: service-with-metric
+  namespace: default
+  annotations:
+    loadbalancer.openstack.org/metrics-enable: "true" # Enable the listener endpoint on the Octavia LoadBalancer (default false)
+    loadbalancer.openstack.org/metrics-port: "9100" # Listener's port (default 9100)
+    loadbalancer.openstack.org/metrics-allow-cidrs: "10.0.0.0/8, fe80::/10" # Listener's allowed cidrs (default none)
+spec:
+  type: LoadBalancer
+```
+
+Then, you can configure a Prometheus scrapper like to get metrics from the LoadBalancer.
+
+e.g. Prometheus Operator configuration:
+
+```yaml
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+    name: octavia-sd-config
+    labels:
+        release: prometheus # adapt it to your Prometheus deployment configuration
+spec:
+    kubernetesSDConfigs:
+    - role: Service
+    relabelings:
+    - sourceLabels: [__meta_kubernetes_namespace]
+      targetLabel: namespace
+      action: replace
+    - sourceLabels: [__meta_kubernetes_service_name]
+      targetLabel: job
+      action: replace
+    - sourceLabels:
+        - __meta_kubernetes_service_annotation_loadbalancer_openstack_org_load_balancer_vip_address
+        - __meta_kubernetes_service_annotation_loadbalancer_openstack_org_metrics_port
+      separator: ":"
+      targetLabel: __address__
+      action: replace
+    - sourceLabels:
+        - __meta_kubernetes_service_annotation_loadbalancer_openstack_org_metrics_enable
+      regex: "true"
+      action: keep
+```
+
+> This configuration use the `loadbalancer.openstack.org/load-balancer-vip-address` annotation that will use the Octavia's VIP to fetch the metric endpoint. Adapt it to your Octavia deployment.
+
+For more information: https://docs.openstack.org/octavia/latest/user/guides/monitoring.html#monitoring-with-prometheus
+
+Grafana dashboard for Octavia Amphora: https://grafana.com/grafana/dashboards/15828-openstack-octavia-amphora-load-balancer/
+
+#### Metric listener allowed CIRD security recommendation
+
+If the Octavia LoadBalancer is exposed with a public IP, the Prometheus listener is also exposed (at least for Amphora). Even if no critical data are exposed by this endpoint, __it's strongly recommended to apply an allowed cidrs on the listener__ via the annotation `loadbalancer.openstack.org/metrics-allow-cidrs`.

go.mod

@@ -46,6 +46,7 @@ require (
 // the below fixes the "go list -m all" execution
 replace (
 	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.31.1
+	k8s.io/cri-client => k8s.io/cri-client v0.31.1
 	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.31.1
 	k8s.io/endpointslice => k8s.io/endpointslice v0.31.1
 	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.31.1

pkg/openstack/events.go

@@ -23,4 +23,5 @@ const (
 	eventLBAZIgnored                   = "LoadBalancerAvailabilityZonesIgnored"
 	eventLBFloatingIPSkipped           = "LoadBalancerFloatingIPSkipped"
 	eventLBRename                      = "LoadBalancerRename"
+	eventLBMetricListenerIgnored       = "LoadBalancerMetricListenerIgnored"
 )

pkg/openstack/loadbalancer.go

@@ -71,6 +71,9 @@ const (
 	ServiceAnnotationLoadBalancerSubnetID             = "loadbalancer.openstack.org/subnet-id"
 	ServiceAnnotationLoadBalancerNetworkID            = "loadbalancer.openstack.org/network-id"
 	ServiceAnnotationLoadBalancerMemberSubnetID       = "loadbalancer.openstack.org/member-subnet-id"
+	ServiceAnnotationLoadBalancerMetricsEnabled       = "loadbalancer.openstack.org/metrics-enable"
+	ServiceAnnotationLoadBalancerMetricsPort          = "loadbalancer.openstack.org/metrics-port"
+	ServiceAnnotationLoadBalancerMetricsAllowCidrs    = "loadbalancer.openstack.org/metrics-allow-cidrs"
 	ServiceAnnotationLoadBalancerTimeoutClientData    = "loadbalancer.openstack.org/timeout-client-data"
 	ServiceAnnotationLoadBalancerTimeoutMemberConnect = "loadbalancer.openstack.org/timeout-member-connect"
 	ServiceAnnotationLoadBalancerTimeoutMemberData    = "loadbalancer.openstack.org/timeout-member-data"
@@ -87,6 +90,7 @@ const (
 	ServiceAnnotationLoadBalancerHealthMonitorMaxRetriesDown = "loadbalancer.openstack.org/health-monitor-max-retries-down"
 	ServiceAnnotationLoadBalancerLoadbalancerHostname        = "loadbalancer.openstack.org/hostname"
 	ServiceAnnotationLoadBalancerAddress                     = "loadbalancer.openstack.org/load-balancer-address"
+	ServiceAnnotationLoadBalancerVIPAddress                  = "loadbalancer.openstack.org/load-balancer-vip-address"
 	// revive:disable:var-naming
 	ServiceAnnotationTlsContainerRef = "loadbalancer.openstack.org/default-tls-container-ref"
 	// revive:enable:var-naming
@@ -95,14 +99,15 @@ const (
 	ServiceAnnotationLoadBalancerID = "loadbalancer.openstack.org/load-balancer-id"
 
 	// Octavia resources name formats
-	servicePrefix  = "kube_service_"
-	lbFormat       = "%s%s_%s_%s"
-	listenerPrefix = "listener_"
-	listenerFormat = listenerPrefix + "%d_%s"
-	poolPrefix     = "pool_"
-	poolFormat     = poolPrefix + "%d_%s"
-	monitorPrefix  = "monitor_"
-	monitorFormat  = monitorPrefix + "%d_%s"
+	servicePrefix        = "kube_service_"
+	lbFormat             = "%s%s_%s_%s"
+	listenerPrefix       = "listener_"
+	listenerFormat       = listenerPrefix + "%d_%s"
+	listenerFormatMetric = listenerPrefix + "metric_%s"
+	poolPrefix           = "pool_"
+	poolFormat           = poolPrefix + "%d_%s"
+	monitorPrefix        = "monitor_"
+	monitorFormat        = monitorPrefix + "%d_%s"
 )
 
 // LbaasV2 is a LoadBalancer implementation based on Octavia
@@ -142,6 +147,9 @@ type serviceConfig struct {
 	healthMonitorTimeout        int
 	healthMonitorMaxRetries     int
 	healthMonitorMaxRetriesDown int
+	metricAllowedCIDRs          []string
+	metricEnabled               bool
+	metricPort                  int
 	preferredIPFamily           corev1.IPFamily // preferred (the first) IP family indicated in service's `spec.ipFamilies`
 }
 
@@ -451,6 +459,20 @@ func getIntFromServiceAnnotation(service *corev1.Service, annotationKey string,
 	return defaultSetting
 }
 
+// getStringArrayFromServiceAnnotationSeparatedByComma  searches a given v1.Service for a specific annotationKey
+// and either returns the annotation's string array value (using comma as separator), or the specified defaultSetting.
+// Each value of the array is TrimSpaced. After the trim, if the string is empty, remove it.
+func getStringArrayFromServiceAnnotationSeparatedByComma(service *corev1.Service, annotationKey string, defaultSetting []string) []string {
+	klog.V(4).Infof("getStringArrayFromServiceAnnotationSeparatedByComma(%s/%s, %v, %q)", service.Namespace, service.Name, annotationKey, defaultSetting)
+	if annotationValue, ok := service.Annotations[annotationKey]; ok {
+		returnValue := cpoutil.SplitTrim(annotationValue, ',')
+		klog.V(4).Infof("Found a Service Annotation: %v = %q", annotationKey, returnValue)
+		return returnValue
+	}
+	klog.V(4).Infof("Could not find a Service Annotation; falling back to default setting: %v = %q", annotationKey, defaultSetting)
+	return defaultSetting
+}
+
 // getBoolFromServiceAnnotation searches a given v1.Service for a specific annotationKey and either returns the annotation's boolean value or a specified defaultSetting
 // If the annotation is not found or is not a valid boolean ("true" or "false"), it falls back to the defaultSetting and logs a message accordingly.
 func getBoolFromServiceAnnotation(service *corev1.Service, annotationKey string, defaultSetting bool) bool {
@@ -1046,16 +1068,39 @@ func (lbaas *LbaasV2) buildCreateMemberOpts(port corev1.ServicePort, nodes []*co
 }
 
 // Make sure the listener is created for Service
-func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, name string, curListenerMapping map[listenerKey]*listeners.Listener, port corev1.ServicePort, svcConf *serviceConfig, _ *corev1.Service) (*listeners.Listener, error) {
-	listener, isPresent := curListenerMapping[listenerKey{
-		Protocol: getListenerProtocol(port.Protocol, svcConf),
-		Port:     int(port.Port),
-	}]
-	if !isPresent {
-		listenerCreateOpt := lbaas.buildListenerCreateOpt(port, svcConf, name)
-		listenerCreateOpt.LoadbalancerID = lbID
+func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, name string, curListenerMapping map[listenerKey]*listeners.Listener, port corev1.ServicePort, svcConf *serviceConfig, isMetricListener bool) (*listeners.Listener, error) {
+	var listener *listeners.Listener
+	var isListenerPresent bool
+
+	if isMetricListener {
+		listener, isListenerPresent = curListenerMapping[listenerKey{
+			Protocol: listeners.ProtocolPrometheus,
+			Port:     svcConf.metricPort,
+		}]
+	} else {
+		listener, isListenerPresent = curListenerMapping[listenerKey{
+			Protocol: getListenerProtocol(port.Protocol, svcConf),
+			Port:     int(port.Port),
+		}]
+	}
+
+	if !isListenerPresent {
+		var listenerCreateOpt listeners.CreateOpts
+		if isMetricListener {
+			listenerCreateOpt = listeners.CreateOpts{
+				Name:           name,
+				Protocol:       listeners.ProtocolPrometheus,
+				ProtocolPort:   svcConf.metricPort,
+				AllowedCIDRs:   svcConf.metricAllowedCIDRs,
+				LoadbalancerID: lbID,
+				Tags:           []string{svcConf.lbName},
+			}
+		} else {
+			listenerCreateOpt = lbaas.buildListenerCreateOpt(port, svcConf, name)
+			listenerCreateOpt.LoadbalancerID = lbID
+		}
 
-		klog.V(2).Infof("Creating listener for port %d using protocol %s", int(port.Port), listenerCreateOpt.Protocol)
+		klog.V(2).Infof("Creating listener for port %d using protocol %s", listenerCreateOpt.ProtocolPort, listenerCreateOpt.Protocol)
 
 		var err error
 		listener, err = openstackutil.CreateListener(lbaas.lb, lbID, listenerCreateOpt)
@@ -1078,50 +1123,57 @@ func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, name string, curListene
 			}
 		}
 
-		if svcConf.connLimit != listener.ConnLimit {
-			updateOpts.ConnLimit = &svcConf.connLimit
-			listenerChanged = true
-		}
-
-		listenerKeepClientIP := listener.InsertHeaders[annotationXForwardedFor] == "true"
-		if svcConf.keepClientIP != listenerKeepClientIP {
-			updateOpts.InsertHeaders = &listener.InsertHeaders
-			if svcConf.keepClientIP {
-				if *updateOpts.InsertHeaders == nil {
-					*updateOpts.InsertHeaders = make(map[string]string)
-				}
-				(*updateOpts.InsertHeaders)[annotationXForwardedFor] = "true"
-			} else {
-				delete(*updateOpts.InsertHeaders, annotationXForwardedFor)
-			}
-			listenerChanged = true
-		}
-		if svcConf.tlsContainerRef != listener.DefaultTlsContainerRef {
-			updateOpts.DefaultTlsContainerRef = &svcConf.tlsContainerRef
-			listenerChanged = true
-		}
-		if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureTimeout, lbaas.opts.LBProvider) {
-			if svcConf.timeoutClientData != listener.TimeoutClientData {
-				updateOpts.TimeoutClientData = &svcConf.timeoutClientData
+		if isMetricListener {
+			if !cpoutil.StringListEqual(svcConf.metricAllowedCIDRs, listener.AllowedCIDRs) {
+				updateOpts.AllowedCIDRs = &svcConf.metricAllowedCIDRs
 				listenerChanged = true
 			}
-			if svcConf.timeoutMemberConnect != listener.TimeoutMemberConnect {
-				updateOpts.TimeoutMemberConnect = &svcConf.timeoutMemberConnect
+		} else {
+			if svcConf.connLimit != listener.ConnLimit {
+				updateOpts.ConnLimit = &svcConf.connLimit
 				listenerChanged = true
 			}
-			if svcConf.timeoutMemberData != listener.TimeoutMemberData {
-				updateOpts.TimeoutMemberData = &svcConf.timeoutMemberData
+
+			listenerKeepClientIP := listener.InsertHeaders[annotationXForwardedFor] == "true"
+			if svcConf.keepClientIP != listenerKeepClientIP {
+				updateOpts.InsertHeaders = &listener.InsertHeaders
+				if svcConf.keepClientIP {
+					if *updateOpts.InsertHeaders == nil {
+						*updateOpts.InsertHeaders = make(map[string]string)
+					}
+					(*updateOpts.InsertHeaders)[annotationXForwardedFor] = "true"
+				} else {
+					delete(*updateOpts.InsertHeaders, annotationXForwardedFor)
+				}
 				listenerChanged = true
 			}
-			if svcConf.timeoutTCPInspect != listener.TimeoutTCPInspect {
-				updateOpts.TimeoutTCPInspect = &svcConf.timeoutTCPInspect
+			if svcConf.tlsContainerRef != listener.DefaultTlsContainerRef {
+				updateOpts.DefaultTlsContainerRef = &svcConf.tlsContainerRef
 				listenerChanged = true
 			}
-		}
-		if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureVIPACL, lbaas.opts.LBProvider) {
-			if !cpoutil.StringListEqual(svcConf.allowedCIDR, listener.AllowedCIDRs) {
-				updateOpts.AllowedCIDRs = &svcConf.allowedCIDR
-				listenerChanged = true
+			if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureTimeout, lbaas.opts.LBProvider) {
+				if svcConf.timeoutClientData != listener.TimeoutClientData {
+					updateOpts.TimeoutClientData = &svcConf.timeoutClientData
+					listenerChanged = true
+				}
+				if svcConf.timeoutMemberConnect != listener.TimeoutMemberConnect {
+					updateOpts.TimeoutMemberConnect = &svcConf.timeoutMemberConnect
+					listenerChanged = true
+				}
+				if svcConf.timeoutMemberData != listener.TimeoutMemberData {
+					updateOpts.TimeoutMemberData = &svcConf.timeoutMemberData
+					listenerChanged = true
+				}
+				if svcConf.timeoutTCPInspect != listener.TimeoutTCPInspect {
+					updateOpts.TimeoutTCPInspect = &svcConf.timeoutTCPInspect
+					listenerChanged = true
+				}
+			}
+			if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureVIPACL, lbaas.opts.LBProvider) {
+				if !cpoutil.StringListEqual(svcConf.allowedCIDR, listener.AllowedCIDRs) {
+					updateOpts.AllowedCIDRs = &svcConf.allowedCIDR
+					listenerChanged = true
+				}
 			}
 		}
 
@@ -1780,7 +1832,7 @@ func (lbaas *LbaasV2) ensureOctaviaLoadBalancer(ctx context.Context, clusterName
 		}
 
 		for portIndex, port := range service.Spec.Ports {
-			listener, err := lbaas.ensureOctaviaListener(loadbalancer.ID, cpoutil.Sprintf255(listenerFormat, portIndex, lbName), curListenerMapping, port, svcConf, service)
+			listener, err := lbaas.ensureOctaviaListener(loadbalancer.ID, cpoutil.Sprintf255(listenerFormat, portIndex, lbName), curListenerMapping, port, svcConf, false)
 			if err != nil {
 				return nil, err
 			}
@@ -1800,6 +1852,25 @@ func (lbaas *LbaasV2) ensureOctaviaLoadBalancer(ctx context.Context, clusterName
 			curListeners = popListener(curListeners, listener.ID)
 		}
 
+		// Check if we need to expose the metric endpoint
+		svcConf.metricEnabled = getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerMetricsEnabled, false)
+		if svcConf.metricEnabled && openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeaturePrometheusListener, lbaas.opts.LBProvider) {
+			// Only a LB owner can add the prometheus listener (to avoid conflict with a shared loadbalancer)
+			if isLBOwner {
+				svcConf.metricPort = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerMetricsPort, 9100)
+				lbaas.updateServiceAnnotation(service, ServiceAnnotationLoadBalancerMetricsPort, strconv.Itoa(svcConf.metricPort))
+				svcConf.metricAllowedCIDRs = getStringArrayFromServiceAnnotationSeparatedByComma(service, ServiceAnnotationLoadBalancerMetricsAllowCidrs, []string{})
+				listener, err := lbaas.ensureOctaviaListener(loadbalancer.ID, cpoutil.Sprintf255(listenerFormatMetric, lbName), curListenerMapping, corev1.ServicePort{}, svcConf, true)
+				if err != nil {
+					return nil, err
+				}
+				curListeners = popListener(curListeners, listener.ID)
+			} else {
+				msg := "Metric Listener cannot be deployed on Service %s, only owner Service can do that"
+				lbaas.eventRecorder.Eventf(service, corev1.EventTypeWarning, eventLBMetricListenerIgnored, msg, serviceName)
+				klog.Warningf(msg, serviceName)
+			}
+		}
 		// Deal with the remaining listeners, delete the listener if it was created by this Service previously.
 		if err := lbaas.deleteOctaviaListeners(loadbalancer.ID, curListeners, isLBOwner, lbName); err != nil {
 			return nil, err
@@ -1819,8 +1890,9 @@ func (lbaas *LbaasV2) ensureOctaviaLoadBalancer(ctx context.Context, clusterName
 		}
 	}
 
-	// save address into the annotation
+	// save addresses into the annotations
 	lbaas.updateServiceAnnotation(service, ServiceAnnotationLoadBalancerAddress, addr)
+	lbaas.updateServiceAnnotation(service, ServiceAnnotationLoadBalancerVIPAddress, loadbalancer.VipAddress)
 
 	// add LB name to load balancer tags.
 	if svcConf.supportLBTags {