feat: use /status/ready as readiness probe of gateway for gateway dis…

…covery (#4368)

Changes the Gateway's Pod readiness probe from `/status` to `/status/ready`.
`/status/ready` returns 200 after receiving the first non-empty configuration.
That allows to make Gateway pods to serve proxy traffic only after they're
initially configured by KIC.

In KIC's Gateway discovery, instead of relying on `EndpointSlices` `Endpoints`'
being ready, we use the discovered endpoints despite their readiness (we only
discard terminating ones) and verify their readiness on our own with the use of
the Admin API's `/status` in the `ReadinessChecker`. It requires running a
periodic readiness reconciliation loop to check whether the endpoints that were
ready should be moved to the pending list and vice versa.

Fundamental changes that were made in this PR:

- `KongAdminAPIServiceReconciler` watches Admin API service endpoints and
  pushes all but terminating ones to the `AdminAPIClientsManager` (via `Notify`
method)
- `AdminAPIClientsManager` accepts the discovered endpoints and verifies their
  readiness with use of a new `ReadinessChecker` first:
  - if they're not ready - they're kept on a pending list
  - if they're ready - they're kept on an active list (that is used to return
    `GatewayClients()` to the upper layers)
- `AdminAPIClientsManager` is responsible for running a readiness
  reconciliation loop in which it uses `ReadinessChecker` to verify if:
  - the clients kept on the active list became not ready; if yes, move them to
    the pending list
  - the clients kept on the pending list became ready; if yes, move them to the
    active list

CHANGELOG.md

@@ -117,6 +117,11 @@ Adding a new version? You'll need three changes:
   KIC restarts, it is now able to fetch the last good configuration from a running
   proxy instance and store it in its internal cache.
   [#4265](https://github.com/Kong/kubernetes-ingress-controller/pull/4265)
+- Gateway Discovery feature was adapted to handle Gateways that are not ready yet
+  in terms of accepting data-plane traffic, but are ready to accept configuration
+  updates. The controller will now send configuration to such Gateways and will
+  actively monitor their readiness for accepting configuration updates.
+  [#4368](https://github.com/Kong/kubernetes-ingress-controller/pull/4368
 
 ### Changed
 
@@ -132,6 +137,10 @@ Adding a new version? You'll need three changes:
   sending stage (we've observed around 35% reduced time in config marshalling
   time but be aware that your mileage may vary).
   [#4222](https://github.com/Kong/kubernetes-ingress-controller/pull/4222)
+- Changed the Gateway's readiness probe in all-in-one manifests from `/status`
+  to `/status/ready`. Gateways will be considered ready only after an initial
+  configuration is applied by the controller.
+  [#4368](https://github.com/Kong/kubernetes-ingress-controller/pull/4368
 
 [gojson]: https://github.com/goccy/go-json
 [httproute-specification]: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRoute

config/variants/multi-gw/base/gateway_deployment.yaml

@@ -99,7 +99,7 @@ spec:
           failureThreshold: 3
         readinessProbe:
           httpGet:
-            path: /status
+            path: /status/ready
             port: 8100
             scheme: HTTP
           initialDelaySeconds: 5

internal/adminapi/client.go

@@ -95,6 +95,12 @@ func (c *Client) NodeID(ctx context.Context) (string, error) {
 	return nodeID, nil
 }
 
+// IsReady returns nil if the Admin API is ready to serve requests.
+func (c *Client) IsReady(ctx context.Context) error {
+	_, err := c.adminAPIClient.Status(ctx)
+	return err
+}
+
 // GetKongVersion returns version of the kong gateway.
 func (c *Client) GetKongVersion(ctx context.Context) (string, error) {
 	if c.isKonnect {

internal/adminapi/endpoints.go

@@ -122,7 +122,7 @@ func (d *Discoverer) AdminAPIsFromEndpointSlice(
 		}
 
 		for _, e := range endpoints.Endpoints {
-			if e.Conditions.Ready == nil || !*e.Conditions.Ready {
+			if e.Conditions.Terminating != nil && *e.Conditions.Terminating {
 				continue
 			}
 

internal/adminapi/endpoints_test.go

@@ -167,7 +167,7 @@ func TestDiscoverer_AddressesFromEndpointSlice(t *testing.T) {
 			dnsStrategy: cfgtypes.ServiceScopedPodDNSStrategy,
 		},
 		{
-			name: "not ready endpoints are not returned",
+			name: "not ready endpoints are returned",
 			endpoints: discoveryv1.EndpointSlice{
 				ObjectMeta:  endpointsSliceObjectMeta,
 				AddressType: discoveryv1.AddressTypeIPv4,
@@ -183,12 +183,18 @@ func TestDiscoverer_AddressesFromEndpointSlice(t *testing.T) {
 				},
 				Ports: builder.NewEndpointPort(8444).WithName("admin").IntoSlice(),
 			},
-			portNames:   sets.New("admin"),
-			want:        sets.New[DiscoveredAdminAPI](),
+			portNames: sets.New("admin"),
+			want: sets.New[DiscoveredAdminAPI](
+				DiscoveredAdminAPI{
+					Address: "https://10.0.0.1:8444",
+					PodRef: k8stypes.NamespacedName{
+						Name: "pod-1", Namespace: namespaceName,
+					},
+				}),
 			dnsStrategy: cfgtypes.IPDNSStrategy,
 		},
 		{
-			name: "not ready and terminating endpoints are not returned",
+			name: "ready and terminating endpoints are not returned",
 			endpoints: discoveryv1.EndpointSlice{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      uuid.NewString(),
@@ -199,7 +205,7 @@ func TestDiscoverer_AddressesFromEndpointSlice(t *testing.T) {
 					{
 						Addresses: []string{"10.0.0.1", "10.0.0.2", "10.0.0.3"},
 						Conditions: discoveryv1.EndpointConditions{
-							Ready:       lo.ToPtr(false),
+							Ready:       lo.ToPtr(true),
 							Terminating: lo.ToPtr(true),
 						},
 						TargetRef: testPodReference(namespaceName, "pod-1"),
@@ -237,7 +243,7 @@ func TestDiscoverer_AddressesFromEndpointSlice(t *testing.T) {
 						Addresses: []string{"10.0.2.1"},
 						Conditions: discoveryv1.EndpointConditions{
 							Ready:       lo.ToPtr(false),
-							Terminating: lo.ToPtr(false),
+							Terminating: lo.ToPtr(true),
 						},
 						TargetRef: testPodReference(namespaceName, "pod-3"),
 					},
@@ -289,7 +295,7 @@ func TestDiscoverer_AddressesFromEndpointSlice(t *testing.T) {
 						Addresses: []string{"10.0.2.1"},
 						Conditions: discoveryv1.EndpointConditions{
 							Ready:       lo.ToPtr(false),
-							Terminating: lo.ToPtr(false),
+							Terminating: lo.ToPtr(true),
 						},
 						TargetRef: testPodReference(namespaceName, "pod-3"),
 					},
@@ -551,7 +557,7 @@ func TestDiscoverer_GetAdminAPIsForService(t *testing.T) {
 									Addresses: []string{"8.0.0.1"},
 									Conditions: discoveryv1.EndpointConditions{
 										Ready:       lo.ToPtr(false),
-										Terminating: lo.ToPtr(false),
+										Terminating: lo.ToPtr(true),
 									},
 									TargetRef: testPodReference(namespaceName, "pod-3"),
 								},
@@ -637,7 +643,7 @@ func TestDiscoverer_GetAdminAPIsForService(t *testing.T) {
 			dnsStrategy: cfgtypes.IPDNSStrategy,
 		},
 		{
-			name: "not Ready Endpoints are not matched",
+			name: "terminating Endpoints are not matched",
 			service: k8stypes.NamespacedName{
 				Namespace: namespaceName,
 				Name:      serviceName,
@@ -652,7 +658,8 @@ func TestDiscoverer_GetAdminAPIsForService(t *testing.T) {
 								{
 									Addresses: []string{"7.0.0.1"},
 									Conditions: discoveryv1.EndpointConditions{
-										Ready: lo.ToPtr(false),
+										Ready:       lo.ToPtr(false),
+										Terminating: lo.ToPtr(true),
 									},
 									TargetRef: testPodReference(namespaceName, "pod-1"),
 								},

internal/clients/config_status_test.go

@@ -5,15 +5,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/go-logr/logr/testr"
+	"github.com/go-logr/logr"
 	"github.com/stretchr/testify/require"
 
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/clients"
 )
 
 func TestChannelConfigNotifier(t *testing.T) {
-	logger := testr.New(t)
-	n := clients.NewChannelConfigNotifier(logger)
+	n := clients.NewChannelConfigNotifier(logr.Discard())
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()