From aef7eb8d95135814fb02d111a74b198493fd535b Mon Sep 17 00:00:00 2001 From: Mattia Lavacca Date: Mon, 22 Apr 2024 17:16:21 +0200 Subject: [PATCH] feat: TLS listeners supported Signed-off-by: Mattia Lavacca --- ...l => gateway-with-multiple-listeners.yaml} | 10 +- .../controlplane/controller_utils_test.go | 12 ++ controller/gateway/controller_conditions.go | 6 + .../gateway/controller_reconciler_utils.go | 128 ++++++++++-------- .../controller_reconciler_utils_test.go | 42 +----- controller/pkg/controlplane/controlplane.go | 10 ++ internal/utils/dataplane/config.go | 3 +- pkg/consts/dataplane.go | 7 +- pkg/utils/kubernetes/resources/deployments.go | 9 +- pkg/utils/kubernetes/resources/services.go | 6 +- test/integration/test_gateway.go | 6 +- 11 files changed, 128 insertions(+), 111 deletions(-) rename config/samples/{gateway-with-custom-listeners.yaml => gateway-with-multiple-listeners.yaml} (75%) diff --git a/config/samples/gateway-with-custom-listeners.yaml b/config/samples/gateway-with-multiple-listeners.yaml similarity index 75% rename from config/samples/gateway-with-custom-listeners.yaml rename to config/samples/gateway-with-multiple-listeners.yaml index fdf607a72..7646c8ad4 100644 --- a/config/samples/gateway-with-custom-listeners.yaml +++ b/config/samples/gateway-with-multiple-listeners.yaml @@ -4,11 +4,6 @@ metadata: name: kong spec: controllerName: konghq.com/gateway-operator - parametersRef: - group: gateway-operator.konghq.com - kind: GatewayConfiguration - name: kong - namespace: default --- kind: Gateway apiVersion: gateway.networking.k8s.io/v1 @@ -24,3 +19,8 @@ spec: - name: http2 protocol: HTTP port: 8089 + - name: tls + protocol: TLS + port: 9443 + tls: + mode: Passthrough diff --git a/controller/controlplane/controller_utils_test.go b/controller/controlplane/controller_utils_test.go index 04e4015b6..9bd030408 100644 --- a/controller/controlplane/controller_utils_test.go +++ b/controller/controlplane/controller_utils_test.go @@ -434,6 +434,14 @@ func TestSetControlPlaneDefaults(t *testing.T) { Name: "CONTROLLER_ADMISSION_WEBHOOK_LISTEN", Value: consts.ControlPlaneAdmissionWebhookEnvVarValue, }, + { + Name: "", + Value: consts.ControlPlaneAdmissionWebhookEnvVarValue, + }, + { + Name: "CONTROLLER_FEATURE_GATES", + Value: "GatewayAlpha=true", + }, }, }, }, @@ -511,6 +519,10 @@ func TestSetControlPlaneDefaults(t *testing.T) { Name: "CONTROLLER_ADMISSION_WEBHOOK_LISTEN", Value: consts.ControlPlaneAdmissionWebhookEnvVarValue, }, + { + Name: "CONTROLLER_FEATURE_GATES", + Value: "GatewayAlpha=true", + }, }, }, }, diff --git a/controller/gateway/controller_conditions.go b/controller/gateway/controller_conditions.go index 6fa8a3511..7206b05d6 100644 --- a/controller/gateway/controller_conditions.go +++ b/controller/gateway/controller_conditions.go @@ -32,4 +32,10 @@ const ( // to express that more than one TLS secret has been set in the listener // TLS configuration. ListenerReasonTooManyTLSSecrets k8sutils.ConditionReason = "TooManyTLSSecrets" + + // ListenereReasonInvalidTLSMode must be used with the Accepted condition + // to express that the listener has an invalid TLS mode. + // HTTPS can only be configured with mode Terminate, while TLS can only be + // be configured with mode Passthrough. + ListenereReasonInvalidTLSMode k8sutils.ConditionReason = "InvalidTLSMode" ) diff --git a/controller/gateway/controller_reconciler_utils.go b/controller/gateway/controller_reconciler_utils.go index 6995691fc..f098c8a30 100644 --- a/controller/gateway/controller_reconciler_utils.go +++ b/controller/gateway/controller_reconciler_utils.go @@ -299,8 +299,8 @@ func generateDataPlaneNetworkPolicy( var ( protocolTCP = corev1.ProtocolTCP adminAPISSLPort = intstr.FromInt(consts.DataPlaneAdminAPIPort) - proxyPort = intstr.FromInt(consts.DataPlaneProxyPort) - proxySSLPort = intstr.FromInt(consts.DataPlaneProxySSLPort) + proxyPort = intstr.FromInt(consts.DataPlaneProxyHTTPPort) + proxySSLPort = intstr.FromInt(consts.DataPlaneProxyHTTPSPort) metricsPort = intstr.FromInt(consts.DataPlaneMetricsPort) ) @@ -527,9 +527,9 @@ func supportedRoutesByProtocol() map[gatewayv1.ProtocolType]map[gatewayv1.Kind]s return map[gatewayv1.ProtocolType]map[gatewayv1.Kind]struct{}{ gatewayv1.HTTPProtocolType: {"HTTPRoute": {}}, gatewayv1.HTTPSProtocolType: {"HTTPRoute": {}}, + gatewayv1.TLSProtocolType: {"TLSRoute": {}}, - // L4 routes not supported yet - // gatewayv1.TLSProtocolType: {"TLSRoute": {}}, + // TCP and UDP routes not supported yet // gatewayv1.TCPProtocolType: {"TCPRoute": {}}, // gatewayv1.UDPProtocolType: {"UDPRoute": {}}, } @@ -590,10 +590,25 @@ func (g *gatewayConditionsAndListenersAwareT) setAccepted() { LastTransitionTime: metav1.Now(), ObservedGeneration: g.Generation, } - if listener.Protocol != gatewayv1.HTTPProtocolType && listener.Protocol != gatewayv1.HTTPSProtocolType { + if listener.Protocol != gatewayv1.HTTPProtocolType && + listener.Protocol != gatewayv1.HTTPSProtocolType && + listener.Protocol != gatewayv1.TLSProtocolType { acceptedCondition.Status = metav1.ConditionFalse acceptedCondition.Reason = string(gatewayv1.ListenerReasonUnsupportedProtocol) } + // Only TLS terminate mode supported with HTTPS Listeners. + if listener.Protocol == gatewayv1.HTTPSProtocolType && *listener.TLS.Mode != gatewayv1.TLSModeTerminate { + acceptedCondition.Status = metav1.ConditionFalse + acceptedCondition.Reason = string(ListenereReasonInvalidTLSMode) + acceptedCondition.Message = "Only Terminate mode is supported with HTTPS listeners" + } + + // Only TLS passthrough mode supported with TLS Listeners. + if listener.Protocol == gatewayv1.TLSProtocolType && *listener.TLS.Mode != gatewayv1.TLSModePassthrough { + acceptedCondition.Status = metav1.ConditionFalse + acceptedCondition.Reason = string(ListenereReasonInvalidTLSMode) + acceptedCondition.Message = "Only Passthrough mode is supported with TLS listeners" + } listenerConditionsAware := listenerConditionsAware(&g.Status.Listeners[i]) listenerConditionsAware.SetConditions(append(listenerConditionsAware.Conditions, acceptedCondition)) } @@ -685,9 +700,11 @@ func setDataPlaneIngressServicePorts(opts *operatorv1beta1.DataPlaneOptions, lis } switch l.Protocol { case gatewayv1.HTTPSProtocolType: - port.TargetPort = intstr.FromInt(consts.DataPlaneProxySSLPort) + port.TargetPort = intstr.FromInt(consts.DataPlaneProxyHTTPSPort) case gatewayv1.HTTPProtocolType: - port.TargetPort = intstr.FromInt(consts.DataPlaneProxyPort) + port.TargetPort = intstr.FromInt(consts.DataPlaneProxyHTTPPort) + case gatewayv1.TLSProtocolType: + port.TargetPort = intstr.FromInt(consts.DataPlaneProxyTLSPort) default: errs = errors.Join(errs, fmt.Errorf("listener %d uses unsupported protocol %s", i, l.Protocol)) continue @@ -712,67 +729,66 @@ func getSupportedKindsWithResolvedRefsCondition(ctx context.Context, c client.Cl message := "" if listener.TLS != nil { - // We currently do not support TLSRoutes, hence only TLS termination supported. - if *listener.TLS.Mode != gatewayv1.TLSModeTerminate { - resolvedRefsCondition.Status = metav1.ConditionFalse - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, "Only Terminate mode is supported") - } // We currently do not support more that one listener certificate. - if len(listener.TLS.CertificateRefs) != 1 { + if len(listener.TLS.CertificateRefs) > 1 { resolvedRefsCondition.Reason = string(ListenerReasonTooManyTLSSecrets) message = conditionMessage(message, "Only one certificate per listener is supported") } else { - isValidGroupKind := true - certificateRef := listener.TLS.CertificateRefs[0] - if certificateRef.Group != nil && *certificateRef.Group != "" && *certificateRef.Group != gatewayv1.Group(corev1.SchemeGroupVersion.Group) { - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, fmt.Sprintf("Group %s not supported in CertificateRef", *certificateRef.Group)) - isValidGroupKind = false - } - if certificateRef.Kind != nil && *certificateRef.Kind != "" && *certificateRef.Kind != gatewayv1.Kind("Secret") { - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, fmt.Sprintf("Kind %s not supported in CertificateRef", *certificateRef.Kind)) - isValidGroupKind = false - } - secretNamespace := gatewayNamespace - if certificateRef.Namespace != nil && *certificateRef.Namespace != "" { - secretNamespace = string(*certificateRef.Namespace) - } - - var secretExists bool - if isValidGroupKind { - // Get the secret and check it exists. - certificateSecret := &corev1.Secret{} - err = c.Get(ctx, types.NamespacedName{ - Namespace: secretNamespace, - Name: string(certificateRef.Name), - }, certificateSecret) - if err != nil { - if !k8serrors.IsNotFound(err) { - return - } + // check certificate references only when Terminate mode is used. + // Passthrough mode does not need a certificate. + if len(listener.TLS.CertificateRefs) != 0 { + isValidGroupKind := true + certificateRef := listener.TLS.CertificateRefs[0] + if certificateRef.Group != nil && *certificateRef.Group != "" && *certificateRef.Group != gatewayv1.Group(corev1.SchemeGroupVersion.Group) { resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, fmt.Sprintf("Referenced secret %s/%s does not exist", secretNamespace, certificateRef.Name)) - } else { - secretExists = true + message = conditionMessage(message, fmt.Sprintf("Group %s not supported in CertificateRef", *certificateRef.Group)) + isValidGroupKind = false + } + if certificateRef.Kind != nil && *certificateRef.Kind != "" && *certificateRef.Kind != gatewayv1.Kind("Secret") { + resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + message = conditionMessage(message, fmt.Sprintf("Kind %s not supported in CertificateRef", *certificateRef.Kind)) + isValidGroupKind = false + } + secretNamespace := gatewayNamespace + if certificateRef.Namespace != nil && *certificateRef.Namespace != "" { + secretNamespace = string(*certificateRef.Namespace) } - } - if secretExists { - // In case there is a cross-namespace reference, check if there is any referenceGrant allowing it. - if secretNamespace != gatewayNamespace { - referenceGrantList := &gatewayv1beta1.ReferenceGrantList{} - err = c.List(ctx, referenceGrantList, client.InNamespace(secretNamespace)) + var secretExists bool + if isValidGroupKind { + // Get the secret and check it exists. + certificateSecret := &corev1.Secret{} + err = c.Get(ctx, types.NamespacedName{ + Namespace: secretNamespace, + Name: string(certificateRef.Name), + }, certificateSecret) if err != nil { - return + if !k8serrors.IsNotFound(err) { + return + } + resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + message = conditionMessage(message, fmt.Sprintf("Referenced secret %s/%s does not exist", secretNamespace, certificateRef.Name)) + } else { + secretExists = true } - if !isSecretCrossReferenceGranted(gatewayv1.Namespace(gatewayNamespace), certificateRef.Name, referenceGrantList.Items) { - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonRefNotPermitted) - message = conditionMessage(message, fmt.Sprintf("Secret %s/%s reference not allowed by any referenceGrant", secretNamespace, certificateRef.Name)) + } + + if secretExists { + // In case there is a cross-namespace reference, check if there is any referenceGrant allowing it. + if secretNamespace != gatewayNamespace { + referenceGrantList := &gatewayv1beta1.ReferenceGrantList{} + err = c.List(ctx, referenceGrantList, client.InNamespace(secretNamespace)) + if err != nil { + return + } + if !isSecretCrossReferenceGranted(gatewayv1.Namespace(gatewayNamespace), certificateRef.Name, referenceGrantList.Items) { + resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonRefNotPermitted) + message = conditionMessage(message, fmt.Sprintf("Secret %s/%s reference not allowed by any referenceGrant", secretNamespace, certificateRef.Name)) + } } } } + } } diff --git a/controller/gateway/controller_reconciler_utils_test.go b/controller/gateway/controller_reconciler_utils_test.go index b3a4825a7..f243b592a 100644 --- a/controller/gateway/controller_reconciler_utils_test.go +++ b/controller/gateway/controller_reconciler_utils_test.go @@ -553,12 +553,12 @@ func TestSetDataPlaneIngressServicePorts(t *testing.T) { { Name: "http", Port: 80, - TargetPort: intstr.FromInt(consts.DataPlaneProxyPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPPort), }, { Name: "https", Port: 443, - TargetPort: intstr.FromInt(consts.DataPlaneProxySSLPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPSPort), }, }, }, @@ -580,7 +580,7 @@ func TestSetDataPlaneIngressServicePorts(t *testing.T) { { Name: "http", Port: 80, - TargetPort: intstr.FromInt(consts.DataPlaneProxyPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPPort), }, }, expectedError: errors.New("listener 1 uses unsupported protocol UDP"), @@ -979,42 +979,6 @@ func TestGetSupportedKindsWithResolvedRefsCondition(t *testing.T) { ObservedGeneration: generation, }, }, - { - name: "tls with passthrough, HTTPS protocol, no allowed routes", - gatewayNamespace: "default", - listener: gatewayv1.Listener{ - Protocol: gatewayv1.HTTPSProtocolType, - TLS: &gatewayv1.GatewayTLSConfig{ - Mode: lo.ToPtr(gatewayv1.TLSModePassthrough), - CertificateRefs: []gatewayv1.SecretObjectReference{ - { - Name: "test-secret", - }, - }, - }, - }, - secrets: []client.Object{ - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret", - Namespace: "default", - }, - }, - }, - expectedSupportedKinds: []gatewayv1.RouteGroupKind{ - { - Group: (*gatewayv1.Group)(&gatewayv1.GroupVersion.Group), - Kind: "HTTPRoute", - }, - }, - expectedResolvedRefsCondition: metav1.Condition{ - Type: string(gatewayv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - Reason: string(gatewayv1.ListenerReasonInvalidCertificateRef), - Message: "Only Terminate mode is supported.", - ObservedGeneration: generation, - }, - }, { name: "tls bad-formed, multiple TLS secrets no cross-namespace reference", gatewayNamespace: "default", diff --git a/controller/pkg/controlplane/controlplane.go b/controller/pkg/controlplane/controlplane.go index a9db4feb7..1ab326fa2 100644 --- a/controller/pkg/controlplane/controlplane.go +++ b/controller/pkg/controlplane/controlplane.go @@ -92,6 +92,16 @@ func SetDefaults( changed = true } + // If the feature gates are not set, set them to the default value. + const ( + featureGates = "CONTROLLER_FEATURE_GATES" + defaultFeatureGates = "GatewayAlpha=true" + ) + if k8sutils.EnvValueByName(container.Env, featureGates) == "" { + container.Env = k8sutils.UpdateEnv(container.Env, featureGates, defaultFeatureGates) + changed = true + } + if args.Namespace != "" && args.DataPlaneIngressServiceName != "" { if _, isOverrideDisabled := dontOverride["CONTROLLER_PUBLISH_SERVICE"]; !isOverrideDisabled { publishServiceNN := k8stypes.NamespacedName{Namespace: args.Namespace, Name: args.DataPlaneIngressServiceName}.String() diff --git a/internal/utils/dataplane/config.go b/internal/utils/dataplane/config.go index 90392d490..d785af235 100644 --- a/internal/utils/dataplane/config.go +++ b/internal/utils/dataplane/config.go @@ -26,8 +26,9 @@ var KongDefaults = map[string]string{ "KONG_PORT_MAPS": "80:8000, 443:8443", "KONG_PROXY_ACCESS_LOG": "/dev/stdout", "KONG_PROXY_ERROR_LOG": "/dev/stderr", - "KONG_PROXY_LISTEN": fmt.Sprintf("0.0.0.0:%d reuseport backlog=16384, 0.0.0.0:%d http2 ssl reuseport backlog=16384", consts.DataPlaneProxyPort, consts.DataPlaneProxySSLPort), + "KONG_PROXY_LISTEN": fmt.Sprintf("0.0.0.0:%d reuseport backlog=16384, 0.0.0.0:%d http2 ssl reuseport backlog=16384", consts.DataPlaneProxyHTTPPort, consts.DataPlaneProxyHTTPSPort), "KONG_STATUS_LISTEN": fmt.Sprintf("0.0.0.0:%d", consts.DataPlaneStatusPort), + "KONG_STREAM_LISTEN": fmt.Sprintf("0.0.0.0:%d ssl", consts.DataPlaneProxyTLSPort), // TODO: reconfigure following https://github.com/Kong/gateway-operator/issues/7 "KONG_ADMIN_LISTEN": fmt.Sprintf("0.0.0.0:%d ssl reuseport backlog=16384", consts.DataPlaneAdminAPIPort), diff --git a/pkg/consts/dataplane.go b/pkg/consts/dataplane.go index a420608bd..af882412a 100644 --- a/pkg/consts/dataplane.go +++ b/pkg/consts/dataplane.go @@ -135,10 +135,13 @@ const ( DataPlaneAdminAPIPort = 8444 // DataPlaneHTTPSPort is the port that the dataplane uses for HTTP. - DataPlaneProxyPort = 8000 + DataPlaneProxyHTTPPort = 8000 // DataPlaneHTTPSPort is the port that the dataplane uses for HTTPS. - DataPlaneProxySSLPort = 8443 + DataPlaneProxyHTTPSPort = 8443 + + // DataPlaneProxyTLSPort is the port that the dataplane uses for TLS. + DataPlaneProxyTLSPort = 9443 // DataPlaneMetricsPort is the port that the dataplane uses for metrics. DataPlaneMetricsPort = 8100 diff --git a/pkg/utils/kubernetes/resources/deployments.go b/pkg/utils/kubernetes/resources/deployments.go index 79d14647d..9aa6acf9b 100644 --- a/pkg/utils/kubernetes/resources/deployments.go +++ b/pkg/utils/kubernetes/resources/deployments.go @@ -295,12 +295,17 @@ func GenerateDataPlaneContainer(image string) corev1.Container { Ports: []corev1.ContainerPort{ { Name: "proxy", - ContainerPort: consts.DataPlaneProxyPort, + ContainerPort: consts.DataPlaneProxyHTTPPort, Protocol: corev1.ProtocolTCP, }, { Name: "proxy-ssl", - ContainerPort: consts.DataPlaneProxySSLPort, + ContainerPort: consts.DataPlaneProxyHTTPSPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "proxy-tls", + ContainerPort: consts.DataPlaneProxyTLSPort, Protocol: corev1.ProtocolTCP, }, { diff --git a/pkg/utils/kubernetes/resources/services.go b/pkg/utils/kubernetes/resources/services.go index b8ffecc38..232876e97 100644 --- a/pkg/utils/kubernetes/resources/services.go +++ b/pkg/utils/kubernetes/resources/services.go @@ -91,13 +91,13 @@ var DefaultDataPlaneIngressServicePorts = []corev1.ServicePort{ Name: "http", Protocol: corev1.ProtocolTCP, Port: consts.DefaultHTTPPort, - TargetPort: intstr.FromInt(consts.DataPlaneProxyPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPPort), }, { Name: "https", Protocol: corev1.ProtocolTCP, Port: consts.DefaultHTTPSPort, - TargetPort: intstr.FromInt(consts.DataPlaneProxySSLPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPSPort), }, } @@ -145,7 +145,7 @@ func ServicePortsFromDataPlaneIngressOpt(dataplane *operatorv1beta1.DataPlane) S newPorts := make([]corev1.ServicePort, 0) alreadyUsedPorts := make(map[int32]struct{}) for _, p := range dataplane.Spec.Network.Services.Ingress.Ports { - targetPort := intstr.FromInt(consts.DataPlaneProxyPort) + targetPort := intstr.FromInt(consts.DataPlaneProxyHTTPPort) if !cmp.Equal(p.TargetPort, intstr.IntOrString{}) { targetPort = p.TargetPort } diff --git a/test/integration/test_gateway.go b/test/integration/test_gateway.go index a971dde5b..cd0ec195f 100644 --- a/test/integration/test_gateway.go +++ b/test/integration/test_gateway.go @@ -729,8 +729,8 @@ func TestGatewayDataPlaneNetworkPolicy(t *testing.T) { t.Log("verifying that the DataPlane's proxy ingress traffic is allowed") var expectAllowProxyIngress networkPolicyIngressRuleDecorator - expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyPort) - expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxySSLPort) + expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyHTTPPort) + expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyHTTPSPort) t.Log("verifying that the DataPlane's metrics ingress traffic is allowed") var expectAllowMetricsIngress networkPolicyIngressRuleDecorator @@ -778,7 +778,7 @@ func TestGatewayDataPlaneNetworkPolicy(t *testing.T) { testutils.GatewayNetworkPolicyForGatewayContainsRules(t, GetCtx(), gateway, clients, expectedUpdatedProxyListenPort.Rule), testutils.SubresourceReadinessWait, time.Second) var notExpectedUpdatedProxyListenPort networkPolicyIngressRuleDecorator - notExpectedUpdatedProxyListenPort.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyPort) + notExpectedUpdatedProxyListenPort.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyHTTPPort) require.Eventually(t, testutils.Not( testutils.GatewayNetworkPolicyForGatewayContainsRules(t, GetCtx(), gateway, clients, notExpectedUpdatedProxyListenPort.Rule),