[WIP] Eventing TLS: add tests for dynamically added CA trust bundles

test/rekt/apiserversource_test.go

@@ -111,6 +111,7 @@ func TestApiServerSourceDataPlaneTLS(t *testing.T) {
 
 	env.ParallelTest(ctx, t, apiserversourcefeatures.SendsEventsWithTLS())
 	env.ParallelTest(ctx, t, apiserversourcefeatures.SendsEventsWithTLSTrustBundle())
+	env.ParallelTest(ctx, t, apiserversourcefeatures.SendsEventsWithTLSWithAdditionalTrustBundle())
 }
 
 func TestApiServerSourceDataPlane_EventModes(t *testing.T) {

test/rekt/channel_test.go

@@ -358,6 +358,7 @@ func TestInMemoryChannelTLS(t *testing.T) {
 
 	env.ParallelTest(ctx, t, channel.SubscriptionTLS())
 	env.ParallelTest(ctx, t, channel.SubscriptionTLSTrustBundle())
+	env.ParallelTest(ctx, t, channel.SubscriptionTLSWithAdditionalTrustBundle())
 }
 
 func TestChannelImplDispatcherAuthenticatesWithOIDC(t *testing.T) {

test/rekt/features/apiserversource/data_plane.go

@@ -26,6 +26,7 @@ import (
 	duckv1 "knative.dev/pkg/apis/duck/v1"
 	"knative.dev/pkg/network"
 	"knative.dev/reconciler-test/pkg/environment"
+	"knative.dev/reconciler-test/pkg/knative"
 
 	"knative.dev/eventing/pkg/eventingtls/eventingtlstesting"
 	"knative.dev/eventing/test/rekt/resources/addressable"
@@ -272,6 +273,62 @@ func SendsEventsWithTLSTrustBundle() *feature.Feature {
 	return f
 }
 
+func SendsEventsWithTLSWithAdditionalTrustBundle() *feature.Feature {
+	src := feature.MakeRandomK8sName("apiserversource")
+	sink := feature.MakeRandomK8sName("sink")
+	trustBundle := feature.MakeRandomK8sName("trust-bundle")
+
+	f := feature.NewFeatureNamed("Send events to TLS sink - additional trust bundle")
+
+	f.Prerequisite("should not run when Istio is enabled", featureflags.IstioDisabled())
+
+	f.Setup("install sink", eventshub.Install(sink, eventshub.StartReceiverTLS))
+
+	f.Setup("Add trust bundle to system namespace", func(ctx context.Context, t feature.T) {
+
+		configmap.Install(trustBundle, knative.KnativeNamespaceFromContext(ctx),
+			configmap.WithLabels(map[string]string{"networking.knative.dev/trust-bundle": "true"}),
+			configmap.WithData("ca.crt", *eventshub.GetCaCerts(ctx)),
+		)(ctx, t)
+	})
+
+	sacmName := feature.MakeRandomK8sName("apiserversource")
+	f.Requirement("Create Service Account for ApiServerSource with RBAC for v1.Event resources",
+		setupAccountAndRoleForPods(sacmName))
+
+	cfg := []manifest.CfgFn{
+		apiserversource.WithServiceAccountName(sacmName),
+		apiserversource.WithEventMode(v1.ResourceMode),
+		apiserversource.WithResources(v1.APIVersionKindSelector{
+			APIVersion: "v1",
+			Kind:       "Event",
+		}),
+	}
+
+	f.Requirement("install ApiServerSource", func(ctx context.Context, t feature.T) {
+		cfg = append(cfg, apiserversource.WithSink(&duckv1.Destination{
+			URI: &apis.URL{
+				Scheme: "https", // Force using https
+				Host:   network.GetServiceHostname(sink, environment.FromContext(ctx).Namespace()),
+			},
+			CACerts: nil, // CA certs are in the new trust-bundle
+		}))
+		apiserversource.Install(src, cfg...)(ctx, t)
+	})
+	f.Requirement("ApiServerSource goes ready", apiserversource.IsReady(src))
+
+	f.Stable("ApiServerSource as event source").
+		Must("delivers events on sink with ref",
+			eventassert.OnStore(sink).
+				Match(eventassert.MatchKind(eventshub.EventReceived)).
+				MatchEvent(test.HasType("dev.knative.apiserver.resource.update")).
+				AtLeast(1),
+		).
+		Must("Set sinkURI to HTTPS endpoint", source.ExpectHTTPSSink(apiserversource.Gvr(), src))
+
+	return f
+}
+
 // SendsEventsWithEventTypes tests apiserversource to a ready broker.
 func SendsEventsWithEventTypes() *feature.Feature {
 	source := feature.MakeRandomK8sName("source")

test/rekt/features/channel/eventing_tls_feature.go

@@ -31,13 +31,15 @@ import (
 	"knative.dev/reconciler-test/pkg/eventshub"
 	"knative.dev/reconciler-test/pkg/eventshub/assert"
 	"knative.dev/reconciler-test/pkg/feature"
+	"knative.dev/reconciler-test/pkg/knative"
 	"knative.dev/reconciler-test/pkg/resources/service"
 	"knative.dev/reconciler-test/resources/certificate"
 
 	"knative.dev/eventing/pkg/eventingtls/eventingtlstesting"
 	"knative.dev/eventing/test/rekt/features/featureflags"
 	"knative.dev/eventing/test/rekt/resources/addressable"
 	"knative.dev/eventing/test/rekt/resources/channel_impl"
+	"knative.dev/eventing/test/rekt/resources/configmap"
 	"knative.dev/eventing/test/rekt/resources/subscription"
 )
 
@@ -243,3 +245,88 @@ func SubscriptionTLSTrustBundle() *feature.Feature {
 
 	return f
 }
+
+func SubscriptionTLSWithAdditionalTrustBundle() *feature.Feature {
+
+	channelName := feature.MakeRandomK8sName("channel")
+	subscriptionName := feature.MakeRandomK8sName("sub")
+	sink := feature.MakeRandomK8sName("sink")
+	source := feature.MakeRandomK8sName("source")
+	dlsName := feature.MakeRandomK8sName("dls")
+	dlsSubscriptionName := feature.MakeRandomK8sName("dls-sub")
+	trustBundle := feature.MakeRandomK8sName("trust-bundle")
+
+	f := feature.NewFeature()
+
+	f.Prerequisite("transport encryption is strict", featureflags.TransportEncryptionStrict())
+	f.Prerequisite("should not run when Istio is enabled", featureflags.IstioDisabled())
+
+	f.Setup("Add trust bundle to system namespace", func(ctx context.Context, t feature.T) {
+
+		configmap.Install(trustBundle, knative.KnativeNamespaceFromContext(ctx),
+			configmap.WithLabels(map[string]string{"networking.knative.dev/trust-bundle": "true"}),
+			configmap.WithData("ca.crt", *eventshub.GetCaCerts(ctx)),
+		)(ctx, t)
+	})
+
+	f.Setup("install sink", eventshub.Install(sink, eventshub.StartReceiverTLS))
+	f.Setup("install sink", eventshub.Install(dlsName, eventshub.StartReceiverTLS))
+	f.Setup("install channel", channel_impl.Install(channelName))
+	f.Setup("channel is ready", channel_impl.IsReady(channelName))
+
+	f.Setup("install subscription", func(ctx context.Context, t feature.T) {
+		d := &duckv1.Destination{
+			URI: &apis.URL{
+				Scheme: "https", // Force using https
+				Host:   network.GetServiceHostname(sink, environment.FromContext(ctx).Namespace()),
+			},
+			CACerts: nil, // CA certs are in the new trust-bundle
+		}
+		subscription.Install(subscriptionName,
+			subscription.WithChannel(channel_impl.AsRef(channelName)),
+			subscription.WithSubscriberFromDestination(d))(ctx, t)
+	})
+	f.Setup("subscription is ready", subscription.IsReady(subscriptionName))
+	f.Setup("install dead letter subscription", func(ctx context.Context, t feature.T) {
+		d := &duckv1.Destination{
+			URI: &apis.URL{
+				Scheme: "https", // Force using https
+				Host:   network.GetServiceHostname(dlsName, environment.FromContext(ctx).Namespace()),
+			},
+			CACerts: nil, // CA certs are in the trust-bundle
+		}
+
+		subscription.Install(dlsSubscriptionName,
+			subscription.WithChannel(channel_impl.AsRef(channelName)),
+			subscription.WithDeadLetterSinkFromDestination(d),
+			subscription.WithSubscriber(nil, "http://127.0.0.1:2468", ""))(ctx, t)
+	})
+	f.Setup("subscription dead letter is ready", subscription.IsReady(dlsSubscriptionName))
+	f.Setup("Channel has HTTPS address", channel_impl.ValidateAddress(channelName, addressable.AssertHTTPSAddress))
+
+	event := cetest.FullEvent()
+	event.SetID(uuid.New().String())
+
+	f.Requirement("install source", eventshub.Install(source,
+		eventshub.StartSenderToResourceTLS(channel_impl.GVR(), channelName, nil),
+		eventshub.InputEvent(event),
+		// Send multiple events so that we take into account that the certificate rotation might
+		// be detected by the server after some time.
+		eventshub.SendMultipleEvents(100, 3*time.Second),
+	))
+
+	f.Assert("Event sent", assert.OnStore(source).
+		MatchSentEvent(cetest.HasId(event.ID())).
+		AtLeast(1),
+	)
+	f.Assert("Event received in sink", assert.OnStore(sink).
+		MatchReceivedEvent(cetest.HasId(event.ID())).
+		AtLeast(1),
+	)
+	f.Assert("Event received in dead letter sink", assert.OnStore(dlsName).
+		MatchReceivedEvent(cetest.HasId(event.ID())).
+		AtLeast(1),
+	)
+
+	return f
+}

test/rekt/features/pingsource/features.go

@@ -27,13 +27,15 @@ import (
 	"knative.dev/reconciler-test/pkg/environment"
 	"knative.dev/reconciler-test/pkg/eventshub"
 	"knative.dev/reconciler-test/pkg/feature"
+	"knative.dev/reconciler-test/pkg/knative"
 	"knative.dev/reconciler-test/pkg/manifest"
 	"knative.dev/reconciler-test/pkg/resources/service"
 
 	sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1"
 	"knative.dev/eventing/pkg/eventingtls/eventingtlstesting"
 	"knative.dev/eventing/test/rekt/resources/addressable"
 	"knative.dev/eventing/test/rekt/resources/broker"
+	"knative.dev/eventing/test/rekt/resources/configmap"
 	"knative.dev/eventing/test/rekt/resources/eventtype"
 	"knative.dev/eventing/test/rekt/resources/trigger"
 
@@ -132,6 +134,48 @@ func SendsEventsTLSTrustBundle() *feature.Feature {
 	return f
 }
 
+func SendsEventsTLSWithAdditionalTrustBundle() *feature.Feature {
+	src := feature.MakeRandomK8sName("pingsource")
+	sink := feature.MakeRandomK8sName("sink")
+	trustBundle := feature.MakeRandomK8sName("trust-bundle")
+
+	f := feature.NewFeature()
+
+	f.Prerequisite("should not run when Istio is enabled", featureflags.IstioDisabled())
+
+	f.Setup("install sink", eventshub.Install(sink, eventshub.StartReceiverTLS))
+
+	f.Setup("Add trust bundle to system namespace", func(ctx context.Context, t feature.T) {
+
+		configmap.Install(trustBundle, knative.KnativeNamespaceFromContext(ctx),
+			configmap.WithLabels(map[string]string{"networking.knative.dev/trust-bundle": "true"}),
+			configmap.WithData("ca.crt", *eventshub.GetCaCerts(ctx)),
+		)(ctx, t)
+	})
+
+	f.Requirement("install pingsource", func(ctx context.Context, t feature.T) {
+		d := &duckv1.Destination{
+			URI: &apis.URL{
+				Scheme: "https", // Force using https
+				Host:   network.GetServiceHostname(sink, environment.FromContext(ctx).Namespace()),
+			},
+			CACerts: nil, // CA certs are in the trust-bundle
+		}
+
+		pingsource.Install(src, pingsource.WithSink(d))(ctx, t)
+	})
+	f.Requirement("pingsource goes ready", pingsource.IsReady(src))
+
+	f.Stable("pingsource as event source").
+		Must("delivers events", assert.OnStore(sink).
+			Match(eventassert.MatchKind(eventshub.EventReceived)).
+			MatchEvent(test.HasType("dev.knative.sources.ping")).
+			AtLeast(1)).
+		Must("Set sinkURI to HTTPS endpoint", source.ExpectHTTPSSink(pingsource.Gvr(), src))
+
+	return f
+}
+
 func SendsEventsWithSinkURI() *feature.Feature {
 	source := feature.MakeRandomK8sName("pingsource")
 	sink := feature.MakeRandomK8sName("sink")

test/rekt/features/trigger/feature.go

@@ -27,6 +27,7 @@ import (
 	"knative.dev/reconciler-test/pkg/environment"
 	"knative.dev/reconciler-test/pkg/eventshub"
 	"knative.dev/reconciler-test/pkg/feature"
+	"knative.dev/reconciler-test/pkg/knative"
 	"knative.dev/reconciler-test/pkg/manifest"
 	"knative.dev/reconciler-test/pkg/resources/service"
 
@@ -36,6 +37,7 @@ import (
 	"knative.dev/eventing/pkg/eventingtls/eventingtlstesting"
 	"knative.dev/eventing/test/rekt/features/featureflags"
 	"knative.dev/eventing/test/rekt/resources/broker"
+	"knative.dev/eventing/test/rekt/resources/configmap"
 	"knative.dev/eventing/test/rekt/resources/pingsource"
 	"knative.dev/eventing/test/rekt/resources/trigger"
 )
@@ -236,3 +238,78 @@ func TriggerWithTLSSubscriberTrustBundle() *feature.Feature {
 
 	return f
 }
+
+func TriggerWithTLSSubscriberWithAdditionalCATrustBundles() *feature.Feature {
+	f := feature.NewFeatureNamed("Trigger with TLS subscriber and additional trust bundle")
+
+	f.Prerequisite("should not run when Istio is enabled", featureflags.IstioDisabled())
+
+	brokerName := feature.MakeRandomK8sName("broker")
+	sourceName := feature.MakeRandomK8sName("source")
+	sinkName := feature.MakeRandomK8sName("sink")
+	triggerName := feature.MakeRandomK8sName("trigger")
+	dlsName := feature.MakeRandomK8sName("dls")
+	dlsTriggerName := feature.MakeRandomK8sName("dls-trigger")
+	trustBundle := feature.MakeRandomK8sName("trust-bundle")
+
+	eventToSend := test.FullEvent()
+
+	// Install Broker
+	f.Setup("Install Broker", broker.Install(brokerName, broker.WithEnvConfig()...))
+	f.Setup("Broker is ready", broker.IsReady(brokerName))
+	f.Setup("Broker is addressable", broker.IsAddressable(brokerName))
+
+	// Install Sink
+	f.Setup("Install Sink", eventshub.Install(sinkName, eventshub.StartReceiverTLS))
+	f.Setup("Install dead letter sink service", eventshub.Install(dlsName, eventshub.StartReceiverTLS))
+
+	f.Setup("Add trust bundle to system namespace", func(ctx context.Context, t feature.T) {
+
+		configmap.Install(trustBundle, knative.KnativeNamespaceFromContext(ctx),
+			configmap.WithLabels(map[string]string{"networking.knative.dev/trust-bundle": "true"}),
+			configmap.WithData("ca.crt", *eventshub.GetCaCerts(ctx)),
+		)(ctx, t)
+	})
+
+	// Install Trigger
+	f.Setup("Install trigger", func(ctx context.Context, t feature.T) {
+		subscriber := &duckv1.Destination{
+			URI: &apis.URL{
+				Scheme: "https", // Force using https
+				Host:   network.GetServiceHostname(sinkName, environment.FromContext(ctx).Namespace()),
+			},
+			CACerts: nil, // CA certs are in the new trust-bundle
+		}
+
+		trigger.Install(triggerName, brokerName,
+			trigger.WithSubscriberFromDestination(subscriber))(ctx, t)
+	})
+	f.Setup("Wait for Trigger to become ready", trigger.IsReady(triggerName))
+
+	f.Setup("Install failing trigger", func(ctx context.Context, t feature.T) {
+		dls := service.AsDestinationRef(dlsName)
+
+		linear := eventingv1.BackoffPolicyLinear
+		trigger.Install(dlsTriggerName, brokerName,
+			trigger.WithRetry(10, &linear, pointer.String("PT1S")),
+			trigger.WithDeadLetterSinkFromDestination(dls),
+			trigger.WithSubscriber(nil, "http://127.0.0.1:2468"))(ctx, t)
+	})
+	f.Setup("Wait for failing Trigger to become ready", trigger.IsReady(dlsTriggerName))
+
+	// Install Source
+	f.Requirement("Install Source", eventshub.Install(
+		sourceName,
+		eventshub.StartSenderToResource(broker.GVR(), brokerName),
+		eventshub.InputEvent(eventToSend),
+	))
+
+	f.Assert("Trigger delivers events to TLS subscriber", assert.OnStore(sinkName).
+		MatchReceivedEvent(test.HasId(eventToSend.ID())).
+		AtLeast(1))
+	f.Assert("Trigger delivers events to TLS dead letter sink", assert.OnStore(dlsName).
+		MatchReceivedEvent(test.HasId(eventToSend.ID())).
+		AtLeast(1))
+
+	return f
+}

test/rekt/pingsource_test.go

@@ -61,6 +61,7 @@ func TestPingSourceTLS(t *testing.T) {
 
 	env.ParallelTest(ctx, t, pingsource.SendsEventsTLS())
 	env.ParallelTest(ctx, t, pingsource.SendsEventsTLSTrustBundle())
+	env.ParallelTest(ctx, t, pingsource.SendsEventsTLSWithAdditionalTrustBundle())
 }
 
 func TestPingSourceWithSinkURI(t *testing.T) {

test/rekt/resources/configmap/config-features.yaml

@@ -3,15 +3,28 @@ kind: ConfigMap
 metadata:
   name: {{ .name }}
   namespace: {{ .namespace }}
+  {{ if .labels }}
+  labels:
+    {{ range $key, $value := .labels }}
+    {{ $key }}: "{{ $value }}"
+    {{ end }}
+  {{ else }}
   labels:
     knative.dev/config-propagation: original
     knative.dev/config-category: eventing
+  {{ end }}
 data:
+  {{ if .data }}
+  {{ range $key, $value := .data }}
+  {{ $key }}: |-
+    {{ $value }}
+  {{ end }}
+  {{ else }}
   _example: |
     my-enabled-flag: "enabled"
     my-disabled-flag: "disabled"
     my-allowed-flag: "allowed"
-    apiserversources-nodeselector-testkey: testvalue
-    apiserversources-nodeselector-testkey1: testvalue1
-    apiserversources-nodeselector-testkey2: testvalue2
-
+    apiserversources.nodeselector.testkey: testvalue
+    apiserversources.nodeselector.testkey1: testvalue1
+    apiserversources.nodeselector.testkey2: testvalue2
+  {{ end }}