Skip to content

Commit

Permalink
feat(konnect): add finalizer handling for unmanaged KongPluginBindings (
Browse files Browse the repository at this point in the history 
#704)
  • Loading branch information
@pmalek
pmalek authored Oct 7, 2024
1 parent 172abaa commit e3cd706
Show file tree
Hide file tree
Showing 3 changed files with 170 additions and 62 deletions.
120 changes: 66 additions & 54 deletions controller/konnect/reconciler_generic_pluginbindingfinalizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ import (
"github.com/kong/gateway-operator/controller/pkg/log"
"github.com/kong/gateway-operator/pkg/consts"

configurationv1 "github.com/kong/kubernetes-configuration/api/configuration/v1"
configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1"
configurationv1beta1 "github.com/kong/kubernetes-configuration/api/configuration/v1beta1"
)

// KonnectEntityPluginBindingFinalizerReconciler reconciles Konnect entities that may be referenced by KongPluginBinding.
Expand Down Expand Up @@ -62,50 +64,60 @@ func (r *KonnectEntityPluginBindingFinalizerReconciler[T, TEnt]) SetupWithManage
return b.Complete(r)
}

func enqueueKongServiceForKongPluginBinding() func(
ctx context.Context, obj client.Object) []reconcile.Request {
// enqueueObjectReferencedByKongPluginBinding watches for KongPluginBinding objects
// that reference the given Konnect entity.
// It returns the reconcile.Request slice containing the entity that the KongPluginBinding references.
func (r *KonnectEntityPluginBindingFinalizerReconciler[T, TEnt]) enqueueObjectReferencedByKongPluginBinding() func(ctx context.Context, obj client.Object) []reconcile.Request {
return func(ctx context.Context, obj client.Object) []reconcile.Request {
kpb, ok := obj.(*configurationv1alpha1.KongPluginBinding)
if !ok {
return nil
}

if kpb.Spec.Targets.ServiceReference == nil ||
kpb.Spec.Targets.ServiceReference.Kind != "KongService" ||
kpb.Spec.Targets.ServiceReference.Group != configurationv1alpha1.GroupVersion.Group {
return nil
}
var (
name string
e T
ent = TEnt(&e)
)

return []ctrl.Request{
{
NamespacedName: types.NamespacedName{
Namespace: kpb.Namespace,
Name: kpb.Spec.Targets.ServiceReference.Name,
},
},
}
}
}
switch any(ent).(type) {
case *configurationv1alpha1.KongService:
if kpb.Spec.Targets.ServiceReference == nil ||
kpb.Spec.Targets.ServiceReference.Kind != "KongService" ||
kpb.Spec.Targets.ServiceReference.Group != configurationv1alpha1.GroupVersion.Group {
return nil
}
name = kpb.Spec.Targets.ServiceReference.Name

func enqueueKongRouteForKongPluginBinding() func(
ctx context.Context, obj client.Object) []reconcile.Request {
return func(ctx context.Context, obj client.Object) []reconcile.Request {
kpb, ok := obj.(*configurationv1alpha1.KongPluginBinding)
if !ok {
return nil
}
case *configurationv1alpha1.KongRoute:
if kpb.Spec.Targets.RouteReference == nil ||
kpb.Spec.Targets.RouteReference.Kind != "KongRoute" ||
kpb.Spec.Targets.RouteReference.Group != configurationv1alpha1.GroupVersion.Group {
return nil
}
name = kpb.Spec.Targets.RouteReference.Name

case *configurationv1.KongConsumer:
if kpb.Spec.Targets.ConsumerReference == nil {
return nil
}
name = kpb.Spec.Targets.ConsumerReference.Name

if kpb.Spec.Targets.RouteReference == nil ||
kpb.Spec.Targets.RouteReference.Kind != "KongRoute" ||
kpb.Spec.Targets.RouteReference.Group != configurationv1alpha1.GroupVersion.Group {
case *configurationv1beta1.KongConsumerGroup:
if kpb.Spec.Targets.ConsumerGroupReference == nil {
return nil
}
name = kpb.Spec.Targets.ConsumerGroupReference.Name

default:
return nil
}

return []ctrl.Request{
{
NamespacedName: types.NamespacedName{
Namespace: kpb.Namespace,
Name: kpb.Spec.Targets.RouteReference.Name,
Name: name,
},
},
}
Expand Down Expand Up @@ -216,6 +228,10 @@ func (r *KonnectEntityPluginBindingFinalizerReconciler[T, TEnt]) getKongPluginBi
return IndexFieldKongPluginBindingKongServiceReference
case *configurationv1alpha1.KongRoute:
return IndexFieldKongPluginBindingKongRouteReference
case *configurationv1.KongConsumer:
return IndexFieldKongPluginBindingKongConsumerReference
case *configurationv1beta1.KongConsumerGroup:
return IndexFieldKongPluginBindingKongConsumerGroupReference
default:
panic(fmt.Sprintf("unsupported entity type %s", constraints.EntityTypeName[T]()))
}
Expand All @@ -229,36 +245,32 @@ func (r *KonnectEntityPluginBindingFinalizerReconciler[T, TEnt]) setControllerBu
ent = TEnt(&e)
)

var pred func(obj client.Object) bool

switch any(ent).(type) {
case *configurationv1alpha1.KongService:
b.
For(&configurationv1alpha1.KongService{},
builder.WithPredicates(
predicate.NewPredicateFuncs(objRefersToKonnectGatewayControlPlane[configurationv1alpha1.KongService]),
kongPluginsAnnotationChangedPredicate,
),
).
Watches(
&configurationv1alpha1.KongPluginBinding{},
handler.EnqueueRequestsFromMapFunc(
enqueueKongServiceForKongPluginBinding(),
),
)
pred = objRefersToKonnectGatewayControlPlane[configurationv1alpha1.KongService]
case *configurationv1alpha1.KongRoute:
b.
For(&configurationv1alpha1.KongRoute{},
builder.WithPredicates(
predicate.NewPredicateFuncs(objRefersToKonnectGatewayControlPlane[configurationv1alpha1.KongRoute]),
kongPluginsAnnotationChangedPredicate,
),
).
Watches(
&configurationv1alpha1.KongPluginBinding{},
handler.EnqueueRequestsFromMapFunc(
enqueueKongRouteForKongPluginBinding(),
),
)
pred = kongRouteRefersToKonnectGatewayControlPlane(r.Client)
case *configurationv1.KongConsumer:
pred = objRefersToKonnectGatewayControlPlane[configurationv1.KongConsumer]
case *configurationv1beta1.KongConsumerGroup:
pred = objRefersToKonnectGatewayControlPlane[configurationv1beta1.KongConsumerGroup]
default:
panic(fmt.Sprintf("unsupported entity type %s", constraints.EntityTypeName[T]()))
}

b.
For(ent,
builder.WithPredicates(
predicate.NewPredicateFuncs(pred),
kongPluginsAnnotationChangedPredicate,
),
).
Watches(
&configurationv1alpha1.KongPluginBinding{},
handler.EnqueueRequestsFromMapFunc(
r.enqueueObjectReferencedByKongPluginBinding(),
),
)
}
44 changes: 36 additions & 8 deletions modules/manager/controller_setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@ const (
KongTargetControllerName = "KongTarget"
// KongServicePluginBindingFinalizerControllerName is the name of the KongService PluginBinding finalizer controller.
KongServicePluginBindingFinalizerControllerName = "KongServicePluginBindingFinalizer"
// KongRoutePluginBindingFinalizerControllerName is the name of the KongRoute PluginBinding finalizer controller.
KongRoutePluginBindingFinalizerControllerName = "KongRoutePluginBindingFinalizer"
// KongConsumerPluginBindingFinalizerControllerName is the name of the KongConsumer PluginBinding finalizer controller.
KongConsumerPluginBindingFinalizerControllerName = "KongConsumerPluginBindingFinalizer"
// KongConsumerGroupPluginBindingFinalizerControllerName is the name of the KongConsumerGroup PluginBinding finalizer controller.
KongConsumerGroupPluginBindingFinalizerControllerName = "KongConsumerGroupPluginBindingFinalizer"
// KongCredentialsSecretControllerName is the name of the Credentials Secret controller.
KongCredentialsSecretControllerName = "KongCredentialSecret"
// KongCredentialBasicAuthControllerName is the name of the KongCredentialBasicAuth controller.
Expand Down Expand Up @@ -542,14 +548,6 @@ func SetupControllers(mgr manager.Manager, c *Config) (map[string]ControllerDef,
mgr.GetClient(),
),
},
// Controllers responsible for cleaning up KongPluginBinding cleanup finalizers.
KongServicePluginBindingFinalizerControllerName: {
Enabled: c.KonnectControllersEnabled,
Controller: konnect.NewKonnectEntityPluginReconciler[configurationv1alpha1.KongService](
c.DevelopmentMode,
mgr.GetClient(),
),
},
KongVaultControllerName: {
Enabled: c.KonnectControllersEnabled,
Controller: konnect.NewKonnectEntityReconciler[configurationv1alpha1.KongVault](
Expand All @@ -568,6 +566,36 @@ func SetupControllers(mgr manager.Manager, c *Config) (map[string]ControllerDef,
konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongSNI](c.KonnectSyncPeriod),
),
},

// Controllers responsible for cleaning up KongPluginBinding cleanup finalizers.
KongServicePluginBindingFinalizerControllerName: {
Enabled: c.KonnectControllersEnabled,
Controller: konnect.NewKonnectEntityPluginReconciler[configurationv1alpha1.KongService](
c.DevelopmentMode,
mgr.GetClient(),
),
},
KongRoutePluginBindingFinalizerControllerName: {
Enabled: c.KonnectControllersEnabled,
Controller: konnect.NewKonnectEntityPluginReconciler[configurationv1alpha1.KongRoute](
c.DevelopmentMode,
mgr.GetClient(),
),
},
KongConsumerPluginBindingFinalizerControllerName: {
Enabled: c.KonnectControllersEnabled,
Controller: konnect.NewKonnectEntityPluginReconciler[configurationv1.KongConsumer](
c.DevelopmentMode,
mgr.GetClient(),
),
},
KongConsumerGroupPluginBindingFinalizerControllerName: {
Enabled: c.KonnectControllersEnabled,
Controller: konnect.NewKonnectEntityPluginReconciler[configurationv1beta1.KongConsumerGroup](
c.DevelopmentMode,
mgr.GetClient(),
),
},
}

// Merge Konnect controllers into the controllers map. This is done this way instead of directly assigning
Expand Down
68 changes: 68 additions & 0 deletions test/envtest/kongplugincleanupfinalizer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import (
"github.com/kong/gateway-operator/pkg/consts"
"github.com/kong/gateway-operator/test/helpers/deploy"

configurationv1 "github.com/kong/kubernetes-configuration/api/configuration/v1"
configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1"
configurationv1beta1 "github.com/kong/kubernetes-configuration/api/configuration/v1beta1"
)

func TestKongPluginFinalizer(t *testing.T) {
Expand All @@ -43,6 +45,8 @@ func TestKongPluginFinalizer(t *testing.T) {
StartReconcilers(ctx, t, mgr, logs,
konnect.NewKonnectEntityPluginReconciler[configurationv1alpha1.KongService](false, mgr.GetClient()),
konnect.NewKonnectEntityPluginReconciler[configurationv1alpha1.KongRoute](false, mgr.GetClient()),
konnect.NewKonnectEntityPluginReconciler[configurationv1.KongConsumer](false, mgr.GetClient()),
konnect.NewKonnectEntityPluginReconciler[configurationv1beta1.KongConsumerGroup](false, mgr.GetClient()),
)

t.Run("KongService", func(t *testing.T) {
Expand Down Expand Up @@ -109,4 +113,68 @@ func TestKongPluginFinalizer(t *testing.T) {
fmt.Sprintf("KongRoute has the %s finalizer set but it shouldn't", consts.CleanupPluginBindingFinalizer),
)
})

t.Run("KongConsumer", func(t *testing.T) {
rateLimitingkongPlugin := deploy.RateLimitingPlugin(t, ctx, clientNamespaced)

wKongConsumer := setupWatch[configurationv1.KongConsumerList](t, ctx, clientWithWatch, client.InNamespace(ns.Name))
kongConsumer := deploy.KongConsumerAttachedToCP(t, ctx, clientNamespaced, "username-1", cp)
kpb := deploy.KongPluginBinding(t, ctx, clientNamespaced,
konnect.NewKongPluginBindingBuilder().
WithControlPlaneRefKonnectNamespaced(cp.Name).
WithPluginRef(rateLimitingkongPlugin.Name).
WithConsumerTarget(kongConsumer.Name).
Build(),
)

_ = watchFor(t, ctx, wKongConsumer, watch.Modified,
func(svc *configurationv1.KongConsumer) bool {
return svc.Name == kongConsumer.Name &&
slices.Contains(svc.GetFinalizers(), consts.CleanupPluginBindingFinalizer)
},
fmt.Sprintf("KongConsumer doesn't have the %s finalizer set", consts.CleanupPluginBindingFinalizer),
)

wKongConsumer = setupWatch[configurationv1.KongConsumerList](t, ctx, clientWithWatch, client.InNamespace(ns.Name))
require.NoError(t, clientNamespaced.Delete(ctx, kpb))
_ = watchFor(t, ctx, wKongConsumer, watch.Modified,
func(svc *configurationv1.KongConsumer) bool {
return svc.Name == kongConsumer.Name &&
!slices.Contains(svc.GetFinalizers(), consts.CleanupPluginBindingFinalizer)
},
fmt.Sprintf("KongConsumer has the %s finalizer set but it shouldn't", consts.CleanupPluginBindingFinalizer),
)
})

t.Run("KongConsumerGroup", func(t *testing.T) {
rateLimitingkongPlugin := deploy.RateLimitingPlugin(t, ctx, clientNamespaced)

wKongConsumerGroup := setupWatch[configurationv1beta1.KongConsumerGroupList](t, ctx, clientWithWatch, client.InNamespace(ns.Name))
kongConsumerGroup := deploy.KongConsumerGroupAttachedToCP(t, ctx, clientNamespaced, cp)
kpb := deploy.KongPluginBinding(t, ctx, clientNamespaced,
konnect.NewKongPluginBindingBuilder().
WithControlPlaneRefKonnectNamespaced(cp.Name).
WithPluginRef(rateLimitingkongPlugin.Name).
WithConsumerGroupTarget(kongConsumerGroup.Name).
Build(),
)

_ = watchFor(t, ctx, wKongConsumerGroup, watch.Modified,
func(svc *configurationv1beta1.KongConsumerGroup) bool {
return svc.Name == kongConsumerGroup.Name &&
slices.Contains(svc.GetFinalizers(), consts.CleanupPluginBindingFinalizer)
},
fmt.Sprintf("KongConsumerGroup doesn't have the %s finalizer set", consts.CleanupPluginBindingFinalizer),
)

wKongConsumerGroup = setupWatch[configurationv1beta1.KongConsumerGroupList](t, ctx, clientWithWatch, client.InNamespace(ns.Name))
require.NoError(t, clientNamespaced.Delete(ctx, kpb))
_ = watchFor(t, ctx, wKongConsumerGroup, watch.Modified,
func(svc *configurationv1beta1.KongConsumerGroup) bool {
return svc.Name == kongConsumerGroup.Name &&
!slices.Contains(svc.GetFinalizers(), consts.CleanupPluginBindingFinalizer)
},
fmt.Sprintf("KongConsumerGroup has the %s finalizer set but it shouldn't", consts.CleanupPluginBindingFinalizer),
)
})
}

0 comments on commit e3cd706

Please sign in to comment.