Fix(custom entity): Generate multiple entities if KongCustomEntity at…

…tached to multiple foreign entities (#6280)

* generate multiple entities if KCE attached to multiple foreign entities

* Add unit tests and integration tests for multiple custom entities

* update changelog

* update compare custom entity

* move custom entities related code

* address comments

CHANGELOG.md

@@ -100,6 +100,9 @@ Adding a new version? You'll need three changes:
 - Services using `Secret`s containing the same certificate as client certificates
   by annotation `konghq.com/client-cert` can be correctly translated.
   [#6228](https://github.com/Kong/kubernetes-ingress-controller/pull/6228)
+- Generate one entity for each attached foreign entity if a `KongCustomEntity`
+  resource is attached to multiple foreign Kong entities.
+  [#6280](https://github.com/Kong/kubernetes-ingress-controller/pull/6280)
 
 ## 3.2.2
 

internal/dataplane/kongstate/customentity.go

@@ -2,11 +2,18 @@ package kongstate
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"sort"
 
+	"github.com/go-logr/logr"
 	"github.com/kong/go-kong/kong"
 	"github.com/kong/go-kong/kong/custom"
+	"github.com/samber/lo"
 
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/failures"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 )
 
@@ -130,6 +137,8 @@ type CustomEntity struct {
 	custom.Object
 	// K8sKongCustomEntity refers to the KongCustomEntity resource that translate to it.
 	K8sKongCustomEntity *kongv1alpha1.KongCustomEntity
+	// ForeignEntityIDs stores the IDs of the foreign Kong entities attached to the entity.
+	ForeignEntityIDs map[kong.EntityType]string
 }
 
 // SchemaGetter is the interface to fetch the schema of a Kong entity by its type.
@@ -138,6 +147,96 @@ type SchemaGetter interface {
 	Get(ctx context.Context, entityType string) (kong.Schema, error)
 }
 
+type entityForeignFieldValue struct {
+	fieldName         string
+	foreignEntityType kong.EntityType
+	foreignEntityID   string
+}
+
+// FillCustomEntities fills custom entities in KongState.
+func (ks *KongState) FillCustomEntities(
+	logger logr.Logger,
+	s store.Storer,
+	failuresCollector *failures.ResourceFailuresCollector,
+	schemaGetter SchemaGetter,
+	workspace string,
+) {
+	entities := s.ListKongCustomEntities()
+	if len(entities) == 0 {
+		return
+	}
+	logger = logger.WithName("fillCustomEntities")
+
+	if ks.CustomEntities == nil {
+		ks.CustomEntities = map[string]*KongCustomEntityCollection{}
+	}
+	// Fetch relations between plugins and services/routes/consumers and store the pointer to translated Kong entities.
+	// Used for fetching entity referred by a custom entity and fill the ID of referred entity.
+	pluginRels := ks.getPluginRelatedEntitiesRef(s, logger)
+
+	for _, entity := range entities {
+		// reject the custom entity if its type is in "known" entity types that are already processed.
+		if IsKnownEntityType(entity.Spec.EntityType) {
+			failuresCollector.PushResourceFailure(
+				fmt.Sprintf("cannot use known entity type %s in custom entity", entity.Spec.EntityType),
+				entity,
+			)
+			continue
+		}
+		// Fetch the entity schema.
+		schema, err := ks.fetchEntitySchema(schemaGetter, entity.Spec.EntityType)
+		if err != nil {
+			failuresCollector.PushResourceFailure(
+				fmt.Sprintf("failed to fetch entity schema for entity type %s: %v", entity.Spec.EntityType, err),
+				entity,
+			)
+			continue
+		}
+
+		// Fill the "foreign" fields if the entity has such fields referencing services/routes/consumers.
+		// First Find out possible foreign field combinations attached to the KCE resource.
+		foreignFieldCombinations := findCustomEntityForeignFields(logger, entity, schema, pluginRels, workspace)
+		// generate Kong entities from the fields in the KCE itself and attached foreign entities.
+		generatedEntities, err := generateCustomEntities(entity, foreignFieldCombinations)
+		if err != nil {
+			failuresCollector.PushResourceFailure(fmt.Sprintf("failed to generate entities from itself and attach foreign entities: %v", err), entity)
+			continue
+		}
+		for _, generatedEntity := range generatedEntities {
+			ks.addCustomEntity(entity.Spec.EntityType, schema, generatedEntity)
+		}
+	}
+
+	ks.sortCustomEntities()
+}
+
+// addCustomEntity adds a custom entity into the collection of its type.
+func (ks *KongState) addCustomEntity(entityType string, schema EntitySchema, e CustomEntity) {
+	// Put the entity into the custom collection to store the entities of its type.
+	if _, ok := ks.CustomEntities[entityType]; !ok {
+		ks.CustomEntities[entityType] = &KongCustomEntityCollection{
+			Schema: schema,
+		}
+	}
+	collection := ks.CustomEntities[entityType]
+	collection.Entities = append(collection.Entities, e)
+}
+
+// fetchEntitySchema fetches schema of an entity by its type and stores the schema in its custom entity collection
+// as a cache to avoid excessive calling of Kong admin APIs.
+func (ks *KongState) fetchEntitySchema(schemaGetter SchemaGetter, entityType string) (EntitySchema, error) {
+	collection, ok := ks.CustomEntities[entityType]
+	if ok {
+		return collection.Schema, nil
+	}
+	// Use `context.Background()` here because `BuildKongConfig` does not provide a context.
+	schema, err := schemaGetter.Get(context.Background(), entityType)
+	if err != nil {
+		return EntitySchema{}, err
+	}
+	return ExtractEntityFieldDefinitions(schema), nil
+}
+
 // sortCustomEntities sorts the custom entities of each type.
 // Since there may not be a consistent field to identify an entity, here we sort them by the k8s namespace/name.
 func (ks *KongState) sortCustomEntities() {
@@ -146,14 +245,192 @@ func (ks *KongState) sortCustomEntities() {
 			e1 := collection.Entities[i]
 			e2 := collection.Entities[j]
 			// Compare namespace first.
-			if e1.K8sKongCustomEntity.Namespace < e2.K8sKongCustomEntity.Namespace {
-				return true
+			if e1.K8sKongCustomEntity.Namespace != e2.K8sKongCustomEntity.Namespace {
+				return e1.K8sKongCustomEntity.Namespace < e2.K8sKongCustomEntity.Namespace
+			}
+			// If namespace are the same, compare names.
+			if e1.K8sKongCustomEntity.Name != e2.K8sKongCustomEntity.Name {
+				return e1.K8sKongCustomEntity.Name < e2.K8sKongCustomEntity.Name
 			}
-			if e1.K8sKongCustomEntity.Namespace > e2.K8sKongCustomEntity.Namespace {
-				return false
+			// Namespace and name are all the same.
+			// This means the two entities are generated from the same KCE resource but attached to different foreign entities.
+			// So we need to compare foreign entities.
+			if e1.ForeignEntityIDs != nil && e2.ForeignEntityIDs != nil {
+				// Compare IDs of attached entities in services, routes, consumers order.
+				foreignEntityTypeList := []kong.EntityType{
+					kong.EntityTypeServices,
+					kong.EntityTypeRoutes,
+					kong.EntityTypeConsumers,
+				}
+				for _, t := range foreignEntityTypeList {
+					if e1.ForeignEntityIDs[t] != e2.ForeignEntityIDs[t] {
+						return e1.ForeignEntityIDs[t] < e2.ForeignEntityIDs[t]
+					}
+				}
 			}
-			// If namespace are the same, compare name.
-			return e1.K8sKongCustomEntity.Name < e2.K8sKongCustomEntity.Name
+			// Should not reach here when k8s namespace/names are the same, and foreign entities are also the same.
+			// This means we generated two Kong entities from one KCE (and attached to the same foreign entities if any).
+			return true
 		})
 	}
 }
+
+func findCustomEntityRelatedPlugin(k8sEntity *kongv1alpha1.KongCustomEntity) (string, bool) {
+	// Find referred entity via the plugin in its spec.parentRef.
+	// Then we can fetch the referred service/route/consumer from the reference relations of the plugin.
+	parentRef := k8sEntity.Spec.ParentRef
+	var namespace string
+	// Abort if the parentRef is empty or does not refer to a plugin.
+	if parentRef == nil ||
+		(parentRef.Group == nil || *parentRef.Group != kongv1alpha1.GroupVersion.Group) {
+		return "", false
+	}
+	if parentRef.Kind == nil || (*parentRef.Kind != "KongPlugin" && *parentRef.Kind != "KongClusterPlugin") {
+		return "", false
+	}
+	// Extract the plugin key to get the plugin relations.
+	if parentRef.Namespace == nil || *parentRef.Namespace == "" {
+		namespace = k8sEntity.Namespace
+	} else {
+		namespace = *parentRef.Namespace
+	}
+	return namespace + ":" + parentRef.Name, true
+}
+
+func findCustomEntityForeignFields(
+	logger logr.Logger,
+	k8sEntity *kongv1alpha1.KongCustomEntity,
+	schema EntitySchema,
+	pluginRelEntities PluginRelatedEntitiesRefs,
+	workspace string,
+) [][]entityForeignFieldValue {
+	pluginKey, ok := findCustomEntityRelatedPlugin(k8sEntity)
+	if !ok {
+		return nil
+	}
+	// Get the relations with other entities of the plugin.
+	rels, ok := pluginRelEntities.RelatedEntities[pluginKey]
+	if !ok {
+		return nil
+	}
+
+	var (
+		foreignRelations      util.ForeignRelations
+		foreignServiceFields  []string
+		foreignRouteFields    []string
+		foreignConsumerFields []string
+	)
+
+	ret := [][]entityForeignFieldValue{}
+	for fieldName, field := range schema.Fields {
+		if field.Type != EntityFieldTypeForeign {
+			continue
+		}
+		switch field.Reference {
+		case string(kong.EntityTypeServices):
+			foreignServiceFields = append(foreignServiceFields, fieldName)
+			foreignRelations.Service = getServiceIDFromPluginRels(logger, rels, pluginRelEntities.RouteAttachedService, workspace)
+		case string(kong.EntityTypeRoutes):
+			foreignRouteFields = append(foreignRouteFields, fieldName)
+			foreignRelations.Route = lo.FilterMap(rels.Routes, func(r *Route, _ int) (string, bool) {
+				if err := r.FillID(workspace); err != nil {
+					return "", false
+				}
+				return *r.ID, true
+			})
+		case string(kong.EntityTypeConsumers):
+			foreignConsumerFields = append(foreignConsumerFields, fieldName)
+			foreignRelations.Consumer = lo.FilterMap(rels.Consumers, func(c *Consumer, _ int) (string, bool) {
+				if err := c.FillID(workspace); err != nil {
+					return "", false
+				}
+				return *c.ID, true
+			})
+		} // end of switch
+	}
+
+	// TODO: Here we inherited the logic of generating combinations of attached foreign entities for plugins.
+	// Actually there are no such case that a custom entity required multiple "foreign" fields in current Kong plugins.
+	// So it is still uncertain how to generate foreign field combinations for custom entities.
+	for _, combination := range foreignRelations.GetCombinations() {
+		foreignFieldValues := []entityForeignFieldValue{}
+		for _, fieldName := range foreignServiceFields {
+			foreignFieldValues = append(foreignFieldValues, entityForeignFieldValue{
+				fieldName:         fieldName,
+				foreignEntityType: kong.EntityTypeServices,
+				foreignEntityID:   combination.Service,
+			})
+		}
+		for _, fieldName := range foreignRouteFields {
+			foreignFieldValues = append(foreignFieldValues, entityForeignFieldValue{
+				fieldName:         fieldName,
+				foreignEntityType: kong.EntityTypeRoutes,
+				foreignEntityID:   combination.Route,
+			})
+		}
+		for _, fieldName := range foreignConsumerFields {
+			foreignFieldValues = append(foreignFieldValues, entityForeignFieldValue{
+				fieldName:         fieldName,
+				foreignEntityType: kong.EntityTypeConsumers,
+				foreignEntityID:   combination.Consumer,
+			})
+		}
+		ret = append(ret, foreignFieldValues)
+	}
+
+	return ret
+}
+
+// generateCustomEntities generates Kong entities from KongCustomEntity resource and combinations of attached foreign entities.
+// If the KCE is attached to any foreign entities, it generates one entity per combination of foreign entities.
+// If the KCE is not attached, generate one entity for itself.
+func generateCustomEntities(
+	entity *kongv1alpha1.KongCustomEntity,
+	foreignFieldCombinations [][]entityForeignFieldValue,
+) ([]CustomEntity, error) {
+	copyEntityFields := func() (map[string]any, error) {
+		// Unmarshal the fields of the entity to have a fresh copy for each combination as we may modify them.
+		fields := map[string]any{}
+		if err := json.Unmarshal(entity.Spec.Fields.Raw, &fields); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal entity fields: %w", err)
+		}
+		return fields, nil
+	}
+	// If there are any foreign fields, generate one entity per each foreign entity combination.
+	if len(foreignFieldCombinations) > 0 {
+		var customEntities []CustomEntity
+		for _, combination := range foreignFieldCombinations {
+			entityFields, err := copyEntityFields()
+			if err != nil {
+				return nil, err
+			}
+			generatedEntity := CustomEntity{
+				K8sKongCustomEntity: entity,
+				ForeignEntityIDs:    make(map[kong.EntityType]string),
+				Object:              entityFields,
+			}
+			// Fill the fields referring to foreign entities.
+			for _, foreignField := range combination {
+				entityFields[foreignField.fieldName] = map[string]any{
+					"id": foreignField.foreignEntityID,
+				}
+				// Save the referred foreign entity IDs for sorting.
+				generatedEntity.ForeignEntityIDs[foreignField.foreignEntityType] = foreignField.foreignEntityID
+			}
+			customEntities = append(customEntities, generatedEntity)
+		}
+		return customEntities, nil
+	}
+
+	// Otherwise (no foreign fields), generate a single entity.
+	entityFields, err := copyEntityFields()
+	if err != nil {
+		return nil, err
+	}
+	return []CustomEntity{
+		{
+			K8sKongCustomEntity: entity,
+			Object:              entityFields,
+		},
+	}, nil
+}