From df3dc81f4063c961f81eac778a3587427c57b07b Mon Sep 17 00:00:00 2001 From: Dragos Gabriel Ghinea <142506926+dgghinea@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:21:37 +0200 Subject: [PATCH] APIGOV-27047 - add unstructured api option (#65) * APIGOV-27047 - add unstructured api option * APIGOV-27047 - MR issues + typo + fix helm name * APIGOV-00003 - fix makefile --------- Co-authored-by: dfeldick --- README.md | 1 + .../templates/discovery-deployment.yaml | 3 ++ .../templates/traceability-statefulset.yaml | 1 + helm/kong-agents/values.yaml | 1 + pkg/discovery/config/config.go | 22 ++++++---- pkg/discovery/config/config_test.go | 4 ++ pkg/discovery/kong/kongclient.go | 44 ++++++++++++------- pkg/discovery/kong/provisioning.go | 2 +- 8 files changed, 52 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 6591dad..903a562 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ All Kong specific environment variables available are listed below | **KONG_SPEC_LOCALPATH** | The local path that the agent will look in for API definitions | | **KONG_SPEC_URLPATHS** | The URL paths that the agent will query on the gateway service for API definitions | | **KONG_SPEC_DEVPORTALENABLED** | Set to true if the agent should look for spec files in the Kong Dev Portal (default: `false`) | +| **KONG_SPEC_CREATEUNSTRUCTUREDAPI** | Set to true to publish unstructured API if spec is not found (default: `false`) | | | | | Traceability Agent Variables | | | **KONG_LOGS_HTTP_PATH** | The path endpoint that the Traceability agent will listen on (default: `/requestlogs`) | diff --git a/helm/kong-agents/templates/discovery-deployment.yaml b/helm/kong-agents/templates/discovery-deployment.yaml index 6af4e61..d9aabf8 100644 --- a/helm/kong-agents/templates/discovery-deployment.yaml +++ b/helm/kong-agents/templates/discovery-deployment.yaml @@ -104,6 +104,8 @@ spec: - name: KONG_SPEC_FILTER value: "{{ .Values.kong.spec.filter }}" {{- end }} + - name: "KONG_SPEC_CREATEUNSTRUCTUREDAPI" + value: "{{ .Values.kong.spec.createUnstructuredAPI }}" {{- if .Values.kong.admin.auth.apikey.value }} - name: KONG_ADMIN_AUTH_APIKEY_VALUE valueFrom: @@ -151,6 +153,7 @@ spec: "KONG_PROXY_PORTS_HTTPS_DISABLE" "KONG_SPEC_LOCALPATH" "KONG_SPEC_URLPATHS" + "KONG_SPEC_CREATEUNSTRUCTUREDAPI" "KONG_LOGS_HTTP_SERVER_PATH" "KONG_LOGS_HTTP_SERVER_PORT" "STATUS_PORT"))) diff --git a/helm/kong-agents/templates/traceability-statefulset.yaml b/helm/kong-agents/templates/traceability-statefulset.yaml index 47b281a..eab5a29 100644 --- a/helm/kong-agents/templates/traceability-statefulset.yaml +++ b/helm/kong-agents/templates/traceability-statefulset.yaml @@ -97,6 +97,7 @@ spec: "KONG_PROXY_PORTS_HTTPS_DISABLE" "KONG_SPEC_LOCALPATH" "KONG_SPEC_URLPATHS" + "KONG_SPEC_CREATEUNSTRUCTUREDAPI" "KONG_LOGS_HTTP_SERVER_PATH" "KONG_LOGS_HTTP_SERVER_PORT" "STATUS_PORT"))) diff --git a/helm/kong-agents/values.yaml b/helm/kong-agents/values.yaml index 1443ebc..186cd06 100644 --- a/helm/kong-agents/values.yaml +++ b/helm/kong-agents/values.yaml @@ -54,6 +54,7 @@ kong: filter: urlPaths: [] localPath: + createUnstructuredAPI: false logs: http: path: diff --git a/pkg/discovery/config/config.go b/pkg/discovery/config/config.go index d1dca64..9646de8 100644 --- a/pkg/discovery/config/config.go +++ b/pkg/discovery/config/config.go @@ -42,6 +42,7 @@ const ( cfgKongSpecLocalPath = "kong.spec.localPath" cfgKongSpecFilter = "kong.spec.filter" cfgKongSpecDevPortal = "kong.spec.devPortalEnabled" + cfgKongSpecCreateUnstructuredAPI = "kong.spec.createUnstructuredAPI" ) func AddKongProperties(rootProps props) { @@ -65,7 +66,8 @@ func AddKongProperties(rootProps props) { rootProps.AddStringSliceProperty(cfgKongSpecURLPaths, []string{}, "URL paths that the agent will look in for spec files") rootProps.AddStringProperty(cfgKongSpecLocalPath, "", "Local paths where the agent will look for spec files") rootProps.AddStringProperty(cfgKongSpecFilter, "", "SDK Filter format. Empty means filters are ignored.") - rootProps.AddBoolProperty(cfgKongSpecDevPortal, false, "Set to true to enable gathering specs from teh Kong's dev portal.") + rootProps.AddBoolProperty(cfgKongSpecDevPortal, false, "Set to true to enable gathering specs from the Kong's dev portal.") + rootProps.AddBoolProperty(cfgKongSpecCreateUnstructuredAPI, false, "Set to true to publish unstructured API if spec is not found.") } // AgentConfig - represents the config for agent @@ -112,10 +114,11 @@ type KongPortSettingsConfig struct { } type KongSpecConfig struct { - URLPaths []string `config:"urlPaths"` - LocalPath string `config:"localPath"` - DevPortalEnabled bool `config:"devPortalEnabled"` - Filter string `config:"filter"` + URLPaths []string `config:"urlPaths"` + LocalPath string `config:"localPath"` + DevPortalEnabled bool `config:"devPortalEnabled"` + Filter string `config:"filter"` + CreateUnstructuredAPI bool `config:"createUnstructuredAPI"` } type KongACLConfig struct { @@ -267,10 +270,11 @@ func ParseProperties(rootProps props) *KongGatewayConfig { BasePath: rootProps.StringPropertyValue(cfgKongProxyBasePath), }, Spec: KongSpecConfig{ - DevPortalEnabled: rootProps.BoolPropertyValue(cfgKongSpecDevPortal), - URLPaths: rootProps.StringSlicePropertyValue(cfgKongSpecURLPaths), - LocalPath: rootProps.StringPropertyValue(cfgKongSpecLocalPath), - Filter: rootProps.StringPropertyValue(cfgKongSpecFilter), + DevPortalEnabled: rootProps.BoolPropertyValue(cfgKongSpecDevPortal), + URLPaths: rootProps.StringSlicePropertyValue(cfgKongSpecURLPaths), + LocalPath: rootProps.StringPropertyValue(cfgKongSpecLocalPath), + Filter: rootProps.StringPropertyValue(cfgKongSpecFilter), + CreateUnstructuredAPI: rootProps.BoolPropertyValue(cfgKongSpecCreateUnstructuredAPI), }, } } diff --git a/pkg/discovery/config/config_test.go b/pkg/discovery/config/config_test.go index 99c6d4c..ea6c98e 100644 --- a/pkg/discovery/config/config_test.go +++ b/pkg/discovery/config/config_test.go @@ -136,6 +136,7 @@ func TestKongProperties(t *testing.T) { assert.Contains(t, newProps.props, cfgKongSpecLocalPath) assert.Contains(t, newProps.props, cfgKongSpecFilter) assert.Contains(t, newProps.props, cfgKongSpecDevPortal) + assert.Contains(t, newProps.props, cfgKongSpecCreateUnstructuredAPI) // validate defaults cfg := ParseProperties(newProps) @@ -155,6 +156,7 @@ func TestKongProperties(t *testing.T) { assert.Equal(t, "", cfg.Spec.LocalPath) assert.Equal(t, "", cfg.Spec.Filter) assert.Equal(t, false, cfg.Spec.DevPortalEnabled) + assert.Equal(t, false, cfg.Spec.CreateUnstructuredAPI) // validate changed values newProps.props[cfgKongACLDisable] = propData{"bool", "", true} @@ -171,6 +173,7 @@ func TestKongProperties(t *testing.T) { newProps.props[cfgKongSpecLocalPath] = propData{"string", "", "/path/to/specs"} newProps.props[cfgKongSpecFilter] = propData{"string", "", "tag_filter"} newProps.props[cfgKongSpecDevPortal] = propData{"bool", "", true} + newProps.props[cfgKongSpecCreateUnstructuredAPI] = propData{"bool", "", true} cfg = ParseProperties(newProps) assert.Equal(t, true, cfg.ACL.Disable) assert.Equal(t, "http://host:port/path", cfg.Admin.Url) @@ -188,6 +191,7 @@ func TestKongProperties(t *testing.T) { assert.Equal(t, "/path/to/specs", cfg.Spec.LocalPath) assert.Equal(t, "tag_filter", cfg.Spec.Filter) assert.Equal(t, true, cfg.Spec.DevPortalEnabled) + assert.Equal(t, true, cfg.Spec.CreateUnstructuredAPI) // validate no port configured when port type disabled newProps.props[cfgKongProxyPortHttpDisable] = propData{"bool", "", true} diff --git a/pkg/discovery/kong/kongclient.go b/pkg/discovery/kong/kongclient.go index d2d0ebd..57b3074 100644 --- a/pkg/discovery/kong/kongclient.go +++ b/pkg/discovery/kong/kongclient.go @@ -22,6 +22,7 @@ import ( ) const tagPrefix = "spec_local_" +const unstructuredSpec = "Unstructured API Example" type KongAPIClient interface { // Provisioning @@ -56,13 +57,14 @@ type KongServiceSpec struct { type KongClient struct { *klib.Client - logger log.FieldLogger - baseClient DoRequest - kongAdminEndpoint string - specURLPaths []string - specLocalPath string - devPortalEnabled bool - clientTimeout time.Duration + logger log.FieldLogger + baseClient DoRequest + kongAdminEndpoint string + specURLPaths []string + specLocalPath string + devPortalEnabled bool + createUnstructuredAPI bool + clientTimeout time.Duration } func NewKongClient(baseClient *http.Client, kongConfig *config.KongGatewayConfig) (*KongClient, error) { @@ -90,14 +92,15 @@ func NewKongClient(baseClient *http.Client, kongConfig *config.KongGatewayConfig } return &KongClient{ - Client: baseKongClient, - logger: log.NewFieldLogger().WithComponent("KongClient").WithPackage("kong"), - baseClient: baseClient, - kongAdminEndpoint: kongEndpoint, - specURLPaths: kongConfig.Spec.URLPaths, - specLocalPath: kongConfig.Spec.LocalPath, - devPortalEnabled: kongConfig.Spec.DevPortalEnabled, - clientTimeout: 60 * time.Second, + Client: baseKongClient, + logger: log.NewFieldLogger().WithComponent("KongClient").WithPackage("kong"), + baseClient: baseClient, + kongAdminEndpoint: kongEndpoint, + specURLPaths: kongConfig.Spec.URLPaths, + specLocalPath: kongConfig.Spec.LocalPath, + devPortalEnabled: kongConfig.Spec.DevPortalEnabled, + createUnstructuredAPI: kongConfig.Spec.CreateUnstructuredAPI, + clientTimeout: 60 * time.Second, }, nil } @@ -132,7 +135,16 @@ func (k KongClient) GetSpecForService(ctx context.Context, service *klib.Service backendURL = backendURL + *service.Path } - return k.getSpecFromBackend(ctx, backendURL) + spec, err := k.getSpecFromBackend(ctx, backendURL) + if spec == nil && err == nil && k.createUnstructuredAPI { + return k.getUnstructuredSpec(ctx), nil + } + return spec, err +} + +func (k KongClient) getUnstructuredSpec(ctx context.Context) []byte { + k.logger.Info("Adding unstructured API to services which had no spec associated") + return []byte(unstructuredSpec) } func (k KongClient) getSpecFromLocal(ctx context.Context, service *klib.Service) ([]byte, error) { diff --git a/pkg/discovery/kong/provisioning.go b/pkg/discovery/kong/provisioning.go index 1239b72..4a3ee79 100644 --- a/pkg/discovery/kong/provisioning.go +++ b/pkg/discovery/kong/provisioning.go @@ -118,7 +118,7 @@ func (k KongClient) CreateOauth2(ctx context.Context, consumerID string, oauth2 func (k KongClient) CreateAuthKey(ctx context.Context, consumerID string, keyAuth *klib.KeyAuth) (*klib.KeyAuth, error) { keyAuth, err := k.KeyAuths.Create(ctx, &consumerID, keyAuth) if err != nil { - k.logger.Errorf("failed to create oauth2 credential for consumerID %s. Reason: %w", consumerID, err) + k.logger.Errorf("failed to create API Key credential for consumerID %s. Reason: %w", consumerID, err) return nil, err } return keyAuth, nil