diff --git a/.gitignore b/.gitignore index 7de5e89..afd972e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,11 +21,15 @@ bin/ .run/ *.log -/kong_discovery_agent.yml -/kong_traceability_agent.yml +**/kong_discovery_agent.yml +**/kong_traceability_agent.yml specs/ +data/ +logs/ secret.yaml overrides.yaml configmap.yaml + +**/__debug_bin* diff --git a/build/traceability/kong_traceability_agent.yml b/build/traceability/kong_traceability_agent.yml index 0285244..5d8d08d 100644 --- a/build/traceability/kong_traceability_agent.yml +++ b/build/traceability/kong_traceability_agent.yml @@ -7,9 +7,9 @@ kong_traceability_agent: apikey: header: ${KONG_ADMIN_AUTH_APIKEY_HEADER} value: ${KONG_ADMIN_AUTH_APIKEY_VALUE} - http_log_plugin_config: - path: ${KONG_LOGS_HTTP_SERVER_PATH} - port: ${KONG_LOGS_HTTP_SERVER_PORT} + httpLogPlugin: + path: ${KONG_HTTPLOGPLUGIN_PATH} + port: ${KONG_HTTPLOGPLUGIN_PORT} # Settings for connecting to Amplify Central central: url: ${CENTRAL_URL:https://apicentral.axway.com} @@ -21,7 +21,6 @@ kong_traceability_agent: agentName: ${CENTRAL_AGENTNAME:""} platformURL: ${CENTRAL_PLATFORMURL:https://platform.axway.com} reportActivityFrequency: ${CENTRAL_REPORTACTIVITYFREQUENCY:5m} - versionChecker: ${CENTRAL_VERSIONCHECKER:true} usageReporting: publish: ${CENTRAL_USAGEREPORTING_PUBLISH} publishMetric: ${CENTRAL_USAGEREPORTING_PUBLISHMETRIC} @@ -49,8 +48,25 @@ kong_traceability_agent: port: ${CENTRAL_GRPC_PORT} proxyUrl: ${CENTRAL_PROXYURL:""} clientTimeout: ${CENTRAL_CLIENTTIMEOUT:60s} + agentFeatures: + persistCache: ${AGENTFEATURES_PERSISTCACHE} + marketplaceProvisioning: ${AGENTFEATURES_MARKETPLACEPROVISIONING} + versionChecker: ${AGENTFEATURES_VERSIONCHECKER} + processSystemSignals: ${AGENTFEATURES_PROCESSSYSTEMSIGNALS} + connectToCentral: ${AGENTFEATURES_CONNECTTOCENTRAL} + status: + port: ${STATUS_PORT:8990} + healthCheckPeriod: ${STATUS_HEALTHCHECKPERIOD:3m} + healthCheckInterval: ${STATUS_HEALTHCHECKINTERVAL:30s} + log: + level: ${LOG_LEVEL:info} + format: ${LOG_FORMAT:json} + output: ${LOG_OUTPUT:stdout} + file: + name: ${LOG_FILE_NAME:traceability_agent.log} + path: ${LOG_FILE_PATH:logs} -# Condor Ingestion service +# Send output to Central Database output.traceability: enabled: true hosts: ${TRACEABILITY_HOST:ingestion.datasearch.axway.com:5044} diff --git a/pkg/beater/custom_beater.go b/pkg/beater/custom_beater.go index 0a75495..80d99d6 100644 --- a/pkg/beater/custom_beater.go +++ b/pkg/beater/custom_beater.go @@ -1,97 +1,107 @@ package beater import ( + "context" "fmt" - "io/ioutil" - "log" + "io" "net/http" - traceabilityconfig "github.com/Axway/agents-kong/pkg/config/traceability" - - "github.com/Axway/agents-kong/pkg/processor" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/google/uuid" agentErrors "github.com/Axway/agent-sdk/pkg/util/errors" hc "github.com/Axway/agent-sdk/pkg/util/healthcheck" - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/Axway/agent-sdk/pkg/util/log" + + config "github.com/Axway/agents-kong/pkg/config/traceability" + "github.com/Axway/agents-kong/pkg/processor" ) -type customLogBeater struct { +type httpLogBeater struct { done chan struct{} eventProcessor *processor.EventProcessor client beat.Client eventChannel chan string + logger log.FieldLogger } // New creates an instance of kong_traceability_agent. func New(*beat.Beat, *common.Config) (beat.Beater, error) { - bt := &customLogBeater{ + b := &httpLogBeater{ done: make(chan struct{}), eventChannel: make(chan string), + logger: log.NewFieldLogger().WithComponent("httpLogBeater").WithPackage("beater"), } - bt.eventProcessor = processor.NewEventProcessor() + b.eventProcessor = processor.NewEventProcessor() // Validate that all necessary services are up and running. If not, return error if hc.RunChecks() != hc.OK { + b.logger.Error("not all services are running") return nil, agentErrors.ErrInitServicesNotReady } - return bt, nil + return b, nil } // Run starts kong_traceability_agent. -func (bt *customLogBeater) Run(b *beat.Beat) error { - logp.Info("kong_traceability_agent is running! Hit CTRL-C to stop it.") +func (b *httpLogBeater) Run(beater *beat.Beat) error { + b.logger.Info("kong_traceability_agent is running! Hit CTRL-C to stop it.") var err error - bt.client, err = b.Publisher.Connect() + b.client, err = beater.Publisher.Connect() if err != nil { return err } - http.HandleFunc(traceabilityconfig.GetAgentConfig().HttpLogPluginConfig.Path, + http.HandleFunc(config.GetAgentConfig().HttpLogPluginConfig.Path, func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { + b.logger.Trace("received a non post request") w.WriteHeader(http.StatusMethodNotAllowed) return } - body, err := ioutil.ReadAll(r.Body) + ctx := context.WithValue(context.Background(), processor.CtxTransactionID, uuid.NewString()) + logData, err := io.ReadAll(r.Body) defer r.Body.Close() if err != nil { - logp.Error(fmt.Errorf("error while reading request body: %s", err)) + b.logger.WithError(err).Error("reading request body") } w.WriteHeader(200) - bt.processAndDispatchEvent(string(body)) - }) + go b.processAndDispatchEvent(ctx, logData) + }, + ) /* Start a new HTTP server in a separate Go routine that will be the target for the HTTP Log plugin. It should write events it gets to eventChannel */ go func() { - if err := http.ListenAndServe(fmt.Sprintf(":%d", traceabilityconfig.GetAgentConfig().HttpLogPluginConfig.Port), - nil); err != nil { - log.Fatalf("Unable to start the HTTP Server: %s", err) + err := http.ListenAndServe(fmt.Sprintf(":%d", config.GetAgentConfig().HttpLogPluginConfig.Port), nil) + if err != nil { + b.logger.WithError(err).Fatalf("unable to start the HTTP Server") } - fmt.Printf("Started HTTP server on port %d to receive request logs", traceabilityconfig.GetAgentConfig().HttpLogPluginConfig.Port) + b.logger.WithField("port", config.GetAgentConfig().HttpLogPluginConfig.Port).WithField("path", config.GetAgentConfig().HttpLogPluginConfig.Path).Info("started HTTP server") }() - <-bt.done + <-b.done return nil } // Stop stops kong_traceability_agent. -func (bt *customLogBeater) Stop() { +func (bt *httpLogBeater) Stop() { bt.client.Close() close(bt.done) } -func (bt *customLogBeater) processAndDispatchEvent(logEvent string) { - eventsToPublish := bt.eventProcessor.ProcessRaw([]byte(logEvent)) +func (bt *httpLogBeater) processAndDispatchEvent(ctx context.Context, logData []byte) { + log := log.UpdateLoggerWithContext(ctx, bt.logger) + log.WithField("data", logData).Trace("handling log data") + eventsToPublish := bt.eventProcessor.ProcessRaw(ctx, logData) if eventsToPublish != nil { + log.Trace("finished handling data") bt.client.PublishAll(eventsToPublish) } } diff --git a/pkg/cmd/discovery/cmd.go b/pkg/cmd/discovery/cmd.go index 0bf4746..21f3ef8 100644 --- a/pkg/cmd/discovery/cmd.go +++ b/pkg/cmd/discovery/cmd.go @@ -6,6 +6,7 @@ import ( corecmd "github.com/Axway/agent-sdk/pkg/cmd" corecfg "github.com/Axway/agent-sdk/pkg/config" "github.com/Axway/agent-sdk/pkg/util/log" + config "github.com/Axway/agents-kong/pkg/config/discovery" "github.com/Axway/agents-kong/pkg/gateway" ) @@ -13,18 +14,6 @@ import ( var DiscoveryCmd corecmd.AgentRootCmd var agentConfig config.AgentConfig -const ( - cfgKongAdminURL = "kong.admin.url" - cfgKongAdminAPIKey = "kong.admin.auth.apikey.value" - cfgKongAdminAPIKeyHeader = "kong.admin.auth.apikey.header" - cfgKongProxyHost = "kong.proxy.host" - cfgKongProxyPortHttp = "kong.proxy.port.http" - cfgKongProxyPortHttps = "kong.proxy.port.https" - cfgKongSpecURLPaths = "kong.spec.urlPaths" - cfgKongSpecLocalPath = "kong.spec.localPath" - cfgKongDevPortalEnabled = "kong.spec.devPortalEnabled" -) - func init() { // Create new root command with callbacks to initialize the agent config and command execution. // The first parameter identifies the name of the yaml file that agent will look for to load the config @@ -38,15 +27,7 @@ func init() { // Get the root command properties and bind the config property in YAML definition rootProps := DiscoveryCmd.GetProperties() - rootProps.AddStringProperty(cfgKongAdminURL, "", "The Kong admin endpoint") - rootProps.AddStringProperty(cfgKongAdminAPIKey, "", "API Key value to authenticate with Kong Gateway") - rootProps.AddStringProperty(cfgKongAdminAPIKeyHeader, "", "API Key header to authenticate with Kong Gateway") - rootProps.AddStringProperty(cfgKongProxyHost, "", "The Kong proxy endpoint") - rootProps.AddIntProperty(cfgKongProxyPortHttp, 80, "The Kong proxy http port") - rootProps.AddIntProperty(cfgKongProxyPortHttps, 443, "The Kong proxy https port") - rootProps.AddStringSliceProperty(cfgKongSpecURLPaths, []string{}, "URL paths that the agent will look in for spec files") - rootProps.AddStringProperty(cfgKongSpecLocalPath, "", "Local paths where the agent will look for spec files") - rootProps.AddBoolProperty(cfgKongDevPortalEnabled, false, "Dev Portal is used to download spec files") + config.AddKongProperties(rootProps) } // Callback that agent will call to process the execution @@ -82,34 +63,9 @@ func run() error { func initConfig(centralConfig corecfg.CentralConfig) (interface{}, error) { rootProps := DiscoveryCmd.GetProperties() - // Parse the config from bound properties and setup gateway config - gatewayConfig := &config.KongGatewayConfig{ - Admin: config.KongAdminConfig{ - URL: rootProps.StringPropertyValue(cfgKongAdminURL), - Auth: config.KongAdminAuthConfig{ - APIKey: config.KongAdminAuthAPIKeyConfig{ - Value: rootProps.StringPropertyValue(cfgKongAdminAPIKey), - Header: rootProps.StringPropertyValue(cfgKongAdminAPIKeyHeader), - }, - }, - }, - Proxy: config.KongProxyConfig{ - Host: rootProps.StringPropertyValue(cfgKongProxyHost), - Port: config.KongProxyPortConfig{ - HTTP: rootProps.IntPropertyValue(cfgKongProxyPortHttp), - HTTPS: rootProps.IntPropertyValue(cfgKongProxyPortHttps), - }, - }, - Spec: config.KongSpecConfig{ - URLPaths: rootProps.StringSlicePropertyValue(cfgKongSpecURLPaths), - LocalPath: rootProps.StringPropertyValue(cfgKongSpecLocalPath), - DevPortalEnabled: rootProps.BoolPropertyValue(cfgKongDevPortalEnabled), - }, - } - agentConfig = config.AgentConfig{ CentralCfg: centralConfig, - KongGatewayCfg: gatewayConfig, + KongGatewayCfg: config.ParseProperties(rootProps), } return agentConfig, nil } diff --git a/pkg/cmd/traceability/cmd.go b/pkg/cmd/traceability/cmd.go index 0ca8e64..4d54f66 100644 --- a/pkg/cmd/traceability/cmd.go +++ b/pkg/cmd/traceability/cmd.go @@ -1,12 +1,14 @@ package traceability import ( + libcmd "github.com/elastic/beats/v7/libbeat/cmd" + "github.com/elastic/beats/v7/libbeat/cmd/instance" + corecmd "github.com/Axway/agent-sdk/pkg/cmd" corecfg "github.com/Axway/agent-sdk/pkg/config" + "github.com/Axway/agents-kong/pkg/beater" config "github.com/Axway/agents-kong/pkg/config/traceability" - libcmd "github.com/elastic/beats/v7/libbeat/cmd" - "github.com/elastic/beats/v7/libbeat/cmd/instance" ) var TraceCmd corecmd.AgentRootCmd @@ -15,8 +17,9 @@ var beatCmd *libcmd.BeatsRootCmd func init() { name := "kong_traceability_agent" settings := instance.Settings{ - Name: name, - HasDashboards: true, + Name: name, + HasDashboards: true, + ConfigOverrides: corecfg.LogConfigOverrides(), } beatCmd = libcmd.GenRootCmdWithSettings(beater.New, settings) @@ -33,8 +36,7 @@ func init() { ) rootProps := TraceCmd.GetProperties() - rootProps.AddStringProperty("http_log_plugin_config.path", "/requestlogs", "Path on which the HTTP Log plugin sends request logs") - rootProps.AddIntProperty("http_log_plugin_config.port", 9000, "Port that listens for request logs from HTTP Log plugin") + config.AddKongProperties(rootProps) } func run() error { @@ -47,14 +49,9 @@ func initConfig(centralConfig corecfg.CentralConfig) (interface{}, error) { rootProps := TraceCmd.GetProperties() - httpLogPluginConfig := &config.HttpLogPluginConfig{ - Port: rootProps.IntPropertyValue("http_log_plugin_config.port"), - Path: rootProps.StringPropertyValue("http_log_plugin_config.path"), - } - agentConfig := &config.AgentConfig{ CentralCfg: centralConfig, - HttpLogPluginConfig: httpLogPluginConfig, + HttpLogPluginConfig: config.ParseProperties(rootProps), } config.SetAgentConfig(agentConfig) diff --git a/pkg/config/discovery/config.go b/pkg/config/discovery/config.go index a7d1996..03b8045 100644 --- a/pkg/config/discovery/config.go +++ b/pkg/config/discovery/config.go @@ -3,9 +3,32 @@ package config import ( "fmt" + "github.com/Axway/agent-sdk/pkg/cmd/properties" corecfg "github.com/Axway/agent-sdk/pkg/config" ) +const ( + cfgKongAdminURL = "kong.admin.url" + cfgKongAdminAPIKey = "kong.admin.auth.apikey.value" + cfgKongAdminAPIKeyHeader = "kong.admin.auth.apikey.header" + cfgKongProxyHost = "kong.proxy.host" + cfgKongProxyPortHttp = "kong.proxy.port.http" + cfgKongProxyPortHttps = "kong.proxy.port.https" + cfgKongSpecURLPaths = "kong.spec.urlPaths" + cfgKongSpecLocalPath = "kong.spec.localPath" +) + +func AddKongProperties(rootProps properties.Properties) { + rootProps.AddStringProperty(cfgKongAdminURL, "", "The Kong admin endpoint") + rootProps.AddStringProperty(cfgKongAdminAPIKey, "", "API Key value to authenticate with Kong Gateway") + rootProps.AddStringProperty(cfgKongAdminAPIKeyHeader, "", "API Key header to authenticate with Kong Gateway") + rootProps.AddStringProperty(cfgKongProxyHost, "", "The Kong proxy endpoint") + rootProps.AddIntProperty(cfgKongProxyPortHttp, 80, "The Kong proxy http port") + rootProps.AddIntProperty(cfgKongProxyPortHttps, 443, "The Kong proxy https port") + rootProps.AddStringSliceProperty(cfgKongSpecURLPaths, []string{}, "URL paths that the agent will look in for spec files") + rootProps.AddStringProperty(cfgKongSpecLocalPath, "", "Local paths where the agent will look for spec files") +} + // AgentConfig - represents the config for agent type AgentConfig struct { CentralCfg corecfg.CentralConfig `config:"central"` @@ -63,3 +86,29 @@ func (c *KongGatewayConfig) ValidateCfg() (err error) { } return } + +func ParseProperties(rootProps properties.Properties) *KongGatewayConfig { + // Parse the config from bound properties and setup gateway config + return &KongGatewayConfig{ + Admin: KongAdminConfig{ + URL: rootProps.StringPropertyValue(cfgKongAdminURL), + Auth: KongAdminAuthConfig{ + APIKey: KongAdminAuthAPIKeyConfig{ + Value: rootProps.StringPropertyValue(cfgKongAdminAPIKey), + Header: rootProps.StringPropertyValue(cfgKongAdminAPIKeyHeader), + }, + }, + }, + Proxy: KongProxyConfig{ + Host: rootProps.StringPropertyValue(cfgKongProxyHost), + Port: KongProxyPortConfig{ + HTTP: rootProps.IntPropertyValue(cfgKongProxyPortHttp), + HTTPS: rootProps.IntPropertyValue(cfgKongProxyPortHttps), + }, + }, + Spec: KongSpecConfig{ + URLPaths: rootProps.StringSlicePropertyValue(cfgKongSpecURLPaths), + LocalPath: rootProps.StringPropertyValue(cfgKongSpecLocalPath), + }, + } +} diff --git a/pkg/config/traceability/config.go b/pkg/config/traceability/config.go index edf1564..e548799 100644 --- a/pkg/config/traceability/config.go +++ b/pkg/config/traceability/config.go @@ -1,16 +1,27 @@ package traceabilityconfig import ( + "github.com/Axway/agent-sdk/pkg/cmd/properties" corecfg "github.com/Axway/agent-sdk/pkg/config" ) +const ( + cfgKongHTTPLogPluginPath = "kong.httpLogPlugin.path" + cfgKongHTTPLogPluginPort = "kong.httpLogPlugin.port" +) + +func AddKongProperties(rootProps properties.Properties) { + rootProps.AddStringProperty(cfgKongHTTPLogPluginPath, "/requestlogs", "Path on which the HTTP Log plugin sends request logs") + rootProps.AddIntProperty(cfgKongHTTPLogPluginPort, 9000, "Port that listens for request logs from HTTP Log plugin") +} + // AgentConfig - represents the config for agent type AgentConfig struct { - CentralCfg corecfg.CentralConfig `config:"central"` - HttpLogPluginConfig *HttpLogPluginConfig `config:"http_log_plugin_config"` + CentralCfg corecfg.CentralConfig `config:"central"` + HttpLogPluginConfig *KongHttpLogPluginConfig `config:"httpLogPlugin"` } -type HttpLogPluginConfig struct { +type KongHttpLogPluginConfig struct { Path string `config:"path"` Port int `config:"port"` } @@ -24,3 +35,11 @@ func SetAgentConfig(cfg *AgentConfig) { func GetAgentConfig() *AgentConfig { return agentConfig } + +func ParseProperties(rootProps properties.Properties) *KongHttpLogPluginConfig { + // Parse the config from bound properties and setup gateway config + return &KongHttpLogPluginConfig{ + Path: rootProps.StringPropertyValue(cfgKongHTTPLogPluginPath), + Port: rootProps.IntPropertyValue(cfgKongHTTPLogPluginPort), + } +} diff --git a/pkg/gateway/client.go b/pkg/gateway/client.go index d8cee07..8b44c67 100644 --- a/pkg/gateway/client.go +++ b/pkg/gateway/client.go @@ -22,10 +22,11 @@ import ( klib "github.com/kong/go-kong/kong" ) -const ( - ardCtx log.ContextField = "accessRequestDefinition" - crdCtx log.ContextField = "credentialRequestDefinition" -) +var kongToCRDMapper = map[string]string{ + "basic-auth": provisioning.BasicAuthCRD, + "key-auth": provisioning.APIKeyCRD, + "oauth2": provisioning.OAuthSecretCRD, +} func NewClient(agentConfig config.AgentConfig) (*Client, error) { kongGatewayConfig := agentConfig.KongGatewayCfg @@ -69,44 +70,6 @@ func hasACLEnabledInPlugins(plugins []*klib.Plugin) error { return fmt.Errorf("failed to find acl plugin is enabled and installed") } -func (gc *Client) createRequestDefinitions(ctx context.Context) (context.Context, error) { - gc.logger.Debug("creating request definitions") - ctx = gc.createAccessRequestDefinition(ctx) - return gc.createCredentialRequestDefinition(ctx) -} - -func (gc *Client) createAccessRequestDefinition(ctx context.Context) context.Context { - return context.WithValue(ctx, ardCtx, true) -} - -func (gc *Client) createCredentialRequestDefinition(ctx context.Context) (context.Context, error) { - ctx = context.WithValue(ctx, crdCtx, []string{}) - allPlugins, err := gc.plugins.ListAll(context.Background()) - if err != nil { - gc.logger.WithError(err).Error("failed list all available plugins") - return ctx, err - } - - uniqueCrds := map[string]string{} - for _, plugin := range allPlugins { - if isValidAuthTypeAndEnabled(plugin) { - uniqueCrds[*plugin.Name] = *plugin.Name - } - } - kongToCRDMapper := map[string]string{ - "basic-auth": provisioning.BasicAuthCRD, - "key-auth": provisioning.APIKeyCRD, - "oauth2": provisioning.OAuthSecretCRD, - } - - for _, crd := range uniqueCrds { - if toAdd, ok := kongToCRDMapper[crd]; ok { - ctx = context.WithValue(ctx, crdCtx, append(ctx.Value(crdCtx).([]string), toAdd)) - } - } - return ctx, nil -} - func (gc *Client) DiscoverAPIs() error { gc.logger.Info("execute discovery process") @@ -115,9 +78,6 @@ func (gc *Client) DiscoverAPIs() error { plugins := kutil.Plugins{PluginLister: gc.kongClient.GetKongPlugins()} gc.plugins = plugins - if ctx, err = gc.createRequestDefinitions(ctx); err != nil { - return err - } services, err := gc.kongClient.ListServices(ctx) if err != nil { @@ -261,10 +221,13 @@ func (gc *Client) processKongAPI( gc.logger.WithError(err).Error("failed to save api to cache") } - if ctx.Value(ardCtx) != nil { - kongAPI.ard = provisioning.APIKeyARD + kongAPI.ard = provisioning.APIKeyARD + kongAPI.crds = []string{} + for k := range apiPlugins { + if crd, ok := kongToCRDMapper[k]; ok { + kongAPI.crds = append(kongAPI.crds, crd) + } } - kongAPI.crds = ctx.Value(crdCtx).([]string) agentDetails := map[string]string{ common.AttrServiceId: *service.ID, diff --git a/pkg/kong/plugins.go b/pkg/kong/plugins.go index c65631d..6be6473 100644 --- a/pkg/kong/plugins.go +++ b/pkg/kong/plugins.go @@ -62,7 +62,8 @@ func (p *Plugins) GetEffectivePlugins(routeID, serviceID string) (map[string]*kl for _, plugin := range plugins { if (plugin.Route != nil && (plugin.Route.ID == nil || *plugin.Route.ID != routeID)) || - (plugin.Service != nil && (plugin.Service.ID == nil || *plugin.Service.ID != serviceID)) { + (plugin.Service != nil && (plugin.Service.ID == nil || *plugin.Service.ID != serviceID)) || + !*plugin.Enabled { continue } diff --git a/pkg/kong/plugins_test.go b/pkg/kong/plugins_test.go index 1909028..9ca0b85 100644 --- a/pkg/kong/plugins_test.go +++ b/pkg/kong/plugins_test.go @@ -10,14 +10,17 @@ import ( type pluginsMock []*klib.Plugin +var truePlugin = true + func (pm pluginsMock) ListAll(_ context.Context) ([]*klib.Plugin, error) { return pm, nil } func p(id, name string) *klib.Plugin { return &klib.Plugin{ - ID: &id, - Name: &name, + ID: &id, + Name: &name, + Enabled: &truePlugin, } } @@ -28,6 +31,7 @@ func pwr(id, name, routeID string) *klib.Plugin { Route: &klib.Route{ ID: &routeID, }, + Enabled: &truePlugin, } } @@ -38,6 +42,7 @@ func pws(id, name, serviceID string) *klib.Plugin { Service: &klib.Service{ ID: &serviceID, }, + Enabled: &truePlugin, } } @@ -51,6 +56,7 @@ func pwrs(id, name, routeID, serviceID string) *klib.Plugin { Service: &klib.Service{ ID: &serviceID, }, + Enabled: &truePlugin, } } diff --git a/pkg/main/discovery/main.go b/pkg/main/discovery/main.go index 475fb6f..a6d8bac 100644 --- a/pkg/main/discovery/main.go +++ b/pkg/main/discovery/main.go @@ -8,7 +8,7 @@ import ( ) func main() { - + os.Setenv("AGENTFEATURES_VERSIONCHECKER", "false") if err := discovery.DiscoveryCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) diff --git a/pkg/main/traceability/main.go b/pkg/main/traceability/main.go index c957c5f..da79b30 100644 --- a/pkg/main/traceability/main.go +++ b/pkg/main/traceability/main.go @@ -5,10 +5,12 @@ import ( "os" _ "github.com/Axway/agent-sdk/pkg/traceability" + "github.com/Axway/agents-kong/pkg/cmd/traceability" ) func main() { + os.Setenv("AGENTFEATURES_VERSIONCHECKER", "false") if err := traceability.TraceCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) diff --git a/pkg/processor/definitions.go b/pkg/processor/definitions.go index 21604b1..a7a09a9 100644 --- a/pkg/processor/definitions.go +++ b/pkg/processor/definitions.go @@ -1,5 +1,15 @@ package processor +import "github.com/Axway/agent-sdk/pkg/util/log" + +const ( + CtxTransactionID log.ContextField = "transactionID" +) + +func init() { + log.RegisterContextField(CtxTransactionID) +} + // KongTrafficLogEntry - Represents the structure of log entry the agent will receive from Kong's HTTP Log Plugin type KongTrafficLogEntry struct { ClientIP string `json:"client_ip"` diff --git a/pkg/processor/eventmapper.go b/pkg/processor/eventmapper.go index 61968f2..4617ad2 100644 --- a/pkg/processor/eventmapper.go +++ b/pkg/processor/eventmapper.go @@ -2,6 +2,7 @@ package processor import ( "bytes" + "context" "encoding/json" "fmt" "net/http" @@ -15,45 +16,53 @@ import ( "github.com/Axway/agent-sdk/pkg/util/log" ) +const ( + host = "host" + userAgent = "user-agent" + leg0 = "leg0" + inbound = "inbound" +) + // EventMapper - type EventMapper struct { + logger log.FieldLogger } -const ( - host = "host" - userAgent = "user-agent" - redactedValue = "****************************" -) +func NewEventMapper() *EventMapper { + return &EventMapper{ + logger: log.NewFieldLogger().WithComponent("eventMapper").WithPackage("processor"), + } +} -func (m *EventMapper) processMapping(kongTrafficLogEntry KongTrafficLogEntry) ([]*transaction.LogEvent, error) { +func (m *EventMapper) processMapping(ctx context.Context, kongTrafficLogEntry KongTrafficLogEntry) ([]*transaction.LogEvent, error) { + log := log.UpdateLoggerWithContext(ctx, m.logger) centralCfg := agent.GetCentralConfig() txnID := uuid.New().String() transactionLegEvent, err := m.createTransactionEvent(kongTrafficLogEntry, txnID) if err != nil { - log.Errorf("Error while building transaction leg event: %s", err) + log.WithError(err).Error("building transaction leg event") return nil, err } jTransactionLegEvent, err := json.Marshal(transactionLegEvent) if err != nil { - log.Errorf("Failed to serialize transaction leg event as json: %s", err) + log.WithError(err).Error("serialize transaction leg event") } - log.Debug("Generated Transaction leg event: ", string(jTransactionLegEvent)) + log.WithField("leg", string(jTransactionLegEvent)).Debug("generated transaction leg event") transSummaryLogEvent, err := m.createSummaryEvent(kongTrafficLogEntry, centralCfg.GetTeamID(), txnID) if err != nil { - log.Errorf("Error while building transaction summary event: %s", err) + log.WithError(err).Error("building transaction summary event") return nil, err } jTransactionSummary, err := json.Marshal(transSummaryLogEvent) if err != nil { - log.Errorf("Failed to serialize transaction summary as json: %s", err) + log.WithError(err).Error("serialize transaction summary event") } - - log.Debug("Generated Transaction summary event: ", string(jTransactionSummary)) + log.WithField("summary", string(jTransactionSummary)).Debug("generated transaction summary event") return []*transaction.LogEvent{ transSummaryLogEvent, @@ -81,10 +90,6 @@ func (m *EventMapper) getTransactionSummaryStatus(statusCode int) transaction.Tx } func (m *EventMapper) buildHeaders(headers map[string]string) string { - if headers["apikey"] != "" { - headers["apikey"] = redactedValue - } - jsonHeader, err := json.Marshal(headers) if err != nil { log.Error(err.Error()) @@ -134,12 +139,11 @@ func (m *EventMapper) createTransactionEvent(ktle KongTrafficLogEntry, txnid str return transaction.NewTransactionEventBuilder(). SetTimestamp(ktle.StartedAt). SetTransactionID(txnid). - SetID("leg0"). - SetParentID(""). + SetID(leg0). SetSource(ktle.ClientIP). SetDestination(ktle.Request.Headers[host]). SetDuration(ktle.Latencies.Request). - SetDirection("inbound"). + SetDirection(inbound). SetStatus(m.getTransactionEventStatus(ktle.Response.Status)). SetProtocolDetail(httpProtocolDetails). Build() diff --git a/pkg/processor/eventprocessor.go b/pkg/processor/eventprocessor.go index bad523c..ae27bf1 100644 --- a/pkg/processor/eventprocessor.go +++ b/pkg/processor/eventprocessor.go @@ -1,6 +1,7 @@ package processor import ( + "context" "encoding/json" "time" @@ -21,23 +22,27 @@ import ( type EventProcessor struct { eventGenerator transaction.EventGenerator eventMapper *EventMapper + logger log.FieldLogger } // NewEventProcessor - return a new EventProcessor func NewEventProcessor() *EventProcessor { ep := &EventProcessor{ eventGenerator: transaction.NewEventGenerator(), - eventMapper: &EventMapper{}, + eventMapper: NewEventMapper(), + logger: log.NewFieldLogger().WithComponent("eventProcessor").WithPackage("processor"), } return ep } // ProcessRaw - process the received log entry and returns the event to be published to AMPLIFY ingestion service -func (p *EventProcessor) ProcessRaw(rawEventData []byte) []beat.Event { +func (p *EventProcessor) ProcessRaw(ctx context.Context, rawEventData []byte) []beat.Event { + log := log.UpdateLoggerWithContext(ctx, p.logger) + var kongTrafficLogEntry KongTrafficLogEntry err := json.Unmarshal(rawEventData, &kongTrafficLogEntry) if err != nil { - log.Error(err.Error()) + log.WithError(err).Error("could not read log data") return nil } @@ -47,9 +52,9 @@ func (p *EventProcessor) ProcessRaw(rawEventData []byte) []beat.Event { } // Map the log entry to log event structure expected by AMPLIFY Central Observer - logEvents, err := p.eventMapper.processMapping(kongTrafficLogEntry) + logEvents, err := p.eventMapper.processMapping(ctx, kongTrafficLogEntry) if err != nil { - log.Error(err.Error()) + log.WithError(err).Error("mapping event") return nil } events := make([]beat.Event, 0) @@ -57,7 +62,7 @@ func (p *EventProcessor) ProcessRaw(rawEventData []byte) []beat.Event { // Generates the beat.Event with attributes by AMPLIFY ingestion service event, err := p.eventGenerator.CreateEvent(*logEvent, time.Now(), nil, nil, nil) if err != nil { - log.Error(err.Error()) + log.WithError(err).Error("creating event") } else { events = append(events, event) } diff --git a/pkg/subscription/credential/credential.go b/pkg/subscription/credential/credential.go index 781bd6f..1e607c8 100644 --- a/pkg/subscription/credential/credential.go +++ b/pkg/subscription/credential/credential.go @@ -2,6 +2,7 @@ package credential import ( "context" + "fmt" "github.com/Axway/agent-sdk/pkg/apic/provisioning" "github.com/Axway/agent-sdk/pkg/util/log" @@ -56,27 +57,33 @@ func (p credentialProvisioner) Deprovision() provisioning.RequestStatus { if credentialID == "" { return rs.SetMessage("CredentialID cannot be empty").Failed() } + log := p.logger.WithField("credentialID", credentialID). + WithField("consumerID", consumerID) + log.Info("Started credential de-provisioning") switch credentialType { case provisioning.APIKeyARD: { if err := p.client.DeleteAuthKey(ctx, consumerID, credentialID); err != nil { - return rs.SetMessage("Could not delete auth key credential").Failed() + return rs.SetMessage("API Key credential does not exist or it has already been deleted").Success() } + log.Info("API Key successful de-provision") return rs.SetMessage("API Key successfully deleted.").Success() } case provisioning.BasicAuthARD: { if err := p.client.DeleteHttpBasic(ctx, consumerID, credentialID); err != nil { - return rs.SetMessage("Could not delete basic auth credential").Failed() + return rs.SetMessage("Basic auth credential does not exist or it has already been deleted").Success() } + log.Info("Basic Auth successful de-provision") return rs.SetMessage("Basic auth credential successfully deleted.").Success() } case provisioning.OAuthSecretCRD: { if err := p.client.DeleteOauth2(ctx, consumerID, credentialID); err != nil { - return rs.SetMessage("Could not delete oauth2 credential").Failed() + return rs.SetMessage("OAuth2 credential does not exist or it has already been deleted").Success() } + log.Info("OAuth2 successful de-provision") return rs.SetMessage("OAuth2 credential successfully deleted.").Success() } } @@ -93,6 +100,8 @@ func (p credentialProvisioner) Provision() (provisioning.RequestStatus, provisio ctx := context.Background() rs := provisioning.NewRequestStatusBuilder() credentialType := p.request.GetCredentialType() + log := p.logger.WithField("consumerID", consumerID) + log.Info("Started credential provisioning") switch credentialType { case provisioning.APIKeyARD: @@ -101,10 +110,12 @@ func (p credentialProvisioner) Provision() (provisioning.RequestStatus, provisio ToKeyAuth() resp, err := p.client.CreateAuthKey(ctx, consumerID, keyAuth) if err != nil { + log.Info("API key unsuccessful provisioning") return rs.SetMessage("Failed to create api-key credential").Failed(), nil } rs.AddProperty(common.AttrAppID, *resp.Consumer.ID) rs.AddProperty(common.AttrCredentialID, *resp.ID) + log.Info("API key successful provisioning") return rs.Success(), provisioning.NewCredentialBuilder().SetAPIKey(*resp.Key) } case provisioning.BasicAuthARD: @@ -116,11 +127,13 @@ func (p credentialProvisioner) Provision() (provisioning.RequestStatus, provisio ToBasicAuth() resp, err := p.client.CreateHttpBasic(ctx, consumerID, basicAuth) if err != nil { + log.Info("Basic auth unsuccessful provisioning") return rs.SetMessage("Failed to create basic auth credential").Failed(), nil } rs.AddProperty(common.AttrAppID, *resp.Consumer.ID) rs.AddProperty(common.AttrCredentialID, *resp.ID) rs.AddProperty(common.AttrCredUpdater, *resp.Username) + log.Info("Basic auth successful provisioning") return rs.Success(), provisioning.NewCredentialBuilder().SetHTTPBasic(user, pass) } case provisioning.OAuthSecretCRD: @@ -131,11 +144,13 @@ func (p credentialProvisioner) Provision() (provisioning.RequestStatus, provisio ToOauth2() resp, err := p.client.CreateOauth2(ctx, consumerID, oauth2) if err != nil { - return rs.SetMessage("Failed to create basic auth credential").Failed(), nil + log.Info("Oauth2 unsuccessful provisioning") + return rs.SetMessage("Failed to create oauth2 credential").Failed(), nil } rs.AddProperty(common.AttrAppID, *resp.Consumer.ID) rs.AddProperty(common.AttrCredentialID, *resp.ID) rs.AddProperty(common.AttrCredUpdater, *resp.ClientID) + log.Info("OAuth2 successful provisioning") return rs.Success(), provisioning.NewCredentialBuilder().SetOAuthIDAndSecret(*resp.ClientID, *resp.ClientSecret) } } @@ -158,43 +173,54 @@ func (p credentialProvisioner) Update() (provisioning.RequestStatus, provisionin return rs.SetMessage("kongCredentialId cannot be empty").Failed(), nil } + log := p.logger.WithField("credentialID", credentialID). + WithField("consumerID", consumerID) + log.Info("Started credential update") + switch credentialType { case provisioning.APIKeyARD: { if err := p.client.DeleteAuthKey(ctx, consumerID, credentialID); err != nil { - return rs.SetMessage("Could not delete api-key credential").Failed(), nil + log.WithError(err).Error("Could not delete api-key credential") + return rs.SetMessage(fmt.Sprintf("Could not delete credential %s for consumer %s", consumerID, credentialID)).Failed(), nil } keyAuth := kongBuilder.WithAuthKey(""). ToKeyAuth() resp, err := p.client.CreateAuthKey(ctx, consumerID, keyAuth) if err != nil { + log.WithError(err).Error("Could not create api-key credential") return rs.SetMessage("Failed to create api-key credential").Failed(), nil } rs.AddProperty(common.AttrAppID, *resp.Consumer.ID) rs.AddProperty(common.AttrCredentialID, *resp.ID) + log.Info("API Key successful update") return rs.Success(), provisioning.NewCredentialBuilder().SetAPIKey(*resp.Key) } case provisioning.BasicAuthARD: { if err := p.client.DeleteHttpBasic(ctx, consumerID, credentialID); err != nil { - return rs.SetMessage("Failed to delete basic auth credential").Failed(), nil + log.WithError(err).Error("Could not delete basic auth credential") + return rs.SetMessage(fmt.Sprintf("Could not delete credential %s for consumer %s", consumerID, credentialID)).Failed(), nil } basicAuth := kongBuilder.WithUsername(key). WithPassword(""). ToBasicAuth() resp, err := p.client.CreateHttpBasic(ctx, consumerID, basicAuth) if err != nil { + log.WithError(err).Error("Could not create basic auth credential") return rs.SetMessage("Failed to create basic auth credential").Failed(), nil } rs.AddProperty(common.AttrAppID, *resp.Consumer.ID) rs.AddProperty(common.AttrCredentialID, *resp.ID) rs.AddProperty(common.AttrCredUpdater, *resp.Username) + log.Info("Basic Auth successful update") return rs.Success(), provisioning.NewCredentialBuilder().SetHTTPBasic(*resp.Username, *resp.Password) } case provisioning.OAuthSecretCRD: { if err := p.client.DeleteOauth2(ctx, consumerID, credentialID); err != nil { - return rs.SetMessage("Failed to delete oauth2 credential").Failed(), nil + log.WithError(err).Error("Could not delete oauth2 credential") + return rs.SetMessage(fmt.Sprintf("Could not delete credential %s for consumer %s", consumerID, credentialID)).Failed(), nil } oauth2 := kongBuilder.WithClientID(key). WithClientSecret(""). @@ -202,11 +228,13 @@ func (p credentialProvisioner) Update() (provisioning.RequestStatus, provisionin ToOauth2() resp, err := p.client.CreateOauth2(ctx, consumerID, oauth2) if err != nil { + log.WithError(err).Error("Could not create oauth2 credential") return rs.SetMessage("Failed to create oauth2 credential").Failed(), nil } rs.AddProperty(common.AttrAppID, *resp.Consumer.ID) rs.AddProperty(common.AttrCredentialID, *resp.ID) rs.AddProperty(common.AttrCredUpdater, *resp.ClientID) + log.Info("Oauth2 successful update") return rs.Success(), provisioning.NewCredentialBuilder().SetOAuthIDAndSecret(*resp.ClientID, *resp.ClientSecret) } }