diff --git a/.csr-profile.json b/.csr-profile.json index c9a597a..aae7ee0 100644 --- a/.csr-profile.json +++ b/.csr-profile.json @@ -3,17 +3,12 @@ "repo_url": "https://github.com/Axway/agents-kong", "security_guide": "https://docs.axway.com/bundle/axway_resources/page/amplify_api_management_platform_security_white_paper.html", "requirements": { - "dependency-check": false, "fortify": true, "irius-risk": false, - "npm-audit": false, "pentest": false, - "retirejs": false, "twistlock": true, - "zap": false, - "yarn": false, - "gosec": false, - "whitesource": true, + "blackduck": true, + "third-party-policy-violation": false, "appspider": false, "insightvm": false }, diff --git a/.github/workflows/package-helm-chart.yml b/.github/workflows/package-helm-chart.yml new file mode 100644 index 0000000..aeb876a --- /dev/null +++ b/.github/workflows/package-helm-chart.yml @@ -0,0 +1,30 @@ +name: Kong Agents Helm +on: + push: + tags: + - v*.*.* + +jobs: + helm-archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: olegtarasov/get-tag@v2.1.2 + id: tagName + with: + tagRegex: "v(.*)" + tagRegexGroup: 1 + - name: Update chart versions + run: | + + sed helm/kong-agents/Chart.yaml -i -e "s/^appVersion.*/appVersion:\ \"v${GIT_TAG_NAME}\"/" + sed helm/kong-agents/Chart.yaml -i -e "s/^version.*/version:\ ${GIT_TAG_NAME}/" + - name: Archive chart + run: | + tar czf kong-agents.tgz -C helm/kong-agents . + - name: Upload chart to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: kong-agents.tgz + tag: ${{ github.ref_name }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index afd972e..b3cbff9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,15 +14,15 @@ bin/ .idea/ **.tgz .log -.env +.env* *.pem *.key .DS_Store .run/ *.log -**/kong_discovery_agent.yml -**/kong_traceability_agent.yml +pkg/discovery/main/kong_discovery_agent.yml +pkg/traceability/main/kong_traceability_agent.yml specs/ data/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3716586..3bf9c50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,10 +10,6 @@ variables: FORTIFY_INCLUDE: "**/*.go" FORTIFY_EXCLUDE: "**/*_test.go" - # Whitesource - WS_PROJECT_ID: "agents-kong" - WS_CONFIG_FILE: "whitesource.config" - # Blackduck BLACKDUCK_PROJECT_NAME: "Amplify - APIC Kong Agents" @@ -31,12 +27,11 @@ include: - "/gitlabci/restrictions.yml" - "/gitlabci/jobs.yml" - project: "scurity/gitlabci" - ref: $SCURITY_V2 + ref: $SCURITY_LATEST file: - "/.gitlab-ci-fortify.yml" - "/.gitlab-ci-twistlock.yml" - "/.gitlab-ci-iriusrisk.yml" - - "/.gitlab-ci-whitesource.yml" - "/.gitlab-ci-blackduck.yml" - "/.gitlab-ci-csr.yml" - project: "apigov/beano_cicd" @@ -103,20 +98,7 @@ twistlock-traceability:on-schedule: - export IMAGE_NAME=ghcr.io/axway/kong_traceability_agent:${GIT_TAG_PREFIX}${LATEST_TAG} - docker pull ${IMAGE_NAME} -whitesource:on-schedule: - extends: .whitesource - rules: - - !reference [.mirror-schedule-csr-rules, rules] - before_script: - - git config --global http.sslVerify false - - git config --global url."ssh://git@git.ecd.axway.org".insteadOf "https://git.ecd.axway.org"'' - - git fetch - - *get-latest-tag - - echo "Checking out ${GIT_TAG_PREFIX}${LATEST_TAG}" - - git checkout ${GIT_TAG_PREFIX}${LATEST_TAG} - blackduck:on-schedule: - extends: .blackduck rules: - !reference [.mirror-schedule-csr-rules, rules] before_script: @@ -136,13 +118,7 @@ fetch-fortify: rules: - !reference [.mirror-branch-csr-rules, rules] -whitesource: - rules: - - !reference [.mirror-branch-csr-rules, rules] - before_script: - - export GOWORK=off - -blackduck: +blackduck-rapid: rules: - !reference [.mirror-branch-csr-rules, rules] diff --git a/CODEOWNERS b/CODEOWNERS index 29c1290..acc3108 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -4,7 +4,6 @@ #[CSR] # .csr-profile.json requires SPOC approval for any modifications .csr-profile.json @dfeldick @jcollins-axway @vivekschauhan -whitesource.config @dfeldick #[CICD] # cicd-related files diff --git a/Makefile b/Makefile index 75e1035..f1e9714 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,10 @@ all: clean test: dep @go vet ${GO_PKG_LIST} @go test -race -v -short -coverprofile=${WORKSPACE}/gocoverage.out -count=1 ${GO_PKG_LIST} + +test-s: dep + @go vet ${GO_PKG_LIST} + @go test -race -short -coverprofile=${WORKSPACE}/gocoverage.out -count=1 ${GO_PKG_LIST} clean: @rm -rf ./bin/ @@ -44,7 +48,7 @@ ${WORKSPACE}/discovery_agent: -X 'github.com/Axway/agent-sdk/pkg/cmd.SDKBuildVersion=$(sdk_version)' \ -X 'github.com/Axway/agent-sdk/pkg/cmd.BuildAgentName=KongDiscoveryAgent' \ -X 'github.com/Axway/agent-sdk/pkg/cmd.BuildAgentDescription=Kong Discovery Agent'" \ - -a -o ${WORKSPACE}/bin/discovery_agent ${WORKSPACE}/pkg/main/discovery/main.go + -a -o ${WORKSPACE}/bin/discovery_agent ${WORKSPACE}/pkg/discovery/main/agent.go build-da: dep ${WORKSPACE}/discovery_agent @echo "Discovery Agent build completed" diff --git a/README.md b/README.md index 76e8378..6591dad 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,13 @@ The Kong agents are used to discover, provision access to, and track usages of Kong Gateway routes. - [Getting started](#getting-started) + - [Discovery process](#discovery-process) + - [Provisioning process](#provisioning-process) + - [Marketplace application](#marketplace-application) + - [Access request](#access-request) + - [Credential](#credential) + - [Traceability process](#traceability-process) + - [Environment variables](#environment-variables) - [Setup](#setup) - [Amplify setup](#amplify-setup) - [Platform - organization ID](#platform---organization-id) @@ -12,25 +19,82 @@ The Kong agents are used to discover, provision access to, and track usages of K - [Kong admin API secured by Kong Gateway](#kong-admin-api-secured-by-kong-gateway) - [Specification discovery methods](#specification-discovery-methods) - [Local specification path](#local-specification-path) + - [Filtering gateway services](#filtering-gateway-services) - [URL specification paths](#url-specification-paths) - [Kong Dev Portal](#kong-dev-portal) + - [HTTP Log plugin](#http-log-plugin) - [Kong agents deployment](#kong-agents-deployment) - [Additional information](#additional-information) - [Docker](#docker) - [Environment variable files](#environment-variable-files) - [Deployment](#deployment) - [Helm](#helm) - - [Download](#download) + - [Traceability agent stateful set](#traceability-agent-stateful-set) - [Create secrets](#create-secrets) - [Create volume, local specification files only](#create-volume-local-specification-files-only) + - [ConfigMap](#configmap) + - [AWS S3 Synchronization](#aws-s3-synchronization) - [Create overrides](#create-overrides) - - [Deploy local helm chart](#deploy-local-helm-chart) - - [Discovery process](#discovery-process) - - [Provisioning process](#provisioning-process) - - [Marketplace application](#marketplace-application) - - [Access request](#access-request) - - [Credential](#credential) - - [Environment variables](#environment-variables) + - [Deploy helm chart](#deploy-helm-chart) + +## Discovery process + +On startup the Kong discovery agent first validates that it is able to connect to all required services. Once connected to Kong the agent begins looking at the Plugins configured, more specifically for the ACL. The default option is to require having it. This can be changed from the config by disabling this check. By having the check disabled, it is assumed that access is allowed for everyone. Then the agent will determine, from the plugins, which credential types the Kong Gateway has configured and create the Central representation of those types. + +After that initial startup process the discovery agent begins running its main discovery loop. In this loop the agent first gets a list of all Gateway Services. With each service the agent looks for all configured routes. The agent then looks to gather the specification file, see [Specification discovery methods](#specification-discovery-methods), if found the process continues. Using the route the agent checks for plugins to determine the types of credentials to associate with it. After gathering all of this information the agent creates a new API service with the specification file and linking the appropriate credentials. The endpoints associated to the API service are constructed using the **KONG_PROXY_HOST**, **KONG_PROXY_PORTS_HTTP**, and **KONG_PROXY_PORTS_HTTPS** settings. + +## Provisioning process + +As described in the [Discovery process](#discovery-process) section the Kong agent creates all supported credential types on Central at startup. Once API services are published they can be made into Assets and Products via Central itself. The Products can then be published to the Marketplace for consumption. In order to receive access to the service a user must first request access to it and the Kong agent provisioning process will execute based off of that request. + +### Marketplace application + +A Marketplace application is created by a Marketplace user. When a resource within the Kong environment is added to that application Central will create a ManagedApplication resource that the agent will execute off of. This ManagedApplication resource event is captured by the Kong agent and the agent creates a Kong consumer. In addition to the creation of the Consumer the agent adds an ACL Group ID to the Consumer, to be used by the Access Request. + +### Access request + +(Note: if the ACL plugin is not required, access request is skipped altogether). When a Marketplace user requests access to a resource, within the Kong environment, Central will create an AccessRequest resource in the same Kong environment. The agent receives this event and makes several changes within Kong. First the agent will add, or update, an ACL configuration on the Route being requested. This ACL will allow the Group ID created during the handling of the [Marketplace application](#marketplace-application) access to the route. Additionally, if a quota for this route has been set in Central in the product being handled the agent will add a Rate limiting plugin to reflect the quota that was set in Central for that product. Note: Quotas in Central can have a Weekly amount, this is not supported by Kong and the agent will reject the Access Request. + +### Credential + +Finally, when a Marketplace user requests a credential, within the Kong environment, Central will create a Credential resource in the same Kong environment. The agent receives this event and creates the proper credential type for the Consumer that the [Marketplace application](#marketplace-application) handling created. After successfully creating this credential the necessary details are returned back to the Central to be viewed and used by the Marketplace user. + +## Traceability process + +On startup the Kong traceability agent first validates that it is able to connect to all required services. Once validation is complete the agent begins listening for log events to be sent to it. The agent receives these events and iterates through them to determine if any of the events should be sampled. If it is to be sampled the agent creates a transaction summary and leg sending that the Amplify Central. Regardless of the event being set for sampling the agent will update the proper API Metric and Usage details to be sent to Amplify Central on the interval configured. See [Usage](https://docs.axway.com/bundle/amplify-central/page/docs/connect_manage_environ/connected_agent_common_reference/traceability_usage/index.html). Note: if the ACL plugin is not required, the traceability agent cannot associate API traffic with a consumer application. + +## Environment variables + +All Kong specific environment variables available are listed below + +| Name | Description | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Discovery Agent Variables | | +| **KONG_ACL_DISABLE** | Set to true to disable the check for a globally enabled ACL plugin on Kong. False by default. | +| **KONG_ADMIN_URL** | The Kong admin API URL that the agent will query against | +| **KONG_ADMIN_AUTH_APIKEY_HEADER** | The API Key header name the agent will use when authenticating | +| **KONG_ADMIN_AUTH_APIKEY_VALUE** | The API Key value the agent will use when authenticating | +| **KONG_ADMIN_AUTH_BASICAUTH_USERNAME** | The HTTP Basic username that the agent will use when authenticating | +| **KONG_ADMIN_AUTH_BASICAUTH_PASSWORD** | The HTTP Basic password that the agent will use when authenticating | +| **KONG_ADMIN_SSL_NEXTPROTOS** | An array of strings. It is a list of supported application level protocols, in order of preference, based on the ALPN protocol list. Allowed values are: h2, http/1.0, http/1.1, h2c | +| **KONG_ADMIN_SSL_INSECURESKIPVERIFY** | Controls whether a client verifies the server’s certificate chain and host name. If true, TLS accepts any certificate presented by the server and any host name in that certificate. In this mode, TLS is susceptible to man-in-the-middle attacks | +| **KONG_ADMIN_SSL_CIPHERSUITES** | An array of strings. It is a list of supported cipher suites for TLS versions up to TLS 1.2. If CipherSuites is nil, a default list of secure cipher suites is used, with a preference order based on hardware performance | +| **KONG_ADMIN_SSL_MAXVERSION** | String value for the maximum SSL/TLS version that is acceptable. If empty, then the maximum version supported by this package is used, which is currently TLS 1.3. Allowed values are: TLS1.0, TLS1.1, TLS1.2, TLS1.3 | +| **KONG_ADMIN_SSL_MINVERSION** | String value for the minimum SSL/TLS version that is acceptable. If empty TLS 1.2 is taken as the minimum. Allowed values are: TLS1.0, TLS1.1, TLS1.2, TLS1.3 | +| **KONG_PROXY_HOST** | The proxy host that the agent will use in API Services when the Kong route does not specify hosts | +| **KONG_PROXY_PORTS_HTTP_VALUE** | The HTTP port value that the agent will set for discovered APIS | +| **KONG_PROXY_PORTS_HTTPS_VALUE** | The HTTPs port value that the agent will set for discovered APIS | +| **KONG_PROXY_PORTS_HTTP_DISABLE** | Set to true if the agent should ignore routes that serve over HTTP | +| **KONG_PROXY_PORTS_HTTPS_DISABLE** | Set to true if the agent should ignore routes that serve over HTTPs | +| **KONG_PROXY_BASEPATH** | The proxy base path that will be added between the proxy host and Kong route path when building endpoints | +| **KONG_SPEC_FILTER** | The Agent SDK specific filter format for filtering out specific Kong services | +| **KONG_SPEC_LOCALPATH** | The local path that the agent will look in for API definitions | +| **KONG_SPEC_URLPATHS** | The URL paths that the agent will query on the gateway service for API definitions | +| **KONG_SPEC_DEVPORTALENABLED** | Set to true if the agent should look for spec files in the Kong Dev Portal (default: `false`) | +| | | +| Traceability Agent Variables | | +| **KONG_LOGS_HTTP_PATH** | The path endpoint that the Traceability agent will listen on (default: `/requestlogs`) | +| **KONG_LOGS_HTTP_PORT** | The port that the Traceability agent HTTP server will listen on (default: `9000`) | ## Setup @@ -86,8 +150,24 @@ You now have the service account information needed for you Kong Agent installat - Finish up the wizard setting values as desired, on the last page click *Save* - Note the *Logical Name* for your new environment +--- +**NOTE:** + +Don't forget to update your Amplify Central Region specific variables, such as the `CENTRAL_URL` setting. + +All CENTRAL_* variables listed on [docs.axway.com](https://docs.axway.com/bundle/amplify-central/page/docs/connect_manage_environ/connect_api_manager/agent-variables/index.html) may be used on the Kong Agent. + +--- + ### Kong setup +--- +**NOTE:** + +The Discovery agent expects that the Kong Gateway utilizes the [ACL](https://docs.konghq.com/hub/kong-inc/acl/) plugin to control access to the various routes provided in the Kong Gateway. On startup the agent checks that this plugin is in use prior to performing any discovery. The agent then uses this plugin while provisioning access to routes in Kong. [Provisioning Process](#provisioning-process). + +--- + #### Kong admin API secured by Kong Gateway See [Kong - Securing the Admin API](https://docs.konghq.com/gateway/latest/production/running-kong/secure-admin-api/) @@ -98,7 +178,7 @@ Once the Kong admin API is secured a gateway service for it must be added to Kon - Basic authentication - API Key authentication -- OAuth2 authentication +- OAuth2 authentication (currently, Kong returns an Internal Server Error if securing the admin api with OAuth2. The plugin can be created in Kong, but further requests will not work when receiving the token. The Agent is also configured to (as of now) not work with OAuth2) #### Specification discovery methods @@ -106,7 +186,7 @@ In order to publish a specification file that properly represents the gateway se ##### Local specification path -The local specification discovery method is configured by providing a value for the `KONG_SPEC_LOCALPATH` variable. When set the Kong agent will look for a tag on each of the available routes from the gateway service that are prefixed by `spec_local_`. When that tag is set the value, after stripping the prefix, is used to find the specification file in directory configured by `KONG_SPEC_LOCALPATH`. When this configuration value is set no other specification discovery methods will be used. +The local specification discovery method is configured by providing a value for the `KONG_SPEC_LOCALPATH` variable. When set the Kong agent will look for a tag on each of the available gateway services that are prefixed by `spec_local_`. When that tag is set the value, after stripping the prefix, is used to find the specification file in directory configured by `KONG_SPEC_LOCALPATH`. When this configuration value is set no other specification discovery methods will be used. Ex. @@ -139,6 +219,16 @@ Configuration on my-service gateway service } ``` +##### Filtering gateway services + +Some possible ways to use the filter for gateway services (all these are done with the env var `KONG_SPEC_FILTER`): + +Ex1: "tag.Any() == \"spec_local_petstore.json\"" -> this will find all the services that have a tag as "spec_local_petstore.json" +Ex2: "tag.discover.Exists()" -> this will find all tags that are equal to "discover" +Note: while both ways can achieve the same functionality, the first one is preferred because it does not restrict you on character usages for Kong tags (note the dot in example 2) + +Currently, functionalities such as tag.Any().Contains() are not implemented in the SDK and only fully equal values are taken into account + ##### URL specification paths The URL specification paths discovery method is configured by value(s) for the `KONG_SPEC_URLPATHS` variable, comma separated. When values are set here, and a local path is not set, The Kong agent will query each of these paths against the gateway service in order to find a specification file. Once a specification file is found none of the other configured URL paths will be queried as that specification file will be used in the creation of the API Service on Central. @@ -154,7 +244,7 @@ KONG_SPEC_URLPATHS=/openapi.json,/swagger.json ##### Kong Dev Portal The Kong Dev Portal discovery method is configured by providing a value for the `KONG_SPEC_DEVPORTALENABLED`, but also the local spec discovery needs to be disabled by setting an empty value for the`KONG_SPEC_LOCALPATH`, otherwise, the local discovery process will be used. - + Ex. Configuration for agent @@ -164,6 +254,38 @@ KONG_SPEC_LOCALPATH="" KONG_SPEC_DEVPORTALENABLED=true ``` +#### HTTP Log plugin + +The Traceability agent utilizes Kong's HTTP log plugin to track transactions. In order to set this up the plugin will have to be added, globally, and configured to send to the endpoint that the Traceability agent will listen on + +- Navigate to the Plugins page +- Click *+ New Plugin* +- In the list of plugins find *HTTP Log* and click *enable* +- Ensure *Global* is selected so the agent receives events for all traffic +- Enter the following, all can be customized as necessary for your infrastructure, [HTTP Log](https://docs.konghq.com/hub/kong-inc/http-log/configuration/) + - An Instance Name (optional) + - Tags (optional) + - content_type - `applicaiton/json` + - custom_fields_by_lua - empty + - flush_timeout - empty + - headers - empty + - http_endpoint - the endpoint the agent will listen on (ie. `http://traceability.host:9000/requestlogs`) + - keepalive - `60000` + - method - `POST` + - queue.initial_retry_delay - `0.01` + - queue.max_batch_size - `1000` + - queue.max_bytes - empty + - queue.max_coalescing_delay - `10` + - queue.max_entries - `100000` + - queue.max_retry_delay - `60` + - queue.max_retry_time - `60` + - queue_size - empty + - retry_count - empty + - timeout - `10000` +- Click *Install* + +Kong is now setup to send transactions to the traceability agent. + ## Kong agents deployment The Kong agents are delivered as containers, kong_discovery_agent and kong_traceability_agent. These containers can be deployed directly to a container server, such as Docker, or using the provided helm chart. In this section you will lean how to deploy the agents directly as containers or within a kubernetes cluster using the helm chart. @@ -172,7 +294,7 @@ The Kong agents are delivered as containers, kong_discovery_agent and kong_trace Before beginning to deploy the agents following information will need to be gathered in addition to the details that were noted in setup. -- The full URL to connect to the Kong admin API, `KONG_ADMIN_URL` +- The full URL to connect to the Kong admin API, `KONG_ADMIN_URL`. Note that if secured by kong, the URL should look like: [https://host:port/secured-route-from-kong] - The host the agent will use when setting the endpoint of a discovered API, (`KONG_PROXY_HOST`) - The HTTP `KONG_PROXY_PORTS_HTTP` and HTTPs `KONG_PROXY_PORTS_HTTPS` ports the agent will use with the endpoint above - The URL paths, hosted by the gateway service, to query for spec files, `KONG_SPEC_URLPATHS` @@ -181,7 +303,7 @@ Before beginning to deploy the agents following information will need to be gath #### Environment variable files -In this section we will use the information gathered within the setup and additional information sections above and create two environment variable files for each agent to use. This is the minimum configuration assuming defaults for all other available settings. Note the setting below expect the use of the API Key authentication method for the [Kong admin api](#kong-admin-api-secured-by-kong-gateway). +In this section we will use the information gathered within the setup and additional information sections above and create two environment variable files for each agent to use. This is the minimum configuration assuming defaults for all other available settings. Note the settings below expect the use of the API Key authentication method for the [Kong admin api](#kong-admin-api-secured-by-kong-gateway). Discovery Agent @@ -190,8 +312,8 @@ KONG_ADMIN_URL=https://kong.url.com:8444 KONG_ADMIN_AUTH_APIKEY_HEADER="apikey" KONG_ADMIN_AUTH_APIKEY_VALUE=123456789abcdefghijkl098765432109 KONG_PROXY_HOST=kong.proxy.endpoint.com -KONG_PROXY_PORTS_HTTP=8000 -KONG_PROXY_PORTS_HTTPS=8443 +KONG_PROXY_PORTS_HTTP_VALUE=8000 +KONG_PROXY_PORTS_HTTPS_VALUE=8443 KONG_SPEC_LOCALPATH=/specs CENTRAL_ORGANIZATIONID=123456789 @@ -205,10 +327,6 @@ AGENTFEATURES_MARKETPLACEPROVISIONING=true Traceability Agent ```shell -KONG_ADMIN_URL=https://kong.url.com:8444 -KONG_ADMIN_AUTH_APIKEY_HEADER="apikey" -KONG_ADMIN_AUTH_APIKEY_VALUE=123456789abcdefghijkl098765432109 - CENTRAL_ORGANIZATIONID=123456789 CENTRAL_AUTH_CLIENTID=kong-agents_123456789-abcd-efgh-ijkl-098765432109 CENTRAL_ENVIRONMENT=kong @@ -235,21 +353,14 @@ docker run -d -v /home/user/keys:/keys -v /home/user/specs:/specs -v /home/user/ Kong Traceability agent ```shell -docker run -d -v /home/user/keys:/keys -v /home/user/traceability/data:/data --env-file traceability-agents.env ghcr.io/axway/kong_traceability_agent:latest +docker run -d -v /home/user/keys:/keys -v /home/user/traceability/data:/data --env-file traceability-agents.env -p 9000:9000 ghcr.io/axway/kong_traceability_agent:latest ``` ### Helm -#### Download +#### Traceability agent stateful set -At the current time the Kong agents helm chart is not hosted on a helm chart repository. To deploy using this helm chart you will first want to download the helm directory from your desired release tag, removing the v, 0.0.1 in the sample below. - -```shell -tag=0.0.1 # tag v0.0.1 with 'v' removed -curl -L https://github.com/Axway/agents-kong/archive/refs/tags/v${tag}.tar.gz --output kong-agents.tar.gz # download release archive -tar xvf kong-agents.tar.gz --strip-components=2 agents-kong-${tag}/helm/kong-agents # extract the helm chart in the current directory -rm kong-agents.tar.gz # remove the archive -``` +The helm deployment of the Traceability agent uses a resource type of Stateful set along with a service to distribute the events to the agent pods. This is to allow scaling of the traceability agent in order to properly handle the load of events being sent through Kong. The agent is expected to be ran in the same kubernetes cluster as the Gateway and the [HTTP Log plugin](#http-log-plugin) should set its endpoint configuration to the [Service](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#services) that is created (ie.`http://kong-traceability-agent.kong-agents.svc.cluster.local:9000` where `kong-traceability-agent` is the service name and `kong-agents` is the namespace for the service) #### Create secrets @@ -280,7 +391,13 @@ stringData: #### Create volume, local specification files only -A volume of with the local specification files is required, given that is the desired [specification discovery method](#specification-discovery-methods). This volume could be of any kubernetes resource type which can be mounted in the Kong agent container. Below is a sample of a ConfigMap that is used for the local specification files. See [Kubernetes Volumes](https://kubernetes.io/docs/concepts/storage/volumes/). +A volume of with the local specification files is required, given that is the desired [specification discovery method](#specification-discovery-methods). This volume could be of any kubernetes resource type which can be mounted in the Kong agent container. See [Kubernetes Volumes](https://kubernetes.io/docs/concepts/storage/volumes/). + +Below are a couple of examples on adding specifications to a volume, of any type, to the agent pod for discovery purposes. + +##### ConfigMap + +Here is a sample of a ConfigMap that is used for the local specification files. ```yaml apiVersion: v1 @@ -292,6 +409,21 @@ data: ...spec file contents... ``` +If a ConfigMap is being used, the kubectl command provides a utility to create the resource file for you. The command that follows will create a ConfigMap named `specs`, in the current kubernetes context and namespace. All files found in the current directories `specs/` folder will be included in the ConfigMap resource. + +```bash +kubectl create configmap specs --from-file=specs/ +``` + +--- +**NOTE:** + +An update to the ConfigMap will *NOT* be seen by any running pods, a pod restart would be required to see changes. + +It is recommended to use a volume type that is more mutable than a ConfigMap. The agent has no knowledge of the volume type being used. + +--- + Once a resource with the files is created, which ever resource type is chosen, the overrides file will need to be updated with that resource information for mounting as a volume. ```yaml @@ -303,19 +435,116 @@ kong: name: my-spec-files # name of the resource created ``` +##### AWS S3 Synchronization + +A kubernetes PersistentVolume resource with a CronJob volume can be set up to regularly synchronize spec files from an S3 bucket to the volume for the agent to utilize. Below you will find the three kubernetes resources that would need to be created as well as the update to the agnet helm chart override file. + +- Create a PersistentVolume - this will store the specification files in the cluster + - In this example a storage class of manual is used with a host path in the kubernetes cluster, however any class type may be used + - [K8S Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + - [EKS Persistent Volumes](https://aws.amazon.com/blogs/storage/persistent-storage-for-kubernetes/) + +```yaml +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: spec-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data" +``` + +- Create a PersistentVolumeClaim - this allows pods to mount this volume, needed for the job and the agent + +```yaml +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: spec-volume-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +``` + +- Create a CronJob - this will run on the specified interval synchronizing the S3 bucket to the volume + - The keys are embedded in this definition, but this can be replaced by a kubernetes secret or service account with the proper role in EKS + - The schedule is to sync the spec files every 15 minutes + - The bucket name is within the command, `specs-bucket` + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: s3-sync +spec: + schedule: "*/15 * * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: s3-sync + image: public.ecr.aws/aws-cli/aws-cli + env: + - name: AWS_ACCESS_KEY_ID + value: XXXXXXXXXXXXXXXXXXX + - name: AWS_SECRET_ACCESS_KEY + value: XXXXXXXXXXXXXXXXXXX + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - aws s3 sync s3://specs-bucket/ /specs/ + volumeMounts: + - name: specs-mount + mountPath: /specs + volumes: + - name: specs-mount + persistentVolumeClaim: + claimName: spec-volume-claim + restartPolicy: Never +``` + +- Override the agent helm chart accordingly + +```yaml +kong: + ... + spec: + localPath: + persistentVolumeClaim: # type of the resource, provided in the deployment as a volume. + claimName: spec-volume-claim # name of the resource created +``` + #### Create overrides overrides.yaml ```yaml kong: + enable: + traceability: true # set this to true to deploy the traceability agent stateful set admin: - url: https://kong.url.com:8444 + url: http://kong-gateway-kong-admin.kong.svc.cluster.local:8001 proxy: host: kong.proxy.endpoint.com ports: - http: 8000 - https: 8443 + http: 80 + https: 443 spec: localPath: configMap: @@ -328,7 +557,7 @@ env: AGENTFEATURES_MARKETPLACEPROVISIONING: true ``` -#### Deploy local helm chart +#### Deploy helm chart Assuming you are already in the desired kubernetes context and namespace, execute the following commands. @@ -338,45 +567,9 @@ Create the secret containing the Central key files used for authentication. kubectl apply -f kong-agent-keys.yaml ``` -Install the helm chart using the created overrides file. +Install the helm chart using the created overrides file. Set the release version to install. ```shell -helm install kong-agents ./kong-agents -f overrides.yaml +release=v0.0.2 +helm upgrade -i kong-agents https://github.com/Axway/agents-kong/releases/download/${release}/kong-agents.tgz -f overrides.yaml ``` - -## Discovery process - -On startup the Kong discovery agent first validates that it is able to connect to all required services. Once connected to Kong the agent begins looking at the Plugins configured, as the ACL plugin is required for handling Amplify Central provisioning events. Then the agent will determine, from the plugins, which credential types the Kong Gateway has configured and create the Central representation of those types. - -After that initial startup process the discovery agent begins running its main discovery loop. In this loop the agent first gets a list of all Gateway Services. With each service the agent looks for all configured routes. The agent then looks to gather the specification file, see [Specification discovery methods](#specification-discovery-methods), if found the process continues. Using the route the agent checks for plugins to determine the types of credentials to associate with it. After gathering all of this information the agent creates a new API service with the specification file and linking the appropriate credentials. The endpoints associated to the API service are constructed using the **KONG_PROXY_HOST**, **KONG_PROXY_PORTS_HTTP**, and **KONG_PROXY_PORTS_HTTPS** settings. - -## Provisioning process - -As described in the [Discovery process](#discovery-process) section the Kong agent creates all supported credential types on Central at startup. Once API services are published they can be made into Assets and Products via Central itself. The Products can then be published to the Marketplace for consumption. In order to receive access to the service a user must first request access to it and the Kong agent provisioning process will execute based off of that request. - -### Marketplace application - -A Marketplace application is created by a Marketplace user. When a resource within the Kong environment is added to that application Central will create a ManagedApplication resource that the agent will execute off of. This ManagedApplication resource event is captured by the Kong agent and the agent creates a Kong consumer. In addition to the creation of the Consumer the agent adds an ACL Group ID to the Consumer, to be used by the Access Request. - -### Access request - -When a Marketplace user requests access to a resource, within the Kong environment, Central will create an AccessRequest resource in the same Kong environment. The agent receives this event and makes several changes within Kong. First the agent will add, or update, an ACL configuration on the Route being requested. This ACL will allow the Group ID created during the handling of the [Marketplace application](#marketplace-application) access to the route. Additionally, if a quota for this route has been set in Central in the product being handled the agent will add a Rate limiting plugin to reflect the quota that was set in Central for that product. Note: Quotas in Central can have a Weekly amount, this is not supported by Kong and the agent will reject the Access Request. - -### Credential - -Finally, when a Marketplace user requests a credential, within the Kong environment, Central will create a Credential resource in the same Kong environment. The agent receives this event and creates the proper credential type for the Consumer that the [Marketplace application](#marketplace-application) handling created. After successfully creating this credential the necessary details are returned back to the Central to be viewed and used by the Marketplace user. - -## Environment variables - -All Kong specific environment variables available are listed below - -| Name | Description | -| --------------------------------- | ------------------------------------------------------------------------------------- | -| **KONG_ADMIN_URL** | The Kong admin API URL that the agent will query against | -| **KONG_ADMIN_AUTH_APIKEY_HEADER** | The API Key header name the agent will use when authenticating | -| **KONG_ADMIN_AUTH_APIKEY_VALUE** | The API Key value the agent will use when authenticating | -| **KONG_PROXY_HOST** | The proxy endpoint that the agent will use in API Services for discovered Kong routes | -| **KONG_PROXY_PORTS_HTTP** | The HTTP port number that the agent will set for discovered APIS | -| **KONG_PROXY_PORTS_HTTPS** | The HTTPs port number that the agent will set for discovered APIS | -| **KONG_SPEC_LOCALPATH** | The local path that the agent will look in for API definitions | -| **KONG_SPEC_URLPATHS** | The URL paths that the agent will query on the gateway service for API definitions | diff --git a/build/discovery/Dockerfile b/build/discovery/Dockerfile index 2f194b3..6587421 100644 --- a/build/discovery/Dockerfile +++ b/build/discovery/Dockerfile @@ -1,6 +1,6 @@ # Build image -# golang:1.21.3-alpine3.18 linux/amd64 -FROM docker.io/golang@sha256:27c76dcf886c5024320f4fa8ceb57d907494a3bb3d477d0aa7ac8385acd871ea AS builder +# golang:1.21.5-alpine3.18 linux/amd64 +FROM docker.io/golang@sha256:2aa0f0960cffcfd8daac2e765b8fdd3aa001a97d967c9ae96d58d06ff11ecdb4 AS builder ARG commit_id ARG version @@ -33,15 +33,15 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ -X 'github.com/Axway/agent-sdk/pkg/cmd.SDKBuildVersion=${sdk_version}' \ -X 'github.com/Axway/agent-sdk/pkg/cmd.BuildAgentName=KongDiscoveryAgent' \ -X 'github.com/Axway/agent-sdk/pkg/cmd.BuildAgentDescription=Kong Discovery Agent'" \ - -a -o /discovery_agent ${BASEPATH}/pkg/main/discovery/main.go + -a -o /discovery_agent ${BASEPATH}/pkg/discovery/main/agent.go # Create non-root user RUN addgroup -g 2500 axway && adduser -u 2500 -D -G axway axway RUN chown -R axway:axway /discovery_agent USER axway -# alpine 3.18.3 -FROM docker.io/alpine@sha256:c5c5fda71656f28e49ac9c5416b3643eaa6a108a8093151d6d1afc9463be8e33 +# alpine 3.18 linux/amd64 +FROM docker.io/alpine@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /etc/passwd /etc/passwd diff --git a/build/traceability/Dockerfile b/build/traceability/Dockerfile index c5e7dcd..56fc2a6 100644 --- a/build/traceability/Dockerfile +++ b/build/traceability/Dockerfile @@ -1,6 +1,6 @@ # Build image -# golang:1.21.3-alpine3.18 linux/amd64 -FROM docker.io/golang@sha256:27c76dcf886c5024320f4fa8ceb57d907494a3bb3d477d0aa7ac8385acd871ea AS builder +# golang:1.21.5-alpine3.18 linux/amd64 +FROM docker.io/golang@sha256:2aa0f0960cffcfd8daac2e765b8fdd3aa001a97d967c9ae96d58d06ff11ecdb4 AS builder ARG commit_id ARG version @@ -40,8 +40,8 @@ RUN addgroup -g 2500 axway && adduser -u 2500 -D -G axway axway RUN chown -R axway:axway /traceability_agent USER axway -# alpine 3.18.3 -FROM docker.io/alpine@sha256:c5c5fda71656f28e49ac9c5416b3643eaa6a108a8093151d6d1afc9463be8e33 +# alpine 3.18 linux/amd64 +FROM docker.io/alpine@sha256:d695c3de6fcd8cfe3a6222b0358425d40adfd129a8a47c3416faff1a8aece389 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /etc/passwd /etc/passwd diff --git a/build/traceability/kong_traceability_agent.yml b/build/traceability/kong_traceability_agent.yml index 5d8d08d..aa70bab 100644 --- a/build/traceability/kong_traceability_agent.yml +++ b/build/traceability/kong_traceability_agent.yml @@ -1,15 +1,10 @@ kong_traceability_agent: # Settings for connecting to Kong kong: - admin: - url: ${KONG_ADMIN_URL} - auth: - apikey: - header: ${KONG_ADMIN_AUTH_APIKEY_HEADER} - value: ${KONG_ADMIN_AUTH_APIKEY_VALUE} - httpLogPlugin: - path: ${KONG_HTTPLOGPLUGIN_PATH} - port: ${KONG_HTTPLOGPLUGIN_PORT} + logs: + http: + path: ${KONG_LOGS_HTTP_PATH} + port: ${KONG_LOGS_HTTP_PORT} # Settings for connecting to Amplify Central central: url: ${CENTRAL_URL:https://apicentral.axway.com} diff --git a/go.mod b/go.mod index b71276b..9284d50 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,13 @@ module github.com/Axway/agents-kong go 1.18 require ( - github.com/Axway/agent-sdk v1.1.68-0.20231120204404-5fa2c5b231e1 + github.com/Axway/agent-sdk v1.1.72-0.20240103224310-c9edbdc66989 github.com/elastic/beats/v7 v7.17.15 github.com/google/uuid v1.3.1 github.com/kong/go-kong v0.47.0 github.com/mitchellh/mapstructure v1.5.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 - github.com/tidwall/gjson v1.16.0 ) require ( @@ -124,6 +123,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.12.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect + github.com/tidwall/gjson v1.16.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect diff --git a/go.sum b/go.sum index 5cdef8d..13b5240 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Axway/agent-sdk v1.1.68-0.20231120204404-5fa2c5b231e1 h1:UPRwbD7Hb/p4CpSHv0hDwOmjbSqYy5Fk7pSnnatDwes= -github.com/Axway/agent-sdk v1.1.68-0.20231120204404-5fa2c5b231e1/go.mod h1:Iuv9KlWksVTbTKdfs4bKVYMDc33ZTLYoHt572z2CbbI= +github.com/Axway/agent-sdk v1.1.72-0.20240103224310-c9edbdc66989 h1:ptdjxNICx+Ftau9usJbKdI0MHxpG2tjgzmW6X+MNhlc= +github.com/Axway/agent-sdk v1.1.72-0.20240103224310-c9edbdc66989/go.mod h1:Iuv9KlWksVTbTKdfs4bKVYMDc33ZTLYoHt572z2CbbI= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= diff --git a/helm/kong-agents/Chart.yaml b/helm/kong-agents/Chart.yaml index 9c86696..fb7679d 100644 --- a/helm/kong-agents/Chart.yaml +++ b/helm/kong-agents/Chart.yaml @@ -19,10 +19,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.0.0" +appVersion: "v0.0.1" diff --git a/helm/kong-agents/templates/_helpers.tpl b/helm/kong-agents/templates/_helpers.tpl index 9c0638d..3a9935c 100644 --- a/helm/kong-agents/templates/_helpers.tpl +++ b/helm/kong-agents/templates/_helpers.tpl @@ -42,6 +42,14 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} +{{/* +Traceability selector labels +*/}} +{{- define "kong-agents.traceability.selectorLabels" -}} +{{ include "kong-agents.selectorLabels" . }} +app.agent.type: traceability +{{- end }} + {{/* Selector labels */}} @@ -62,8 +70,22 @@ Create the name of the service account to use {{- end }} {{/* -Create the name of the service account to use +Create the env var value for spec download paths */}} -{{- define "kong-agents.specDownloadPathsString" -}} +{{- define "kong-agents.spec.urlPaths.string" -}} {{- join "," .Values.kong.spec.urlPaths }} +{{- end -}} + +{{/* +Create the env var value for ssl next protos +*/}} +{{- define "kong-agents.admin.ssl.nextProtos.string" -}} +{{- join "," .Values.kong.admin.ssl.nextProtos }} +{{- end -}} + +{{/* +Create the env var value for ssl cipher suites +*/}} +{{- define "kong-agents.admin.ssl.cipherSuites.string" -}} +{{- join "," .Values.kong.admin.ssl.cipherSuites }} {{- end -}} \ No newline at end of file diff --git a/helm/kong-agents/templates/discovery-deployment.yaml b/helm/kong-agents/templates/discovery-deployment.yaml index e55c05b..6af4e61 100644 --- a/helm/kong-agents/templates/discovery-deployment.yaml +++ b/helm/kong-agents/templates/discovery-deployment.yaml @@ -66,22 +66,44 @@ spec: {{- end }} {{- end }} env: - - name: KONG_ADMIN_URL - value: "{{ .Values.kong.admin.url }}" - name: KONG_PROXY_HOST value: "{{ .Values.kong.proxy.host }}" - - name: KONG_PROXY_PORTS_HTTP - value: "{{ .Values.kong.proxy.ports.http }}" - - name: KONG_PROXY_PORTS_HTTPS - value: "{{ .Values.kong.proxy.ports.https }}" - {{- if (include "kong-agents.specDownloadPathsString" .) }} + - name: KONG_ACL_DISABLE + value: "{{ .Values.kong.acl.disable }}" + - name: KONG_PROXY_BASEPATH + value: "{{ .Values.kong.proxy.basePath }}" + - name: KONG_PROXY_PORTS_HTTP_VALUE + value: "{{ .Values.kong.proxy.ports.http.value }}" + - name: KONG_PROXY_PORTS_HTTPS_VALUE + value: "{{ .Values.kong.proxy.ports.https.value }}" + - name: KONG_PROXY_PORTS_HTTP_DISABLE + value: "{{ .Values.kong.proxy.ports.http.disable }}" + - name: KONG_PROXY_PORTS_HTTPS_DISABLE + value: "{{ .Values.kong.proxy.ports.https.disable }}" + - name: KONG_ADMIN_URL + value: "{{ .Values.kong.admin.url }}" + - name: KONG_ADMIN_SSL_NEXTPROTOS + value: {{ include "kong-agents.admin.ssl.nextProtos.string" . }} + - name: KONG_ADMIN_SSL_CIPHERSUITES + value: {{ include "kong-agents.admin.ssl.cipherSuites.string" . }} + - name: KONG_ADMIN_SSL_INSECURESKIPVERIFY + value: "{{ .Values.kong.admin.ssl.insecureSkipVerify }}" + - name: KONG_ADMIN_SSL_MAXVERSION + value: "{{ .Values.kong.admin.ssl.maxVersion }}" + - name: KONG_ADMIN_SSL_MINVERSION + value: "{{ .Values.kong.admin.ssl.minVersion }}" + {{- if (include "kong-agents.spec.urlPaths.string" .) }} - name: KONG_SPEC_URLPATHS - value: {{ include "kong-agents.specDownloadPathsString" . }} + value: {{ include "kong-agents.spec.urlPaths.string" . }} {{- end }} {{- if .Values.kong.spec.localPath }} - name: KONG_SPEC_LOCALPATH value: /specs {{- end }} + {{- if .Values.kong.spec.filter }} + - name: KONG_SPEC_FILTER + value: "{{ .Values.kong.spec.filter }}" + {{- end }} {{- if .Values.kong.admin.auth.apikey.value }} - name: KONG_ADMIN_AUTH_APIKEY_VALUE valueFrom: @@ -94,18 +116,44 @@ spec: name: kong-admin-auth-apikey key: header {{- end }} + {{- if .Values.kong.admin.auth.basicAuth.username }} + - name: KONG_ADMIN_AUTH_BASICAUTH_USERNAME + valueFrom: + secretKeyRef: + name: kong-admin-auth-basicauth + key: username + - name: KONG_ADMIN_AUTH_BASICAUTH_PASSWORD + valueFrom: + secretKeyRef: + name: kong-admin-auth-basicauth + key: password + {{- end }} {{- with .Values.env }} {{- range $key, $value := . }} {{- if and (not (eq (toString $value) "")) (not (eq (toString $key) "")) - (not (has (toString $key) (list "KONG_ADMIN_URL" + (not (has (toString $key) (list "KONG_ACL_DISABLE" + "KONG_ADMIN_URL" "KONG_ADMIN_AUTH_APIKEY_HEADER" - "KONG_ADMIN_AUTH_APIKEY_VALUE" + "KONG_ADMIN_AUTH_APIKEY_VALUE" + "KONG_ADMIN_AUTH_BASICAUTH_USERNAME" + "KONG_ADMIN_AUTH_BASICAUTH_PASSWORD" + "KONG_ADMIN_SSL_NEXTPROTOS" + "KONG_ADMIN_SSL_CIPHERSUITES" + "KONG_ADMIN_SSL_INSECURESKIPVERIFY" + "KONG_ADMIN_SSL_MAXVERSION" + "KONG_ADMIN_SSL_MINVERSION" "KONG_PROXY_HOST" - "KONG_PROXY_PORTS_HTTP" - "KONG_PROXY_PORTS_HTTPS" + "KONG_PROXY_BASEPATH" + "KONG_PROXY_PORTS_HTTP_VALUE" + "KONG_PROXY_PORTS_HTTPS_VALUE" + "KONG_PROXY_PORTS_HTTP_DISABLE" + "KONG_PROXY_PORTS_HTTPS_DISABLE" "KONG_SPEC_LOCALPATH" - "KONG_SPEC_URLPATHS"))) + "KONG_SPEC_URLPATHS" + "KONG_LOGS_HTTP_SERVER_PATH" + "KONG_LOGS_HTTP_SERVER_PORT" + "STATUS_PORT"))) }} - name: {{ $key }} value: {{ $value | quote }} @@ -116,6 +164,8 @@ spec: value: "/keys/private_key.pem" - name: CENTRAL_AUTH_PUBLICKEY value: "/keys/public_key.pem" + - name: STATUS_PORT + value: "{{ .Values.statusPort }}" volumeMounts: - name: "kong-agent-keys" mountPath: "/keys" diff --git a/helm/kong-agents/templates/kong-admin-auth-basicauth.yaml b/helm/kong-agents/templates/kong-admin-auth-basicauth.yaml new file mode 100644 index 0000000..fde0aa5 --- /dev/null +++ b/helm/kong-agents/templates/kong-admin-auth-basicauth.yaml @@ -0,0 +1,10 @@ +{{- if .Values.kong.admin.auth.basicAuth.username }} +apiVersion: v1 +kind: Secret +metadata: + name: kong-admin-auth-basicauth +type: Opaque +stringData: + username: "{{ .Values.kong.admin.auth.basicAuth.username }}" + password: "{{ .Values.kong.admin.auth.basicAuth.password }}" +{{- end }} \ No newline at end of file diff --git a/helm/kong-agents/templates/service.yaml b/helm/kong-agents/templates/service.yaml new file mode 100644 index 0000000..023a238 --- /dev/null +++ b/helm/kong-agents/templates/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.kong.enable.traceability }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kong-agents.fullname" . }}-traceability + labels: + {{- include "kong-agents.labels" . | nindent 4 }} +spec: + selector: + {{- include "kong-agents.selectorLabels" . | nindent 4 }} + app.agent.type: traceability + ports: + - name: logs + protocol: TCP + port: {{ .Values.kong.logs.http.port }} + targetPort: logs +{{- end -}} \ No newline at end of file diff --git a/helm/kong-agents/templates/traceability-deployment.yaml b/helm/kong-agents/templates/traceability-statefulset.yaml similarity index 62% rename from helm/kong-agents/templates/traceability-deployment.yaml rename to helm/kong-agents/templates/traceability-statefulset.yaml index c1a85d0..47b281a 100644 --- a/helm/kong-agents/templates/traceability-deployment.yaml +++ b/helm/kong-agents/templates/traceability-statefulset.yaml @@ -1,6 +1,6 @@ {{- if .Values.kong.enable.traceability }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ include "kong-agents.fullname" . }}-traceability labels: @@ -9,7 +9,7 @@ spec: replicas: {{ .Values.traceability.replicaCount }} selector: matchLabels: - {{- include "kong-agents.selectorLabels" . | nindent 6 }} + {{- include "kong-agents.traceability.selectorLabels" . | nindent 6 }} {{- with .Values.additionalLabels }} {{- range $key, $value := . }} {{ default "none" $key }}: {{ default "none" $value | quote }} @@ -22,7 +22,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "kong-agents.selectorLabels" . | nindent 8 }} + {{- include "kong-agents.traceability.selectorLabels" . | nindent 8 }} {{- with .Values.additionalLabels }} {{- range $key, $value := . }} {{ default "none" $key }}: {{ default "none" $value | quote }} @@ -48,6 +48,9 @@ spec: - name: probe-port containerPort: {{ .Values.statusPort }} protocol: TCP + - name: logs + containerPort: {{ .Values.kong.logs.http.port }} + protocol: TCP livenessProbe: httpGet: path: /status @@ -67,44 +70,62 @@ spec: {{- end }} {{- end }} env: - - name: KONG_ADMIN_URL - value: "{{ .Values.kong.admin.url }}" - {{- if .Values.kong.admin.auth.apikey.value }} - - name: KONG_ADMIN_AUTH_APIKEY_VALUE - valueFrom: - secretKeyRef: - name: kong-admin-auth-apikey - key: value - - name: KONG_ADMIN_AUTH_APIKEY_HEADER - valueFrom: - secretKeyRef: - name: kong-admin-auth-apikey - key: header - {{- end }} + - name: KONG_LOGS_HTTP_SERVER_PATH + value: "{{ .Values.kong.logs.http.path }}" + - name: KONG_LOGS_HTTP_SERVER_PORT + value: "{{ .Values.kong.logs.http.port }}" {{- with .Values.env }} {{- range $key, $value := . }} {{- if and (not (eq (toString $value) "")) (not (eq (toString $key) "")) - (not (has (toString $key) (list "KONG_ADMIN_URL" + (not (has (toString $key) (list "KONG_ACL_DISABLE" + "KONG_ADMIN_URL" "KONG_ADMIN_AUTH_APIKEY_HEADER" - "KONG_ADMIN_AUTH_APIKEY_VALUE" + "KONG_ADMIN_AUTH_APIKEY_VALUE" + "KONG_ADMIN_AUTH_BASICAUTH_USERNAME" + "KONG_ADMIN_AUTH_BASICAUTH_PASSWORD" + "KONG_ADMIN_SSL_NEXTPROTOS" + "KONG_ADMIN_SSL_CIPHERSUITES" + "KONG_ADMIN_SSL_INSECURESKIPVERIFY" + "KONG_ADMIN_SSL_MAXVERSION" + "KONG_ADMIN_SSL_MINVERSION" "KONG_PROXY_HOST" - "KONG_PROXY_PORTS_HTTP" - "KONG_PROXY_PORTS_HTTPS" + "KONG_PROXY_BASEPATH" + "KONG_PROXY_PORTS_HTTP_VALUE" + "KONG_PROXY_PORTS_HTTPS_VALUE" + "KONG_PROXY_PORTS_HTTP_DISABLE" + "KONG_PROXY_PORTS_HTTPS_DISABLE" "KONG_SPEC_LOCALPATH" - "KONG_SPEC_URLPATHS"))) + "KONG_SPEC_URLPATHS" + "KONG_LOGS_HTTP_SERVER_PATH" + "KONG_LOGS_HTTP_SERVER_PORT" + "STATUS_PORT"))) }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} {{- end }} {{- end }} + - name: CENTRAL_AUTH_PRIVATEKEY + value: "/keys/private_key.pem" + - name: CENTRAL_AUTH_PUBLICKEY + value: "/keys/public_key.pem" + - name: STATUS_PORT + value: "{{ .Values.statusPort }}" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name volumeMounts: + - name: beat-storage + mountPath: /data - name: "kong-agent-keys" mountPath: "/keys" resources: {{- toYaml .Values.resources | nindent 12 }} volumes: + - name: beat-storage + emptyDir: {} - name: kong-agent-keys secret: secretName: {{ .Values.secrets.keys }} diff --git a/helm/kong-agents/values.yaml b/helm/kong-agents/values.yaml index 0fb0630..1443ebc 100644 --- a/helm/kong-agents/values.yaml +++ b/helm/kong-agents/values.yaml @@ -21,6 +21,8 @@ fullnameOverride: "" statusPort: 8989 kong: + acl: + disable: false enable: traceability: false admin: @@ -29,14 +31,33 @@ kong: apikey: header: value: + basicAuth: + username: + password: + ssl: + nextProtos: [] + insecureSkipVerify: + cipherSuites: [] + minVersion: + maxVersion: proxy: host: + basePath: ports: - http: 8000 - https: 8443 + http: + disable: + value: + https: + disable: + value: spec: + filter: urlPaths: [] localPath: + logs: + http: + path: + port: # Add any environment variable overrides here env: {} diff --git a/pkg/common/common.go b/pkg/common/common.go index 80be816..77650ff 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -2,11 +2,12 @@ package common const ( AttrServiceID = "serviceID" - AttrRouteID = "routeID" - AttrChecksum = "checksum" - AttrAppID = "kongApplicationID" AttrServiceName = "serviceName" AttrRouteName = "routeName" + AttrRouteID = "routeID" + AttrServiceTag = "serviceTag" + AttrChecksum = "checksum" + AttrAppID = "kongApplicationId" AttrCredentialID = "kongCredentialID" AttrCredUpdater = "kongCredentialUpdate" diff --git a/pkg/config/discovery/config.go b/pkg/config/discovery/config.go deleted file mode 100644 index 03b8045..0000000 --- a/pkg/config/discovery/config.go +++ /dev/null @@ -1,114 +0,0 @@ -package config - -import ( - "fmt" - - "github.com/Axway/agent-sdk/pkg/cmd/properties" - corecfg "github.com/Axway/agent-sdk/pkg/config" -) - -const ( - cfgKongAdminURL = "kong.admin.url" - cfgKongAdminAPIKey = "kong.admin.auth.apikey.value" - cfgKongAdminAPIKeyHeader = "kong.admin.auth.apikey.header" - cfgKongProxyHost = "kong.proxy.host" - cfgKongProxyPortHttp = "kong.proxy.port.http" - cfgKongProxyPortHttps = "kong.proxy.port.https" - cfgKongSpecURLPaths = "kong.spec.urlPaths" - cfgKongSpecLocalPath = "kong.spec.localPath" -) - -func AddKongProperties(rootProps properties.Properties) { - rootProps.AddStringProperty(cfgKongAdminURL, "", "The Kong admin endpoint") - rootProps.AddStringProperty(cfgKongAdminAPIKey, "", "API Key value to authenticate with Kong Gateway") - rootProps.AddStringProperty(cfgKongAdminAPIKeyHeader, "", "API Key header to authenticate with Kong Gateway") - rootProps.AddStringProperty(cfgKongProxyHost, "", "The Kong proxy endpoint") - rootProps.AddIntProperty(cfgKongProxyPortHttp, 80, "The Kong proxy http port") - rootProps.AddIntProperty(cfgKongProxyPortHttps, 443, "The Kong proxy https port") - rootProps.AddStringSliceProperty(cfgKongSpecURLPaths, []string{}, "URL paths that the agent will look in for spec files") - rootProps.AddStringProperty(cfgKongSpecLocalPath, "", "Local paths where the agent will look for spec files") -} - -// AgentConfig - represents the config for agent -type AgentConfig struct { - CentralCfg corecfg.CentralConfig `config:"central"` - KongGatewayCfg *KongGatewayConfig `config:"kong"` -} - -type KongAdminConfig struct { - URL string `config:"url"` - Auth KongAdminAuthConfig `config:"auth"` -} - -type KongAdminAuthConfig struct { - APIKey KongAdminAuthAPIKeyConfig `config:"apikey"` -} - -type KongAdminAuthAPIKeyConfig struct { - Header string `config:"header"` - Value string `config:"value"` -} - -type KongProxyConfig struct { - Host string `config:"host"` - Port KongProxyPortConfig `config:"port"` -} - -type KongProxyPortConfig struct { - HTTP int `config:"http"` - HTTPS int `config:"https"` -} - -type KongSpecConfig struct { - URLPaths []string `config:"urlPaths"` - LocalPath string `config:"localPath"` - DevPortalEnabled bool `config:"devPortalEnabled"` -} - -// KongGatewayConfig - represents the config for gateway -type KongGatewayConfig struct { - corecfg.IConfigValidator - Admin KongAdminConfig `config:"admin"` - Proxy KongProxyConfig `config:"proxy"` - Spec KongSpecConfig `config:"spec"` -} - -// ValidateCfg - Validates the gateway config -func (c *KongGatewayConfig) ValidateCfg() (err error) { - if c.Admin.URL == "" { - return fmt.Errorf("error: admin url is required") - } - if c.Proxy.Host == "" { - return fmt.Errorf("error: proxy host is required") - } - if c.Proxy.Port.HTTP == 0 && c.Proxy.Port.HTTPS == 0 { - return fmt.Errorf("error: at least one proxy port value of either http or https is required") - } - return -} - -func ParseProperties(rootProps properties.Properties) *KongGatewayConfig { - // Parse the config from bound properties and setup gateway config - return &KongGatewayConfig{ - Admin: KongAdminConfig{ - URL: rootProps.StringPropertyValue(cfgKongAdminURL), - Auth: KongAdminAuthConfig{ - APIKey: KongAdminAuthAPIKeyConfig{ - Value: rootProps.StringPropertyValue(cfgKongAdminAPIKey), - Header: rootProps.StringPropertyValue(cfgKongAdminAPIKeyHeader), - }, - }, - }, - Proxy: KongProxyConfig{ - Host: rootProps.StringPropertyValue(cfgKongProxyHost), - Port: KongProxyPortConfig{ - HTTP: rootProps.IntPropertyValue(cfgKongProxyPortHttp), - HTTPS: rootProps.IntPropertyValue(cfgKongProxyPortHttps), - }, - }, - Spec: KongSpecConfig{ - URLPaths: rootProps.StringSlicePropertyValue(cfgKongSpecURLPaths), - LocalPath: rootProps.StringPropertyValue(cfgKongSpecLocalPath), - }, - } -} diff --git a/pkg/cmd/discovery/cmd.go b/pkg/discovery/cmd/cmd.go similarity index 91% rename from pkg/cmd/discovery/cmd.go rename to pkg/discovery/cmd/cmd.go index 21f3ef8..5f39177 100644 --- a/pkg/cmd/discovery/cmd.go +++ b/pkg/discovery/cmd/cmd.go @@ -7,8 +7,8 @@ import ( corecfg "github.com/Axway/agent-sdk/pkg/config" "github.com/Axway/agent-sdk/pkg/util/log" - config "github.com/Axway/agents-kong/pkg/config/discovery" - "github.com/Axway/agents-kong/pkg/gateway" + "github.com/Axway/agents-kong/pkg/discovery/config" + "github.com/Axway/agents-kong/pkg/discovery/gateway" ) var DiscoveryCmd corecmd.AgentRootCmd @@ -62,14 +62,9 @@ func run() error { // and passed to the callback allowing the agent code to access the central config func initConfig(centralConfig corecfg.CentralConfig) (interface{}, error) { rootProps := DiscoveryCmd.GetProperties() - agentConfig = config.AgentConfig{ CentralCfg: centralConfig, KongGatewayCfg: config.ParseProperties(rootProps), } return agentConfig, nil } - -func GetAgentConfig() config.AgentConfig { - return agentConfig -} diff --git a/pkg/discovery/config/config.go b/pkg/discovery/config/config.go new file mode 100644 index 0000000..d1dca64 --- /dev/null +++ b/pkg/discovery/config/config.go @@ -0,0 +1,276 @@ +package config + +import ( + "fmt" + "net/url" + "strings" + + corecfg "github.com/Axway/agent-sdk/pkg/config" + "github.com/Axway/agent-sdk/pkg/util/log" +) + +type props interface { + AddStringProperty(name string, defaultVal string, description string) + AddStringSliceProperty(name string, defaultVal []string, description string) + AddIntProperty(name string, defaultVal int, description string) + AddBoolProperty(name string, defaultVal bool, description string) + StringPropertyValue(name string) string + StringSlicePropertyValue(name string) []string + IntPropertyValue(name string) int + BoolPropertyValue(name string) bool +} + +const ( + cfgKongACLDisable = "kong.acl.disable" + cfgKongAdminUrl = "kong.admin.url" + cfgKongAdminAPIKey = "kong.admin.auth.apiKey.value" + cfgKongAdminAPIKeyHeader = "kong.admin.auth.apiKey.header" + cfgKongAdminBasicUsername = "kong.admin.auth.basicauth.username" + cfgKongAdminBasicPassword = "kong.admin.auth.basicauth.password" + cfgKongAdminSSLNextProto = "kong.admin.ssl.nextProtos" + cfgKongAdminSSLInsecureSkipVerify = "kong.admin.ssl.insecureSkipVerify" + cfgKongAdminSSLCipherSuites = "kong.admin.ssl.cipherSuites" + cfgKongAdminSSLMinVersion = "kong.admin.ssl.minVersion" + cfgKongAdminSSLMaxVersion = "kong.admin.ssl.maxVersion" + cfgKongProxyHost = "kong.proxy.host" + cfgKongProxyPortHttp = "kong.proxy.ports.http.value" + cfgKongProxyPortHttpDisable = "kong.proxy.ports.http.disable" + cfgKongProxyPortHttps = "kong.proxy.ports.https.value" + cfgKongProxyPortHttpsDisable = "kong.proxy.ports.https.disable" + cfgKongProxyBasePath = "kong.proxy.basePath" + cfgKongSpecURLPaths = "kong.spec.urlPaths" + cfgKongSpecLocalPath = "kong.spec.localPath" + cfgKongSpecFilter = "kong.spec.filter" + cfgKongSpecDevPortal = "kong.spec.devPortalEnabled" +) + +func AddKongProperties(rootProps props) { + rootProps.AddBoolProperty(cfgKongACLDisable, false, "Disable the check for a globally enabled ACL plugin on Kong. False by default.") + rootProps.AddStringProperty(cfgKongAdminUrl, "", "The Admin API url") + rootProps.AddStringProperty(cfgKongAdminAPIKey, "", "API Key value to authenticate with Kong Gateway") + rootProps.AddStringProperty(cfgKongAdminAPIKeyHeader, "", "API Key header to authenticate with Kong Gateway") + rootProps.AddStringProperty(cfgKongAdminBasicUsername, "", "Username for basic auth to authenticate with Kong Admin API") + rootProps.AddStringProperty(cfgKongAdminBasicPassword, "", "Password for basic auth to authenticate with Kong Admin API") + rootProps.AddStringSliceProperty(cfgKongAdminSSLNextProto, []string{}, "List of supported application level protocols, comma separated") + rootProps.AddBoolProperty(cfgKongAdminSSLInsecureSkipVerify, false, "Controls whether a client verifies the server's certificate chain and host name") + rootProps.AddStringSliceProperty(cfgKongAdminSSLCipherSuites, corecfg.TLSDefaultCipherSuitesStringSlice(), "List of supported cipher suites, comma separated") + rootProps.AddStringProperty(cfgKongAdminSSLMinVersion, corecfg.TLSDefaultMinVersionString(), "Minimum acceptable SSL/TLS protocol version") + rootProps.AddStringProperty(cfgKongAdminSSLMaxVersion, "0", "Maximum acceptable SSL/TLS protocol version") + rootProps.AddStringProperty(cfgKongProxyHost, "", "The Kong proxy endpoint") + rootProps.AddIntProperty(cfgKongProxyPortHttp, 80, "The Kong proxy http port") + rootProps.AddBoolProperty(cfgKongProxyPortHttpDisable, false, "Set to true to disable adding an http endpoint to discovered routes") + rootProps.AddIntProperty(cfgKongProxyPortHttps, 443, "The Kong proxy https port") + rootProps.AddBoolProperty(cfgKongProxyPortHttpsDisable, false, "Set to true to disable adding an https endpoint to discovered routes") + rootProps.AddStringProperty(cfgKongProxyBasePath, "", "The base path for the Kong proxy endpoint") + rootProps.AddStringSliceProperty(cfgKongSpecURLPaths, []string{}, "URL paths that the agent will look in for spec files") + rootProps.AddStringProperty(cfgKongSpecLocalPath, "", "Local paths where the agent will look for spec files") + rootProps.AddStringProperty(cfgKongSpecFilter, "", "SDK Filter format. Empty means filters are ignored.") + rootProps.AddBoolProperty(cfgKongSpecDevPortal, false, "Set to true to enable gathering specs from teh Kong's dev portal.") +} + +// AgentConfig - represents the config for agent +type AgentConfig struct { + CentralCfg corecfg.CentralConfig `config:"central"` + KongGatewayCfg *KongGatewayConfig `config:"kong"` +} + +type KongAdminConfig struct { + Url string `config:"url"` + Auth KongAdminAuthConfig `config:"auth"` + TLS corecfg.TLSConfig `config:"ssl"` +} + +type KongAdminAuthConfig struct { + APIKey KongAdminAuthAPIKeyConfig `config:"apiKey"` + BasicAuth KongAdminBasicAuthConfig `config:"basicAuth"` +} + +type KongAdminBasicAuthConfig struct { + Username string `config:"username"` + Password string `config:"password"` +} + +type KongAdminAuthAPIKeyConfig struct { + Header string `config:"header"` + Value string `config:"value"` +} + +type KongProxyConfig struct { + Host string `config:"host"` + Ports KongPortConfig `config:"ports"` + BasePath string `config:"basePath"` +} + +type KongPortConfig struct { + HTTP KongPortSettingsConfig `config:"http"` + HTTPS KongPortSettingsConfig `config:"https"` +} + +type KongPortSettingsConfig struct { + Value int `config:"value"` + Disable bool `config:"disable"` +} + +type KongSpecConfig struct { + URLPaths []string `config:"urlPaths"` + LocalPath string `config:"localPath"` + DevPortalEnabled bool `config:"devPortalEnabled"` + Filter string `config:"filter"` +} + +type KongACLConfig struct { + Disable bool `config:"disable"` +} + +// KongGatewayConfig - represents the config for gateway +type KongGatewayConfig struct { + corecfg.IConfigValidator + Admin KongAdminConfig `config:"admin"` + Proxy KongProxyConfig `config:"proxy"` + Spec KongSpecConfig `config:"spec"` + ACL KongACLConfig `config:"acl"` +} + +const ( + hostErr = "kong host must be provided" + httpPortErr = "a non-zero value is required for the http port number when it is enabled" + httpsPortErr = "a non-zero value is required for the https port number when it is enabled" + basePathPrefixErr = "the base path must start with a '/' character" + basePathSuffixErr = "the base path must not end with a '/' character" + portErr = "at least one port endpoint needs to be enabled" + invalidUrlErr = "invalid Admin API url provided. Must contain protocol + hostname + port." + + "Examples: , " + credentialConfigErr = "invalid authorization configuration provided. " + + "If provided, (Username and Password) or (ClientID and ClientSecret) must be non-empty" +) + +// ValidateCfg - Validates the gateway config +func (c *KongGatewayConfig) ValidateCfg() error { + logger := log.NewFieldLogger().WithPackage("config").WithComponent("ValidateConfig") + if c.Proxy.Host == "" { + return fmt.Errorf(hostErr) + } + if !c.Proxy.Ports.HTTP.Disable && c.Proxy.Ports.HTTP.Value == 0 { + return fmt.Errorf(httpPortErr) + } + if len(c.Proxy.BasePath) > 0 && !strings.HasPrefix(c.Proxy.BasePath, "/") { + return fmt.Errorf(basePathPrefixErr) + } + if len(c.Proxy.BasePath) > 0 && strings.HasSuffix(c.Proxy.BasePath, "/") { + return fmt.Errorf(basePathSuffixErr) + } + if !c.Proxy.Ports.HTTPS.Disable && c.Proxy.Ports.HTTPS.Value == 0 { + return fmt.Errorf(httpsPortErr) + } + if c.Proxy.Ports.HTTP.Disable && c.Proxy.Ports.HTTPS.Disable { + return fmt.Errorf(portErr) + } + if invalidAdminUrl(c.Admin.Url) { + return fmt.Errorf(invalidUrlErr) + } + if noCredentialsProvided(c) { + logger.Warn("No credentials provided. Assuming Kong Admin API requires no authorization.") + } + if invalidCredentialConfig(c) { + return fmt.Errorf(credentialConfigErr) + } + if tlsValidate, validator := c.Admin.TLS.(corecfg.IConfigValidator); validator { + if err := tlsValidate.ValidateCfg(); err != nil { + return fmt.Errorf("kong.admin.%s", err.Error()) + } + } + return nil +} + +func noCredentialsProvided(c *KongGatewayConfig) bool { + apiKey := c.Admin.Auth.APIKey.Value + user := c.Admin.Auth.BasicAuth.Username + pass := c.Admin.Auth.BasicAuth.Password + + if apiKey == "" && user == "" && pass == "" { + return true + } + return false +} + +func invalidAdminUrl(u string) bool { + parsedUrl, err := url.Parse(u) + if err != nil { + return true + } + if parsedUrl.Port() == "" || + strings.HasPrefix(parsedUrl.Host, "http://") || strings.HasPrefix(parsedUrl.Host, "https://") { + return true + } + return false +} + +func invalidCredentialConfig(c *KongGatewayConfig) bool { + user := c.Admin.Auth.BasicAuth.Username + pass := c.Admin.Auth.BasicAuth.Password + + if (user == "" && pass != "") || + (user != "" && pass == "") { + return true + } + return false +} + +func ParseProperties(rootProps props) *KongGatewayConfig { + // Parse the config from bound properties and setup gateway config + httpPortConf := KongPortSettingsConfig{ + Disable: rootProps.BoolPropertyValue(cfgKongProxyPortHttpDisable), + Value: rootProps.IntPropertyValue(cfgKongProxyPortHttp), + } + if httpPortConf.Disable { + httpPortConf.Value = 0 + } + + httpsPortConf := KongPortSettingsConfig{ + Disable: rootProps.BoolPropertyValue(cfgKongProxyPortHttpsDisable), + Value: rootProps.IntPropertyValue(cfgKongProxyPortHttps), + } + if httpsPortConf.Disable { + httpsPortConf.Value = 0 + } + + return &KongGatewayConfig{ + ACL: KongACLConfig{ + Disable: rootProps.BoolPropertyValue(cfgKongACLDisable), + }, + Admin: KongAdminConfig{ + Url: rootProps.StringPropertyValue(cfgKongAdminUrl), + Auth: KongAdminAuthConfig{ + APIKey: KongAdminAuthAPIKeyConfig{ + Value: rootProps.StringPropertyValue(cfgKongAdminAPIKey), + Header: rootProps.StringPropertyValue(cfgKongAdminAPIKeyHeader), + }, + BasicAuth: KongAdminBasicAuthConfig{ + Username: rootProps.StringPropertyValue(cfgKongAdminBasicUsername), + Password: rootProps.StringPropertyValue(cfgKongAdminBasicPassword), + }, + }, + TLS: &corecfg.TLSConfiguration{ + NextProtos: rootProps.StringSlicePropertyValue(cfgKongAdminSSLNextProto), + InsecureSkipVerify: rootProps.BoolPropertyValue(cfgKongAdminSSLInsecureSkipVerify), + CipherSuites: corecfg.NewCipherArray(rootProps.StringSlicePropertyValue(cfgKongAdminSSLCipherSuites)), + MinVersion: corecfg.TLSVersionAsValue(rootProps.StringPropertyValue(cfgKongAdminSSLMinVersion)), + MaxVersion: corecfg.TLSVersionAsValue(rootProps.StringPropertyValue(cfgKongAdminSSLMaxVersion)), + }, + }, + Proxy: KongProxyConfig{ + Host: rootProps.StringPropertyValue(cfgKongProxyHost), + Ports: KongPortConfig{ + HTTP: httpPortConf, + HTTPS: httpsPortConf, + }, + BasePath: rootProps.StringPropertyValue(cfgKongProxyBasePath), + }, + Spec: KongSpecConfig{ + DevPortalEnabled: rootProps.BoolPropertyValue(cfgKongSpecDevPortal), + URLPaths: rootProps.StringSlicePropertyValue(cfgKongSpecURLPaths), + LocalPath: rootProps.StringPropertyValue(cfgKongSpecLocalPath), + Filter: rootProps.StringPropertyValue(cfgKongSpecFilter), + }, + } +} diff --git a/pkg/discovery/config/config_test.go b/pkg/discovery/config/config_test.go new file mode 100644 index 0000000..99c6d4c --- /dev/null +++ b/pkg/discovery/config/config_test.go @@ -0,0 +1,200 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKongGatewayCfg(t *testing.T) { + cfg := &KongGatewayConfig{} + + err := cfg.ValidateCfg() + assert.Equal(t, hostErr, err.Error()) + + cfg.Proxy.Host = "localhost" + err = cfg.ValidateCfg() + assert.Equal(t, httpPortErr, err.Error()) + + cfg.Proxy.Ports.HTTP.Value = 8000 + err = cfg.ValidateCfg() + assert.Equal(t, httpsPortErr, err.Error()) + + cfg.Proxy.Ports.HTTPS.Value = 8443 + cfg.Proxy.Ports.HTTP.Disable = true + cfg.Proxy.Ports.HTTPS.Disable = true + err = cfg.ValidateCfg() + assert.Equal(t, portErr, err.Error()) + + cfg.Proxy.Ports.HTTP.Disable = false + cfg.Proxy.BasePath = "base" + err = cfg.ValidateCfg() + assert.Equal(t, basePathPrefixErr, err.Error()) + + cfg.Proxy.BasePath = "/base/" + err = cfg.ValidateCfg() + assert.Equal(t, basePathSuffixErr, err.Error()) + + cfg.Proxy.BasePath = "/base" + cfg.Admin.Url = "sdl.com:8000" + err = cfg.ValidateCfg() + assert.Equal(t, invalidUrlErr, err.Error()) + + cfg.Admin.Url = "http://sdl.com" + err = cfg.ValidateCfg() + assert.Equal(t, invalidUrlErr, err.Error()) + + cfg.Admin.Url = "https://sds.com:8000" + cfg.Admin.Auth.BasicAuth.Username = "test" + err = cfg.ValidateCfg() + assert.Equal(t, credentialConfigErr, err.Error()) + + cfg.Admin.Auth.BasicAuth.Username = "" + cfg.Admin.Auth.BasicAuth.Password = "sas" + err = cfg.ValidateCfg() + assert.Equal(t, credentialConfigErr, err.Error()) + + cfg.Admin.Auth.BasicAuth.Password = "" + + err = cfg.ValidateCfg() + assert.Equal(t, nil, err) + +} + +type propData struct { + pType string + desc string + val interface{} +} + +type fakeProps struct { + props map[string]propData +} + +func (f *fakeProps) AddStringProperty(name string, defaultVal string, description string) { + f.props[name] = propData{"string", description, defaultVal} +} + +func (f *fakeProps) AddStringSliceProperty(name string, defaultVal []string, description string) { + f.props[name] = propData{"string", description, defaultVal} +} + +func (f *fakeProps) AddIntProperty(name string, defaultVal int, description string) { + f.props[name] = propData{"int", description, defaultVal} +} + +func (f *fakeProps) AddBoolProperty(name string, defaultVal bool, description string) { + f.props[name] = propData{"bool", description, defaultVal} +} + +func (f *fakeProps) StringPropertyValue(name string) string { + if prop, ok := f.props[name]; ok { + return prop.val.(string) + } + return "" +} + +func (f *fakeProps) StringSlicePropertyValue(name string) []string { + if prop, ok := f.props[name]; ok { + return prop.val.([]string) + } + return []string{} +} + +func (f *fakeProps) IntPropertyValue(name string) int { + if prop, ok := f.props[name]; ok { + return prop.val.(int) + } + return 0 +} + +func (f *fakeProps) BoolPropertyValue(name string) bool { + if prop, ok := f.props[name]; ok { + return prop.val.(bool) + } + return false +} + +func TestKongProperties(t *testing.T) { + newProps := &fakeProps{props: map[string]propData{}} + + // validate add props + AddKongProperties(newProps) + assert.Contains(t, newProps.props, cfgKongACLDisable) + assert.Contains(t, newProps.props, cfgKongAdminUrl) + assert.Contains(t, newProps.props, cfgKongAdminAPIKey) + assert.Contains(t, newProps.props, cfgKongAdminAPIKeyHeader) + assert.Contains(t, newProps.props, cfgKongAdminBasicUsername) + assert.Contains(t, newProps.props, cfgKongAdminBasicPassword) + assert.Contains(t, newProps.props, cfgKongProxyHost) + assert.Contains(t, newProps.props, cfgKongProxyPortHttp) + assert.Contains(t, newProps.props, cfgKongProxyPortHttpDisable) + assert.Contains(t, newProps.props, cfgKongProxyPortHttps) + assert.Contains(t, newProps.props, cfgKongProxyPortHttpsDisable) + assert.Contains(t, newProps.props, cfgKongProxyBasePath) + assert.Contains(t, newProps.props, cfgKongSpecURLPaths) + assert.Contains(t, newProps.props, cfgKongSpecLocalPath) + assert.Contains(t, newProps.props, cfgKongSpecFilter) + assert.Contains(t, newProps.props, cfgKongSpecDevPortal) + + // validate defaults + cfg := ParseProperties(newProps) + assert.Equal(t, false, cfg.ACL.Disable) + assert.Equal(t, "", cfg.Admin.Url) + assert.Equal(t, "", cfg.Admin.Auth.APIKey.Value) + assert.Equal(t, "", cfg.Admin.Auth.APIKey.Header) + assert.Equal(t, "", cfg.Admin.Auth.BasicAuth.Username) + assert.Equal(t, "", cfg.Admin.Auth.BasicAuth.Password) + assert.Equal(t, "", cfg.Proxy.Host) + assert.Equal(t, 80, cfg.Proxy.Ports.HTTP.Value) + assert.Equal(t, 443, cfg.Proxy.Ports.HTTPS.Value) + assert.Equal(t, false, cfg.Proxy.Ports.HTTP.Disable) + assert.Equal(t, false, cfg.Proxy.Ports.HTTPS.Disable) + assert.Equal(t, "", cfg.Proxy.BasePath) + assert.Equal(t, []string{}, cfg.Spec.URLPaths) + assert.Equal(t, "", cfg.Spec.LocalPath) + assert.Equal(t, "", cfg.Spec.Filter) + assert.Equal(t, false, cfg.Spec.DevPortalEnabled) + + // validate changed values + newProps.props[cfgKongACLDisable] = propData{"bool", "", true} + newProps.props[cfgKongAdminUrl] = propData{"string", "", "http://host:port/path"} + newProps.props[cfgKongAdminAPIKey] = propData{"string", "", "apikey"} + newProps.props[cfgKongAdminAPIKeyHeader] = propData{"string", "", "header"} + newProps.props[cfgKongAdminBasicUsername] = propData{"string", "", "username"} + newProps.props[cfgKongAdminBasicPassword] = propData{"string", "", "password"} + newProps.props[cfgKongProxyHost] = propData{"string", "", "proxyhost"} + newProps.props[cfgKongProxyPortHttp] = propData{"int", "", 8080} + newProps.props[cfgKongProxyPortHttps] = propData{"int", "", 8443} + newProps.props[cfgKongProxyHost] = propData{"string", "", "proxyhost"} + newProps.props[cfgKongSpecURLPaths] = propData{"string", "", []string{"path1", "path2"}} + newProps.props[cfgKongSpecLocalPath] = propData{"string", "", "/path/to/specs"} + newProps.props[cfgKongSpecFilter] = propData{"string", "", "tag_filter"} + newProps.props[cfgKongSpecDevPortal] = propData{"bool", "", true} + cfg = ParseProperties(newProps) + assert.Equal(t, true, cfg.ACL.Disable) + assert.Equal(t, "http://host:port/path", cfg.Admin.Url) + assert.Equal(t, "apikey", cfg.Admin.Auth.APIKey.Value) + assert.Equal(t, "header", cfg.Admin.Auth.APIKey.Header) + assert.Equal(t, "username", cfg.Admin.Auth.BasicAuth.Username) + assert.Equal(t, "password", cfg.Admin.Auth.BasicAuth.Password) + assert.Equal(t, "proxyhost", cfg.Proxy.Host) + assert.Equal(t, 8080, cfg.Proxy.Ports.HTTP.Value) + assert.Equal(t, 8443, cfg.Proxy.Ports.HTTPS.Value) + assert.Equal(t, false, cfg.Proxy.Ports.HTTP.Disable) + assert.Equal(t, false, cfg.Proxy.Ports.HTTPS.Disable) + assert.Equal(t, "", cfg.Proxy.BasePath) + assert.Equal(t, []string{"path1", "path2"}, cfg.Spec.URLPaths) + assert.Equal(t, "/path/to/specs", cfg.Spec.LocalPath) + assert.Equal(t, "tag_filter", cfg.Spec.Filter) + assert.Equal(t, true, cfg.Spec.DevPortalEnabled) + + // validate no port configured when port type disabled + newProps.props[cfgKongProxyPortHttpDisable] = propData{"bool", "", true} + newProps.props[cfgKongProxyPortHttpsDisable] = propData{"bool", "", true} + cfg = ParseProperties(newProps) + assert.Equal(t, 0, cfg.Proxy.Ports.HTTP.Value) + assert.Equal(t, 0, cfg.Proxy.Ports.HTTPS.Value) + assert.Equal(t, true, cfg.Proxy.Ports.HTTP.Disable) + assert.Equal(t, true, cfg.Proxy.Ports.HTTPS.Disable) +} diff --git a/pkg/gateway/client.go b/pkg/discovery/gateway/client.go similarity index 63% rename from pkg/gateway/client.go rename to pkg/discovery/gateway/client.go index 746186a..25263d9 100644 --- a/pkg/gateway/client.go +++ b/pkg/discovery/gateway/client.go @@ -7,19 +7,20 @@ import ( "net/http" "sync" - "github.com/Axway/agent-sdk/pkg/apic/provisioning" - "github.com/Axway/agents-kong/pkg/common" - "github.com/Axway/agents-kong/pkg/subscription" + klib "github.com/kong/go-kong/kong" "github.com/Axway/agent-sdk/pkg/agent" "github.com/Axway/agent-sdk/pkg/apic" + "github.com/Axway/agent-sdk/pkg/apic/provisioning" "github.com/Axway/agent-sdk/pkg/cache" - + "github.com/Axway/agent-sdk/pkg/filter" "github.com/Axway/agent-sdk/pkg/util" "github.com/Axway/agent-sdk/pkg/util/log" - config "github.com/Axway/agents-kong/pkg/config/discovery" - kutil "github.com/Axway/agents-kong/pkg/kong" - klib "github.com/kong/go-kong/kong" + + "github.com/Axway/agents-kong/pkg/common" + "github.com/Axway/agents-kong/pkg/discovery/config" + kutil "github.com/Axway/agents-kong/pkg/discovery/kong" + "github.com/Axway/agents-kong/pkg/discovery/subscription" ) var kongToCRDMapper = map[string]string{ @@ -43,12 +44,22 @@ func NewClient(agentConfig config.AgentConfig) (*Client, error) { return nil, err } - if err := hasACLEnabledInPlugins(plugins); err != nil { + discoveryFilter, err := filter.NewFilter(agentConfig.KongGatewayCfg.Spec.Filter) + if err != nil { + return nil, err + } + + if err = hasGlobalACLEnabledInPlugins(logger, plugins, agentConfig.KongGatewayCfg.ACL.Disable); err != nil { + logger.WithError(err).Error("ACL Plugin configured as required, but none found in Kong plugins.") return nil, err } provisionLogger := log.NewFieldLogger().WithComponent("provision").WithPackage("kong") - subscription.NewProvisioner(kongClient, provisionLogger) + opts := []subscription.ProvisionerOption{} + if agentConfig.KongGatewayCfg.ACL.Disable { + opts = append(opts, subscription.WithACLDisable()) + } + subscription.NewProvisioner(kongClient, provisionLogger, opts...) return &Client{ logger: logger, @@ -57,17 +68,30 @@ func NewClient(agentConfig config.AgentConfig) (*Client, error) { kongClient: kongClient, cache: daCache, mode: common.Marketplace, + filter: discoveryFilter, }, nil } -// Returns no error in case an ACL plugin which is enabled is found -func hasACLEnabledInPlugins(plugins []*klib.Plugin) error { +func pluginIsGlobal(p *klib.Plugin) bool { + if p.Service == nil && p.Route == nil { + return true + } + return false +} + +// Returns no error in case a global ACL plugin which is enabled is found +func hasGlobalACLEnabledInPlugins(logger log.FieldLogger, plugins []*klib.Plugin, aclDisable bool) error { + if aclDisable { + logger.Warn("ACL Plugin check disabled. Assuming global access is allowed for all services.") + return nil + } for _, plugin := range plugins { - if *plugin.Name == "acl" && *plugin.Enabled { + if *plugin.Name == "acl" && *plugin.Enabled && pluginIsGlobal(plugin) { return nil } } - return fmt.Errorf("failed to find acl plugin is enabled and installed") + return fmt.Errorf("failed to find acl plugin is enabled and installed on the Kong Gateway. " + + "Enable in on the Gateway or change the config to disable this check.") } func (gc *Client) DiscoverAPIs() error { @@ -92,6 +116,10 @@ func (gc *Client) DiscoverAPIs() error { func (gc *Client) processKongServicesList(ctx context.Context, services []*klib.Service) { wg := new(sync.WaitGroup) for _, service := range services { + if !gc.filter.Evaluate(toTagsMap(service)) { + gc.logger.WithField(common.AttrServiceName, *service.Name).Info("Service not passing tag filters. Skipping discovery for this service.") + continue + } wg.Add(1) go func(service *klib.Service, wg *sync.WaitGroup) { defer wg.Done() @@ -104,6 +132,15 @@ func (gc *Client) processKongServicesList(ctx context.Context, services []*klib. wg.Wait() } +func toTagsMap(service *klib.Service) map[string]string { + // The SDK currently only supports map[string]string format. + filters := make(map[string]string) + for _, tag := range service.Tags { + filters[*tag] = *tag + } + return filters +} + func (gc *Client) processSingleKongService(ctx context.Context, service *klib.Service) error { log := gc.logger.WithField(common.AttrServiceName, *service.Name) log.Info("processing service") @@ -113,19 +150,29 @@ func (gc *Client) processSingleKongService(ctx context.Context, service *klib.Se log.WithError(err).Errorf("failed to get routes for service") return err } + kongServiceSpec, err := gc.kongClient.GetSpecForService(ctx, service) + if err != nil { + return + } + // don't publish an empty spec + if kongServiceSpec == nil { + log.Warn("no spec found") + return nil + } + + // parse the spec file that was found and get the spec processor + spec := apic.NewSpecResourceParser(kongServiceSpec, "") + spec.Parse() for _, route := range routes { - gc.specPreparation(ctx, route, service) + gc.specPreparation(ctx, route, service, spec.GetSpecProcessor()) } return nil } -func (gc *Client) specPreparation(ctx context.Context, route *klib.Route, service *klib.Service) { +func (gc *Client) specPreparation(ctx context.Context, route *klib.Route, service *klib.Service, spec apic.SpecProcessor) { log := gc.logger.WithField(common.AttrRouteID, *route.ID). WithField(common.AttrServiceID, *service.ID) - proxyHost := gc.kongGatewayCfg.Proxy.Host - httpPort := gc.kongGatewayCfg.Proxy.Port.HTTP - httpsPort := gc.kongGatewayCfg.Proxy.Port.HTTPS apiPlugins, err := gc.plugins.GetEffectivePlugins(*route.ID, *service.ID) if err != nil { @@ -133,21 +180,12 @@ func (gc *Client) specPreparation(ctx context.Context, route *klib.Route, servic return } - kongServiceSpec, err := gc.kongClient.GetSpecForService(ctx, service, route) - if err != nil { + endpoints := gc.processKongRoute(route) + if len(endpoints) == 0 { + log.Info("not processing route as no enabled endpoints detected") return } - // don't publish an empty spec - if kongServiceSpec == nil { - log.Warn("no spec found") - return - } - oasSpec := Openapi{ - spec: string(kongServiceSpec), - } - - endpoints := gc.processKongRoute(proxyHost, oasSpec.BasePath(), route, httpPort, httpsPort) - serviceBody, err := gc.processKongAPI(ctx, *route.ID, service, oasSpec, endpoints, apiPlugins) + serviceBody, err := gc.processKongAPI(ctx, route, service, spec, endpoints, apiPlugins) if err != nil { log.WithError(err).Error("failed to process kong API") return @@ -166,50 +204,31 @@ func (gc *Client) specPreparation(ctx context.Context, route *klib.Route, servic log.Info("Successfully published to central") } -func (gc *Client) processKongRoute(defaultHost string, basePath string, route *klib.Route, httpPort, httpsPort int) []apic.EndpointDefinition { - var endpoints []apic.EndpointDefinition +func (gc *Client) processKongRoute(route *klib.Route) []apic.EndpointDefinition { if route == nil { - return endpoints - } - - hosts := route.Hosts - hosts = append(hosts, &defaultHost) - - for _, host := range hosts { - for _, path := range route.Paths { - for _, protocol := range route.Protocols { - port := httpPort - if *protocol == "https" { - port = httpsPort - } - - routingBasePath := *path - if *route.StripPath { - routingBasePath = routingBasePath + basePath - } - endpoint := apic.EndpointDefinition{ - Host: *host, - Port: int32(port), - Protocol: *protocol, - BasePath: routingBasePath, - } - endpoints = append(endpoints, endpoint) - } - } + return []apic.EndpointDefinition{} + } + + kRoute := KongRoute{ + Route: route, + defaultHost: gc.kongGatewayCfg.Proxy.Host, + httpPort: gc.kongGatewayCfg.Proxy.Ports.HTTP.Value, + httpsPort: gc.kongGatewayCfg.Proxy.Ports.HTTPS.Value, + basePath: gc.kongGatewayCfg.Proxy.BasePath, } - return endpoints + return kRoute.GetEndpoints() } func (gc *Client) processKongAPI( ctx context.Context, - routeID string, + route *klib.Route, service *klib.Service, - oasSpec Openapi, + spec apic.SpecProcessor, endpoints []apic.EndpointDefinition, apiPlugins map[string]*klib.Plugin, ) (*apic.ServiceBody, error) { - kongAPI := newKongAPI(routeID, service, oasSpec, endpoints) + kongAPI := newKongAPI(route, service, spec, endpoints) isAlreadyPublished, checksum := isPublished(&kongAPI, gc.cache) // If true, then the api is published and there were no changes detected if isAlreadyPublished { @@ -231,7 +250,7 @@ func (gc *Client) processKongAPI( agentDetails := map[string]string{ common.AttrServiceID: *service.ID, - common.AttrRouteID: routeID, + common.AttrRouteID: *route.ID, common.AttrChecksum: checksum, } kongAPI.agentDetails = agentDetails @@ -244,21 +263,29 @@ func (gc *Client) processKongAPI( } func newKongAPI( - routeID string, + route *klib.Route, service *klib.Service, - oasSpec Openapi, + spec apic.SpecProcessor, endpoints []apic.EndpointDefinition, ) KongAPI { + // strip any security from spec if it is an oas spec + resType := spec.GetResourceType() + if resType == apic.Oas2 || resType == apic.Oas3 { + spec.(apic.OasSpecProcessor).StripSpecAuth() + } + return KongAPI{ - id: routeID, + id: *service.ID, name: *service.Name, - description: oasSpec.Description(), - version: oasSpec.Version(), + description: spec.GetDescription(), + version: spec.GetVersion(), url: *service.Host, - resourceType: oasSpec.ResourceType(), + resourceType: resType, documentation: []byte(*service.Name), - swaggerSpec: []byte(oasSpec.spec), + spec: spec.GetSpecBytes(), endpoints: endpoints, + stageName: *route.Name, + stage: *route.ID, } } @@ -274,9 +301,9 @@ func (ka *KongAPI) buildServiceBody() (apic.ServiceBody, error) { "GatewayType": "Kong API Gateway", } - return apic.NewServiceBodyBuilder(). + builder := apic.NewServiceBodyBuilder(). SetAPIName(ka.name). - SetAPISpec(ka.swaggerSpec). + SetAPISpec(ka.spec). SetAPIUpdateSeverity(ka.apiUpdateSeverity). SetDescription(ka.description). SetDocumentation(ka.documentation). @@ -287,15 +314,21 @@ func (ka *KongAPI) buildServiceBody() (apic.ServiceBody, error) { SetServiceAgentDetails(util.MapStringStringToMapStringInterface(ka.agentDetails)). SetServiceAttribute(serviceAttributes). SetStage(ka.stage). + SetStageDisplayName(ka.stageName). + SetStageDescriptor("Route"). SetState(apic.PublishedStatus). SetStatus(apic.PublishedStatus). SetTags(tags). SetTitle(ka.name). SetURL(ka.url). SetVersion(ka.version). - SetServiceEndpoints(ka.endpoints). - SetAccessRequestDefinitionName(ka.ard, false). - SetCredentialRequestDefinitions(ka.crds).Build() + SetServiceEndpoints(ka.endpoints) + + if len(ka.crds) > 0 { + return builder.SetAccessRequestDefinitionName(ka.ard, false). + SetCredentialRequestDefinitions(ka.crds).Build() + } + return builder.SetAuthPolicy(apic.Passthrough).Build() } // makeChecksum generates a makeChecksum for the api for change detection @@ -314,15 +347,3 @@ func isPublished(api *KongAPI, c cache.Cache) (bool, string) { } return true, checksum } - -func isValidAuthTypeAndEnabled(p *klib.Plugin) bool { - if !*p.Enabled { - return false - } - for _, availableAuthName := range []string{"basic-auth", "oauth2", "key-auth"} { - if *p.Name == availableAuthName { - return true - } - } - return false -} diff --git a/pkg/gateway/client_test.go b/pkg/discovery/gateway/client_test.go similarity index 52% rename from pkg/gateway/client_test.go rename to pkg/discovery/gateway/client_test.go index f7c44f1..f8fcc00 100644 --- a/pkg/gateway/client_test.go +++ b/pkg/discovery/gateway/client_test.go @@ -4,18 +4,11 @@ import ( "testing" corecfg "github.com/Axway/agent-sdk/pkg/config" - config "github.com/Axway/agents-kong/pkg/config/discovery" + config "github.com/Axway/agents-kong/pkg/discovery/config" ) func TestKongClient(t *testing.T) { - gatewayConfig := &config.KongGatewayConfig{ - Admin: config.KongAdminConfig{ - URL: "http://localhost", - }, - Proxy: config.KongProxyConfig{ - Host: "localhost", - }, - } + gatewayConfig := &config.KongGatewayConfig{} _ = config.AgentConfig{ CentralCfg: corecfg.NewCentralConfig(corecfg.DiscoveryAgent), KongGatewayCfg: gatewayConfig, diff --git a/pkg/gateway/definitions.go b/pkg/discovery/gateway/definitions.go similarity index 79% rename from pkg/gateway/definitions.go rename to pkg/discovery/gateway/definitions.go index 2b3acb2..c3ffe63 100644 --- a/pkg/gateway/definitions.go +++ b/pkg/discovery/gateway/definitions.go @@ -4,10 +4,11 @@ import ( "github.com/Axway/agent-sdk/pkg/apic" "github.com/Axway/agent-sdk/pkg/cache" corecfg "github.com/Axway/agent-sdk/pkg/config" + "github.com/Axway/agent-sdk/pkg/filter" "github.com/Axway/agent-sdk/pkg/util/log" - config "github.com/Axway/agents-kong/pkg/config/discovery" - "github.com/Axway/agents-kong/pkg/kong" + config "github.com/Axway/agents-kong/pkg/discovery/config" + "github.com/Axway/agents-kong/pkg/discovery/kong" ) type Client struct { @@ -18,10 +19,11 @@ type Client struct { plugins kong.Plugins cache cache.Cache mode string + filter filter.Filter } type KongAPI struct { - swaggerSpec []byte + spec []byte id string name string description string @@ -37,5 +39,6 @@ type KongAPI struct { agentDetails map[string]string tags []string stage string + stageName string ard string } diff --git a/pkg/discovery/gateway/route.go b/pkg/discovery/gateway/route.go new file mode 100644 index 0000000..a7c9d56 --- /dev/null +++ b/pkg/discovery/gateway/route.go @@ -0,0 +1,68 @@ +package gateway + +import ( + "fmt" + + "github.com/Axway/agent-sdk/pkg/apic" + klib "github.com/kong/go-kong/kong" +) + +type KongRoute struct { + *klib.Route + defaultHost string + basePath string + httpPort int + httpsPort int +} + +func (r *KongRoute) GetEndpoints() []apic.EndpointDefinition { + endpoints := r.handleHosts() + if len(endpoints) == 0 { + return r.handlePaths(r.defaultHost, r.basePath) + } + return endpoints +} + +func (r *KongRoute) handleHosts() []apic.EndpointDefinition { + endpoints := make([]apic.EndpointDefinition, 0) + for _, host := range r.Hosts { + endpoints = append(endpoints, r.handlePaths(*host, "")...) + } + return endpoints +} + +func (r *KongRoute) handlePaths(host, basePath string) []apic.EndpointDefinition { + endpoints := make([]apic.EndpointDefinition, 0) + for _, path := range r.Paths { + fullPath := *path + if basePath != "" { + // prepend the base path to the path + fullPath = fmt.Sprintf("%s%s", basePath, fullPath) + } + endpoints = append(endpoints, r.handleProtocols(host, fullPath)...) + } + return endpoints +} + +func (r *KongRoute) handleProtocols(host, path string) []apic.EndpointDefinition { + endpoints := make([]apic.EndpointDefinition, 0) + for _, protocol := range r.Protocols { + if *protocol == "http" && r.httpPort != 0 { + endpoints = append(endpoints, apic.EndpointDefinition{ + Host: host, + Port: int32(r.httpPort), + Protocol: "http", + BasePath: path, + }) + } + if *protocol == "https" && r.httpsPort != 0 { + endpoints = append(endpoints, apic.EndpointDefinition{ + Host: host, + Port: int32(r.httpsPort), + Protocol: "https", + BasePath: path, + }) + } + } + return endpoints +} diff --git a/pkg/discovery/gateway/route_test.go b/pkg/discovery/gateway/route_test.go new file mode 100644 index 0000000..de1de09 --- /dev/null +++ b/pkg/discovery/gateway/route_test.go @@ -0,0 +1,230 @@ +package gateway + +import ( + "testing" + + "github.com/Axway/agent-sdk/pkg/apic" + "github.com/kong/go-kong/kong" + "github.com/stretchr/testify/assert" +) + +var ( + kHttp = kong.String("http") + kHttps = kong.String("https") +) + +func TestKongRoute(t *testing.T) { + testCases := map[string]struct { + cfgHost string + cfgHttpPort int + cfgHttpsPort int + cfgBasePath string + route *kong.Route + expectedEndpoints []apic.EndpointDefinition + }{ + "http default route, no base path": { + cfgHost: "my.host.com", + cfgHttpPort: 8080, + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttp, kHttps}, + Paths: []*string{kong.String("/path")}, + }, + expectedEndpoints: []apic.EndpointDefinition{ + { + Host: "my.host.com", + Port: 8080, + Protocol: "http", + BasePath: "/path", + }, + }, + }, + "https only route only has http": { + cfgHost: "my.host.com", + cfgHttpsPort: 8443, + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttp}, + Paths: []*string{kong.String("/path")}, + }, + expectedEndpoints: []apic.EndpointDefinition{}, + }, + "http only route only has https": { + cfgHost: "my.host.com", + cfgHttpPort: 8080, + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttps}, + Paths: []*string{kong.String("/path")}, + }, + expectedEndpoints: []apic.EndpointDefinition{}, + }, + "https default route, no base path": { + cfgHost: "my.host.com", + cfgHttpsPort: 8443, + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttp, kHttps}, + Paths: []*string{kong.String("/path")}, + }, + expectedEndpoints: []apic.EndpointDefinition{ + { + Host: "my.host.com", + Port: 8443, + Protocol: "https", + BasePath: "/path", + }, + }, + }, + "http and https allowed, no base path, route only has http": { + cfgHost: "my.host.com", + cfgHttpPort: 8080, + cfgHttpsPort: 8443, + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttp}, + Paths: []*string{kong.String("/path")}, + }, + expectedEndpoints: []apic.EndpointDefinition{ + { + Host: "my.host.com", + Port: 8080, + Protocol: "http", + BasePath: "/path", + }, + }, + }, + "http and https allowed, no base path, route only has https": { + cfgHost: "my.host.com", + cfgHttpPort: 8080, + cfgHttpsPort: 8443, + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttps}, + Paths: []*string{kong.String("/path")}, + }, + expectedEndpoints: []apic.EndpointDefinition{ + { + Host: "my.host.com", + Port: 8443, + Protocol: "https", + BasePath: "/path", + }, + }, + }, + "http and https default routes, no base path": { + cfgHost: "my.host.com", + cfgHttpPort: 8080, + cfgHttpsPort: 8443, + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttp, kHttps}, + Paths: []*string{kong.String("/path")}, + }, + expectedEndpoints: []apic.EndpointDefinition{ + { + Host: "my.host.com", + Port: 8080, + Protocol: "http", + BasePath: "/path", + }, + { + Host: "my.host.com", + Port: 8443, + Protocol: "https", + BasePath: "/path", + }, + }, + }, + "http and https default routes, with base path": { + cfgHost: "my.host.com", + cfgHttpPort: 8080, + cfgHttpsPort: 8443, + cfgBasePath: "/base", + route: &kong.Route{ + Hosts: []*string{}, + Protocols: []*string{kHttp, kHttps}, + Paths: []*string{kong.String("/path1"), kong.String("/path2")}, + }, + expectedEndpoints: []apic.EndpointDefinition{ + { + Host: "my.host.com", + Port: 8080, + Protocol: "http", + BasePath: "/base/path1", + }, + { + Host: "my.host.com", + Port: 8443, + Protocol: "https", + BasePath: "/base/path1", + }, + { + Host: "my.host.com", + Port: 8080, + Protocol: "http", + BasePath: "/base/path2", + }, + { + Host: "my.host.com", + Port: 8443, + Protocol: "https", + BasePath: "/base/path2", + }, + }, + }, + "http and https only configured routes": { + cfgHost: "my.host.com", + cfgHttpPort: 8080, + cfgHttpsPort: 8443, + cfgBasePath: "/base", + route: &kong.Route{ + Hosts: []*string{kong.String("kong.host.com")}, + Protocols: []*string{kHttp, kHttps}, + Paths: []*string{kong.String("/path1"), kong.String("/path2")}, + }, + expectedEndpoints: []apic.EndpointDefinition{ + { + Host: "kong.host.com", + Port: 8080, + Protocol: "http", + BasePath: "/path1", + }, + { + Host: "kong.host.com", + Port: 8443, + Protocol: "https", + BasePath: "/path1", + }, + { + Host: "kong.host.com", + Port: 8080, + Protocol: "http", + BasePath: "/path2", + }, + { + Host: "kong.host.com", + Port: 8443, + Protocol: "https", + BasePath: "/path2", + }, + }, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + route := KongRoute{ + Route: tc.route, + defaultHost: tc.cfgHost, + httpPort: tc.cfgHttpPort, + httpsPort: tc.cfgHttpsPort, + basePath: tc.cfgBasePath, + } + + endpoints := route.GetEndpoints() + + assert.Equal(t, len(endpoints), len(tc.expectedEndpoints)) + assert.ElementsMatch(t, endpoints, tc.expectedEndpoints) + }) + } +} diff --git a/pkg/kong/definitions.go b/pkg/discovery/kong/definitions.go similarity index 100% rename from pkg/kong/definitions.go rename to pkg/discovery/kong/definitions.go diff --git a/pkg/kong/kongclient.go b/pkg/discovery/kong/kongclient.go similarity index 87% rename from pkg/kong/kongclient.go rename to pkg/discovery/kong/kongclient.go index 5cdeba9..28362da 100644 --- a/pkg/kong/kongclient.go +++ b/pkg/discovery/kong/kongclient.go @@ -2,9 +2,8 @@ package kong import ( "context" - "crypto/tls" + "encoding/base64" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -13,13 +12,13 @@ import ( "strings" "time" - "github.com/Axway/agent-sdk/pkg/util/log" + klib "github.com/kong/go-kong/kong" "github.com/Axway/agent-sdk/pkg/apic" - "github.com/Axway/agents-kong/pkg/common" - config "github.com/Axway/agents-kong/pkg/config/discovery" + "github.com/Axway/agent-sdk/pkg/util/log" - klib "github.com/kong/go-kong/kong" + "github.com/Axway/agents-kong/pkg/common" + config "github.com/Axway/agents-kong/pkg/discovery/config" ) const tagPrefix = "spec_local_" @@ -67,29 +66,34 @@ type KongClient struct { } func NewKongClient(baseClient *http.Client, kongConfig *config.KongGatewayConfig) (*KongClient, error) { - if kongConfig.Admin.Auth.APIKey.Value != "" { - defaultTransport := http.DefaultTransport.(*http.Transport) - http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - baseClient.Transport = defaultTransport + headers := make(http.Header) + var kongEndpoint string + kongTransport := http.DefaultTransport.(*http.Transport) + kongTransport.TLSClientConfig = kongConfig.Admin.TLS.BuildTLSConfig() + baseClient.Transport = kongTransport + kongEndpoint = kongConfig.Admin.Url - headers := make(http.Header) + if kongConfig.Admin.Auth.APIKey.Value != "" { headers.Set(kongConfig.Admin.Auth.APIKey.Header, kongConfig.Admin.Auth.APIKey.Value) - client := klib.HTTPClientWithHeaders(baseClient, headers) - baseClient = client } + if kongConfig.Admin.Auth.BasicAuth.Username != "" { + headers.Set("Authorization", "Basic "+basicAuth(kongConfig.Admin.Auth.BasicAuth.Username, kongConfig.Admin.Auth.BasicAuth.Password)) + } + headers.Set("Host", kongConfig.Proxy.Host) + baseClient = klib.HTTPClientWithHeaders(baseClient, headers) logger := log.NewFieldLogger().WithComponent("client").WithPackage("kong") - - baseKongClient, err := klib.NewClient(&kongConfig.Admin.URL, baseClient) + baseKongClient, err := klib.NewClient(&kongEndpoint, baseClient) if err != nil { logger.WithError(err).Error("failed to create kong client") return nil, err } + return &KongClient{ Client: baseKongClient, logger: log.NewFieldLogger().WithComponent("KongClient").WithPackage("kong"), baseClient: baseClient, - kongAdminEndpoint: kongConfig.Admin.URL, + kongAdminEndpoint: kongEndpoint, specURLPaths: kongConfig.Spec.URLPaths, specLocalPath: kongConfig.Spec.LocalPath, devPortalEnabled: kongConfig.Spec.DevPortalEnabled, @@ -106,9 +110,8 @@ func (k KongClient) ListRoutesForService(ctx context.Context, serviceId string) return routes, err } -func (k KongClient) GetSpecForService(ctx context.Context, service *klib.Service, route *klib.Route) ([]byte, error) { - log := k.logger.WithField(common.AttrServiceName, *service.Name). - WithField(common.AttrRouteName, *route.Name) +func (k KongClient) GetSpecForService(ctx context.Context, service *klib.Service) ([]byte, error) { + log := k.logger.WithField(common.AttrServiceName, *service.Name) if k.specLocalPath != "" { return k.getSpecFromLocal(ctx, service, route) @@ -132,9 +135,8 @@ func (k KongClient) GetSpecForService(ctx context.Context, service *klib.Service return k.getSpecFromBackend(ctx, backendURL) } -func (k KongClient) getSpecFromLocal(ctx context.Context, service *klib.Service, route *klib.Route) ([]byte, error) { - log := k.logger.WithField(common.AttrServiceName, *service.Name). - WithField(common.AttrRouteName, *route.Name) +func (k KongClient) getSpecFromLocal(ctx context.Context, service *klib.Service) ([]byte, error) { + log := k.logger.WithField(common.AttrServiceName, *service.Name) specTag := "" for _, tag := range route.Tags { @@ -145,8 +147,8 @@ func (k KongClient) getSpecFromLocal(ctx context.Context, service *klib.Service, } if specTag == "" { - log.Error("In order to map local specs to the desired routes, a tag with format 'spec_local_fileName.extension' must be present") - return nil, errors.New("No specification tag found.") + log.Error("in order to map local specs to the desired services, a tag with format 'spec_local_fileName.extension' must be present") + return nil, fmt.Errorf("no specification tag found") } filename := specTag[len(tagPrefix):] @@ -286,3 +288,8 @@ func (k KongClient) getSpec(ctx context.Context, endpoint string, fromDevPortal func (k KongClient) GetKongPlugins() *Plugins { return &Plugins{PluginLister: k.Plugins} } + +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} diff --git a/pkg/kong/plugins.go b/pkg/discovery/kong/plugins.go similarity index 100% rename from pkg/kong/plugins.go rename to pkg/discovery/kong/plugins.go diff --git a/pkg/kong/plugins_test.go b/pkg/discovery/kong/plugins_test.go similarity index 97% rename from pkg/kong/plugins_test.go rename to pkg/discovery/kong/plugins_test.go index 9ca0b85..7a02d03 100644 --- a/pkg/kong/plugins_test.go +++ b/pkg/discovery/kong/plugins_test.go @@ -4,8 +4,9 @@ import ( "context" "testing" - "github.com/Axway/agents-kong/pkg/kong" klib "github.com/kong/go-kong/kong" + + "github.com/Axway/agents-kong/pkg/discovery/kong" ) type pluginsMock []*klib.Plugin diff --git a/pkg/kong/provisioning.go b/pkg/discovery/kong/provisioning.go similarity index 96% rename from pkg/kong/provisioning.go rename to pkg/discovery/kong/provisioning.go index a8397b8..1239b72 100644 --- a/pkg/kong/provisioning.go +++ b/pkg/discovery/kong/provisioning.go @@ -360,15 +360,15 @@ func getSpecificPlugin(plugins []*klib.Plugin, serviceID, routeID, consumerID, p continue } - if consumerID == "" || plugin.Consumer == nil || (plugin.Consumer != nil && *plugin.Consumer.ID == consumerID) { + if (consumerID == "" && plugin.Consumer == nil) || (consumerID != "" && plugin.Consumer != nil && *plugin.Consumer.ID == consumerID) { consumerMatch = true } - if routeID == "" || plugin.Route == nil || (plugin.Route != nil && *plugin.Route.ID == routeID) { + if (routeID == "" && plugin.Route == nil) || (routeID != "" && plugin.Route != nil && *plugin.Route.ID == routeID) { routeMatch = true } - if serviceID == "" || plugin.Service == nil || (plugin.Service != nil && *plugin.Service.ID == serviceID) { + if (serviceID == "" && plugin.Service == nil) || (serviceID != "" && plugin.Service != nil && *plugin.Service.ID == serviceID) { serviceMatch = true } diff --git a/pkg/kong/provisioning_test.go b/pkg/discovery/kong/provisioning_test.go similarity index 95% rename from pkg/kong/provisioning_test.go rename to pkg/discovery/kong/provisioning_test.go index fcccc07..d3bd7e0 100644 --- a/pkg/kong/provisioning_test.go +++ b/pkg/discovery/kong/provisioning_test.go @@ -6,14 +6,18 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/url" + "strconv" "testing" klib "github.com/kong/go-kong/kong" "github.com/stretchr/testify/assert" "github.com/Axway/agent-sdk/pkg/apic/provisioning" + corecfg "github.com/Axway/agent-sdk/pkg/config" + "github.com/Axway/agents-kong/pkg/common" - config "github.com/Axway/agents-kong/pkg/config/discovery" + config "github.com/Axway/agents-kong/pkg/discovery/config" ) func formatRequestKey(method, path string) string { @@ -86,11 +90,28 @@ func createClient(responses map[string]response) KongAPIClient { return } })) + u, _ := url.Parse(s.URL) + port, _ := strconv.Atoi(u.Port()) cfg := &config.KongGatewayConfig{ + Proxy: config.KongProxyConfig{ + Host: u.Hostname(), + Ports: config.KongPortConfig{ + HTTP: config.KongPortSettingsConfig{ + Value: port, + }, + HTTPS: config.KongPortSettingsConfig{ + Value: port, + }, + }, + }, Admin: config.KongAdminConfig{ - URL: s.URL, + Url: s.URL, + TLS: corecfg.NewTLSConfig(), }, } + if err := cfg.ValidateCfg(); err != nil { + panic(err) + } client, _ := NewKongClient(&http.Client{}, cfg) return client } @@ -570,6 +591,9 @@ func TestAddQuota(t *testing.T) { ID: klib.String("routeID"), }, Enabled: klib.Bool(true), + Consumer: &klib.Consumer{ + ID: klib.String("consumerID"), + }, }, }, "next": "null", @@ -592,6 +616,9 @@ func TestAddQuota(t *testing.T) { Route: &klib.Route{ ID: klib.String("routeID"), }, + Consumer: &klib.Consumer{ + ID: klib.String("consumerID"), + }, Enabled: klib.Bool(false), }, }, diff --git a/pkg/discovery/main/agent.go b/pkg/discovery/main/agent.go new file mode 100644 index 0000000..78a92d3 --- /dev/null +++ b/pkg/discovery/main/agent.go @@ -0,0 +1,22 @@ +package main + +import ( + "fmt" + "os" + + discovery "github.com/Axway/agents-kong/pkg/discovery/cmd" +) + +func main() { + os.Setenv("AGENTFEATURES_VERSIONCHECKER", "false") + + // update to set the default pattern for kong discovery + pattern := os.Getenv("CENTRAL_APISERVICEREVISIONPATTERN") + if pattern == "" { + os.Setenv("CENTRAL_APISERVICEREVISIONPATTERN", "{{.APIServiceName}} - {{.Date:YYYY/MM/DD}} - r {{.Revision}}") + } + if err := discovery.DiscoveryCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/pkg/subscription/access/access.go b/pkg/discovery/subscription/access/access.go similarity index 79% rename from pkg/subscription/access/access.go rename to pkg/discovery/subscription/access/access.go index 817aacc..dc9aa4e 100644 --- a/pkg/subscription/access/access.go +++ b/pkg/discovery/subscription/access/access.go @@ -29,27 +29,30 @@ type accessRequest interface { } type AccessProvisioner struct { - ctx context.Context - logger log.FieldLogger - client accessClient - quota provisioning.Quota - routeID string - appID string + ctx context.Context + logger log.FieldLogger + client accessClient + quota provisioning.Quota + routeID string + appID string + aclDisable bool } -func NewAccessProvisioner(ctx context.Context, client accessClient, request accessRequest) AccessProvisioner { +func NewAccessProvisioner(ctx context.Context, client accessClient, request accessRequest, aclDisable bool) AccessProvisioner { instDetails := request.GetInstanceDetails() routeID := sdkUtil.ToString(instDetails[common.AttrRouteID]) + logger := log.NewFieldLogger(). + WithComponent("AccessProvisioner"). + WithPackage("access") a := AccessProvisioner{ - ctx: context.Background(), - logger: log.NewFieldLogger(). - WithComponent("AccessProvisioner"). - WithPackage("access"), - client: client, - quota: request.GetQuota(), - routeID: routeID, - appID: request.GetApplicationDetailsValue(common.AttrAppID), + ctx: context.Background(), + logger: logger, + client: client, + quota: request.GetQuota(), + routeID: routeID, + appID: request.GetApplicationDetailsValue(common.AttrAppID), + aclDisable: aclDisable, } if a.routeID != "" { @@ -63,8 +66,8 @@ func NewAccessProvisioner(ctx context.Context, client accessClient, request acce func (a AccessProvisioner) Provision() (provisioning.RequestStatus, provisioning.AccessData) { a.logger.Info("provisioning access") - rs := provisioning.NewRequestStatusBuilder() + if a.appID == "" { a.logger.Error("could not find the managed application ID on the resource") return rs.SetMessage("managed application ID not found").Failed(), nil @@ -75,6 +78,11 @@ func (a AccessProvisioner) Provision() (provisioning.RequestStatus, provisioning return rs.SetMessage("route ID not found").Failed(), nil } + if a.aclDisable { + a.logger.Debug("ACL plugin check is disabled or not existing for current spec. Skipping access request provisioning") + return rs.Success(), nil + } + if a.quota != nil && a.quota.GetInterval().String() == provisioning.Weekly.String() { a.logger.Debug("weekly quota interval is not supported") return rs.SetMessage("weekly quota is not supported by kong").Failed(), nil @@ -105,8 +113,8 @@ func (a AccessProvisioner) Provision() (provisioning.RequestStatus, provisioning func (a AccessProvisioner) Deprovision() provisioning.RequestStatus { a.logger.Info("deprovisioning access") - rs := provisioning.NewRequestStatusBuilder() + if a.appID == "" { a.logger.Error("could not find the managed application ID on the resource") return rs.SetMessage("managed application ID not found").Failed() @@ -117,6 +125,11 @@ func (a AccessProvisioner) Deprovision() provisioning.RequestStatus { return rs.SetMessage("route ID not found").Failed() } + if a.aclDisable { + a.logger.Debug("ACL plugin check is disabled or not existing for current spec. Skipping access request deprovisioning") + return rs.Success() + } + err := a.client.RemoveRouteACL(a.ctx, a.routeID, a.appID) if err != nil { a.logger.WithError(err).Error("failed to remove managed app from ACL") diff --git a/pkg/subscription/access/access_test.go b/pkg/discovery/subscription/access/access_test.go similarity index 85% rename from pkg/subscription/access/access_test.go rename to pkg/discovery/subscription/access/access_test.go index d46ad66..63085a2 100644 --- a/pkg/subscription/access/access_test.go +++ b/pkg/discovery/subscription/access/access_test.go @@ -93,9 +93,10 @@ func (q *mockQuota) GetPlanName() string { func TestProvision(t *testing.T) { cases := map[string]struct { - client mockAccessClient - request mockAccessRequest - result provisioning.Status + client mockAccessClient + request mockAccessRequest + result provisioning.Status + aclDisable bool }{ "no app id configured": { result: provisioning.Error, @@ -108,6 +109,18 @@ func TestProvision(t *testing.T) { }, result: provisioning.Error, }, + "ACL disable is active": { + request: mockAccessRequest{ + values: map[string]string{ + common.AttrAppID: "appID", + }, + details: map[string]interface{}{ + common.AttrRouteID: "routeID", + }, + }, + result: provisioning.Success, + aclDisable: true, + }, "unsupported quota interval": { request: mockAccessRequest{ values: map[string]string{ @@ -184,7 +197,7 @@ func TestProvision(t *testing.T) { t.Run(name, func(t *testing.T) { ctx := context.WithValue(context.Background(), testName, name) - result, _ := NewAccessProvisioner(ctx, tc.client, &tc.request).Provision() + result, _ := NewAccessProvisioner(ctx, tc.client, &tc.request, tc.aclDisable).Provision() assert.Equal(t, tc.result, result.GetStatus()) }) } @@ -192,9 +205,10 @@ func TestProvision(t *testing.T) { func TestDeprovision(t *testing.T) { cases := map[string]struct { - client mockAccessClient - request mockAccessRequest - result provisioning.Status + client mockAccessClient + request mockAccessRequest + result provisioning.Status + aclDisable bool }{ "no app id configured": { result: provisioning.Error, @@ -207,6 +221,18 @@ func TestDeprovision(t *testing.T) { }, result: provisioning.Error, }, + "ACL disable is active": { + request: mockAccessRequest{ + values: map[string]string{ + common.AttrAppID: "appID", + }, + details: map[string]interface{}{ + common.AttrRouteID: "routeID", + }, + }, + result: provisioning.Success, + aclDisable: true, + }, "error revoking access for managed app": { client: mockAccessClient{ removeManagedAppErr: true, @@ -248,7 +274,7 @@ func TestDeprovision(t *testing.T) { t.Run(name, func(t *testing.T) { ctx := context.WithValue(context.Background(), testName, name) - result := NewAccessProvisioner(ctx, tc.client, &tc.request).Deprovision() + result := NewAccessProvisioner(ctx, tc.client, &tc.request, tc.aclDisable).Deprovision() assert.Equal(t, tc.result, result.GetStatus()) }) } diff --git a/pkg/subscription/application/application.go b/pkg/discovery/subscription/application/application.go similarity index 100% rename from pkg/subscription/application/application.go rename to pkg/discovery/subscription/application/application.go diff --git a/pkg/subscription/application/application_test.go b/pkg/discovery/subscription/application/application_test.go similarity index 100% rename from pkg/subscription/application/application_test.go rename to pkg/discovery/subscription/application/application_test.go diff --git a/pkg/subscription/credential/builder.go b/pkg/discovery/subscription/credential/builder.go similarity index 100% rename from pkg/subscription/credential/builder.go rename to pkg/discovery/subscription/credential/builder.go diff --git a/pkg/subscription/credential/credential.go b/pkg/discovery/subscription/credential/credential.go similarity index 100% rename from pkg/subscription/credential/credential.go rename to pkg/discovery/subscription/credential/credential.go diff --git a/pkg/subscription/credential/credential_test.go b/pkg/discovery/subscription/credential/credential_test.go similarity index 100% rename from pkg/subscription/credential/credential_test.go rename to pkg/discovery/subscription/credential/credential_test.go diff --git a/pkg/subscription/provision.go b/pkg/discovery/subscription/provision.go similarity index 76% rename from pkg/subscription/provision.go rename to pkg/discovery/subscription/provision.go index e2fa7da..c340455 100644 --- a/pkg/subscription/provision.go +++ b/pkg/discovery/subscription/provision.go @@ -3,37 +3,46 @@ package subscription import ( "context" - klib "github.com/kong/go-kong/kong" - "github.com/Axway/agent-sdk/pkg/agent" "github.com/Axway/agent-sdk/pkg/apic/provisioning" "github.com/Axway/agent-sdk/pkg/util/log" - "github.com/Axway/agents-kong/pkg/kong" - "github.com/Axway/agents-kong/pkg/subscription/access" - "github.com/Axway/agents-kong/pkg/subscription/application" - "github.com/Axway/agents-kong/pkg/subscription/credential" + "github.com/Axway/agents-kong/pkg/discovery/kong" + "github.com/Axway/agents-kong/pkg/discovery/subscription/access" + "github.com/Axway/agents-kong/pkg/discovery/subscription/application" + "github.com/Axway/agents-kong/pkg/discovery/subscription/credential" ) +type ProvisionerOption func(*provisioner) + type provisioner struct { - logger log.FieldLogger - client kong.KongAPIClient - kc *klib.Client + logger log.FieldLogger + client kong.KongAPIClient + aclDisable bool } // NewProvisioner creates a type to implement the SDK Provisioning methods for handling subscriptions -func NewProvisioner(client kong.KongAPIClient, logger log.FieldLogger) { +func NewProvisioner(client kong.KongAPIClient, logger log.FieldLogger, opts ...ProvisionerOption) { logger.Info("Registering provisioning callbacks") provisioner := &provisioner{ client: client, logger: logger, } + for _, o := range opts { + o(provisioner) + } agent.RegisterProvisioner(provisioner) registerOauth2() registerBasicAuth() registerKeyAuth() } +func WithACLDisable() ProvisionerOption { + return func(p *provisioner) { + p.aclDisable = true + } +} + func (p provisioner) ApplicationRequestProvision(request provisioning.ApplicationRequest) provisioning.RequestStatus { return application.NewApplicationProvisioner(context.Background(), p.client, request).Provision() } @@ -55,9 +64,9 @@ func (p provisioner) CredentialUpdate(request provisioning.CredentialRequest) (p } func (p provisioner) AccessRequestProvision(request provisioning.AccessRequest) (provisioning.RequestStatus, provisioning.AccessData) { - return access.NewAccessProvisioner(context.Background(), p.client, request).Provision() + return access.NewAccessProvisioner(context.Background(), p.client, request, p.aclDisable).Provision() } func (p provisioner) AccessRequestDeprovision(request provisioning.AccessRequest) provisioning.RequestStatus { - return access.NewAccessProvisioner(context.Background(), p.client, request).Deprovision() + return access.NewAccessProvisioner(context.Background(), p.client, request, p.aclDisable).Deprovision() } diff --git a/pkg/subscription/register.go b/pkg/discovery/subscription/register.go similarity index 100% rename from pkg/subscription/register.go rename to pkg/discovery/subscription/register.go diff --git a/pkg/gateway/openapi.go b/pkg/gateway/openapi.go deleted file mode 100644 index 38273b2..0000000 --- a/pkg/gateway/openapi.go +++ /dev/null @@ -1,36 +0,0 @@ -package gateway - -import ( - "github.com/Axway/agent-sdk/pkg/apic" - "github.com/Axway/agent-sdk/pkg/util/log" - "github.com/tidwall/gjson" -) - -type Openapi struct { - spec string -} - -func (oas *Openapi) ResourceType() string { - oas2 := gjson.Get(oas.spec, "swagger").Str - oas3 := gjson.Get(oas.spec, "openapi").Str - if len(oas2) > 0 { - return apic.Oas2 - } - if len(oas3) > 0 { - return apic.Oas3 - } - log.Error("not a valid spec") - return "" -} - -func (oas *Openapi) Description() string { - return gjson.Get(oas.spec, "info.description").Str -} - -func (oas *Openapi) Version() string { - return gjson.Get(oas.spec, "info.version").Str -} - -func (oas *Openapi) BasePath() string { - return gjson.Get(oas.spec, "basePath").Str -} diff --git a/pkg/main/discovery/main.go b/pkg/main/discovery/main.go deleted file mode 100644 index a6d8bac..0000000 --- a/pkg/main/discovery/main.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/Axway/agents-kong/pkg/cmd/discovery" -) - -func main() { - os.Setenv("AGENTFEATURES_VERSIONCHECKER", "false") - if err := discovery.DiscoveryCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/pkg/traceability/beater/beater.go b/pkg/traceability/beater/beater.go index 2907d43..750d257 100644 --- a/pkg/traceability/beater/beater.go +++ b/pkg/traceability/beater/beater.go @@ -5,11 +5,16 @@ import ( "fmt" "io" "net/http" + "os" + "sync" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/google/uuid" + "github.com/Axway/agent-sdk/pkg/agent" + management "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/management/v1alpha1" + "github.com/Axway/agent-sdk/pkg/transaction/metric" agentErrors "github.com/Axway/agent-sdk/pkg/util/errors" hc "github.com/Axway/agent-sdk/pkg/util/healthcheck" "github.com/Axway/agent-sdk/pkg/util/log" @@ -19,15 +24,19 @@ import ( ) type httpLogBeater struct { - client beat.Client - logger log.FieldLogger - server http.Server + client beat.Client + logger log.FieldLogger + server http.Server + processing sync.WaitGroup + shutdownDone sync.WaitGroup } // New creates an instance of kong_traceability_agent. func New(*beat.Beat, *common.Config) (beat.Beater, error) { b := &httpLogBeater{ - logger: log.NewFieldLogger().WithComponent("httpLogBeater").WithPackage("beater"), + logger: log.NewFieldLogger().WithComponent("httpLogBeater").WithPackage("beater"), + processing: sync.WaitGroup{}, + shutdownDone: sync.WaitGroup{}, } // Validate that all necessary services are up and running. If not, return error @@ -48,18 +57,23 @@ func (b *httpLogBeater) Run(beater *beat.Beat) error { if err != nil { return err } + agent.RegisterShutdownHandler(b.shutdownHandler) mux := http.NewServeMux() - mux.HandleFunc(config.GetAgentConfig().HttpLogPluginConfig.Path, b.HandleHello) + mux.HandleFunc(config.GetAgentConfig().KongGatewayCfg.Logs.HTTP.Path, b.HandleEvent) // other handlers can be assigned to separate paths - b.server = http.Server{Handler: mux, Addr: fmt.Sprintf(":%d", config.GetAgentConfig().HttpLogPluginConfig.Port)} - b.logger.Fatal(b.server.ListenAndServe()) + b.server = http.Server{Handler: mux, Addr: fmt.Sprintf(":%d", config.GetAgentConfig().KongGatewayCfg.Logs.HTTP.Port)} + b.server.ListenAndServe() + + // wait for the shutdown process to finish prior to exit + b.shutdownDone.Add(1) + b.shutdownDone.Wait() return nil } -func (b *httpLogBeater) HandleHello(w http.ResponseWriter, r *http.Request) { +func (b *httpLogBeater) HandleEvent(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { b.logger.Trace("received a non post request") w.WriteHeader(http.StatusMethodNotAllowed) @@ -74,19 +88,58 @@ func (b *httpLogBeater) HandleHello(w http.ResponseWriter, r *http.Request) { return } + b.processing.Add(1) go func(data []byte) { + defer b.processing.Done() ctx := context.WithValue(context.Background(), processor.CtxTransactionID, uuid.NewString()) eventProcessor, err := processor.NewEventsHandler(ctx, data) if err == nil { - eventsToPublish, err := eventProcessor.Handle() - if err == nil { - b.client.PublishAll(eventsToPublish) - } + eventsToPublish := eventProcessor.Handle() + b.client.PublishAll(eventsToPublish) } }(logData) } +func (b *httpLogBeater) shutdownHandler() { + b.logger.Info("waiting for current processing to finish") + defer b.shutdownDone.Done() + + // wait for all processing to finish + b.processing.Wait() + + // publish the metrics and usage + b.logger.Info("publishing cached metrics and usage") + metric.GetMetricCollector().ShutdownPublish() + + // clean the agent resource, if necessary + b.cleanResource() +} + +func (b *httpLogBeater) cleanResource() { + // if pod name is empty do nothing further + podName := os.Getenv("POD_NAME") + if podName == "" { + b.logger.Debug("not cleaning the agent resource, does not seem to be a kubernetes pod") + return + } + + // check if this agent resource reported an error + if agent.GetStatus() == agent.AgentFailed || agent.GetStatus() == agent.AgentUnhealthy { + b.logger.Debug("not cleaning the agent resource, agent not gracefully stopping") + return + } + + // check that this is not the last agent resource to be removed + agentRes := management.NewTraceabilityAgent(config.GetAgentConfig().CentralCfg.GetAgentName(), config.GetAgentConfig().CentralCfg.GetEnvironmentName()) + res, err := agent.GetCentralClient().GetResources(agentRes) + if len(res) > 1 && err == nil { + b.logger.Info("cleaning the agent resource") + // cleanup the agent resource + agent.GetCentralClient().DeleteResourceInstance(agentRes) + } +} + // Stop stops kong_traceability_agent. func (b *httpLogBeater) Stop() { b.server.Shutdown(context.Background()) diff --git a/pkg/traceability/beater/beater_test.go b/pkg/traceability/beater/beater_test.go new file mode 100644 index 0000000..f9204f6 --- /dev/null +++ b/pkg/traceability/beater/beater_test.go @@ -0,0 +1,95 @@ +package beater + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/Axway/agent-sdk/pkg/agent" + v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" + management "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/management/v1alpha1" + "github.com/Axway/agent-sdk/pkg/apic/mock" + corecfg "github.com/Axway/agent-sdk/pkg/config" + + "github.com/Axway/agents-kong/pkg/traceability/config" +) + +func Test_httpLogBeater_cleanResource(t *testing.T) { + testCases := map[string]struct { + podName string + agentStatus string + numTAs int + getResErr bool + expectGetCalled bool + expectDeleteCalled bool + }{ + "no pod name, nothing deleted": { + agentStatus: agent.AgentRunning, + }, + "agent in failed status, nothing deleted": { + podName: "pod", + agentStatus: agent.AgentFailed, + }, + "only one ta exists, nothing deleted": { + podName: "pod", + agentStatus: agent.AgentRunning, + numTAs: 1, + expectGetCalled: true, + }, + "error returned getting resources, nothing deleted": { + podName: "pod", + agentStatus: agent.AgentRunning, + numTAs: 2, + getResErr: true, + expectGetCalled: true, + }, + "all validations pass, delete agent res": { + podName: "pod", + agentStatus: agent.AgentRunning, + numTAs: 2, + expectGetCalled: true, + expectDeleteCalled: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + os.Setenv("POD_NAME", tc.podName) + config.SetAgentConfig(&config.AgentConfig{CentralCfg: &corecfg.CentralConfiguration{}}) + agent.UpdateStatus(tc.agentStatus, "") + + beater, err := New(nil, nil) + assert.Nil(t, err) + assert.NotNil(t, beater) + httpBeater := beater.(*httpLogBeater) + getResCalled := false + deleteResCalled := false + + mockAPICClient := &mock.Client{ + GetResourcesMock: func(ri v1.Interface) ([]v1.Interface, error) { + getResCalled = true + assert.NotNil(t, ri) + if tc.getResErr { + return nil, fmt.Errorf("error") + } + tas := []v1.Interface{} + for i := 0; i < tc.numTAs; i++ { + tas = append(tas, management.NewTraceabilityAgent("name", "env")) + } + return tas, nil + }, + DeleteResourceInstanceMock: func(ri v1.Interface) error { + deleteResCalled = true + assert.NotNil(t, ri) + return nil + }, + } + agent.InitializeForTest(mockAPICClient) + + httpBeater.cleanResource() + assert.Equal(t, tc.expectGetCalled, getResCalled) + assert.Equal(t, tc.expectDeleteCalled, deleteResCalled) + }) + } +} diff --git a/pkg/traceability/cmd/cmd.go b/pkg/traceability/cmd/cmd.go index 674c973..0765528 100644 --- a/pkg/traceability/cmd/cmd.go +++ b/pkg/traceability/cmd/cmd.go @@ -50,8 +50,8 @@ func initConfig(centralConfig corecfg.CentralConfig) (interface{}, error) { rootProps := TraceCmd.GetProperties() agentConfig := &config.AgentConfig{ - CentralCfg: centralConfig, - HttpLogPluginConfig: config.ParseProperties(rootProps), + CentralCfg: centralConfig, + KongGatewayCfg: config.ParseProperties(rootProps), } config.SetAgentConfig(agentConfig) diff --git a/pkg/traceability/config/config.go b/pkg/traceability/config/config.go index 254c08b..1e240a3 100644 --- a/pkg/traceability/config/config.go +++ b/pkg/traceability/config/config.go @@ -1,27 +1,62 @@ package config import ( - "github.com/Axway/agent-sdk/pkg/cmd/properties" + "fmt" + "strings" + corecfg "github.com/Axway/agent-sdk/pkg/config" ) +type props interface { + AddStringProperty(name string, defaultVal string, description string) + AddIntProperty(name string, defaultVal int, description string) + StringPropertyValue(name string) string + IntPropertyValue(name string) int +} + const ( - cfgKongHTTPLogPluginPath = "kong.httpLogPlugin.path" - cfgKongHTTPLogPluginPort = "kong.httpLogPlugin.port" + cfgKongHTTPLogsPath = "kong.logs.http.path" + cfgKongHTTPLogsPort = "kong.logs.http.port" ) -func AddKongProperties(rootProps properties.Properties) { - rootProps.AddStringProperty(cfgKongHTTPLogPluginPath, "/requestlogs", "Path on which the HTTP Log plugin sends request logs") - rootProps.AddIntProperty(cfgKongHTTPLogPluginPort, 9000, "Port that listens for request logs from HTTP Log plugin") +func AddKongProperties(rootProps props) { + rootProps.AddStringProperty(cfgKongHTTPLogsPath, "/requestlogs", "Path on which the HTTP Log plugin sends request logs") + rootProps.AddIntProperty(cfgKongHTTPLogsPort, 9000, "Port that listens for request logs from HTTP Log plugin") } // AgentConfig - represents the config for agent type AgentConfig struct { - CentralCfg corecfg.CentralConfig `config:"central"` - HttpLogPluginConfig *KongHttpLogPluginConfig `config:"httpLogPlugin"` + CentralCfg corecfg.CentralConfig `config:"central"` + KongGatewayCfg KongGatewayConfig `config:"kong"` +} + +// KongGatewayConfig - represents the config for gateway +type KongGatewayConfig struct { + corecfg.IConfigValidator + Logs KongLogsConfig `config:"logs"` } -type KongHttpLogPluginConfig struct { +const ( + pathErr = "a path for the http server to listen on is required" + portErr = "a port for the http server to listen on is required" +) + +// ValidateCfg - Validates the gateway config +func (c *KongGatewayConfig) ValidateCfg() (err error) { + if c.Logs.HTTP.Path == "" { + return fmt.Errorf(pathErr) + } + if c.Logs.HTTP.Port == 0 { + return fmt.Errorf(portErr) + } + return +} + +type KongLogsConfig struct { + HTTP KongLogsHTTPConfig `config:"http"` +} + +type KongLogsHTTPConfig struct { Path string `config:"path"` Port int `config:"port"` } @@ -36,10 +71,20 @@ func GetAgentConfig() *AgentConfig { return agentConfig } -func ParseProperties(rootProps properties.Properties) *KongHttpLogPluginConfig { +func ParseProperties(rootProps props) KongGatewayConfig { // Parse the config from bound properties and setup gateway config - return &KongHttpLogPluginConfig{ - Path: rootProps.StringPropertyValue(cfgKongHTTPLogPluginPath), - Port: rootProps.IntPropertyValue(cfgKongHTTPLogPluginPort), + cfg := KongGatewayConfig{ + Logs: KongLogsConfig{ + HTTP: KongLogsHTTPConfig{ + Path: rootProps.StringPropertyValue(cfgKongHTTPLogsPath), + Port: rootProps.IntPropertyValue(cfgKongHTTPLogsPort), + }, + }, + } + + if !strings.HasPrefix(cfg.Logs.HTTP.Path, "/") { + cfg.Logs.HTTP.Path = fmt.Sprintf("/%s", cfg.Logs.HTTP.Path) } + + return cfg } diff --git a/pkg/traceability/config/config_test.go b/pkg/traceability/config/config_test.go new file mode 100644 index 0000000..e322af8 --- /dev/null +++ b/pkg/traceability/config/config_test.go @@ -0,0 +1,75 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateConfig(t *testing.T) { + cfg := &KongGatewayConfig{} + + err := cfg.ValidateCfg() + assert.Equal(t, pathErr, err.Error()) + + cfg.Logs.HTTP.Path = "/" + err = cfg.ValidateCfg() + assert.Equal(t, portErr, err.Error()) + + cfg.Logs.HTTP.Port = 9000 + err = cfg.ValidateCfg() + assert.Equal(t, nil, err) +} + +type propData struct { + pType string + desc string + val interface{} +} + +type fakeProps struct { + props map[string]propData +} + +func (f *fakeProps) AddStringProperty(name string, defaultVal string, description string) { + f.props[name] = propData{"string", description, defaultVal} +} + +func (f *fakeProps) AddIntProperty(name string, defaultVal int, description string) { + f.props[name] = propData{"int", description, defaultVal} +} + +func (f *fakeProps) StringPropertyValue(name string) string { + if prop, ok := f.props[name]; ok { + return prop.val.(string) + } + return "" +} + +func (f *fakeProps) IntPropertyValue(name string) int { + if prop, ok := f.props[name]; ok { + return prop.val.(int) + } + return 0 +} + +func TestKongProperties(t *testing.T) { + newProps := &fakeProps{props: map[string]propData{}} + + // validate add props + AddKongProperties(newProps) + assert.Contains(t, newProps.props, cfgKongHTTPLogsPath) + assert.Contains(t, newProps.props, cfgKongHTTPLogsPort) + + // validate defaults + cfg := ParseProperties(newProps) + assert.Equal(t, "/requestlogs", cfg.Logs.HTTP.Path) + assert.Equal(t, 9000, cfg.Logs.HTTP.Port) + + // validate changed values + newProps.props[cfgKongHTTPLogsPath] = propData{"string", "", "another/path"} + newProps.props[cfgKongHTTPLogsPort] = propData{"int", "", 30123} + cfg = ParseProperties(newProps) + assert.Equal(t, "/another/path", cfg.Logs.HTTP.Path) + assert.Equal(t, 30123, cfg.Logs.HTTP.Port) +} diff --git a/pkg/traceability/main/agent.go b/pkg/traceability/main/agent.go index f274969..0fba253 100644 --- a/pkg/traceability/main/agent.go +++ b/pkg/traceability/main/agent.go @@ -11,6 +11,13 @@ import ( func main() { os.Setenv("AGENTFEATURES_VERSIONCHECKER", "false") + + // use the pod name as the agent name + pod_name := os.Getenv("POD_NAME") + if pod_name != "" { + os.Setenv("CENTRAL_AGENTNAME", pod_name) + } + if err := traceability.TraceCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) diff --git a/pkg/traceability/processor/definitions.go b/pkg/traceability/processor/definitions.go index a264d70..6644428 100644 --- a/pkg/traceability/processor/definitions.go +++ b/pkg/traceability/processor/definitions.go @@ -1,10 +1,14 @@ package processor -import "github.com/Axway/agent-sdk/pkg/util/log" +import ( + "github.com/Axway/agent-sdk/pkg/transaction/metric" + "github.com/Axway/agent-sdk/pkg/util/log" +) const ( CtxTransactionID log.ContextField = "transactionID" ctxEntryIndex log.ContextField = "entryIndex" + ctxRequestID log.ContextField = "requestID" ) func init() { @@ -31,19 +35,19 @@ type Latencies struct { } type Request struct { - QueryString map[string]string `json:"querystring"` - Size int `json:"size"` - URI string `json:"uri"` - URL string `json:"url"` - Headers map[string]string `json:"headers"` - Method string `json:"method"` - TLS *TLS `json:"tls"` + QueryString map[string]string `json:"querystring"` + Size int `json:"size"` + URI string `json:"uri"` + URL string `json:"url"` + Headers map[string]interface{} `json:"headers"` + Method string `json:"method"` + TLS *TLS `json:"tls"` } type Response struct { - Headers map[string]string `json:"headers"` - Status int `json:"status"` - Size int `json:"size"` + Headers map[string]interface{} `json:"headers"` + Status int `json:"status"` + Size int `json:"size"` } type Route struct { @@ -89,9 +93,13 @@ type TLS struct { } type Consumer struct { - CustomID string `json:"custom_id"` - CreatedAt int64 `json:"created_at"` - ID string `json:"id"` - Tags []string `json:"tags"` - Username string `json:"username"` + CustomID string `json:"custom_id"` + CreatedAt int64 `json:"created_at"` + ID string `json:"id"` + Tags interface{} `json:"tags"` + Username string `json:"username"` +} + +type metricCollector interface { + AddMetricDetail(metricDetail metric.Detail) } diff --git a/pkg/traceability/processor/handler.go b/pkg/traceability/processor/handler.go index d4c411d..146df2b 100644 --- a/pkg/traceability/processor/handler.go +++ b/pkg/traceability/processor/handler.go @@ -4,24 +4,37 @@ import ( "context" "encoding/json" + "github.com/Axway/agent-sdk/pkg/transaction" "github.com/Axway/agent-sdk/pkg/util/log" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/google/uuid" ) // EventsHandler - type EventsHandler struct { - ctx context.Context - logger log.FieldLogger - logEntries []TrafficLogEntry + ctx context.Context + logger log.FieldLogger + metrics MetricsProcessor + logEntries []TrafficLogEntry + requestID string + eventGenerator func() transaction.EventGenerator + collectorGetter func() metricCollector } // NewEventsHandler - return a new EventProcessor func NewEventsHandler(ctx context.Context, logData []byte) (*EventsHandler, error) { + requestID := uuid.NewString() + p := &EventsHandler{ - ctx: ctx, - logger: log.NewLoggerFromContext(ctx).WithComponent("eventsHandler").WithPackage("processor"), + ctx: ctx, + logger: log.NewLoggerFromContext(ctx).WithComponent("eventsHandler").WithPackage("processor").WithField(string(ctxRequestID), requestID), + requestID: requestID, + metrics: NewMetricsProcessor(ctx), + eventGenerator: transaction.NewEventGenerator, + collectorGetter: getMetricCollector, } + p.logger.WithField("inputData", string(logData)).Debug("data sent from kong") err := json.Unmarshal(logData, &p.logEntries) if err != nil { p.logger.WithError(err).Error("could not read log data") @@ -32,21 +45,37 @@ func NewEventsHandler(ctx context.Context, logData []byte) (*EventsHandler, erro } // Handle - processes the batch of events from the http request -func (p *EventsHandler) Handle() ([]beat.Event, error) { +func (p *EventsHandler) Handle() []beat.Event { events := make([]beat.Event, 0) p.logger.WithField("numEvents", len(p.logEntries)).Info("handling events in request") + p.metrics.setCollector(p.collectorGetter()) for i, entry := range p.logEntries { - log := p.logger.WithField(string(ctxEntryIndex), i) - processor, _ := NewTransactionProcessor(context.WithValue(p.ctx, ctxEntryIndex, i), entry) + ctx := context.WithValue(p.ctx, ctxEntryIndex, i) - // Map the log entry to log event structure expected by AMPLIFY Central Observer - newEvents, err := processor.process() + sample, err := p.metrics.process(entry) if err != nil { - log.WithError(err).Error("creating event") + p.logger.WithError(err).Error("handling event for metric") + continue + } + if !sample { continue } - events = append(events, newEvents...) + + // Map the log entry to log event structure expected by AMPLIFY Central Observer + events = append(events, p.handleTransaction(ctx, entry)...) + } + + return events +} + +func (p *EventsHandler) handleTransaction(ctx context.Context, entry TrafficLogEntry) []beat.Event { + log := p.logger.WithField(string(ctxEntryIndex), ctx.Value(ctxEntryIndex)) + + newEvents, err := NewTransactionProcessor(ctx).setEventGenerator(p.eventGenerator()).setEntry(entry).process() + if err != nil { + log.WithError(err).Error("executing transaction processor") + return []beat.Event{} } - return events, nil + return newEvents } diff --git a/pkg/traceability/processor/metrics.go b/pkg/traceability/processor/metrics.go new file mode 100644 index 0000000..041d296 --- /dev/null +++ b/pkg/traceability/processor/metrics.go @@ -0,0 +1,83 @@ +package processor + +import ( + "context" + "fmt" + + "github.com/Axway/agent-sdk/pkg/traceability/sampling" + "github.com/Axway/agent-sdk/pkg/transaction/metric" + "github.com/Axway/agent-sdk/pkg/transaction/models" + sdkUtil "github.com/Axway/agent-sdk/pkg/transaction/util" + "github.com/Axway/agent-sdk/pkg/util/log" +) + +// MetricsProcessor - +type MetricsProcessor struct { + ctx context.Context + logger log.FieldLogger + collector metricCollector +} + +func NewMetricsProcessor(ctx context.Context) MetricsProcessor { + return MetricsProcessor{ + ctx: ctx, + logger: log.NewLoggerFromContext(ctx).WithComponent("eventMapper").WithPackage("processor"), + } +} + +func (m *MetricsProcessor) setCollector(collector metricCollector) { + m.collector = collector +} + +// process - receives the log event and returns if the transaction should be sampled +func (m *MetricsProcessor) process(entry TrafficLogEntry) (bool, error) { + details := sampling.TransactionDetails{} + if entry.Response != nil { + details.Status = sdkUtil.GetTransactionSummaryStatus(entry.Response.Status) + } + if entry.Service != nil { + details.APIID = entry.Service.ID + if entry.Route != nil { + details.APIID = fmt.Sprintf("%s-%s", entry.Service.ID, entry.Route.ID) + } + } + if entry.Consumer != nil { + details.SubID = entry.Consumer.ID + } + + sample, err := sampling.ShouldSampleTransaction(details) + if err != nil { + return false, err + } + m.updateMetric(entry) + + return sample, nil +} + +func (m *MetricsProcessor) updateMetric(entry TrafficLogEntry) { + apiDetails := models.APIDetails{ + ID: sdkUtil.FormatProxyID(entry.Service.ID), + Name: entry.Service.Name, + Stage: entry.Route.ID, + Revision: 1, + } + + statusCode := entry.Response.Status + duration := entry.Latencies.Request + appDetails := models.AppDetails{} + if entry.Consumer != nil { + appDetails.Name = entry.Consumer.Username + appDetails.ID = sdkUtil.FormatApplicationID(entry.Consumer.ID) + } + + if m.collector != nil { + metricDetail := metric.Detail{ + APIDetails: apiDetails, + StatusCode: fmt.Sprint(statusCode), + Duration: int64(duration), + Bytes: int64(entry.Request.Size), + AppDetails: appDetails, + } + m.collector.AddMetricDetail(metricDetail) + } +} diff --git a/pkg/traceability/processor/mock/collector_mock.go b/pkg/traceability/processor/mock/collector_mock.go new file mode 100644 index 0000000..27cec73 --- /dev/null +++ b/pkg/traceability/processor/mock/collector_mock.go @@ -0,0 +1,28 @@ +package mock + +import ( + "sync" + + "github.com/Axway/agent-sdk/pkg/transaction/metric" +) + +var collector *CollectorMock + +func GetMockCollector() *CollectorMock { + return collector +} + +func SetMockCollector(c *CollectorMock) { + collector = c +} + +type CollectorMock struct { + sync.WaitGroup + Details []metric.Detail + Expected int +} + +func (c *CollectorMock) AddMetricDetail(metricDetail metric.Detail) { + c.Details = append(c.Details, metricDetail) + c.Done() +} diff --git a/pkg/traceability/processor/mock/eventgenerator_mock.go b/pkg/traceability/processor/mock/eventgenerator_mock.go new file mode 100644 index 0000000..2003a87 --- /dev/null +++ b/pkg/traceability/processor/mock/eventgenerator_mock.go @@ -0,0 +1,68 @@ +package mock + +import ( + "encoding/json" + "time" + + "github.com/Axway/agent-sdk/pkg/transaction" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" +) + +// EventGeneratorMock - mock event generator +type EventGeneratorMock struct { + shouldUseTrafficForAggregation bool +} + +// NewEventGeneratorMock - Create a new mock event generator +func NewEventGeneratorMock() transaction.EventGenerator { + return &EventGeneratorMock{} +} + +// CreateEvent - Creates a new mocked event for tests +func (c *EventGeneratorMock) CreateEvent(logEvent transaction.LogEvent, eventTime time.Time, metaData, eventFields common.MapStr, privateData interface{}) (event beat.Event, err error) { + serializedLogEvent, _ := json.Marshal(logEvent) + eventData := make(map[string]interface{}) + eventData["message"] = string(serializedLogEvent) + event = beat.Event{ + Timestamp: eventTime, + Meta: metaData, + Private: privateData, + Fields: eventData, + } + return +} + +// CreateEvents - Creates a new mocked event for tests +func (c *EventGeneratorMock) CreateEvents(summaryEvent transaction.LogEvent, detailEvents []transaction.LogEvent, eventTime time.Time, metaData, eventFields common.MapStr, privateData interface{}) ([]beat.Event, error) { + serializedSumEvent, _ := json.Marshal(summaryEvent) + sumEventData := make(map[string]interface{}) + sumEventData["message"] = string(serializedSumEvent) + events := []beat.Event{ + { + Timestamp: eventTime, + Meta: metaData, + Private: privateData, + Fields: sumEventData, + }, + } + + for _, detailEvent := range detailEvents { + serializedEvent, _ := json.Marshal(detailEvent) + eventData := make(map[string]interface{}) + eventData["message"] = string(serializedEvent) + events = append(events, beat.Event{ + Timestamp: eventTime, + Meta: metaData, + Private: privateData, + Fields: eventData, + }) + } + + return events, nil +} + +// SetUseTrafficForAggregation - set the flag to use traffic events for aggregation. +func (c *EventGeneratorMock) SetUseTrafficForAggregation(useTrafficForAggregation bool) { + c.shouldUseTrafficForAggregation = useTrafficForAggregation +} diff --git a/pkg/traceability/processor/processor.go b/pkg/traceability/processor/processor.go deleted file mode 100644 index 858e2f8..0000000 --- a/pkg/traceability/processor/processor.go +++ /dev/null @@ -1,170 +0,0 @@ -package processor - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "strconv" - "time" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/google/uuid" - - "github.com/Axway/agent-sdk/pkg/agent" - "github.com/Axway/agent-sdk/pkg/transaction" - sdkUtil "github.com/Axway/agent-sdk/pkg/transaction/util" - "github.com/Axway/agent-sdk/pkg/util/log" -) - -const ( - host = "host" - userAgent = "user-agent" - leg0 = "leg0" - inbound = "inbound" -) - -// TransactionProcessor - -type TransactionProcessor struct { - ctx context.Context - logger log.FieldLogger - eventGenerator transaction.EventGenerator - event TrafficLogEntry -} - -func NewTransactionProcessor(ctx context.Context, entry TrafficLogEntry) (*TransactionProcessor, bool) { - p := &TransactionProcessor{ - ctx: ctx, - logger: log.NewLoggerFromContext(ctx).WithComponent("eventMapper").WithPackage("processor"), - eventGenerator: transaction.NewEventGenerator(), - event: entry, - } - - return p, true -} - -func (m *TransactionProcessor) process() ([]beat.Event, error) { - centralCfg := agent.GetCentralConfig() - txnID := uuid.New().String() - - // leg 0 - transactionLogEvent, err := m.createTransactionEvent(m.event, txnID) - if err != nil { - m.logger.WithError(err).Error("building transaction leg event") - return nil, err - } - - // summary - summaryLogEvent, err := m.createSummaryEvent(m.event, centralCfg.GetTeamID(), txnID) - if err != nil { - m.logger.WithError(err).Error("building transaction summary event") - return nil, err - } - - // create Central log events - events, err := m.eventGenerator.CreateEvents(*summaryLogEvent, []transaction.LogEvent{*transactionLogEvent}, time.Unix(m.event.StartedAt, 0), nil, nil, nil) - if err != nil { - m.logger.WithError(err).Error("building Central events") - return nil, err - } - - return events, nil -} - -func (m *TransactionProcessor) getTransactionEventStatus(code int) transaction.TxEventStatus { - if code >= 400 { - return transaction.TxEventStatusFail - } - return transaction.TxEventStatusPass -} - -func (m *TransactionProcessor) getTransactionSummaryStatus(statusCode int) transaction.TxSummaryStatus { - transSummaryStatus := transaction.TxSummaryStatusUnknown - if statusCode >= http.StatusOK && statusCode < http.StatusBadRequest { - transSummaryStatus = transaction.TxSummaryStatusSuccess - } else if statusCode >= http.StatusBadRequest && statusCode < http.StatusInternalServerError { - transSummaryStatus = transaction.TxSummaryStatusFailure - } else if statusCode >= http.StatusInternalServerError && statusCode < http.StatusNetworkAuthenticationRequired { - transSummaryStatus = transaction.TxSummaryStatusException - } - return transSummaryStatus -} - -func (m *TransactionProcessor) buildHeaders(headers map[string]string) string { - jsonHeader, err := json.Marshal(headers) - if err != nil { - log.Error(err.Error()) - } - - return string(jsonHeader) -} - -func (m *TransactionProcessor) buildSSLInfoIfAvailable(ktle TrafficLogEntry) (string, string, string) { - if ktle.Request.TLS != nil { - return ktle.Request.TLS.Version, - ktle.Request.URL, - ktle.Request.URL // Using SSL server name as SSL subject name for now - } - return "", "", "" -} - -func (m *TransactionProcessor) processQueryArgs(args map[string]string) string { - b := new(bytes.Buffer) - for key, value := range args { - fmt.Fprintf(b, "%s=\"%s\",", key, value) - } - return b.String() -} - -func (m *TransactionProcessor) createTransactionEvent(ktle TrafficLogEntry, txnid string) (*transaction.LogEvent, error) { - - httpProtocolDetails, err := transaction.NewHTTPProtocolBuilder(). - SetURI(ktle.Request.URI). - SetMethod(ktle.Request.Method). - SetArgs(m.processQueryArgs(ktle.Request.QueryString)). - SetStatus(ktle.Response.Status, http.StatusText(ktle.Response.Status)). - SetHost(ktle.Request.Headers[host]). - SetHeaders(m.buildHeaders(ktle.Request.Headers), m.buildHeaders(ktle.Response.Headers)). - SetByteLength(ktle.Request.Size, ktle.Response.Size). - SetLocalAddress(ktle.ClientIP, 0). // Could not determine local port for now - SetRemoteAddress("", "", ktle.Service.Port). - SetSSLProperties(m.buildSSLInfoIfAvailable(ktle)). - SetUserAgent(ktle.Request.Headers[userAgent]). - Build() - - if err != nil { - log.Errorf("Error while filling protocol details for transaction event: %s", err) - return nil, err - } - - return transaction.NewTransactionEventBuilder(). - SetTimestamp(ktle.StartedAt). - SetTransactionID(txnid). - SetID(leg0). - SetSource(ktle.ClientIP). - SetDestination(ktle.Request.Headers[host]). - SetDuration(ktle.Latencies.Request). - SetDirection(inbound). - SetStatus(m.getTransactionEventStatus(ktle.Response.Status)). - SetProtocolDetail(httpProtocolDetails). - Build() -} - -func (m *TransactionProcessor) createSummaryEvent(ktle TrafficLogEntry, teamID string, txnid string) (*transaction.LogEvent, error) { - - builder := transaction.NewTransactionSummaryBuilder(). - SetTimestamp(ktle.StartedAt). - SetTransactionID(txnid). - SetStatus(m.getTransactionSummaryStatus(ktle.Response.Status), strconv.Itoa(ktle.Response.Status)). - SetTeam(teamID). - SetEntryPoint(ktle.Service.Protocol, ktle.Request.Method, ktle.Request.URI, ktle.Request.URL). - SetDuration(ktle.Latencies.Request). - SetProxy(sdkUtil.FormatProxyID(ktle.Route.ID), ktle.Service.Name, 1) - - if ktle.Consumer != nil { - builder.SetApplication(sdkUtil.FormatApplicationID(ktle.Consumer.ID), ktle.Consumer.Username) - } - - return builder.Build() -} diff --git a/pkg/traceability/processor/processor_test.go b/pkg/traceability/processor/processor_test.go new file mode 100644 index 0000000..27e079e --- /dev/null +++ b/pkg/traceability/processor/processor_test.go @@ -0,0 +1,103 @@ +package processor + +import ( + "context" + "testing" + + "github.com/Axway/agent-sdk/pkg/traceability/redaction" + "github.com/Axway/agent-sdk/pkg/traceability/sampling" + "github.com/Axway/agent-sdk/pkg/transaction/metric" + "github.com/Axway/agents-kong/pkg/traceability/processor/mock" + "github.com/stretchr/testify/assert" +) + +var testData = []byte(`[{ + "service": {"host": "httpbin.org","created_at": 1614232642,"connect_timeout": 60000,"id": "167290ee-c682-4ebf-bdea-e49a3ac5e260","protocol": "http","read_timeout": 60000,"port": 80,"path": "/anything","updated_at": 1614232642,"write_timeout": 60000,"retries": 5,"ws_id": "54baa5a9-23d6-41e0-9c9a-02434b010b25"}, + "route": {"id": "78f79740-c410-4fd9-a998-d0a60a99dc9b","paths": ["/log"],"protocols": ["http"],"strip_path": true,"created_at": 1614232648,"ws_id": "54baa5a9-23d6-41e0-9c9a-02434b010b25","request_buffering": true,"updated_at": 1614232648,"preserve_host": false,"regex_priority": 0,"response_buffering": true,"https_redirect_status_code": 426,"path_handling": "v0","service": {"id": "167290ee-c682-4ebf-bdea-e49a3ac5e260"}}, + "request": {"querystring": {"status": "available"},"size": 138,"uri": "/log","url": "http://localhost:8000/log","headers": {"host": "localhost:8000","accept-encoding": "gzip, deflate","user-agent": "HTTPie/2.4.0","accept": "*/*","connection": "keep-alive"},"method": "GET"}, + "response": {"headers": {"content-type": "application/json","date": "Thu, 25 Feb 2021 05:57:48 GMT","connection": "close","access-control-allow-credentials": "true","content-length": "503","server": "gunicorn/19.9.0","via": "kong/2.2.1.0-enterprise-edition","x-kong-proxy-latency": "57","x-kong-upstream-latency": "457","access-control-allow-origin": "*"},"status": 200,"size": 827}, + "latencies": {"request": 515,"kong": 58,"proxy": 457}, + "tries": [{"balancer_latency": 0,"port": 80,"balancer_start": 1614232668399,"ip": "18.211.130.98"}], + "client_ip": "192.168.144.1", + "workspace": "54baa5a9-23d6-41e0-9c9a-02434b010b25", + "workspace_name": "default", + "upstream_uri": "/anything", + "authenticated_entity": {"id": "c62c1455-9b1d-4f2d-8797-509ba83b8ae8"}, + "consumer": {"id": "ae974d6c-0f8a-4dc5-b701-fa0aa38592bd","created_at": 1674035962,"username_lower": "foo","username": "foo","type": 0}, + "started_at": 1614232668342 +},{ + "service": {"host": "httpbin.org","created_at": 1614232642,"connect_timeout": 60000,"id": "167290ee-c682-4ebf-bdea-e49a3ac5e260","protocol": "http","read_timeout": 60000,"port": 80,"path": "/anything","updated_at": 1614232642,"write_timeout": 60000,"retries": 5,"ws_id": "54baa5a9-23d6-41e0-9c9a-02434b010b25"}, + "route": {"id": "78f79740-c410-4fd9-a998-d0a60a99dc9b","paths": ["/log"],"protocols": ["http"],"strip_path": true,"created_at": 1614232648,"ws_id": "54baa5a9-23d6-41e0-9c9a-02434b010b25","request_buffering": true,"updated_at": 1614232648,"preserve_host": false,"regex_priority": 0,"response_buffering": true,"https_redirect_status_code": 426,"path_handling": "v0","service": {"id": "167290ee-c682-4ebf-bdea-e49a3ac5e260"}}, + "request": {"querystring": {"status": "available"},"size": 138,"uri": "/log","url": "http://localhost:8000/log","headers": {"host": "localhost:8000","accept-encoding": "gzip, deflate","user-agent": "HTTPie/2.4.0","accept": "*/*","connection": "keep-alive"},"method": "GET"}, + "response": {"headers": {"content-type": "application/json","date": "Thu, 25 Feb 2021 05:57:48 GMT","connection": "close","access-control-allow-credentials": "true","content-length": "503","server": "gunicorn/19.9.0","via": "kong/2.2.1.0-enterprise-edition","x-kong-proxy-latency": "57","x-kong-upstream-latency": "457","access-control-allow-origin": "*"},"status": 200,"size": 827}, + "latencies": {"request": 515,"kong": 58,"proxy": 457}, + "tries": [{"balancer_latency": 0,"port": 80,"balancer_start": 1614232668399,"ip": "18.211.130.98"}], + "client_ip": "192.168.144.1", + "workspace_name": "default", + "upstream_uri": "/anything", + "authenticated_entity": {"id": "c62c1455-9b1d-4f2d-8797-509ba83b8ae8"}, + "consumer": {"id": "ae974d6c-0f8a-4dc5-b701-fa0aa38592bd","created_at": 1674035962,"username_lower": "foo","username": "foo","type": 0}, + "started_at": 1614232668342 +}]`) + +func TestNewHandler(t *testing.T) { + cases := map[string]struct { + data []byte + constructorErr bool + expectedEvents int + expectedMetricDetails int + }{ + "expect error creating handler, when no data sent into handler": { + data: []byte{}, + constructorErr: true, + }, + "expect no error when empty array data sent into handler": { + data: []byte("[]"), + }, + "handle data with sampling setup": { + data: testData, + expectedEvents: 4, + expectedMetricDetails: 2, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctx := context.WithValue(context.Background(), "test", name) + + redaction.SetupGlobalRedaction(redaction.DefaultConfig()) + sampling.SetupSampling(sampling.DefaultConfig(), false) + + // create the handler + h, err := NewEventsHandler(ctx, tc.data) + if tc.constructorErr { + assert.NotNil(t, err) + assert.Nil(t, h) + return + } + assert.Nil(t, err) + assert.NotNil(t, h) + + // setup collector + collector := &mock.CollectorMock{Details: make([]metric.Detail, 0), Expected: tc.expectedMetricDetails} + mock.SetMockCollector(collector) + h.collectorGetter = func() metricCollector { + return mock.GetMockCollector() + } + + // setup event generator + h.eventGenerator = mock.NewEventGeneratorMock + + // if metric details are expected + if tc.expectedMetricDetails > 1 { + collector.Add(tc.expectedMetricDetails) + } + + // execute the handler + events := h.Handle() + collector.Wait() + assert.Nil(t, err) + assert.Len(t, events, tc.expectedEvents) + assert.Equal(t, tc.expectedMetricDetails, len(mock.GetMockCollector().Details)) + }) + } +} diff --git a/pkg/traceability/processor/transaction.go b/pkg/traceability/processor/transaction.go new file mode 100644 index 0000000..fe11b3c --- /dev/null +++ b/pkg/traceability/processor/transaction.go @@ -0,0 +1,144 @@ +package processor + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/google/uuid" + + "github.com/Axway/agent-sdk/pkg/transaction" + sdkUtil "github.com/Axway/agent-sdk/pkg/transaction/util" + "github.com/Axway/agent-sdk/pkg/util/log" +) + +const ( + host = "host" + userAgent = "user-agent" + leg0 = "leg0" + inbound = "inbound" +) + +// TransactionProcessor - +type TransactionProcessor struct { + ctx context.Context + logger log.FieldLogger + eventGenerator transaction.EventGenerator + event TrafficLogEntry +} + +func NewTransactionProcessor(ctx context.Context) *TransactionProcessor { + p := &TransactionProcessor{ + ctx: ctx, + logger: log.NewLoggerFromContext(ctx).WithComponent("eventMapper").WithPackage("processor"). + WithField(string(ctxEntryIndex), ctx.Value(ctxEntryIndex)). + WithField(string(ctxRequestID), ctx.Value(ctxRequestID)), + } + return p +} + +func (p *TransactionProcessor) setEntry(entry TrafficLogEntry) *TransactionProcessor { + p.event = entry + return p +} + +func (p *TransactionProcessor) setEventGenerator(eventGenerator transaction.EventGenerator) *TransactionProcessor { + p.eventGenerator = eventGenerator + p.eventGenerator.SetUseTrafficForAggregation(false) + return p +} + +func (p *TransactionProcessor) process() ([]beat.Event, error) { + if p.eventGenerator == nil { + return nil, fmt.Errorf("an event generator is required") + } + txnID := uuid.New().String() + + // leg 0 + transactionLogEvent, err := p.createTransactionEvent(txnID) + if err != nil { + p.logger.WithError(err).Error("building transaction leg event") + return nil, err + } + legEvent, err := p.eventGenerator.CreateEvent(*transactionLogEvent, time.Unix(p.event.StartedAt, 0), nil, nil, nil) + if err != nil { + p.logger.WithError(err).Error("creating transaction leg event") + return nil, err + } + + // summary + summaryLogEvent, err := p.createSummaryEvent("id", txnID) + if err != nil { + p.logger.WithError(err).Error("building transaction summary event") + return nil, err + } + summaryEvent, err := p.eventGenerator.CreateEvent(*summaryLogEvent, time.Unix(p.event.StartedAt, 0), nil, nil, nil) + if err != nil { + p.logger.WithError(err).Error("creating transaction summary event") + return nil, err + } + + return []beat.Event{summaryEvent, legEvent}, nil +} + +func (p *TransactionProcessor) createTransactionEvent(txnid string) (*transaction.LogEvent, error) { + requestHost := "" + if value, found := p.event.Request.Headers[host]; found { + requestHost = fmt.Sprintf("%v", value) + } + + userAgentVal := "" + if value, found := p.event.Request.Headers[userAgent]; found { + userAgentVal = fmt.Sprintf("%v", value) + } + + httpProtocolDetails, err := transaction.NewHTTPProtocolBuilder(). + SetURI(p.event.Request.URI). + SetMethod(p.event.Request.Method). + SetArgsMap(processQueryArgs(p.event.Request.QueryString)). + SetStatus(p.event.Response.Status, http.StatusText(p.event.Response.Status)). + SetHost(requestHost). + SetHeaders(buildHeaders(p.event.Request.Headers), buildHeaders(p.event.Response.Headers)). + SetByteLength(p.event.Request.Size, p.event.Response.Size). + SetLocalAddress(p.event.ClientIP, 0). // Could not determine local port for now + SetRemoteAddress("", "", p.event.Service.Port). + SetSSLProperties(buildSSLInfoIfAvailable(p.event)). + SetUserAgent(userAgentVal). + Build() + + if err != nil { + return nil, err + } + + return transaction.NewTransactionEventBuilder(). + SetTimestamp(p.event.StartedAt). + SetTransactionID(txnid). + SetID(leg0). + SetSource(p.event.ClientIP). + SetDestination(requestHost). + SetDuration(p.event.Latencies.Request). + SetDirection(inbound). + SetStatus(getTransactionEventStatus(p.event.Response.Status)). + SetProtocolDetail(httpProtocolDetails). + Build() +} + +func (p *TransactionProcessor) createSummaryEvent(teamID string, txnid string) (*transaction.LogEvent, error) { + builder := transaction.NewTransactionSummaryBuilder(). + SetTimestamp(p.event.StartedAt). + SetTransactionID(txnid). + SetStatus(getTransactionSummaryStatus(p.event.Response.Status), strconv.Itoa(p.event.Response.Status)). + SetTeam(teamID). + SetEntryPoint(p.event.Service.Protocol, p.event.Request.Method, p.event.Request.URI, p.event.Request.URL). + SetDuration(p.event.Latencies.Request). + SetProxyWithStage(sdkUtil.FormatProxyID(p.event.Service.ID), p.event.Service.Name, p.event.Route.ID, 1) + + if p.event.Consumer != nil { + builder.SetApplication(sdkUtil.FormatApplicationID(p.event.Consumer.ID), p.event.Consumer.Username) + } + + return builder.Build() +} diff --git a/pkg/traceability/processor/util.go b/pkg/traceability/processor/util.go new file mode 100644 index 0000000..32ba611 --- /dev/null +++ b/pkg/traceability/processor/util.go @@ -0,0 +1,64 @@ +package processor + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/Axway/agent-sdk/pkg/transaction" + "github.com/Axway/agent-sdk/pkg/transaction/metric" + "github.com/Axway/agent-sdk/pkg/util/log" +) + +func getTransactionEventStatus(code int) transaction.TxEventStatus { + if code >= 400 { + return transaction.TxEventStatusFail + } + return transaction.TxEventStatusPass +} + +func getTransactionSummaryStatus(statusCode int) transaction.TxSummaryStatus { + transSummaryStatus := transaction.TxSummaryStatusUnknown + if statusCode >= http.StatusOK && statusCode < http.StatusBadRequest { + transSummaryStatus = transaction.TxSummaryStatusSuccess + } else if statusCode >= http.StatusBadRequest && statusCode < http.StatusInternalServerError { + transSummaryStatus = transaction.TxSummaryStatusFailure + } else if statusCode >= http.StatusInternalServerError && statusCode < http.StatusNetworkAuthenticationRequired { + transSummaryStatus = transaction.TxSummaryStatusException + } + return transSummaryStatus +} + +func buildHeaders(headers map[string]interface{}) string { + newHeaders := make(map[string]string) + for key, val := range headers { + newHeaders[key] = fmt.Sprintf("%v", val) + } + + jsonHeader, err := json.Marshal(newHeaders) + if err != nil { + log.Error(err.Error()) + } + return string(jsonHeader) +} + +func buildSSLInfoIfAvailable(ktle TrafficLogEntry) (string, string, string) { + if ktle.Request.TLS != nil { + return ktle.Request.TLS.Version, + ktle.Request.URL, + ktle.Request.URL // Using SSL server name as SSL subject name for now + } + return "", "", "" +} + +func processQueryArgs(args map[string]string) map[string][]string { + newArgs := make(map[string][]string) + for key, value := range args { + newArgs[key] = []string{value} + } + return newArgs +} + +func getMetricCollector() metricCollector { + return metric.GetMetricCollector() +} diff --git a/png/KongAgentHTTP.png b/png/KongAgentHTTP.png deleted file mode 100644 index f9298fc..0000000 Binary files a/png/KongAgentHTTP.png and /dev/null differ diff --git a/png/KongFilePlugin.png b/png/KongFilePlugin.png deleted file mode 100644 index 919fa76..0000000 Binary files a/png/KongFilePlugin.png and /dev/null differ diff --git a/png/KongHTTPPlugin.png b/png/KongHTTPPlugin.png deleted file mode 100644 index 881457f..0000000 Binary files a/png/KongHTTPPlugin.png and /dev/null differ diff --git a/sample_env_vars.env b/sample_env_vars.env new file mode 100644 index 0000000..d30e9f2 --- /dev/null +++ b/sample_env_vars.env @@ -0,0 +1,26 @@ +CENTRAL_ENVIRONMENT="kong-env-name", +CENTRAL_ORGANIZATIONID="28123456789", +CENTRAL_PLATFORM_URL="central_platform_url", +CENTRAL_POLLINTERVAL="30m", +CENTRAL_TEAM="Default Team", +CENTRAL_URL="central_url", +CENTRAL_GRPC_ENABLED="true", +CENTRAL_AUTH_CLIENTID="central_auth_clientid", +CENTRAL_AUTH_PRIVATEKEY="/path/to/private_key.pem", +CENTRAL_AUTH_PUBLICKEY="/path/to/public_key.pem", +CENTRAL_AUTH_URL="central_auth_url", +LOG_LEVEL="info", +LOG_FORMAT="json", +LOG_OUTPUT="stdout", +LOG_PATH="logs", +KONG_PROXY_HOST="kong.host.name.where.gateway.lives", +KONG_PROXY_PORTS_HTTP="8000", +KONG_PROXY_PORTS_HTTPS="8443", +KONG_ADMIN_ROUTEPATH="/admin-api", +KONG_ADMIN_AUTH_BASICAUTH_USERNAME="gh", +KONG_ADMIN_AUTH_BASICAUTH_PASSWORD="12", +KONG_ADMIN_AUTH_APIKEY_HEADER="apikey", +KONG_ADMIN_AUTH_APIKEY_VALUE="12", +KONG_SPEC_LOCALPATH="/local/path/to/specs", +KONG_SPEC_URLPATHS="openapi.json,swagger.json", +AGENTFEATURES_MARKETPLACEPROVISIONING="true" \ No newline at end of file diff --git a/specs/petstore.json b/specs/petstore.json deleted file mode 100644 index c00d4bf..0000000 --- a/specs/petstore.json +++ /dev/null @@ -1,1054 +0,0 @@ -{ - "swagger": "2.0", - "info": { - "description": "This is a sample server Petstore server. You can find out more about Swagger at [http://swagger.io](http://swagger.io) or on [irc.freenode.net, #swagger](http://swagger.io/irc/). For this sample, you can use the api key `special-key` to test the authorization filters.", - "version": "1.0.5", - "title": "Swagger Petstore", - "termsOfService": "http://swagger.io/terms/", - "contact": { - "email": "apiteam@swagger.io" - }, - "license": { - "name": "Apache 2.0", - "url": "http://www.apache.org/licenses/LICENSE-2.0.html" - } - }, - "host": "petstore.swagger.io", - "basePath": "/v2", - "tags": [ - { - "name": "pet", - "description": "Everything about your Pets", - "externalDocs": { - "description": "Find out more", - "url": "http://swagger.io" - } - }, - { - "name": "store", - "description": "Access to Petstore orders" - }, - { - "name": "user", - "description": "Operations about user", - "externalDocs": { - "description": "Find out more about our store", - "url": "http://swagger.io" - } - } - ], - "schemes": [ - "https", - "http" - ], - "paths": { - "/pet/{petId}/uploadImage": { - "post": { - "tags": [ - "pet" - ], - "summary": "uploads an image", - "description": "", - "operationId": "uploadFile", - "consumes": [ - "multipart/form-data" - ], - "produces": [ - "application/json" - ], - "parameters": [ - { - "name": "petId", - "in": "path", - "description": "ID of pet to update", - "required": true, - "type": "integer", - "format": "int64" - }, - { - "name": "additionalMetadata", - "in": "formData", - "description": "Additional data to pass to server", - "required": false, - "type": "string" - }, - { - "name": "file", - "in": "formData", - "description": "file to upload", - "required": false, - "type": "file" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "$ref": "#/definitions/ApiResponse" - } - } - }, - "security": [ - { - "petstore_auth": [ - "write:pets", - "read:pets" - ] - } - ] - } - }, - "/pet": { - "post": { - "tags": [ - "pet" - ], - "summary": "Add a new pet to the store", - "description": "", - "operationId": "addPet", - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Pet object that needs to be added to the store", - "required": true, - "schema": { - "$ref": "#/definitions/Pet" - } - } - ], - "responses": { - "405": { - "description": "Invalid input" - } - }, - "security": [ - { - "petstore_auth": [ - "write:pets", - "read:pets" - ] - } - ] - }, - "put": { - "tags": [ - "pet" - ], - "summary": "Update an existing pet", - "description": "", - "operationId": "updatePet", - "consumes": [ - "application/json", - "application/xml" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Pet object that needs to be added to the store", - "required": true, - "schema": { - "$ref": "#/definitions/Pet" - } - } - ], - "responses": { - "400": { - "description": "Invalid ID supplied" - }, - "404": { - "description": "Pet not found" - }, - "405": { - "description": "Validation exception" - } - }, - "security": [ - { - "petstore_auth": [ - "write:pets", - "read:pets" - ] - } - ] - } - }, - "/pet/findByStatus": { - "get": { - "tags": [ - "pet" - ], - "summary": "Finds Pets by status", - "description": "Multiple status values can be provided with comma separated strings", - "operationId": "findPetsByStatus", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "status", - "in": "query", - "description": "Status values that need to be considered for filter", - "required": true, - "type": "array", - "items": { - "type": "string", - "enum": [ - "available", - "pending", - "sold" - ], - "default": "available" - }, - "collectionFormat": "multi" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Pet" - } - } - }, - "400": { - "description": "Invalid status value" - } - }, - "security": [ - { - "petstore_auth": [ - "write:pets", - "read:pets" - ] - } - ] - } - }, - "/pet/findByTags": { - "get": { - "tags": [ - "pet" - ], - "summary": "Finds Pets by tags", - "description": "Multiple tags can be provided with comma separated strings. Use tag1, tag2, tag3 for testing.", - "operationId": "findPetsByTags", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "tags", - "in": "query", - "description": "Tags to filter by", - "required": true, - "type": "array", - "items": { - "type": "string" - }, - "collectionFormat": "multi" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Pet" - } - } - }, - "400": { - "description": "Invalid tag value" - } - }, - "security": [ - { - "petstore_auth": [ - "write:pets", - "read:pets" - ] - } - ], - "deprecated": true - } - }, - "/pet/{petId}": { - "get": { - "tags": [ - "pet" - ], - "summary": "Find pet by ID", - "description": "Returns a single pet", - "operationId": "getPetById", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "petId", - "in": "path", - "description": "ID of pet to return", - "required": true, - "type": "integer", - "format": "int64" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "$ref": "#/definitions/Pet" - } - }, - "400": { - "description": "Invalid ID supplied" - }, - "404": { - "description": "Pet not found" - } - }, - "security": [ - { - "api_key": [] - } - ] - }, - "post": { - "tags": [ - "pet" - ], - "summary": "Updates a pet in the store with form data", - "description": "", - "operationId": "updatePetWithForm", - "consumes": [ - "application/x-www-form-urlencoded" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "petId", - "in": "path", - "description": "ID of pet that needs to be updated", - "required": true, - "type": "integer", - "format": "int64" - }, - { - "name": "name", - "in": "formData", - "description": "Updated name of the pet", - "required": false, - "type": "string" - }, - { - "name": "status", - "in": "formData", - "description": "Updated status of the pet", - "required": false, - "type": "string" - } - ], - "responses": { - "405": { - "description": "Invalid input" - } - }, - "security": [ - { - "petstore_auth": [ - "write:pets", - "read:pets" - ] - } - ] - }, - "delete": { - "tags": [ - "pet" - ], - "summary": "Deletes a pet", - "description": "", - "operationId": "deletePet", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "api_key", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "petId", - "in": "path", - "description": "Pet id to delete", - "required": true, - "type": "integer", - "format": "int64" - } - ], - "responses": { - "400": { - "description": "Invalid ID supplied" - }, - "404": { - "description": "Pet not found" - } - }, - "security": [ - { - "petstore_auth": [ - "write:pets", - "read:pets" - ] - } - ] - } - }, - "/store/order": { - "post": { - "tags": [ - "store" - ], - "summary": "Place an order for a pet", - "description": "", - "operationId": "placeOrder", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "order placed for purchasing the pet", - "required": true, - "schema": { - "$ref": "#/definitions/Order" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "$ref": "#/definitions/Order" - } - }, - "400": { - "description": "Invalid Order" - } - } - } - }, - "/store/order/{orderId}": { - "get": { - "tags": [ - "store" - ], - "summary": "Find purchase order by ID", - "description": "For valid response try integer IDs with value >= 1 and <= 10. Other values will generated exceptions", - "operationId": "getOrderById", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "orderId", - "in": "path", - "description": "ID of pet that needs to be fetched", - "required": true, - "type": "integer", - "maximum": 10, - "minimum": 1, - "format": "int64" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "$ref": "#/definitions/Order" - } - }, - "400": { - "description": "Invalid ID supplied" - }, - "404": { - "description": "Order not found" - } - } - }, - "delete": { - "tags": [ - "store" - ], - "summary": "Delete purchase order by ID", - "description": "For valid response try integer IDs with positive integer value. Negative or non-integer values will generate API errors", - "operationId": "deleteOrder", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "orderId", - "in": "path", - "description": "ID of the order that needs to be deleted", - "required": true, - "type": "integer", - "minimum": 1, - "format": "int64" - } - ], - "responses": { - "400": { - "description": "Invalid ID supplied" - }, - "404": { - "description": "Order not found" - } - } - } - }, - "/store/inventory": { - "get": { - "tags": [ - "store" - ], - "summary": "Returns pet inventories by status", - "description": "Returns a map of status codes to quantities", - "operationId": "getInventory", - "produces": [ - "application/json" - ], - "parameters": [], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "type": "object", - "additionalProperties": { - "type": "integer", - "format": "int32" - } - } - } - }, - "security": [ - { - "api_key": [] - } - ] - } - }, - "/user/createWithArray": { - "post": { - "tags": [ - "user" - ], - "summary": "Creates list of users with given input array", - "description": "", - "operationId": "createUsersWithArrayInput", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "List of user object", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/User" - } - } - } - ], - "responses": { - "default": { - "description": "successful operation" - } - } - } - }, - "/user/createWithList": { - "post": { - "tags": [ - "user" - ], - "summary": "Creates list of users with given input array", - "description": "", - "operationId": "createUsersWithListInput", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "List of user object", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/User" - } - } - } - ], - "responses": { - "default": { - "description": "successful operation" - } - } - } - }, - "/user/{username}": { - "get": { - "tags": [ - "user" - ], - "summary": "Get user by user name", - "description": "", - "operationId": "getUserByName", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "username", - "in": "path", - "description": "The name that needs to be fetched. Use user1 for testing. ", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "successful operation", - "schema": { - "$ref": "#/definitions/User" - } - }, - "400": { - "description": "Invalid username supplied" - }, - "404": { - "description": "User not found" - } - } - }, - "put": { - "tags": [ - "user" - ], - "summary": "Updated user", - "description": "This can only be done by the logged in user.", - "operationId": "updateUser", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "username", - "in": "path", - "description": "name that need to be updated", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "description": "Updated user object", - "required": true, - "schema": { - "$ref": "#/definitions/User" - } - } - ], - "responses": { - "400": { - "description": "Invalid user supplied" - }, - "404": { - "description": "User not found" - } - } - }, - "delete": { - "tags": [ - "user" - ], - "summary": "Delete user", - "description": "This can only be done by the logged in user.", - "operationId": "deleteUser", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "username", - "in": "path", - "description": "The name that needs to be deleted", - "required": true, - "type": "string" - } - ], - "responses": { - "400": { - "description": "Invalid username supplied" - }, - "404": { - "description": "User not found" - } - } - } - }, - "/user/login": { - "get": { - "tags": [ - "user" - ], - "summary": "Logs user into the system", - "description": "", - "operationId": "loginUser", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "name": "username", - "in": "query", - "description": "The user name for login", - "required": true, - "type": "string" - }, - { - "name": "password", - "in": "query", - "description": "The password for login in clear text", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "successful operation", - "headers": { - "X-Expires-After": { - "type": "string", - "format": "date-time", - "description": "date in UTC when token expires" - }, - "X-Rate-Limit": { - "type": "integer", - "format": "int32", - "description": "calls per hour allowed by the user" - } - }, - "schema": { - "type": "string" - } - }, - "400": { - "description": "Invalid username/password supplied" - } - } - } - }, - "/user/logout": { - "get": { - "tags": [ - "user" - ], - "summary": "Logs out current logged in user session", - "description": "", - "operationId": "logoutUser", - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [], - "responses": { - "default": { - "description": "successful operation" - } - } - } - }, - "/user": { - "post": { - "tags": [ - "user" - ], - "summary": "Create user", - "description": "This can only be done by the logged in user.", - "operationId": "createUser", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json", - "application/xml" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "Created user object", - "required": true, - "schema": { - "$ref": "#/definitions/User" - } - } - ], - "responses": { - "default": { - "description": "successful operation" - } - } - } - } - }, - "securityDefinitions": { - "api_key": { - "type": "apiKey", - "name": "api_key", - "in": "header" - }, - "petstore_auth": { - "type": "oauth2", - "authorizationUrl": "https://petstore.swagger.io/oauth/authorize", - "flow": "implicit", - "scopes": { - "read:pets": "read your pets", - "write:pets": "modify pets in your account" - } - } - }, - "definitions": { - "ApiResponse": { - "type": "object", - "properties": { - "code": { - "type": "integer", - "format": "int32" - }, - "type": { - "type": "string" - }, - "message": { - "type": "string" - } - } - }, - "Category": { - "type": "object", - "properties": { - "id": { - "type": "integer", - "format": "int64" - }, - "name": { - "type": "string" - } - }, - "xml": { - "name": "Category" - } - }, - "Pet": { - "type": "object", - "required": [ - "name", - "photoUrls" - ], - "properties": { - "id": { - "type": "integer", - "format": "int64" - }, - "category": { - "$ref": "#/definitions/Category" - }, - "name": { - "type": "string", - "example": "doggie" - }, - "photoUrls": { - "type": "array", - "xml": { - "wrapped": true - }, - "items": { - "type": "string", - "xml": { - "name": "photoUrl" - } - } - }, - "tags": { - "type": "array", - "xml": { - "wrapped": true - }, - "items": { - "xml": { - "name": "tag" - }, - "$ref": "#/definitions/Tag" - } - }, - "status": { - "type": "string", - "description": "pet status in the store", - "enum": [ - "available", - "pending", - "sold" - ] - } - }, - "xml": { - "name": "Pet" - } - }, - "Tag": { - "type": "object", - "properties": { - "id": { - "type": "integer", - "format": "int64" - }, - "name": { - "type": "string" - } - }, - "xml": { - "name": "Tag" - } - }, - "Order": { - "type": "object", - "properties": { - "id": { - "type": "integer", - "format": "int64" - }, - "petId": { - "type": "integer", - "format": "int64" - }, - "quantity": { - "type": "integer", - "format": "int32" - }, - "shipDate": { - "type": "string", - "format": "date-time" - }, - "status": { - "type": "string", - "description": "Order Status", - "enum": [ - "placed", - "approved", - "delivered" - ] - }, - "complete": { - "type": "boolean" - } - }, - "xml": { - "name": "Order" - } - }, - "User": { - "type": "object", - "properties": { - "id": { - "type": "integer", - "format": "int64" - }, - "username": { - "type": "string" - }, - "firstName": { - "type": "string" - }, - "lastName": { - "type": "string" - }, - "email": { - "type": "string" - }, - "password": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "userStatus": { - "type": "integer", - "format": "int32", - "description": "User Status" - } - }, - "xml": { - "name": "User" - } - } - }, - "externalDocs": { - "description": "Find out more about Swagger", - "url": "http://swagger.io" - } -} \ No newline at end of file diff --git a/whitesource.config b/whitesource.config deleted file mode 100644 index 3e83429..0000000 --- a/whitesource.config +++ /dev/null @@ -1,26 +0,0 @@ -############################################################### -# WhiteSource Unified-Agent configuration file -############################################################### - -# Change the below URL to your WhiteSource server. -# Use the 'WhiteSource Server URL' which can be retrieved -# from your 'Profile' page on the 'Server URLs' panel. -# Then, add the '/agent' path to it. -wss.url=https://axway.whitesourcesoftware.com/agent - -######################################## -# Package Manager Dependency resolvers # -######################################## -resolveAllDependencies=false -go.collectDependenciesAtRuntime=false -go.resolveDependencies=false -go.modules.resolveDependencies=true - -########################################################################################### -# Includes/Excludes Glob patterns - Please use only one exclude line and one include line # -########################################################################################### -includes=**/*.go go.mod - -excludes=**/*_test.go - -wss.url=https://axway.whitesourcesoftware.com/agent