Skip to content

Commit

Permalink
APIGOV-27481 - proxy discovery updates (#199)
Browse files Browse the repository at this point in the history 
* use local map for func

* INT - match to spec resources by name

* add new match on url config

* use the url from association file without checking cache

* add new var to docs

* clean unused code

* update handling of endpoints in context

* check response code when retrieving spec
  • Loading branch information
@jcollins-axway
jcollins-axway authored Mar 15, 2024
1 parent cf728ec commit ec46516
Show file tree
Hide file tree
Showing 8 changed files with 178 additions and 75 deletions.
3 changes: 3 additions & 0 deletions client/pkg/apigee/specs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ func (a *ApigeeClient) GetSpecFile(specPath string) ([]byte, error) {
if err != nil {
return nil, err
}
if response.Code != http.StatusOK {
return nil, fmt.Errorf("spec file not found at path")
}

return response.Body, nil
}
Expand Down
4 changes: 4 additions & 0 deletions client/pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type ApigeeConfig struct {
type ApigeeSpecConfig struct {
DisablePollForSpecs bool `config:"disablePollForSpecs"`
Unstructured bool `config:"unstructured"`
MatchOnURL bool `config:"matchOnURL"`
LocalPath string `config:"localDirectory"`
SpecExtensions string `config:"extensions"`
Extensions []string
Expand Down Expand Up @@ -125,6 +126,7 @@ const (
pathSpecWorkers = "apigee.workers.spec"
pathProxyWorkers = "apigee.workers.proxy"
pathProductWorkers = "apigee.workers.product"
pathSpecMatchOnURL = "apigee.specConfig.matchOnURL"
pathSpecLocalPath = "apigee.specConfig.localPath"
pathSpecExtensions = "apigee.specConfig.extensions"
pathSpecUnstructured = "apigee.specConfig.unstructured"
Expand Down Expand Up @@ -156,6 +158,7 @@ func AddProperties(rootProps props) {
rootProps.AddIntProperty(pathProxyWorkers, 10, "Max number of workers discovering proxies")
rootProps.AddIntProperty(pathSpecWorkers, 20, "Max number of workers discovering specs")
rootProps.AddIntProperty(pathProductWorkers, 10, "Max number of workers discovering products")
rootProps.AddBoolProperty(pathSpecMatchOnURL, true, "Set to false to skip matching spec URLs to proxy URLs")
rootProps.AddStringProperty(pathSpecLocalPath, "", "Path to a local directory that contains the spec files")
rootProps.AddStringProperty(pathSpecExtensions, "json,yaml,yml", "Comma separated list of spec file extensions, needed for proxy mode")
rootProps.AddBoolProperty(pathSpecUnstructured, false, "Set to true to enable discovering apis that have no associated spec")
Expand Down Expand Up @@ -200,6 +203,7 @@ func ParseConfig(rootProps props) *ApigeeConfig {
BasicAuth: rootProps.BoolPropertyValue(pathAuthBasicAuth),
},
Specs: &ApigeeSpecConfig{
MatchOnURL: rootProps.BoolPropertyValue(pathSpecMatchOnURL),
LocalPath: rootProps.StringPropertyValue(pathSpecLocalPath),
DisablePollForSpecs: rootProps.BoolPropertyValue(pathSpecDisablePollForSpecs),
Unstructured: rootProps.BoolPropertyValue(pathSpecUnstructured),
Expand Down
53 changes: 27 additions & 26 deletions discovery/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,32 +104,33 @@ Here is a sample Quota policy that may be added to the desired Proxies.
* TimeUnit - in this case using the API Key policy gets the quota time unit from the product definition
Í

| Environment Variable | Description | Default (if applicable) |
| ------------------------------------- | ----------------------------------------------------------------------------------------------- | --------------------------------- |
| APIGEE_URL | The base Apigee URL for this agent to connect to | https://api.enterprise.apigee.com |
| APIGEE_APIVERSION | The version of the API for the agent to use | v1 |
| APIGEE_DATAURL | The base Apigee Data API URL for this agent to connect to | https://apigee.com/dapi/api |
| APIGEE_ORGANIZATION | The Apigee organization name | |
| APIGEE_DEVELOPERID | The Apigee developer, email, that will own all apps | |
| APIGEE_DISCOVERYMODE | The mode in which the agent operates, discover proxies (proxy) or products (product) | proxy |
| APIGEE_FILTER | The tag filter to use against an Apigee product's attributes, only in product mode | |
| APIGEE_CLONEATTRIBUTES | Set this to true if the tags on a product should also be cloned on provisioning | false |
| APIGEE_INTERVAL_PROXY | The polling interval checking for API Proxy changes, only in proxy mode | 30s (30 seconds), >=30s, <=5m |
| APIGEE_INTERVAL_PRODUCT | The polling interval checking for Product changes, only in product mode | 30s (30 seconds), >=30s, <=5m |
| APIGEE_INTERVAL_SPEC | The polling interval for checking for new Specs | 30m (30 minute), >=1m |
| APIGEE_WORKERS_PROXY | The number of workers processing API Proxies, only in proxy mode | 10 |
| APIGEE_WORKERS_PRODUCT | The number of workers processing Products, only in product mode | 10 |
| APIGEE_WORKERS_SPEC | The number of workers processing API Specs | 20 |
| APIGEE_AUTH_USERNAME | The Apigee account username/email address | |
| APIGEE_AUTH_PASSWORD | The Apigee account password | |
| APIGEE_AUTH_USEBASICAUTH | Set this to true to have the Apigee api client use HTTP Basic Authentication | false |
| APIGEE_AUTH_URL | The IDP URL | https://login.apigee.com |
| APIGEE_AUTH_SERVERUSERNAME | The IDP username for requesting tokens | edgecli |
| APIGEE_AUTH_SERVERPASSWORD | The IDP password for requesting tokens | edgeclisecret |
| APIGEE_SPECCONFIG_LOCALPATH | Path to a local directory that contains the spec files | |
| APIGEE_SPECCONFIG_EXTENSIONS | Comma separated list of file extensions that the agent will look for spec in the local path for | json,yaml,yml |
| APIGEE_SPECCONFIG_UNSTRUCTURED | Set to true to enable discovering apis that have no associated spec | false |
| APIGEE_SPECCONFIG_DISABLEPOLLFORSPECS | Set to true to disable polling apigee for specs, rely on the local directory or spec URLs | false |
| Environment Variable | Description | Default (if applicable) |
| ------------------------------------- | --------------------------------------------------------------------------------------------------- | --------------------------------- |
| APIGEE_URL | The base Apigee URL for this agent to connect to | https://api.enterprise.apigee.com |
| APIGEE_APIVERSION | The version of the API for the agent to use | v1 |
| APIGEE_DATAURL | The base Apigee Data API URL for this agent to connect to | https://apigee.com/dapi/api |
| APIGEE_ORGANIZATION | The Apigee organization name | |
| APIGEE_DEVELOPERID | The Apigee developer, email, that will own all apps | |
| APIGEE_DISCOVERYMODE | The mode in which the agent operates, discover proxies (proxy) or products (product) | proxy |
| APIGEE_FILTER | The tag filter to use against an Apigee product's attributes, only in product mode | |
| APIGEE_CLONEATTRIBUTES | Set this to true if the tags on a product should also be cloned on provisioning | false |
| APIGEE_INTERVAL_PROXY | The polling interval checking for API Proxy changes, only in proxy mode | 30s (30 seconds), >=30s, <=5m |
| APIGEE_INTERVAL_PRODUCT | The polling interval checking for Product changes, only in product mode | 30s (30 seconds), >=30s, <=5m |
| APIGEE_INTERVAL_SPEC | The polling interval for checking for new Specs | 30m (30 minute), >=1m |
| APIGEE_WORKERS_PROXY | The number of workers processing API Proxies, only in proxy mode | 10 |
| APIGEE_WORKERS_PRODUCT | The number of workers processing Products, only in product mode | 10 |
| APIGEE_WORKERS_SPEC | The number of workers processing API Specs | 20 |
| APIGEE_AUTH_USERNAME | The Apigee account username/email address | |
| APIGEE_AUTH_PASSWORD | The Apigee account password | |
| APIGEE_AUTH_USEBASICAUTH | Set this to true to have the Apigee api client use HTTP Basic Authentication | false |
| APIGEE_AUTH_URL | The IDP URL | https://login.apigee.com |
| APIGEE_AUTH_SERVERUSERNAME | The IDP username for requesting tokens | edgecli |
| APIGEE_AUTH_SERVERPASSWORD | The IDP password for requesting tokens | edgeclisecret |
| APIGEE_SPECCONFIG_MATCHONURL | Set to false to skip parsing specs for URLs and matching to computed proxy url for spec association | true |
| APIGEE_SPECCONFIG_LOCALPATH | Path to a local directory that contains the spec files | |
| APIGEE_SPECCONFIG_EXTENSIONS | Comma separated list of file extensions that the agent will look for spec in the local path for | json,yaml,yml |
| APIGEE_SPECCONFIG_UNSTRUCTURED | Set to true to enable discovering apis that have no associated spec | false |
| APIGEE_SPECCONFIG_DISABLEPOLLFORSPECS | Set to true to disable polling apigee for specs, rely on the local directory or spec URLs | false |


## Development
Expand Down
16 changes: 14 additions & 2 deletions discovery/pkg/apigee/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,14 @@ func (a *Agent) registerJobs() error {
return true
}

parseSpec := a.cfg.ApigeeCfg.IsProxyMode() && a.cfg.ApigeeCfg.Specs.MatchOnURL // parse specs if proxy mode and match on url set
if !a.cfg.ApigeeCfg.Specs.DisablePollForSpecs {
specsJob := newPollSpecsJob(a.apigeeClient, a.agentCache, a.cfg.ApigeeCfg.GetWorkers().Spec, a.cfg.ApigeeCfg.IsProxyMode())
specsJob := newPollSpecsJob().
SetSpecClient(a.apigeeClient).
SetSpecCache(a.agentCache).
SetWorkers(a.cfg.ApigeeCfg.GetWorkers().Spec).
SetParseSpec(parseSpec)

_, err = jobs.RegisterIntervalJobWithName(specsJob, a.apigeeClient.GetConfig().GetIntervals().Spec, "Poll Specs")
if err != nil {
return err
Expand All @@ -89,7 +95,13 @@ func (a *Agent) registerJobs() error {
var validatorReady jobFirstRunDone

if a.cfg.ApigeeCfg.IsProxyMode() {
proxiesJob := newPollProxiesJob(a.apigeeClient, a.agentCache, startPollingJob, a.cfg.ApigeeCfg.GetWorkers().Proxy)
proxiesJob := newPollProxiesJob().
SetSpecClient(a.apigeeClient).
SetSpecCache(a.agentCache).
SetSpecsReady(startPollingJob).
SetWorkers(a.cfg.ApigeeCfg.GetWorkers().Proxy).
SetMatchOnURL(a.cfg.ApigeeCfg.Specs.MatchOnURL)

_, err = jobs.RegisterIntervalJobWithName(proxiesJob, a.apigeeClient.GetConfig().GetIntervals().Proxy, "Poll Proxies")
if err != nil {
return err
Expand Down
108 changes: 69 additions & 39 deletions discovery/pkg/apigee/pollproxiesjob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,43 +49,64 @@ type proxyClient interface {

type proxyCache interface {
GetSpecWithPath(path string) (*specCacheItem, error)
GetSpecWithName(name string) (*specCacheItem, error)
GetSpecPathWithEndpoint(endpoint string) (string, error)
AddPublishedServiceToCache(cacheKey string, serviceBody *apic.ServiceBody)
}

// job that will poll for any new portals on APIGEE Edge
type pollProxiesJob struct {
jobs.Job
client proxyClient
firstRun bool
cache proxyCache
logger log.FieldLogger
specsReady jobFirstRunDone
pubLock sync.Mutex
publishFunc agent.PublishAPIFunc
workers int
running bool
runningLock sync.Mutex
virtualHostURLs map[string]map[string][]string
lastTime int
runTime int
client proxyClient
firstRun bool
cache proxyCache
logger log.FieldLogger
specsReady jobFirstRunDone
pubLock sync.Mutex
publishFunc agent.PublishAPIFunc
workers int
running bool
matchOnURL bool
runningLock sync.Mutex
lastTime int
runTime int
}

func newPollProxiesJob(client proxyClient, cache proxyCache, specsReady jobFirstRunDone, workers int) *pollProxiesJob {
func newPollProxiesJob() *pollProxiesJob {
job := &pollProxiesJob{
client: client,
cache: cache,
firstRun: true,
specsReady: specsReady,
logger: log.NewFieldLogger().WithComponent("pollProxies").WithPackage("apigee"),
publishFunc: agent.PublishAPI,
workers: workers,
runningLock: sync.Mutex{},
virtualHostURLs: make(map[string]map[string][]string),
firstRun: true,
logger: log.NewFieldLogger().WithComponent("pollProxies").WithPackage("apigee"),
publishFunc: agent.PublishAPI,
runningLock: sync.Mutex{},
}
return job
}

func (j *pollProxiesJob) SetSpecClient(client proxyClient) *pollProxiesJob {
j.client = client
return j
}

func (j *pollProxiesJob) SetSpecCache(cache proxyCache) *pollProxiesJob {
j.cache = cache
return j
}

func (j *pollProxiesJob) SetSpecsReady(specsReady jobFirstRunDone) *pollProxiesJob {
j.specsReady = specsReady
return j
}

func (j *pollProxiesJob) SetWorkers(workers int) *pollProxiesJob {
j.workers = workers
return j
}

func (j *pollProxiesJob) SetMatchOnURL(matchOnURL bool) *pollProxiesJob {
j.matchOnURL = matchOnURL
return j
}

func (j *pollProxiesJob) FirstRunComplete() bool {
return !j.firstRun
}
Expand Down Expand Up @@ -204,9 +225,7 @@ func (j *pollProxiesJob) handleRevision(ctx context.Context, revName string) {
logger := getLoggerFromContext(ctx).WithField(revNameField.String(), revName)
addLoggerToContext(ctx, logger)
logger.Debug("handling revision")
pName := getStringFromContext(ctx, proxyNameField)

_ = pName
revision, err := j.client.GetRevision(getStringFromContext(ctx, proxyNameField), revName)
if err != nil {
logger.WithError(err).Error("getting revision")
Expand Down Expand Up @@ -278,11 +297,21 @@ func (j *pollProxiesJob) specFromRevision(ctx context.Context) string {
logger := getLoggerFromContext(ctx)
logger.Trace("checking revision resource files")

// get the spec using the association.json file, if it exists
revision := ctx.Value(revNameField).(*models.ApiProxyRevision)
for _, resource := range revision.ResourceFiles.ResourceFile {
if resource.Type == openapi && resource.Name == association {
return j.getSpecFromResourceFile(ctx, resource.Type, resource.Name)
if resource.Type != openapi || resource.Name != association {
continue
}
if path := j.getSpecFromResourceFile(ctx, resource.Type, resource.Name); path != "" {
return path
}
}

// get a spec match based off the proxy name to the spec name
specData, _ := j.cache.GetSpecWithName(revision.Name)
if specData != nil {
return specData.ContentPath
}

return j.getSpecFromVirtualHosts(ctx)
Expand All @@ -293,35 +322,41 @@ func (j *pollProxiesJob) getVirtualHostURLs(ctx context.Context) context.Context
revision := ctx.Value(revNameField).(*models.ApiProxyRevision)
envName := getStringFromContext(ctx, envNameField)
proxyName := getStringFromContext(ctx, proxyNameField)
allURLs := []string{}
allURLs := getStringArrayFromContext(ctx, endpointsField)

connection, err := j.client.GetRevisionConnectionType(proxyName, revision.Revision)
if err != nil {
logger.WithError(err).Error("could not get the revision connection type")
return context.WithValue(ctx, endpointsField, allURLs)
}

if _, ok := j.virtualHostURLs[envName]; !ok {
j.virtualHostURLs[envName] = make(map[string][]string)
virtualHostURLs := make(map[string]map[string][]string)

if _, ok := virtualHostURLs[envName]; !ok {
virtualHostURLs[envName] = make(map[string][]string)
}

if _, ok := j.virtualHostURLs[envName][connection.VirtualHost]; !ok {
if _, ok := virtualHostURLs[envName][connection.VirtualHost]; !ok {
virtualHost, err := j.client.GetVirtualHost(envName, connection.VirtualHost)
if err != nil {
logger.WithError(err).Error("could not get the virtual host info")
return context.WithValue(ctx, endpointsField, allURLs)
}
j.virtualHostURLs[envName][connection.VirtualHost] = urlsFromVirtualHost(virtualHost)
virtualHostURLs[envName][connection.VirtualHost] = urlsFromVirtualHost(virtualHost)
}

for _, url := range j.virtualHostURLs[envName][connection.VirtualHost] {
for _, url := range virtualHostURLs[envName][connection.VirtualHost] {
allURLs = append(allURLs, fmt.Sprintf("%s%s", url, connection.BasePath))
}

return context.WithValue(ctx, endpointsField, allURLs)
}

func (j *pollProxiesJob) getSpecFromVirtualHosts(ctx context.Context) string {
if !j.matchOnURL {
return ""
}

logger := getLoggerFromContext(ctx)
revision := ctx.Value(revNameField).(*models.ApiProxyRevision)

Expand Down Expand Up @@ -360,12 +395,7 @@ func (j *pollProxiesJob) getSpecFromResourceFile(ctx context.Context, resourceTy
logger.WithError(err).Debug("could not read resource file content")
}

// get the association.json file content
_, err = j.cache.GetSpecWithPath(associationFile.URL)
if err != nil {
logger.WithError(err).Error("spec path not found in cache")
return ""
}
// return the association.json file content
return associationFile.URL
}

Expand Down
Loading

0 comments on commit ec46516

Please sign in to comment.