APIGOV-24021 - agent attribute and config (#77)

client/pkg/config/config.go

@@ -12,16 +12,17 @@ import (
 // ApigeeConfig - represents the config for gateway
 type ApigeeConfig struct {
 	corecfg.IConfigValidator
-	Organization string           `config:"organization"`
-	URL          string           `config:"url"`
-	DataURL      string           `config:"dataURL"`
-	APIVersion   string           `config:"apiVersion"`
-	Auth         *AuthConfig      `config:"auth"`
-	Intervals    *ApigeeIntervals `config:"intervals"`
-	Workers      *ApigeeWorkers   `config:"workers"`
-	Filter       string           `config:"filter"`
-	DeveloperID  string           `config:"developerID"`
-	mode         discoveryMode
+	Organization    string           `config:"organization"`
+	URL             string           `config:"url"`
+	DataURL         string           `config:"dataURL"`
+	APIVersion      string           `config:"apiVersion"`
+	Filter          string           `config:"filter"`
+	DeveloperID     string           `config:"developerID"`
+	Auth            *AuthConfig      `config:"auth"`
+	Intervals       *ApigeeIntervals `config:"intervals"`
+	Workers         *ApigeeWorkers   `config:"workers"`
+	CloneAttributes bool             `config:"cloneAttributes"`
+	mode            discoveryMode
 }
 
 // ApigeeIntervals - intervals for the apigee agent to use
@@ -74,6 +75,7 @@ const (
 	pathOrganization       = "apigee.organization"
 	pathMode               = "apigee.discoveryMode"
 	pathFilter             = "apigee.filter"
+	pathCloneAttributes    = "apigee.cloneAttributes"
 	pathAuthURL            = "apigee.auth.url"
 	pathAuthServerUsername = "apigee.auth.serverUsername"
 	pathAuthServerPassword = "apigee.auth.serverPassword"
@@ -99,6 +101,7 @@ func AddProperties(rootProps properties.Properties) {
 	rootProps.AddStringProperty(pathAuthURL, "https://login.apigee.com", "URL to use when authenticating to APIGEE")
 	rootProps.AddStringProperty(pathAuthServerUsername, "edgecli", "Username to use to when requesting APIGEE token")
 	rootProps.AddStringProperty(pathAuthServerPassword, "edgeclisecret", "Password to use to when requesting APIGEE token")
+	rootProps.AddBoolProperty(pathCloneAttributes, false, "Set to true to copy the tags when provisioning a Product in product mode.")
 	rootProps.AddStringProperty(pathAuthUsername, "", "Username to use to authenticate to APIGEE")
 	rootProps.AddStringProperty(pathAuthPassword, "", "Password for the user to authenticate to APIGEE")
 	rootProps.AddDurationProperty(pathSpecInterval, 30*time.Minute, "The time interval between checking for updated specs")
@@ -113,13 +116,14 @@ func AddProperties(rootProps properties.Properties) {
 // ParseConfig - parse the config on startup
 func ParseConfig(rootProps properties.Properties) *ApigeeConfig {
 	return &ApigeeConfig{
-		Organization: rootProps.StringPropertyValue(pathOrganization),
-		URL:          strings.TrimSuffix(rootProps.StringPropertyValue(pathURL), "/"),
-		APIVersion:   rootProps.StringPropertyValue(pathAPIVersion),
-		DataURL:      strings.TrimSuffix(rootProps.StringPropertyValue(pathDataURL), "/"),
-		DeveloperID:  rootProps.StringPropertyValue(pathDeveloper),
-		mode:         stringToDiscoveryMode(rootProps.StringPropertyValue(pathMode)),
-		Filter:       rootProps.StringPropertyValue(pathFilter),
+		Organization:    rootProps.StringPropertyValue(pathOrganization),
+		URL:             strings.TrimSuffix(rootProps.StringPropertyValue(pathURL), "/"),
+		APIVersion:      rootProps.StringPropertyValue(pathAPIVersion),
+		DataURL:         strings.TrimSuffix(rootProps.StringPropertyValue(pathDataURL), "/"),
+		DeveloperID:     rootProps.StringPropertyValue(pathDeveloper),
+		mode:            stringToDiscoveryMode(rootProps.StringPropertyValue(pathMode)),
+		Filter:          rootProps.StringPropertyValue(pathFilter),
+		CloneAttributes: rootProps.BoolPropertyValue(pathCloneAttributes),
 		Intervals: &ApigeeIntervals{
 			Proxy:   rootProps.DurationPropertyValue(pathProxyInterval),
 			Spec:    rootProps.DurationPropertyValue(pathSpecInterval),
@@ -203,3 +207,7 @@ func (a *ApigeeConfig) IsProxyMode() bool {
 func (a *ApigeeConfig) IsProductMode() bool {
 	return a.mode == discoveryModeProduct
 }
+
+func (a *ApigeeConfig) ShouldCloneAttributes() bool {
+	return a.CloneAttributes
+}

discovery/README.md

@@ -141,6 +141,7 @@ Here is a sample Quota policy that may be added to the desired Proxies.
 | APIGEE_DEVELOPERID         | The Apigee developer, email, that will own all apps                                  |                                   |
 | APIGEE_DISCOVERYMODE       | The mode in which the agent operates, discover proxies (proxy) or products (product) | proxy                             |
 | APIGEE_FILTER              | The tag filter to use against an Apigee product's attributes, only in product mode   |                                   |
+| APIGEE_CLONEATTRIBUTES     | Set this to true if the tags on a product should also be cloned on provisioning      | false                             |
 | APIGEE_INTERVAL_PROXY      | The polling interval checking for API Proxy changes, only in proxy mode              | 30s (30 seconds)                  |
 | APIGEE_INTERVAL_PRODUCT    | The polling interval checking for Product changes, only in product mode              | 30s (30 seconds)                  |
 | APIGEE_INTERVAL_SPEC       | The polling interval for checking for new Specs                                      | 30m (30 minute)                   |

discovery/pkg/apigee/agent.go

@@ -51,6 +51,7 @@ func NewAgent(agentCfg *AgentConfig) (*Agent, error) {
 		agentCfg.CentralCfg.GetCredentialConfig().GetExpirationDays(),
 		agent.GetCacheManager(),
 		agentCfg.ApigeeCfg.IsProductMode(),
+		agentCfg.ApigeeCfg.ShouldCloneAttributes(),
 	)
 	agent.RegisterProvisioner(provisioner)
 

discovery/pkg/apigee/definitions.go

@@ -18,5 +18,7 @@ const (
 )
 
 const (
-	cacheKeyAttribute = "cacheKey"
+	cacheKeyAttribute    = "cacheKey"
+	agentProductTagName  = "AgentCreated"
+	agentProductTagValue = "true"
 )

discovery/pkg/apigee/pollproductsjob.go

@@ -144,6 +144,7 @@ func (j *pollProductsJob) Execute() error {
 	wg.Wait()
 	close(limiter)
 
+	j.firstRun = false
 	return nil
 }
 
@@ -286,7 +287,14 @@ func (j *pollProductsJob) shouldPublishProduct(ctx context.Context) bool {
 		}
 		attributes[att.Name] = att.Value
 	}
-	j.logger.WithField("attributes", attributes).Trace("checking against discovery filter")
+	logger := j.logger.WithField("attributes", attributes)
+
+	if val, ok := attributes[agentProductTagName]; ok && val == agentProductTagValue {
+		logger.Trace("product was created by agent, skipping")
+		return false
+	}
+
+	logger.WithField("attributes", attributes).Trace("checking against discovery filter")
 	return j.shouldPushAPI(attributes)
 }
 

discovery/pkg/apigee/provision.go

@@ -21,11 +21,12 @@ const (
 )
 
 type provisioner struct {
-	client        client
-	credExpDays   int
-	cacheManager  cacheManager
-	isProductMode bool
-	logger        log.FieldLogger
+	client                client
+	credExpDays           int
+	cacheManager          cacheManager
+	isProductMode         bool
+	shouldCloneAttributes bool
+	logger                log.FieldLogger
 }
 
 type cacheManager interface {
@@ -50,13 +51,14 @@ type client interface {
 }
 
 // NewProvisioner creates a type to implement the SDK Provisioning methods for handling subscriptions
-func NewProvisioner(client client, credExpDays int, cacheMan cacheManager, isProductMode bool) prov.Provisioning {
+func NewProvisioner(client client, credExpDays int, cacheMan cacheManager, isProductMode, cloneAttributes bool) prov.Provisioning {
 	return &provisioner{
-		client:        client,
-		credExpDays:   credExpDays,
-		cacheManager:  cacheMan,
-		isProductMode: isProductMode,
-		logger:        log.NewFieldLogger().WithComponent("provision").WithPackage("apigee"),
+		client:                client,
+		credExpDays:           credExpDays,
+		cacheManager:          cacheMan,
+		isProductMode:         isProductMode,
+		shouldCloneAttributes: cloneAttributes,
+		logger:                log.NewFieldLogger().WithComponent("provision").WithPackage("apigee"),
 	}
 }
 
@@ -240,10 +242,19 @@ func (p provisioner) productModeCreateProduct(logger log.FieldLogger, targetProd
 
 	// only create a product if one is not found
 	if err != nil {
+		attributes := []models.Attribute{}
+		if p.shouldCloneAttributes {
+			attributes = curProduct.Attributes
+		}
+		attributes = append(attributes, models.Attribute{
+			Name:  agentProductTagName,
+			Value: agentProductTagValue,
+		})
+
 		product = &models.ApiProduct{
 			ApiResources:  curProduct.ApiResources,
 			ApprovalType:  curProduct.ApprovalType,
-			Attributes:    curProduct.Attributes,
+			Attributes:    attributes,
 			Description:   curProduct.Description,
 			DisplayName:   targetProductName,
 			Environments:  curProduct.Environments,

discovery/pkg/apigee/provision_test.go

@@ -87,7 +87,7 @@ func TestAccessRequestDeprovision(t *testing.T) {
 				appName:     tc.appName,
 				key:         app.Credentials[0].ConsumerKey,
 				productName: tc.apiID,
-			}, 30, &mockCache{t: t}, false)
+			}, 30, &mockCache{t: t}, false, false)
 
 			if tc.missingCred {
 				app.Credentials = nil
@@ -200,7 +200,7 @@ func TestAccessRequestProvision(t *testing.T) {
 				getAppErr:   tc.getAppErr,
 				productName: tc.apiID,
 				t:           t,
-			}, 30, &mockCache{t: t}, false)
+			}, 30, &mockCache{t: t}, false, false)
 
 			mar := mock.MockAccessRequest{
 				InstanceDetails: map[string]interface{}{
@@ -269,7 +269,7 @@ func TestApplicationRequestDeprovision(t *testing.T) {
 				productName: tc.apiID,
 				t:           t,
 				rmAppErr:    tc.rmAppErr,
-			}, 30, &mockCache{t: t}, false)
+			}, 30, &mockCache{t: t}, false, false)
 
 			mar := mock.MockApplicationRequest{
 				AppName:  tc.appName,
@@ -317,7 +317,7 @@ func TestApplicationRequestProvision(t *testing.T) {
 				productName:  tc.apiID,
 				t:            t,
 				createAppErr: tc.createAppErr,
-			}, 30, &mockCache{t: t}, false)
+			}, 30, &mockCache{t: t}, false, false)
 
 			mar := mock.MockApplicationRequest{
 				AppName:  tc.appName,
@@ -393,7 +393,7 @@ func TestCredentialDeprovision(t *testing.T) {
 				productName: tc.apiID,
 				t:           t,
 				getAppErr:   tc.getAppErr,
-			}, 30, &mockCache{t: t, appName: tc.appName}, false)
+			}, 30, &mockCache{t: t, appName: tc.appName}, false, false)
 
 			thisHash, _ := util.ComputeHash(key)
 			mcr := mock.MockCredentialRequest{
@@ -462,7 +462,7 @@ func TestCredentialProvision(t *testing.T) {
 				productName: tc.apiID,
 				t:           t,
 				getAppErr:   tc.getAppErr,
-			}, 30, &mockCache{t: t, appName: tc.appName}, false)
+			}, 30, &mockCache{t: t, appName: tc.appName}, false, false)
 
 			mcr := mock.MockCredentialRequest{
 				AppName:     tc.appName,
@@ -567,7 +567,7 @@ func TestCredentialUpdate(t *testing.T) {
 				t:           t,
 				getAppErr:   tc.getAppErr,
 				enable:      tc.action == provisioning.Enable,
-			}, 30, &mockCache{t: t, appName: tc.appName}, false)
+			}, 30, &mockCache{t: t, appName: tc.appName}, false, false)
 
 			thisHash, _ := util.ComputeHash(key)
 			details := map[string]string{

discovery/pkg/apigee/registervalidatorjob.go

@@ -6,20 +6,20 @@ import (
 
 type registerAPIValidatorJob struct {
 	jobs.Job
-	proxiesReady      jobFirstRunDone
+	validatorReady    jobFirstRunDone
 	registerValidator func()
 }
 
 func newRegisterAPIValidatorJob(proxiesReady jobFirstRunDone, registerValidator func()) *registerAPIValidatorJob {
 	job := &registerAPIValidatorJob{
-		proxiesReady:      proxiesReady,
+		validatorReady:    proxiesReady,
 		registerValidator: registerValidator,
 	}
 	return job
 }
 
 func (j *registerAPIValidatorJob) Ready() bool {
-	return j.proxiesReady()
+	return j.validatorReady()
 }
 
 func (j *registerAPIValidatorJob) Status() error {

traceability/main.go

@@ -11,7 +11,7 @@ import (
 )
 
 func main() {
-	// os.Setenv("AGENTFEATURES_VERSIONCHECKER", "false")
+	os.Setenv("AGENTFEATURES_VERSIONCHECKER", "false")
 	if err := cmd.RootCmd.Execute(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)