From 0c97a039004b70ce220b219b503ffd2571e035d1 Mon Sep 17 00:00:00 2001 From: Miccah Castorina Date: Wed, 18 Sep 2024 11:36:49 -0700 Subject: [PATCH 1/5] Refresh EndpointCustomizer for more explicit configuration Also add CloudProvider interface. --- pkg/detectors/detectors.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index b4f643f97aff..5bf35b1d854d 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -73,7 +73,12 @@ type MultiPartCredentialProvider interface { // support verifying against user-supplied endpoints. type EndpointCustomizer interface { SetEndpoints(...string) error - DefaultEndpoint() string + UseCloudEndpoint(bool) + UseFoundEndpoints(bool) +} + +type CloudProvider interface { + CloudEndpoint() string } type Result struct { From a328390977f3089c023e2b2a7e50cc6053b4fb02 Mon Sep 17 00:00:00 2001 From: Miccah Castorina Date: Thu, 19 Sep 2024 10:18:15 -0700 Subject: [PATCH 2/5] WIP: Update EndpointSetter --- pkg/detectors/detectors.go | 3 ++- pkg/detectors/endpoint_customizer.go | 38 ++++++++++++++++++---------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index 5bf35b1d854d..d3cf2f98050a 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -72,7 +72,8 @@ type MultiPartCredentialProvider interface { // EndpointCustomizer is an optional interface that a detector can implement to // support verifying against user-supplied endpoints. type EndpointCustomizer interface { - SetEndpoints(...string) error + SetConfiguredEndpoints(...string) error + SetCloudEndpoint(string) UseCloudEndpoint(bool) UseFoundEndpoints(bool) } diff --git a/pkg/detectors/endpoint_customizer.go b/pkg/detectors/endpoint_customizer.go index 85510cd9eb70..e4c6ccfb1701 100644 --- a/pkg/detectors/endpoint_customizer.go +++ b/pkg/detectors/endpoint_customizer.go @@ -10,27 +10,39 @@ import ( // of the EndpointCustomizer interface. A detector can embed this struct to // gain the functionality. type EndpointSetter struct { - endpoints []string + configuredEndpoints []string + cloudEndpoint string + useCloudEndpoint bool + useFoundEndpoints bool } -func (e *EndpointSetter) SetEndpoints(endpoints ...string) error { - if len(endpoints) == 0 { +func (e *EndpointSetter) SetConfiguredEndpoints(userConfiguredEndpoints ...string) error { + if len(userConfiguredEndpoints) == 0 { return fmt.Errorf("at least one endpoint required") } - deduped := make([]string, 0, len(endpoints)) - for _, endpoint := range endpoints { + deduped := make([]string, 0, len(userConfiguredEndpoints)) + for _, endpoint := range userConfiguredEndpoints { common.AddStringSliceItem(endpoint, &deduped) } - e.endpoints = deduped + e.configuredEndpoints = deduped return nil } -func (e *EndpointSetter) Endpoints(defaultEndpoint string) []string { - // The only valid time len(e.endpoints) == 0 is when EndpointSetter is - // initializetd to its default state. That means SetEndpoints was never - // called and we should use the default. - if len(e.endpoints) == 0 { - return []string{defaultEndpoint} +func (e *EndpointSetter) UseCloudEndpoint(enabled bool) { + e.useCloudEndpoint = true +} + +func (e *EndpointSetter) UseFoundEndpoints(enabled bool) { + e.useFoundEndpoints = true +} + +func (e *EndpointSetter) Endpoints(foundEndpoints ...string) []string { + endpoints := e.configuredEndpoints + if e.useCloudEndpoint { + endpoints = append(endpoints, e.cloudEndpoint) + } + if e.useFoundEndpoints { + endpoints = append(endpoints, foundEndpoints...) } - return e.endpoints + return endpoints } From 040786dacbcc3e1b588b6184f4cd9677b8e9fd60 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Thu, 19 Sep 2024 22:54:19 +0500 Subject: [PATCH 3/5] Updated detectors with new endpoint customizer --- pkg/detectors/artifactory/artifactory.go | 31 +++++++++++++--------- pkg/detectors/datadogtoken/datadogtoken.go | 8 +++--- pkg/detectors/endpoint_customizer.go | 8 ++++-- pkg/detectors/github/v1/github_old.go | 7 ++--- pkg/detectors/github/v2/github.go | 3 ++- pkg/detectors/gitlab/v1/gitlab.go | 7 ++--- pkg/engine/engine.go | 12 +++++++-- 7 files changed, 49 insertions(+), 27 deletions(-) diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go index 98841f984fb8..c281cb4a4484 100644 --- a/pkg/detectors/artifactory/artifactory.go +++ b/pkg/detectors/artifactory/artifactory.go @@ -15,11 +15,13 @@ import ( type Scanner struct { client *http.Client detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter } var ( // Ensure the Scanner satisfies the interface at compile time. - _ detectors.Detector = (*Scanner)(nil) + _ detectors.Detector = (*Scanner)(nil) + _ detectors.EndpointCustomizer = (*Scanner)(nil) defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses @@ -52,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if len(URLmatch) != 2 { continue } + resURLMatch = strings.TrimSpace(URLmatch[1]) } @@ -61,20 +64,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } resMatch := strings.TrimSpace(match[1]) - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, - Raw: []byte(resMatch), - RawV2: []byte(resMatch + resURLMatch), - } + client := s.getClient() + + for _, URL := range s.Endpoints(resURLMatch) { + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, + Raw: []byte(resMatch), + RawV2: []byte(resMatch + URL), + } + + if verify { + isVerified, verificationErr := verifyArtifactory(ctx, client, URL, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } - if verify { - client := s.getClient() - isVerified, verificationErr := verifyArtifactory(ctx, client, resURLMatch, resMatch) - s1.Verified = isVerified - s1.SetVerificationError(verificationErr, resMatch) + results = append(results, s1) } - results = append(results, s1) } return results, nil diff --git a/pkg/detectors/datadogtoken/datadogtoken.go b/pkg/detectors/datadogtoken/datadogtoken.go index e4743701ca97..b0a1c07d350e 100644 --- a/pkg/detectors/datadogtoken/datadogtoken.go +++ b/pkg/detectors/datadogtoken/datadogtoken.go @@ -21,8 +21,9 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var _ detectors.EndpointCustomizer = (*Scanner)(nil) +var _ detectors.CloudProvider = (*Scanner)(nil) -func (Scanner) DefaultEndpoint() string { return "https://api.datadoghq.com" } +func (Scanner) CloudEndpoint() string { return "https://api.datadoghq.com" } var ( client = common.SaneHttpClient() @@ -126,7 +127,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) { + for _, baseURL := range s.Endpoints() { req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil) if err != nil { continue @@ -169,8 +170,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - - for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) { + for _, baseURL := range s.Endpoints() { req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v1/validate", nil) if err != nil { continue diff --git a/pkg/detectors/endpoint_customizer.go b/pkg/detectors/endpoint_customizer.go index e4c6ccfb1701..4c53ed1af818 100644 --- a/pkg/detectors/endpoint_customizer.go +++ b/pkg/detectors/endpoint_customizer.go @@ -28,12 +28,16 @@ func (e *EndpointSetter) SetConfiguredEndpoints(userConfiguredEndpoints ...strin return nil } +func (e *EndpointSetter) SetCloudEndpoint(url string) { + e.cloudEndpoint = url +} + func (e *EndpointSetter) UseCloudEndpoint(enabled bool) { - e.useCloudEndpoint = true + e.useCloudEndpoint = enabled } func (e *EndpointSetter) UseFoundEndpoints(enabled bool) { - e.useFoundEndpoints = true + e.useFoundEndpoints = enabled } func (e *EndpointSetter) Endpoints(foundEndpoints ...string) []string { diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.go index 0a78a8671d57..5f706e5c2866 100644 --- a/pkg/detectors/github/v1/github_old.go +++ b/pkg/detectors/github/v1/github_old.go @@ -19,9 +19,10 @@ type Scanner struct{ detectors.EndpointSetter } var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Versioner = (*Scanner)(nil) var _ detectors.EndpointCustomizer = (*Scanner)(nil) +var _ detectors.CloudProvider = (*Scanner)(nil) -func (Scanner) Version() int { return 1 } -func (Scanner) DefaultEndpoint() string { return "https://api.github.com" } +func (Scanner) Version() int { return 1 } +func (Scanner) CloudEndpoint() string { return "https://api.github.com" } var ( // Oauth token @@ -112,7 +113,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result func (s Scanner) VerifyGithub(ctx context.Context, client *http.Client, token string) (bool, *UserRes, *HeaderInfo, error) { // https://developer.github.com/v3/users/#get-the-authenticated-user var requestErr error - for _, url := range s.Endpoints(s.DefaultEndpoint()) { + for _, url := range s.Endpoints() { requestErr = nil req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil) diff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.go index 08ec567580f4..80dede25a4b6 100644 --- a/pkg/detectors/github/v2/github.go +++ b/pkg/detectors/github/v2/github.go @@ -21,11 +21,12 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Versioner = (*Scanner)(nil) var _ detectors.EndpointCustomizer = (*Scanner)(nil) +var _ detectors.CloudProvider = (*Scanner)(nil) func (s Scanner) Version() int { return 2 } -func (Scanner) DefaultEndpoint() string { return "https://api.github.com" } +func (Scanner) CloudEndpoint() string { return "https://api.github.com" } var ( // Oauth token diff --git a/pkg/detectors/gitlab/v1/gitlab.go b/pkg/detectors/gitlab/v1/gitlab.go index c718366222a0..062863822003 100644 --- a/pkg/detectors/gitlab/v1/gitlab.go +++ b/pkg/detectors/gitlab/v1/gitlab.go @@ -25,10 +25,11 @@ var ( _ detectors.Detector = (*Scanner)(nil) _ detectors.EndpointCustomizer = (*Scanner)(nil) _ detectors.Versioner = (*Scanner)(nil) + _ detectors.CloudProvider = (*Scanner)(nil) ) -func (Scanner) Version() int { return 1 } -func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" } +func (Scanner) Version() int { return 1 } +func (Scanner) CloudEndpoint() string { return "https://gitlab.com" } var ( defaultClient = common.SaneHttpClient() @@ -87,7 +88,7 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, error if client == nil { client = defaultClient } - for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) { + for _, baseURL := range s.Endpoints() { // test `read_user` scope req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil) if err != nil { diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index d2ca05a888af..dc7c61362a04 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -280,11 +280,19 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) { } if !cfg.CustomVerifiersOnly || len(urls) == 0 { - urls = append(urls, customizer.DefaultEndpoint()) + customizer.UseFoundEndpoints(true) + customizer.UseCloudEndpoint(true) } - if err := customizer.SetEndpoints(urls...); err != nil { + + if err := customizer.SetConfiguredEndpoints(urls...); err != nil { return false } + + cloudProvider, ok := d.(detectors.CloudProvider) + if ok { + customizer.SetCloudEndpoint(cloudProvider.CloudEndpoint()) + } + return true }) } From d2de90884de5eddf96220661a64f80bf86b7c87e Mon Sep 17 00:00:00 2001 From: kashif khan Date: Fri, 20 Sep 2024 10:33:02 +0500 Subject: [PATCH 4/5] Fixed linter --- pkg/detectors/endpoint_customizer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/detectors/endpoint_customizer_test.go b/pkg/detectors/endpoint_customizer_test.go index dc85422933bc..229cf915a1ae 100644 --- a/pkg/detectors/endpoint_customizer_test.go +++ b/pkg/detectors/endpoint_customizer_test.go @@ -10,7 +10,7 @@ func TestEmbeddedEndpointSetter(t *testing.T) { type Scanner struct{ EndpointSetter } var s Scanner assert.Equal(t, []string{"baz"}, s.Endpoints("baz")) - assert.NoError(t, s.SetEndpoints("foo", "bar")) - assert.Error(t, s.SetEndpoints()) + assert.NoError(t, s.SetConfiguredEndpoints("foo", "bar")) + assert.Error(t, s.SetConfiguredEndpoints()) assert.Equal(t, []string{"foo", "bar"}, s.Endpoints("baz")) } From 8515fa0cf09163fbeef2726672601fd1df4dfb58 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 24 Sep 2024 11:04:43 +0500 Subject: [PATCH 5/5] Added check for appending cloud endpoints --- pkg/detectors/endpoint_customizer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/endpoint_customizer.go b/pkg/detectors/endpoint_customizer.go index 4c53ed1af818..7cc917d8528f 100644 --- a/pkg/detectors/endpoint_customizer.go +++ b/pkg/detectors/endpoint_customizer.go @@ -42,7 +42,7 @@ func (e *EndpointSetter) UseFoundEndpoints(enabled bool) { func (e *EndpointSetter) Endpoints(foundEndpoints ...string) []string { endpoints := e.configuredEndpoints - if e.useCloudEndpoint { + if e.useCloudEndpoint && e.cloudEndpoint != "" { endpoints = append(endpoints, e.cloudEndpoint) } if e.useFoundEndpoints {