Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Endpoint customizer refresh #3308

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+82 −42
Open

Endpoint customizer refresh #3308

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0c97a03
Refresh EndpointCustomizer for more explicit configuration
mcastorina Sep 18, 2024
a328390
WIP: Update EndpointSetter
mcastorina Sep 19, 2024
040786d
Updated detectors with new endpoint customizer
kashifkhan0771 Sep 19, 2024
d2de908
Fixed linter
kashifkhan0771 Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions pkg/detectors/artifactory/artifactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@ import (
type Scanner struct {
client *http.Client
detectors.DefaultMultiPartCredentialProvider
detectors.EndpointSetter
}

var (
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
_ detectors.Detector = (*Scanner)(nil)
_ detectors.EndpointCustomizer = (*Scanner)(nil)

defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses

Expand Down Expand Up @@ -52,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if len(URLmatch) != 2 {
continue
}

resURLMatch = strings.TrimSpace(URLmatch[1])
}

Expand All @@ -61,20 +64,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
resMatch := strings.TrimSpace(match[1])

s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resURLMatch),
}
client := s.getClient()

for _, URL := range s.Endpoints(resURLMatch) {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + URL),
}

if verify {
isVerified, verificationErr := verifyArtifactory(ctx, client, URL, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}

if verify {
client := s.getClient()
isVerified, verificationErr := verifyArtifactory(ctx, client, resURLMatch, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
results = append(results, s1)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't a verified token break the loop? A token can't be valid on more than one instance, AFAIK.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea is to capture the result of all URLs we verify against even if the verification passes or not.

Copy link
Contributor

@abmussani abmussani Sep 20, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rgmz I believe these are access token. If yes, then from jfrog artifcatory documentation,

Cross-instance authentication

Access tokens can be used for authentication, not only by the instance or cluster where they were created but also for other instances and clusters that are all part of the same circle of trust.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting. Just be careful: Artifactory cloud has aggressive rate limiting.

More than a few failed auth in a minute and you won't be able to hit it anymore, in my experience.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rgmz If Artifactory has any rate limiting, I believe that should be on the same endpoint. Here the endpoint will be changing for each request unless user pass same endpoint multiple times.

}

results = append(results, s1)
}

return results, nil
Expand Down
8 changes: 4 additions & 4 deletions pkg/detectors/datadogtoken/datadogtoken.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ type Scanner struct {
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
var _ detectors.CloudProvider = (*Scanner)(nil)

func (Scanner) DefaultEndpoint() string { return "https://api.datadoghq.com" }
func (Scanner) CloudEndpoint() string { return "https://api.datadoghq.com" }

var (
client = common.SaneHttpClient()
Expand Down Expand Up @@ -126,7 +127,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

if verify {
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
for _, baseURL := range s.Endpoints() {
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil)
if err != nil {
continue
Expand Down Expand Up @@ -169,8 +170,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

if verify {

for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
for _, baseURL := range s.Endpoints() {
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v1/validate", nil)
if err != nil {
continue
Expand Down
10 changes: 8 additions & 2 deletions pkg/detectors/detectors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,14 @@ type MultiPartCredentialProvider interface {
// EndpointCustomizer is an optional interface that a detector can implement to
// support verifying against user-supplied endpoints.
type EndpointCustomizer interface {
SetEndpoints(...string) error
DefaultEndpoint() string
SetConfiguredEndpoints(...string) error
SetCloudEndpoint(string)
UseCloudEndpoint(bool)
UseFoundEndpoints(bool)
}

type CloudProvider interface {
CloudEndpoint() string
}

type Result struct {
Expand Down
42 changes: 29 additions & 13 deletions pkg/detectors/endpoint_customizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,43 @@ import (
// of the EndpointCustomizer interface. A detector can embed this struct to
// gain the functionality.
type EndpointSetter struct {
endpoints []string
configuredEndpoints []string
cloudEndpoint string
useCloudEndpoint bool
useFoundEndpoints bool
}

func (e *EndpointSetter) SetEndpoints(endpoints ...string) error {
if len(endpoints) == 0 {
func (e *EndpointSetter) SetConfiguredEndpoints(userConfiguredEndpoints ...string) error {
if len(userConfiguredEndpoints) == 0 {
return fmt.Errorf("at least one endpoint required")
}
deduped := make([]string, 0, len(endpoints))
for _, endpoint := range endpoints {
deduped := make([]string, 0, len(userConfiguredEndpoints))
for _, endpoint := range userConfiguredEndpoints {
common.AddStringSliceItem(endpoint, &deduped)
}
e.endpoints = deduped
e.configuredEndpoints = deduped
return nil
}

func (e *EndpointSetter) Endpoints(defaultEndpoint string) []string {
// The only valid time len(e.endpoints) == 0 is when EndpointSetter is
// initializetd to its default state. That means SetEndpoints was never
// called and we should use the default.
if len(e.endpoints) == 0 {
return []string{defaultEndpoint}
func (e *EndpointSetter) SetCloudEndpoint(url string) {
e.cloudEndpoint = url
}

func (e *EndpointSetter) UseCloudEndpoint(enabled bool) {
e.useCloudEndpoint = enabled
}

func (e *EndpointSetter) UseFoundEndpoints(enabled bool) {
e.useFoundEndpoints = enabled
}

func (e *EndpointSetter) Endpoints(foundEndpoints ...string) []string {
endpoints := e.configuredEndpoints
if e.useCloudEndpoint {
endpoints = append(endpoints, e.cloudEndpoint)
}
if e.useFoundEndpoints {
endpoints = append(endpoints, foundEndpoints...)
}
return e.endpoints
return endpoints
}
7 changes: 4 additions & 3 deletions pkg/detectors/github/v1/github_old.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ type Scanner struct{ detectors.EndpointSetter }
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
var _ detectors.CloudProvider = (*Scanner)(nil)

func (Scanner) Version() int { return 1 }
func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }
func (Scanner) Version() int { return 1 }
func (Scanner) CloudEndpoint() string { return "https://api.github.com" }

var (
// Oauth token
Expand Down Expand Up @@ -112,7 +113,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
func (s Scanner) VerifyGithub(ctx context.Context, client *http.Client, token string) (bool, *UserRes, *HeaderInfo, error) {
// https://developer.github.com/v3/users/#get-the-authenticated-user
var requestErr error
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
for _, url := range s.Endpoints() {
requestErr = nil

req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
Expand Down
3 changes: 2 additions & 1 deletion pkg/detectors/github/v2/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,12 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
var _ detectors.CloudProvider = (*Scanner)(nil)

func (s Scanner) Version() int {
return 2
}
func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }
func (Scanner) CloudEndpoint() string { return "https://api.github.com" }

var (
// Oauth token
Expand Down
7 changes: 4 additions & 3 deletions pkg/detectors/gitlab/v1/gitlab.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@ var (
_ detectors.Detector = (*Scanner)(nil)
_ detectors.EndpointCustomizer = (*Scanner)(nil)
_ detectors.Versioner = (*Scanner)(nil)
_ detectors.CloudProvider = (*Scanner)(nil)
)

func (Scanner) Version() int { return 1 }
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
func (Scanner) Version() int { return 1 }
func (Scanner) CloudEndpoint() string { return "https://gitlab.com" }

var (
defaultClient = common.SaneHttpClient()
Expand Down Expand Up @@ -87,7 +88,7 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, error
if client == nil {
client = defaultClient
}
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
for _, baseURL := range s.Endpoints() {
// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
if err != nil {
Expand Down
12 changes: 10 additions & 2 deletions pkg/engine/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,11 +280,19 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {
}

if !cfg.CustomVerifiersOnly || len(urls) == 0 {
urls = append(urls, customizer.DefaultEndpoint())
customizer.UseFoundEndpoints(true)
customizer.UseCloudEndpoint(true)
}
if err := customizer.SetEndpoints(urls...); err != nil {

if err := customizer.SetConfiguredEndpoints(urls...); err != nil {
return false
}

cloudProvider, ok := d.(detectors.CloudProvider)
if ok {
customizer.SetCloudEndpoint(cloudProvider.CloudEndpoint())
}

return true
})
}
Expand Down
Loading