From 0d9508378e7e5feee66e6a23d205bc0ac5a8e03e Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:25:20 +0300 Subject: [PATCH] introduce pagination (#1105) * introduce pagination * add default rate limit * make request to free api endpoint in case of 403 --- v2/pkg/runner/options.go | 1 + .../sources/securitytrails/securitytrails.go | 81 ++++++++++++++----- 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 72bd4801c..63066f9da 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -253,4 +253,5 @@ var defaultRateLimits = []string{ // "threatminer=10/m", "waybackarchive=15/m", "whoisxmlapi=50/s", + "securitytrails=2/s", } diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go index c8c6ca88c..3ed732989 100644 --- a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -2,8 +2,10 @@ package securitytrails import ( + "bytes" "context" "fmt" + "net/http" "strings" "time" @@ -13,6 +15,12 @@ import ( ) type response struct { + Meta struct { + ScrollID string `json:"scroll_id"` + } `json:"meta"` + Records []struct { + Hostname string `json:"hostname"` + } `json:"records"` Subdomains []string `json:"subdomains"` } @@ -43,34 +51,63 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": randomApiKey}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } + var scrollId string + headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey} + + for { + var resp *http.Response + var err error + + if scrollId == "" { + var requestBody = []byte(fmt.Sprintf(`{"query":"apex_domain='%s'"}`, domain)) + resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", + headers, bytes.NewReader(requestBody)) + } else { + resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) + } + + if err != nil && resp.StatusCode == 403 { + resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) + } + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var securityTrailsResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + resp.Body.Close() + return + } - var securityTrailsResponse response - err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ resp.Body.Close() - return - } - resp.Body.Close() + for _, record := range securityTrailsResponse.Records { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } - for _, subdomain := range securityTrailsResponse.Subdomains { - if strings.HasSuffix(subdomain, ".") { - subdomain += domain - } else { - subdomain = subdomain + "." + domain + for _, subdomain := range securityTrailsResponse.Subdomains { + if strings.HasSuffix(subdomain, ".") { + subdomain += domain + } else { + subdomain = subdomain + "." + domain + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + scrollId = securityTrailsResponse.Meta.ScrollID + + if scrollId == "" { + break + } } }()