introduce pagination (#1105)

* introduce pagination

* add default rate limit

* make request to free api endpoint in case of 403

v2/pkg/runner/options.go

@@ -253,4 +253,5 @@ var defaultRateLimits = []string{
 	// "threatminer=10/m",
 	"waybackarchive=15/m",
 	"whoisxmlapi=50/s",
+	"securitytrails=2/s",
 }

v2/pkg/subscraping/sources/securitytrails/securitytrails.go

@@ -2,8 +2,10 @@
 package securitytrails
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"net/http"
 	"strings"
 	"time"
 
@@ -13,6 +15,12 @@ import (
 )
 
 type response struct {
+	Meta struct {
+		ScrollID string `json:"scroll_id"`
+	} `json:"meta"`
+	Records []struct {
+		Hostname string `json:"hostname"`
+	} `json:"records"`
 	Subdomains []string `json:"subdomains"`
 }
 
@@ -43,34 +51,63 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			return
 		}
 
-		resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": randomApiKey})
-		if err != nil {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-			s.errors++
-			session.DiscardHTTPResponse(resp)
-			return
-		}
+		var scrollId string
+		headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey}
+
+		for {
+			var resp *http.Response
+			var err error
+
+			if scrollId == "" {
+				var requestBody = []byte(fmt.Sprintf(`{"query":"apex_domain='%s'"}`, domain))
+				resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "",
+					headers, bytes.NewReader(requestBody))
+			} else {
+				resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers)
+			}
+
+			if err != nil && resp.StatusCode == 403 {
+				resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers)
+			}
+
+			if err != nil {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				s.errors++
+				session.DiscardHTTPResponse(resp)
+				return
+			}
+
+			var securityTrailsResponse response
+			err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse)
+			if err != nil {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				s.errors++
+				resp.Body.Close()
+				return
+			}
 
-		var securityTrailsResponse response
-		err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse)
-		if err != nil {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-			s.errors++
 			resp.Body.Close()
-			return
-		}
 
-		resp.Body.Close()
+			for _, record := range securityTrailsResponse.Records {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
+				s.results++
+			}
 
-		for _, subdomain := range securityTrailsResponse.Subdomains {
-			if strings.HasSuffix(subdomain, ".") {
-				subdomain += domain
-			} else {
-				subdomain = subdomain + "." + domain
+			for _, subdomain := range securityTrailsResponse.Subdomains {
+				if strings.HasSuffix(subdomain, ".") {
+					subdomain += domain
+				} else {
+					subdomain = subdomain + "." + domain
+				}
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
+				s.results++
 			}
 
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
-			s.results++
+			scrollId = securityTrailsResponse.Meta.ScrollID
+
+			if scrollId == "" {
+				break
+			}
 		}
 	}()