add alienvault

pkg/engine/passive/passive.go

@@ -60,6 +60,7 @@ func (c *Crawler) Close() error {
 
 // Crawl crawls a URL with the specified options
 func (c *Crawler) Crawl(rootURL string) error {
+	rootUrlParsed, _ := urlutil.ParseURL(rootURL, true)
 	results := make(chan source.Result)
 	go func() {
 		defer close(results)
@@ -69,33 +70,27 @@ func (c *Crawler) Crawl(rootURL string) error {
 		for _, s := range c.sources {
 			wg.Add(1)
 			go func(source source.Source) {
-				for resp := range source.Run(ctx, c.Shared, rootURL) {
-					results <- resp
+				for result := range source.Run(ctx, c.Shared, rootURL) {
+					results <- result
 				}
 				wg.Done()
 			}(s)
 		}
 		wg.Wait()
 	}()
 
-	URLs := map[string]struct{}{rootURL: {}}
 	for result := range results {
-		URLs[result.Value] = struct{}{}
-	}
-
-	rootUrlParsed, _ := urlutil.ParseURL(rootURL, true)
-	for URL := range URLs {
-		if !utils.IsURL(URL) {
-			gologger.Debug().Msgf("`%v` not a url. skipping", URL)
+		if !utils.IsURL(result.Value) {
+			gologger.Debug().Msgf("`%v` not a url. skipping", result.Value)
 			continue
 		}
 
-		if ok, err := c.Options.ValidateScope(URL, rootUrlParsed.Hostname()); err != nil || !ok {
-			gologger.Debug().Msgf("`%v` not in scope. skipping", URL)
+		if ok, err := c.Options.ValidateScope(result.Value, rootUrlParsed.Hostname()); err != nil || !ok {
+			gologger.Debug().Msgf("`%v` not in scope. skipping", result.Value)
 			continue
 		}
 
-		req := &navigation.Request{Method: "GET", URL: URL}
+		req := &navigation.Request{Method: "GET", URL: result.Value}
 		resp := &navigation.Response{}
 		c.Output(req, resp, nil)
 	}

pkg/engine/passive/registry.go

@@ -2,11 +2,13 @@ package passive
 
 import (
 	"github.com/projectdiscovery/katana/pkg/engine/passive/source"
+	"github.com/projectdiscovery/katana/pkg/engine/passive/source/alienvault"
 	"github.com/projectdiscovery/katana/pkg/engine/passive/source/commoncrawl"
 	"github.com/projectdiscovery/katana/pkg/engine/passive/source/waybackarchive"
 )
 
 var Sources = map[string]source.Source{
 	"waybackarchive": &waybackarchive.Source{},
 	"commoncrawl":    &commoncrawl.Source{},
+	"alienvault":     &alienvault.Source{},
 }

pkg/engine/passive/source/alienvault/alienvault.go

@@ -0,0 +1,81 @@
+package alienvault
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/projectdiscovery/katana/pkg/engine/common"
+	"github.com/projectdiscovery/katana/pkg/engine/passive/httpclient"
+	"github.com/projectdiscovery/katana/pkg/engine/passive/source"
+	urlutil "github.com/projectdiscovery/utils/url"
+)
+
+type alienvaultResponse struct {
+	URLList []url `json:"url_list"`
+	HasNext bool  `json:"has_next"`
+}
+
+type url struct {
+	URL string `json:"url"`
+}
+
+type Source struct {
+}
+
+func (s *Source) Run(ctx context.Context, sharedCtx *common.Shared, rootUrl string) <-chan source.Result {
+	results := make(chan source.Result)
+
+	go func() {
+		defer close(results)
+
+		if parsedRootUrl, err := urlutil.Parse(rootUrl); err == nil {
+			rootUrl = parsedRootUrl.Hostname()
+		}
+
+		page := 1
+		for {
+			httpClient := httpclient.NewHttpClient(sharedCtx.Options.Options.Timeout)
+			apiURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?page=%d", rootUrl, page)
+			resp, err := httpClient.SimpleGet(ctx, apiURL)
+			if err != nil && resp == nil {
+				results <- source.Result{Source: s.Name(), Error: err}
+				httpClient.DiscardHTTPResponse(resp)
+				return
+			}
+
+			var response alienvaultResponse
+			// Get the response body and decode
+			err = json.NewDecoder(resp.Body).Decode(&response)
+			if err != nil {
+				results <- source.Result{Source: s.Name(), Error: err}
+				resp.Body.Close()
+				return
+			}
+			resp.Body.Close()
+
+			for _, record := range response.URLList {
+				results <- source.Result{Source: s.Name(), Value: record.URL}
+			}
+
+			if !response.HasNext {
+				break
+			}
+			page++
+		}
+	}()
+
+	return results
+}
+
+func (s *Source) Name() string {
+	return "alienvault"
+}
+
+func (s *Source) NeedsKey() bool {
+	return false
+}
+
+func (s *Source) AddApiKeys(_ []string) {
+	// no key needed
+}