Skip to content

Commit

Permalink
fix vt pagination (#1110)
Browse files Browse the repository at this point in the history
  • Loading branch information
dogancanbakir authored Jan 8, 2024
1 parent 0d95083 commit 6921d55
Showing 1 changed file with 4 additions and 7 deletions.
11 changes: 4 additions & 7 deletions v2/pkg/subscraping/sources/virustotal/virustotal.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
if randomApiKey == "" {
return
}
var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000", domain)
var hasMore bool = true
var cursor string = ""
for hasMore {
for {
var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000", domain)
if cursor != "" {
url = fmt.Sprintf("%s&cursor=%s", url, cursor)
}
Expand All @@ -63,25 +62,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
session.DiscardHTTPResponse(resp)
return
}
defer resp.Body.Close()

var data response
err = jsoniter.NewDecoder(resp.Body).Decode(&data)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
resp.Body.Close()
return
}

resp.Body.Close()

for _, subdomain := range data.Data {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}
s.results++
}
cursor = data.Meta.Cursor
if cursor == "" {
hasMore = false
break
}
}
}()
Expand Down

0 comments on commit 6921d55

Please sign in to comment.