Skip to content

Commit

Permalink
update non-headless crawling to not follow redirects when the disable…
Browse files Browse the repository at this point in the history
…-redirects flag is enabled
  • Loading branch information
ErikOwen committed Oct 16, 2023
1 parent 3558f38 commit 4700ebb
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions pkg/engine/common/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,10 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error {
if resp.Resp == nil || resp.Reader == nil {
return
}
isRedirectResponse := resp.StatusCode >= 300 && resp.StatusCode < 400
if s.Options.Options.DisableRedirects && isRedirectResponse {
return
}

navigationRequests := parser.ParseResponse(resp)
s.Enqueue(crawlSession.Queue, navigationRequests...)
Expand Down

0 comments on commit 4700ebb

Please sign in to comment.