From 4700ebb38ffc45ca56d23af66e50e62ca3d8da5a Mon Sep 17 00:00:00 2001
From: ErikOwen <erik@jelo.com>
Date: Mon, 16 Oct 2023 13:06:11 -0700
Subject: [PATCH] update non-headless crawling to not follow redirects when the
 disable-redirects flag is enabled

---
 pkg/engine/common/base.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/engine/common/base.go b/pkg/engine/common/base.go
index d9c9a0ed..6cf939ff 100644
--- a/pkg/engine/common/base.go
+++ b/pkg/engine/common/base.go
@@ -239,6 +239,10 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error {
 			if resp.Resp == nil || resp.Reader == nil {
 				return
 			}
+			isRedirectResponse := resp.StatusCode >= 300 && resp.StatusCode < 400
+			if s.Options.Options.DisableRedirects && isRedirectResponse {
+				return
+			}
 
 			navigationRequests := parser.ParseResponse(resp)
 			s.Enqueue(crawlSession.Queue, navigationRequests...)