From 0569f251a41355ac5165c23357279a14c2da6bd9 Mon Sep 17 00:00:00 2001 From: Corentin Barreau Date: Thu, 24 Aug 2023 22:03:50 +0200 Subject: [PATCH] refactor: change condition for pausing crawls during redirections - Change the condition for pausing crawls if it is not a redirection Signed-off-by: Corentin Barreau --- internal/pkg/crawl/capture.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/crawl/capture.go b/internal/pkg/crawl/capture.go index 3b633279..f5c4dbd6 100644 --- a/internal/pkg/crawl/capture.go +++ b/internal/pkg/crawl/capture.go @@ -50,7 +50,7 @@ func (c *Crawl) executeGET(item *frontier.Item, req *http.Request, isRedirection // Temporarily pause crawls for individual hosts if they are over our configured maximum concurrent requests per domain. // If the request is a redirection, we do not pause the crawl because we want to follow the redirection. - if isRedirection { + if !isRedirection { for c.shouldPause(item.Host) { time.Sleep(time.Millisecond * time.Duration(c.RateLimitDelay)) }