From 23fcff58b4abd007cbca16d75c7cc651375eb13a Mon Sep 17 00:00:00 2001 From: primal Date: Fri, 6 Feb 2026 00:58:55 -0500 Subject: [PATCH] Skip fetching feeds when work channel is >50% full to prevent DB pool exhaustion --- crawler.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crawler.go b/crawler.go index 70bbd8f..d4e22a8 100644 --- a/crawler.go +++ b/crawler.go @@ -182,6 +182,12 @@ func (c *Crawler) StartFeedCheckLoop() { return } + // Only fetch more work when the channel has space + if len(workChan) > fetchSize/2 { + time.Sleep(1 * time.Second) + continue + } + feeds, err := c.GetFeedsDueForCheck(fetchSize) if err != nil { fmt.Printf("Error fetching feeds: %v\n", err)