Skip fetching feeds when work channel is >50% full to prevent DB pool exhaustion

2026-02-06 00:58:55 -05:00
parent 0270d72920
commit 23fcff58b4
1 changed files with 6 additions and 0 deletions
@@ -182,6 +182,12 @@ func (c *Crawler) StartFeedCheckLoop() {
 			return
 		}

+		// Only fetch more work when the channel has space
+		if len(workChan) > fetchSize/2 {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
 		feeds, err := c.GetFeedsDueForCheck(fetchSize)
 		if err != nil {
 			fmt.Printf("Error fetching feeds: %v\n", err)