diff --git a/crawler.go b/crawler.go index 70bbd8f..d4e22a8 100644 --- a/crawler.go +++ b/crawler.go @@ -182,6 +182,12 @@ func (c *Crawler) StartFeedCheckLoop() { return } + // Only fetch more work when the channel has space + if len(workChan) > fetchSize/2 { + time.Sleep(1 * time.Second) + continue + } + feeds, err := c.GetFeedsDueForCheck(fetchSize) if err != nil { fmt.Printf("Error fetching feeds: %v\n", err)