Increase feed check parallelism for 1Gbps bandwidth

- Workers: 1000 -> 4000
- Work channel buffer: 1000 -> 4000
- Fetch batch size: 1000 -> 4000
- MaxIdleConns: 100 -> 2000

Should improve throughput from ~15 feeds/sec to ~50-60 feeds/sec.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
primal
2026-02-05 12:11:58 -05:00
parent 1f90b7d6a0
commit d4a1928fa6
+4 -4
View File
@@ -36,7 +36,7 @@ func NewCrawler(connString string) (*Crawler, error) {
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: false,
MaxIdleConns: 100,
MaxIdleConns: 2000,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 30 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
@@ -161,10 +161,10 @@ func (c *Crawler) StartMaintenanceLoop() {
// StartFeedCheckLoop runs the feed_check loop (checking feeds for new items)
func (c *Crawler) StartFeedCheckLoop() {
numWorkers := 1000
numWorkers := 4000
// Buffered channel for feed work
workChan := make(chan *Feed, 1000)
workChan := make(chan *Feed, 4000)
// Start workers
for i := 0; i < numWorkers; i++ {
@@ -175,7 +175,7 @@ func (c *Crawler) StartFeedCheckLoop() {
}()
}
const fetchSize = 1000
const fetchSize = 4000
for {
if c.IsShuttingDown() {
close(workChan)