Increase feed check parallelism for 1Gbps bandwidth
- Workers: 1000 -> 4000 - Work channel buffer: 1000 -> 4000 - Fetch batch size: 1000 -> 4000 - MaxIdleConns: 100 -> 2000 Should improve throughput from ~15 feeds/sec to ~50-60 feeds/sec. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
+4
-4
@@ -36,7 +36,7 @@ func NewCrawler(connString string) (*Crawler, error) {
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
ForceAttemptHTTP2: false,
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConns: 2000,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 30 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
@@ -161,10 +161,10 @@ func (c *Crawler) StartMaintenanceLoop() {
|
||||
|
||||
// StartFeedCheckLoop runs the feed_check loop (checking feeds for new items)
|
||||
func (c *Crawler) StartFeedCheckLoop() {
|
||||
numWorkers := 1000
|
||||
numWorkers := 4000
|
||||
|
||||
// Buffered channel for feed work
|
||||
workChan := make(chan *Feed, 1000)
|
||||
workChan := make(chan *Feed, 4000)
|
||||
|
||||
// Start workers
|
||||
for i := 0; i < numWorkers; i++ {
|
||||
@@ -175,7 +175,7 @@ func (c *Crawler) StartFeedCheckLoop() {
|
||||
}()
|
||||
}
|
||||
|
||||
const fetchSize = 1000
|
||||
const fetchSize = 4000
|
||||
for {
|
||||
if c.IsShuttingDown() {
|
||||
close(workChan)
|
||||
|
||||
Reference in New Issue
Block a user