diff --git a/main.go b/main.go index 7cb240b..372bcc6 100644 --- a/main.go +++ b/main.go @@ -30,7 +30,7 @@ func main() { go crawler.UpdateStats() // Start all loops independently - fmt.Println("Starting import, crawl, and stats loops...") + fmt.Println("Starting import and processing loops...") // Import loop (background) - imports .com domains from vertices.txt.gz go crawler.ImportDomainsInBackground("vertices.txt.gz") @@ -41,8 +41,8 @@ func main() { "ycombinator.com", }) - // Check loop (background) - go crawler.StartCheckLoop() + // feed_check loop (background) - checks feeds for new items + go crawler.StartFeedCheckLoop() // Stats loop (background) - updates once per minute go crawler.StartStatsLoop() @@ -53,11 +53,14 @@ func main() { // Maintenance loop (background) - WAL checkpoints and integrity checks go crawler.StartMaintenanceLoop() + // TLD sync loop (background) - syncs with IANA, marks dead TLDs, adds new ones + go crawler.startTLDSyncLoop() + // Publish loop (background) - autopublishes items for approved feeds go crawler.StartPublishLoop() - // Crawl loop (background) - crawls approved domains for feeds - go crawler.StartCrawlLoop() + // Domain loop (background) - domain_check + feed_crawl + go crawler.StartDomainLoop() // Wait for shutdown signal sig := <-sigChan