ec53ad59db
Removed dashboard-related files (now in standalone dashboard/ service): - api_domains.go, api_feeds.go, api_publish.go, api_search.go - dashboard.go, templates.go - oauth.go, oauth_handlers.go, oauth_middleware.go, oauth_session.go - routes.go - static/dashboard.css, static/dashboard.js Updated crawler.go: - Removed cachedStats, cachedAllDomains, statsMu fields - Removed StartStatsLoop function Updated main.go: - Removed dashboard startup - Removed stats loop and UpdateStats calls The crawler now runs independently without dashboard. Use the standalone dashboard/ service for web interface.
64 lines
1.7 KiB
Go
64 lines
1.7 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
)
|
|
|
|
func main() {
|
|
// Connection string from environment (DATABASE_URL or DB_* vars)
|
|
crawler, err := NewCrawler("")
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error initializing crawler: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
// Setup graceful shutdown
|
|
sigChan := make(chan os.Signal, 1)
|
|
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
|
|
|
// Start all loops independently
|
|
fmt.Println("Starting import and processing loops...")
|
|
|
|
// Import loop (background) - imports .com domains from vertices.txt.gz
|
|
go crawler.ImportDomainsInBackground("vertices.txt.gz")
|
|
|
|
// Add test domains (in addition to imported domains)
|
|
go crawler.ImportTestDomains([]string{
|
|
"news.ycombinator.com",
|
|
"ycombinator.com",
|
|
})
|
|
|
|
// feed_check loop (background) - checks feeds for new items
|
|
go crawler.StartFeedCheckLoop()
|
|
|
|
// Cleanup loop (background) - removes old items once per week
|
|
go crawler.StartCleanupLoop()
|
|
|
|
// Maintenance loop (background) - WAL checkpoints and integrity checks
|
|
go crawler.StartMaintenanceLoop()
|
|
|
|
// TLD sync loop (background) - syncs with IANA, marks dead TLDs, adds new ones
|
|
go crawler.startTLDSyncLoop()
|
|
|
|
// Publish loop (background) - autopublishes items for approved feeds
|
|
go crawler.StartPublishLoop()
|
|
|
|
// Domain loop (background) - domain_check + feed_crawl
|
|
go crawler.StartDomainLoop()
|
|
|
|
// Wait for shutdown signal
|
|
sig := <-sigChan
|
|
fmt.Printf("\nReceived %v, shutting down gracefully...\n", sig)
|
|
|
|
// Close crawler (checkpoints WAL and closes database)
|
|
if err := crawler.Close(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error closing crawler: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
fmt.Println("Shutdown complete")
|
|
}
|