f4afb29980
- Replace modernc.org/sqlite with jackc/pgx/v5 - Update all SQL queries for PostgreSQL syntax ($1, $2 placeholders) - Use snake_case column names throughout - Replace SQLite FTS5 with PostgreSQL tsvector/tsquery full-text search - Add connection pooling with pgxpool - Support Docker secrets for database password - Add trigger to normalize feed URLs (strip https://, http://, www.) - Fix anchor feed detection regex to avoid false positives - Connect app container to atproto network for PostgreSQL access - Add version indicator to dashboard UI Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
74 lines
1.8 KiB
Go
74 lines
1.8 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
)
|
|
|
|
func main() {
|
|
// Connection string from environment (DATABASE_URL or DB_* vars)
|
|
crawler, err := NewCrawler("")
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error initializing crawler: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
// Setup graceful shutdown
|
|
sigChan := make(chan os.Signal, 1)
|
|
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
|
|
|
// Start dashboard in background
|
|
go func() {
|
|
if err := crawler.StartDashboard("0.0.0.0:4321"); err != nil {
|
|
fmt.Fprintf(os.Stderr, "Dashboard error: %v\n", err)
|
|
}
|
|
}()
|
|
|
|
// Initialize stats in background (can be slow with large DBs)
|
|
go crawler.UpdateStats()
|
|
|
|
// Start all loops independently
|
|
fmt.Println("Starting import, crawl, check, and stats loops...")
|
|
|
|
// Import loop (background) - DISABLED for testing, using manual domains
|
|
// go crawler.ImportDomainsInBackground("vertices.txt.gz")
|
|
|
|
// Add only ycombinator domains for testing
|
|
go crawler.ImportTestDomains([]string{
|
|
"news.ycombinator.com",
|
|
"ycombinator.com",
|
|
})
|
|
|
|
// Check loop (background)
|
|
go crawler.StartCheckLoop()
|
|
|
|
// Stats loop (background) - updates once per minute
|
|
go crawler.StartStatsLoop()
|
|
|
|
// Cleanup loop (background) - removes old items once per week
|
|
go crawler.StartCleanupLoop()
|
|
|
|
// Maintenance loop (background) - WAL checkpoints and integrity checks
|
|
go crawler.StartMaintenanceLoop()
|
|
|
|
// Publish loop (background) - autopublishes items for approved feeds
|
|
go crawler.StartPublishLoop()
|
|
|
|
// Crawl loop (background)
|
|
go crawler.StartCrawlLoop()
|
|
|
|
// Wait for shutdown signal
|
|
sig := <-sigChan
|
|
fmt.Printf("\nReceived %v, shutting down gracefully...\n", sig)
|
|
|
|
// Close crawler (checkpoints WAL and closes database)
|
|
if err := crawler.Close(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error closing crawler: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
fmt.Println("Shutdown complete")
|
|
}
|