package main import ( "fmt" "sync" "time" "github.com/1440news/shared" ) // Dashboard is the main dashboard service type Dashboard struct { db *shared.DB statsMu sync.RWMutex cachedStats *DashboardStats cachedAllDomains []DomainStat startTime time.Time } // NewDashboard creates a new dashboard instance func NewDashboard(connString string) (*Dashboard, error) { db, err := shared.OpenDatabase(connString) if err != nil { return nil, err } return &Dashboard{ db: db, startTime: time.Now(), }, nil } // Close closes the database connection func (d *Dashboard) Close() error { return d.db.Close() } // DashboardStats holds all statistics for the dashboard type DashboardStats struct { // Domain stats TotalDomains int `json:"total_domains"` HoldDomains int `json:"hold_domains"` PassDomains int `json:"pass_domains"` SkipDomains int `json:"skip_domains"` DeadDomains int `json:"dead_domains"` // Feed stats TotalFeeds int `json:"total_feeds"` AliveFeeds int `json:"alive_feeds"` // status='pass' (healthy feeds) PublishFeeds int `json:"publish_feeds"` // publish_status='pass' (approved for publishing) SkipFeeds int `json:"skip_feeds"` HoldFeeds int `json:"hold_feeds"` DeadFeeds int `json:"dead_feeds"` EmptyFeeds int `json:"empty_feeds"` RSSFeeds int `json:"rss_feeds"` AtomFeeds int `json:"atom_feeds"` JSONFeeds int `json:"json_feeds"` UnknownFeeds int `json:"unknown_feeds"` // Processing rates (per minute) - populated by crawler API if available DomainsCrawled int32 `json:"domains_crawled"` DomainCheckRate int `json:"domain_check_rate"` FeedCrawlRate int `json:"feed_crawl_rate"` FeedCheckRate int `json:"feed_check_rate"` // Timing UpdatedAt time.Time `json:"updated_at"` } type TLDStat struct { TLD string `json:"tld"` Count int `json:"count"` } type RecentFeed struct { URL string `json:"url"` Title string `json:"title"` Type string `json:"type"` DiscoveredAt time.Time `json:"discovered_at"` } type DomainStat struct { Host string `json:"host"` FeedsFound int `json:"feeds_found"` } // commaFormat formats an integer with comma separators func commaFormat(n int) string { s := fmt.Sprintf("%d", n) if len(s) <= 3 { return s } var result []byte for i, c := range s { if i > 0 && (len(s)-i)%3 == 0 { result = append(result, ',') } result = append(result, byte(c)) } return string(result) } // UpdateStats recalculates and caches dashboard statistics func (d *Dashboard) UpdateStats() { fmt.Println("UpdateStats: calculating stats...") stats, err := d.calculateStats() if err != nil { fmt.Printf("UpdateStats: error calculating stats: %v\n", err) return } // Cache all domains with feeds (runs in background, so slow query is OK) fmt.Println("UpdateStats: fetching all domains...") allDomains := d.fetchAllDomainsFromDB() fmt.Printf("UpdateStats: got %d domains\n", len(allDomains)) d.statsMu.Lock() d.cachedStats = stats d.cachedAllDomains = allDomains d.statsMu.Unlock() fmt.Println("UpdateStats: complete") } func (d *Dashboard) fetchAllDomainsFromDB() []DomainStat { rows, err := d.db.Query(` SELECT domain_tld as tld, domain_host as domain_host, COUNT(*) as cnt FROM feeds GROUP BY domain_tld, domain_host ORDER BY domain_tld, domain_host `) if err != nil { fmt.Printf("fetchAllDomainsFromDB error: %v\n", err) return nil } defer rows.Close() var domains []DomainStat for rows.Next() { var ds DomainStat var tld string if err := rows.Scan(&tld, &ds.Host, &ds.FeedsFound); err != nil { continue } domains = append(domains, ds) } return domains } // GetDashboardStats returns cached statistics (returns empty stats if not yet cached) func (d *Dashboard) GetDashboardStats() (*DashboardStats, error) { d.statsMu.RLock() stats := d.cachedStats d.statsMu.RUnlock() if stats != nil { return stats, nil } // Return empty stats while background calculation runs (don't block HTTP requests) return &DashboardStats{UpdatedAt: time.Now()}, nil } // GetCachedAllDomains returns the cached list of all domains func (d *Dashboard) GetCachedAllDomains() []DomainStat { d.statsMu.RLock() defer d.statsMu.RUnlock() return d.cachedAllDomains } // calculateStats collects all statistics for the dashboard func (d *Dashboard) calculateStats() (*DashboardStats, error) { stats := &DashboardStats{ UpdatedAt: time.Now(), // Runtime rates not available in standalone dashboard // TODO: fetch from crawler API if needed } // Get domain stats if err := d.collectDomainStats(stats); err != nil { return nil, err } // Get feed stats if err := d.collectFeedStats(stats); err != nil { return nil, err } return stats, nil } func (d *Dashboard) collectDomainStats(stats *DashboardStats) error { // Use COUNT(*) for total count err := d.db.QueryRow("SELECT COUNT(*) FROM domains").Scan(&stats.TotalDomains) if err != nil { return err } // Single query to get all status counts (one index scan instead of three) rows, err := d.db.Query("SELECT status, COUNT(*) FROM domains GROUP BY status") if err != nil { return err } defer rows.Close() for rows.Next() { var status string var count int if err := rows.Scan(&status, &count); err != nil { continue } switch status { case "hold": stats.HoldDomains = count case "pass": stats.PassDomains = count case "skip": stats.SkipDomains = count case "dead": stats.DeadDomains = count } } if err := rows.Err(); err != nil { return err } return rows.Err() } func (d *Dashboard) collectFeedStats(stats *DashboardStats) error { // Use COUNT(*) for total count err := d.db.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&stats.TotalFeeds) if err != nil { return err } // Get status counts statusRows, err := d.db.Query("SELECT status, COUNT(*) FROM feeds GROUP BY status") if err != nil { return err } defer statusRows.Close() for statusRows.Next() { var status *string var count int if err := statusRows.Scan(&status, &count); err != nil { continue } if status != nil { switch *status { case "pass": stats.AliveFeeds = count case "skip": stats.SkipFeeds = count case "hold": stats.HoldFeeds = count case "dead": stats.DeadFeeds = count } } } // Count feeds approved for publishing (publish_status='pass') d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE publish_status = 'pass'").Scan(&stats.PublishFeeds) // Count empty feeds (item_count = 0 or NULL) d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE item_count IS NULL OR item_count = 0").Scan(&stats.EmptyFeeds) // Single query to get all type counts (one index scan instead of three) rows, err := d.db.Query("SELECT type, COUNT(*) FROM feeds GROUP BY type") if err != nil { return err } defer rows.Close() for rows.Next() { var feedType *string var count int if err := rows.Scan(&feedType, &count); err != nil { continue } if feedType == nil { stats.UnknownFeeds += count } else { switch *feedType { case "rss": stats.RSSFeeds = count case "atom": stats.AtomFeeds = count case "json": stats.JSONFeeds = count default: stats.UnknownFeeds += count } } } return rows.Err() } // StartStatsLoop starts a background loop that updates stats every minute func (d *Dashboard) StartStatsLoop() { ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() for range ticker.C { d.UpdateStats() } }