Add dashboard.go - stats types and calculation

Migrated from app/dashboard.go:
- Dashboard struct with DB connection and stats caching
- DashboardStats, TLDStat, RecentFeed, DomainStat types
- Stats calculation methods (collectDomainStats, collectFeedStats)
- Background stats update loop

Note: Runtime rates (domains/min, etc.) not available in standalone
dashboard - these are crawler-specific metrics.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

dashboard.go

@@ -0,0 +1,305 @@
+package main
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/1440news/shared"
+)
+
+// Dashboard is the main dashboard service
+type Dashboard struct {
+	db               *shared.DB
+	statsMu          sync.RWMutex
+	cachedStats      *DashboardStats
+	cachedAllDomains []DomainStat
+	startTime        time.Time
+}
+
+// NewDashboard creates a new dashboard instance
+func NewDashboard(connString string) (*Dashboard, error) {
+	db, err := shared.OpenDatabase(connString)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Dashboard{
+		db:        db,
+		startTime: time.Now(),
+	}, nil
+}
+
+// Close closes the database connection
+func (d *Dashboard) Close() error {
+	return d.db.Close()
+}
+
+// DashboardStats holds all statistics for the dashboard
+type DashboardStats struct {
+	// Domain stats
+	TotalDomains int `json:"total_domains"`
+	HoldDomains  int `json:"hold_domains"`
+	PassDomains  int `json:"pass_domains"`
+	SkipDomains  int `json:"skip_domains"`
+	DeadDomains  int `json:"dead_domains"`
+
+	// Feed stats
+	TotalFeeds   int `json:"total_feeds"`
+	AliveFeeds   int `json:"alive_feeds"`   // status='pass' (healthy feeds)
+	PublishFeeds int `json:"publish_feeds"` // publish_status='pass' (approved for publishing)
+	SkipFeeds    int `json:"skip_feeds"`
+	HoldFeeds    int `json:"hold_feeds"`
+	DeadFeeds    int `json:"dead_feeds"`
+	EmptyFeeds   int `json:"empty_feeds"`
+	RSSFeeds     int `json:"rss_feeds"`
+	AtomFeeds    int `json:"atom_feeds"`
+	JSONFeeds    int `json:"json_feeds"`
+	UnknownFeeds int `json:"unknown_feeds"`
+
+	// Processing rates (per minute) - populated by crawler API if available
+	DomainsCrawled  int32 `json:"domains_crawled"`
+	DomainCheckRate int   `json:"domain_check_rate"`
+	FeedCrawlRate   int   `json:"feed_crawl_rate"`
+	FeedCheckRate   int   `json:"feed_check_rate"`
+
+	// Timing
+	UpdatedAt time.Time `json:"updated_at"`
+}
+
+type TLDStat struct {
+	TLD   string `json:"tld"`
+	Count int    `json:"count"`
+}
+
+type RecentFeed struct {
+	URL          string    `json:"url"`
+	Title        string    `json:"title"`
+	Type         string    `json:"type"`
+	DiscoveredAt time.Time `json:"discovered_at"`
+}
+
+type DomainStat struct {
+	Host       string `json:"host"`
+	FeedsFound int    `json:"feeds_found"`
+}
+
+// commaFormat formats an integer with comma separators
+func commaFormat(n int) string {
+	s := fmt.Sprintf("%d", n)
+	if len(s) <= 3 {
+		return s
+	}
+	var result []byte
+	for i, c := range s {
+		if i > 0 && (len(s)-i)%3 == 0 {
+			result = append(result, ',')
+		}
+		result = append(result, byte(c))
+	}
+	return string(result)
+}
+
+// UpdateStats recalculates and caches dashboard statistics
+func (d *Dashboard) UpdateStats() {
+	fmt.Println("UpdateStats: calculating stats...")
+	stats, err := d.calculateStats()
+	if err != nil {
+		fmt.Printf("UpdateStats: error calculating stats: %v\n", err)
+		return
+	}
+	// Cache all domains with feeds (runs in background, so slow query is OK)
+	fmt.Println("UpdateStats: fetching all domains...")
+	allDomains := d.fetchAllDomainsFromDB()
+	fmt.Printf("UpdateStats: got %d domains\n", len(allDomains))
+
+	d.statsMu.Lock()
+	d.cachedStats = stats
+	d.cachedAllDomains = allDomains
+	d.statsMu.Unlock()
+	fmt.Println("UpdateStats: complete")
+}
+
+func (d *Dashboard) fetchAllDomainsFromDB() []DomainStat {
+	rows, err := d.db.Query(`
+		SELECT domain_tld as tld, domain_host as domain_host, COUNT(*) as cnt FROM feeds
+		GROUP BY domain_tld, domain_host
+		ORDER BY domain_tld, domain_host
+	`)
+	if err != nil {
+		fmt.Printf("fetchAllDomainsFromDB error: %v\n", err)
+		return nil
+	}
+	defer rows.Close()
+
+	var domains []DomainStat
+	for rows.Next() {
+		var ds DomainStat
+		var tld string
+		if err := rows.Scan(&tld, &ds.Host, &ds.FeedsFound); err != nil {
+			continue
+		}
+		domains = append(domains, ds)
+	}
+	return domains
+}
+
+// GetDashboardStats returns cached statistics (returns empty stats if not yet cached)
+func (d *Dashboard) GetDashboardStats() (*DashboardStats, error) {
+	d.statsMu.RLock()
+	stats := d.cachedStats
+	d.statsMu.RUnlock()
+
+	if stats != nil {
+		return stats, nil
+	}
+	// Return empty stats while background calculation runs (don't block HTTP requests)
+	return &DashboardStats{UpdatedAt: time.Now()}, nil
+}
+
+// GetCachedAllDomains returns the cached list of all domains
+func (d *Dashboard) GetCachedAllDomains() []DomainStat {
+	d.statsMu.RLock()
+	defer d.statsMu.RUnlock()
+	return d.cachedAllDomains
+}
+
+// calculateStats collects all statistics for the dashboard
+func (d *Dashboard) calculateStats() (*DashboardStats, error) {
+	stats := &DashboardStats{
+		UpdatedAt: time.Now(),
+		// Runtime rates not available in standalone dashboard
+		// TODO: fetch from crawler API if needed
+	}
+
+	// Get domain stats
+	if err := d.collectDomainStats(stats); err != nil {
+		return nil, err
+	}
+
+	// Get feed stats
+	if err := d.collectFeedStats(stats); err != nil {
+		return nil, err
+	}
+
+	return stats, nil
+}
+
+func (d *Dashboard) collectDomainStats(stats *DashboardStats) error {
+	// Use COUNT(*) for total count
+	err := d.db.QueryRow("SELECT COUNT(*) FROM domains").Scan(&stats.TotalDomains)
+	if err != nil {
+		return err
+	}
+
+	// Single query to get all status counts (one index scan instead of three)
+	rows, err := d.db.Query("SELECT status, COUNT(*) FROM domains GROUP BY status")
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var status string
+		var count int
+		if err := rows.Scan(&status, &count); err != nil {
+			continue
+		}
+		switch status {
+		case "hold":
+			stats.HoldDomains = count
+		case "pass":
+			stats.PassDomains = count
+		case "skip":
+			stats.SkipDomains = count
+		case "dead":
+			stats.DeadDomains = count
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+
+	return rows.Err()
+}
+
+func (d *Dashboard) collectFeedStats(stats *DashboardStats) error {
+	// Use COUNT(*) for total count
+	err := d.db.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&stats.TotalFeeds)
+	if err != nil {
+		return err
+	}
+
+	// Get status counts
+	statusRows, err := d.db.Query("SELECT status, COUNT(*) FROM feeds GROUP BY status")
+	if err != nil {
+		return err
+	}
+	defer statusRows.Close()
+
+	for statusRows.Next() {
+		var status *string
+		var count int
+		if err := statusRows.Scan(&status, &count); err != nil {
+			continue
+		}
+		if status != nil {
+			switch *status {
+			case "pass":
+				stats.AliveFeeds = count
+			case "skip":
+				stats.SkipFeeds = count
+			case "hold":
+				stats.HoldFeeds = count
+			case "dead":
+				stats.DeadFeeds = count
+			}
+		}
+	}
+
+	// Count feeds approved for publishing (publish_status='pass')
+	d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE publish_status = 'pass'").Scan(&stats.PublishFeeds)
+
+	// Count empty feeds (item_count = 0 or NULL)
+	d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE item_count IS NULL OR item_count = 0").Scan(&stats.EmptyFeeds)
+
+	// Single query to get all type counts (one index scan instead of three)
+	rows, err := d.db.Query("SELECT type, COUNT(*) FROM feeds GROUP BY type")
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var feedType *string
+		var count int
+		if err := rows.Scan(&feedType, &count); err != nil {
+			continue
+		}
+		if feedType == nil {
+			stats.UnknownFeeds += count
+		} else {
+			switch *feedType {
+			case "rss":
+				stats.RSSFeeds = count
+			case "atom":
+				stats.AtomFeeds = count
+			case "json":
+				stats.JSONFeeds = count
+			default:
+				stats.UnknownFeeds += count
+			}
+		}
+	}
+	return rows.Err()
+}
+
+// StartStatsLoop starts a background loop that updates stats every minute
+func (d *Dashboard) StartStatsLoop() {
+	ticker := time.NewTicker(1 * time.Minute)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		d.UpdateStats()
+	}
+}

go.mod

@@ -2,11 +2,16 @@ module github.com/1440news/dashboard
 
 go 1.24.0
 
+require github.com/1440news/shared v0.0.0
+
 require (
-	github.com/1440news/shared v0.0.0
+	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/haileyok/atproto-oauth-golang v0.0.0-20250101000000-000000000000
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/pgx/v5 v5.7.5
+	github.com/jackc/pgx/v5 v5.7.5 // indirect
-	github.com/lestrrat-go/jwx/v2 v2.1.6
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	golang.org/x/crypto v0.47.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/text v0.33.0 // indirect
 )
 
 replace github.com/1440news/shared => ../shared

go.sum

@@ -0,0 +1,28 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
+github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=