From a998168c598f27e255568f300d7f71821c3074fc Mon Sep 17 00:00:00 2001 From: primal Date: Mon, 2 Feb 2026 12:42:17 -0500 Subject: [PATCH] Add dashboard.go - stats types and calculation Migrated from app/dashboard.go: - Dashboard struct with DB connection and stats caching - DashboardStats, TLDStat, RecentFeed, DomainStat types - Stats calculation methods (collectDomainStats, collectFeedStats) - Background stats update loop Note: Runtime rates (domains/min, etc.) not available in standalone dashboard - these are crawler-specific metrics. Co-Authored-By: Claude Opus 4.5 --- dashboard.go | 305 +++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 13 ++- go.sum | 28 +++++ 3 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 dashboard.go create mode 100644 go.sum diff --git a/dashboard.go b/dashboard.go new file mode 100644 index 0000000..669243e --- /dev/null +++ b/dashboard.go @@ -0,0 +1,305 @@ +package main + +import ( + "fmt" + "sync" + "time" + + "github.com/1440news/shared" +) + +// Dashboard is the main dashboard service +type Dashboard struct { + db *shared.DB + statsMu sync.RWMutex + cachedStats *DashboardStats + cachedAllDomains []DomainStat + startTime time.Time +} + +// NewDashboard creates a new dashboard instance +func NewDashboard(connString string) (*Dashboard, error) { + db, err := shared.OpenDatabase(connString) + if err != nil { + return nil, err + } + + return &Dashboard{ + db: db, + startTime: time.Now(), + }, nil +} + +// Close closes the database connection +func (d *Dashboard) Close() error { + return d.db.Close() +} + +// DashboardStats holds all statistics for the dashboard +type DashboardStats struct { + // Domain stats + TotalDomains int `json:"total_domains"` + HoldDomains int `json:"hold_domains"` + PassDomains int `json:"pass_domains"` + SkipDomains int `json:"skip_domains"` + DeadDomains int `json:"dead_domains"` + + // Feed stats + TotalFeeds int `json:"total_feeds"` + AliveFeeds int `json:"alive_feeds"` // status='pass' (healthy feeds) + PublishFeeds int `json:"publish_feeds"` // publish_status='pass' (approved for publishing) + SkipFeeds int `json:"skip_feeds"` + HoldFeeds int `json:"hold_feeds"` + DeadFeeds int `json:"dead_feeds"` + EmptyFeeds int `json:"empty_feeds"` + RSSFeeds int `json:"rss_feeds"` + AtomFeeds int `json:"atom_feeds"` + JSONFeeds int `json:"json_feeds"` + UnknownFeeds int `json:"unknown_feeds"` + + // Processing rates (per minute) - populated by crawler API if available + DomainsCrawled int32 `json:"domains_crawled"` + DomainCheckRate int `json:"domain_check_rate"` + FeedCrawlRate int `json:"feed_crawl_rate"` + FeedCheckRate int `json:"feed_check_rate"` + + // Timing + UpdatedAt time.Time `json:"updated_at"` +} + +type TLDStat struct { + TLD string `json:"tld"` + Count int `json:"count"` +} + +type RecentFeed struct { + URL string `json:"url"` + Title string `json:"title"` + Type string `json:"type"` + DiscoveredAt time.Time `json:"discovered_at"` +} + +type DomainStat struct { + Host string `json:"host"` + FeedsFound int `json:"feeds_found"` +} + +// commaFormat formats an integer with comma separators +func commaFormat(n int) string { + s := fmt.Sprintf("%d", n) + if len(s) <= 3 { + return s + } + var result []byte + for i, c := range s { + if i > 0 && (len(s)-i)%3 == 0 { + result = append(result, ',') + } + result = append(result, byte(c)) + } + return string(result) +} + +// UpdateStats recalculates and caches dashboard statistics +func (d *Dashboard) UpdateStats() { + fmt.Println("UpdateStats: calculating stats...") + stats, err := d.calculateStats() + if err != nil { + fmt.Printf("UpdateStats: error calculating stats: %v\n", err) + return + } + // Cache all domains with feeds (runs in background, so slow query is OK) + fmt.Println("UpdateStats: fetching all domains...") + allDomains := d.fetchAllDomainsFromDB() + fmt.Printf("UpdateStats: got %d domains\n", len(allDomains)) + + d.statsMu.Lock() + d.cachedStats = stats + d.cachedAllDomains = allDomains + d.statsMu.Unlock() + fmt.Println("UpdateStats: complete") +} + +func (d *Dashboard) fetchAllDomainsFromDB() []DomainStat { + rows, err := d.db.Query(` + SELECT domain_tld as tld, domain_host as domain_host, COUNT(*) as cnt FROM feeds + GROUP BY domain_tld, domain_host + ORDER BY domain_tld, domain_host + `) + if err != nil { + fmt.Printf("fetchAllDomainsFromDB error: %v\n", err) + return nil + } + defer rows.Close() + + var domains []DomainStat + for rows.Next() { + var ds DomainStat + var tld string + if err := rows.Scan(&tld, &ds.Host, &ds.FeedsFound); err != nil { + continue + } + domains = append(domains, ds) + } + return domains +} + +// GetDashboardStats returns cached statistics (returns empty stats if not yet cached) +func (d *Dashboard) GetDashboardStats() (*DashboardStats, error) { + d.statsMu.RLock() + stats := d.cachedStats + d.statsMu.RUnlock() + + if stats != nil { + return stats, nil + } + // Return empty stats while background calculation runs (don't block HTTP requests) + return &DashboardStats{UpdatedAt: time.Now()}, nil +} + +// GetCachedAllDomains returns the cached list of all domains +func (d *Dashboard) GetCachedAllDomains() []DomainStat { + d.statsMu.RLock() + defer d.statsMu.RUnlock() + return d.cachedAllDomains +} + +// calculateStats collects all statistics for the dashboard +func (d *Dashboard) calculateStats() (*DashboardStats, error) { + stats := &DashboardStats{ + UpdatedAt: time.Now(), + // Runtime rates not available in standalone dashboard + // TODO: fetch from crawler API if needed + } + + // Get domain stats + if err := d.collectDomainStats(stats); err != nil { + return nil, err + } + + // Get feed stats + if err := d.collectFeedStats(stats); err != nil { + return nil, err + } + + return stats, nil +} + +func (d *Dashboard) collectDomainStats(stats *DashboardStats) error { + // Use COUNT(*) for total count + err := d.db.QueryRow("SELECT COUNT(*) FROM domains").Scan(&stats.TotalDomains) + if err != nil { + return err + } + + // Single query to get all status counts (one index scan instead of three) + rows, err := d.db.Query("SELECT status, COUNT(*) FROM domains GROUP BY status") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var status string + var count int + if err := rows.Scan(&status, &count); err != nil { + continue + } + switch status { + case "hold": + stats.HoldDomains = count + case "pass": + stats.PassDomains = count + case "skip": + stats.SkipDomains = count + case "dead": + stats.DeadDomains = count + } + } + if err := rows.Err(); err != nil { + return err + } + + return rows.Err() +} + +func (d *Dashboard) collectFeedStats(stats *DashboardStats) error { + // Use COUNT(*) for total count + err := d.db.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&stats.TotalFeeds) + if err != nil { + return err + } + + // Get status counts + statusRows, err := d.db.Query("SELECT status, COUNT(*) FROM feeds GROUP BY status") + if err != nil { + return err + } + defer statusRows.Close() + + for statusRows.Next() { + var status *string + var count int + if err := statusRows.Scan(&status, &count); err != nil { + continue + } + if status != nil { + switch *status { + case "pass": + stats.AliveFeeds = count + case "skip": + stats.SkipFeeds = count + case "hold": + stats.HoldFeeds = count + case "dead": + stats.DeadFeeds = count + } + } + } + + // Count feeds approved for publishing (publish_status='pass') + d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE publish_status = 'pass'").Scan(&stats.PublishFeeds) + + // Count empty feeds (item_count = 0 or NULL) + d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE item_count IS NULL OR item_count = 0").Scan(&stats.EmptyFeeds) + + // Single query to get all type counts (one index scan instead of three) + rows, err := d.db.Query("SELECT type, COUNT(*) FROM feeds GROUP BY type") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var feedType *string + var count int + if err := rows.Scan(&feedType, &count); err != nil { + continue + } + if feedType == nil { + stats.UnknownFeeds += count + } else { + switch *feedType { + case "rss": + stats.RSSFeeds = count + case "atom": + stats.AtomFeeds = count + case "json": + stats.JSONFeeds = count + default: + stats.UnknownFeeds += count + } + } + } + return rows.Err() +} + +// StartStatsLoop starts a background loop that updates stats every minute +func (d *Dashboard) StartStatsLoop() { + ticker := time.NewTicker(1 * time.Minute) + defer ticker.Stop() + + for range ticker.C { + d.UpdateStats() + } +} diff --git a/go.mod b/go.mod index 56707d1..7b613cc 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,16 @@ module github.com/1440news/dashboard go 1.24.0 +require github.com/1440news/shared v0.0.0 + require ( - github.com/1440news/shared v0.0.0 - github.com/haileyok/atproto-oauth-golang v0.0.0-20250101000000-000000000000 - github.com/jackc/pgx/v5 v5.7.5 - github.com/lestrrat-go/jwx/v2 v2.1.6 + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/text v0.33.0 // indirect ) replace github.com/1440news/shared => ../shared diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0c35200 --- /dev/null +++ b/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=