Add dashboard.go - stats types and calculation
Migrated from app/dashboard.go: - Dashboard struct with DB connection and stats caching - DashboardStats, TLDStat, RecentFeed, DomainStat types - Stats calculation methods (collectDomainStats, collectFeedStats) - Background stats update loop Note: Runtime rates (domains/min, etc.) not available in standalone dashboard - these are crawler-specific metrics. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
primal
+305
@@ -0,0 +1,305 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/1440news/shared"
|
||||
)
|
||||
|
||||
// Dashboard is the main dashboard service
|
||||
type Dashboard struct {
|
||||
db *shared.DB
|
||||
statsMu sync.RWMutex
|
||||
cachedStats *DashboardStats
|
||||
cachedAllDomains []DomainStat
|
||||
startTime time.Time
|
||||
}
|
||||
|
||||
// NewDashboard creates a new dashboard instance
|
||||
func NewDashboard(connString string) (*Dashboard, error) {
|
||||
db, err := shared.OpenDatabase(connString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Dashboard{
|
||||
db: db,
|
||||
startTime: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the database connection
|
||||
func (d *Dashboard) Close() error {
|
||||
return d.db.Close()
|
||||
}
|
||||
|
||||
// DashboardStats holds all statistics for the dashboard
|
||||
type DashboardStats struct {
|
||||
// Domain stats
|
||||
TotalDomains int `json:"total_domains"`
|
||||
HoldDomains int `json:"hold_domains"`
|
||||
PassDomains int `json:"pass_domains"`
|
||||
SkipDomains int `json:"skip_domains"`
|
||||
DeadDomains int `json:"dead_domains"`
|
||||
|
||||
// Feed stats
|
||||
TotalFeeds int `json:"total_feeds"`
|
||||
AliveFeeds int `json:"alive_feeds"` // status='pass' (healthy feeds)
|
||||
PublishFeeds int `json:"publish_feeds"` // publish_status='pass' (approved for publishing)
|
||||
SkipFeeds int `json:"skip_feeds"`
|
||||
HoldFeeds int `json:"hold_feeds"`
|
||||
DeadFeeds int `json:"dead_feeds"`
|
||||
EmptyFeeds int `json:"empty_feeds"`
|
||||
RSSFeeds int `json:"rss_feeds"`
|
||||
AtomFeeds int `json:"atom_feeds"`
|
||||
JSONFeeds int `json:"json_feeds"`
|
||||
UnknownFeeds int `json:"unknown_feeds"`
|
||||
|
||||
// Processing rates (per minute) - populated by crawler API if available
|
||||
DomainsCrawled int32 `json:"domains_crawled"`
|
||||
DomainCheckRate int `json:"domain_check_rate"`
|
||||
FeedCrawlRate int `json:"feed_crawl_rate"`
|
||||
FeedCheckRate int `json:"feed_check_rate"`
|
||||
|
||||
// Timing
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type TLDStat struct {
|
||||
TLD string `json:"tld"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type RecentFeed struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Type string `json:"type"`
|
||||
DiscoveredAt time.Time `json:"discovered_at"`
|
||||
}
|
||||
|
||||
type DomainStat struct {
|
||||
Host string `json:"host"`
|
||||
FeedsFound int `json:"feeds_found"`
|
||||
}
|
||||
|
||||
// commaFormat formats an integer with comma separators
|
||||
func commaFormat(n int) string {
|
||||
s := fmt.Sprintf("%d", n)
|
||||
if len(s) <= 3 {
|
||||
return s
|
||||
}
|
||||
var result []byte
|
||||
for i, c := range s {
|
||||
if i > 0 && (len(s)-i)%3 == 0 {
|
||||
result = append(result, ',')
|
||||
}
|
||||
result = append(result, byte(c))
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
// UpdateStats recalculates and caches dashboard statistics
|
||||
func (d *Dashboard) UpdateStats() {
|
||||
fmt.Println("UpdateStats: calculating stats...")
|
||||
stats, err := d.calculateStats()
|
||||
if err != nil {
|
||||
fmt.Printf("UpdateStats: error calculating stats: %v\n", err)
|
||||
return
|
||||
}
|
||||
// Cache all domains with feeds (runs in background, so slow query is OK)
|
||||
fmt.Println("UpdateStats: fetching all domains...")
|
||||
allDomains := d.fetchAllDomainsFromDB()
|
||||
fmt.Printf("UpdateStats: got %d domains\n", len(allDomains))
|
||||
|
||||
d.statsMu.Lock()
|
||||
d.cachedStats = stats
|
||||
d.cachedAllDomains = allDomains
|
||||
d.statsMu.Unlock()
|
||||
fmt.Println("UpdateStats: complete")
|
||||
}
|
||||
|
||||
func (d *Dashboard) fetchAllDomainsFromDB() []DomainStat {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT domain_tld as tld, domain_host as domain_host, COUNT(*) as cnt FROM feeds
|
||||
GROUP BY domain_tld, domain_host
|
||||
ORDER BY domain_tld, domain_host
|
||||
`)
|
||||
if err != nil {
|
||||
fmt.Printf("fetchAllDomainsFromDB error: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var domains []DomainStat
|
||||
for rows.Next() {
|
||||
var ds DomainStat
|
||||
var tld string
|
||||
if err := rows.Scan(&tld, &ds.Host, &ds.FeedsFound); err != nil {
|
||||
continue
|
||||
}
|
||||
domains = append(domains, ds)
|
||||
}
|
||||
return domains
|
||||
}
|
||||
|
||||
// GetDashboardStats returns cached statistics (returns empty stats if not yet cached)
|
||||
func (d *Dashboard) GetDashboardStats() (*DashboardStats, error) {
|
||||
d.statsMu.RLock()
|
||||
stats := d.cachedStats
|
||||
d.statsMu.RUnlock()
|
||||
|
||||
if stats != nil {
|
||||
return stats, nil
|
||||
}
|
||||
// Return empty stats while background calculation runs (don't block HTTP requests)
|
||||
return &DashboardStats{UpdatedAt: time.Now()}, nil
|
||||
}
|
||||
|
||||
// GetCachedAllDomains returns the cached list of all domains
|
||||
func (d *Dashboard) GetCachedAllDomains() []DomainStat {
|
||||
d.statsMu.RLock()
|
||||
defer d.statsMu.RUnlock()
|
||||
return d.cachedAllDomains
|
||||
}
|
||||
|
||||
// calculateStats collects all statistics for the dashboard
|
||||
func (d *Dashboard) calculateStats() (*DashboardStats, error) {
|
||||
stats := &DashboardStats{
|
||||
UpdatedAt: time.Now(),
|
||||
// Runtime rates not available in standalone dashboard
|
||||
// TODO: fetch from crawler API if needed
|
||||
}
|
||||
|
||||
// Get domain stats
|
||||
if err := d.collectDomainStats(stats); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get feed stats
|
||||
if err := d.collectFeedStats(stats); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (d *Dashboard) collectDomainStats(stats *DashboardStats) error {
|
||||
// Use COUNT(*) for total count
|
||||
err := d.db.QueryRow("SELECT COUNT(*) FROM domains").Scan(&stats.TotalDomains)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Single query to get all status counts (one index scan instead of three)
|
||||
rows, err := d.db.Query("SELECT status, COUNT(*) FROM domains GROUP BY status")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var status string
|
||||
var count int
|
||||
if err := rows.Scan(&status, &count); err != nil {
|
||||
continue
|
||||
}
|
||||
switch status {
|
||||
case "hold":
|
||||
stats.HoldDomains = count
|
||||
case "pass":
|
||||
stats.PassDomains = count
|
||||
case "skip":
|
||||
stats.SkipDomains = count
|
||||
case "dead":
|
||||
stats.DeadDomains = count
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dashboard) collectFeedStats(stats *DashboardStats) error {
|
||||
// Use COUNT(*) for total count
|
||||
err := d.db.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&stats.TotalFeeds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get status counts
|
||||
statusRows, err := d.db.Query("SELECT status, COUNT(*) FROM feeds GROUP BY status")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer statusRows.Close()
|
||||
|
||||
for statusRows.Next() {
|
||||
var status *string
|
||||
var count int
|
||||
if err := statusRows.Scan(&status, &count); err != nil {
|
||||
continue
|
||||
}
|
||||
if status != nil {
|
||||
switch *status {
|
||||
case "pass":
|
||||
stats.AliveFeeds = count
|
||||
case "skip":
|
||||
stats.SkipFeeds = count
|
||||
case "hold":
|
||||
stats.HoldFeeds = count
|
||||
case "dead":
|
||||
stats.DeadFeeds = count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Count feeds approved for publishing (publish_status='pass')
|
||||
d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE publish_status = 'pass'").Scan(&stats.PublishFeeds)
|
||||
|
||||
// Count empty feeds (item_count = 0 or NULL)
|
||||
d.db.QueryRow("SELECT COUNT(*) FROM feeds WHERE item_count IS NULL OR item_count = 0").Scan(&stats.EmptyFeeds)
|
||||
|
||||
// Single query to get all type counts (one index scan instead of three)
|
||||
rows, err := d.db.Query("SELECT type, COUNT(*) FROM feeds GROUP BY type")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var feedType *string
|
||||
var count int
|
||||
if err := rows.Scan(&feedType, &count); err != nil {
|
||||
continue
|
||||
}
|
||||
if feedType == nil {
|
||||
stats.UnknownFeeds += count
|
||||
} else {
|
||||
switch *feedType {
|
||||
case "rss":
|
||||
stats.RSSFeeds = count
|
||||
case "atom":
|
||||
stats.AtomFeeds = count
|
||||
case "json":
|
||||
stats.JSONFeeds = count
|
||||
default:
|
||||
stats.UnknownFeeds += count
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
// StartStatsLoop starts a background loop that updates stats every minute
|
||||
func (d *Dashboard) StartStatsLoop() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
d.UpdateStats()
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,16 @@ module github.com/1440news/dashboard
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require github.com/1440news/shared v0.0.0
|
||||
|
||||
require (
|
||||
github.com/1440news/shared v0.0.0
|
||||
github.com/haileyok/atproto-oauth-golang v0.0.0-20250101000000-000000000000
|
||||
github.com/jackc/pgx/v5 v5.7.5
|
||||
github.com/lestrrat-go/jwx/v2 v2.1.6
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/pgx/v5 v5.7.5 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
golang.org/x/crypto v0.47.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/1440news/shared => ../shared
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
|
||||
github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
|
||||
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
Reference in New Issue
Block a user