package main import ( "encoding/json" "fmt" "net/http" "strings" "time" "github.com/1440news/commons" "github.com/jackc/pgx/v5" ) // GetItemsByFeed retrieves items for a feed func (d *Dashboard) GetItemsByFeed(feedURL string, limit int) ([]*commons.Item, error) { rows, err := d.db.Query(` SELECT feed_url, link, title, description, author, pub_date FROM items WHERE feed_url = $1 ORDER BY pub_date DESC LIMIT $2 `, feedURL, limit) if err != nil { return nil, err } defer rows.Close() var items []*commons.Item for rows.Next() { var item commons.Item var title, description, author *string var pubDate *time.Time if err := rows.Scan(&item.FeedURL, &item.Link, &title, &description, &author, &pubDate); err != nil { continue } item.Title = commons.StringValue(title) item.Description = commons.StringValue(description) item.Author = commons.StringValue(author) if pubDate != nil { item.PubDate = *pubDate } items = append(items, &item) } return items, nil } func (d *Dashboard) handleAPIFeedInfo(w http.ResponseWriter, r *http.Request) { feedURL := r.URL.Query().Get("url") if feedURL == "" { http.Error(w, "url parameter required", http.StatusBadRequest) return } type FeedDetails struct { URL string `json:"url"` Type string `json:"type,omitempty"` Title string `json:"title,omitempty"` Description string `json:"description,omitempty"` Language string `json:"language,omitempty"` LastCheckedAt string `json:"lastCheckedAt,omitempty"` Status string `json:"status,omitempty"` LastError string `json:"lastError,omitempty"` ItemCount int `json:"itemCount,omitempty"` } var f FeedDetails var title, description, language *string var lastCheckedAt *time.Time var status, lastError *string var itemCount *int err := d.db.QueryRow(` SELECT url, type, title, description, language, last_checked_at, status, last_error, (SELECT COUNT(*) FROM items WHERE feed_url = feeds.url AND rkey IS NULL) as item_count FROM feeds WHERE url = $1 `, feedURL).Scan( &f.URL, &f.Type, &title, &description, &language, &lastCheckedAt, &status, &lastError, &itemCount, ) if err == pgx.ErrNoRows { http.Error(w, "feed not found", http.StatusNotFound) return } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } f.Title = commons.StringValue(title) f.Description = commons.StringValue(description) f.Language = commons.StringValue(language) if lastCheckedAt != nil { f.LastCheckedAt = lastCheckedAt.Format(time.RFC3339) } f.Status = commons.StringValue(status) f.LastError = commons.StringValue(lastError) if itemCount != nil { f.ItemCount = *itemCount } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(f) } func (d *Dashboard) handleAPIFeedItems(w http.ResponseWriter, r *http.Request) { feedURL := r.URL.Query().Get("url") if feedURL == "" { http.Error(w, "url parameter required", http.StatusBadRequest) return } limit := 50 if l := r.URL.Query().Get("limit"); l != "" { fmt.Sscanf(l, "%d", &limit) if limit > 100 { limit = 100 } } items, err := d.GetItemsByFeed(feedURL, limit) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } if items == nil { items = []*commons.Item{} } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(items) } func (d *Dashboard) handleAPIFeedsByStatus(w http.ResponseWriter, r *http.Request) { status := r.URL.Query().Get("status") if status == "" { http.Error(w, "status parameter required", http.StatusBadRequest) return } limit := 100 offset := 0 if l := r.URL.Query().Get("limit"); l != "" { fmt.Sscanf(l, "%d", &limit) if limit > 500 { limit = 500 } } if o := r.URL.Query().Get("offset"); o != "" { fmt.Sscanf(o, "%d", &offset) } rows, err := d.db.Query(` SELECT url, title, type, status, last_error, (SELECT COUNT(*) FROM items WHERE feed_url = feeds.url AND rkey IS NULL) as item_count FROM feeds WHERE status = $1 ORDER BY url ASC LIMIT $2 OFFSET $3 `, status, limit, offset) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type FeedInfo struct { URL string `json:"url"` Title string `json:"title,omitempty"` Type string `json:"type"` SourceHost string `json:"source_host"` Status string `json:"status"` LastError string `json:"last_error,omitempty"` ItemCount int `json:"item_count,omitempty"` } var feeds []FeedInfo for rows.Next() { var f FeedInfo var title, lastError *string var itemCount *int if err := rows.Scan(&f.URL, &title, &f.Type, &f.Status, &lastError, &itemCount); err != nil { continue } f.Title = commons.StringValue(title) f.SourceHost = commons.GetHostFromURL(f.URL) f.LastError = commons.StringValue(lastError) if itemCount != nil { f.ItemCount = *itemCount } feeds = append(feeds, f) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(feeds) } // handleAPIFeeds lists feeds with optional status filter func (d *Dashboard) handleAPIFeeds(w http.ResponseWriter, r *http.Request) { statusFilter := r.URL.Query().Get("status") limit := 100 offset := 0 if l := r.URL.Query().Get("limit"); l != "" { fmt.Sscanf(l, "%d", &limit) if limit > 500 { limit = 500 } } if o := r.URL.Query().Get("offset"); o != "" { fmt.Sscanf(o, "%d", &offset) } var rows pgx.Rows var err error if statusFilter != "" { rows, err = d.db.Query(` SELECT url, title, type, status, last_error, (SELECT COUNT(*) FROM items WHERE feed_url = feeds.url AND rkey IS NULL) as item_count, language FROM feeds WHERE status = $1 ORDER BY url ASC LIMIT $2 OFFSET $3 `, statusFilter, limit, offset) } else { rows, err = d.db.Query(` SELECT url, title, type, status, last_error, (SELECT COUNT(*) FROM items WHERE feed_url = feeds.url AND rkey IS NULL) as item_count, language FROM feeds ORDER BY url ASC LIMIT $1 OFFSET $2 `, limit, offset) } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type FeedInfo struct { URL string `json:"url"` Title string `json:"title,omitempty"` Type string `json:"type"` SourceHost string `json:"source_host"` Status string `json:"status"` LastError string `json:"last_error,omitempty"` ItemCount int `json:"item_count,omitempty"` Language string `json:"language,omitempty"` } var feeds []FeedInfo for rows.Next() { var f FeedInfo var title, lastError, language *string var itemCount *int if err := rows.Scan(&f.URL, &title, &f.Type, &f.Status, &lastError, &itemCount, &language); err != nil { continue } f.Title = commons.StringValue(title) f.SourceHost = commons.GetHostFromURL(f.URL) f.LastError = commons.StringValue(lastError) f.Language = commons.StringValue(language) if itemCount != nil { f.ItemCount = *itemCount } feeds = append(feeds, f) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(feeds) } func (d *Dashboard) filterFeeds(w http.ResponseWriter, tld, domain, status string, languages []string, limit, offset int) { var args []interface{} argNum := 1 query := ` SELECT url, title, type, status, last_error, (SELECT COUNT(*) FROM items WHERE feed_url = feeds.url AND rkey IS NULL) as item_count, language FROM feeds WHERE 1=1` if tld != "" { // Filter by TLD using URL pattern: feeds ending with .tld/ query += fmt.Sprintf(" AND LOWER(url) LIKE '%%.' || LOWER($%d) || '/%%'", argNum) args = append(args, tld) argNum++ } if domain != "" { // Filter by domain using URL pattern query += fmt.Sprintf(" AND LOWER(url) LIKE LOWER($%d) || '/%%'", argNum) args = append(args, domain) argNum++ } if status != "" { query += fmt.Sprintf(" AND status = $%d", argNum) args = append(args, status) argNum++ } if len(languages) > 0 { // Build IN clause for languages, handling 'unknown' as empty string placeholders := make([]string, len(languages)) for i, lang := range languages { placeholders[i] = fmt.Sprintf("$%d", argNum) if lang == "unknown" { args = append(args, "") } else { args = append(args, lang) } argNum++ } query += fmt.Sprintf(" AND COALESCE(language, '') IN (%s)", strings.Join(placeholders, ",")) } query += fmt.Sprintf(" ORDER BY url ASC LIMIT $%d OFFSET $%d", argNum, argNum+1) args = append(args, limit, offset) rows, err := d.db.Query(query, args...) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type FeedInfo struct { URL string `json:"url"` Title string `json:"title,omitempty"` Type string `json:"type"` SourceHost string `json:"source_host"` Status string `json:"status"` LastError string `json:"last_error,omitempty"` ItemCount int `json:"item_count,omitempty"` Language string `json:"language,omitempty"` } var feeds []FeedInfo for rows.Next() { var f FeedInfo var title, lastError, language *string var itemCount *int if err := rows.Scan(&f.URL, &title, &f.Type, &f.Status, &lastError, &itemCount, &language); err != nil { continue } f.Title = commons.StringValue(title) f.SourceHost = commons.GetHostFromURL(f.URL) f.LastError = commons.StringValue(lastError) if itemCount != nil { f.ItemCount = *itemCount } f.Language = commons.StringValue(language) feeds = append(feeds, f) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]interface{}{ "type": "feeds", "data": feeds, }) } // handleAPICheckFeed - NOTE: This requires crawler functionality (feed checking) // For now, returns an error. In the future, could call crawler API. func (d *Dashboard) handleAPICheckFeed(w http.ResponseWriter, r *http.Request) { http.Error(w, "Feed checking not available in standalone dashboard. Use crawler service.", http.StatusNotImplemented) } // handleAPIFeedsBrowse returns a flat list of feeds with optional search filtering func (d *Dashboard) handleAPIFeedsBrowse(w http.ResponseWriter, r *http.Request) { search := r.URL.Query().Get("search") limit := 100 offset := 0 if l := r.URL.Query().Get("limit"); l != "" { fmt.Sscanf(l, "%d", &limit) if limit > 500 { limit = 500 } } if o := r.URL.Query().Get("offset"); o != "" { fmt.Sscanf(o, "%d", &offset) } // Build query with optional search filter query := ` SELECT url, title, type, status, (SELECT COUNT(*) FROM items WHERE feed_url = feeds.url AND rkey IS NULL) as item_count, language FROM feeds WHERE ($1 = '' OR LOWER(url) LIKE '%' || LOWER($1) || '%' OR LOWER(COALESCE(title, '')) LIKE '%' || LOWER($1) || '%') ORDER BY url LIMIT $2 OFFSET $3 ` rows, err := d.db.Query(query, search, limit, offset) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type BrowseFeed struct { URL string `json:"url"` Title string `json:"title,omitempty"` Type string `json:"type"` Status string `json:"status"` SourceHost string `json:"source_host"` ItemCount int `json:"item_count"` Language string `json:"language,omitempty"` } var feeds []BrowseFeed for rows.Next() { var f BrowseFeed var title, feedType, status, language *string var itemCount *int if err := rows.Scan(&f.URL, &title, &feedType, &status, &itemCount, &language); err != nil { continue } f.Title = commons.StringValue(title) f.Type = commons.StringValue(feedType) f.Status = commons.StringValue(status) f.SourceHost = commons.GetHostFromURL(f.URL) f.Language = commons.StringValue(language) if itemCount != nil { f.ItemCount = *itemCount } feeds = append(feeds, f) } if feeds == nil { feeds = []BrowseFeed{} } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(feeds) } // handleAPILanguages returns distinct languages with counts func (d *Dashboard) handleAPILanguages(w http.ResponseWriter, r *http.Request) { rows, err := d.db.Query(` SELECT COALESCE(NULLIF(language, ''), 'unknown') as lang, COUNT(*) as cnt FROM feeds GROUP BY lang ORDER BY cnt DESC `) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type LangInfo struct { Language string `json:"language"` Count int `json:"count"` } var languages []LangInfo for rows.Next() { var l LangInfo if err := rows.Scan(&l.Language, &l.Count); err != nil { continue } languages = append(languages, l) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(languages) }