package main import ( "encoding/json" "fmt" "net/http" "strings" "time" "github.com/jackc/pgx/v5" ) func (c *Crawler) handleAPIFeedInfo(w http.ResponseWriter, r *http.Request) { feedURL := r.URL.Query().Get("url") if feedURL == "" { http.Error(w, "url parameter required", http.StatusBadRequest) return } type FeedDetails struct { URL string `json:"url"` Type string `json:"type,omitempty"` Category string `json:"category,omitempty"` Title string `json:"title,omitempty"` Description string `json:"description,omitempty"` Language string `json:"language,omitempty"` SiteURL string `json:"siteUrl,omitempty"` DiscoveredAt string `json:"discoveredAt,omitempty"` LastCheckedAt string `json:"lastCheckedAt,omitempty"` NextCheckAt string `json:"nextCheckAt,omitempty"` LastBuildDate string `json:"lastBuildDate,omitempty"` Status string `json:"status,omitempty"` LastError string `json:"lastError,omitempty"` ItemCount int `json:"itemCount,omitempty"` OldestItemDate string `json:"oldestItemDate,omitempty"` NewestItemDate string `json:"newestItemDate,omitempty"` PublishStatus string `json:"publishStatus,omitempty"` PublishAccount string `json:"publishAccount,omitempty"` } var f FeedDetails var category, title, description, language, siteUrl *string var lastCheckedAt, nextCheckAt, lastBuildDate *time.Time var status, lastError *string var oldestItemDate, newestItemDate *time.Time var itemCount *int var discoveredAt time.Time var publishStatus, publishAccount *string err := c.db.QueryRow(` SELECT url, type, category, title, description, language, site_url, discovered_at, last_checked_at, next_check_at, last_build_date, status, last_error, (SELECT COUNT(*) FROM items WHERE feed_url = feeds.url) as item_count, oldest_item_date, newest_item_date, publish_status, publish_account FROM feeds WHERE url = $1 `, feedURL).Scan( &f.URL, &f.Type, &category, &title, &description, &language, &siteUrl, &discoveredAt, &lastCheckedAt, &nextCheckAt, &lastBuildDate, &status, &lastError, &itemCount, &oldestItemDate, &newestItemDate, &publishStatus, &publishAccount, ) if err == pgx.ErrNoRows { http.Error(w, "feed not found", http.StatusNotFound) return } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } f.Category = StringValue(category) f.Title = StringValue(title) f.Description = StringValue(description) f.Language = StringValue(language) f.SiteURL = StringValue(siteUrl) f.DiscoveredAt = discoveredAt.Format(time.RFC3339) if lastCheckedAt != nil { f.LastCheckedAt = lastCheckedAt.Format(time.RFC3339) } if nextCheckAt != nil { f.NextCheckAt = nextCheckAt.Format(time.RFC3339) } if lastBuildDate != nil { f.LastBuildDate = lastBuildDate.Format(time.RFC3339) } f.Status = StringValue(status) f.LastError = StringValue(lastError) if itemCount != nil { f.ItemCount = *itemCount } if oldestItemDate != nil { f.OldestItemDate = oldestItemDate.Format(time.RFC3339) } if newestItemDate != nil { f.NewestItemDate = newestItemDate.Format(time.RFC3339) } f.PublishStatus = StringValue(publishStatus) f.PublishAccount = StringValue(publishAccount) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(f) } func (c *Crawler) handleAPIFeedItems(w http.ResponseWriter, r *http.Request) { feedURL := r.URL.Query().Get("url") if feedURL == "" { http.Error(w, "url parameter required", http.StatusBadRequest) return } limit := 50 if l := r.URL.Query().Get("limit"); l != "" { fmt.Sscanf(l, "%d", &limit) if limit > 100 { limit = 100 } } items, err := c.GetItemsByFeed(feedURL, limit) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } if items == nil { items = []*Item{} } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(items) } func (c *Crawler) handleAPIFeedsByStatus(w http.ResponseWriter, r *http.Request) { status := r.URL.Query().Get("status") if status == "" { http.Error(w, "status parameter required", http.StatusBadRequest) return } limit := 100 offset := 0 if l := r.URL.Query().Get("limit"); l != "" { fmt.Sscanf(l, "%d", &limit) if limit > 500 { limit = 500 } } if o := r.URL.Query().Get("offset"); o != "" { fmt.Sscanf(o, "%d", &offset) } rows, err := c.db.Query(` SELECT url, title, type, domain_host || '.' || domain_tld as source_host, domain_tld as tld, status, last_error, item_count FROM feeds WHERE status = $1 ORDER BY url ASC LIMIT $2 OFFSET $3 `, status, limit, offset) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type FeedInfo struct { URL string `json:"url"` Title string `json:"title,omitempty"` Type string `json:"type"` SourceHost string `json:"source_host"` TLD string `json:"tld"` Status string `json:"status"` LastError string `json:"last_error,omitempty"` ItemCount int `json:"item_count,omitempty"` } var feeds []FeedInfo for rows.Next() { var f FeedInfo var title, sourceHost, tld, lastError *string var itemCount *int if err := rows.Scan(&f.URL, &title, &f.Type, &sourceHost, &tld, &f.Status, &lastError, &itemCount); err != nil { continue } f.Title = StringValue(title) f.SourceHost = StringValue(sourceHost) f.TLD = StringValue(tld) f.LastError = StringValue(lastError) if itemCount != nil { f.ItemCount = *itemCount } feeds = append(feeds, f) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(feeds) } // handleAPIFeeds lists feeds with optional publish_status filter func (c *Crawler) handleAPIFeeds(w http.ResponseWriter, r *http.Request) { publishStatus := r.URL.Query().Get("publish_status") limit := 100 offset := 0 if l := r.URL.Query().Get("limit"); l != "" { fmt.Sscanf(l, "%d", &limit) if limit > 500 { limit = 500 } } if o := r.URL.Query().Get("offset"); o != "" { fmt.Sscanf(o, "%d", &offset) } var rows pgx.Rows var err error if publishStatus != "" { rows, err = c.db.Query(` SELECT url, title, type, domain_host || '.' || domain_tld as source_host, domain_tld as tld, status, last_error, item_count, publish_status, language FROM feeds WHERE publish_status = $1 ORDER BY url ASC LIMIT $2 OFFSET $3 `, publishStatus, limit, offset) } else { rows, err = c.db.Query(` SELECT url, title, type, domain_host || '.' || domain_tld as source_host, domain_tld as tld, status, last_error, item_count, publish_status, language FROM feeds ORDER BY url ASC LIMIT $1 OFFSET $2 `, limit, offset) } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type FeedInfo struct { URL string `json:"url"` Title string `json:"title,omitempty"` Type string `json:"type"` SourceHost string `json:"source_host"` TLD string `json:"tld"` Status string `json:"status"` LastError string `json:"last_error,omitempty"` ItemCount int `json:"item_count,omitempty"` PublishStatus string `json:"publish_status,omitempty"` Language string `json:"language,omitempty"` } var feeds []FeedInfo for rows.Next() { var f FeedInfo var title, sourceHost, tld, lastError, publishStatus, language *string var itemCount *int if err := rows.Scan(&f.URL, &title, &f.Type, &sourceHost, &tld, &f.Status, &lastError, &itemCount, &publishStatus, &language); err != nil { continue } f.Title = StringValue(title) f.SourceHost = StringValue(sourceHost) f.TLD = StringValue(tld) f.LastError = StringValue(lastError) f.PublishStatus = StringValue(publishStatus) f.Language = StringValue(language) if itemCount != nil { f.ItemCount = *itemCount } feeds = append(feeds, f) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(feeds) } func (c *Crawler) filterFeeds(w http.ResponseWriter, tld, domain, status string, languages []string, limit, offset int) { var args []interface{} argNum := 1 query := ` SELECT url, title, type, category, domain_host || '.' || domain_tld as source_host, domain_tld as tld, status, last_error, item_count, language FROM feeds WHERE 1=1` if tld != "" { query += fmt.Sprintf(" AND domain_tld = $%d", argNum) args = append(args, tld) argNum++ } if domain != "" { // Parse domain into host and tld parts domainHost := stripTLD(domain) domainTLD := getTLD(domain) query += fmt.Sprintf(" AND domain_host = $%d AND domain_tld = $%d", argNum, argNum+1) args = append(args, domainHost, domainTLD) argNum += 2 } if status != "" { query += fmt.Sprintf(" AND status = $%d", argNum) args = append(args, status) argNum++ } if len(languages) > 0 { // Build IN clause for languages, handling 'unknown' as empty string placeholders := make([]string, len(languages)) for i, lang := range languages { placeholders[i] = fmt.Sprintf("$%d", argNum) if lang == "unknown" { args = append(args, "") } else { args = append(args, lang) } argNum++ } query += fmt.Sprintf(" AND COALESCE(language, '') IN (%s)", strings.Join(placeholders, ",")) } query += fmt.Sprintf(" ORDER BY url ASC LIMIT $%d OFFSET $%d", argNum, argNum+1) args = append(args, limit, offset) rows, err := c.db.Query(query, args...) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type FeedInfo struct { URL string `json:"url"` Title string `json:"title,omitempty"` Type string `json:"type"` Category string `json:"category"` SourceHost string `json:"source_host"` TLD string `json:"tld"` Status string `json:"status"` LastError string `json:"last_error,omitempty"` ItemCount int `json:"item_count,omitempty"` Language string `json:"language,omitempty"` } var feeds []FeedInfo for rows.Next() { var f FeedInfo var title, category, sourceHost, tldVal, lastError, language *string var itemCount *int if err := rows.Scan(&f.URL, &title, &f.Type, &category, &sourceHost, &tldVal, &f.Status, &lastError, &itemCount, &language); err != nil { continue } f.Title = StringValue(title) if category != nil && *category != "" { f.Category = *category } else { f.Category = "main" } f.SourceHost = StringValue(sourceHost) f.TLD = StringValue(tldVal) f.LastError = StringValue(lastError) if itemCount != nil { f.ItemCount = *itemCount } f.Language = StringValue(language) feeds = append(feeds, f) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]interface{}{ "type": "feeds", "data": feeds, }) } // handleAPICheckFeed immediately checks a feed and returns items func (c *Crawler) handleAPICheckFeed(w http.ResponseWriter, r *http.Request) { feedURL := r.URL.Query().Get("url") if feedURL == "" { http.Error(w, "url parameter required", http.StatusBadRequest) return } force := r.URL.Query().Get("force") == "true" feedURL = normalizeURL(feedURL) // Get the feed feed, err := c.getFeed(feedURL) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } if feed == nil { http.Error(w, "feed not found", http.StatusNotFound) return } // Clear cache headers if force is requested if force { feed.ETag = "" feed.LastModified = "" } // Force check the feed fmt.Printf("Force check feed: %s (force=%v)\n", feedURL, force) changed, checkErr := c.CheckFeed(feed) // Get updated feed info feed, _ = c.getFeed(feedURL) // Get items items, _ := c.GetItemsByFeed(feedURL, 20) type ItemSummary struct { Title string `json:"title"` Link string `json:"link"` PubDate string `json:"pub_date,omitempty"` Author string `json:"author,omitempty"` } var itemSummaries []ItemSummary for _, item := range items { is := ItemSummary{ Title: item.Title, Link: item.Link, Author: item.Author, } if !item.PubDate.IsZero() { is.PubDate = item.PubDate.Format("2006-01-02 15:04") } itemSummaries = append(itemSummaries, is) } result := map[string]interface{}{ "url": feedURL, "title": feed.Title, "type": feed.Type, "category": feed.Category, "status": feed.Status, "changed": changed, "itemCount": feed.ItemCount, "items": itemSummaries, } if checkErr != nil { result["error"] = checkErr.Error() } if feed.LastError != "" { result["lastError"] = feed.LastError } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(result) } // handleAPILanguages returns distinct languages with counts func (c *Crawler) handleAPILanguages(w http.ResponseWriter, r *http.Request) { rows, err := c.db.Query(` SELECT COALESCE(NULLIF(language, ''), 'unknown') as lang, COUNT(*) as cnt FROM feeds GROUP BY lang ORDER BY cnt DESC `) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } defer rows.Close() type LangInfo struct { Language string `json:"language"` Count int `json:"count"` } var languages []LangInfo for rows.Next() { var l LangInfo if err := rows.Scan(&l.Language, &l.Count); err != nil { continue } languages = append(languages, l) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(languages) }