81c92b5b51
- Container: atproto-1440news-watcher - Endpoints: dashboard.1440.news and watcher.1440.news (alias) - Updated Dockerfile and docker-compose.yml - Updated go.mod module name
576 lines
17 KiB
Go
576 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/1440news/shared"
|
|
)
|
|
|
|
// Database helper methods for publish-related queries
|
|
|
|
// SetPublishStatus updates a feed's publish status and account
|
|
func (d *Dashboard) SetPublishStatus(feedURL, status, account string) error {
|
|
_, err := d.db.Exec(`
|
|
UPDATE feeds SET publish_status = $1, publish_account = $2 WHERE url = $3
|
|
`, status, shared.NullableString(account), feedURL)
|
|
return err
|
|
}
|
|
|
|
// GetFeedsByPublishStatus returns feeds with a specific publish status
|
|
func (d *Dashboard) GetFeedsByPublishStatus(status string) ([]*shared.Feed, error) {
|
|
rows, err := d.db.Query(`
|
|
SELECT url, type, category, title, description, language, site_url,
|
|
discovered_at, last_checked_at, next_check_at, last_build_date,
|
|
status, last_error, last_error_at,
|
|
source_url, domain_host, domain_tld,
|
|
item_count, oldest_item_date, newest_item_date, no_update,
|
|
publish_status, publish_account
|
|
FROM feeds
|
|
WHERE publish_status = $1
|
|
ORDER BY url ASC
|
|
`, status)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var feeds []*shared.Feed
|
|
for rows.Next() {
|
|
f := &shared.Feed{}
|
|
var category, title, description, language, siteUrl *string
|
|
var lastCheckedAt, nextCheckAt, lastBuildDate, lastErrorAt *time.Time
|
|
var feedStatus, lastError *string
|
|
var sourceUrl, domainHost, domainTLD *string
|
|
var itemCount, noUpdate *int
|
|
var oldestItemDate, newestItemDate *time.Time
|
|
var publishStatus, publishAccount *string
|
|
|
|
if err := rows.Scan(&f.URL, &f.Type, &category, &title, &description, &language, &siteUrl,
|
|
&f.DiscoveredAt, &lastCheckedAt, &nextCheckAt, &lastBuildDate,
|
|
&feedStatus, &lastError, &lastErrorAt,
|
|
&sourceUrl, &domainHost, &domainTLD,
|
|
&itemCount, &oldestItemDate, &newestItemDate, &noUpdate,
|
|
&publishStatus, &publishAccount); err != nil {
|
|
continue
|
|
}
|
|
f.Category = shared.StringValue(category)
|
|
f.Title = shared.StringValue(title)
|
|
f.Description = shared.StringValue(description)
|
|
f.Language = shared.StringValue(language)
|
|
f.SiteURL = shared.StringValue(siteUrl)
|
|
if lastCheckedAt != nil {
|
|
f.LastCheckedAt = *lastCheckedAt
|
|
}
|
|
if nextCheckAt != nil {
|
|
f.NextCheckAt = *nextCheckAt
|
|
}
|
|
if lastBuildDate != nil {
|
|
f.LastBuildDate = *lastBuildDate
|
|
}
|
|
f.Status = shared.StringValue(feedStatus)
|
|
f.LastError = shared.StringValue(lastError)
|
|
if lastErrorAt != nil {
|
|
f.LastErrorAt = *lastErrorAt
|
|
}
|
|
f.SourceURL = shared.StringValue(sourceUrl)
|
|
f.DomainHost = shared.StringValue(domainHost)
|
|
f.DomainTLD = shared.StringValue(domainTLD)
|
|
if itemCount != nil {
|
|
f.ItemCount = *itemCount
|
|
}
|
|
if oldestItemDate != nil {
|
|
f.OldestItemDate = *oldestItemDate
|
|
}
|
|
if newestItemDate != nil {
|
|
f.NewestItemDate = *newestItemDate
|
|
}
|
|
if noUpdate != nil {
|
|
f.NoUpdate = *noUpdate
|
|
}
|
|
f.PublishStatus = shared.StringValue(publishStatus)
|
|
f.PublishAccount = shared.StringValue(publishAccount)
|
|
|
|
feeds = append(feeds, f)
|
|
}
|
|
return feeds, nil
|
|
}
|
|
|
|
// GetUnpublishedItemCount returns count of unpublished items for a feed
|
|
func (d *Dashboard) GetUnpublishedItemCount(feedURL string) (int, error) {
|
|
var count int
|
|
err := d.db.QueryRow(`
|
|
SELECT COUNT(*) FROM items WHERE feed_url = $1 AND published_at IS NULL
|
|
`, feedURL).Scan(&count)
|
|
return count, err
|
|
}
|
|
|
|
// GetPublishCandidates returns feeds pending review that have items
|
|
func (d *Dashboard) GetPublishCandidates(limit int) ([]*shared.Feed, error) {
|
|
rows, err := d.db.Query(`
|
|
SELECT url, type, category, title, description, domain_host, domain_tld, item_count
|
|
FROM feeds
|
|
WHERE publish_status = 'hold'
|
|
AND status = 'pass'
|
|
AND item_count > 0
|
|
AND language = 'en'
|
|
ORDER BY item_count DESC
|
|
LIMIT $1
|
|
`, limit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var feeds []*shared.Feed
|
|
for rows.Next() {
|
|
f := &shared.Feed{}
|
|
var category, title, description, domainHost, domainTLD *string
|
|
var itemCount *int
|
|
if err := rows.Scan(&f.URL, &f.Type, &category, &title, &description, &domainHost, &domainTLD, &itemCount); err != nil {
|
|
continue
|
|
}
|
|
f.Category = shared.StringValue(category)
|
|
if f.Category == "" {
|
|
f.Category = "main"
|
|
}
|
|
f.Title = shared.StringValue(title)
|
|
f.Description = shared.StringValue(description)
|
|
f.DomainHost = shared.StringValue(domainHost)
|
|
f.DomainTLD = shared.StringValue(domainTLD)
|
|
if itemCount != nil {
|
|
f.ItemCount = *itemCount
|
|
}
|
|
feeds = append(feeds, f)
|
|
}
|
|
return feeds, nil
|
|
}
|
|
|
|
// GetUnpublishedItems returns unpublished items for a feed
|
|
func (d *Dashboard) GetUnpublishedItems(feedURL string, limit int) ([]*shared.Item, error) {
|
|
rows, err := d.db.Query(`
|
|
SELECT feed_url, guid, title, link, description, content, author, pub_date, discovered_at, updated_at
|
|
FROM items
|
|
WHERE feed_url = $1 AND published_at IS NULL
|
|
ORDER BY pub_date ASC
|
|
LIMIT $2
|
|
`, feedURL, limit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var items []*shared.Item
|
|
for rows.Next() {
|
|
var item shared.Item
|
|
var guid, title, link, description, content, author *string
|
|
var pubDate, discoveredAt, updatedAt *time.Time
|
|
if err := rows.Scan(&item.FeedURL, &guid, &title, &link, &description, &content, &author, &pubDate, &discoveredAt, &updatedAt); err != nil {
|
|
continue
|
|
}
|
|
item.GUID = shared.StringValue(guid)
|
|
item.Title = shared.StringValue(title)
|
|
item.Link = shared.StringValue(link)
|
|
item.Description = shared.StringValue(description)
|
|
item.Content = shared.StringValue(content)
|
|
item.Author = shared.StringValue(author)
|
|
if pubDate != nil {
|
|
item.PubDate = *pubDate
|
|
}
|
|
if discoveredAt != nil {
|
|
item.DiscoveredAt = *discoveredAt
|
|
}
|
|
if updatedAt != nil {
|
|
item.UpdatedAt = *updatedAt
|
|
}
|
|
items = append(items, &item)
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// getFeed retrieves a single feed by URL
|
|
func (d *Dashboard) getFeed(feedURL string) (*shared.Feed, error) {
|
|
f := &shared.Feed{}
|
|
var category, title, description, language, siteUrl *string
|
|
var lastCheckedAt, nextCheckAt, lastBuildDate, lastErrorAt *time.Time
|
|
var status, lastError *string
|
|
var sourceUrl, domainHost, domainTLD *string
|
|
var itemCount, noUpdate *int
|
|
var oldestItemDate, newestItemDate *time.Time
|
|
var publishStatus, publishAccount *string
|
|
var etag, lastModified *string
|
|
|
|
err := d.db.QueryRow(`
|
|
SELECT url, type, category, title, description, language, site_url,
|
|
discovered_at, last_checked_at, next_check_at, last_build_date,
|
|
etag, last_modified,
|
|
status, last_error, last_error_at,
|
|
source_url, domain_host, domain_tld,
|
|
item_count, oldest_item_date, newest_item_date, no_update,
|
|
publish_status, publish_account
|
|
FROM feeds WHERE url = $1
|
|
`, feedURL).Scan(&f.URL, &f.Type, &category, &title, &description, &language, &siteUrl,
|
|
&f.DiscoveredAt, &lastCheckedAt, &nextCheckAt, &lastBuildDate,
|
|
&etag, &lastModified,
|
|
&status, &lastError, &lastErrorAt,
|
|
&sourceUrl, &domainHost, &domainTLD,
|
|
&itemCount, &oldestItemDate, &newestItemDate, &noUpdate,
|
|
&publishStatus, &publishAccount)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
f.Category = shared.StringValue(category)
|
|
f.Title = shared.StringValue(title)
|
|
f.Description = shared.StringValue(description)
|
|
f.Language = shared.StringValue(language)
|
|
f.SiteURL = shared.StringValue(siteUrl)
|
|
if lastCheckedAt != nil {
|
|
f.LastCheckedAt = *lastCheckedAt
|
|
}
|
|
if nextCheckAt != nil {
|
|
f.NextCheckAt = *nextCheckAt
|
|
}
|
|
if lastBuildDate != nil {
|
|
f.LastBuildDate = *lastBuildDate
|
|
}
|
|
f.ETag = shared.StringValue(etag)
|
|
f.LastModified = shared.StringValue(lastModified)
|
|
f.Status = shared.StringValue(status)
|
|
f.LastError = shared.StringValue(lastError)
|
|
if lastErrorAt != nil {
|
|
f.LastErrorAt = *lastErrorAt
|
|
}
|
|
f.SourceURL = shared.StringValue(sourceUrl)
|
|
f.DomainHost = shared.StringValue(domainHost)
|
|
f.DomainTLD = shared.StringValue(domainTLD)
|
|
if itemCount != nil {
|
|
f.ItemCount = *itemCount
|
|
}
|
|
if oldestItemDate != nil {
|
|
f.OldestItemDate = *oldestItemDate
|
|
}
|
|
if newestItemDate != nil {
|
|
f.NewestItemDate = *newestItemDate
|
|
}
|
|
if noUpdate != nil {
|
|
f.NoUpdate = *noUpdate
|
|
}
|
|
f.PublishStatus = shared.StringValue(publishStatus)
|
|
f.PublishAccount = shared.StringValue(publishAccount)
|
|
|
|
return f, nil
|
|
}
|
|
|
|
// API Handlers
|
|
|
|
// handleAPIEnablePublish sets a feed's publish status to 'pass' (database only, no PDS account creation)
|
|
func (d *Dashboard) handleAPIEnablePublish(w http.ResponseWriter, r *http.Request) {
|
|
feedURL := r.URL.Query().Get("url")
|
|
account := r.URL.Query().Get("account")
|
|
if feedURL == "" {
|
|
http.Error(w, "url parameter required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
feedURL = shared.NormalizeURL(feedURL)
|
|
|
|
// Auto-derive account handle if not provided
|
|
if account == "" {
|
|
account = shared.DeriveHandleFromFeed(feedURL)
|
|
if account == "" {
|
|
http.Error(w, "could not derive account handle from URL", http.StatusBadRequest)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check feed exists
|
|
feed, err := d.getFeed(feedURL)
|
|
if err != nil {
|
|
http.Error(w, "feed not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
if feed == nil {
|
|
http.Error(w, "feed not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
if err := d.SetPublishStatus(feedURL, "pass", account); err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
// Get unpublished count
|
|
count, _ := d.GetUnpublishedItemCount(feedURL)
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": "pass",
|
|
"url": feedURL,
|
|
"account": account,
|
|
"unpublished_items": count,
|
|
"note": "PDS account must be created via publisher service",
|
|
})
|
|
}
|
|
|
|
// handleAPIDeriveHandle shows what handle would be derived from a feed URL
|
|
func (d *Dashboard) handleAPIDeriveHandle(w http.ResponseWriter, r *http.Request) {
|
|
feedURL := r.URL.Query().Get("url")
|
|
if feedURL == "" {
|
|
http.Error(w, "url parameter required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
handle := shared.DeriveHandleFromFeed(feedURL)
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"url": feedURL,
|
|
"handle": handle,
|
|
})
|
|
}
|
|
|
|
// handleAPIDisablePublish sets a feed's publish status to 'skip'
|
|
func (d *Dashboard) handleAPIDisablePublish(w http.ResponseWriter, r *http.Request) {
|
|
feedURL := r.URL.Query().Get("url")
|
|
if feedURL == "" {
|
|
http.Error(w, "url parameter required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
feedURL = shared.NormalizeURL(feedURL)
|
|
|
|
if err := d.SetPublishStatus(feedURL, "skip", ""); err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": "skip",
|
|
"url": feedURL,
|
|
})
|
|
}
|
|
|
|
// handleAPIPublishEnabled returns all feeds with publish status 'pass'
|
|
func (d *Dashboard) handleAPIPublishEnabled(w http.ResponseWriter, r *http.Request) {
|
|
feeds, err := d.GetFeedsByPublishStatus("pass")
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
type FeedPublishInfo struct {
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
Account string `json:"account"`
|
|
UnpublishedCount int `json:"unpublished_count"`
|
|
}
|
|
|
|
var result []FeedPublishInfo
|
|
for _, f := range feeds {
|
|
count, _ := d.GetUnpublishedItemCount(f.URL)
|
|
result = append(result, FeedPublishInfo{
|
|
URL: f.URL,
|
|
Title: f.Title,
|
|
Account: f.PublishAccount,
|
|
UnpublishedCount: count,
|
|
})
|
|
}
|
|
|
|
if result == nil {
|
|
result = []FeedPublishInfo{}
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(result)
|
|
}
|
|
|
|
// handleAPIPublishDenied returns all feeds with publish status 'skip'
|
|
func (d *Dashboard) handleAPIPublishDenied(w http.ResponseWriter, r *http.Request) {
|
|
feeds, err := d.GetFeedsByPublishStatus("skip")
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
type FeedDeniedInfo struct {
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
SourceHost string `json:"source_host"`
|
|
}
|
|
|
|
var result []FeedDeniedInfo
|
|
for _, f := range feeds {
|
|
result = append(result, FeedDeniedInfo{
|
|
URL: f.URL,
|
|
Title: f.Title,
|
|
SourceHost: f.DomainHost,
|
|
})
|
|
}
|
|
|
|
if result == nil {
|
|
result = []FeedDeniedInfo{}
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(result)
|
|
}
|
|
|
|
// handleAPIPublishCandidates returns feeds pending review that have items
|
|
func (d *Dashboard) handleAPIPublishCandidates(w http.ResponseWriter, r *http.Request) {
|
|
limit := 50
|
|
if l := r.URL.Query().Get("limit"); l != "" {
|
|
fmt.Sscanf(l, "%d", &limit)
|
|
if limit > 200 {
|
|
limit = 200
|
|
}
|
|
}
|
|
|
|
feeds, err := d.GetPublishCandidates(limit)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
type CandidateInfo struct {
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
Category string `json:"category"`
|
|
SourceHost string `json:"source_host"`
|
|
ItemCount int `json:"item_count"`
|
|
DerivedHandle string `json:"derived_handle"`
|
|
}
|
|
|
|
var result []CandidateInfo
|
|
for _, f := range feeds {
|
|
result = append(result, CandidateInfo{
|
|
URL: f.URL,
|
|
Title: f.Title,
|
|
Category: f.Category,
|
|
SourceHost: f.DomainHost,
|
|
ItemCount: f.ItemCount,
|
|
DerivedHandle: shared.DeriveHandleFromFeed(f.URL),
|
|
})
|
|
}
|
|
|
|
if result == nil {
|
|
result = []CandidateInfo{}
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(result)
|
|
}
|
|
|
|
// handleAPISetPublishStatus sets the publish status for a feed (database only)
|
|
func (d *Dashboard) handleAPISetPublishStatus(w http.ResponseWriter, r *http.Request) {
|
|
feedURL := r.URL.Query().Get("url")
|
|
status := r.URL.Query().Get("status")
|
|
account := r.URL.Query().Get("account")
|
|
|
|
if feedURL == "" {
|
|
http.Error(w, "url parameter required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if status != "pass" && status != "skip" && status != "hold" {
|
|
http.Error(w, "status must be 'pass', 'hold', or 'skip' (use publisher service for 'drop')", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
feedURL = shared.NormalizeURL(feedURL)
|
|
|
|
result := map[string]interface{}{
|
|
"url": feedURL,
|
|
"status": status,
|
|
}
|
|
|
|
// Handle 'pass' - set account
|
|
if status == "pass" {
|
|
if account == "" {
|
|
account = shared.DeriveHandleFromFeed(feedURL)
|
|
}
|
|
result["account"] = account
|
|
result["note"] = "PDS account must be created via publisher service"
|
|
}
|
|
|
|
// Handle 'hold' and 'skip' - preserve current account
|
|
if status == "hold" || status == "skip" {
|
|
feed, _ := d.getFeed(feedURL)
|
|
if feed != nil {
|
|
account = feed.PublishAccount
|
|
}
|
|
}
|
|
|
|
if err := d.SetPublishStatus(feedURL, status, account); err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
result["account"] = account
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(result)
|
|
}
|
|
|
|
// handleAPIUnpublishedItems returns unpublished items for a feed
|
|
func (d *Dashboard) handleAPIUnpublishedItems(w http.ResponseWriter, r *http.Request) {
|
|
feedURL := r.URL.Query().Get("url")
|
|
if feedURL == "" {
|
|
http.Error(w, "url parameter required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
limit := 50
|
|
if l := r.URL.Query().Get("limit"); l != "" {
|
|
fmt.Sscanf(l, "%d", &limit)
|
|
if limit > 200 {
|
|
limit = 200
|
|
}
|
|
}
|
|
|
|
items, err := d.GetUnpublishedItems(feedURL, limit)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
if items == nil {
|
|
items = []*shared.Item{}
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(items)
|
|
}
|
|
|
|
// The following handlers require PDS interaction and should use the publisher service
|
|
|
|
func (d *Dashboard) handleAPITestPublish(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "Publishing requires the publisher service. This endpoint is not available in standalone dashboard.", http.StatusNotImplemented)
|
|
}
|
|
|
|
func (d *Dashboard) handleAPIPublishFeed(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "Publishing requires the publisher service. This endpoint is not available in standalone dashboard.", http.StatusNotImplemented)
|
|
}
|
|
|
|
func (d *Dashboard) handleAPICreateAccount(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "Account creation requires the publisher service. This endpoint is not available in standalone dashboard.", http.StatusNotImplemented)
|
|
}
|
|
|
|
func (d *Dashboard) handleAPIPublishFeedFull(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "Publishing requires the publisher service. This endpoint is not available in standalone dashboard.", http.StatusNotImplemented)
|
|
}
|
|
|
|
func (d *Dashboard) handleAPIUpdateProfile(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "Profile updates require the publisher service. This endpoint is not available in standalone dashboard.", http.StatusNotImplemented)
|
|
}
|
|
|
|
func (d *Dashboard) handleAPIResetAllPublishing(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "This destructive operation requires the publisher service. This endpoint is not available in standalone dashboard.", http.StatusNotImplemented)
|
|
}
|
|
|
|
func (d *Dashboard) handleAPIRefreshProfiles(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "Profile refresh requires the publisher service. This endpoint is not available in standalone dashboard.", http.StatusNotImplemented)
|
|
}
|