crawler/templates.go

package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"strings"
	"time"
)

// PDSAccount represents a Bluesky account on the PDS
type PDSAccount struct {
	DID         string `json:"did"`
	Handle      string `json:"handle"`
	DisplayName string `json:"displayName"`
	Description string `json:"description"`
	Avatar      string `json:"avatar"`
}

// handleAccountsDirectory serves the 1440.news accounts directory page
func (c *Crawler) handleAccountsDirectory(w http.ResponseWriter, r *http.Request) {
	pdsHost := os.Getenv("PDS_HOST")
	if pdsHost == "" {
		pdsHost = "https://pds.1440.news"
	}

	// Fetch all repos from PDS
	listReposURL := pdsHost + "/xrpc/com.atproto.sync.listRepos?limit=1000"
	resp, err := http.Get(listReposURL)
	if err != nil {
		http.Error(w, "Failed to fetch accounts: "+err.Error(), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	body, _ := io.ReadAll(resp.Body)
	var reposResp struct {
		Repos []struct {
			DID    string `json:"did"`
			Head   string `json:"head"`
			Active bool   `json:"active"`
		} `json:"repos"`
	}
	if err := json.Unmarshal(body, &reposResp); err != nil {
		http.Error(w, "Failed to parse repos: "+err.Error(), http.StatusInternalServerError)
		return
	}

	// Fetch profile for each account using unauthenticated endpoints
	var accounts []PDSAccount
	client := &http.Client{Timeout: 5 * time.Second}

	for _, repo := range reposResp.Repos {
		if !repo.Active {
			continue
		}

		// Get handle using describeRepo
		describeURL := pdsHost + "/xrpc/com.atproto.repo.describeRepo?repo=" + repo.DID
		describeResp, err := client.Get(describeURL)
		if err != nil {
			continue
		}
		describeBody, _ := io.ReadAll(describeResp.Body)
		describeResp.Body.Close()

		var repoInfo struct {
			Handle string `json:"handle"`
			DID    string `json:"did"`
		}
		if err := json.Unmarshal(describeBody, &repoInfo); err != nil {
			continue
		}

		// Skip the main 1440.news account (directory account itself)
		if repoInfo.Handle == "1440.news" {
			continue
		}

		account := PDSAccount{
			DID:    repoInfo.DID,
			Handle: repoInfo.Handle,
		}

		// Get profile record for display name, description, avatar
		recordURL := pdsHost + "/xrpc/com.atproto.repo.getRecord?repo=" + repo.DID + "&collection=app.bsky.actor.profile&rkey=self"
		recordResp, err := client.Get(recordURL)
		if err == nil {
			recordBody, _ := io.ReadAll(recordResp.Body)
			recordResp.Body.Close()

			var record struct {
				Value struct {
					DisplayName string `json:"displayName"`
					Description string `json:"description"`
					Avatar      struct {
						Ref struct {
							Link string `json:"$link"`
						} `json:"ref"`
					} `json:"avatar"`
				} `json:"value"`
			}
			if json.Unmarshal(recordBody, &record) == nil {
				account.DisplayName = record.Value.DisplayName
				account.Description = record.Value.Description
				if record.Value.Avatar.Ref.Link != "" {
					account.Avatar = pdsHost + "/xrpc/com.atproto.sync.getBlob?did=" + repo.DID + "&cid=" + record.Value.Avatar.Ref.Link
				}
			}
		}

		accounts = append(accounts, PDSAccount{
			DID:         account.DID,
			Handle:      account.Handle,
			DisplayName: account.DisplayName,
			Description: account.Description,
			Avatar:      account.Avatar,
		})
	}

	// Render the page
	tmpl := template.Must(template.New("accounts").Parse(accountsDirectoryHTML))
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	tmpl.Execute(w, map[string]interface{}{
		"Accounts": accounts,
		"Count":    len(accounts),
	})
}

func (c *Crawler) handleDashboard(w http.ResponseWriter, r *http.Request) {
	stats, err := c.GetDashboardStats()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	funcMap := template.FuncMap{
		"pct": func(a, b int) float64 {
			if b == 0 {
				return 0
			}
			return float64(a) * 100.0 / float64(b)
		},
		"comma": func(n interface{}) string {
			var val int
			switch v := n.(type) {
			case int:
				val = v
			case int32:
				val = int(v)
			case int64:
				val = int(v)
			default:
				return "0"
			}
			if val < 0 {
				return "-" + commaFormat(-val)
			}
			return commaFormat(val)
		},
	}

	tmpl, err := template.New("dashboard").Funcs(funcMap).Parse(dashboardHTML)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "text/html")
	tmpl.Execute(w, stats)
}

func (c *Crawler) handleAPIStats(w http.ResponseWriter, r *http.Request) {
	stats, err := c.GetDashboardStats()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(stats)
}

// handleRedirect handles short URL redirects for url.1440.news
func (c *Crawler) handleRedirect(w http.ResponseWriter, r *http.Request) {
	code := strings.TrimPrefix(r.URL.Path, "/")
	if code == "" {
		http.NotFound(w, r)
		return
	}

	// Look up the short URL
	shortURL, err := c.GetShortURL(code)
	if err != nil {
		http.NotFound(w, r)
		return
	}

	// Record the click asynchronously
	go func() {
		if err := c.RecordClick(code, r); err != nil {
			fmt.Printf("Failed to record click for %s: %v\n", code, err)
		}
	}()

	// Redirect to original URL
	http.Redirect(w, r, shortURL.OriginalURL, http.StatusFound)
}

const accountsDirectoryHTML = `<!DOCTYPE html>
<html>
<head>
    <title>1440.news - News Feed Directory</title>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <style>
        * { box-sizing: border-box; margin: 0; padding: 0; }
        body {
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
            background: #0a0a0a;
            color: #e0e0e0;
            min-height: 100vh;
            padding: 20px;
        }
        .container {
            max-width: 800px;
            margin: 0 auto;
        }
        header {
            text-align: center;
            margin-bottom: 40px;
            padding-bottom: 20px;
            border-bottom: 1px solid #333;
        }
        h1 {
            font-size: 2.5em;
            color: #fff;
            margin-bottom: 10px;
        }
        .tagline {
            color: #888;
            font-size: 1.1em;
        }
        .count {
            color: #0af;
            margin-top: 10px;
        }
        .accounts {
            display: flex;
            flex-direction: column;
            gap: 15px;
        }
        .account {
            display: flex;
            align-items: center;
            gap: 15px;
            padding: 15px;
            background: #151515;
            border: 1px solid #252525;
            border-radius: 12px;
            transition: all 0.2s;
        }
        .account:hover {
            border-color: #0af;
            background: #1a1a1a;
        }
        .avatar {
            width: 60px;
            height: 60px;
            border-radius: 50%;
            background: #333;
            flex-shrink: 0;
            object-fit: cover;
        }
        .avatar-placeholder {
            width: 60px;
            height: 60px;
            border-radius: 50%;
            background: linear-gradient(135deg, #0af, #08f);
            flex-shrink: 0;
            display: flex;
            align-items: center;
            justify-content: center;
            font-size: 24px;
            color: #fff;
            font-weight: bold;
        }
        .info {
            flex: 1;
            min-width: 0;
        }
        .display-name {
            font-size: 1.1em;
            font-weight: 600;
            color: #fff;
            margin-bottom: 2px;
        }
        .handle {
            color: #0af;
            font-size: 0.9em;
            margin-bottom: 5px;
        }
        .description {
            color: #888;
            font-size: 0.85em;
            overflow: hidden;
            text-overflow: ellipsis;
            white-space: nowrap;
        }
        .follow-btn {
            padding: 8px 20px;
            background: #0af;
            color: #000;
            border: none;
            border-radius: 20px;
            font-weight: 600;
            cursor: pointer;
            text-decoration: none;
            flex-shrink: 0;
            transition: all 0.2s;
        }
        .follow-btn:hover {
            background: #0cf;
            transform: scale(1.05);
        }
        footer {
            margin-top: 40px;
            padding-top: 20px;
            border-top: 1px solid #333;
            text-align: center;
            color: #666;
            font-size: 0.9em;
        }
        footer a {
            color: #0af;
            text-decoration: none;
        }
        .search-box {
            margin-top: 20px;
        }
        .search-input {
            width: 100%;
            padding: 12px 20px;
            font-size: 1em;
            background: #151515;
            border: 1px solid #333;
            border-radius: 25px;
            color: #fff;
            outline: none;
            transition: border-color 0.2s;
        }
        .search-input:focus {
            border-color: #0af;
        }
        .search-input::placeholder {
            color: #666;
        }
        .no-results {
            text-align: center;
            color: #666;
            padding: 40px;
            display: none;
        }
        .account.hidden {
            display: none;
        }
        @media (max-width: 600px) {
            .account {
                flex-wrap: wrap;
            }
            .follow-btn {
                width: 100%;
                text-align: center;
                margin-top: 10px;
            }
        }
    </style>
</head>
<body>
    <div class="container">
        <header>
            <h1>1440.news</h1>
            <p class="tagline">Curated news feeds on Bluesky</p>
            <p class="count"><span id="visibleCount">{{.Count}}</span> feeds available</p>
            <div class="search-box">
                <input type="text" class="search-input" id="searchInput" placeholder="Search feeds..." autocomplete="off">
            </div>
        </header>

        <div class="accounts" id="accountsList">
            {{range .Accounts}}
            <div class="account">
                {{if .Avatar}}
                <img class="avatar" src="{{.Avatar}}" alt="{{.DisplayName}}">
                {{else}}
                <div class="avatar-placeholder">{{slice .Handle 0 1}}</div>
                {{end}}
                <div class="info">
                    <div class="display-name">{{if .DisplayName}}{{.DisplayName}}{{else}}{{.Handle}}{{end}}</div>
                    <div class="handle">@{{.Handle}}</div>
                    {{if .Description}}<div class="description">{{.Description}}</div>{{end}}
                </div>
                <a class="follow-btn" href="https://bsky.app/profile/{{.Handle}}" target="_blank">View</a>
            </div>
            {{else}}
            <p style="text-align: center; color: #666;">No feeds available yet.</p>
            {{end}}
        </div>
        <p class="no-results" id="noResults">No feeds match your search.</p>

        <footer>
            <p>Follow <a href="https://bsky.app/profile/1440.news" target="_blank">@1440.news</a> for updates</p>
        </footer>
    </div>
    <script>
        document.getElementById('searchInput').addEventListener('input', function() {
            const query = this.value.toLowerCase().trim();
            const accounts = document.querySelectorAll('.account');
            let visibleCount = 0;

            accounts.forEach(function(account) {
                const text = account.textContent.toLowerCase();
                if (query === '' || text.includes(query)) {
                    account.classList.remove('hidden');
                    visibleCount++;
                } else {
                    account.classList.add('hidden');
                }
            });

            document.getElementById('visibleCount').textContent = visibleCount;
            document.getElementById('noResults').style.display = visibleCount === 0 && query !== '' ? 'block' : 'none';
        });
    </script>
</body>
</html>
`

const dashboardHTML = `<!DOCTYPE html>
<html>
<head>
    <title>1440.news Feed Crawler</title>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <link rel="stylesheet" href="/static/dashboard.css?v=1770011013">
    <script src="/static/dashboard.js?v=1770011013"></script>
</head>
<body>
    <div id="topSection">
    <h2>Domains</h2>
    <div class="grid grid-narrow">
        <div class="card clickable" data-filter="domain" data-status="all">
            <div class="stat-value" id="totalDomains">{{comma .TotalDomains}}</div>
            <div class="stat-label">All</div>
        </div>
        <div class="card clickable" data-filter="domain" data-status="pass">
            <div class="stat-value" id="passDomains" style="color: #0f0;">{{comma .PassDomains}}</div>
            <div class="stat-label">Pass</div>
        </div>
        <div class="card clickable" data-filter="domain" data-status="skip">
            <div class="stat-value" id="skipDomains" style="color: #f66;">{{comma .SkipDomains}}</div>
            <div class="stat-label">Skip</div>
        </div>
        <div class="card clickable" data-filter="domain" data-status="hold">
            <div class="stat-value" id="holdDomains" style="color: #f90;">{{comma .HoldDomains}}</div>
            <div class="stat-label">Hold</div>
        </div>
        <div class="card clickable" data-filter="domain" data-status="dead">
            <div class="stat-value" id="deadDomains" style="color: #888;">{{comma .DeadDomains}}</div>
            <div class="stat-label">Dead</div>
        </div>
        <div class="card">
            <div class="stat-value" id="domainCheckRate">{{comma .DomainCheckRate}}</div>
            <div class="stat-label">alive/min</div>
        </div>
        <div class="card">
            <div class="stat-value" id="feedCrawlRate">{{comma .FeedCrawlRate}}</div>
            <div class="stat-label">crawl/min</div>
        </div>
        <div class="card">
            <div class="stat-value" id="feedCheckRate">{{comma .FeedCheckRate}}</div>
            <div class="stat-label">check/min</div>
        </div>
    </div>

    <h2>Feeds</h2>
    <div class="grid grid-narrow">
        <div class="card clickable" data-filter="feed" data-status="all">
            <div class="stat-value" id="totalFeeds">{{comma .TotalFeeds}}</div>
            <div class="stat-label">All</div>
        </div>
        <div class="card clickable" data-filter="feed" data-status="alive">
            <div class="stat-value" id="aliveFeeds" style="color: #0f0;">{{comma .AliveFeeds}}</div>
            <div class="stat-label">Alive</div>
        </div>
        <div class="card clickable" data-filter="feed" data-status="publish">
            <div class="stat-value" id="publishFeeds" style="color: #0ff;">{{comma .PublishFeeds}}</div>
            <div class="stat-label">Pass</div>
        </div>
        <div class="card clickable" data-filter="feed" data-status="skip">
            <div class="stat-value" id="skipFeeds" style="color: #f66;">{{comma .SkipFeeds}}</div>
            <div class="stat-label">Skip</div>
        </div>
        <div class="card clickable" data-filter="feed" data-status="hold">
            <div class="stat-value" id="holdFeeds" style="color: #f90;">{{comma .HoldFeeds}}</div>
            <div class="stat-label">Hold</div>
        </div>
        <div class="card clickable" data-filter="feed" data-status="dead">
            <div class="stat-value" id="deadFeeds" style="color: #888;">{{comma .DeadFeeds}}</div>
            <div class="stat-label">Dead</div>
        </div>
        <div class="card clickable" data-filter="feed" data-type="empty">
            <div class="stat-value" id="emptyFeeds" style="color: #555;">{{comma .EmptyFeeds}}</div>
            <div class="stat-label">Empty</div>
        </div>
        <div class="card clickable" data-filter="feed" data-type="rss">
            <div class="stat-value" style="color: #f90" id="rssFeeds">{{comma .RSSFeeds}}</div>
            <div class="stat-label">RSS</div>
        </div>
        <div class="card clickable" data-filter="feed" data-type="atom">
            <div class="stat-value" style="color: #09f" id="atomFeeds">{{comma .AtomFeeds}}</div>
            <div class="stat-label">Atom</div>
        </div>
        <div class="card clickable" data-filter="feed" data-type="json">
            <div class="stat-value" style="color: #0a0" id="jsonFeeds">{{comma .JSONFeeds}}</div>
            <div class="stat-label">JSON</div>
        </div>
        <div class="card clickable" data-filter="feed" data-type="unknown">
            <div class="stat-value" style="color: #666" id="unknownFeeds">{{comma .UnknownFeeds}}</div>
            <div class="stat-label">Unknown</div>
        </div>
    </div>

    <div class="card" id="inputCard" style="background: #0a0a0a; display: flex; gap: 10px;">
        <button id="clearBtn" class="cmd-btn" style="padding: 12px 16px; margin: 0; background: #1a1a1a;" title="Clear search">✕</button>
        <input type="text" id="searchInput" placeholder="Search domains and feeds..."
               style="flex: 1; padding: 12px; background: #0a0a0a; border: 1px solid #333; border-radius: 4px; color: #fff;">
        <button id="searchBtn" class="cmd-btn" style="padding: 12px 16px; margin: 0; font-size: 18px;" title="Search">↵</button>
    </div>
    </div>
    <div id="topSectionSpacer"></div>

    <div class="card" id="outputCard">
        <div id="output"></div>
    </div>

    <div class="updated" id="updatedAt">Last updated: {{.UpdatedAt.Format "2006-01-02 15:04:05"}}</div>
</body>
</html>`