553 lines
18 KiB
Go
553 lines
18 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"html/template"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// PDSAccount represents a Bluesky account on the PDS
|
|
type PDSAccount struct {
|
|
DID string `json:"did"`
|
|
Handle string `json:"handle"`
|
|
DisplayName string `json:"displayName"`
|
|
Description string `json:"description"`
|
|
Avatar string `json:"avatar"`
|
|
}
|
|
|
|
// handleAccountsDirectory serves the 1440.news accounts directory page
|
|
func (c *Crawler) handleAccountsDirectory(w http.ResponseWriter, r *http.Request) {
|
|
pdsHost := os.Getenv("PDS_HOST")
|
|
if pdsHost == "" {
|
|
pdsHost = "https://pds.1440.news"
|
|
}
|
|
|
|
// Fetch all repos from PDS
|
|
listReposURL := pdsHost + "/xrpc/com.atproto.sync.listRepos?limit=1000"
|
|
resp, err := http.Get(listReposURL)
|
|
if err != nil {
|
|
http.Error(w, "Failed to fetch accounts: "+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, _ := io.ReadAll(resp.Body)
|
|
var reposResp struct {
|
|
Repos []struct {
|
|
DID string `json:"did"`
|
|
Head string `json:"head"`
|
|
Active bool `json:"active"`
|
|
} `json:"repos"`
|
|
}
|
|
if err := json.Unmarshal(body, &reposResp); err != nil {
|
|
http.Error(w, "Failed to parse repos: "+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
// Fetch profile for each account using unauthenticated endpoints
|
|
var accounts []PDSAccount
|
|
client := &http.Client{Timeout: 5 * time.Second}
|
|
|
|
for _, repo := range reposResp.Repos {
|
|
if !repo.Active {
|
|
continue
|
|
}
|
|
|
|
// Get handle using describeRepo
|
|
describeURL := pdsHost + "/xrpc/com.atproto.repo.describeRepo?repo=" + repo.DID
|
|
describeResp, err := client.Get(describeURL)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
describeBody, _ := io.ReadAll(describeResp.Body)
|
|
describeResp.Body.Close()
|
|
|
|
var repoInfo struct {
|
|
Handle string `json:"handle"`
|
|
DID string `json:"did"`
|
|
}
|
|
if err := json.Unmarshal(describeBody, &repoInfo); err != nil {
|
|
continue
|
|
}
|
|
|
|
// Skip the main 1440.news account (directory account itself)
|
|
if repoInfo.Handle == "1440.news" {
|
|
continue
|
|
}
|
|
|
|
account := PDSAccount{
|
|
DID: repoInfo.DID,
|
|
Handle: repoInfo.Handle,
|
|
}
|
|
|
|
// Get profile record for display name, description, avatar
|
|
recordURL := pdsHost + "/xrpc/com.atproto.repo.getRecord?repo=" + repo.DID + "&collection=app.bsky.actor.profile&rkey=self"
|
|
recordResp, err := client.Get(recordURL)
|
|
if err == nil {
|
|
recordBody, _ := io.ReadAll(recordResp.Body)
|
|
recordResp.Body.Close()
|
|
|
|
var record struct {
|
|
Value struct {
|
|
DisplayName string `json:"displayName"`
|
|
Description string `json:"description"`
|
|
Avatar struct {
|
|
Ref struct {
|
|
Link string `json:"$link"`
|
|
} `json:"ref"`
|
|
} `json:"avatar"`
|
|
} `json:"value"`
|
|
}
|
|
if json.Unmarshal(recordBody, &record) == nil {
|
|
account.DisplayName = record.Value.DisplayName
|
|
account.Description = record.Value.Description
|
|
if record.Value.Avatar.Ref.Link != "" {
|
|
account.Avatar = pdsHost + "/xrpc/com.atproto.sync.getBlob?did=" + repo.DID + "&cid=" + record.Value.Avatar.Ref.Link
|
|
}
|
|
}
|
|
}
|
|
|
|
accounts = append(accounts, PDSAccount{
|
|
DID: account.DID,
|
|
Handle: account.Handle,
|
|
DisplayName: account.DisplayName,
|
|
Description: account.Description,
|
|
Avatar: account.Avatar,
|
|
})
|
|
}
|
|
|
|
// Render the page
|
|
tmpl := template.Must(template.New("accounts").Parse(accountsDirectoryHTML))
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
tmpl.Execute(w, map[string]interface{}{
|
|
"Accounts": accounts,
|
|
"Count": len(accounts),
|
|
})
|
|
}
|
|
|
|
func (c *Crawler) handleDashboard(w http.ResponseWriter, r *http.Request) {
|
|
stats, err := c.GetDashboardStats()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
funcMap := template.FuncMap{
|
|
"pct": func(a, b int) float64 {
|
|
if b == 0 {
|
|
return 0
|
|
}
|
|
return float64(a) * 100.0 / float64(b)
|
|
},
|
|
"comma": func(n interface{}) string {
|
|
var val int
|
|
switch v := n.(type) {
|
|
case int:
|
|
val = v
|
|
case int32:
|
|
val = int(v)
|
|
case int64:
|
|
val = int(v)
|
|
default:
|
|
return "0"
|
|
}
|
|
if val < 0 {
|
|
return "-" + commaFormat(-val)
|
|
}
|
|
return commaFormat(val)
|
|
},
|
|
}
|
|
|
|
tmpl, err := template.New("dashboard").Funcs(funcMap).Parse(dashboardHTML)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "text/html")
|
|
tmpl.Execute(w, stats)
|
|
}
|
|
|
|
func (c *Crawler) handleAPIStats(w http.ResponseWriter, r *http.Request) {
|
|
stats, err := c.GetDashboardStats()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(stats)
|
|
}
|
|
|
|
// handleRedirect handles short URL redirects for url.1440.news
|
|
func (c *Crawler) handleRedirect(w http.ResponseWriter, r *http.Request) {
|
|
code := strings.TrimPrefix(r.URL.Path, "/")
|
|
if code == "" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
// Look up the short URL
|
|
shortURL, err := c.GetShortURL(code)
|
|
if err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
// Record the click asynchronously
|
|
go func() {
|
|
if err := c.RecordClick(code, r); err != nil {
|
|
fmt.Printf("Failed to record click for %s: %v\n", code, err)
|
|
}
|
|
}()
|
|
|
|
// Redirect to original URL
|
|
http.Redirect(w, r, shortURL.OriginalURL, http.StatusFound)
|
|
}
|
|
|
|
const accountsDirectoryHTML = `<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>1440.news - News Feed Directory</title>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<style>
|
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
min-height: 100vh;
|
|
padding: 20px;
|
|
}
|
|
.container {
|
|
max-width: 800px;
|
|
margin: 0 auto;
|
|
}
|
|
header {
|
|
text-align: center;
|
|
margin-bottom: 40px;
|
|
padding-bottom: 20px;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
h1 {
|
|
font-size: 2.5em;
|
|
color: #fff;
|
|
margin-bottom: 10px;
|
|
}
|
|
.tagline {
|
|
color: #888;
|
|
font-size: 1.1em;
|
|
}
|
|
.count {
|
|
color: #0af;
|
|
margin-top: 10px;
|
|
}
|
|
.accounts {
|
|
display: flex;
|
|
flex-direction: column;
|
|
gap: 15px;
|
|
}
|
|
.account {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 15px;
|
|
padding: 15px;
|
|
background: #151515;
|
|
border: 1px solid #252525;
|
|
border-radius: 12px;
|
|
transition: all 0.2s;
|
|
}
|
|
.account:hover {
|
|
border-color: #0af;
|
|
background: #1a1a1a;
|
|
}
|
|
.avatar {
|
|
width: 60px;
|
|
height: 60px;
|
|
border-radius: 50%;
|
|
background: #333;
|
|
flex-shrink: 0;
|
|
object-fit: cover;
|
|
}
|
|
.avatar-placeholder {
|
|
width: 60px;
|
|
height: 60px;
|
|
border-radius: 50%;
|
|
background: linear-gradient(135deg, #0af, #08f);
|
|
flex-shrink: 0;
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
font-size: 24px;
|
|
color: #fff;
|
|
font-weight: bold;
|
|
}
|
|
.info {
|
|
flex: 1;
|
|
min-width: 0;
|
|
}
|
|
.display-name {
|
|
font-size: 1.1em;
|
|
font-weight: 600;
|
|
color: #fff;
|
|
margin-bottom: 2px;
|
|
}
|
|
.handle {
|
|
color: #0af;
|
|
font-size: 0.9em;
|
|
margin-bottom: 5px;
|
|
}
|
|
.description {
|
|
color: #888;
|
|
font-size: 0.85em;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
.follow-btn {
|
|
padding: 8px 20px;
|
|
background: #0af;
|
|
color: #000;
|
|
border: none;
|
|
border-radius: 20px;
|
|
font-weight: 600;
|
|
cursor: pointer;
|
|
text-decoration: none;
|
|
flex-shrink: 0;
|
|
transition: all 0.2s;
|
|
}
|
|
.follow-btn:hover {
|
|
background: #0cf;
|
|
transform: scale(1.05);
|
|
}
|
|
footer {
|
|
margin-top: 40px;
|
|
padding-top: 20px;
|
|
border-top: 1px solid #333;
|
|
text-align: center;
|
|
color: #666;
|
|
font-size: 0.9em;
|
|
}
|
|
footer a {
|
|
color: #0af;
|
|
text-decoration: none;
|
|
}
|
|
.search-box {
|
|
margin-top: 20px;
|
|
}
|
|
.search-input {
|
|
width: 100%;
|
|
padding: 12px 20px;
|
|
font-size: 1em;
|
|
background: #151515;
|
|
border: 1px solid #333;
|
|
border-radius: 25px;
|
|
color: #fff;
|
|
outline: none;
|
|
transition: border-color 0.2s;
|
|
}
|
|
.search-input:focus {
|
|
border-color: #0af;
|
|
}
|
|
.search-input::placeholder {
|
|
color: #666;
|
|
}
|
|
.no-results {
|
|
text-align: center;
|
|
color: #666;
|
|
padding: 40px;
|
|
display: none;
|
|
}
|
|
.account.hidden {
|
|
display: none;
|
|
}
|
|
@media (max-width: 600px) {
|
|
.account {
|
|
flex-wrap: wrap;
|
|
}
|
|
.follow-btn {
|
|
width: 100%;
|
|
text-align: center;
|
|
margin-top: 10px;
|
|
}
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<header>
|
|
<h1>1440.news</h1>
|
|
<p class="tagline">Curated news feeds on Bluesky</p>
|
|
<p class="count"><span id="visibleCount">{{.Count}}</span> feeds available</p>
|
|
<div class="search-box">
|
|
<input type="text" class="search-input" id="searchInput" placeholder="Search feeds..." autocomplete="off">
|
|
</div>
|
|
</header>
|
|
|
|
<div class="accounts" id="accountsList">
|
|
{{range .Accounts}}
|
|
<div class="account">
|
|
{{if .Avatar}}
|
|
<img class="avatar" src="{{.Avatar}}" alt="{{.DisplayName}}">
|
|
{{else}}
|
|
<div class="avatar-placeholder">{{slice .Handle 0 1}}</div>
|
|
{{end}}
|
|
<div class="info">
|
|
<div class="display-name">{{if .DisplayName}}{{.DisplayName}}{{else}}{{.Handle}}{{end}}</div>
|
|
<div class="handle">@{{.Handle}}</div>
|
|
{{if .Description}}<div class="description">{{.Description}}</div>{{end}}
|
|
</div>
|
|
<a class="follow-btn" href="https://bsky.app/profile/{{.Handle}}" target="_blank">View</a>
|
|
</div>
|
|
{{else}}
|
|
<p style="text-align: center; color: #666;">No feeds available yet.</p>
|
|
{{end}}
|
|
</div>
|
|
<p class="no-results" id="noResults">No feeds match your search.</p>
|
|
|
|
<footer>
|
|
<p>Follow <a href="https://bsky.app/profile/1440.news" target="_blank">@1440.news</a> for updates</p>
|
|
</footer>
|
|
</div>
|
|
<script>
|
|
document.getElementById('searchInput').addEventListener('input', function() {
|
|
const query = this.value.toLowerCase().trim();
|
|
const accounts = document.querySelectorAll('.account');
|
|
let visibleCount = 0;
|
|
|
|
accounts.forEach(function(account) {
|
|
const text = account.textContent.toLowerCase();
|
|
if (query === '' || text.includes(query)) {
|
|
account.classList.remove('hidden');
|
|
visibleCount++;
|
|
} else {
|
|
account.classList.add('hidden');
|
|
}
|
|
});
|
|
|
|
document.getElementById('visibleCount').textContent = visibleCount;
|
|
document.getElementById('noResults').style.display = visibleCount === 0 && query !== '' ? 'block' : 'none';
|
|
});
|
|
</script>
|
|
</body>
|
|
</html>
|
|
`
|
|
|
|
const dashboardHTML = `<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>1440.news Feed Crawler</title>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<link rel="stylesheet" href="/static/dashboard.css?v=222">
|
|
<script src="/static/dashboard.js?v=222"></script>
|
|
</head>
|
|
<body>
|
|
<div id="topSection">
|
|
<h2>Domains</h2>
|
|
<div class="grid grid-narrow">
|
|
<div class="card clickable" data-filter="domain" data-status="all">
|
|
<div class="stat-value" id="totalDomains">{{comma .TotalDomains}}</div>
|
|
<div class="stat-label">All</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="domain" data-status="pass">
|
|
<div class="stat-value" id="passDomains" style="color: #0f0;">{{comma .PassDomains}}</div>
|
|
<div class="stat-label">Pass</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="domain" data-status="skip">
|
|
<div class="stat-value" id="skipDomains" style="color: #f66;">{{comma .SkipDomains}}</div>
|
|
<div class="stat-label">Skip</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="domain" data-status="hold">
|
|
<div class="stat-value" id="holdDomains" style="color: #f90;">{{comma .HoldDomains}}</div>
|
|
<div class="stat-label">Hold</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="domain" data-status="dead">
|
|
<div class="stat-value" id="deadDomains" style="color: #888;">{{comma .DeadDomains}}</div>
|
|
<div class="stat-label">Dead</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="domainCheckRate">{{comma .DomainCheckRate}}</div>
|
|
<div class="stat-label">alive/min</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="feedCrawlRate">{{comma .FeedCrawlRate}}</div>
|
|
<div class="stat-label">crawl/min</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="feedCheckRate">{{comma .FeedCheckRate}}</div>
|
|
<div class="stat-label">check/min</div>
|
|
</div>
|
|
</div>
|
|
|
|
<h2>Feeds</h2>
|
|
<div class="grid grid-narrow">
|
|
<div class="card clickable" data-filter="feed" data-status="all">
|
|
<div class="stat-value" id="totalFeeds">{{comma .TotalFeeds}}</div>
|
|
<div class="stat-label">All</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-status="alive">
|
|
<div class="stat-value" id="aliveFeeds" style="color: #0f0;">{{comma .AliveFeeds}}</div>
|
|
<div class="stat-label">Alive</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-status="publish">
|
|
<div class="stat-value" id="publishFeeds" style="color: #0ff;">{{comma .PublishFeeds}}</div>
|
|
<div class="stat-label">Pass</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-status="skip">
|
|
<div class="stat-value" id="skipFeeds" style="color: #f66;">{{comma .SkipFeeds}}</div>
|
|
<div class="stat-label">Skip</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-status="hold">
|
|
<div class="stat-value" id="holdFeeds" style="color: #f90;">{{comma .HoldFeeds}}</div>
|
|
<div class="stat-label">Hold</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-status="dead">
|
|
<div class="stat-value" id="deadFeeds" style="color: #888;">{{comma .DeadFeeds}}</div>
|
|
<div class="stat-label">Dead</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-type="empty">
|
|
<div class="stat-value" id="emptyFeeds" style="color: #555;">{{comma .EmptyFeeds}}</div>
|
|
<div class="stat-label">Empty</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-type="rss">
|
|
<div class="stat-value" style="color: #f90" id="rssFeeds">{{comma .RSSFeeds}}</div>
|
|
<div class="stat-label">RSS</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-type="atom">
|
|
<div class="stat-value" style="color: #09f" id="atomFeeds">{{comma .AtomFeeds}}</div>
|
|
<div class="stat-label">Atom</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-type="json">
|
|
<div class="stat-value" style="color: #0a0" id="jsonFeeds">{{comma .JSONFeeds}}</div>
|
|
<div class="stat-label">JSON</div>
|
|
</div>
|
|
<div class="card clickable" data-filter="feed" data-type="unknown">
|
|
<div class="stat-value" style="color: #666" id="unknownFeeds">{{comma .UnknownFeeds}}</div>
|
|
<div class="stat-label">Unknown</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card" id="inputCard" style="background: #0a0a0a; display: flex; gap: 10px;">
|
|
<button id="clearBtn" class="cmd-btn" style="padding: 12px 16px; margin: 0; background: #1a1a1a;" title="Clear search">✕</button>
|
|
<input type="text" id="searchInput" placeholder="Search domains and feeds..."
|
|
style="flex: 1; padding: 12px; background: #0a0a0a; border: 1px solid #333; border-radius: 4px; color: #fff;">
|
|
<button id="searchBtn" class="cmd-btn" style="padding: 12px 16px; margin: 0; font-size: 18px;" title="Search">↵</button>
|
|
</div>
|
|
</div>
|
|
<div id="topSectionSpacer"></div>
|
|
|
|
<div class="card" id="outputCard">
|
|
<div id="output"></div>
|
|
</div>
|
|
|
|
<div class="updated" id="updatedAt">Last updated: {{.UpdatedAt.Format "2006-01-02 15:04:05"}}</div>
|
|
</body>
|
|
</html>`
|