542 lines
17 KiB
Go
542 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"html/template"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// PDSAccount represents a Bluesky account on the PDS
|
|
type PDSAccount struct {
|
|
DID string `json:"did"`
|
|
Handle string `json:"handle"`
|
|
DisplayName string `json:"displayName"`
|
|
Description string `json:"description"`
|
|
Avatar string `json:"avatar"`
|
|
}
|
|
|
|
// handleAccountsDirectory serves the 1440.news accounts directory page
|
|
func (c *Crawler) handleAccountsDirectory(w http.ResponseWriter, r *http.Request) {
|
|
pdsHost := os.Getenv("PDS_HOST")
|
|
if pdsHost == "" {
|
|
pdsHost = "https://pds.1440.news"
|
|
}
|
|
|
|
// Fetch all repos from PDS
|
|
listReposURL := pdsHost + "/xrpc/com.atproto.sync.listRepos?limit=1000"
|
|
resp, err := http.Get(listReposURL)
|
|
if err != nil {
|
|
http.Error(w, "Failed to fetch accounts: "+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, _ := io.ReadAll(resp.Body)
|
|
var reposResp struct {
|
|
Repos []struct {
|
|
DID string `json:"did"`
|
|
Head string `json:"head"`
|
|
Active bool `json:"active"`
|
|
} `json:"repos"`
|
|
}
|
|
if err := json.Unmarshal(body, &reposResp); err != nil {
|
|
http.Error(w, "Failed to parse repos: "+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
// Fetch profile for each account using unauthenticated endpoints
|
|
var accounts []PDSAccount
|
|
client := &http.Client{Timeout: 5 * time.Second}
|
|
|
|
for _, repo := range reposResp.Repos {
|
|
if !repo.Active {
|
|
continue
|
|
}
|
|
|
|
// Get handle using describeRepo
|
|
describeURL := pdsHost + "/xrpc/com.atproto.repo.describeRepo?repo=" + repo.DID
|
|
describeResp, err := client.Get(describeURL)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
describeBody, _ := io.ReadAll(describeResp.Body)
|
|
describeResp.Body.Close()
|
|
|
|
var repoInfo struct {
|
|
Handle string `json:"handle"`
|
|
DID string `json:"did"`
|
|
}
|
|
if err := json.Unmarshal(describeBody, &repoInfo); err != nil {
|
|
continue
|
|
}
|
|
|
|
// Skip the main 1440.news account (directory account itself)
|
|
if repoInfo.Handle == "1440.news" {
|
|
continue
|
|
}
|
|
|
|
account := PDSAccount{
|
|
DID: repoInfo.DID,
|
|
Handle: repoInfo.Handle,
|
|
}
|
|
|
|
// Get profile record for display name, description, avatar
|
|
recordURL := pdsHost + "/xrpc/com.atproto.repo.getRecord?repo=" + repo.DID + "&collection=app.bsky.actor.profile&rkey=self"
|
|
recordResp, err := client.Get(recordURL)
|
|
if err == nil {
|
|
recordBody, _ := io.ReadAll(recordResp.Body)
|
|
recordResp.Body.Close()
|
|
|
|
var record struct {
|
|
Value struct {
|
|
DisplayName string `json:"displayName"`
|
|
Description string `json:"description"`
|
|
Avatar struct {
|
|
Ref struct {
|
|
Link string `json:"$link"`
|
|
} `json:"ref"`
|
|
} `json:"avatar"`
|
|
} `json:"value"`
|
|
}
|
|
if json.Unmarshal(recordBody, &record) == nil {
|
|
account.DisplayName = record.Value.DisplayName
|
|
account.Description = record.Value.Description
|
|
if record.Value.Avatar.Ref.Link != "" {
|
|
account.Avatar = pdsHost + "/xrpc/com.atproto.sync.getBlob?did=" + repo.DID + "&cid=" + record.Value.Avatar.Ref.Link
|
|
}
|
|
}
|
|
}
|
|
|
|
accounts = append(accounts, PDSAccount{
|
|
DID: account.DID,
|
|
Handle: account.Handle,
|
|
DisplayName: account.DisplayName,
|
|
Description: account.Description,
|
|
Avatar: account.Avatar,
|
|
})
|
|
}
|
|
|
|
// Render the page
|
|
tmpl := template.Must(template.New("accounts").Parse(accountsDirectoryHTML))
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
tmpl.Execute(w, map[string]interface{}{
|
|
"Accounts": accounts,
|
|
"Count": len(accounts),
|
|
})
|
|
}
|
|
|
|
func (c *Crawler) handleDashboard(w http.ResponseWriter, r *http.Request) {
|
|
stats, err := c.GetDashboardStats()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
funcMap := template.FuncMap{
|
|
"pct": func(a, b int) float64 {
|
|
if b == 0 {
|
|
return 0
|
|
}
|
|
return float64(a) * 100.0 / float64(b)
|
|
},
|
|
"comma": func(n interface{}) string {
|
|
var val int
|
|
switch v := n.(type) {
|
|
case int:
|
|
val = v
|
|
case int32:
|
|
val = int(v)
|
|
case int64:
|
|
val = int(v)
|
|
default:
|
|
return "0"
|
|
}
|
|
if val < 0 {
|
|
return "-" + commaFormat(-val)
|
|
}
|
|
return commaFormat(val)
|
|
},
|
|
}
|
|
|
|
tmpl, err := template.New("dashboard").Funcs(funcMap).Parse(dashboardHTML)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "text/html")
|
|
tmpl.Execute(w, stats)
|
|
}
|
|
|
|
func (c *Crawler) handleAPIStats(w http.ResponseWriter, r *http.Request) {
|
|
stats, err := c.GetDashboardStats()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(stats)
|
|
}
|
|
|
|
// handleRedirect handles short URL redirects for url.1440.news
|
|
func (c *Crawler) handleRedirect(w http.ResponseWriter, r *http.Request) {
|
|
code := strings.TrimPrefix(r.URL.Path, "/")
|
|
if code == "" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
// Look up the short URL
|
|
shortURL, err := c.GetShortURL(code)
|
|
if err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
// Record the click asynchronously
|
|
go func() {
|
|
if err := c.RecordClick(code, r); err != nil {
|
|
fmt.Printf("Failed to record click for %s: %v\n", code, err)
|
|
}
|
|
}()
|
|
|
|
// Redirect to original URL
|
|
http.Redirect(w, r, shortURL.OriginalURL, http.StatusFound)
|
|
}
|
|
|
|
const accountsDirectoryHTML = `<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>1440.news - News Feed Directory</title>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<style>
|
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
min-height: 100vh;
|
|
padding: 20px;
|
|
}
|
|
.container {
|
|
max-width: 800px;
|
|
margin: 0 auto;
|
|
}
|
|
header {
|
|
text-align: center;
|
|
margin-bottom: 40px;
|
|
padding-bottom: 20px;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
h1 {
|
|
font-size: 2.5em;
|
|
color: #fff;
|
|
margin-bottom: 10px;
|
|
}
|
|
.tagline {
|
|
color: #888;
|
|
font-size: 1.1em;
|
|
}
|
|
.count {
|
|
color: #0af;
|
|
margin-top: 10px;
|
|
}
|
|
.accounts {
|
|
display: flex;
|
|
flex-direction: column;
|
|
gap: 15px;
|
|
}
|
|
.account {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 15px;
|
|
padding: 15px;
|
|
background: #151515;
|
|
border: 1px solid #252525;
|
|
border-radius: 12px;
|
|
transition: all 0.2s;
|
|
}
|
|
.account:hover {
|
|
border-color: #0af;
|
|
background: #1a1a1a;
|
|
}
|
|
.avatar {
|
|
width: 60px;
|
|
height: 60px;
|
|
border-radius: 50%;
|
|
background: #333;
|
|
flex-shrink: 0;
|
|
object-fit: cover;
|
|
}
|
|
.avatar-placeholder {
|
|
width: 60px;
|
|
height: 60px;
|
|
border-radius: 50%;
|
|
background: linear-gradient(135deg, #0af, #08f);
|
|
flex-shrink: 0;
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
font-size: 24px;
|
|
color: #fff;
|
|
font-weight: bold;
|
|
}
|
|
.info {
|
|
flex: 1;
|
|
min-width: 0;
|
|
}
|
|
.display-name {
|
|
font-size: 1.1em;
|
|
font-weight: 600;
|
|
color: #fff;
|
|
margin-bottom: 2px;
|
|
}
|
|
.handle {
|
|
color: #0af;
|
|
font-size: 0.9em;
|
|
margin-bottom: 5px;
|
|
}
|
|
.description {
|
|
color: #888;
|
|
font-size: 0.85em;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
.follow-btn {
|
|
padding: 8px 20px;
|
|
background: #0af;
|
|
color: #000;
|
|
border: none;
|
|
border-radius: 20px;
|
|
font-weight: 600;
|
|
cursor: pointer;
|
|
text-decoration: none;
|
|
flex-shrink: 0;
|
|
transition: all 0.2s;
|
|
}
|
|
.follow-btn:hover {
|
|
background: #0cf;
|
|
transform: scale(1.05);
|
|
}
|
|
footer {
|
|
margin-top: 40px;
|
|
padding-top: 20px;
|
|
border-top: 1px solid #333;
|
|
text-align: center;
|
|
color: #666;
|
|
font-size: 0.9em;
|
|
}
|
|
footer a {
|
|
color: #0af;
|
|
text-decoration: none;
|
|
}
|
|
.search-box {
|
|
margin-top: 20px;
|
|
}
|
|
.search-input {
|
|
width: 100%;
|
|
padding: 12px 20px;
|
|
font-size: 1em;
|
|
background: #151515;
|
|
border: 1px solid #333;
|
|
border-radius: 25px;
|
|
color: #fff;
|
|
outline: none;
|
|
transition: border-color 0.2s;
|
|
}
|
|
.search-input:focus {
|
|
border-color: #0af;
|
|
}
|
|
.search-input::placeholder {
|
|
color: #666;
|
|
}
|
|
.no-results {
|
|
text-align: center;
|
|
color: #666;
|
|
padding: 40px;
|
|
display: none;
|
|
}
|
|
.account.hidden {
|
|
display: none;
|
|
}
|
|
@media (max-width: 600px) {
|
|
.account {
|
|
flex-wrap: wrap;
|
|
}
|
|
.follow-btn {
|
|
width: 100%;
|
|
text-align: center;
|
|
margin-top: 10px;
|
|
}
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<header>
|
|
<h1>1440.news</h1>
|
|
<p class="tagline">Curated news feeds on Bluesky</p>
|
|
<p class="count"><span id="visibleCount">{{.Count}}</span> feeds available</p>
|
|
<div class="search-box">
|
|
<input type="text" class="search-input" id="searchInput" placeholder="Search feeds..." autocomplete="off">
|
|
</div>
|
|
</header>
|
|
|
|
<div class="accounts" id="accountsList">
|
|
{{range .Accounts}}
|
|
<div class="account">
|
|
{{if .Avatar}}
|
|
<img class="avatar" src="{{.Avatar}}" alt="{{.DisplayName}}">
|
|
{{else}}
|
|
<div class="avatar-placeholder">{{slice .Handle 0 1}}</div>
|
|
{{end}}
|
|
<div class="info">
|
|
<div class="display-name">{{if .DisplayName}}{{.DisplayName}}{{else}}{{.Handle}}{{end}}</div>
|
|
<div class="handle">@{{.Handle}}</div>
|
|
{{if .Description}}<div class="description">{{.Description}}</div>{{end}}
|
|
</div>
|
|
<a class="follow-btn" href="https://bsky.app/profile/{{.Handle}}" target="_blank">View</a>
|
|
</div>
|
|
{{else}}
|
|
<p style="text-align: center; color: #666;">No feeds available yet.</p>
|
|
{{end}}
|
|
</div>
|
|
<p class="no-results" id="noResults">No feeds match your search.</p>
|
|
|
|
<footer>
|
|
<p>Follow <a href="https://bsky.app/profile/1440.news" target="_blank">@1440.news</a> for updates</p>
|
|
</footer>
|
|
</div>
|
|
<script>
|
|
document.getElementById('searchInput').addEventListener('input', function() {
|
|
const query = this.value.toLowerCase().trim();
|
|
const accounts = document.querySelectorAll('.account');
|
|
let visibleCount = 0;
|
|
|
|
accounts.forEach(function(account) {
|
|
const text = account.textContent.toLowerCase();
|
|
if (query === '' || text.includes(query)) {
|
|
account.classList.remove('hidden');
|
|
visibleCount++;
|
|
} else {
|
|
account.classList.add('hidden');
|
|
}
|
|
});
|
|
|
|
document.getElementById('visibleCount').textContent = visibleCount;
|
|
document.getElementById('noResults').style.display = visibleCount === 0 && query !== '' ? 'block' : 'none';
|
|
});
|
|
</script>
|
|
</body>
|
|
</html>
|
|
`
|
|
|
|
const dashboardHTML = `<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>1440.news Feed Crawler</title>
|
|
<meta charset="utf-8">
|
|
<link rel="stylesheet" href="/static/dashboard.css">
|
|
<script src="/static/dashboard.js?v=37"></script>
|
|
</head>
|
|
<body>
|
|
<h1>1440.news Feed Crawler</h1>
|
|
|
|
<h2>Domain Status</h2>
|
|
<div class="grid">
|
|
<div class="card">
|
|
<div class="stat-value" id="totalDomains">{{comma .TotalDomains}}</div>
|
|
<div class="stat-label">Total</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="holdDomains" style="color: #f90;">{{comma .HoldDomains}}</div>
|
|
<div class="stat-label">Hold</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="passDomains" style="color: #0f0;">{{comma .PassDomains}}</div>
|
|
<div class="stat-label">Pass</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="skipDomains" style="color: #f66;">{{comma .SkipDomains}}</div>
|
|
<div class="stat-label">Skip</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="failDomains" style="color: #f00;">{{comma .FailDomains}}</div>
|
|
<div class="stat-label">Fail</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="crawlRate">{{comma .CrawlRate}}</div>
|
|
<div class="stat-label">crawls/min</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" id="checkRate">{{comma .CheckRate}}</div>
|
|
<div class="stat-label">checks/min</div>
|
|
</div>
|
|
</div>
|
|
|
|
<h2>Feeds Discovered</h2>
|
|
<div class="grid">
|
|
<div class="card">
|
|
<div class="stat-value" id="totalFeeds">{{comma .TotalFeeds}}</div>
|
|
<div class="stat-label">Total Feeds</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" style="color: #f90" id="rssFeeds">{{comma .RSSFeeds}}</div>
|
|
<div class="stat-label">RSS Feeds</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" style="color: #09f" id="atomFeeds">{{comma .AtomFeeds}}</div>
|
|
<div class="stat-label">Atom Feeds</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="stat-value" style="color: #666" id="unknownFeeds">{{comma .UnknownFeeds}}</div>
|
|
<div class="stat-label">Unknown Type</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card" id="inputCard">
|
|
<div id="commandButtons" style="margin-bottom: 10px;">
|
|
<button class="cmd-btn" data-cmd="/domains">domains</button>
|
|
<button class="cmd-btn" data-cmd="/feeds">feeds</button>
|
|
<button class="cmd-btn" id="tldToggleBtn">tlds</button>
|
|
<span style="color: #333; margin: 0 4px;">|</span>
|
|
<button class="cmd-btn" data-cmd="domains:hold">d:hold</button>
|
|
<button class="cmd-btn" data-cmd="domains:pass">d:pass</button>
|
|
<button class="cmd-btn" data-cmd="domains:skip">d:skip</button>
|
|
<button class="cmd-btn" data-cmd="domains:fail">d:fail</button>
|
|
<button class="cmd-btn" data-cmd="domains:feeds">d:feeds</button>
|
|
<span style="color: #333; margin: 0 4px;">|</span>
|
|
<button class="cmd-btn" data-cmd="feeds:hold">f:hold</button>
|
|
<button class="cmd-btn" data-cmd="feeds:pass">f:pass</button>
|
|
<button class="cmd-btn" data-cmd="feeds:skip">f:skip</button>
|
|
<button class="cmd-btn" data-cmd="feeds:fail">f:fail</button>
|
|
</div>
|
|
<div id="langDropdown" style="display: none; margin-bottom: 10px; padding: 10px; background: #0a0a0a; border: 1px solid #333; border-radius: 4px; max-height: 200px; overflow-y: auto;">
|
|
<div id="langList"></div>
|
|
</div>
|
|
<div id="tldDropdown" style="display: none; margin-bottom: 10px; padding: 10px; background: #0a0a0a; border: 1px solid #333; border-radius: 4px;">
|
|
<div id="tldList" style="display: flex; flex-wrap: wrap; gap: 6px;"></div>
|
|
</div>
|
|
<input type="text" id="commandInput" value="/help"
|
|
style="width: 100%; padding: 12px; background: #0a0a0a; border: 1px solid #333; border-radius: 4px; color: #fff; font-size: 14px; font-family: monospace;">
|
|
</div>
|
|
|
|
<div class="card" id="outputCard">
|
|
<div id="breadcrumb" style="margin-bottom: 10px; display: none;"></div>
|
|
<div id="output"></div>
|
|
</div>
|
|
|
|
<div style="color: #333; font-size: 11px; margin-top: 10px;">v56</div>
|
|
|
|
<div class="updated" id="updatedAt">Last updated: {{.UpdatedAt.Format "2006-01-02 15:04:05"}}</div>
|
|
</body>
|
|
</html>`
|