Add AT Protocol publishing, media support, and SQLite stability

Publishing:
- Add publisher.go for posting feed items to AT Protocol PDS
- Support deterministic rkeys from SHA256(guid + discoveredAt)
- Handle multiple URLs in posts with facets for each link
- Image embed support (app.bsky.embed.images) for up to 4 images
- External embed with thumbnail fallback
- Podcast/audio enclosure URLs included in post text

Media extraction:
- Parse RSS enclosures (audio, video, images)
- Extract Media RSS content and thumbnails
- Extract images from HTML content in descriptions
- Store enclosure and imageUrls in items table

SQLite stability improvements:
- Add synchronous=NORMAL and wal_autocheckpoint pragmas
- Connection pool tuning (idle conns, max lifetime)
- Periodic WAL checkpoint every 5 minutes
- Hourly integrity checks with PRAGMA quick_check
- Daily hot backup via VACUUM INTO
- Docker stop_grace_period: 30s for graceful shutdown

Dashboard:
- Feed publishing UI and API endpoints
- Account creation with invite codes

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
primal
2026-01-28 15:30:02 -05:00
parent aa6f571215
commit 75835d771d
11 changed files with 3723 additions and 635 deletions
+43 -5
View File
@@ -3,6 +3,7 @@ package main
import (
"database/sql"
"fmt"
"time"
_ "modernc.org/sqlite"
)
@@ -25,6 +26,7 @@ CREATE INDEX IF NOT EXISTS idx_domains_feedsFound ON domains(feedsFound DESC) WH
CREATE TABLE IF NOT EXISTS feeds (
url TEXT PRIMARY KEY,
type TEXT,
category TEXT DEFAULT 'main',
title TEXT,
description TEXT,
language TEXT,
@@ -56,14 +58,20 @@ CREATE TABLE IF NOT EXISTS feeds (
oldestItemDate DATETIME,
newestItemDate DATETIME,
noUpdate INTEGER DEFAULT 0
noUpdate INTEGER DEFAULT 0,
-- Publishing to PDS
publishStatus TEXT DEFAULT 'held' CHECK(publishStatus IN ('held', 'pass', 'fail')),
publishAccount TEXT
);
CREATE INDEX IF NOT EXISTS idx_feeds_sourceHost ON feeds(sourceHost);
CREATE INDEX IF NOT EXISTS idx_feeds_publishStatus ON feeds(publishStatus);
CREATE INDEX IF NOT EXISTS idx_feeds_sourceHost_url ON feeds(sourceHost, url);
CREATE INDEX IF NOT EXISTS idx_feeds_tld ON feeds(tld);
CREATE INDEX IF NOT EXISTS idx_feeds_tld_sourceHost ON feeds(tld, sourceHost);
CREATE INDEX IF NOT EXISTS idx_feeds_type ON feeds(type);
CREATE INDEX IF NOT EXISTS idx_feeds_category ON feeds(category);
CREATE INDEX IF NOT EXISTS idx_feeds_status ON feeds(status);
CREATE INDEX IF NOT EXISTS idx_feeds_discoveredAt ON feeds(discoveredAt);
CREATE INDEX IF NOT EXISTS idx_feeds_title ON feeds(title);
@@ -80,6 +88,17 @@ CREATE TABLE IF NOT EXISTS items (
pubDate DATETIME,
discoveredAt DATETIME NOT NULL,
updatedAt DATETIME,
-- Media attachments
enclosureUrl TEXT,
enclosureType TEXT,
enclosureLength INTEGER,
imageUrls TEXT, -- JSON array of image URLs
-- Publishing to PDS
publishedAt DATETIME,
publishedUri TEXT,
UNIQUE(feedUrl, guid)
);
@@ -87,6 +106,7 @@ CREATE INDEX IF NOT EXISTS idx_items_feedUrl ON items(feedUrl);
CREATE INDEX IF NOT EXISTS idx_items_pubDate ON items(pubDate DESC);
CREATE INDEX IF NOT EXISTS idx_items_link ON items(link);
CREATE INDEX IF NOT EXISTS idx_items_feedUrl_pubDate ON items(feedUrl, pubDate DESC);
CREATE INDEX IF NOT EXISTS idx_items_unpublished ON items(feedUrl, publishedAt) WHERE publishedAt IS NULL;
-- Full-text search for feeds
CREATE VIRTUAL TABLE IF NOT EXISTS feeds_fts USING fts5(
@@ -148,15 +168,22 @@ func OpenDatabase(dbPath string) (*sql.DB, error) {
fmt.Printf("Opening database: %s\n", dbPath)
// Use pragmas in connection string for consistent application
connStr := dbPath + "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)"
// - busy_timeout: wait up to 10s for locks instead of failing immediately
// - journal_mode: WAL for better concurrency and crash recovery
// - synchronous: NORMAL is safe with WAL (fsync at checkpoint, not every commit)
// - wal_autocheckpoint: checkpoint every 1000 pages (~4MB) to prevent WAL bloat
// - foreign_keys: enforce referential integrity
connStr := dbPath + "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=wal_autocheckpoint(1000)&_pragma=foreign_keys(ON)"
db, err := sql.Open("sqlite", connStr)
if err != nil {
return nil, fmt.Errorf("failed to open database: %v", err)
}
// Allow multiple readers (WAL mode supports concurrent reads)
// SQLite is single-writer, but reads can happen concurrently
db.SetMaxOpenConns(4)
// Connection pool settings for stability
db.SetMaxOpenConns(4) // Limit concurrent connections
db.SetMaxIdleConns(2) // Keep some connections warm
db.SetConnMaxLifetime(5 * time.Minute) // Recycle connections periodically
db.SetConnMaxIdleTime(1 * time.Minute) // Close idle connections
// Verify connection and show journal mode
var journalMode string
@@ -173,6 +200,17 @@ func OpenDatabase(dbPath string) (*sql.DB, error) {
}
fmt.Println(" Schema OK")
// Migrations for existing databases
migrations := []string{
"ALTER TABLE items ADD COLUMN enclosureUrl TEXT",
"ALTER TABLE items ADD COLUMN enclosureType TEXT",
"ALTER TABLE items ADD COLUMN enclosureLength INTEGER",
"ALTER TABLE items ADD COLUMN imageUrls TEXT",
}
for _, m := range migrations {
db.Exec(m) // Ignore errors (column may already exist)
}
// Run stats and ANALYZE in background to avoid blocking startup with large databases
go func() {
var domainCount, feedCount int