Add AT Protocol publishing, media support, and SQLite stability
Publishing: - Add publisher.go for posting feed items to AT Protocol PDS - Support deterministic rkeys from SHA256(guid + discoveredAt) - Handle multiple URLs in posts with facets for each link - Image embed support (app.bsky.embed.images) for up to 4 images - External embed with thumbnail fallback - Podcast/audio enclosure URLs included in post text Media extraction: - Parse RSS enclosures (audio, video, images) - Extract Media RSS content and thumbnails - Extract images from HTML content in descriptions - Store enclosure and imageUrls in items table SQLite stability improvements: - Add synchronous=NORMAL and wal_autocheckpoint pragmas - Connection pool tuning (idle conns, max lifetime) - Periodic WAL checkpoint every 5 minutes - Hourly integrity checks with PRAGMA quick_check - Daily hot backup via VACUUM INTO - Docker stop_grace_period: 30s for graceful shutdown Dashboard: - Feed publishing UI and API endpoints - Account creation with invite codes Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
@@ -25,6 +26,7 @@ CREATE INDEX IF NOT EXISTS idx_domains_feedsFound ON domains(feedsFound DESC) WH
|
||||
CREATE TABLE IF NOT EXISTS feeds (
|
||||
url TEXT PRIMARY KEY,
|
||||
type TEXT,
|
||||
category TEXT DEFAULT 'main',
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
language TEXT,
|
||||
@@ -56,14 +58,20 @@ CREATE TABLE IF NOT EXISTS feeds (
|
||||
oldestItemDate DATETIME,
|
||||
newestItemDate DATETIME,
|
||||
|
||||
noUpdate INTEGER DEFAULT 0
|
||||
noUpdate INTEGER DEFAULT 0,
|
||||
|
||||
-- Publishing to PDS
|
||||
publishStatus TEXT DEFAULT 'held' CHECK(publishStatus IN ('held', 'pass', 'fail')),
|
||||
publishAccount TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_sourceHost ON feeds(sourceHost);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_publishStatus ON feeds(publishStatus);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_sourceHost_url ON feeds(sourceHost, url);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_tld ON feeds(tld);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_tld_sourceHost ON feeds(tld, sourceHost);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_type ON feeds(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_category ON feeds(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_status ON feeds(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_discoveredAt ON feeds(discoveredAt);
|
||||
CREATE INDEX IF NOT EXISTS idx_feeds_title ON feeds(title);
|
||||
@@ -80,6 +88,17 @@ CREATE TABLE IF NOT EXISTS items (
|
||||
pubDate DATETIME,
|
||||
discoveredAt DATETIME NOT NULL,
|
||||
updatedAt DATETIME,
|
||||
|
||||
-- Media attachments
|
||||
enclosureUrl TEXT,
|
||||
enclosureType TEXT,
|
||||
enclosureLength INTEGER,
|
||||
imageUrls TEXT, -- JSON array of image URLs
|
||||
|
||||
-- Publishing to PDS
|
||||
publishedAt DATETIME,
|
||||
publishedUri TEXT,
|
||||
|
||||
UNIQUE(feedUrl, guid)
|
||||
);
|
||||
|
||||
@@ -87,6 +106,7 @@ CREATE INDEX IF NOT EXISTS idx_items_feedUrl ON items(feedUrl);
|
||||
CREATE INDEX IF NOT EXISTS idx_items_pubDate ON items(pubDate DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_items_link ON items(link);
|
||||
CREATE INDEX IF NOT EXISTS idx_items_feedUrl_pubDate ON items(feedUrl, pubDate DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_items_unpublished ON items(feedUrl, publishedAt) WHERE publishedAt IS NULL;
|
||||
|
||||
-- Full-text search for feeds
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS feeds_fts USING fts5(
|
||||
@@ -148,15 +168,22 @@ func OpenDatabase(dbPath string) (*sql.DB, error) {
|
||||
fmt.Printf("Opening database: %s\n", dbPath)
|
||||
|
||||
// Use pragmas in connection string for consistent application
|
||||
connStr := dbPath + "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)"
|
||||
// - busy_timeout: wait up to 10s for locks instead of failing immediately
|
||||
// - journal_mode: WAL for better concurrency and crash recovery
|
||||
// - synchronous: NORMAL is safe with WAL (fsync at checkpoint, not every commit)
|
||||
// - wal_autocheckpoint: checkpoint every 1000 pages (~4MB) to prevent WAL bloat
|
||||
// - foreign_keys: enforce referential integrity
|
||||
connStr := dbPath + "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=wal_autocheckpoint(1000)&_pragma=foreign_keys(ON)"
|
||||
db, err := sql.Open("sqlite", connStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open database: %v", err)
|
||||
}
|
||||
|
||||
// Allow multiple readers (WAL mode supports concurrent reads)
|
||||
// SQLite is single-writer, but reads can happen concurrently
|
||||
db.SetMaxOpenConns(4)
|
||||
// Connection pool settings for stability
|
||||
db.SetMaxOpenConns(4) // Limit concurrent connections
|
||||
db.SetMaxIdleConns(2) // Keep some connections warm
|
||||
db.SetConnMaxLifetime(5 * time.Minute) // Recycle connections periodically
|
||||
db.SetConnMaxIdleTime(1 * time.Minute) // Close idle connections
|
||||
|
||||
// Verify connection and show journal mode
|
||||
var journalMode string
|
||||
@@ -173,6 +200,17 @@ func OpenDatabase(dbPath string) (*sql.DB, error) {
|
||||
}
|
||||
fmt.Println(" Schema OK")
|
||||
|
||||
// Migrations for existing databases
|
||||
migrations := []string{
|
||||
"ALTER TABLE items ADD COLUMN enclosureUrl TEXT",
|
||||
"ALTER TABLE items ADD COLUMN enclosureType TEXT",
|
||||
"ALTER TABLE items ADD COLUMN enclosureLength INTEGER",
|
||||
"ALTER TABLE items ADD COLUMN imageUrls TEXT",
|
||||
}
|
||||
for _, m := range migrations {
|
||||
db.Exec(m) // Ignore errors (column may already exist)
|
||||
}
|
||||
|
||||
// Run stats and ANALYZE in background to avoid blocking startup with large databases
|
||||
go func() {
|
||||
var domainCount, feedCount int
|
||||
|
||||
Reference in New Issue
Block a user