v100

db.go

@@ -27,8 +27,7 @@ CREATE TABLE IF NOT EXISTS domains (
 CREATE INDEX IF NOT EXISTS idx_domains_status ON domains(status);
 CREATE INDEX IF NOT EXISTS idx_domains_tld ON domains(tld);
 CREATE INDEX IF NOT EXISTS idx_domains_feeds_found ON domains(feeds_found DESC) WHERE feeds_found > 0;
-CREATE INDEX IF NOT EXISTS idx_domains_to_check ON domains(status) WHERE last_checked_at IS NULL;
-CREATE INDEX IF NOT EXISTS idx_domains_to_crawl ON domains(status) WHERE last_checked_at IS NOT NULL AND last_crawled_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_domains_to_crawl ON domains(status) WHERE last_crawled_at IS NULL;
 
 CREATE TABLE IF NOT EXISTS feeds (
 	url TEXT PRIMARY KEY,
@@ -47,12 +46,7 @@ CREATE TABLE IF NOT EXISTS feeds (
 	etag TEXT,
 	last_modified TEXT,
 
-	ttl_minutes INTEGER,
-	update_period TEXT,
-	update_freq INTEGER,
-
-	status TEXT DEFAULT 'active',
-	error_count INTEGER DEFAULT 0,
+	status TEXT DEFAULT 'pass' CHECK(status IN ('hold', 'pass', 'skip')),
 	last_error TEXT,
 	last_error_at TIMESTAMPTZ,
 
@@ -61,7 +55,6 @@ CREATE TABLE IF NOT EXISTS feeds (
 	tld TEXT,
 
 	item_count INTEGER,
-	avg_post_freq_hrs DOUBLE PRECISION,
 	oldest_item_date TIMESTAMPTZ,
 	newest_item_date TIMESTAMPTZ,
 
@@ -90,6 +83,7 @@ CREATE INDEX IF NOT EXISTS idx_feeds_status ON feeds(status);
 CREATE INDEX IF NOT EXISTS idx_feeds_discovered_at ON feeds(discovered_at);
 CREATE INDEX IF NOT EXISTS idx_feeds_title ON feeds(title);
 CREATE INDEX IF NOT EXISTS idx_feeds_search ON feeds USING GIN(search_vector);
+CREATE INDEX IF NOT EXISTS idx_feeds_due_check ON feeds(next_crawl_at, no_update DESC) WHERE status = 'pass';
 
 CREATE TABLE IF NOT EXISTS items (
 	id BIGSERIAL PRIMARY KEY,