primal/crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Auto-deny all-digit domains, whitelist 1440.news

Browse Source 
- Deny domains where hostname is all digits (e.g., 0000114.com)
- Never auto-deny 1440.news or subdomains
- Auto-pass feeds from 1440.news sources
- Updated 554,085 domains and 3,213 feeds in database

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
primal
2026-01-29 13:27:48 -05:00
parent 897ae66e81
commit 2386d551fc
2 changed files with 31 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
feed.go
+4 -1
View File
@@ -179,9 +179,12 @@ type Feed struct {
func (c *Crawler) saveFeed(feed *Feed) error {
// Default publishStatus to "held" if not set
// Auto-deny feeds with no language or unsupported type
// Auto-pass feeds from our own domain
publishStatus := feed.PublishStatus
if publishStatus == "" {
if feed.Language == "" {
if strings.HasSuffix(feed.SourceHost, "1440.news") || feed.SourceHost == "1440.news" {
publishStatus = "pass"
} else if feed.Language == "" {
publishStatus = "deny"
} else if feed.Type != "rss" && feed.Type != "atom" && feed.Type != "json" {
publishStatus = "deny"