Skip bare TLDs during domain import

Domains without a dot (e.g., "com", "net", "org") are bare TLDs
from Common Crawl data and should not be processed as domains.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
primal
2026-01-29 22:28:09 -05:00
parent 1066f42189
commit edce82f1af
+4
View File
@@ -33,6 +33,10 @@ func shouldAutoSkipDomain(host string) bool {
if strings.HasSuffix(host, "1440.news") || host == "1440.news" {
return false
}
// Skip bare TLDs (no dot means it's just "com", "net", etc.)
if !strings.Contains(host, ".") {
return true
}
// Skip domains starting with a digit (spam pattern)
if len(host) > 0 && host[0] >= '0' && host[0] <= '9' {
return true