Skip bare TLDs during domain import
Domains without a dot (e.g., "com", "net", "org") are bare TLDs from Common Crawl data and should not be processed as domains. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,10 @@ func shouldAutoSkipDomain(host string) bool {
|
||||
if strings.HasSuffix(host, "1440.news") || host == "1440.news" {
|
||||
return false
|
||||
}
|
||||
// Skip bare TLDs (no dot means it's just "com", "net", etc.)
|
||||
if !strings.Contains(host, ".") {
|
||||
return true
|
||||
}
|
||||
// Skip domains starting with a digit (spam pattern)
|
||||
if len(host) > 0 && host[0] >= '0' && host[0] <= '9' {
|
||||
return true
|
||||
|
||||
Reference in New Issue
Block a user