package main import ( "regexp" "strings" "golang.org/x/net/html" ) // simpleFeed is a lightweight feed reference used during HTML extraction type simpleFeed struct { URL string Type string } func (c *Crawler) isFeedContent(body, contentType string) bool { if strings.Contains(contentType, "application/rss+xml") || strings.Contains(contentType, "application/atom+xml") || strings.Contains(contentType, "application/xml") || strings.Contains(contentType, "text/xml") || strings.Contains(contentType, "application/feed+json") || strings.Contains(contentType, "application/json") { return true } body = strings.TrimSpace(body) if strings.HasPrefix(body, "