package main import ( "bytes" "fmt" "image" _ "image/gif" "image/jpeg" _ "image/png" "io" "net/http" "net/url" "strings" "time" "go.deanishe.net/favicon" "golang.org/x/image/draw" _ "golang.org/x/image/webp" ) // ImageUploadResult contains the uploaded blob and image dimensions type ImageUploadResult struct { Blob *BlobRef Width int Height int } // FetchFavicon tries to get a favicon URL for a site func (p *Publisher) FetchFavicon(siteURL string) string { if siteURL == "" { return "" } if !strings.Contains(siteURL, "://") { siteURL = "https://" + siteURL } u, err := url.Parse(siteURL) if err != nil { return "" } finder := favicon.New( favicon.WithClient(p.httpClient), ) icons, err := finder.Find(siteURL) if err == nil && len(icons) > 0 { var bestIcon string var bestScore int for _, icon := range icons { if icon.Width > 0 && icon.Width < 32 { continue } lowerURL := strings.ToLower(icon.URL) if strings.Contains(lowerURL, "og-image") || strings.Contains(lowerURL, "og_image") || strings.Contains(lowerURL, "opengraph") || strings.Contains(lowerURL, "twitter") { continue } if icon.Width > 0 && icon.Height > 0 { ratio := float64(icon.Width) / float64(icon.Height) if ratio > 1.5 || ratio < 0.67 { continue } } score := 0 if strings.Contains(lowerURL, "favicon") || strings.Contains(lowerURL, "icon") || strings.Contains(lowerURL, "apple-touch") { score += 100 } if icon.MimeType == "image/png" { score += 50 } else if icon.MimeType == "image/x-icon" || strings.HasSuffix(lowerURL, ".ico") { score += 40 } else if icon.MimeType == "image/jpeg" { score += 10 } if icon.Width >= 64 && icon.Width <= 512 { score += 30 } else if icon.Width > 0 { score += 10 } if score > bestScore { bestScore = score bestIcon = icon.URL } } if bestIcon != "" { return bestIcon } for _, icon := range icons { lowerURL := strings.ToLower(icon.URL) if !strings.Contains(lowerURL, "og-image") && !strings.Contains(lowerURL, "og_image") { return icon.URL } } } return fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host) } func (p *Publisher) fetchAndUploadImage(session *PDSSession, imageURL string) *BlobRef { result := p.fetchAndUploadImageWithDimensions(session, imageURL) if result == nil { return nil } return result.Blob } func upgradeImageURL(imageURL string) string { if strings.Contains(imageURL, "ichef.bbci.co.uk") { imageURL = strings.Replace(imageURL, "/standard/240/", "/standard/800/", 1) imageURL = strings.Replace(imageURL, "/standard/480/", "/standard/800/", 1) } return imageURL } func (p *Publisher) fetchAndUploadImageWithDimensions(session *PDSSession, imageURL string) *ImageUploadResult { imageURL = upgradeImageURL(imageURL) resp, err := p.httpClient.Get(imageURL) if err != nil { return nil } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil } contentType := resp.Header.Get("Content-Type") if contentType == "" { if strings.HasSuffix(strings.ToLower(imageURL), ".png") { contentType = "image/png" } else if strings.HasSuffix(strings.ToLower(imageURL), ".gif") { contentType = "image/gif" } else if strings.HasSuffix(strings.ToLower(imageURL), ".webp") { contentType = "image/webp" } else { contentType = "image/jpeg" } } if !strings.HasPrefix(contentType, "image/") { return nil } data, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) if err != nil || len(data) == 0 { return nil } imgConfig, _, err := image.DecodeConfig(bytes.NewReader(data)) width, height := 1, 1 if err == nil { width, height = imgConfig.Width, imgConfig.Height } const maxBlobSize = 900 * 1024 if len(data) > maxBlobSize { img, _, err := image.Decode(bytes.NewReader(data)) if err != nil { return nil } scaleFactor := 0.9 for attempt := 0; attempt < 5; attempt++ { newWidth := int(float64(width) * scaleFactor) newHeight := int(float64(height) * scaleFactor) if newWidth < 100 { newWidth = 100 } if newHeight < 100 { newHeight = 100 } resized := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) draw.CatmullRom.Scale(resized, resized.Bounds(), img, img.Bounds(), draw.Over, nil) var buf bytes.Buffer if err := jpeg.Encode(&buf, resized, &jpeg.Options{Quality: 85}); err != nil { return nil } if buf.Len() <= maxBlobSize { data = buf.Bytes() width = newWidth height = newHeight contentType = "image/jpeg" break } scaleFactor *= 0.8 } if len(data) > maxBlobSize { return nil } } blob, err := p.UploadBlob(session, data, contentType) if err != nil { return nil } return &ImageUploadResult{ Blob: blob, Width: width, Height: height, } } // FetchFaviconBytes downloads a favicon/icon from a URL func FetchFaviconBytes(siteURL string) ([]byte, string, error) { if !strings.HasPrefix(siteURL, "http") { siteURL = "https://" + siteURL } u, err := url.Parse(siteURL) if err != nil { return nil, "", err } client := &http.Client{Timeout: 10 * time.Second} finder := favicon.New( favicon.WithClient(client), favicon.IgnoreNoSize, ) icons, err := finder.Find(siteURL) if err != nil || len(icons) == 0 { googleURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host) return fetchIconBytes(client, googleURL) } var iconURLs []string for _, icon := range icons { if icon.Width > 0 && icon.Width < 32 { continue } if icon.MimeType == "image/png" || icon.MimeType == "image/jpeg" { iconURLs = append([]string{icon.URL}, iconURLs...) } else { iconURLs = append(iconURLs, icon.URL) } } if len(iconURLs) == 0 { for _, icon := range icons { iconURLs = append(iconURLs, icon.URL) } } for _, iconURL := range iconURLs { data, mimeType, err := fetchIconBytes(client, iconURL) if err == nil && len(data) > 0 { return data, mimeType, nil } } googleURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", u.Host) return fetchIconBytes(client, googleURL) } func fetchIconBytes(client *http.Client, iconURL string) ([]byte, string, error) { resp, err := client.Get(iconURL) if err != nil { return nil, "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, "", fmt.Errorf("HTTP %d", resp.StatusCode) } data, err := io.ReadAll(resp.Body) if err != nil { return nil, "", err } contentType := resp.Header.Get("Content-Type") if contentType == "" { if strings.HasSuffix(iconURL, ".png") { contentType = "image/png" } else if strings.HasSuffix(iconURL, ".ico") { contentType = "image/x-icon" } else { contentType = "image/png" } } return data, contentType, nil }