Add JSON Feed support

- Detect JSON Feed format (jsonfeed.org) via version field
- Parse JSON Feed metadata and items
- Support application/feed+json MIME type for feed discovery
- Include "json" as valid feed type (not auto-denied)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
primal
2026-01-29 13:16:50 -05:00
parent 798f79bfe9
commit ad78c1a4c0
3 changed files with 158 additions and 4 deletions
+137
View File
@@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"encoding/xml"
"fmt"
"regexp"
@@ -500,3 +501,139 @@ func extractImgTags(html string) []string {
return urls
}
// JSON Feed structs (https://jsonfeed.org/version/1.1)
type JSONFeed struct {
Version string `json:"version"`
Title string `json:"title"`
HomePageURL string `json:"home_page_url"`
FeedURL string `json:"feed_url"`
Description string `json:"description"`
Language string `json:"language"`
Items []JSONFeedItem `json:"items"`
}
type JSONFeedItem struct {
ID string `json:"id"`
URL string `json:"url"`
Title string `json:"title"`
ContentHTML string `json:"content_html"`
ContentText string `json:"content_text"`
Summary string `json:"summary"`
Image string `json:"image"`
DatePublished string `json:"date_published"`
DateModified string `json:"date_modified"`
Authors []JSONFeedAuthor `json:"authors"`
Attachments []JSONFeedAttachment `json:"attachments"`
}
type JSONFeedAuthor struct {
Name string `json:"name"`
URL string `json:"url"`
}
type JSONFeedAttachment struct {
URL string `json:"url"`
MimeType string `json:"mime_type"`
Size int64 `json:"size_in_bytes"`
}
func (c *Crawler) parseJSONFeedMetadata(body string, feed *Feed) []*Item {
var jf JSONFeed
if err := json.Unmarshal([]byte(body), &jf); err != nil {
return nil
}
feed.Title = jf.Title
feed.Description = jf.Description
feed.Language = jf.Language
feed.SiteURL = normalizeURL(jf.HomePageURL)
feed.ItemCount = len(jf.Items)
// Parse items
now := time.Now()
var items []*Item
var dates []time.Time
for _, ji := range jf.Items {
item := &Item{
FeedURL: feed.URL,
Title: ji.Title,
Link: ji.URL,
DiscoveredAt: now,
}
// Use ID as GUID, fall back to URL
if ji.ID != "" {
item.GUID = ji.ID
} else if ji.URL != "" {
item.GUID = ji.URL
}
// Content: prefer HTML, fall back to text
if ji.ContentHTML != "" {
item.Content = ji.ContentHTML
} else if ji.ContentText != "" {
item.Content = ji.ContentText
}
item.Description = ji.Summary
// Author
if len(ji.Authors) > 0 {
item.Author = ji.Authors[0].Name
}
// Parse date
dateStr := ji.DatePublished
if dateStr == "" {
dateStr = ji.DateModified
}
if dateStr != "" {
if t, err := time.Parse(time.RFC3339, dateStr); err == nil {
item.PubDate = t
dates = append(dates, t)
}
}
// Images
if ji.Image != "" {
item.ImageURLs = []string{ji.Image}
}
// Attachments (enclosures)
for _, att := range ji.Attachments {
if att.URL != "" {
item.Enclosure = &Enclosure{
URL: att.URL,
Type: att.MimeType,
Length: att.Size,
}
break // Only use first attachment as enclosure
}
}
items = append(items, item)
}
// Calculate date stats
if len(dates) > 0 {
oldest, newest := dates[0], dates[0]
for _, d := range dates {
if d.Before(oldest) {
oldest = d
}
if d.After(newest) {
newest = d
}
}
feed.OldestItemDate = oldest
feed.NewestItemDate = newest
if len(dates) > 1 {
totalHours := newest.Sub(oldest).Hours()
feed.AvgPostFreqHrs = totalHours / float64(len(dates)-1)
}
}
return items
}