520 lines
24 KiB
JavaScript
520 lines
24 KiB
JavaScript
function initDashboard() {
|
|
function commaFormat(n) {
|
|
return n.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
|
|
}
|
|
|
|
function escapeHtml(text) {
|
|
if (text == null) return '';
|
|
const div = document.createElement('div');
|
|
div.textContent = text;
|
|
return div.innerHTML;
|
|
}
|
|
|
|
// All domains state
|
|
let allDomainsOffset = 0;
|
|
let allDomainsLoading = false;
|
|
let allDomainsEnd = false;
|
|
let expandedDomain = null;
|
|
let expandedFeed = null;
|
|
const PAGE_SIZE = 100;
|
|
const PREFETCH_THRESHOLD = 100; // Prefetch when within 100 domains of bottom
|
|
|
|
// Search state
|
|
let searchTimeout = null;
|
|
let isSearching = false;
|
|
|
|
async function loadMoreDomains() {
|
|
if (allDomainsLoading || allDomainsEnd) return;
|
|
|
|
allDomainsLoading = true;
|
|
const loadingEl = document.getElementById('allDomainsLoading');
|
|
loadingEl.style.display = 'block';
|
|
|
|
try {
|
|
const response = await fetch('/api/allDomains?offset=' + allDomainsOffset + '&limit=' + PAGE_SIZE);
|
|
const domains = await response.json();
|
|
|
|
if (!domains || domains.length === 0) {
|
|
allDomainsEnd = true;
|
|
loadingEl.style.display = 'none';
|
|
return;
|
|
}
|
|
|
|
const container = document.getElementById('allDomains');
|
|
domains.forEach(d => {
|
|
const row = document.createElement('div');
|
|
row.className = 'domain-row';
|
|
row.innerHTML =
|
|
'<div class="stat-row" style="cursor: pointer;">' +
|
|
'<span>' + escapeHtml(d.host) + '</span>' +
|
|
'<span>' + commaFormat(d.feeds_found) + '</span>' +
|
|
'</div>' +
|
|
'<div class="domain-feeds" style="display: none;"></div>';
|
|
|
|
row.querySelector('.stat-row').addEventListener('click', () => toggleDomainFeeds(d.host, row));
|
|
container.appendChild(row);
|
|
});
|
|
|
|
allDomainsOffset += domains.length;
|
|
loadingEl.style.display = 'none';
|
|
|
|
// If we got fewer than PAGE_SIZE, we've reached the end
|
|
if (domains.length < PAGE_SIZE) {
|
|
allDomainsEnd = true;
|
|
}
|
|
} catch (err) {
|
|
console.error('Failed to load domains:', err);
|
|
} finally {
|
|
allDomainsLoading = false;
|
|
}
|
|
}
|
|
|
|
async function toggleDomainFeeds(host, rowEl) {
|
|
const feedsDiv = rowEl.querySelector('.domain-feeds');
|
|
|
|
// Close previously expanded domain
|
|
if (expandedDomain && expandedDomain !== rowEl) {
|
|
expandedDomain.querySelector('.domain-feeds').style.display = 'none';
|
|
}
|
|
|
|
// Toggle current
|
|
if (feedsDiv.style.display === 'none') {
|
|
feedsDiv.style.display = 'block';
|
|
feedsDiv.innerHTML = '<div style="padding: 10px; color: #666;">Loading feeds...</div>';
|
|
expandedDomain = rowEl;
|
|
|
|
try {
|
|
const response = await fetch('/api/domainFeeds?host=' + encodeURIComponent(host));
|
|
const feeds = await response.json();
|
|
|
|
if (!feeds || feeds.length === 0) {
|
|
feedsDiv.innerHTML = '<div style="padding: 10px; color: #666;">No feeds found</div>';
|
|
} else {
|
|
feedsDiv.innerHTML = '';
|
|
feeds.forEach(f => {
|
|
const feedItem = document.createElement('div');
|
|
feedItem.className = 'feed-item';
|
|
feedItem.style.cssText = 'padding: 5px 10px; border-top: 1px solid #333; cursor: pointer;';
|
|
feedItem.innerHTML =
|
|
'<div class="feed-header">' +
|
|
'<div style="color: #0af;">' + escapeHtml(f.url) + '</div>' +
|
|
(f.title ? '<div style="color: #888; font-size: 0.9em;">' + escapeHtml(f.title) + '</div>' : '') +
|
|
'<div style="color: #666; font-size: 0.8em;">' + (f.type || 'unknown') + '</div>' +
|
|
'</div>' +
|
|
'<div class="feed-details" style="display: none;"></div>';
|
|
|
|
feedItem.querySelector('.feed-header').addEventListener('click', (e) => {
|
|
e.stopPropagation();
|
|
toggleFeedInfo(f.url, feedItem);
|
|
});
|
|
feedsDiv.appendChild(feedItem);
|
|
});
|
|
}
|
|
} catch (err) {
|
|
feedsDiv.innerHTML = '<div style="padding: 10px; color: #f66;">Error loading feeds</div>';
|
|
}
|
|
} else {
|
|
feedsDiv.style.display = 'none';
|
|
expandedDomain = null;
|
|
}
|
|
}
|
|
|
|
async function toggleFeedInfo(feedUrl, feedItemEl) {
|
|
const detailsDiv = feedItemEl.querySelector('.feed-details');
|
|
|
|
// Close previously expanded feed
|
|
if (expandedFeed && expandedFeed !== feedItemEl) {
|
|
expandedFeed.querySelector('.feed-details').style.display = 'none';
|
|
}
|
|
|
|
// Toggle current
|
|
if (detailsDiv.style.display === 'none') {
|
|
detailsDiv.style.display = 'block';
|
|
detailsDiv.innerHTML = '<div style="padding: 10px; color: #666;">Loading feed info...</div>';
|
|
expandedFeed = feedItemEl;
|
|
|
|
// Scroll the feed item to the top of the viewport
|
|
feedItemEl.scrollIntoView({ behavior: 'smooth', block: 'start' });
|
|
|
|
try {
|
|
// Fetch feed info and items in parallel
|
|
const [infoResponse, itemsResponse] = await Promise.all([
|
|
fetch('/api/feedInfo?url=' + encodeURIComponent(feedUrl)),
|
|
fetch('/api/feedItems?url=' + encodeURIComponent(feedUrl) + '&limit=50')
|
|
]);
|
|
const info = await infoResponse.json();
|
|
const items = await itemsResponse.json();
|
|
|
|
let html = '<div style="padding: 10px; background: #1a1a1a; margin-top: 5px; border-radius: 4px; font-size: 0.85em;">';
|
|
|
|
if (info.description) {
|
|
html += '<div style="margin-bottom: 8px; color: #aaa;">' + escapeHtml(info.description) + '</div>';
|
|
}
|
|
|
|
html += '<table style="width: 100%; color: #888;">';
|
|
|
|
if (info.siteUrl) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Site</td><td>' + escapeHtml(info.siteUrl) + '</td></tr>';
|
|
}
|
|
if (info.language) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Language</td><td>' + escapeHtml(info.language) + '</td></tr>';
|
|
}
|
|
if (info.status) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Status</td><td>' + escapeHtml(info.status) + '</td></tr>';
|
|
}
|
|
if (info.itemCount) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Items</td><td>' + commaFormat(info.itemCount) + '</td></tr>';
|
|
}
|
|
if (info.avgPostFreqHrs) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Avg Post Freq</td><td>' + info.avgPostFreqHrs.toFixed(1) + ' hrs</td></tr>';
|
|
}
|
|
if (info.ttlMinutes) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">TTL</td><td>' + info.ttlMinutes + ' min</td></tr>';
|
|
}
|
|
if (info.updatePeriod) {
|
|
let updateStr = info.updatePeriod;
|
|
if (info.updateFreq) updateStr += ' (' + info.updateFreq + ')';
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Update</td><td>' + escapeHtml(updateStr) + '</td></tr>';
|
|
}
|
|
if (info.lastBuildDate) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Last Build</td><td>' + escapeHtml(info.lastBuildDate) + '</td></tr>';
|
|
}
|
|
if (info.newestItemDate) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Newest Item</td><td>' + escapeHtml(info.newestItemDate) + '</td></tr>';
|
|
}
|
|
if (info.oldestItemDate) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Oldest Item</td><td>' + escapeHtml(info.oldestItemDate) + '</td></tr>';
|
|
}
|
|
if (info.discoveredAt) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Discovered</td><td>' + escapeHtml(info.discoveredAt) + '</td></tr>';
|
|
}
|
|
if (info.lastCrawledAt) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Last Crawled</td><td>' + escapeHtml(info.lastCrawledAt) + '</td></tr>';
|
|
}
|
|
if (info.errorCount > 0) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Errors</td><td style="color: #f66;">' + info.errorCount + '</td></tr>';
|
|
}
|
|
if (info.lastError) {
|
|
html += '<tr><td style="padding: 2px 8px 2px 0; color: #666;">Last Error</td><td style="color: #f66;">' + escapeHtml(info.lastError) + '</td></tr>';
|
|
}
|
|
|
|
html += '</table>';
|
|
|
|
// Display items
|
|
if (items && items.length > 0) {
|
|
html += '<div style="margin-top: 12px; border-top: 1px solid #333; padding-top: 8px;">';
|
|
html += '<div style="color: #666; margin-bottom: 6px; font-weight: bold;">Recent Items (' + items.length + ')</div>';
|
|
|
|
items.forEach(item => {
|
|
html += '<div style="padding: 6px 0; border-bottom: 1px solid #222;">';
|
|
|
|
// Title with link
|
|
if (item.title) {
|
|
if (item.link) {
|
|
html += '<div><a href="' + escapeHtml(item.link) + '" target="_blank" style="color: #0af; text-decoration: none;">' + escapeHtml(item.title) + '</a></div>';
|
|
} else {
|
|
html += '<div style="color: #ccc;">' + escapeHtml(item.title) + '</div>';
|
|
}
|
|
} else if (item.link) {
|
|
html += '<div><a href="' + escapeHtml(item.link) + '" target="_blank" style="color: #0af; text-decoration: none;">' + escapeHtml(item.link) + '</a></div>';
|
|
}
|
|
|
|
// Metadata line (date, author)
|
|
let meta = [];
|
|
if (item.pub_date) {
|
|
const date = new Date(item.pub_date);
|
|
meta.push(date.toLocaleDateString() + ' ' + date.toLocaleTimeString());
|
|
}
|
|
if (item.author) {
|
|
meta.push(escapeHtml(item.author));
|
|
}
|
|
if (meta.length > 0) {
|
|
html += '<div style="color: #666; font-size: 0.85em;">' + meta.join(' • ') + '</div>';
|
|
}
|
|
|
|
html += '</div>';
|
|
});
|
|
|
|
html += '</div>';
|
|
}
|
|
|
|
html += '</div>';
|
|
|
|
detailsDiv.innerHTML = html;
|
|
} catch (err) {
|
|
detailsDiv.innerHTML = '<div style="padding: 10px; color: #f66;">Error loading feed info</div>';
|
|
}
|
|
} else {
|
|
detailsDiv.style.display = 'none';
|
|
expandedFeed = null;
|
|
}
|
|
}
|
|
|
|
// Infinite scroll handler with prefetch (uses window scroll)
|
|
function setupInfiniteScroll() {
|
|
window.addEventListener('scroll', () => {
|
|
// Check if we're near the bottom of the page
|
|
const scrollBottom = window.scrollY + window.innerHeight;
|
|
const docHeight = document.documentElement.scrollHeight;
|
|
const remainingPixels = docHeight - scrollBottom;
|
|
|
|
// Prefetch when within 500px of the bottom
|
|
if (remainingPixels < 500) {
|
|
loadMoreDomains();
|
|
}
|
|
});
|
|
}
|
|
|
|
// Search functionality
|
|
function setupSearch() {
|
|
const searchInput = document.getElementById('searchInput');
|
|
const searchResults = document.getElementById('searchResults');
|
|
const domainsContainer = document.getElementById('allDomainsContainer');
|
|
|
|
if (!searchInput || !searchResults || !domainsContainer) {
|
|
console.error('Search elements not found');
|
|
return;
|
|
}
|
|
|
|
searchInput.addEventListener('input', (e) => {
|
|
const query = e.target.value.trim();
|
|
|
|
// Clear previous timeout
|
|
if (searchTimeout) {
|
|
clearTimeout(searchTimeout);
|
|
}
|
|
|
|
// If empty, show domains list
|
|
if (!query) {
|
|
searchResults.style.display = 'none';
|
|
domainsContainer.style.display = 'block';
|
|
isSearching = false;
|
|
return;
|
|
}
|
|
|
|
// Debounce search
|
|
searchTimeout = setTimeout(() => performSearch(query), 300);
|
|
});
|
|
|
|
// Handle Enter key
|
|
searchInput.addEventListener('keydown', (e) => {
|
|
if (e.key === 'Enter') {
|
|
const query = e.target.value.trim();
|
|
if (query) {
|
|
if (searchTimeout) clearTimeout(searchTimeout);
|
|
performSearch(query);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
async function performSearch(query) {
|
|
const searchResults = document.getElementById('searchResults');
|
|
const domainsContainer = document.getElementById('allDomainsContainer');
|
|
|
|
isSearching = true;
|
|
domainsContainer.style.display = 'none';
|
|
searchResults.style.display = 'block';
|
|
searchResults.innerHTML = '<div style="padding: 20px; color: #666; text-align: center;">Searching...</div>';
|
|
|
|
try {
|
|
const response = await fetch('/api/search?q=' + encodeURIComponent(query) + '&limit=200');
|
|
const results = await response.json();
|
|
|
|
if (!results || results.length === 0) {
|
|
searchResults.innerHTML = '<div style="padding: 20px; color: #666; text-align: center;">No results found</div>';
|
|
return;
|
|
}
|
|
|
|
// Group results by host
|
|
const byHost = {};
|
|
results.forEach(r => {
|
|
const host = r.feed.source_host || 'unknown';
|
|
if (!byHost[host]) {
|
|
byHost[host] = [];
|
|
}
|
|
byHost[host].push(r);
|
|
});
|
|
|
|
// Render results
|
|
searchResults.innerHTML = '';
|
|
|
|
Object.keys(byHost).sort().forEach(host => {
|
|
const hostDiv = document.createElement('div');
|
|
hostDiv.className = 'search-host';
|
|
|
|
// Host header
|
|
const hostHeader = document.createElement('div');
|
|
hostHeader.className = 'stat-row';
|
|
hostHeader.style.cssText = 'cursor: pointer; background: #1a1a1a; padding: 8px; margin-bottom: 2px;';
|
|
hostHeader.innerHTML = '<span style="color: #0af;">' + escapeHtml(host) + '</span><span style="color: #666;">' + byHost[host].length + ' feed(s)</span>';
|
|
|
|
const feedsContainer = document.createElement('div');
|
|
feedsContainer.style.display = 'block';
|
|
|
|
byHost[host].forEach(result => {
|
|
const feedDiv = document.createElement('div');
|
|
feedDiv.className = 'search-feed';
|
|
feedDiv.style.cssText = 'padding: 8px 8px 8px 20px; border-bottom: 1px solid #222;';
|
|
|
|
// Feed header
|
|
let feedHtml = '<div style="color: #0af; cursor: pointer;" class="feed-url">' + escapeHtml(result.feed.url) + '</div>';
|
|
if (result.feed.title) {
|
|
feedHtml += '<div style="color: #aaa; font-size: 0.9em;">' + escapeHtml(result.feed.title) + '</div>';
|
|
}
|
|
if (result.feed.description) {
|
|
feedHtml += '<div style="color: #666; font-size: 0.85em; margin-top: 2px;">' + escapeHtml(result.feed.description.substring(0, 200)) + '</div>';
|
|
}
|
|
|
|
// Items
|
|
if (result.items && result.items.length > 0) {
|
|
feedHtml += '<div class="search-items" style="margin-top: 8px; padding-left: 10px; border-left: 2px solid #333;">';
|
|
result.items.forEach(item => {
|
|
feedHtml += '<div style="padding: 4px 0; border-bottom: 1px solid #1a1a1a;">';
|
|
if (item.title) {
|
|
if (item.link) {
|
|
feedHtml += '<a href="' + escapeHtml(item.link) + '" target="_blank" style="color: #6cf; text-decoration: none;">' + escapeHtml(item.title) + '</a>';
|
|
} else {
|
|
feedHtml += '<span style="color: #ccc;">' + escapeHtml(item.title) + '</span>';
|
|
}
|
|
}
|
|
let meta = [];
|
|
if (item.pub_date) {
|
|
meta.push(item.pub_date.substring(0, 10));
|
|
}
|
|
if (item.author) {
|
|
meta.push(escapeHtml(item.author));
|
|
}
|
|
if (meta.length > 0) {
|
|
feedHtml += '<div style="color: #555; font-size: 0.8em;">' + meta.join(' • ') + '</div>';
|
|
}
|
|
feedHtml += '</div>';
|
|
});
|
|
feedHtml += '</div>';
|
|
}
|
|
|
|
feedDiv.innerHTML = feedHtml;
|
|
|
|
// Click on feed URL to toggle full feed info
|
|
feedDiv.querySelector('.feed-url').addEventListener('click', () => {
|
|
toggleSearchFeedInfo(result.feed.url, feedDiv);
|
|
});
|
|
|
|
feedsContainer.appendChild(feedDiv);
|
|
});
|
|
|
|
hostHeader.addEventListener('click', () => {
|
|
feedsContainer.style.display = feedsContainer.style.display === 'none' ? 'block' : 'none';
|
|
});
|
|
|
|
hostDiv.appendChild(hostHeader);
|
|
hostDiv.appendChild(feedsContainer);
|
|
searchResults.appendChild(hostDiv);
|
|
});
|
|
|
|
} catch (err) {
|
|
console.error('Search failed:', err);
|
|
searchResults.innerHTML = '<div style="padding: 20px; color: #f66; text-align: center;">Search failed: ' + escapeHtml(err.message) + '</div>';
|
|
}
|
|
}
|
|
|
|
async function toggleSearchFeedInfo(feedUrl, feedDiv) {
|
|
let detailsDiv = feedDiv.querySelector('.feed-details-expanded');
|
|
|
|
if (detailsDiv) {
|
|
detailsDiv.remove();
|
|
return;
|
|
}
|
|
|
|
detailsDiv = document.createElement('div');
|
|
detailsDiv.className = 'feed-details-expanded';
|
|
detailsDiv.style.cssText = 'padding: 10px; background: #111; margin-top: 8px; border-radius: 4px;';
|
|
detailsDiv.innerHTML = '<div style="color: #666;">Loading feed info...</div>';
|
|
feedDiv.appendChild(detailsDiv);
|
|
|
|
try {
|
|
const [infoResponse, itemsResponse] = await Promise.all([
|
|
fetch('/api/feedInfo?url=' + encodeURIComponent(feedUrl)),
|
|
fetch('/api/feedItems?url=' + encodeURIComponent(feedUrl) + '&limit=20')
|
|
]);
|
|
const info = await infoResponse.json();
|
|
const items = await itemsResponse.json();
|
|
|
|
let html = '<table style="width: 100%; color: #888; font-size: 0.85em;">';
|
|
if (info.siteUrl) html += '<tr><td style="color: #555; padding: 2px 8px 2px 0;">Site</td><td>' + escapeHtml(info.siteUrl) + '</td></tr>';
|
|
if (info.language) html += '<tr><td style="color: #555; padding: 2px 8px 2px 0;">Language</td><td>' + escapeHtml(info.language) + '</td></tr>';
|
|
if (info.status) html += '<tr><td style="color: #555; padding: 2px 8px 2px 0;">Status</td><td>' + escapeHtml(info.status) + '</td></tr>';
|
|
if (info.itemCount) html += '<tr><td style="color: #555; padding: 2px 8px 2px 0;">Items</td><td>' + commaFormat(info.itemCount) + '</td></tr>';
|
|
if (info.avgPostFreqHrs) html += '<tr><td style="color: #555; padding: 2px 8px 2px 0;">Avg Freq</td><td>' + info.avgPostFreqHrs.toFixed(1) + ' hrs</td></tr>';
|
|
if (info.newestItemDate) html += '<tr><td style="color: #555; padding: 2px 8px 2px 0;">Newest</td><td>' + escapeHtml(info.newestItemDate) + '</td></tr>';
|
|
html += '</table>';
|
|
|
|
if (items && items.length > 0) {
|
|
html += '<div style="margin-top: 10px; border-top: 1px solid #222; padding-top: 8px;">';
|
|
html += '<div style="color: #555; margin-bottom: 4px;">All Items (' + items.length + ')</div>';
|
|
items.forEach(item => {
|
|
html += '<div style="padding: 3px 0; border-bottom: 1px solid #1a1a1a;">';
|
|
if (item.title && item.link) {
|
|
html += '<a href="' + escapeHtml(item.link) + '" target="_blank" style="color: #0af; text-decoration: none; font-size: 0.9em;">' + escapeHtml(item.title) + '</a>';
|
|
} else if (item.title) {
|
|
html += '<span style="color: #aaa; font-size: 0.9em;">' + escapeHtml(item.title) + '</span>';
|
|
}
|
|
html += '</div>';
|
|
});
|
|
html += '</div>';
|
|
}
|
|
|
|
detailsDiv.innerHTML = html;
|
|
} catch (err) {
|
|
detailsDiv.innerHTML = '<div style="color: #f66;">Failed to load feed info</div>';
|
|
}
|
|
}
|
|
|
|
async function updateStats() {
|
|
try {
|
|
const response = await fetch('/api/stats');
|
|
const stats = await response.json();
|
|
|
|
// Update domain stats
|
|
document.getElementById('totalDomains').textContent = commaFormat(stats.total_domains);
|
|
document.getElementById('checkedDomains').textContent = commaFormat(stats.checked_domains);
|
|
document.getElementById('uncheckedDomains').textContent = commaFormat(stats.unchecked_domains);
|
|
document.getElementById('crawlRate').textContent = commaFormat(stats.crawl_rate);
|
|
document.getElementById('checkRate').textContent = commaFormat(stats.check_rate);
|
|
|
|
// Update progress bar
|
|
const progress = stats.total_domains > 0
|
|
? (stats.checked_domains * 100 / stats.total_domains).toFixed(1)
|
|
: 0;
|
|
document.getElementById('crawlProgress').style.width = progress + '%';
|
|
|
|
// Update feed stats
|
|
document.getElementById('totalFeeds').textContent = commaFormat(stats.total_feeds);
|
|
document.getElementById('rssFeeds').textContent = commaFormat(stats.rss_feeds);
|
|
document.getElementById('atomFeeds').textContent = commaFormat(stats.atom_feeds);
|
|
document.getElementById('unknownFeeds').textContent = commaFormat(stats.unknown_feeds);
|
|
|
|
// Update timestamp
|
|
const updatedAt = new Date(stats.updated_at);
|
|
document.getElementById('updatedAt').textContent = 'Last updated: ' +
|
|
updatedAt.toISOString().replace('T', ' ').substring(0, 19);
|
|
|
|
} catch (err) {
|
|
console.error('Failed to update stats:', err);
|
|
}
|
|
}
|
|
|
|
// Initialize
|
|
try {
|
|
setupSearch();
|
|
} catch (e) {
|
|
console.error('setupSearch failed:', e);
|
|
}
|
|
setupInfiniteScroll();
|
|
loadMoreDomains();
|
|
updateStats();
|
|
setInterval(updateStats, 1000);
|
|
}
|
|
|
|
window.onload = initDashboard;
|