Files
crawler/static/dashboard.js

1288 lines
57 KiB
JavaScript

function initDashboard() {
function commaFormat(n) {
return n.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
}
function escapeHtml(text) {
if (text == null) return '';
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
// State
let currentView = 'domains'; // 'domains', 'feeds', 'domain-feeds'
let currentDomain = null;
let infiniteScrollState = null;
let isLoadingMore = false;
let selectedTLDs = new Set(); // Empty means show all
let allTLDs = []; // All available TLDs
// Event delegation for domain-spacer clicks (toggle feeds)
document.addEventListener('click', (e) => {
const spacer = e.target.closest('.domain-spacer');
if (spacer) {
const block = spacer.closest('.domain-block');
if (block) {
const feedsDiv = block.querySelector('.domain-feeds');
if (feedsDiv) {
const isVisible = feedsDiv.style.display !== 'none';
feedsDiv.style.display = isVisible ? 'none' : 'block';
// Load items for all feeds when opening
if (!isVisible) {
feedsDiv.querySelectorAll('.inline-feed-block').forEach(feedBlock => {
const itemsDiv = feedBlock.querySelector('.feed-items');
if (itemsDiv && !itemsDiv.dataset.loaded) {
itemsDiv.dataset.loaded = 'true';
loadFeedItems(feedBlock.dataset.url, itemsDiv);
}
});
}
}
}
}
});
// Event delegation for feed-url-toggle clicks (toggle feed info)
document.addEventListener('click', (e) => {
const urlToggle = e.target.closest('.feed-url-toggle');
if (urlToggle) {
const feedBlock = urlToggle.closest('.inline-feed-block');
if (feedBlock) {
const infoDiv = feedBlock.querySelector('.feed-info');
if (infoDiv) {
const isVisible = infoDiv.style.display !== 'none';
infoDiv.style.display = isVisible ? 'none' : 'block';
if (!isVisible && !infoDiv.dataset.loaded) {
infoDiv.dataset.loaded = 'true';
loadFeedInfo(feedBlock.dataset.url, infoDiv);
}
}
}
}
});
// Event delegation for feed-title-toggle and feed-filler-toggle clicks (toggle items)
document.addEventListener('click', (e) => {
const titleToggle = e.target.closest('.feed-title-toggle');
const fillerToggle = e.target.closest('.feed-filler-toggle');
if (titleToggle || fillerToggle) {
const feedBlock = (titleToggle || fillerToggle).closest('.inline-feed-block');
if (feedBlock) {
const itemsDiv = feedBlock.querySelector('.feed-items');
if (itemsDiv) {
const isVisible = itemsDiv.style.display !== 'none';
itemsDiv.style.display = isVisible ? 'none' : 'block';
}
}
}
});
// Load feed info into info div
async function loadFeedInfo(feedUrl, infoDiv) {
infoDiv.innerHTML = '<span style="color: #666;">Loading feed info...</span>';
try {
const resp = await fetch(`/api/feedInfo?url=${encodeURIComponent(feedUrl)}`);
if (!resp.ok) throw new Error('Failed to load');
const f = await resp.json();
let html = '<div style="display: grid; grid-template-columns: auto 1fr; gap: 4px 12px; color: #888;">';
const fields = [
['URL', f.url],
['Title', f.title],
['Description', f.description],
['Type', f.type],
['Language', f.language],
['Category', f.category],
['Site URL', f.siteUrl],
['Status', f.status],
['Error Count', f.errorCount],
['Last Error', f.lastError],
['Item Count', f.itemCount],
['Avg Post Freq', f.avgPostFreqHrs ? f.avgPostFreqHrs.toFixed(1) + ' hrs' : null],
['Oldest Item', f.oldestItemDate],
['Newest Item', f.newestItemDate],
['Discovered', f.discoveredAt],
['Last Crawled', f.lastCrawledAt],
['Next Crawl', f.nextCrawlAt],
['TTL', f.ttlMinutes ? f.ttlMinutes + ' min' : null],
['Publish Status', f.publishStatus],
['Publish Account', f.publishAccount],
];
fields.forEach(([label, value]) => {
if (value != null && value !== '' && value !== 0) {
html += `<span style="color: #666;">${escapeHtml(label)}:</span><span style="color: #fff;">${escapeHtml(String(value))}</span>`;
}
});
html += '</div>';
infoDiv.innerHTML = html;
} catch (err) {
infoDiv.innerHTML = `<span style="color: #f66;">Error: ${escapeHtml(err.message)}</span>`;
}
}
// Load feed items into items div
async function loadFeedItems(feedUrl, itemsDiv) {
itemsDiv.innerHTML = '<span style="color: #666;">Loading items...</span>';
try {
const resp = await fetch(`/api/feedItems?url=${encodeURIComponent(feedUrl)}&limit=50`);
if (!resp.ok) throw new Error('Failed to load');
const items = await resp.json();
if (!items || items.length === 0) {
itemsDiv.innerHTML = '<span style="color: #666;">No items</span>';
return;
}
let html = '';
items.forEach(item => {
const date = item.pub_date ? new Date(item.pub_date).toLocaleDateString() : '';
html += `<div style="padding: 2px 0; border-bottom: 1px solid #1a1a1a;">`;
html += `<span style="color: #666; margin-right: 8px;">${escapeHtml(date)}</span>`;
if (item.link) {
html += `<a href="${escapeHtml(item.link)}" target="_blank" style="color: #0af; text-decoration: none;">${escapeHtml(item.title || item.link)}</a>`;
} else {
html += `<span style="color: #fff;">${escapeHtml(item.title || '(no title)')}</span>`;
}
html += '</div>';
});
itemsDiv.innerHTML = html;
} catch (err) {
itemsDiv.innerHTML = `<span style="color: #f66;">Error: ${escapeHtml(err.message)}</span>`;
}
}
// Status colors and labels
const statusConfig = {
hold: { color: '#f90', bg: '#330', border: '#550', dimColor: '#664', dimBg: '#1a1a00', label: 'hold' },
skip: { color: '#f66', bg: '#400', border: '#600', dimColor: '#844', dimBg: '#200', label: 'skip' },
pass: { color: '#0f0', bg: '#040', border: '#060', dimColor: '#484', dimBg: '#020', label: 'pass' },
fail: { color: '#f00', bg: '#400', border: '#600', dimColor: '#333', dimBg: '#111', label: 'fail' },
drop: { color: '#f0f', bg: '#404', border: '#606', dimColor: '#848', dimBg: '#202', label: 'drop' }
};
// Render status button group
// For domains: pass, hold, skip (+ fail indicator or spacer)
// For feeds: pass, hold, skip, drop (+ fail indicator or spacer)
function renderStatusBtns(currentStatus, type, id, errorStatus) {
const order = type === 'feed' ? ['pass', 'hold', 'skip', 'drop'] : ['pass', 'hold', 'skip'];
const showFail = errorStatus === 'error' || errorStatus === 'dead';
let html = '<div class="status-btn-group" style="display: inline-flex; margin-right: 10px;">';
order.forEach((s, i) => {
const cfg = statusConfig[s];
const isActive = currentStatus === s;
const bg = isActive ? cfg.bg : cfg.dimBg;
const color = isActive ? cfg.color : cfg.dimColor;
const border = isActive ? cfg.border : '#333';
html += `<button class="status-btn" data-type="${type}" data-id="${escapeHtml(id)}" data-status="${s}"
style="padding: 2px 6px; font-family: monospace;
background: ${bg}; border: 1px solid ${border}; border-radius: 3px;
color: ${color}; cursor: pointer; margin-left: ${i > 0 ? '1px' : '0'};">${cfg.label}</button>`;
});
// Fail indicator (not clickable) or hidden spacer for alignment
const cfg = statusConfig.fail;
const failBtnStyle = `padding: 2px 6px; font-family: monospace;
background: ${cfg.bg}; border: 1px solid ${cfg.border}; border-radius: 3px;
color: ${cfg.color}; cursor: default; margin-left: 1px; box-sizing: border-box;`;
if (showFail) {
html += `<button disabled style="${failBtnStyle}" title="Feed has errors">${cfg.label}</button>`;
} else {
// Hidden spacer - same style but invisible
html += `<button disabled style="${failBtnStyle} opacity: 0;" aria-hidden="true">${cfg.label}</button>`;
}
html += '</div>';
return html;
}
// Infinite scroll
function setupInfiniteScroll(loadMoreFn) {
infiniteScrollState = { loadMore: loadMoreFn, ended: false };
}
function clearInfiniteScroll() {
infiniteScrollState = null;
}
function checkInfiniteScroll() {
if (!infiniteScrollState || infiniteScrollState.ended || isLoadingMore) return;
const scrollBottom = window.scrollY + window.innerHeight;
const docHeight = document.documentElement.scrollHeight;
if (docHeight - scrollBottom < 300) {
isLoadingMore = true;
infiniteScrollState.loadMore().finally(() => {
isLoadingMore = false;
});
}
}
window.addEventListener('scroll', checkInfiniteScroll);
// Render domain row with feeds
function renderDomainRow(d) {
const status = d.status || 'hold';
const hasError = !!d.last_error;
const rowStyle = hasError ? 'opacity: 0.8;' : '';
let html = `<div class="domain-block" data-host="${escapeHtml(d.host)}" data-status="${status}">`;
// Domain header row
html += `<div class="domain-row" style="display: flex; align-items: center; padding: 8px 10px; border-bottom: 1px solid #202020; ${rowStyle}">`;
// Status buttons (pass/hold/skip + fail indicator if error)
html += renderStatusBtns(status, 'domain', d.host, hasError ? 'error' : null);
// Domain name (links to site)
html += `<a class="domain-name" href="https://${escapeHtml(d.host)}" target="_blank" style="color: #0af; text-decoration: none;">${escapeHtml(d.host)}</a>`;
// Spacer (clickable to toggle feeds) - uses &nbsp; to ensure clickable area
if (d.last_error) {
html += `<span class="domain-spacer" style="color: #f66; margin-left: 10px; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; cursor: pointer;" title="${escapeHtml(d.last_error)}">${escapeHtml(d.last_error)}</span>`;
} else {
html += '<span class="domain-spacer" style="flex: 1; cursor: pointer;">&nbsp;</span>';
}
// Drop button (only for skipped domains)
if (status === 'skip') {
html += `<button class="drop-btn" data-host="${escapeHtml(d.host)}"
style="padding: 2px 8px; font-family: monospace; margin-left: 8px;
background: #400; border: 1px solid #800; border-radius: 3px;
color: #f88; cursor: pointer;" title="Permanently delete all data">drop</button>`;
}
html += '</div>';
// Feeds under this domain (hidden by default, toggled by clicking spacer)
if (d.feeds && d.feeds.length > 0) {
html += '<div class="domain-feeds" style="display: none; margin-left: 10px; border-left: 2px solid #333; padding-left: 6px;">';
d.feeds.forEach(f => {
const feedStatus = f.publish_status || 'hold';
html += `<div class="inline-feed-block" data-url="${escapeHtml(f.url)}" data-status="${feedStatus}">`;
html += `<div class="feed-row" style="display: flex; align-items: center; padding: 4px 0;">`;
// Language indicator (fixed width)
const lang = f.language || '';
html += `<span style="display: inline-block; width: 32px; margin-right: 6px; color: #666; font-family: monospace; text-align: center;">${escapeHtml(lang)}</span>`;
// Feed publish status buttons (pass/hold/skip)
html += renderStatusBtns(feedStatus, 'feed', f.url, f.status);
// Feed crawl status (active/error/dead)
const statusColor = f.status === 'active' ? '#484' : f.status === 'error' ? '#a66' : '#666';
html += `<span style="color: ${statusColor}; font-family: monospace; width: 50px; margin-right: 6px;">${escapeHtml(f.status || 'active')}</span>`;
// Item count if > 0
if (f.item_count > 0) {
html += `<span style="color: #888; font-family: monospace; width: 55px; margin-right: 6px; text-align: right;">${commaFormat(f.item_count)}</span>`;
} else {
html += `<span style="width: 55px; margin-right: 6px;"></span>`;
}
// Feed path (URL without domain) - click to toggle feed info
let feedPath = f.url;
try {
const urlObj = new URL(f.url.startsWith('http') ? f.url : 'https://' + f.url);
feedPath = urlObj.pathname + urlObj.search;
} catch (e) {}
html += `<span class="feed-url-toggle" style="color: #0af; margin-right: 8px; white-space: nowrap; cursor: pointer;" title="Click to show feed info">${escapeHtml(feedPath)}</span>`;
// Feed title - click to toggle items
if (f.title) {
html += `<span class="feed-title-toggle" style="color: #fff; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; cursor: pointer;" title="Click to toggle items">${escapeHtml(f.title)}</span>`;
}
// Filler span - also toggles items
html += '<span class="feed-filler-toggle" style="flex: 1; cursor: pointer;">&nbsp;</span>';
html += '</div>';
// Feed info section (hidden by default)
html += '<div class="feed-info" style="display: none; padding: 6px 10px; margin-left: 10px; border-left: 2px solid #444; background: #0a0a0a;"></div>';
// Feed items section (shown by default)
html += '<div class="feed-items" style="display: block; padding: 4px 10px; margin-left: 10px; border-left: 2px solid #333;"></div>';
html += '</div>';
});
html += '</div>';
}
html += '</div>';
return html;
}
// Render feed row
function renderFeedRow(f) {
const status = f.publish_status || 'hold';
let html = `<div class="feed-block" data-url="${escapeHtml(f.url)}" data-status="${status}">`;
html += `<div class="feed-row" style="display: flex; align-items: flex-start; padding: 8px 10px; border-bottom: 1px solid #202020;">`;
// Language indicator (fixed width)
const lang = f.language || '';
html += `<span style="display: inline-block; width: 32px; margin-right: 6px; color: #666; font-family: monospace; text-align: center; padding-top: 4px;">${escapeHtml(lang)}</span>`;
// Status buttons (pass/hold/skip + fail indicator if errors)
html += `<div style="padding-top: 2px;">${renderStatusBtns(status, 'feed', f.url, f.status)}</div>`;
// Feed info
html += '<div style="flex: 1; min-width: 0;">';
// Title or URL
const title = f.title || f.url;
html += `<div class="feed-title" style="color: #0af; cursor: pointer; word-break: break-word;">${escapeHtml(title)}</div>`;
// URL if different from title
if (f.title) {
html += `<div style="color: #666; font-size: 0.85em; word-break: break-all;">${escapeHtml(f.url)}</div>`;
}
// Meta info
let meta = [];
if (f.type) meta.push(f.type);
if (f.language) meta.push(f.language);
if (f.item_count > 0) meta.push(commaFormat(f.item_count) + ' items');
if (f.status && f.status !== 'active') {
const color = f.status === 'error' ? '#f66' : '#888';
meta.push(`<span style="color: ${color};">${f.status}</span>`);
}
if (meta.length > 0) {
html += `<div style="color: #666; font-size: 0.8em;">${meta.join(' · ')}</div>`;
}
html += '</div>';
// External link
html += `<a href="https://${escapeHtml(f.url)}" target="_blank" style="color: #666; margin-left: 8px; text-decoration: none; flex-shrink: 0;">↗</a>`;
html += '</div>';
// Collapsible detail section (hidden by default)
html += '<div class="feed-detail" style="display: none; margin-left: 20px; padding: 10px; border-left: 2px solid #333; border-bottom: 1px solid #202020; background: #0a0a0a;"></div>';
html += '</div>';
return html;
}
// Attach status button handlers
function attachStatusHandlers(container) {
container.querySelectorAll('.status-btn:not(.handled)').forEach(btn => {
btn.classList.add('handled');
btn.addEventListener('click', async (e) => {
e.stopPropagation();
const type = btn.dataset.type;
const id = btn.dataset.id;
const newStatus = btn.dataset.status;
const group = btn.closest('.status-btn-group');
// Disable all buttons in group
const allBtns = group.querySelectorAll('.status-btn');
allBtns.forEach(b => b.disabled = true);
try {
let endpoint;
if (type === 'domain') {
endpoint = `/api/setDomainStatus?host=${encodeURIComponent(id)}&status=${newStatus}`;
} else {
endpoint = `/api/setPublishStatus?url=${encodeURIComponent(id)}&status=${newStatus}`;
}
const resp = await fetch(endpoint);
if (resp.ok) {
// Update all buttons in group
allBtns.forEach(b => {
const s = b.dataset.status;
const cfg = statusConfig[s];
const isActive = s === newStatus;
b.style.background = isActive ? cfg.bg : cfg.dimBg;
b.style.color = isActive ? cfg.color : cfg.dimColor;
b.style.borderColor = isActive ? cfg.border : '#333';
});
// Update row data attribute
const row = btn.closest('.domain-row, .feed-row');
if (row) row.dataset.status = newStatus;
}
} catch (err) {
console.error('Status update failed:', err);
}
allBtns.forEach(b => b.disabled = false);
});
});
}
// Attach domain row handlers
function attachDomainHandlers(container) {
attachStatusHandlers(container);
container.querySelectorAll('.domain-block:not(.block-handled)').forEach(block => {
block.classList.add('block-handled');
const row = block.querySelector('.domain-row');
const host = block.dataset.host;
// Hover effect on domain row
row.addEventListener('mouseenter', () => row.style.background = '#1a1a1a');
row.addEventListener('mouseleave', () => row.style.background = 'transparent');
// Drop button handler (for skipped domains)
const dropBtn = row.querySelector('.drop-btn');
if (dropBtn) {
dropBtn.addEventListener('click', async (e) => {
e.stopPropagation();
const host = dropBtn.dataset.host;
if (!confirm(`Permanently delete all data for ${host}?\n\nThis will:\n- Delete all PDS accounts\n- Delete all feed items\n- Delete all feeds\n\nThis cannot be undone.`)) {
return;
}
dropBtn.disabled = true;
dropBtn.textContent = '...';
try {
const resp = await fetch(`/api/dropDomain?host=${encodeURIComponent(host)}`);
if (resp.ok) {
const result = await resp.json();
// Update status to "drop" visually
block.dataset.status = 'drop';
const statusGroup = row.querySelector('.status-btn-group');
if (statusGroup) {
statusGroup.innerHTML = '<span style="padding: 2px 6px; font-family: monospace; background: #300; border: 1px solid #600; border-radius: 3px; color: #f66;">dropped</span>';
}
dropBtn.remove();
console.log('Drop result:', result);
} else {
alert('Drop failed: ' + await resp.text());
dropBtn.disabled = false;
dropBtn.textContent = 'drop';
}
} catch (err) {
console.error('Drop failed:', err);
alert('Drop failed: ' + err.message);
dropBtn.disabled = false;
dropBtn.textContent = 'drop';
}
});
}
// Handle inline feed clicks - toggle detail
block.querySelectorAll('.inline-feed-block').forEach(feedBlock => {
const title = feedBlock.querySelector('.feed-title');
const detailDiv = feedBlock.querySelector('.feed-detail');
const feedUrl = feedBlock.dataset.url;
if (title && detailDiv) {
title.addEventListener('click', () => {
const isVisible = detailDiv.style.display !== 'none';
if (isVisible) {
detailDiv.style.display = 'none';
} else {
detailDiv.style.display = 'block';
if (!detailDiv.dataset.loaded) {
detailDiv.dataset.loaded = 'true';
loadFeedDetail(feedUrl, detailDiv);
}
}
});
}
});
});
}
// Load and render feed detail content
async function loadFeedDetail(feedUrl, detailDiv) {
detailDiv.innerHTML = '<div style="color: #666;">Loading...</div>';
try {
const [infoResp, itemsResp] = await Promise.all([
fetch('/api/feedInfo?url=' + encodeURIComponent(feedUrl)),
fetch('/api/feedItems?url=' + encodeURIComponent(feedUrl) + '&limit=20')
]);
const info = await infoResp.json();
const items = await itemsResp.json();
let html = '';
// Description
if (info.description) {
html += `<div style="color: #fff; margin-bottom: 10px; font-size: 0.9em;">${escapeHtml(info.description)}</div>`;
}
// Info row
let infoParts = [];
if (info.type) infoParts.push(info.type);
if (info.status) {
const statusColor = info.status === 'active' ? '#0a0' : '#f66';
infoParts.push(`<span style="color: ${statusColor};">${info.status}</span>`);
}
if (info.language) infoParts.push(info.language);
if (info.itemCount) infoParts.push(commaFormat(info.itemCount) + ' items');
if (info.publishAccount) {
infoParts.push(`<span style="color: #0af;">@${info.publishAccount}</span>`);
}
if (infoParts.length > 0) {
html += `<div style="color: #666; font-size: 0.85em; margin-bottom: 10px;">${infoParts.join(' · ')}</div>`;
}
// Items
if (items && items.length > 0) {
html += '<div style="border-top: 1px solid #333; padding-top: 10px; margin-top: 5px;">';
items.forEach(item => {
html += '<div style="padding: 4px 0; font-size: 0.9em;">';
if (item.title && item.link) {
html += `<a href="${escapeHtml(item.link)}" target="_blank" style="color: #0af; text-decoration: none;">${escapeHtml(item.title)}</a>`;
} else if (item.title) {
html += `<span style="color: #fff;">${escapeHtml(item.title)}</span>`;
}
if (item.pub_date) {
const date = new Date(item.pub_date);
html += `<span style="color: #555; margin-left: 8px; font-size: 0.85em;">${date.toLocaleDateString()}</span>`;
}
html += '</div>';
});
html += '</div>';
} else {
html += '<div style="color: #555; font-size: 0.9em;">No items</div>';
}
detailDiv.innerHTML = html;
} catch (err) {
detailDiv.innerHTML = `<div style="color: #f66;">Error: ${escapeHtml(err.message)}</div>`;
}
}
// Attach feed row handlers
function attachFeedHandlers(container) {
attachStatusHandlers(container);
container.querySelectorAll('.feed-block:not(.block-handled)').forEach(block => {
block.classList.add('block-handled');
const row = block.querySelector('.feed-row');
const detailDiv = block.querySelector('.feed-detail');
const feedUrl = block.dataset.url;
// Click feed title to toggle detail
const title = row.querySelector('.feed-title');
if (title) {
title.addEventListener('click', () => {
const isVisible = detailDiv.style.display !== 'none';
if (isVisible) {
detailDiv.style.display = 'none';
} else {
detailDiv.style.display = 'block';
// Load content if empty
if (!detailDiv.dataset.loaded) {
detailDiv.dataset.loaded = 'true';
loadFeedDetail(feedUrl, detailDiv);
}
}
});
}
// Hover effect
row.addEventListener('mouseenter', () => row.style.background = '#1a1a1a');
row.addEventListener('mouseleave', () => row.style.background = 'transparent');
});
}
// Show all domains view
// statusFilter can be: 'hold', 'pass', 'skip', 'fail', or 'feeds' (special: has_feeds=true)
async function showDomains(statusFilter = null) {
currentView = 'domains';
currentDomain = null;
clearInfiniteScroll();
const output = document.getElementById('output');
const breadcrumb = document.getElementById('breadcrumb');
// Breadcrumb
let bc = '<span class="bc-link" data-view="domains" style="color: #0af; cursor: pointer;">domains</span>';
if (statusFilter) {
bc += ` <span style="color: #666;">/</span> <span style="color: #f90;">${statusFilter}</span>`;
}
breadcrumb.innerHTML = bc;
breadcrumb.style.display = 'block';
// Add breadcrumb handler
breadcrumb.querySelector('.bc-link').addEventListener('click', () => showDomains());
output.innerHTML = '<div class="domain-list"></div><div id="infiniteLoader" style="text-align: center; padding: 10px; color: #666;">Loading...</div>';
let offset = 0;
const limit = 100;
let currentTLD = null;
async function loadMore() {
try {
let url = `/api/domains?limit=${limit}&offset=${offset}&sort=alpha`;
if (statusFilter === 'feeds') {
url += `&has_feeds=true`;
} else if (statusFilter) {
url += `&status=${statusFilter}`;
}
const resp = await fetch(url);
const domains = await resp.json();
if (!domains || domains.length === 0) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = offset === 0 ? 'No domains found' : 'End of list';
return;
}
const container = output.querySelector('.domain-list');
domains.forEach(d => {
// Insert TLD header if TLD changed
if (d.tld && d.tld !== currentTLD) {
currentTLD = d.tld;
container.insertAdjacentHTML('beforeend', `<div class="tld-header" data-tld="${escapeHtml(currentTLD)}" style="padding: 12px 10px 6px; color: #888; font-size: 0.9em; border-bottom: 1px solid #333; margin-top: ${offset === 0 && !container.querySelector('.domain-block') ? '0' : '15px'}; cursor: pointer;"><span style="margin-right: 6px; font-size: 0.8em;">▼</span>.${escapeHtml(currentTLD)}</div>`);
}
container.insertAdjacentHTML('beforeend', renderDomainRow(d));
});
attachDomainHandlers(container);
applyTLDFilter();
// Auto-expand feeds in d:feeds mode
if (statusFilter === 'feeds') {
// Show the domain-feeds containers and load items
container.querySelectorAll('.domain-feeds').forEach(feedsDiv => {
feedsDiv.style.display = 'block';
});
// Load items for all feeds
container.querySelectorAll('.inline-feed-block').forEach(feedBlock => {
const itemsDiv = feedBlock.querySelector('.feed-items');
const feedUrl = feedBlock.dataset.url;
if (itemsDiv && !itemsDiv.dataset.loaded) {
itemsDiv.dataset.loaded = 'true';
loadFeedItems(feedUrl, itemsDiv);
}
});
}
offset += domains.length;
if (domains.length < limit) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = 'End of list';
}
} catch (err) {
document.getElementById('infiniteLoader').textContent = 'Error: ' + err.message;
}
}
await loadMore();
setupInfiniteScroll(loadMore);
}
// Show feeds for a specific domain
async function showDomainFeeds(host) {
currentView = 'domain-feeds';
currentDomain = host;
clearInfiniteScroll();
const output = document.getElementById('output');
const breadcrumb = document.getElementById('breadcrumb');
// Breadcrumb
breadcrumb.innerHTML = `
<span class="bc-link" data-view="domains" style="color: #0af; cursor: pointer;">domains</span>
<span style="color: #666;"> / </span>
<span style="color: #fff;">${escapeHtml(host)}</span>
<a href="https://${escapeHtml(host)}" target="_blank" style="color: #666; margin-left: 8px; text-decoration: none;">↗</a>`;
breadcrumb.style.display = 'block';
breadcrumb.querySelector('.bc-link').addEventListener('click', () => showDomains());
output.innerHTML = '<div class="feed-list"></div><div id="infiniteLoader" style="text-align: center; padding: 10px; color: #666;">Loading...</div>';
let offset = 0;
const limit = 100;
async function loadMore() {
try {
const resp = await fetch(`/api/domainFeeds?host=${encodeURIComponent(host)}&limit=${limit}&offset=${offset}`);
const feeds = await resp.json();
if (!feeds || feeds.length === 0) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = offset === 0 ? 'No feeds found' : 'End of list';
return;
}
const container = output.querySelector('.feed-list');
feeds.forEach(f => container.insertAdjacentHTML('beforeend', renderFeedRow(f)));
attachFeedHandlers(container);
offset += feeds.length;
if (feeds.length < limit) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = 'End of list';
}
} catch (err) {
document.getElementById('infiniteLoader').textContent = 'Error: ' + err.message;
}
}
await loadMore();
setupInfiniteScroll(loadMore);
}
// Show all feeds view (shows domains with feeds, or flat feed list if filtered)
async function showFeeds(statusFilter = null) {
currentView = 'feeds';
currentDomain = null;
clearInfiniteScroll();
const output = document.getElementById('output');
const breadcrumb = document.getElementById('breadcrumb');
// Breadcrumb
let bc = '<span class="bc-link" data-view="feeds" style="color: #0af; cursor: pointer;">feeds</span>';
if (statusFilter) {
bc += ` <span style="color: #666;">/</span> <span style="color: #f90;">${statusFilter}</span>`;
}
breadcrumb.innerHTML = bc;
breadcrumb.style.display = 'block';
breadcrumb.querySelector('.bc-link').addEventListener('click', () => showFeeds());
// When no filter, show domains with feeds; when filtered, show flat feed list
if (!statusFilter) {
output.innerHTML = '<div class="domain-list"></div><div id="infiniteLoader" style="text-align: center; padding: 10px; color: #666;">Loading...</div>';
let offset = 0;
const limit = 100;
let currentTLD = null;
async function loadMore() {
try {
const url = `/api/domains?has_feeds=true&limit=${limit}&offset=${offset}&sort=alpha`;
const resp = await fetch(url);
const domains = await resp.json();
if (!domains || domains.length === 0) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = offset === 0 ? 'No domains with feeds found' : 'End of list';
return;
}
const container = output.querySelector('.domain-list');
domains.forEach(d => {
// Insert TLD header if TLD changed
if (d.tld && d.tld !== currentTLD) {
currentTLD = d.tld;
container.insertAdjacentHTML('beforeend', `<div class="tld-header" data-tld="${escapeHtml(currentTLD)}" style="padding: 12px 10px 6px; color: #888; font-size: 0.9em; border-bottom: 1px solid #333; margin-top: ${offset === 0 && !container.querySelector('.domain-block') ? '0' : '15px'}; cursor: pointer;"><span style="margin-right: 6px; font-size: 0.8em;">▼</span>.${escapeHtml(currentTLD)}</div>`);
}
container.insertAdjacentHTML('beforeend', renderDomainRow(d));
});
attachDomainHandlers(container);
applyTLDFilter();
offset += domains.length;
if (domains.length < limit) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = 'End of list';
}
} catch (err) {
document.getElementById('infiniteLoader').textContent = 'Error: ' + err.message;
}
}
await loadMore();
setupInfiniteScroll(loadMore);
} else {
// Filtered view: show flat feed list
output.innerHTML = '<div class="feed-list"></div><div id="infiniteLoader" style="text-align: center; padding: 10px; color: #666;">Loading...</div>';
let offset = 0;
const limit = 100;
async function loadMore() {
try {
let url = `/api/feeds?limit=${limit}&offset=${offset}&sort=alpha`;
url += `&publish_status=${statusFilter}`;
const resp = await fetch(url);
const feeds = await resp.json();
if (!feeds || feeds.length === 0) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = offset === 0 ? 'No feeds found' : 'End of list';
return;
}
const container = output.querySelector('.feed-list');
feeds.forEach(f => container.insertAdjacentHTML('beforeend', renderFeedRow(f)));
attachFeedHandlers(container);
offset += feeds.length;
if (feeds.length < limit) {
if (infiniteScrollState) infiniteScrollState.ended = true;
document.getElementById('infiniteLoader').textContent = 'End of list';
}
} catch (err) {
document.getElementById('infiniteLoader').textContent = 'Error: ' + err.message;
}
}
await loadMore();
setupInfiniteScroll(loadMore);
}
}
// Show feed info
async function showFeedInfo(feedUrl) {
clearInfiniteScroll();
const output = document.getElementById('output');
const breadcrumb = document.getElementById('breadcrumb');
breadcrumb.innerHTML = `
<span class="bc-link" data-view="feeds" style="color: #0af; cursor: pointer;">feeds</span>
<span style="color: #666;"> / </span>
<span style="color: #fff; word-break: break-all;">${escapeHtml(feedUrl)}</span>`;
breadcrumb.style.display = 'block';
breadcrumb.querySelector('.bc-link').addEventListener('click', () => {
if (currentDomain) showDomainFeeds(currentDomain);
else showFeeds();
});
output.innerHTML = '<div style="color: #666; padding: 10px;">Loading feed info...</div>';
try {
const [infoResp, itemsResp] = await Promise.all([
fetch('/api/feedInfo?url=' + encodeURIComponent(feedUrl)),
fetch('/api/feedItems?url=' + encodeURIComponent(feedUrl) + '&limit=50')
]);
const info = await infoResp.json();
const items = await itemsResp.json();
let html = '<div style="padding: 10px;">';
// Title and status buttons
html += '<div style="display: flex; align-items: center; margin-bottom: 15px;">';
html += renderStatusBtns(info.publishStatus || 'hold', 'feed', feedUrl, info.status);
html += `<span style="color: #fff; font-size: 1.2em;">${escapeHtml(info.title || feedUrl)}</span>`;
html += '</div>';
// URL
html += `<div style="color: #666; margin-bottom: 15px; word-break: break-all;">${escapeHtml(feedUrl)}</div>`;
// Description
if (info.description) {
html += `<div style="color: #fff; margin-bottom: 15px;">${escapeHtml(info.description)}</div>`;
}
// Info table
html += '<table style="color: #888; font-size: 0.9em; margin-bottom: 15px;">';
const addRow = (label, value, color) => {
if (!value) return '';
return `<tr><td style="padding: 4px 20px 4px 0; color: #666;">${label}</td><td style="color: ${color || '#888'};">${escapeHtml(String(value))}</td></tr>`;
};
html += addRow('Type', info.type);
html += addRow('Status', info.status, info.status === 'active' ? '#0a0' : '#f66');
html += addRow('Language', info.language);
html += addRow('Items', info.itemCount ? commaFormat(info.itemCount) : null);
html += addRow('Source', info.sourceHost);
if (info.publishAccount) {
html += addRow('Account', info.publishAccount, '#0af');
}
html += '</table>';
// Items
if (items && items.length > 0) {
html += '<div style="border-top: 1px solid #333; padding-top: 15px;">';
html += `<div style="color: #666; margin-bottom: 10px; font-weight: bold;">Recent Items (${items.length})</div>`;
items.forEach(item => {
html += '<div style="padding: 8px 0; border-bottom: 1px solid #222;">';
if (item.title && item.link) {
html += `<a href="${escapeHtml(item.link)}" target="_blank" style="color: #0af; text-decoration: none;">${escapeHtml(item.title)}</a>`;
} else if (item.title) {
html += `<span style="color: #fff;">${escapeHtml(item.title)}</span>`;
}
if (item.pub_date) {
const date = new Date(item.pub_date);
html += `<span style="color: #666; margin-left: 10px; font-size: 0.85em;">${date.toLocaleDateString()}</span>`;
}
html += '</div>';
});
html += '</div>';
}
html += '</div>';
output.innerHTML = html;
// Attach status handler
attachStatusHandlers(output);
} catch (err) {
output.innerHTML = `<div style="color: #f66; padding: 10px;">Error: ${escapeHtml(err.message)}</div>`;
}
}
// Search
async function performSearch(query) {
clearInfiniteScroll();
const output = document.getElementById('output');
const breadcrumb = document.getElementById('breadcrumb');
breadcrumb.innerHTML = `
<span class="bc-link" data-view="domains" style="color: #0af; cursor: pointer;">domains</span>
<span style="color: #666;"> / </span>
<span style="color: #f90;">search: ${escapeHtml(query)}</span>`;
breadcrumb.style.display = 'block';
breadcrumb.querySelector('.bc-link').addEventListener('click', () => showDomains());
output.innerHTML = '<div style="color: #666; padding: 10px;">Searching...</div>';
try {
const resp = await fetch('/api/search?q=' + encodeURIComponent(query) + '&limit=200');
const results = await resp.json();
if (!results || results.length === 0) {
output.innerHTML = '<div style="color: #666; padding: 20px; text-align: center;">No results found</div>';
return;
}
let html = `<div style="color: #888; padding: 10px; border-bottom: 1px solid #333;">${results.length} result(s)</div>`;
html += '<div class="feed-list">';
results.forEach(r => {
const f = r.feed;
html += renderFeedRow(f);
});
html += '</div>';
output.innerHTML = html;
attachFeedHandlers(output.querySelector('.feed-list'));
} catch (err) {
output.innerHTML = `<div style="color: #f66; padding: 10px;">Search error: ${escapeHtml(err.message)}</div>`;
}
}
// Process command
function processCommand(cmd) {
const trimmed = cmd.trim().toLowerCase();
if (!trimmed || trimmed === '/help') {
showHelp();
return;
}
if (trimmed === '/domains' || trimmed === 'domains') {
showDomains();
return;
}
if (trimmed === '/feeds' || trimmed === 'feeds') {
showFeeds();
return;
}
// Domain status filters (domains:hold or d:hold)
if (trimmed.startsWith('domains:')) {
const status = trimmed.substring(8);
showDomains(status);
return;
}
if (trimmed.startsWith('d:')) {
const status = trimmed.substring(2);
showDomains(status);
return;
}
// Feed status filters (feeds:hold or f:hold)
if (trimmed.startsWith('feeds:')) {
const status = trimmed.substring(6);
showFeeds(status);
return;
}
if (trimmed.startsWith('f:')) {
const status = trimmed.substring(2);
showFeeds(status);
return;
}
// Otherwise, treat as search
performSearch(cmd.trim());
}
// Show help
function showHelp() {
clearInfiniteScroll();
const output = document.getElementById('output');
const breadcrumb = document.getElementById('breadcrumb');
breadcrumb.innerHTML = '';
breadcrumb.style.display = 'none';
output.innerHTML = `
<div style="color: #888; padding: 20px;">
<div style="color: #fff; font-size: 1.2em; margin-bottom: 15px;">Commands</div>
<div style="margin-bottom: 10px;"><span style="color: #0af;">/domains</span> - List all domains</div>
<div style="margin-bottom: 10px;"><span style="color: #0af;">/feeds</span> - List all feeds</div>
<div style="margin-bottom: 10px;"><span style="color: #0af;">d:hold d:pass d:skip d:fail</span> - Filter domains by status</div>
<div style="margin-bottom: 10px;"><span style="color: #0af;">d:feeds</span> - Filter domains with discovered feeds</div>
<div style="margin-bottom: 10px;"><span style="color: #0af;">f:hold f:pass f:skip f:fail</span> - Filter feeds by status</div>
<div style="margin-bottom: 20px;"><span style="color: #0af;">[text]</span> - Search feeds</div>
<div style="color: #666;">Click domain/feed name to drill down. Click status buttons to set status.</div>
</div>`;
}
// Setup command input
function setupCommandInput() {
const input = document.getElementById('commandInput');
input.addEventListener('keydown', (e) => {
if (e.key === 'Enter') processCommand(input.value);
});
input.addEventListener('focus', () => input.select());
// Command buttons
document.querySelectorAll('.cmd-btn[data-cmd]').forEach(btn => {
btn.addEventListener('click', () => {
const cmd = btn.dataset.cmd;
input.value = cmd;
processCommand(cmd);
});
});
}
// Setup TLD filter
async function setupTLDFilter() {
const tldToggleBtn = document.getElementById('tldToggleBtn');
const tldDropdown = document.getElementById('tldDropdown');
const tldList = document.getElementById('tldList');
// Toggle dropdown
tldToggleBtn.addEventListener('click', async () => {
const isVisible = tldDropdown.style.display !== 'none';
if (isVisible) {
tldDropdown.style.display = 'none';
} else {
tldDropdown.style.display = 'block';
if (allTLDs.length === 0) {
await loadTLDs();
}
}
});
}
async function loadTLDs() {
const tldList = document.getElementById('tldList');
tldList.innerHTML = '<span style="color: #666;">Loading...</span>';
try {
const resp = await fetch('/api/tlds');
allTLDs = await resp.json();
renderTLDButtons();
} catch (err) {
tldList.innerHTML = `<span style="color: #f66;">Error: ${escapeHtml(err.message)}</span>`;
}
}
function renderTLDButtons() {
const tldList = document.getElementById('tldList');
const allSelected = selectedTLDs.size === 0 || selectedTLDs.size === allTLDs.length;
let html = '';
allTLDs.forEach(t => {
const isSelected = selectedTLDs.has(t.tld);
const bg = isSelected ? '#036' : '#1a1a1a';
const border = isSelected ? '#06c' : '#333';
const color = isSelected ? '#0af' : '#fff';
html += `<button class="tld-btn" data-tld="${escapeHtml(t.tld)}"
style="padding: 4px 8px; font-family: monospace;
background: ${bg}; border: 1px solid ${border}; border-radius: 3px;
color: ${color}; cursor: pointer;">.${escapeHtml(t.tld)} <span style="color: #888;">(${t.domain_count})</span></button>`;
});
// Add clear button if any selected
if (selectedTLDs.size > 0) {
html += `<button id="clearTLDs" style="padding: 4px 8px; font-family: monospace;
background: #400; border: 1px solid #600; border-radius: 3px;
color: #f66; cursor: pointer; margin-left: 10px;">clear</button>`;
}
tldList.innerHTML = html;
// Attach click handlers
tldList.querySelectorAll('.tld-btn').forEach(btn => {
btn.addEventListener('click', () => {
const tld = btn.dataset.tld;
if (selectedTLDs.has(tld)) {
selectedTLDs.delete(tld);
} else {
selectedTLDs.add(tld);
}
renderTLDButtons();
applyTLDFilter();
});
});
const clearBtn = document.getElementById('clearTLDs');
if (clearBtn) {
clearBtn.addEventListener('click', () => {
selectedTLDs.clear();
renderTLDButtons();
applyTLDFilter();
});
}
}
// Track manually collapsed TLDs (collapsed by clicking header)
let collapsedTLDs = new Set();
function applyTLDFilter() {
const container = document.querySelector('.domain-list');
if (!container) return;
const showAll = selectedTLDs.size === 0;
// Handle TLD headers and their domain blocks
container.querySelectorAll('.tld-header:not(.tld-handled)').forEach(header => {
header.classList.add('tld-handled');
header.style.cursor = 'pointer';
header.addEventListener('click', () => {
const tldText = header.dataset.tld;
// If TLD filter is active, clicking adds/removes from filter
if (selectedTLDs.size > 0) {
if (selectedTLDs.has(tldText)) {
selectedTLDs.delete(tldText);
} else {
selectedTLDs.add(tldText);
}
renderTLDButtons();
} else {
// No filter active - toggle manual collapse
if (collapsedTLDs.has(tldText)) {
collapsedTLDs.delete(tldText);
} else {
collapsedTLDs.add(tldText);
}
}
applyTLDVisibility();
});
});
// Store TLD in data attribute for headers that don't have it
container.querySelectorAll('.tld-header').forEach(header => {
if (!header.dataset.tld) {
const match = header.textContent.trim().match(/^\.(.+?)(?:\s|$)/);
if (match) {
header.dataset.tld = match[1];
}
}
});
applyTLDVisibility();
}
function applyTLDVisibility() {
const container = document.querySelector('.domain-list');
if (!container) return;
const showAll = selectedTLDs.size === 0;
container.querySelectorAll('.tld-header').forEach(header => {
const tldText = header.dataset.tld;
if (!tldText) return;
// Determine if this TLD should be expanded
let isExpanded;
if (selectedTLDs.size > 0) {
// Filter mode: show only selected TLDs
isExpanded = selectedTLDs.has(tldText);
} else {
// No filter: show all except manually collapsed
isExpanded = !collapsedTLDs.has(tldText);
}
// Find all domain blocks until next TLD header, track last domain block
let nextEl = header.nextElementSibling;
let lastDomainBlock = null;
while (nextEl && !nextEl.classList.contains('tld-header')) {
if (nextEl.classList.contains('domain-block')) {
nextEl.style.display = isExpanded ? 'block' : 'none';
lastDomainBlock = nextEl;
}
if (nextEl.classList.contains('tld-footer')) {
nextEl.style.display = isExpanded ? 'block' : 'none';
}
nextEl = nextEl.nextElementSibling;
}
// Insert footer if it doesn't exist and we have domain blocks
if (lastDomainBlock) {
let footer = lastDomainBlock.nextElementSibling;
if (!footer || !footer.classList.contains('tld-footer')) {
const footerHtml = `<div class="tld-footer" data-tld="${escapeHtml(tldText)}" style="padding: 6px 10px 12px; color: #555; font-size: 0.85em; cursor: pointer; display: ${isExpanded ? 'block' : 'none'};"><span style="margin-right: 6px; font-size: 0.8em;">▲</span>.${escapeHtml(tldText)}</div>`;
lastDomainBlock.insertAdjacentHTML('afterend', footerHtml);
// Add click handler to new footer
footer = lastDomainBlock.nextElementSibling;
footer.addEventListener('click', () => {
collapsedTLDs.add(tldText);
applyTLDVisibility();
});
}
}
// Style header based on expanded state
header.style.color = isExpanded ? '#888' : '#555';
// Add expand/collapse indicator
const indicator = isExpanded ? '▼' : '▶';
const tldDisplay = '.' + tldText;
if (!header.textContent.includes('▼') && !header.textContent.includes('▶')) {
header.innerHTML = `<span style="margin-right: 6px; font-size: 0.8em;">${indicator}</span>${tldDisplay}`;
} else {
header.innerHTML = header.innerHTML.replace(/[▼▶]/, indicator);
}
});
}
// Stats update
async function updateStats() {
try {
const resp = await fetch('/api/stats');
const stats = await resp.json();
document.getElementById('totalDomains').textContent = commaFormat(stats.total_domains);
document.getElementById('holdDomains').textContent = commaFormat(stats.hold_domains);
document.getElementById('passDomains').textContent = commaFormat(stats.pass_domains);
document.getElementById('skipDomains').textContent = commaFormat(stats.skip_domains);
document.getElementById('failDomains').textContent = commaFormat(stats.fail_domains);
document.getElementById('crawlRate').textContent = commaFormat(stats.crawl_rate);
document.getElementById('checkRate').textContent = commaFormat(stats.check_rate);
document.getElementById('totalFeeds').textContent = commaFormat(stats.total_feeds);
document.getElementById('rssFeeds').textContent = commaFormat(stats.rss_feeds);
document.getElementById('atomFeeds').textContent = commaFormat(stats.atom_feeds);
document.getElementById('unknownFeeds').textContent = commaFormat(stats.unknown_feeds);
const updatedAt = new Date(stats.updated_at);
document.getElementById('updatedAt').textContent = 'Last updated: ' + updatedAt.toISOString().replace('T', ' ').substring(0, 19);
} catch (err) {
console.error('Failed to update stats:', err);
}
}
// Initialize
setupCommandInput();
setupTLDFilter();
showHelp();
updateStats();
setInterval(updateStats, 1000);
}
window.onload = initDashboard;