115 lines
4.1 KiB
JavaScript
115 lines
4.1 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
const DIST_DIR = path.resolve('dist');
|
|
|
|
if (!fs.existsSync(DIST_DIR)) {
|
|
console.error(`❌ Dist directory not found at ${DIST_DIR}. Please run "pnpm build" first.`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(`\n======================================================`);
|
|
console.log(`🗺️ Sitemap 100% Coverage & Index Integrity Validator`);
|
|
console.log(`======================================================\n`);
|
|
|
|
// 1. Locate all sitemap XML files
|
|
const allFiles = fs.readdirSync(DIST_DIR);
|
|
const sitemapFiles = allFiles.filter(f => f.startsWith('sitemap') && f.endsWith('.xml'));
|
|
|
|
if (sitemapFiles.length === 0) {
|
|
console.error(`❌ No sitemap XML files found in ${DIST_DIR}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(`Found ${sitemapFiles.length} sitemap file(s):`);
|
|
sitemapFiles.forEach(f => console.log(` - ${f}`));
|
|
|
|
// 2. Parse all URLs from sitemaps
|
|
const sitemapUrls = new Set();
|
|
const sitemapIndexFiles = [];
|
|
|
|
for (const sitemapFile of sitemapFiles) {
|
|
const content = fs.readFileSync(path.join(DIST_DIR, sitemapFile), 'utf-8');
|
|
if (content.includes('<sitemapindex')) {
|
|
sitemapIndexFiles.push(sitemapFile);
|
|
}
|
|
const locMatches = [...content.matchAll(/<loc>([^<]+)<\/loc>/g)];
|
|
for (const match of locMatches) {
|
|
const url = match[1].trim();
|
|
// Only count content page URLs, not nested sitemap pointers
|
|
if (!url.endsWith('.xml')) {
|
|
sitemapUrls.add(url);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`\nTotal unique page URLs extracted from sitemaps: ${sitemapUrls.size}`);
|
|
if (sitemapIndexFiles.length > 0) {
|
|
console.log(`Sitemap Index properly configured: ${sitemapIndexFiles.join(', ')}`);
|
|
}
|
|
|
|
// 3. Scan all generated HTML pages (excluding 404 and admin)
|
|
function getIndexableHtmlPaths(dir, base = dir, paths = []) {
|
|
const items = fs.readdirSync(dir);
|
|
for (const item of items) {
|
|
const fullPath = path.join(dir, item);
|
|
const stat = fs.statSync(fullPath);
|
|
if (stat.isDirectory()) {
|
|
getIndexableHtmlPaths(fullPath, base, paths);
|
|
} else if (item.endsWith('.html')) {
|
|
const relPath = path.relative(base, fullPath);
|
|
// Skip 404 or preview
|
|
if (!relPath.includes('404') && !relPath.includes('preview')) {
|
|
let route = '/' + relPath.replace(/index\.html$/, '').replace(/\.html$/, '');
|
|
if (route !== '/' && route.endsWith('/')) {
|
|
route = route.slice(0, -1);
|
|
}
|
|
paths.push({ relPath, route });
|
|
}
|
|
}
|
|
}
|
|
return paths;
|
|
}
|
|
|
|
const htmlPages = getIndexableHtmlPaths(DIST_DIR);
|
|
console.log(`Total generated indexable HTML routes: ${htmlPages.length}`);
|
|
|
|
// 4. Compare HTML routes vs Sitemap URLs
|
|
const siteBaseUrl = 'https://agentskillshub.top';
|
|
const missingFromSitemap = [];
|
|
|
|
for (const page of htmlPages) {
|
|
const expectedUrl = page.route === '/' ? `${siteBaseUrl}/` : `${siteBaseUrl}${page.route}`;
|
|
const normalizedExpected = expectedUrl.replace(/\/+$/, '');
|
|
|
|
let found = false;
|
|
for (const sitemapUrl of sitemapUrls) {
|
|
if (sitemapUrl.replace(/\/+$/, '') === normalizedExpected) {
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!found) {
|
|
missingFromSitemap.push(expectedUrl);
|
|
}
|
|
}
|
|
|
|
// 5. Output Report
|
|
console.log(`\n📊 Coverage Verification:`);
|
|
const coveragePercent = (( (htmlPages.length - missingFromSitemap.length) / htmlPages.length ) * 100).toFixed(1);
|
|
console.log(` Sitemap Coverage: ${coveragePercent}% (${htmlPages.length - missingFromSitemap.length} / ${htmlPages.length})`);
|
|
|
|
if (missingFromSitemap.length > 0) {
|
|
console.error(`\n⚠️ WARNING: Found ${missingFromSitemap.length} generated URL(s) MISSING from sitemaps:`);
|
|
missingFromSitemap.slice(0, 10).forEach(url => console.error(` ❌ Missing: ${url}`));
|
|
if (missingFromSitemap.length > 10) {
|
|
console.error(` ... and ${missingFromSitemap.length - 10} more.`);
|
|
}
|
|
console.error(`\n💡 This is the exact root cause of the "19,600 generated but only 4,050 submitted" problem.`);
|
|
console.error(` Ensure @astrojs/sitemap filter and entryLimit match all programmatic dynamic routes!`);
|
|
process.exit(1);
|
|
} else {
|
|
console.log(`\n✅ 100% Sitemap Coverage! Every single generated indexable page is submitted.`);
|
|
}
|