discovery: plain sitemaps enter shadow at tier C (news sitemaps stay B) — 168 shadow sitemaps at 15 min were the heaviest new load
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +1 −1
modified
packages/connectors/src/discovery-deep.ts
+1 −1
@@ -326,7 +326,7 @@ export async function discoverOrganization(domain: string, opts: DeepDiscoveryOp | ||
| 326 | 326 | if (!n) return; |
| 327 | 327 | const lastmods = s.entries.filter((e) => e.lastmod).length; |
| 328 | 328 | const isNews = /news/i.test(url) || s.entries.some((e) => e.publishedAt); |
| 329 | − sitemaps.push({ url: o.meta.finalUrl || url, type: "SITEMAP", connector: "sitemap", kind: "sitemap", name: isNews ? "news sitemap" : "sitemap", config: { maxChildren: 4, maxUrls: 3000 }, suggestedTier: "B", evidence: `${sitemapsFromRobots.includes(url) ? "robots.txt" : "well-known path"} · ${n} ${s.kind === "sitemapindex" ? "children" : "urls"}`, value: 0.45 + (s.entries.length ? 0.3 * (lastmods / s.entries.length) : 0.15) + (isNews ? 0.2 : 0), itemCount: n, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } }); | |
| 329 | + sitemaps.push({ url: o.meta.finalUrl || url, type: "SITEMAP", connector: "sitemap", kind: "sitemap", name: isNews ? "news sitemap" : "sitemap", config: { maxChildren: 4, maxUrls: 3000 }, suggestedTier: isNews ? "B" : "C", evidence: `${sitemapsFromRobots.includes(url) ? "robots.txt" : "well-known path"} · ${n} ${s.kind === "sitemapindex" ? "children" : "urls"}`, value: 0.45 + (s.entries.length ? 0.3 * (lastmods / s.entries.length) : 0.15) + (isNews ? 0.2 : 0), itemCount: n, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } }); | |
| 330 | 330 | } catch { |
| 331 | 331 | // not a sitemap |
| 332 | 332 | } |
| 333 | 333 | |