HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1import 'server-only';2import { api, safe } from '@/lib/api';3import { routes, SITE_URL, TYPE_PATH } from '@/lib/site';45export const SITEMAP_HEADERS = { 'content-type': 'application/xml; charset=utf-8', 'cache-control': 'public, s-maxage=3600, stale-while-revalidate=86400' };6export const SHARD = 5000;7/** Static public routes (1.1 surface). Query-driven pages are listed once at their canonical default URL. */8const STATIC = [9 '/',10 '/models',11 '/companies',12 '/papers',13 '/providers',14 '/benchmarks',15 '/benchmarks?view=matrix',16 '/hardware',17 '/hardware/fit',18 '/frameworks',19 '/datasets',20 '/tools',21 '/agents',22 '/families',23 '/licenses',24 '/open',25 '/frontier',26 '/prices',27 '/pulse',28 '/calculator',29 '/run-locally',30 '/find-a-model',31 '/compare',32 '/graph',33 '/changes',34 '/timeline',35 '/time-machine',36 '/diff',37 '/explore',38 '/methodology',39 '/sources',40 '/about',41 '/developers',42 '/bot',43];44/** Types with their own sitemap series (in this order); everything else goes into the "other" series. */45const SERIES = ['model', 'company', 'paper', 'provider', 'benchmark', 'hardware', 'framework', 'dataset', 'tool', 'repository', 'model_family', 'artifact', 'license'] as const;4647function esc(s: string): string {48 return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');49}5051export function toIndex(locs: string[]): string {52 return `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${locs.map((l) => ` <sitemap><loc>${esc(SITE_URL + l)}</loc></sitemap>`).join('\n')}\n</sitemapindex>\n`;53}54export function toUrlset(entries: { loc: string; lastmod?: string }[]): string {55 return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${entries.map((e) => ` <url><loc>${esc(SITE_URL + e.loc)}</loc>${e.lastmod ? `<lastmod>${e.lastmod.slice(0, 10)}</lastmod>` : ''}</url>`).join('\n')}\n</urlset>\n`;56}5758/** Shard ids: `static`, `changes` (daily digests of the observation history), then `<type>-<n>` per type with entities. */59export async function shardIds(): Promise<string[]> {60 const ids = ['static', 'changes'];61 const stats = await safe(api.stats());62 for (const t of SERIES) {63 const n = Number(stats?.entities?.[t] ?? 0);64 const shards = n > 0 ? Math.ceil(n / SHARD) : 0;65 for (let i = 0; i < shards; i++) ids.push(`${t}-${i}`);66 }67 return ids;68}6970/** One /changes/<date> URL per UTC day from the first entity to today (bounded to 3 years). */71async function changesEntries(): Promise<{ loc: string; lastmod?: string }[]> {72 const stats = await safe(api.stats());73 const first = stats?.first_entity_at ? new Date(stats.first_entity_at) : null;74 if (!first || Number.isNaN(first.getTime())) return [];75 const today = new Date();76 const start = Math.max(first.getTime(), today.getTime() - 3 * 365 * 86400000);77 const out: { loc: string; lastmod?: string }[] = [];78 for (let t = start; t <= today.getTime(); t += 86400000) {79 const day = new Date(t).toISOString().slice(0, 10);80 out.push({ loc: routes.changesDay(day) });81 }82 return out;83}8485export async function shardEntries(id: string): Promise<{ loc: string; lastmod?: string }[] | null> {86 if (id === 'static') return STATIC.map((loc) => ({ loc }));87 if (id === 'changes') return changesEntries();88 const m = /^([a-z_]+)-(\d+)$/.exec(id);89 if (!m) return null;90 const type = m[1] as string;91 const n = Number(m[2]);92 if (!(SERIES as readonly string[]).includes(type) || !TYPE_PATH[type]) return null;93 const res = await safe(api.sitemap(type, SHARD, n * SHARD));94 if (!res) return [];95 return res.items.map((it) => ({ loc: routes.entity({ entity_type: it.entity_type, slug: it.slug }), lastmod: it.updated_at }));96}97