spb/rareindex

1import { gunzipSync } from 'node:zlib';2import type { CrawlContext } from '../types.js';34/** Sitemap ingestion (SPEC §1): sitemap indexes, url sets, gzip, lastmod filtering. */5export interface SitemapEntry {6  loc: string;7  lastmod: string | null;8  changefreq?: string | null;9  priority?: number | null;10}1112export function decodeXml(s: string): string {13  return s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&apos;/g, "'");14}1516/** Parse <urlset> entries. */17export function parseUrlset(xml: string): SitemapEntry[] {18  const out: SitemapEntry[] = [];19  const re = /<url>([\s\S]*?)<\/url>/g;20  let m: RegExpExecArray | null;21  while ((m = re.exec(xml))) {22    const loc = m[1]!.match(/<loc>\s*([^<\s]+)\s*<\/loc>/)?.[1];23    if (!loc) continue;24    const pr = m[1]!.match(/<priority>\s*([\d.]+)\s*<\/priority>/)?.[1];25    out.push({ loc: decodeXml(loc), lastmod: m[1]!.match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/)?.[1] ?? null, changefreq: m[1]!.match(/<changefreq>\s*([^<\s]+)\s*<\/changefreq>/)?.[1] ?? null, priority: pr ? Number(pr) : null });26  }27  return out;28}2930/** Parse <sitemapindex> entries (child sitemap locations). */31export function parseSitemapIndex(xml: string): SitemapEntry[] {32  const out: SitemapEntry[] = [];33  const re = /<sitemap>([\s\S]*?)<\/sitemap>/g;34  let m: RegExpExecArray | null;35  while ((m = re.exec(xml))) {36    const loc = m[1]!.match(/<loc>\s*([^<\s]+)\s*<\/loc>/)?.[1];37    if (!loc) continue;38    out.push({ loc: decodeXml(loc), lastmod: m[1]!.match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/)?.[1] ?? null });39  }40  return out;41}4243export function isSitemapIndex(xml: string): boolean {44  return /<sitemapindex[\s>]/i.test(xml.slice(0, 2000));45}4647function toText(res: { html: string | null; buffer?: Uint8Array | null }, url: string): string | null {48  if (res.buffer && res.buffer.byteLength) {49    const b = Buffer.from(res.buffer);50    const gz = b.length >= 2 && b[0] === 0x1f && b[1] === 0x8b;51    return (gz || url.endsWith('.gz') ? gunzipSync(b) : b).toString('utf8');52  }53  return res.html;54}5556export interface DiscoverOptions {57  /** keep only URLs matching this pattern */58  match?: RegExp;59  /** keep only child sitemaps whose URL matches (sitemap indexes) */60  matchSitemap?: RegExp;61  /** skip entries whose lastmod is older than this */62  since?: Date | null;63  /** cap on URLs returned */64  limit?: number;65  /** cap on child sitemaps fetched */66  maxSitemaps?: number;67  /** newest first when lastmod is present */68  newestFirst?: boolean;69}7071/**72 * Discover page URLs from a sitemap (or sitemap index), following children, decompressing .gz and73 * filtering by pattern/date. Uses the connector's routed fetch (budget + stats + politeness).74 */75export async function discoverFromSitemap(ctx: Pick<CrawlContext, 'fetch' | 'anomaly' | 'signal'>, sitemapUrl: string, opts: DiscoverOptions = {}): Promise<SitemapEntry[]> {76  const out: SitemapEntry[] = [];77  const queue = [sitemapUrl];78  let fetched = 0;79  const maxSitemaps = opts.maxSitemaps ?? 50;80  while (queue.length && fetched < maxSitemaps) {81    if (ctx.signal?.aborted) break;82    const url = queue.shift()!;83    fetched++;84    const res = await ctx.fetch(url, { engines: ['api'], responseType: url.endsWith('.gz') ? 'binary' : 'text', minQuality: 0, force: true });85    const xml = res.success ? toText(res, url) : null;86    if (!xml) {87      ctx.anomaly('sitemap_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);88      continue;89    }90    if (isSitemapIndex(xml)) {91      let children = parseSitemapIndex(xml);92      if (opts.matchSitemap) children = children.filter((c) => opts.matchSitemap!.test(c.loc));93      if (opts.since) children = children.filter((c) => !c.lastmod || new Date(c.lastmod) >= opts.since!);94      if (opts.newestFirst) children.sort((a, b) => (b.lastmod ?? '').localeCompare(a.lastmod ?? ''));95      queue.push(...children.map((c) => c.loc));96      continue;97    }98    let entries = parseUrlset(xml);99    if (opts.match) entries = entries.filter((e) => opts.match!.test(e.loc));100    if (opts.since) entries = entries.filter((e) => !e.lastmod || new Date(e.lastmod) >= opts.since!);101    out.push(...entries);102    if (opts.limit && out.length >= opts.limit) break;103  }104  if (opts.newestFirst) out.sort((a, b) => (b.lastmod ?? '').localeCompare(a.lastmod ?? ''));105  return opts.limit ? out.slice(0, opts.limit) : out;106}107108/** robots.txt: Sitemap: directives + whether a path is disallowed for our UA / '*'. */109export function parseRobots(txt: string, agent = 'rareindexbot'): { sitemaps: string[]; disallow: string[]; allow: string[]; crawlDelay: number | null } {110  const lines = txt.split(/\r?\n/).map((l) => l.replace(/#.*$/, '').trim()).filter(Boolean);111  const sitemaps: string[] = [];112  const groups: Array<{ agents: string[]; allow: string[]; disallow: string[]; delay: number | null }> = [];113  let cur: (typeof groups)[number] | null = null;114  for (const l of lines) {115    const [kRaw, ...rest] = l.split(':');116    const k = kRaw!.trim().toLowerCase();117    const v = rest.join(':').trim();118    if (k === 'sitemap') sitemaps.push(v);119    else if (k === 'user-agent') {120      if (!cur || cur.allow.length || cur.disallow.length) {121        cur = { agents: [], allow: [], disallow: [], delay: null };122        groups.push(cur);123      }124      cur.agents.push(v.toLowerCase());125    } else if (cur && k === 'disallow') cur.disallow.push(v);126    else if (cur && k === 'allow') cur.allow.push(v);127    else if (cur && k === 'crawl-delay') cur.delay = Number(v) || null;128  }129  const g = groups.find((x) => x.agents.some((a) => agent.includes(a) && a !== '*')) ?? groups.find((x) => x.agents.includes('*'));130  return { sitemaps, disallow: g?.disallow ?? [], allow: g?.allow ?? [], crawlDelay: g?.delay ?? null };131}132133export function robotsAllows(rules: { allow: string[]; disallow: string[] }, path: string): boolean {134  const match = (rule: string) => {135    if (!rule) return false;136    const re = new RegExp('^' + rule.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\\\$$/, '$'));137    return re.test(path);138  };139  const longest = (rules_: string[]) => rules_.filter(match).sort((a, b) => b.length - a.length)[0] ?? '';140  const a = longest(rules.allow);141  const d = longest(rules.disallow);142  if (!d) return true;143  return a.length >= d.length;144}145