spb/rareindex

1/** Live smoke: pnpm tsx connectors/scrapfly/chrono24/_smoke.ts [/brand/model--modN.htm …]  (CAPTURE=1 to save fixtures). ~40 Scrapfly credits per page. */2import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';3import { saveFixture } from '@rareindex/connectors/testing';4import { env } from '@rareindex/shared';5import createConnector from './index.js';67const e = env();8const meta = getConnectorMeta('chrono24');9const connector = createConnector(meta);10const router = createRouter({ firecrawlApiKey: e.FIRECRAWL_API_KEY, scrapflyApiKey: e.SCRAPFLY_API_KEY });11const seeds = process.argv.slice(2);12const ctx = createCrawlContext({ router, meta: { ...meta, config: { ...meta.config, pagesPerSeed: 1 } }, options: { mode: 'probe', limit: Number(process.env.LIMIT ?? 1), seeds: seeds.length ? seeds : undefined } });1314for await (const raw of connector.crawl(ctx)) {15  const recs = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });16  const p = raw.payload as { brand: string; model: string; offers: unknown[]; totalListings: number | null };17  console.log(`\n# ${raw.url} — ${p.brand} ${p.model}: ${p.offers.length} offers on page, ${p.totalListings} total → ${recs.length} listings`);18  for (const r of recs.slice(0, 3)) console.log(JSON.stringify(r).slice(0, 700));19  if (process.env.CAPTURE) {20    const slug = raw.url.split('/').slice(-2).join('__').replace(/\.htm$/, '');21    saveFixture('chrono24', slug, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 10, kinds: ['listing'] }, note: `captured live from ${raw.url}` });22  }23}24console.log('\nengine stats', ctx.engineStats, 'anomalies', ctx.anomalies);25