import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
import { saveFixture } from '@rareindex/connectors/testing';
import metaJson from './meta.json' with { type: 'json' };
import createConnector from './index.js';

const save = process.argv.includes('--save');
const meta = ConnectorMetaSchema.parse(metaJson);
const connector = createConnector(meta);
const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2 } });
let i = 0;
for await (const raw of connector.crawl(ctx)) {
  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
  console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} records (${out.filter((r) => r.kind === 'price_observation').length} observations)`);
  for (const r of out.filter((r) => r.kind === 'price_observation').slice(0, 3)) console.log('  ', JSON.stringify(r).slice(0, 420));
  if (save && i < 2) saveFixture(meta.id, i === 0 ? 'bordeaux-page-1' : 'bordeaux-page-2', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 40, kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.name'] }, note: 'Captured live from idealwine.com /en/cote/bordeaux (plain HTTPS)' });
  i++;
}
console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);