/**
 * Connector scaffolder (SPEC §15).
 *
 *   pnpm connector:new <id> --url https://www.example.com --name "Example" [--engine api|firecrawl|scrapfly]
 *                       [--adapter shopify|woocommerce] [--type marketplace|auction_house|dealer|…]
 *                       [--categories pokemon,magic_the_gathering] [--country US] [--currency USD]
 *                       [--kind listing|sale|auction_lot|price_observation|catalog_item|population_report] [--group <entries-file>]
 *
 * Creates connectors/<engine>/<id>/{meta.json,index.ts,index.test.ts,README.md}, data/fixtures/<id>/,
 * a stub entry in data/sources/sources.json, then rebuilds the registry.
 */
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { execSync } from 'node:child_process';

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const [id, ...rest] = process.argv.slice(2);
const flags: Record<string, string> = {};
for (let i = 0; i < rest.length; i++) {
  const a = rest[i]!;
  if (a.startsWith('--')) {
    flags[a.slice(2)] = rest[i + 1] && !rest[i + 1]!.startsWith('--') ? rest[++i]! : 'true';
  }
}
if (!id || !/^[a-z0-9-]+$/.test(id) || !flags.url) {
  console.error('usage: pnpm connector:new <id> --url https://… --name "Name" [--engine api|firecrawl|scrapfly] [--adapter shopify|woocommerce] [--type marketplace] [--categories a,b] [--country US] [--currency USD] [--kind listing]');
  process.exit(1);
}
const adapter = flags.adapter as 'shopify' | 'woocommerce' | undefined;
const engine = (flags.engine ?? 'api') as 'api' | 'firecrawl' | 'scrapfly';
const name = flags.name ?? id;
const url = flags.url.replace(/\/+$/, '');
const host = new URL(url).hostname.replace(/^www\./, '');
const sourceType = flags.type ?? (adapter ? 'dealer' : 'marketplace');
const categories = (flags.categories ?? 'other_tcg').split(',').map((s) => s.trim()).filter(Boolean);
const country = flags.country ?? 'US';
const currency = flags.currency ?? 'USD';
const kind = flags.kind ?? (adapter ? 'listing' : 'sale');
const dir = path.join(root, 'connectors', engine, id);
if (existsSync(dir)) {
  console.error(`connector ${engine}/${id} already exists`);
  process.exit(1);
}
mkdirSync(dir, { recursive: true });
mkdirSync(path.join(root, 'data/fixtures', id), { recursive: true });

const meta = {
  id,
  displayName: name,
  sourceId: id,
  sourceName: name,
  sourceType,
  sourceUrl: url,
  module: `${engine}/${id}`,
  enginePriority: adapter ? ['api'] : engine === 'api' ? ['api'] : engine === 'firecrawl' ? ['firecrawl', 'scrapfly'] : ['scrapfly'],
  categories,
  regions: [country],
  languages: ['en'],
  currency: [currency],
  supportsListings: kind === 'listing',
  supportsSold: kind === 'sale',
  supportsAuctions: kind === 'auction_lot',
  supportsImages: true,
  supportsCatalog: kind === 'catalog_item',
  supportsPopulation: kind === 'population_report',
  supportsLookup: Boolean(adapter),
  refreshFrequencyMinutes: kind === 'listing' ? 720 : 1440,
  priority: 'medium',
  trustScore: 0.7,
  attributionRequired: true,
  termsUrl: `${url}/terms`,
  accessNotes: `TODO (SPEC §36): describe exactly which public pages/endpoints are read, robots.txt findings, rate limit, anti-bot observations and what is deliberately NOT fetched.`,
  enabled: true,
  schemaVersion: '1.0',
  acquisitionMethod: adapter === 'shopify' ? 'Shopify storefront products.json' : adapter === 'woocommerce' ? 'WooCommerce Store API' : engine === 'api' ? 'public JSON/HTML (direct HTTP)' : engine === 'firecrawl' ? 'Firecrawl markdown/HTML' : 'Scrapfly render',
  historicalDepth: kind === 'sale' || kind === 'auction_lot' ? 'years' : 'none',
  requires: [],
  config: adapter
    ? { currency, seller: name, collections: [{ handle: 'TODO-collection-handle', categorySlug: categories[0] }], rules: [], defaultCategory: null }
    : { seeds: [], pagesPerSeed: 2 },
};
writeFileSync(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n');

const adapterIndex = (cls: string) => `import { ${cls}, type ConnectorMeta } from '@rareindex/connectors';

/**
 * ${name} — ${adapter} storefront read through the shared ${adapter} adapter.
 * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency).
 */
export default (meta: ConnectorMeta) => new ${cls}(meta);
`;

const customIndex = `import { z } from 'zod';
import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
import { ${kind === 'sale' ? 'NormalizedSaleSchema' : kind === 'listing' ? 'NormalizedListingSchema' : kind === 'auction_lot' ? 'NormalizedAuctionLotSchema' : kind === 'price_observation' ? 'NormalizedPriceObservationSchema' : kind === 'population_report' ? 'NormalizedPopulationReportSchema' : 'NormalizedCatalogItemSchema'}, type NormalizedRecord } from '@rareindex/shared';

/**
 * ${name} (${host}) — ${kind} connector.
 * Acquisition: ${meta.acquisitionMethod}. See meta.json accessNotes for what is (not) fetched and why.
 */
const PARSER_VERSION = '1.0.0';
const SITE = '${url}';

/** One captured page. Keep the payload small and source-shaped; parsing happens in normalize(). */
export const RawPayloadSchema = z.object({
  seed: z.string(),
  page: z.number().int(),
  items: z.array(z.object({ id: z.string(), title: z.string(), url: z.string(), priceText: z.string().nullable(), dateText: z.string().nullable(), image: z.string().nullable() })),
});
export type RawPayload = z.infer<typeof RawPayloadSchema>;

/** Parse a fetched document into items. Exported so tests can exercise it on saved HTML/markdown. */
export function parsePage(doc: { html: string | null; markdown: string | null; json: unknown }): RawPayload['items'] {
  // TODO: prefer JSON (embedded __NEXT_DATA__/JSON-LD via html.jsonLd / adapters.productsFromHtml) over selectors.
  void doc;
  return [];
}

export class ${toClass(id)}Connector extends BaseConnector {
  readonly version = '1.0.0';
  readonly parserVersion = PARSER_VERSION;
  protected override minIntervalMs = 2000;

  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
    const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []);
    const maxPages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? this.policy.crawlDepth);
    const cursor = (ctx.options.cursor ?? {}) as { seed?: string; page?: number };
    let count = 0;
    for (const seed of seeds) {
      if (cursor.seed && cursor.seed !== seed && seeds.indexOf(seed) < seeds.indexOf(cursor.seed)) continue;
      for (let page = cursor.seed === seed && cursor.page ? cursor.page : 1; page <= maxPages; page++) {
        if (ctx.signal?.aborted || this.reached(ctx, count)) return;
        const url = \`\${SITE}/\${seed}?page=\${page}\`;
        await this.throttle(url);
        const res = await ctx.fetch(url, { expect: ['title', 'price'], parse: (r) => { const items = parsePage(r); return { title: items[0]?.title ?? null, price: items[0]?.priceText ?? null }; } });
        if (!res.success) {
          ctx.anomaly('page_fetch_failed', \`\${url}: \${res.error ?? res.httpStatus}\`);
          break;
        }
        const items = parsePage(res);
        if (!items.length) {
          ctx.anomaly(page === 1 ? 'parse_failure_page' : 'pagination_end', url);
          break;
        }
        count++;
        const payload: RawPayload = { seed, page, items };
        yield { url, externalId: \`\${seed}:p\${page}\`, kind: '${kind}', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
        await ctx.setCursor({ seed, page: page + 1, at: new Date().toISOString() });
        await ctx.progress({ page, itemsProcessed: count });
      }
    }
    await ctx.setCursor({ done: true, at: new Date().toISOString() });
  }

  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
    const p = RawPayloadSchema.parse(raw.payload);
    const out: NormalizedRecord[] = [];
    for (const it of p.items) {
      void it; // TODO: map to the canonical schema; keep native currency and the source's own date; never invent fields.
    }
    return out;
  }
}

function toClass(s: string): string {
  return s.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('');
}

export default (meta: ConnectorMeta) => new ${toClass(id)}Connector(meta);
`;

function toClass(s: string): string {
  return s.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('');
}

writeFileSync(path.join(dir, 'index.ts'), adapter ? adapterIndex(adapter === 'shopify' ? 'ShopifyStoreConnector' : 'WooCommerceStoreConnector') : customIndex.replace(/\nfunction toClass[\s\S]*?\n}\n/, '\n'));

const test = `import { describe, expect, it } from 'vitest';
import meta from './meta.json' with { type: 'json' };
import { localMeta } from '../../api/_lib/local-meta.js';
import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
import createConnector from './index.js';

const connector = createConnector(localMeta(meta));

describe('${id}', () => {
  // Standard invariants on every saved fixture (schema validity, prices > 0, source dates, currency, dedupe keys).
  runFixtureSuite(connector, it, expect);

  it('normalises fixtures into ${kind} records with native currency', async () => {
    for (const name of listFixtures('${id}')) {
      const out = await connector.normalize(loadFixture('${id}', name).raw);
      expect(out.length).toBeGreaterThan(0);
      for (const r of out) {
        expect(r.kind).toBe('${kind}');
        if ('currency' in r && r.currency) expect(meta.currency).toContain(r.currency);
      }
    }
  });
});
`;
writeFileSync(path.join(dir, 'index.test.ts'), test);

writeFileSync(
  path.join(dir, 'README.md'),
  `# ${name} connector (\`${id}\`)

- Source: ${url} (${host}) · type: ${sourceType} · country: ${country} · currency: ${currency}
- Acquisition: ${meta.acquisitionMethod}
- Records: \`${kind}\`
- Status: scaffolded — complete the TODOs, capture at least one real fixture, run \`pnpm vitest run connectors/${engine}/${id}\`.

## Access & compliance
Fill \`accessNotes\` in meta.json: which public pages/endpoints are read, robots.txt findings, rate limit, anti-bot behaviour, what is intentionally not fetched (logins, paywalls, personal data).

## Fixtures
\`data/fixtures/${id}/<name>.json\` = \`{ raw: { url, externalId, kind, engine, fetchedAt, payload }, expect: { minCount, kinds, requiredFields } }\` captured from a live page (never hand-written). Large HTML goes next to it as \`<name>.html\` (\`payload.snapshot\`).

## Checklist (SPEC §35)
discovery · pagination · normalisation · stable ids · error handling · rate limiting (domains.json) · retries · fixtures · tests · health · logging · persistence · refresh class · source metadata (data/sources/sources.json) · documentation
`,
);

// source registry stub → data/sources/entries/<group>.json (fragments are merged into sources.json by build-sources)
const group = flags.group ?? 'scaffold';
mkdirSync(path.join(root, 'data/sources/entries'), { recursive: true });
const sourcesPath = path.join(root, 'data/sources/entries', `${group}.json`);
const sources = { sources: existsSync(sourcesPath) ? (JSON.parse(readFileSync(sourcesPath, 'utf8')) as Array<Record<string, unknown>>) : [] };
if (!sources.sources.some((s) => s.id === id)) {
  sources.sources.push({ id, name, domain: host, country, supportedCountries: [country], languages: ['en'], sourceType, categories, api: { available: engine === 'api' && !adapter, docsUrl: null, auth: 'none', notes: null }, urls: { search: null, product: null, sold: null, sitemap: `${url}/sitemap.xml`, robots: `${url}/robots.txt` }, data: { liveListings: kind === 'listing', soldResults: kind === 'sale', auctionResults: kind === 'auction_lot', historical: meta.historicalDepth, catalog: kind === 'catalog_item', population: kind === 'population_report' }, access: { pagination: 'page_number', jsRendering: engine !== 'api', antiBot: 'none', cloudflare: false, preferredEngine: adapter ?? engine }, currency: [currency], freshness: 'daily', priority: 'wave3', difficulty: adapter ? 'easy' : 'medium', reliability: 0.6, legal: null, status: 'planned', connectors: [id], statusReason: 'scaffolded, not yet verified', verified: false, verifiedAt: null, notes: null });
  sources.sources.sort((a, b) => String(a.id).localeCompare(String(b.id)));
  writeFileSync(sourcesPath, JSON.stringify(sources.sources, null, 2) + '\n');
}

execSync('pnpm tsx scripts/build-registry.ts', { cwd: root, stdio: 'inherit' });
console.log(`\n✔ scaffolded connectors/${engine}/${id}\n  next: fill meta.json (collections/seeds, accessNotes), implement parsePage/normalize, capture a fixture, run:\n  pnpm vitest run connectors/${engine}/${id} && pnpm tsx scripts/build-sources.ts`);