import { z } from 'zod';
import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
import { parseGradeFromTitle } from '@rareindex/taxonomy';
import { extractYear, parsePrice, parseSourceDate, type NormalizedCatalogItem, type NormalizedPriceObservation, type NormalizedRecord, type NormalizedSale, type AssetAttributes, type Grade } from '@rareindex/shared';

/**
 * Shared core for the PriceCharting family of sites (pricecharting.com, sportscardspro.com).
 *
 * A product page carries (a) guide values per condition/grade column and (b) tables of recently
 * completed eBay sales per condition/grade tab (date · title · price · eBay id). The page is
 * stored compactly as one raw record and normalised into one catalog_item, one price_observation
 * per non-empty guide cell ('guide_value') and one sale per completed-sale row.
 *
 * Families: video_games · lego · funko · comics (PriceCharting) · cards (Pokémon / Magic / Yu-Gi-Oh!
 * on PriceCharting) · sports (SportsCardsPro). Card pages use grade columns instead of conditions.
 */

export const PARSER_VERSION = '2.0.0';

export const PriceCellSchema = z.object({ key: z.string(), value: z.number().nullable(), raw: z.string() });
export const SaleRowSchema = z.object({ tab: z.string(), date: z.string(), title: z.string(), price: z.number(), ebayId: z.string().nullable(), listedPrice: z.number().nullable() });
export const SetRefSchema = z.object({ id: z.string().nullable(), code: z.string().nullable(), name: z.string().nullable(), source: z.string() });
export const CardRefSchema = z.object({ scryfall_id: z.string().optional(), number: z.string().optional(), pokemontcg_id: z.string().optional(), tcgplayer_id: z.string().optional() });
export const ProductPayloadSchema = z.object({
  kind: z.literal('product'),
  url: z.string(),
  productId: z.string().nullable(),
  consoleUri: z.string(),
  consoleName: z.string(),
  title: z.string(),
  flags: z.object({ isComic: z.boolean(), isLegoSet: z.boolean(), isFunkoPop: z.boolean(), isCard: z.boolean(), isCoin: z.boolean(), isSystem: z.boolean() }),
  columnLabels: z.array(z.string()),
  prices: z.array(PriceCellSchema),
  /** card pages: every row of the "full prices" table (Ungraded, Grade 1 … PSA 10, BGS 10 Black…) */
  fullPrices: z.array(z.object({ label: z.string(), value: z.number().nullable() })).default([]),
  /** tab id → human label ("grade-seventeen" → "CGC 10"), parsed from the page's tab selector */
  tabLabels: z.record(z.string(), z.string()).default({}),
  sales: z.array(SaleRowSchema),
  details: z.record(z.string(), z.string()),
  images: z.array(z.string()),
  /** set resolved against a reference catalog at crawl time (TCGdex / pokemontcg / Scryfall) */
  setRef: SetRefSchema.nullable().default(null),
  /** card resolved against a reference catalog at crawl time (Scryfall exact-name lookup) */
  cardRef: CardRefSchema.nullable().default(null),
  site: z.enum(['pricecharting', 'sportscardspro']).default('pricecharting'),
});
export type ProductPayload = z.infer<typeof ProductPayloadSchema>;

export type Family = 'video_games' | 'lego' | 'funko' | 'comics' | 'cards' | 'sports';

interface ColumnMeaning {
  condition: string | null;
  completeness: string | null;
  conditionRaw: string;
  grade?: string | null;
}

export const PRICE_KEYS = ['used_price', 'complete_price', 'new_price', 'graded_price', 'box_only_price', 'manual_only_price'] as const;
export const TAB_TO_KEY: Record<string, (typeof PRICE_KEYS)[number]> = {
  used: 'used_price',
  cib: 'complete_price',
  new: 'new_price',
  graded: 'graded_price',
  'box-only': 'box_only_price',
  'manual-only': 'manual_only_price',
  'loose-and-manual': 'graded_price',
};

/** Column semantics per family (the site reuses the same six cells with different labels). */
export const COLUMN_MEANING: Record<Exclude<Family, 'cards' | 'sports'>, Partial<Record<(typeof PRICE_KEYS)[number], ColumnMeaning>>> = {
  video_games: {
    used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Loose' },
    complete_price: { condition: 'cib', completeness: 'cib', conditionRaw: 'Complete in box' },
    new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },
    graded_price: { condition: null, completeness: null, conditionRaw: 'Graded (any grader)' },
    box_only_price: { condition: null, completeness: 'box_only', conditionRaw: 'Box only' },
    manual_only_price: { condition: null, completeness: 'manual_only', conditionRaw: 'Manual only' },
  },
  lego: {
    used_price: { condition: 'used_complete', completeness: 'used_complete', conditionRaw: 'Pieces only (used, complete pieces)' },
    complete_price: { condition: 'opened_complete', completeness: 'opened_complete', conditionRaw: 'Complete (pieces, box, manual)' },
    new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },
    manual_only_price: { condition: null, completeness: 'instructions_only', conditionRaw: 'Manual only' },
  },
  funko: {
    used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Out of box' },
    complete_price: { condition: 'boxed', completeness: 'boxed', conditionRaw: 'In damaged box' },
    new_price: { condition: 'mint_in_box', completeness: 'boxed', conditionRaw: 'New (mint in box)' },
  },
  comics: {
    used_price: { condition: null, completeness: null, conditionRaw: 'Ungraded (raw)', grade: null },
    complete_price: { condition: 'very_good', completeness: null, conditionRaw: 'Graded 4.0 / VG (any grader)', grade: '4.0' },
    new_price: { condition: 'fine', completeness: null, conditionRaw: 'Graded 6.0 / Fine (any grader)', grade: '6.0' },
    graded_price: { condition: 'very_fine', completeness: null, conditionRaw: 'Graded 8.0 / VF (any grader)', grade: '8.0' },
    box_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.2 / NM- (any grader)', grade: '9.2' },
    manual_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.8 (any grader)', grade: '9.8' },
  },
};

/** Card pages: the six main cells map to Ungraded / Grade 7 / Grade 8 / Grade 9 / Grade 9.5 / PSA 10. */
export const CARD_CELL_LABEL: Record<(typeof PRICE_KEYS)[number], string> = {
  used_price: 'Ungraded',
  complete_price: 'Grade 7',
  new_price: 'Grade 8',
  graded_price: 'Grade 9',
  box_only_price: 'Grade 9.5',
  manual_only_price: 'PSA 10',
};
const CARD_TAB_LABEL: Record<string, string> = { used: 'Ungraded', cib: 'Grade 7', new: 'Grade 8', graded: 'Grade 9', 'box-only': 'Grade 9.5', 'manual-only': 'PSA 10' };

/** Generic grader used when the site reports a grade without a grading company ("Grade 9"). */
export const GENERIC_GRADER = 'graded';

export interface CardGrade {
  grader: string | null;
  grade: string | null;
  qualifier: string | null;
  conditionRaw: string;
}

/** "PSA 10" | "BGS 10 Black" | "CGC 10 Pristine" | "Grade 9.5" | "Ungraded" → structured grade. */
export function gradeFromLabel(label: string): CardGrade | null {
  const l = label.replace(/\s*\(\d+\)\s*$/, '').trim();
  if (!l) return null;
  if (/^ungraded$/i.test(l)) return { grader: null, grade: null, qualifier: null, conditionRaw: 'Ungraded (raw)' };
  let m = l.match(/^grade\s+(\d+(?:\.\d)?)$/i);
  if (m) return { grader: GENERIC_GRADER, grade: m[1]!, qualifier: null, conditionRaw: `Grade ${m[1]} (any grader)` };
  m = l.match(/^(PSA|BGS|CGC|SGC|TAG|ACE)\s+(\d+(?:\.\d)?)\s*(Black|Pristine|Black Label)?$/i);
  if (m) {
    const q = m[3] ? (/black/i.test(m[3]) ? 'Black Label' : 'Pristine') : null;
    return { grader: m[1]!.toLowerCase(), grade: m[2]!, qualifier: q, conditionRaw: l };
  }
  return null;
}

/** Refine a generic grade with the grader named in the eBay row title when it agrees on the grade. */
export function refineGradeFromTitle(base: CardGrade, rowTitle: string): CardGrade {
  if (base.grader !== GENERIC_GRADER || !base.grade) return base;
  const t = parseGradeFromTitle(rowTitle);
  if (t.grader && t.grader !== 'raw' && t.grade && Number(t.grade) === Number(base.grade)) {
    return { grader: t.grader, grade: base.grade, qualifier: t.qualifier ?? null, conditionRaw: `${t.grader.toUpperCase()} ${base.grade}` };
  }
  return base;
}

const NINTENDO = /^(nes|famicom|super-nintendo|super-famicom|nintendo-64|gamecube|wii|wii-u|nintendo-switch|gameboy|gameboy-color|gameboy-advance|nintendo-ds|nintendo-3ds|virtual-boy|jp-|pal-)/;
const SEGA = /^(sega-|pal-sega|jp-sega)/;
const PLAYSTATION = /^(playstation|psp|playstation-vita|jp-playstation|pal-playstation)/;
const XBOX = /^xbox/;
const RETRO = /^(atari|intellivision|colecovision|neo-geo|turbografx|pc-engine|commodore|amiga|vectrex|3do|jaguar|philips-cd-i|magnavox|odyssey|msx|sharp|wonderswan|n-gage|game-com|tiger|evercade|super-cassette|fairchild|bally|arcadia|action-max|amiga-cd32)/;
const SPORTS = ['basketball', 'baseball', 'football', 'hockey', 'soccer', 'wrestling', 'golf', 'racing', 'boxing', 'tennis', 'ufc', 'mma', 'formula', 'multi-sport', 'non-sport'] as const;

export function familyOf(p: ProductPayload): Family | null {
  if (p.site === 'sportscardspro') return 'sports';
  if (p.flags.isComic || p.consoleUri.startsWith('comic-books')) return 'comics';
  if (p.flags.isLegoSet || p.consoleUri.startsWith('lego')) return 'lego';
  if (p.flags.isFunkoPop || p.consoleUri.startsWith('funko')) return 'funko';
  if (p.flags.isCoin) return null; // coins handled elsewhere
  if (p.flags.isCard || /^(pokemon|magic|yugioh)-/.test(p.consoleUri)) {
    return /^(pokemon|magic|yugioh)-/.test(p.consoleUri) ? 'cards' : null; // other card games not mapped yet
  }
  return 'video_games';
}

export function categorySlug(p: ProductPayload, family: Family): string | null {
  const c = p.consoleUri;
  switch (family) {
    case 'lego':
      return 'lego_sets';
    case 'funko':
      return 'funko';
    case 'comics': {
      const pub = (p.details['Publisher'] ?? '').toLowerCase();
      if (/marvel/.test(pub)) return 'marvel_comics';
      if (/\bdc\b|dc comics|vertigo|wildstorm/.test(pub)) return 'dc_comics';
      return 'independent_comics';
    }
    case 'cards':
      if (c.startsWith('pokemon-')) return 'pokemon';
      if (c.startsWith('magic-')) return 'magic_the_gathering';
      if (c.startsWith('yugioh-')) return 'yugioh';
      return null;
    case 'sports': {
      const sport = SPORTS.find((s) => c.startsWith(s));
      switch (sport) {
        case 'basketball':
          return 'basketball_cards';
        case 'baseball':
          return 'baseball_cards';
        case 'football':
          return 'football_cards';
        case 'hockey':
          return 'hockey_cards';
        case 'soccer':
          return 'soccer_cards';
        case 'racing':
        case 'formula':
          return 'f1_cards';
        case 'non-sport':
          return 'non_sport_cards';
        default:
          return 'other_sports_cards';
      }
    }
    case 'video_games':
      if (c === 'pc-games') return 'pc_games';
      if (NINTENDO.test(c)) return 'nintendo_games';
      if (SEGA.test(c)) return 'sega_games';
      if (PLAYSTATION.test(c)) return 'playstation_games';
      if (XBOX.test(c)) return 'xbox_games';
      if (RETRO.test(c)) return 'atari_retro_games';
      return 'video_games';
  }
}

const YGO_CODE = /\b([A-Z0-9]{2,6}-[A-Z]{0,3}\d{2,4})\b/;
/** Tags that describe the card's status, not a distinct printing (kept in metadata, not in variant). */
const NON_VARIANT_TAGS = /^(rookie|rc|key issue|hall of fame|hof|error|misprint)$/i;

/** "Super Mario 64 [Player's Choice]" → { name, variant } ; "Cloud City #10123" → { name, number } ; "Blue-Eyes White Dragon [1st Edition] LOB-001" → number LOB-001 */
export function splitTitle(title: string, family?: Family | null): { name: string; variant: string | null; number: string | null; tags: string[] } {
  let t = title.trim();
  const variants: string[] = [];
  const tags: string[] = [];
  t = t.replace(/\[([^\]]+)\]/g, (_m, v: string) => {
    const tag = v.trim();
    if (NON_VARIANT_TAGS.test(tag)) tags.push(tag);
    else variants.push(tag);
    return ' ';
  });
  let number: string | null = null;
  const num = t.match(/#\s*([A-Za-z0-9.\-/]+)/);
  if (num) {
    number = num[1]!;
    t = t.replace(num[0], ' ');
  } else if (family === 'cards') {
    const code = t.match(YGO_CODE);
    if (code) {
      number = code[1]!;
      t = t.replace(code[0], ' ');
    }
  }
  return { name: t.replace(/\s+/g, ' ').trim(), variant: variants.length ? variants.join(' · ') : null, number, tags };
}

/** Pokémon/Magic set name from a console name: "Pokemon Base Set" → "Base Set"; "Magic Alpha" → "Alpha". */
export function setNameFromConsole(consoleName: string, family: Family): string {
  if (family === 'cards') return consoleName.replace(/^(Pokemon|Pokémon|Magic|YuGiOh|Yu-Gi-Oh!?)\s+/i, '').trim();
  return consoleName.trim();
}

export function parseProductPage(htmlText: string, url: string, site: ProductPayload['site'] = 'pricecharting'): ProductPayload {
  const $ = H.load(htmlText);
  const h1 = $('h1#product_name');
  const consoleName = H.text(h1.find('a')) ?? '';
  const consoleUri = (h1.find('a').attr('href') ?? '').replace(/^(https?:\/\/[^/]+)?\/console\//, '') || (url.match(/\/game\/([^/]+)\//)?.[1] ?? '');
  const title = h1
    .clone()
    .children('a')
    .remove()
    .end()
    .text()
    .replace(/\s+/g, ' ')
    .trim();
  const flagsBlock = htmlText.match(/VGPC\.product\s*=\s*\{([\s\S]*?)\}/)?.[1] ?? '';
  const flag = (k: string) => new RegExp(`${k}\\s*:\\s*true`).test(flagsBlock);
  let productId = flagsBlock.match(/id\s*:\s*(\d+)/)?.[1] ?? null;

  const prices: z.infer<typeof PriceCellSchema>[] = [];
  const seen = new Set<string>();
  for (const key of PRICE_KEYS) {
    if (seen.has(key)) continue;
    const cell = $(`td#${key}`).first();
    if (!cell.length) continue;
    seen.add(key);
    const raw = H.text(cell.find('.price').first()) ?? H.text(cell) ?? '';
    const parsed = parsePrice(raw, 'USD');
    prices.push({ key, value: parsed && parsed.amount > 0 ? parsed.amount : null, raw });
  }
  const columnLabels: string[] = [];
  $('#price_data thead th, table.price_data thead th').each((_, el) => {
    const t = H.text($(el));
    if (t) columnLabels.push(t);
  });
  const fullPrices: Array<{ label: string; value: number | null }> = [];
  $('#full-prices tr').each((_, tr) => {
    const tds = $(tr).find('td');
    if (tds.length < 2) return;
    const label = H.text(tds.eq(0)) ?? '';
    const v = parsePrice(H.text(tds.eq(1)) ?? '', 'USD');
    if (label) fullPrices.push({ label, value: v && v.amount > 0 ? v.amount : null });
  });
  const tabLabels: Record<string, string> = {};
  $('option[value^="completed-auctions-"]').each((_, el) => {
    const tab = ($(el).attr('value') ?? '').replace('completed-auctions-', '');
    const label = (H.text($(el)) ?? '').replace(/\s*\(\d+\)\s*$/, '').trim();
    if (tab && label) tabLabels[tab] = label;
  });

  const sales: z.infer<typeof SaleRowSchema>[] = [];
  $('div[class^="completed-auctions-"]').each((_, sec) => {
    const cls = ($(sec).attr('class') ?? '').split(/\s+/).find((c) => c.startsWith('completed-auctions-')) ?? '';
    const tab = cls.replace('completed-auctions-', '');
    if (!tab || tab === 'condition' || !$(sec).find('table').length) return;
    $(sec)
      .find('tbody tr')
      .each((__, tr) => {
        const $tr = $(tr);
        const date = H.text($tr.find('td.date')) ?? '';
        const titleEl = $tr.find('td.title a').first();
        const rowTitle = H.text(titleEl) ?? H.text($tr.find('td.title')) ?? '';
        const priceTxt = H.text($tr.find('td.numeric .js-price').first()) ?? H.text($tr.find('td.numeric').first()) ?? '';
        const price = parsePrice(priceTxt, 'USD');
        const listedTxt = H.text($tr.find('td.listed-price'));
        const listed = listedTxt ? parsePrice(listedTxt, 'USD') : null;
        const ebayId = ($tr.attr('id') ?? '').match(/ebay-(\d+)/)?.[1] ?? titleEl.attr('href')?.match(/\/itm\/(\d+)/)?.[1] ?? null;
        if (!date || !rowTitle || !price || price.amount <= 0) return;
        sales.push({ tab, date, title: rowTitle, price: price.amount, ebayId, listedPrice: listed && listed.amount > 0 ? listed.amount : null });
      });
  });

  const details: Record<string, string> = {};
  $('td.title').each((_, el) => {
    const k = (H.text($(el)) ?? '').replace(/:$/, '').trim();
    const v = H.text($(el).next('td.details'));
    if (k && v && v !== 'none' && v !== 'n/a') details[k] = v;
  });
  if (!productId && details['PriceCharting ID']) productId = details['PriceCharting ID'].trim();
  const images: string[] = [];
  $('img[src*="images.pricecharting.com"]').each((_, el) => {
    const src = $(el).attr('src');
    if (src && /\/(240|1600|400)\.jpg$/.test(src) && !images.includes(src)) images.push(src);
  });

  return {
    kind: 'product',
    url,
    productId,
    consoleUri,
    consoleName,
    title,
    flags: { isComic: flag('is_comic'), isLegoSet: flag('is_lego_set'), isFunkoPop: flag('is_funko_pop'), isCard: flag('is_card'), isCoin: flag('is_coin'), isSystem: flag('is_system') },
    columnLabels,
    prices,
    fullPrices,
    tabLabels,
    sales,
    details,
    images: images.slice(0, 4),
    setRef: null,
    cardRef: null,
    site,
  };
}

export interface ConsoleProduct {
  id: string;
  productUri: string;
  productName: string;
  consoleUri: string;
}

/** Parse a JSON body that may arrive wrapped in <html><body>…</body></html> by a rendering engine. */
export function parseJsonBody<T>(res: { json: unknown; html: string | null }): T | null {
  if (res.json && typeof res.json === 'object') return res.json as T;
  const text = res.html ?? '';
  if (!text) return null;
  const stripped = text.replace(/^[\s\S]*?<body[^>]*>/i, '').replace(/<\/body>[\s\S]*$/i, '').replace(/<[^>]+>/g, '').trim();
  const candidate = stripped.startsWith('{') || stripped.startsWith('[') ? stripped : text.trim();
  try {
    return JSON.parse(candidate) as T;
  } catch {
    return null;
  }
}

export interface SiteOptions {
  base: string;
  site: ProductPayload['site'];
  /** category slugs (site "/category/<slug>") whose console lists are discovered automatically */
  categoryPrefix: (categorySlug: string) => string;
}

/**
 * Base connector for PriceCharting-like sites. Subclasses provide the site and may enrich a
 * parsed product with reference-catalog lookups (setRef / cardRef) before it is yielded.
 */
export abstract class PriceChartingLikeConnector extends BaseConnector {
  readonly version = '2.0.0';
  readonly parserVersion = PARSER_VERSION;
  protected override minIntervalMs = 1500;
  protected abstract readonly siteOptions: SiteOptions;

  protected get seeds(): string[] {
    const s = this.meta.config.seeds;
    return Array.isArray(s) ? (s as string[]) : [];
  }
  protected get cardCategories(): string[] {
    const s = this.meta.config.cardCategories;
    return Array.isArray(s) ? (s as string[]) : [];
  }

  /** Hook: enrich a product payload with reference-catalog data (set code, card number, ids). */
  protected async enrich(_ctx: CrawlContext, payload: ProductPayload): Promise<ProductPayload> {
    return payload;
  }
  /** Hook: called once per crawl before iterating consoles (load reference maps). */
  protected async prepare(_ctx: CrawlContext): Promise<void> {}

  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
    const perConsole = Number(this.meta.config.productsPerConsole ?? 200);
    const sort = String(this.meta.config.sort ?? 'popularity');
    const maxConsoles = Number(this.meta.config.maxConsolesPerCategory ?? 40);
    const cursor = ctx.options.cursor ?? {};
    const doneConsoles = new Set<string>((cursor.doneConsoles as string[] | undefined) ?? []);
    let count = 0;

    await this.prepare(ctx);
    let seeds = ctx.options.seeds?.length ? [...ctx.options.seeds] : [...this.seeds];
    if (!ctx.options.seeds?.length) {
      for (const cat of this.cardCategories) {
        const discovered = await this.discoverConsoles(ctx, cat, maxConsoles);
        for (const c of discovered) if (!seeds.includes(c)) seeds.push(c);
      }
    }
    seeds = [...new Set(seeds)];

    for (const consoleUri of seeds) {
      if (doneConsoles.has(consoleUri) && ctx.options.mode !== 'backfill') continue;
      if (ctx.signal?.aborted) return;
      const products = await this.listConsole(ctx, consoleUri, perConsole, sort);
      if (products.length === 0) ctx.anomaly('empty_console', consoleUri);
      for (const p of products) {
        if (this.reached(ctx, count)) return;
        const url = `${this.siteOptions.base}/game/${p.consoleUri}/${p.productUri}`;
        if (!(await ctx.shouldFetch(url))) continue;
        await this.throttle();
        const rec = await this.fetchProduct(ctx, url);
        if (rec) {
          count++;
          yield rec;
        }
      }
      doneConsoles.add(consoleUri);
      await ctx.setCursor({ doneConsoles: [...doneConsoles], updatedAt: new Date().toISOString() });
    }
    // Full pass complete: reset so the next incremental run starts again.
    await ctx.setCursor({ doneConsoles: [], updatedAt: new Date().toISOString() });
  }

  /** Console (set) slugs listed on a category page, in page order (most popular first). */
  protected async discoverConsoles(ctx: CrawlContext, category: string, max: number): Promise<string[]> {
    await this.throttle();
    const url = `${this.siteOptions.base}/category/${category}`;
    const res = await ctx.fetch(url, { responseType: 'text', minQuality: 0 });
    if (!res.success || !res.html) {
      ctx.anomaly('category_discovery_failed', `${category}: ${res.error ?? res.httpStatus}`);
      return [];
    }
    const prefix = this.siteOptions.categoryPrefix(category);
    const out: string[] = [];
    for (const m of res.html.matchAll(/href="(?:https?:\/\/[^/"]+)?\/console\/([a-z0-9-]+)"/g)) {
      const slug = m[1]!;
      if (slug.startsWith(prefix) && !out.includes(slug)) out.push(slug);
      if (out.length >= max) break;
    }
    return out;
  }

  protected async listConsole(ctx: CrawlContext, consoleUri: string, max: number, sort: string): Promise<ConsoleProduct[]> {
    const out: ConsoleProduct[] = [];
    let cursorPos = 0;
    while (out.length < max) {
      await this.throttle();
      const url = `${this.siteOptions.base}/console/${consoleUri}?sort=${encodeURIComponent(sort)}&cursor=${cursorPos}&format=json`;
      const res = await ctx.fetch(url, { responseType: 'json', minQuality: 0 });
      const json = parseJsonBody<{ products?: Array<Record<string, unknown>>; cursor?: string }>(res);
      if (!res.success || !json?.products) {
        if (cursorPos === 0) ctx.anomaly('console_list_failed', `${consoleUri}: ${res.error ?? 'no products'}`);
        break;
      }
      for (const p of json.products) {
        if (typeof p.productUri === 'string' && typeof p.consoleUri === 'string') {
          out.push({ id: String(p.id ?? ''), productUri: p.productUri, productName: String(p.productName ?? ''), consoleUri: p.consoleUri });
        }
      }
      const next = Number(json.cursor);
      if (!Number.isFinite(next) || next <= cursorPos || json.products.length === 0) break;
      cursorPos = next;
    }
    return out.slice(0, max);
  }

  protected async fetchProduct(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {
    const site = this.siteOptions.site;
    const res = await ctx.fetch(url, {
      responseType: 'text',
      expect: ['title', 'price', 'identifiers', 'category'],
      parse: (r) => {
        if (!r.html) return null;
        const p = parseProductPage(r.html, url, site);
        return { title: p.title, price: p.prices.find((x) => x.value !== null)?.value ?? p.fullPrices.find((x) => x.value !== null)?.value ?? null, identifiers: p.productId ? { id: p.productId } : null, category: p.consoleUri };
      },
    });
    if (!res.success || !res.html) {
      ctx.anomaly('product_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
      return null;
    }
    let payload = parseProductPage(res.html, url, site);
    if (!payload.title) {
      ctx.anomaly('parse_failure_title', url);
      return null;
    }
    try {
      payload = await this.enrich(ctx, payload);
    } catch (err) {
      ctx.anomaly('enrich_failed', `${url}: ${err instanceof Error ? err.message : String(err)}`);
    }
    return { url, externalId: payload.productId, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
  }

  async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
    const clean = url.split('?')[0]!;
    await this.prepare(ctx);
    const rec = await this.fetchProduct(ctx, clean);
    return rec ? [rec] : [];
  }

  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
    const p = ProductPayloadSchema.parse(raw.payload);
    return normalizeProduct(p, { connectorId: this.meta.id, sourceId: this.meta.sourceId, fetchedAt: raw.fetchedAt });
  }
}

// ---------------------------------------------------------------------------------------------
// Normalisation
// ---------------------------------------------------------------------------------------------

export interface NormalizeCtx {
  connectorId: string;
  sourceId: string;
  fetchedAt: Date;
}

const RAW_GRADE: Grade = { grader: null, grade: null, qualifier: null, certificationNumber: null };

export function normalizeProduct(p: ProductPayload, nctx: NormalizeCtx): NormalizedRecord[] {
  const family = familyOf(p);
  if (!family) return [];
  const slug = categorySlug(p, family);
  if (!slug) return [];
  const { name, variant, number, tags } = splitTitle(p.title, family);
  const isCardFamily = family === 'cards' || family === 'sports';
  const setName = isCardFamily ? (p.setRef?.name ?? setNameFromConsole(p.consoleName, family)) : null;
  const year = isCardFamily ? (extractYear(p.consoleName) ?? extractYear(p.details['Release Date'] ?? '') ?? null) : (extractYear(p.details['Release Date'] ?? '') ?? extractYear(p.title) ?? null);

  const identifiers: Record<string, string> = {};
  if (p.productId) identifiers.pricecharting_id = p.site === 'sportscardspro' ? `scp:${p.productId}` : p.productId;
  if (p.details['UPC']) identifiers.upc = p.details['UPC'].replace(/\s+/g, '');
  if (p.details['ASIN (Amazon)']) identifiers.asin = p.details['ASIN (Amazon)'];
  if (p.details['ePID (eBay)']) identifiers.ebay_epid = p.details['ePID (eBay)'];
  if (p.details['Comic.org ID']) identifiers.comics_org_id = p.details['Comic.org ID'];
  if (p.details['TCGPlayer ID']) identifiers.tcgplayer_id = p.details['TCGPlayer ID'];
  if (family === 'lego' && number) identifiers.lego_set_number = number;
  if (p.details['Model Number']) identifiers.model_number = p.details['Model Number'];
  if (family === 'funko' && p.details['Box Number']) identifiers.funko_box_number = p.details['Box Number'];
  if (p.cardRef?.scryfall_id) identifiers.scryfall_id = p.cardRef.scryfall_id;
  if (p.cardRef?.pokemontcg_id) identifiers.pokemontcg_id = p.cardRef.pokemontcg_id;
  if (p.cardRef?.tcgplayer_id) identifiers.tcgplayer_id = p.cardRef.tcgplayer_id;

  let cardNumber = number;
  if (isCardFamily && !cardNumber && p.cardRef?.number) cardNumber = p.cardRef.number;
  const language = /japanese/i.test(p.consoleName) ? 'Japanese' : /korean/i.test(p.consoleName) ? 'Korean' : /chinese/i.test(p.consoleName) ? 'Chinese' : 'English';
  const cardVariant = isCardFamily ? normalizeCardVariant(variant, slug) : variant;
  const setCode = isCardFamily ? (p.setRef?.code ?? (slug === 'yugioh' && cardNumber ? cardNumber.split('-')[0]! : null)) : null;

  const attributes: AssetAttributes = {
    categorySlug: slug,
    subcategorySlug: null,
    franchise: family === 'funko' || family === 'lego' ? p.consoleName.replace(/^(Funko POP|LEGO)\s*/i, '') || null : slug === 'pokemon' ? 'Pokémon' : slug === 'magic_the_gathering' ? 'Magic: The Gathering' : slug === 'yugioh' ? 'Yu-Gi-Oh!' : null,
    brand: family === 'lego' ? 'LEGO' : family === 'funko' ? 'Funko' : family === 'sports' ? brandFromSet(setName ?? '') : slug === 'pokemon' ? 'The Pokémon Company' : slug === 'magic_the_gathering' ? 'Wizards of the Coast' : slug === 'yugioh' ? 'Konami' : (p.details['Publisher'] ?? null),
    series: family === 'funko' ? (p.details['Series'] ?? p.consoleName) : null,
    set: isCardFamily ? setName : family === 'comics' ? p.consoleName : family === 'lego' ? p.consoleName.replace(/^LEGO\s*/i, '') : p.consoleName,
    setCode,
    name: family === 'comics' ? p.consoleName : name,
    model: null,
    reference: null,
    number: cardNumber ?? (family === 'funko' ? (p.details['Box Number'] ?? null) : null),
    year,
    edition: null,
    variant: cardVariant,
    language: isCardFamily ? language : 'English',
    region: isCardFamily ? null : /^(jp|pal)-/.test(p.consoleUri) ? p.consoleUri.split('-')[0]!.toUpperCase() : 'NTSC-U',
    country: null,
    material: null,
    size: null,
    color: null,
    rarity: null,
    productionQuantity: null,
    originalMsrp: null,
    originalMsrpCurrency: null,
    identifiers,
    metadata: {
      pricecharting_console: p.consoleUri,
      pricecharting_site: p.site,
      key_issue: p.details['Is Key Issue'] === 'Yes' ? true : undefined,
      rookie: family === 'sports' && (p.details['Is Rookie Card'] === 'Yes' || tags.some((t) => /rookie|^rc$/i.test(t))) ? true : undefined,
      tags: tags.length ? tags : undefined,
      genre: p.details['Genre'],
      notes: p.details['Notes'],
      set_ref_source: p.setRef?.source,
    },
  };
  if (family === 'comics') {
    attributes.name = p.consoleName;
    attributes.year = extractYear(p.title) ?? year;
  }

  const observedAt = nctx.fetchedAt;
  const base = {
    connectorId: nctx.connectorId,
    sourceId: nctx.sourceId,
    sourceUrl: p.url,
    externalId: p.productId ? (p.site === 'sportscardspro' ? `scp:${p.productId}` : p.productId) : null,
    rawTitle: family === 'comics' ? p.title : `${p.title} (${p.consoleName})`,
    description: p.details['Description'] ?? null,
    imageUrls: p.images,
    attributes,
    observedAt,
    confidence: 0.9,
    parserVersion: PARSER_VERSION,
  };
  const out: NormalizedRecord[] = [];
  const catalog: NormalizedCatalogItem = { kind: 'catalog_item', ...base, grade: RAW_GRADE, condition: { condition: null, conditionRaw: null, completeness: null }, releaseDate: parseSourceDate(p.details['Release Date']) ?? null };
  out.push(catalog);

  const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));

  if (isCardFamily) {
    // Guide values: prefer the full price table (all grades); fall back to the six main cells.
    const cells: Array<{ label: string; value: number | null }> = p.fullPrices.length ? p.fullPrices : p.prices.map((c) => ({ label: CARD_CELL_LABEL[c.key as (typeof PRICE_KEYS)[number]] ?? c.key, value: c.value }));
    const seenLabel = new Set<string>();
    for (const cell of cells) {
      const g = gradeFromLabel(cell.label);
      if (!g || cell.value === null || seenLabel.has(cell.label)) continue;
      seenLabel.add(cell.label);
      out.push({
        kind: 'price_observation',
        ...base,
        externalId: `${base.externalId ?? p.url}:guide:${cell.label.toLowerCase().replace(/[^a-z0-9.]+/g, '-')}`,
        grade: { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: null },
        condition: { condition: null, conditionRaw: g.conditionRaw, completeness: null },
        priceKind: 'guide_value',
        price: cell.value,
        currency: 'USD',
        observationDate: obsDate,
        sampleSize: null,
        confidence: 0.85,
      } satisfies NormalizedPriceObservation);
    }
    const seenSale = new Set<string>();
    for (const row of p.sales) {
      const label = p.tabLabels[row.tab] ?? CARD_TAB_LABEL[row.tab];
      if (!label) continue;
      const g0 = gradeFromLabel(label);
      if (!g0) continue;
      const g = refineGradeFromTitle(g0, row.title);
      const saleDate = parseSourceDate(row.date);
      if (!saleDate) continue;
      const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;
      if (seenSale.has(dedupe)) continue;
      seenSale.add(dedupe);
      out.push(makeSale(base, p, row, saleDate, { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, { condition: null, conditionRaw: g.conditionRaw, completeness: null }));
    }
    return out;
  }

  const meanings = COLUMN_MEANING[family];
  for (const cell of p.prices) {
    const m = meanings[cell.key as (typeof PRICE_KEYS)[number]];
    if (!m || cell.value === null) continue;
    out.push({
      kind: 'price_observation',
      ...base,
      externalId: `${base.externalId ?? p.url}:guide:${cell.key}`,
      grade: { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null },
      condition: { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness },
      priceKind: 'guide_value',
      price: cell.value,
      currency: 'USD',
      observationDate: obsDate,
      sampleSize: null,
      confidence: 0.85,
    } satisfies NormalizedPriceObservation);
  }
  const seenSale = new Set<string>();
  for (const row of p.sales) {
    const key = TAB_TO_KEY[row.tab];
    const m = key ? meanings[key] : undefined;
    if (!m) continue;
    const saleDate = parseSourceDate(row.date);
    if (!saleDate) continue;
    const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;
    if (seenSale.has(dedupe)) continue;
    seenSale.add(dedupe);
    out.push(makeSale(base, p, row, saleDate, { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null }, { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness }));
  }
  return out;
}

function makeSale(base: Omit<NormalizedCatalogItem, 'kind' | 'grade' | 'condition' | 'releaseDate'>, p: ProductPayload, row: z.infer<typeof SaleRowSchema>, saleDate: Date, grade: Grade, condition: NormalizedSale['condition']): NormalizedSale {
  return {
    kind: 'sale',
    ...base,
    externalId: row.ebayId ? `ebay:${row.ebayId}` : `${p.productId ?? p.url}:${row.date}:${row.price}`,
    rawTitle: row.title,
    description: null,
    grade,
    condition,
    saleType: 'unknown',
    saleDate,
    price: row.price,
    currency: 'USD',
    buyerPremiumIncluded: false,
    quantity: 1,
    isBundle: /\b(lot|bundle)\b/i.test(row.title),
    location: null,
    auctionHouse: null,
    lotNumber: null,
    confidence: 0.75,
  };
}

/** Align bracket tags with the API catalogs' variant vocabulary (pokemontcg: 'Holo', 'Reverse Holo', '1st Edition', '1st Edition Holo'; scryfall: 'Foil'). */
export function normalizeCardVariant(variant: string | null, slug: string): string | null {
  if (!variant) return null;
  const parts = variant.split(' · ').map((v) => v.trim()).filter(Boolean);
  const mapped = parts.map((v) => {
    const l = v.toLowerCase();
    if (slug === 'pokemon') {
      if (l === 'reverse holo' || l === 'reverse holofoil') return 'Reverse Holo';
      if (l === 'holo' || l === 'holofoil') return 'Holo';
      if (l === '1st edition') return '1st Edition';
      if (l === 'shadowless') return 'Shadowless';
    }
    if (slug === 'magic_the_gathering') {
      if (l === 'foil') return 'Foil';
      if (l === 'etched foil' || l === 'foil etched') return 'Etched Foil';
    }
    return v;
  });
  // Pokémon: "1st Edition · Holo" → "1st Edition Holo" (pokemontcg vocabulary)
  if (slug === 'pokemon' && mapped.includes('1st Edition') && mapped.includes('Holo')) {
    return ['1st Edition Holo', ...mapped.filter((v) => v !== '1st Edition' && v !== 'Holo')].join(' · ');
  }
  return mapped.join(' · ');
}

const CARD_BRANDS = ['Topps', 'Panini', 'Upper Deck', 'Bowman', 'Fleer', 'Donruss', 'Score', 'Leaf', 'Futera', 'O-Pee-Chee', 'Skybox', 'Hoops', 'Pinnacle', 'Pro Set', 'Goudey', 'Playoff', 'Pacific', 'Select', 'Prizm', 'Mosaic', 'Optic'];
export function brandFromSet(setName: string): string | null {
  for (const b of CARD_BRANDS) if (new RegExp(`\\b${b.replace('-', '\\-')}\\b`, 'i').test(setName)) return b === 'Prizm' || b === 'Select' || b === 'Mosaic' || b === 'Optic' ? 'Panini' : b;
  return null;
}