/**
 * Shared base for grading-company certificate-verification connectors (SPEC §3 "Grading/certification
 * data", §22 cert tracking; group g2-grading).
 *
 * A cert connector verifies ONE public certification page per cert number: it never enumerates the
 * grader's cert space. Certs come from `ctx.options.seeds` (bare numbers or public verify URLs) or from
 * `meta.config.certs`; `lookup(url)` resolves a single verify URL (the worker calls
 * `lookup(certUrl(cert))` for cert numbers seen on marketplaces). Each found cert yields one raw record
 * whose payload is a trimmed snapshot of the public page; `normalize` turns it into a `catalog_item`
 * (the graded object + grade + cert identifiers) and, when the page shows it, a `population_report`.
 */
import { z } from 'zod';
import { BaseConnector, type ConnectorMeta, type CrawlContext, type FetchOptions, type QualityField, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPopulationReportSchema, type ExtractionResult, type NormalizedRecord } from '@rareindex/shared';
import { dayOf } from '../../api/_lib/wave4.js';

export const CertPayloadSchema = z.object({
  grader: z.string(),
  cert: z.string(),
  /** the public verify URL that was fetched */
  url: z.string(),
  format: z.enum(['html', 'markdown']),
  status: z.enum(['found', 'not_found']),
  /** trimmed page fragment (HTML or markdown) — everything `normalize` needs */
  snapshot: z.string(),
});
export type CertPayload = z.infer<typeof CertPayloadSchema>;

export type AttrInput = z.input<typeof AssetAttributesSchema>;

/** What a connector's page parser returns for a found certificate. */
export interface ParsedCert {
  /** the grader's headline description, kept verbatim as rawTitle */
  title: string;
  attributes: AttrInput;
  grade: string | null;
  qualifier: string | null;
  /** grader's printed grade label (e.g. "GEM MT 10", "MS 61", "67 EPQ") */
  gradeLabel: string | null;
  description?: string | null;
  images?: string[];
  /**
   * Population shown on the page for this exact grade, plus either the count in higher grades
   * (PSA/CGC/PCGS/NGC/PMG) or the total graded across all grades (TAG). Partial by nature: the
   * report keeps `population_scope` in metadata so consumers never mistake it for a full distribution.
   */
  population?: { gradeKey: string; atGrade: number | null; higher: number | null; total?: number | null; url: string | null; asOf?: Date | null } | null;
  confidence: number;
}

export type PageStatus = 'found' | 'not_found' | 'unknown';

export const CertConfigSchema = z.object({
  /** taxonomy grader slug this connector verifies (coordinator convention) */
  grader: z.string(),
  /** bounded list of cert numbers (or verify URLs) verified on every scheduled run */
  certs: z.array(z.string()).default([]),
  /** hard cap of pages per run */
  maxPerRun: z.number().int().positive().default(50),
});

export abstract class CertLookupConnector extends BaseConnector {
  /** taxonomy grader slug (data/taxonomy/graders.json) */
  abstract readonly grader: string;
  /** identifiers key carrying the cert number (e.g. psa_cert) */
  abstract readonly idKey: string;
  /** which engine output the parser consumes */
  abstract readonly format: 'html' | 'markdown';
  /** Firecrawl wait for client-rendered pages */
  protected waitForMs = 3000;
  protected override minIntervalMs = 4000;
  /** URL patterns understood by `lookup` (subclasses assign their own list). */
  abstract override readonly urlPatterns: RegExp[];

  /** Canonical public verify URL for a cert number (the worker builds these). */
  abstract certUrl(cert: string): string;
  /** Extract the cert number from any public URL variant; null when the URL is not a cert page. */
  abstract certFromUrl(url: string): string | null;
  /** found = the page shows a certificate; not_found = the grader says the cert does not exist; unknown = unexpected page. */
  abstract classify(doc: string): PageStatus;
  /** Trim the full document to the fragment `parse` needs (fixtures store this). */
  abstract trim(doc: string): string;
  /** Pure parser over the trimmed snapshot. Null when the fragment carries no certificate. */
  abstract parse(snapshot: string, info: { cert: string; url: string }): ParsedCert | null;

  protected get config() {
    return CertConfigSchema.parse({ grader: this.grader, ...this.meta.config });
  }

  /** The value stored in identifiers[idKey] / grade.certificationNumber (subclasses strip URL-only parts such as NGC's grade segment). */
  certIdentifier(cert: string): string {
    return cert;
  }

  /** Normalise a seed (bare cert number or URL) to a cert number. */
  protected seedToCert(seed: string): string | null {
    const s = seed.trim();
    if (!s) return null;
    if (/^https?:\/\//i.test(s)) return this.certFromUrl(s);
    return s;
  }

  protected docOf(res: ExtractionResult): string | null {
    return this.format === 'markdown' ? (res.markdown ?? null) : (res.html ?? res.markdown ?? null);
  }

  protected fetchOptions(): Partial<FetchOptions> {
    return {};
  }

  /**
   * Fetch and classify one certificate page. Returns a raw record for a found cert; for a not-found
   * cert returns null (or a `status: not_found` record when `includeNotFound`, used to build fixtures).
   */
  async fetchCert(ctx: CrawlContext, cert: string, opts: { includeNotFound?: boolean } = {}): Promise<RawRecordInput | null> {
    const url = this.certUrl(cert);
    await this.throttle(url);
    const expect: QualityField[] = ['title', 'identifiers', 'status'];
    const res = await ctx.fetch(url, {
      engines: this.meta.enginePriority,
      waitForMs: this.waitForMs,
      timeoutMs: 90_000,
      minQuality: 0.3,
      expect,
      parse: (r) => {
        const doc = this.docOf(r);
        if (!doc) return null;
        const status = this.classify(doc);
        if (status === 'not_found') return { title: 'not found', status: 'not_found' };
        if (status === 'unknown') return null;
        const parsed = this.parse(this.trim(doc), { cert, url });
        return parsed ? { title: parsed.title, identifiers: cert, status: 'found' } : null;
      },
      ...this.fetchOptions(),
    });
    const gone = res.httpStatus === 404 || res.httpStatus === 410;
    if (!res.success && !gone) {
      ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
      return null;
    }
    const doc = this.docOf(res);
    // A 404/410 from the grader is a definitive answer (PSA answers unknown certs with 404), never a host failure.
    const status: PageStatus = gone ? 'not_found' : doc ? this.classify(doc) : 'unknown';
    if (status === 'unknown') {
      ctx.anomaly('selector_missing', `${url}: page is neither a certificate nor a not-found response`);
      return null;
    }
    const snapshot = doc && !gone ? this.trim(doc) : doc ? `<div class="g2-not-found" data-http-status="${res.httpStatus}">${this.trim(doc).slice(0, 4000)}</div>` : '';
    if (status === 'not_found') {
      ctx.log.info({ cert, url }, 'cert not found at grader');
      if (!opts.includeNotFound) return null;
    }
    const payload: CertPayload = { grader: this.grader, cert, url, format: this.format, status, snapshot };
    return { url, externalId: `${this.grader}:${cert}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
  }

  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
    const cfg = this.config;
    const seeds = ctx.options.seeds?.length ? ctx.options.seeds : cfg.certs;
    const certs = [...new Set(seeds.map((s) => this.seedToCert(s)).filter((c): c is string => Boolean(c)))];
    if (!certs.length) {
      ctx.log.info({ connector: this.meta.id }, 'no certs to verify (pass seeds or config.certs)');
      return;
    }
    let idx = Number(ctx.options.cursor?.idx ?? 0);
    if (!Number.isFinite(idx) || idx < 0 || idx >= certs.length) idx = 0;
    let count = 0;
    let pages = 0;
    for (; idx < certs.length && pages < cfg.maxPerRun; idx++) {
      if (ctx.signal?.aborted) return;
      if (this.reached(ctx, count)) break;
      const cert = certs[idx]!;
      pages++;
      const raw = await this.fetchCert(ctx, cert);
      if (raw) {
        count++;
        yield raw;
      }
      await ctx.setCursor({ idx: idx + 1, total: certs.length });
      if (ctx.options.mode === 'backfill') await ctx.progress({ page: idx + 1, totalPages: certs.length, itemsProcessed: count });
    }
    if (idx >= certs.length) await ctx.setCursor({ idx: 0, total: certs.length, completedAt: new Date().toISOString(), ...(ctx.options.mode === 'backfill' ? { done: true } : {}) });
  }

  async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
    const cert = this.certFromUrl(url);
    if (!cert) return [];
    const raw = await this.fetchCert(ctx, cert);
    return raw ? [raw] : [];
  }

  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
    const p = CertPayloadSchema.parse(raw.payload);
    if (p.status === 'not_found') return [];
    const parsed = this.parse(p.snapshot, { cert: p.cert, url: p.url });
    if (!parsed) return [];
    const certId = this.certIdentifier(p.cert);
    const identifiers = { ...(parsed.attributes.identifiers ?? {}), [this.idKey]: certId };
    const attributes = AssetAttributesSchema.parse({ ...parsed.attributes, identifiers });
    const out: NormalizedRecord[] = [];
    out.push(
      NormalizedCatalogItemSchema.parse({
        kind: 'catalog_item',
        connectorId: this.meta.id,
        sourceId: this.meta.sourceId,
        sourceUrl: p.url,
        externalId: `${this.grader}:${certId}`,
        rawTitle: parsed.title,
        description: parsed.description ?? null,
        imageUrls: parsed.images ?? [],
        attributes,
        grade: { grader: this.grader, grade: parsed.grade, qualifier: parsed.qualifier, certificationNumber: certId },
        condition: {},
        observedAt: raw.fetchedAt,
        confidence: parsed.confidence,
        parserVersion: this.parserVersion,
        releaseDate: null,
      }),
    );
    const pop = parsed.population;
    if (pop && pop.atGrade !== null) {
      const byGrade: Record<string, number> = { [pop.gradeKey]: pop.atGrade };
      if (pop.higher !== null) byGrade.higher = pop.higher;
      const hasTotal = pop.total !== undefined && pop.total !== null && pop.total >= pop.atGrade;
      const total = hasTotal ? (pop.total as number) : pop.atGrade + (pop.higher ?? 0);
      if (hasTotal && total > pop.atGrade) byGrade.other = total - pop.atGrade;
      out.push(
        NormalizedPopulationReportSchema.parse({
          kind: 'population_report',
          connectorId: this.meta.id,
          sourceId: this.meta.sourceId,
          sourceUrl: pop.url ?? p.url,
          grader: this.grader,
          attributes: AssetAttributesSchema.parse({ ...attributes, metadata: { ...attributes.metadata, population_scope: hasTotal ? 'grade_and_total' : 'grade_and_higher', grade_label: parsed.gradeLabel } }),
          reportDate: pop.asOf ?? dayOf(raw.fetchedAt),
          total,
          byGrade,
          parserVersion: this.parserVersion,
          confidence: Math.min(parsed.confidence, 0.9),
        }),
      );
    }
    return out;
  }
}

/** Shared meta factory signature used by every cert connector module. */
export type CertConnectorFactory = (meta: ConnectorMeta) => CertLookupConnector;