spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';5import { UA_HEADERS, dayOf } from '../_lib/tcg-shared.js';67/** SWU-DB — Star Wars: Unlimited catalog with relayed TCGplayer market prices. */8const API = 'https://api.swu-db.com';9const PARSER_VERSION = '1.0.0';1011const CardSchema = z12  .object({13    Set: z.string(),14    Number: z.string(),15    Name: z.string(),16    Subtitle: z.string().nullable().optional(),17    Type: z.string().nullable().optional(),18    Rarity: z.string().nullable().optional(),19    VariantType: z.string().nullable().optional(),20    Unique: z.boolean().optional(),21    Artist: z.string().nullable().optional(),22    cid: z.string().nullable().optional(),23    tcgplayerId: z.string().nullable().optional(),24    MarketPrice: z.string().nullable().optional(),25    LowPrice: z.string().nullable().optional(),26    FoilPrice: z.string().nullable().optional(),27    LowFoilPrice: z.string().nullable().optional(),28    FrontArt: z.string().nullable().optional(),29    Aspects: z.array(z.union([z.string(), z.record(z.string(), z.string())])).optional(),30  })31  .loose();32export type SwuCard = z.infer<typeof CardSchema>;3334export function trimCard(raw: Record<string, unknown>): SwuCard {35  const keep = ['Set', 'Number', 'Name', 'Subtitle', 'Type', 'Rarity', 'VariantType', 'Unique', 'Artist', 'cid', 'tcgplayerId', 'MarketPrice', 'LowPrice', 'FoilPrice', 'LowFoilPrice', 'FrontArt', 'Aspects'];36  const out: Record<string, unknown> = {};37  for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];38  return CardSchema.parse(out);39}4041const RawPayloadSchema = z.object({ card: CardSchema, setName: z.string().nullable() });4243export class SwuDbConnector extends BaseConnector {44  readonly version = '1.0.0';45  readonly parserVersion = PARSER_VERSION;46  protected override minIntervalMs = 500;47  override readonly urlPatterns = [/swu-db\.com\/cards?\/[a-z]{3}\/\d+/i];4849  private sets(): Record<string, string> {50    return (this.meta.config.sets ?? {}) as Record<string, string>;51  }5253  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {54    const sets = Object.entries(this.sets()).filter(([code]) => !ctx.options.seeds?.length || ctx.options.seeds.map((s) => s.toUpperCase()).includes(code));55    let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);56    let count = 0;57    for (; setIdx < sets.length; setIdx++) {58      const [code, setName] = sets[setIdx]!;59      await this.throttle();60      const res = await withRetries(() => ctx.fetch(`${API}/cards/${code.toLowerCase()}`, { engines: ['api'], headers: UA_HEADERS, timeoutMs: 60_000 }), (r) => r.success && r.json !== null, 3, 2000);61      const data = (res.json as { data?: unknown[] } | null)?.data;62      if (!res.success || !Array.isArray(data)) {63        ctx.anomaly('page_fetch_failed', `${code}: ${res.error ?? res.httpStatus}`);64        continue;65      }66      for (const raw of data) {67        let card: SwuCard;68        try {69          card = trimCard(raw as Record<string, unknown>);70        } catch (err) {71          ctx.anomaly('parse_failure_card', `${code}: ${err instanceof Error ? err.message : String(err)}`);72          continue;73        }74        if (this.reached(ctx, count)) return;75        count++;76        yield { url: `https://www.swu-db.com/card/${card.Set}/${card.Number}`, externalId: `${card.Set}-${card.Number}`, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, setName }, fetchedAt: res.fetchedAt };77      }78      await ctx.setCursor({ setIdx: setIdx + 1, updatedAt: new Date().toISOString() });79    }80    await ctx.setCursor({ setIdx: 0, updatedAt: new Date().toISOString() });81  }8283  async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {84    const m = url.match(/swu-db\.com\/cards?\/([a-z]{3})\/(\d+)/i);85    if (!m) return [];86    const res = await ctx.fetch(`${API}/cards/${m[1]!.toLowerCase()}/${m[2]}`, { engines: ['api'], headers: UA_HEADERS });87    if (!res.success || !res.json) return [];88    const card = trimCard(res.json as Record<string, unknown>);89    return [{ url, externalId: `${card.Set}-${card.Number}`, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, setName: this.sets()[card.Set] ?? null }, fetchedAt: res.fetchedAt }];90  }9192  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {93    const { card, setName } = RawPayloadSchema.parse(raw.payload);94    const identifiers: Record<string, string> = { swudb_id: `${card.Set}-${card.Number}` };95    if (card.cid) identifiers.swudb_cid = card.cid;96    if (card.tcgplayerId) identifiers.tcgplayer_id = card.tcgplayerId;97    const name = card.Subtitle ? `${card.Name} - ${card.Subtitle}` : card.Name;98    const foilEntry = /F$/i.test(card.Number);99    const number = foilEntry ? card.Number.replace(/F$/i, '') : card.Number;100    const typeVariant = card.VariantType && card.VariantType !== 'Normal' && card.VariantType !== 'Foil' ? card.VariantType : null;101    const baseVariant = foilEntry || card.VariantType === 'Foil' ? (typeVariant ? `${typeVariant} Foil` : 'Foil') : typeVariant;102    const images = card.FrontArt ? [card.FrontArt] : [];103    const observedAt = raw.fetchedAt;104    const obsDate = dayOf(observedAt);105    const build = (variant: string | null) =>106      attrs({107        categorySlug: 'star_wars_tcg',108        franchise: 'Star Wars: Unlimited',109        brand: 'Fantasy Flight Games',110        set: setName ?? card.Set,111        setCode: card.Set,112        name,113        number,114        year: null,115        variant,116        language: 'English',117        rarity: card.Rarity ?? null,118        identifiers,119        metadata: { type: card.Type, unique: card.Unique ?? null, artist: card.Artist, variant_type: card.VariantType ?? 'Normal', foil_price_hint: num(card.FoilPrice) },120      });121    const out: NormalizedRecord[] = [];122    const emit = (variant: string | null, market: number | null, low: number | null) => {123      const a = build(variant);124      const rawTitle = makeTitle({ name, set: setName ?? card.Set, number, variant });125      out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.Set}-${card.Number}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null }));126      for (const [kind, price] of [['market', market], ['low', low]] as const) {127        if (!price) continue;128        // Prices are relayed from TCGplayer without a timestamp → fetch-day observation, moderate confidence.129        out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.Set}-${card.Number}:${variant ?? 'normal'}:${kind}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: kind, price, currency: 'USD', observationDate: obsDate, sampleSize: null }));130      }131    };132    // Foil printings are separate API entries (Number "059F"); FoilPrice on the base entry is informational only.133    emit(baseVariant, num(card.MarketPrice), num(card.LowPrice));134    return out;135  }136}137138export default (meta: ConnectorMeta) => new SwuDbConnector(meta);139