import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { type NormalizedRecord } from '@rareindex/shared'; import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js'; import { UA_HEADERS, dayOf } from '../_lib/tcg-shared.js'; /** SWU-DB — Star Wars: Unlimited catalog with relayed TCGplayer market prices. */ const API = 'https://api.swu-db.com'; const PARSER_VERSION = '1.0.0'; const CardSchema = z .object({ Set: z.string(), Number: z.string(), Name: z.string(), Subtitle: z.string().nullable().optional(), Type: z.string().nullable().optional(), Rarity: z.string().nullable().optional(), VariantType: z.string().nullable().optional(), Unique: z.boolean().optional(), Artist: z.string().nullable().optional(), cid: z.string().nullable().optional(), tcgplayerId: z.string().nullable().optional(), MarketPrice: z.string().nullable().optional(), LowPrice: z.string().nullable().optional(), FoilPrice: z.string().nullable().optional(), LowFoilPrice: z.string().nullable().optional(), FrontArt: z.string().nullable().optional(), Aspects: z.array(z.union([z.string(), z.record(z.string(), z.string())])).optional(), }) .loose(); export type SwuCard = z.infer; export function trimCard(raw: Record): SwuCard { const keep = ['Set', 'Number', 'Name', 'Subtitle', 'Type', 'Rarity', 'VariantType', 'Unique', 'Artist', 'cid', 'tcgplayerId', 'MarketPrice', 'LowPrice', 'FoilPrice', 'LowFoilPrice', 'FrontArt', 'Aspects']; const out: Record = {}; for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k]; return CardSchema.parse(out); } const RawPayloadSchema = z.object({ card: CardSchema, setName: z.string().nullable() }); export class SwuDbConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 500; override readonly urlPatterns = [/swu-db\.com\/cards?\/[a-z]{3}\/\d+/i]; private sets(): Record { return (this.meta.config.sets ?? {}) as Record; } async *crawl(ctx: CrawlContext): AsyncIterable { const sets = Object.entries(this.sets()).filter(([code]) => !ctx.options.seeds?.length || ctx.options.seeds.map((s) => s.toUpperCase()).includes(code)); let setIdx = Number(ctx.options.cursor?.setIdx ?? 0); let count = 0; for (; setIdx < sets.length; setIdx++) { const [code, setName] = sets[setIdx]!; await this.throttle(); const res = await withRetries(() => ctx.fetch(`${API}/cards/${code.toLowerCase()}`, { engines: ['api'], headers: UA_HEADERS, timeoutMs: 60_000 }), (r) => r.success && r.json !== null, 3, 2000); const data = (res.json as { data?: unknown[] } | null)?.data; if (!res.success || !Array.isArray(data)) { ctx.anomaly('page_fetch_failed', `${code}: ${res.error ?? res.httpStatus}`); continue; } for (const raw of data) { let card: SwuCard; try { card = trimCard(raw as Record); } catch (err) { ctx.anomaly('parse_failure_card', `${code}: ${err instanceof Error ? err.message : String(err)}`); continue; } if (this.reached(ctx, count)) return; count++; yield { url: `https://www.swu-db.com/card/${card.Set}/${card.Number}`, externalId: `${card.Set}-${card.Number}`, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, setName }, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ setIdx: setIdx + 1, updatedAt: new Date().toISOString() }); } await ctx.setCursor({ setIdx: 0, updatedAt: new Date().toISOString() }); } async lookup(url: string, ctx: CrawlContext): Promise { const m = url.match(/swu-db\.com\/cards?\/([a-z]{3})\/(\d+)/i); if (!m) return []; const res = await ctx.fetch(`${API}/cards/${m[1]!.toLowerCase()}/${m[2]}`, { engines: ['api'], headers: UA_HEADERS }); if (!res.success || !res.json) return []; const card = trimCard(res.json as Record); return [{ url, externalId: `${card.Set}-${card.Number}`, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, setName: this.sets()[card.Set] ?? null }, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const { card, setName } = RawPayloadSchema.parse(raw.payload); const identifiers: Record = { swudb_id: `${card.Set}-${card.Number}` }; if (card.cid) identifiers.swudb_cid = card.cid; if (card.tcgplayerId) identifiers.tcgplayer_id = card.tcgplayerId; const name = card.Subtitle ? `${card.Name} - ${card.Subtitle}` : card.Name; const foilEntry = /F$/i.test(card.Number); const number = foilEntry ? card.Number.replace(/F$/i, '') : card.Number; const typeVariant = card.VariantType && card.VariantType !== 'Normal' && card.VariantType !== 'Foil' ? card.VariantType : null; const baseVariant = foilEntry || card.VariantType === 'Foil' ? (typeVariant ? `${typeVariant} Foil` : 'Foil') : typeVariant; const images = card.FrontArt ? [card.FrontArt] : []; const observedAt = raw.fetchedAt; const obsDate = dayOf(observedAt); const build = (variant: string | null) => attrs({ categorySlug: 'star_wars_tcg', franchise: 'Star Wars: Unlimited', brand: 'Fantasy Flight Games', set: setName ?? card.Set, setCode: card.Set, name, number, year: null, variant, language: 'English', rarity: card.Rarity ?? null, identifiers, metadata: { type: card.Type, unique: card.Unique ?? null, artist: card.Artist, variant_type: card.VariantType ?? 'Normal', foil_price_hint: num(card.FoilPrice) }, }); const out: NormalizedRecord[] = []; const emit = (variant: string | null, market: number | null, low: number | null) => { const a = build(variant); const rawTitle = makeTitle({ name, set: setName ?? card.Set, number, variant }); out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.Set}-${card.Number}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null })); for (const [kind, price] of [['market', market], ['low', low]] as const) { if (!price) continue; // Prices are relayed from TCGplayer without a timestamp → fetch-day observation, moderate confidence. out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.Set}-${card.Number}:${variant ?? 'normal'}:${kind}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: kind, price, currency: 'USD', observationDate: obsDate, sampleSize: null })); } }; // Foil printings are separate API entries (Number "059F"); FoilPrice on the base entry is informational only. emit(baseVariant, num(card.MarketPrice), num(card.LowPrice)); return out; } } export default (meta: ConnectorMeta) => new SwuDbConnector(meta);