SPB Git
spboucher.ai

spb/rareindex

Public

Connectors: TCG data wave (tcgcsv, Digimon, SWU, Grand Archive, Sorcery, MTGGoldfish, pokemonprice; agent K); registry 61 connectors; more deterministic identifiers

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 7, 2026) parent 5270c73 Browse files

42 changed files +7,711 −356

42 changed files
Load all 42 files Unified Split
added connectors/api/_lib/capture-k.ts +89 −0
@@ -0,0 +1,89 @@
1 +/**
2 + * Live capture + smoke for the wave-3 TCG connectors. Runs each connector in probe mode through the real
3 + * router/crawl context, saves small fixtures and prints sample normalized records.
4 + * Usage: pnpm tsx connectors/api/_lib/capture-k.ts [ids…] (default: all seven)
5 + */
6 +import { createCrawlContext, createRouter, type RareIndexConnector, type RawRecordInput } from '@rareindex/connectors';
7 +import { localMeta } from './local-meta.js';
8 +import { saveFixture } from '@rareindex/connectors/testing';
9 +import { childLogger } from '@rareindex/shared';
10 +
11 +interface Plan {
12 + limit: number;
13 + seeds?: string[];
14 + /** pick which raw records become fixtures: returns a fixture name or null */
15 + pick: (raw: RawRecordInput, i: number) => string | null;
16 + kinds: string[];
17 + requiredFields: string[];
18 +}
19 +
20 +const PLANS: Record<string, Plan> = {
21 + tcgcsv: {
22 + limit: 40,
23 + seeds: ['3'],
24 + pick: (raw, i) => {
25 + const p = raw.payload as { product: { name: string }; prices: unknown[] };
26 + if (i === 0) return 'pokemon-first-product';
27 + if (p.prices.length > 1 && /charizard|blastoise|venusaur/i.test(p.product.name)) return 'pokemon-multi-subtype';
28 + return null;
29 + },
30 + kinds: ['catalog_item', 'price_observation'],
31 + requiredFields: ['attributes.identifiers.tcgplayer_id', 'attributes.set'],
32 + },
33 + digimoncard: { limit: 12, seeds: ['BT5'], pick: (_r, i) => (i === 0 ? 'bt5-first' : i === 5 ? 'bt5-sixth' : null), kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.digimoncard_id', 'attributes.number'] },
34 + 'swu-db': { limit: 8, seeds: ['SOR'], pick: (_r, i) => (i === 0 ? 'sor-first' : i === 3 ? 'sor-fourth' : null), kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.setCode', 'attributes.number'] },
35 + 'grand-archive': { limit: 6, pick: (_r, i) => (i === 0 ? 'page1-first' : i === 2 ? 'page1-third' : null), kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.gatcg_edition_id'] },
36 + 'sorcery-tcg': { limit: 6, pick: (_r, i) => (i === 0 ? 'first' : i === 5 ? 'sixth' : null), kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.sorcery_printing_id', 'attributes.set'] },
37 + mtggoldfish: {
38 + limit: 400,
39 + seeds: ['Limited+Edition+Alpha'],
40 + pick: (raw) => {
41 + const p = raw.payload as { kind: string; card: { display_name: string; foil?: boolean } };
42 + if (p.kind === 'card' && p.card.display_name === 'Black Lotus' && !p.card.foil) return 'lea-black-lotus';
43 + if (p.kind === 'history' && p.card.display_name === 'Black Lotus') return 'lea-black-lotus-history';
44 + if (p.kind === 'card' && p.card.display_name === 'Air Elemental') return 'lea-air-elemental';
45 + return null;
46 + },
47 + kinds: ['catalog_item', 'price_observation'],
48 + requiredFields: ['attributes.setCode', 'attributes.number'],
49 + },
50 + pokemonprice: { limit: 3, seeds: ['base-set-1st-edition'], pick: (raw, i) => (i === 0 ? 'base-set-1st-edition-first' : i === 2 ? 'base-set-1st-edition-third' : null), kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.set', 'attributes.number'] },
51 +};
52 +
53 +const ids = process.argv.slice(2).length ? process.argv.slice(2) : Object.keys(PLANS);
54 +const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
55 +for (const id of ids) {
56 + const plan = PLANS[id];
57 + if (!plan) {
58 + console.log(`[${id}] no plan`);
59 + continue;
60 + }
61 + const started = Date.now();
62 + const mod = (await import(`../${id}/index.ts`)) as { default: (m: ReturnType<typeof localMeta>) => RareIndexConnector };
63 + const metaJson = (await import(`../${id}/meta.json`, { with: { type: 'json' } })) as { default: unknown };
64 + const connector = mod.default(localMeta(metaJson.default));
65 + const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: plan.limit, ...(plan.seeds ? { seeds: plan.seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) });
66 + let raws = 0;
67 + let normalized = 0;
68 + let printed = 0;
69 + let saved = 0;
70 + for await (const raw of connector.crawl(ctx)) {
71 + const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() };
72 + const out = await connector.normalize(rawLike);
73 + normalized += out.length;
74 + const name = plan.pick(raw, raws);
75 + raws++;
76 + if (name) {
77 + saveFixture(id, name, { raw: rawLike, expect: { minCount: 1, kinds: plan.kinds, requiredFields: plan.requiredFields }, note: `Live capture ${new Date().toISOString().slice(0, 10)} from ${raw.url}` });
78 + saved++;
79 + }
80 + if (printed < 3) {
81 + for (const r of out.slice(0, 3)) {
82 + const summary = r.kind === 'price_observation' ? `${r.priceKind} ${r.price} ${r.currency} @ ${r.observationDate.toISOString().slice(0, 10)}${r.grade.grader ? ` [${r.grade.grader} ${r.grade.grade ?? ''}]` : ''}` : r.kind === 'catalog_item' ? `variant=${r.attributes.variant ?? '-'} set=${r.attributes.set ?? '-'} #${r.attributes.number ?? '-'} ids=${JSON.stringify(r.attributes.identifiers)}` : '';
83 + console.log(` [${id}] ${r.kind}: ${'rawTitle' in r ? r.rawTitle : ''} → ${summary}`);
84 + }
85 + printed++;
86 + }
87 + }
88 + console.log(`[${id}] raw=${raws} normalized=${normalized} fixtures=${saved} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${ctx.anomalies.slice(0, 3).join(' | ')} ms=${Date.now() - started}`);
89 +}
added connectors/api/_lib/local-meta.ts +6 −0
@@ -0,0 +1,6 @@
1 +import { ConnectorMetaSchema, type ConnectorMeta } from '@rareindex/connectors';
2 +
3 +/** Parse a connector's own meta.json (lets tests/smoke run before the registry is rebuilt). */
4 +export function localMeta(meta: unknown): ConnectorMeta {
5 + return ConnectorMetaSchema.parse(meta);
6 +}
added connectors/api/_lib/tcg-shared.ts +68 −0
@@ -0,0 +1,68 @@
1 +/**
2 + * Helpers shared by the TCG catalog/price connectors added in wave 3 (tcgcsv, digimoncard, swu-db,
3 + * grand-archive, sorcery-tcg, mtggoldfish, pokemonprice). Kept inside connectors/api (not the framework).
4 + */
5 +
6 +export const UA_HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json, text/html;q=0.9, */*;q=0.8' };
7 +
8 +/** UTC midnight of a Date (observation dates are day-precise). */
9 +export function dayOf(d: Date): Date {
10 + return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
11 +}
12 +
13 +/** "2026-09-06T20:05:30+0000" | "2026-09-06" → UTC midnight Date, else null. */
14 +export function isoDay(s: string | null | undefined): Date | null {
15 + if (!s) return null;
16 + const m = s.match(/^(\d{4})-(\d{2})-(\d{2})/);
17 + if (!m) return null;
18 + const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])));
19 + return Number.isNaN(d.getTime()) ? null : d;
20 +}
21 +
22 +/**
23 + * TCGplayer "subTypeName" / finish label → RareIndex variant vocabulary shared with the API catalogs
24 + * (pokemontcg: 'Holo', 'Reverse Holo', '1st Edition', '1st Edition Holo'; scryfall: 'Foil', 'Etched Foil').
25 + * Returns null for the base printing.
26 + */
27 +export function variantFromSubType(subType: string | null | undefined): string | null {
28 + if (!subType) return null;
29 + const s = subType.trim().toLowerCase();
30 + if (!s || s === 'normal' || s === 'regular' || s === 'standard' || s === 'non-foil' || s === 'nonfoil' || s === 'unlimited') return null;
31 + if (s === 'holofoil' || s === 'holo') return 'Holo';
32 + if (s === 'reverse holofoil' || s === 'reverse holo') return 'Reverse Holo';
33 + if (s === '1st edition holofoil' || s === '1st edition holo') return '1st Edition Holo';
34 + if (s === '1st edition normal' || s === '1st edition') return '1st Edition';
35 + if (s === 'unlimited holofoil') return 'Holo';
36 + if (s === 'foil') return 'Foil';
37 + if (s === 'etched foil' || s === 'foil etched') return 'Etched Foil';
38 + // Title-case anything else ("Hyperspace Foil", "Showcase", "Textured Foil")
39 + return subType.trim().replace(/\s+/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
40 +}
41 +
42 +/** "PSA9" | "PSA 10" | "BGS9.5" | "CGC 9.8" | "Raw" → grader/grade. */
43 +export function parseCompactGrade(label: string | null | undefined): { grader: string | null; grade: string | null } {
44 + if (!label) return { grader: null, grade: null };
45 + const s = label.trim();
46 + if (/^(raw|ungraded)$/i.test(s)) return { grader: 'raw', grade: null };
47 + const m = s.match(/^(PSA|BGS|CGC|SGC|TAG|ACE|CBCS)\s*(\d{1,2}(?:\.\d)?)$/i);
48 + if (!m) return { grader: null, grade: null };
49 + return { grader: m[1]!.toLowerCase(), grade: m[2]! };
50 +}
51 +
52 +/** "101/102" → { number: "101", total: 102 }; "SV049" → { number: "SV049", total: null } */
53 +export function splitNumber(s: string | null | undefined): { number: string | null; total: number | null } {
54 + if (!s) return { number: null, total: null };
55 + const t = s.trim();
56 + const m = t.match(/^([A-Za-z0-9-]+)\s*\/\s*([A-Za-z0-9-]+)$/);
57 + if (m) {
58 + const total = Number(m[2]);
59 + return { number: m[1]!, total: Number.isFinite(total) ? total : null };
60 + }
61 + return { number: t || null, total: null };
62 +}
63 +
64 +/** Upscale a TCGplayer CDN thumbnail (…_200w.jpg → …_400w.jpg). */
65 +export function tcgplayerImage(url: string | null | undefined): string[] {
66 + if (!url) return [];
67 + return [url.replace(/_200w\.jpg$/, '_400w.jpg')];
68 +}
added connectors/api/digimoncard/index.test.ts +29 −0
@@ -0,0 +1,29 @@
1 +import { describe, expect, it } from 'vitest';
2 +import meta from './meta.json' with { type: 'json' };
3 +import { localMeta } from '../_lib/local-meta.js';
4 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { primarySet, setPrefix } from './index.js';
6 +
7 +const connector = createConnector(localMeta(meta));
8 +
9 +describe('digimoncard', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('derives the primary set from the card number prefix', () => {
13 + expect(setPrefix('BT5-103')).toEqual({ prefix: 'BT5', code: 'BT-05' });
14 + expect(setPrefix('ST1-03').code).toBe('ST-1');
15 + expect(primarySet('BT5-103', ['Promo', 'BT-05: Battle of Omni'])).toBe('BT-05: Battle of Omni');
16 + expect(primarySet('ST1-03', ['ST-1: Starter Deck Gaia Red', 'Trial Deck'])).toBe('ST-1: Starter Deck Gaia Red');
17 + });
18 +
19 + it('emits digimon_tcg catalog items with tcgplayer ids when present', async () => {
20 + const [name] = listFixtures('digimoncard');
21 + const out = await connector.normalize(loadFixture('digimoncard', name!).raw);
22 + expect(out).toHaveLength(1);
23 + const c = out[0]!;
24 + if (c.kind !== 'catalog_item') throw new Error('expected catalog item');
25 + expect(c.attributes.categorySlug).toBe('digimon_tcg');
26 + expect(c.attributes.identifiers.digimoncard_id).toMatch(/^[A-Z]+\d*-\d+$/);
27 + expect(c.imageUrls[0]).toContain('images.digimoncard.io');
28 + });
29 +});
added connectors/api/digimoncard/index.ts +147 −0
@@ -0,0 +1,147 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, withRetries } from '../_lib/shared.js';
5 +import { UA_HEADERS } from '../_lib/tcg-shared.js';
6 +
7 +/** DigimonCard.io — full Digimon Card Game catalog in one public JSON call (no prices). */
8 +const API = 'https://digimoncard.io/api-public';
9 +const PARSER_VERSION = '1.0.0';
10 +
11 +const CardSchema = z
12 + .object({
13 + name: z.string(),
14 + type: z.string().nullable().optional(),
15 + id: z.string(),
16 + color: z.string().nullable().optional(),
17 + color2: z.string().nullable().optional(),
18 + rarity: z.string().nullable().optional(),
19 + stage: z.string().nullable().optional(),
20 + attribute: z.string().nullable().optional(),
21 + level: z.number().nullable().optional(),
22 + dp: z.number().nullable().optional(),
23 + artist: z.string().nullable().optional(),
24 + series: z.string().nullable().optional(),
25 + pretty_url: z.string().nullable().optional(),
26 + date_added: z.string().nullable().optional(),
27 + tcgplayer_name: z.string().nullable().optional(),
28 + tcgplayer_id: z.number().nullable().optional(),
29 + set_name: z.array(z.string()).nullable().optional(),
30 + })
31 + .loose();
32 +export type DigimonCard = z.infer<typeof CardSchema>;
33 +
34 +export function trimCard(raw: Record<string, unknown>): DigimonCard {
35 + const keep = ['name', 'type', 'id', 'color', 'color2', 'rarity', 'stage', 'attribute', 'level', 'dp', 'artist', 'series', 'pretty_url', 'date_added', 'tcgplayer_name', 'tcgplayer_id', 'set_name'];
36 + const out: Record<string, unknown> = {};
37 + for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];
38 + return CardSchema.parse(out);
39 +}
40 +
41 +/** "BT5-103" → { prefix: "BT5", code: "BT-05" }; "ST1-03" → { prefix "ST1", code "ST-1" }; "P-021" → { prefix "P", code "P" } */
42 +export function setPrefix(id: string): { prefix: string; code: string } {
43 + const m = id.match(/^([A-Za-z]+)(\d*)-/);
44 + if (!m) return { prefix: id, code: id };
45 + const letters = m[1]!.toUpperCase();
46 + const digits = m[2] ?? '';
47 + const code = digits ? `${letters}-${letters === 'BT' || letters === 'EX' ? digits.padStart(2, '0') : digits}` : letters;
48 + return { prefix: `${letters}${digits}`, code };
49 +}
50 +
51 +/** Pick the primary set name for a card from its set list using the number prefix. */
52 +export function primarySet(id: string, sets: string[] | null | undefined): string | null {
53 + if (!sets?.length) return null;
54 + const { code, prefix } = setPrefix(id);
55 + const letters = prefix.replace(/\d+$/, '');
56 + const digits = prefix.slice(letters.length);
57 + const candidates = [code, `${letters}-${digits}`, `${letters}-${digits.padStart(2, '0')}`, `${letters}${digits}`];
58 + for (const s of sets) {
59 + const head = s.split(':')[0]!.trim().toUpperCase();
60 + if (candidates.includes(head)) return s;
61 + }
62 + return sets[0] ?? null;
63 +}
64 +
65 +export class DigimonCardConnector extends BaseConnector {
66 + readonly version = '1.0.0';
67 + readonly parserVersion = PARSER_VERSION;
68 + protected override minIntervalMs = 1000;
69 + override readonly urlPatterns = [/digimoncard\.io\/card\/[a-z0-9-]+/i];
70 +
71 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
72 + await this.throttle();
73 + const res = await withRetries(() => ctx.fetch(`${API}/search?series=${encodeURIComponent('Digimon Card Game')}`, { engines: ['api'], headers: UA_HEADERS, timeoutMs: 120_000 }), (r) => r.success && Array.isArray(r.json), 3, 3000);
74 + if (!res.success || !Array.isArray(res.json)) throw new Error(`digimoncard: catalog unavailable (${res.error ?? res.httpStatus})`);
75 + const seen = new Map<string, number>();
76 + let count = 0;
77 + for (const raw of res.json as Record<string, unknown>[]) {
78 + let card: DigimonCard;
79 + try {
80 + card = trimCard(raw);
81 + } catch (err) {
82 + ctx.anomaly('parse_failure_card', err instanceof Error ? err.message : String(err));
83 + continue;
84 + }
85 + if (ctx.options.seeds?.length && !ctx.options.seeds.includes(setPrefix(card.id).prefix)) continue;
86 + const n = (seen.get(card.id) ?? 0) + 1;
87 + seen.set(card.id, n);
88 + if (this.reached(ctx, count)) return;
89 + count++;
90 + yield { url: `https://digimoncard.io/card/${card.pretty_url ?? card.id.toLowerCase()}`, externalId: n > 1 ? `${card.id}#${n}` : card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, alt: n }, fetchedAt: res.fetchedAt };
91 + }
92 + await ctx.setCursor({ updatedAt: new Date().toISOString(), cards: count });
93 + }
94 +
95 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
96 + const m = url.match(/digimoncard\.io\/card\/([a-z0-9-]+)/i);
97 + if (!m) return [];
98 + const id = m[1]!.split('-').slice(-2).join('-').toUpperCase(); // pretty_url ends with the card number
99 + const res = await ctx.fetch(`${API}/search?card=${encodeURIComponent(id)}`, { engines: ['api'], headers: UA_HEADERS });
100 + if (!res.success || !Array.isArray(res.json) || !res.json.length) return [];
101 + const card = trimCard(res.json[0] as Record<string, unknown>);
102 + return [{ url, externalId: card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, alt: 1 }, fetchedAt: res.fetchedAt }];
103 + }
104 +
105 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
106 + const { card, alt } = z.object({ card: CardSchema, alt: z.number().default(1) }).parse(raw.payload);
107 + const set = primarySet(card.id, card.set_name);
108 + const { code } = setPrefix(card.id);
109 + const identifiers: Record<string, string> = { digimoncard_id: card.id };
110 + if (card.tcgplayer_id) identifiers.tcgplayer_id = String(card.tcgplayer_id);
111 + const variant = alt > 1 ? `Alternate Art ${alt - 1}` : null;
112 + const a = attrs({
113 + categorySlug: 'digimon_tcg',
114 + franchise: 'Digimon',
115 + brand: 'Bandai',
116 + set,
117 + setCode: code,
118 + name: card.name,
119 + number: card.id,
120 + year: null,
121 + variant,
122 + language: 'English',
123 + rarity: card.rarity ?? null,
124 + identifiers,
125 + metadata: { type: card.type, color: card.color, color2: card.color2, stage: card.stage, attribute: card.attribute, level: card.level, dp: card.dp, artist: card.artist, sets: card.set_name ?? [] },
126 + });
127 + const rawTitle = makeTitle({ name: card.name, set, number: card.id, variant });
128 + return [
129 + catalogItem({
130 + kind: 'catalog_item',
131 + connectorId: this.meta.id,
132 + sourceId: this.meta.sourceId,
133 + sourceUrl: raw.url,
134 + externalId: raw.externalId ?? card.id,
135 + rawTitle,
136 + imageUrls: [`https://images.digimoncard.io/images/cards/${card.id}.jpg`],
137 + attributes: a,
138 + observedAt: raw.fetchedAt,
139 + confidence: 0.85,
140 + parserVersion: PARSER_VERSION,
141 + releaseDate: null,
142 + }),
143 + ];
144 + }
145 +}
146 +
147 +export default (meta: ConnectorMeta) => new DigimonCardConnector(meta);
added connectors/api/digimoncard/meta.json +30 −0
@@ -0,0 +1,30 @@
1 +{
2 + "id": "digimoncard",
3 + "displayName": "DigimonCard.io (Digimon Card Game)",
4 + "sourceId": "digimoncard",
5 + "sourceName": "DigimonCard.io",
6 + "sourceType": "catalog",
7 + "sourceUrl": "https://digimoncard.io",
8 + "module": "api/digimoncard",
9 + "enginePriority": ["api"],
10 + "categories": ["digimon_tcg"],
11 + "regions": ["US", "JP"],
12 + "languages": ["en"],
13 + "currency": [],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 10080,
22 + "priority": "low",
23 + "trustScore": 0.7,
24 + "attributionRequired": true,
25 + "termsUrl": "https://digimoncard.io/api-public/",
26 + "accessNotes": "Free public JSON API (https://digimoncard.io/api-public/search?series=Digimon%20Card%20Game returns every card in one call; the documented '.php' paths 301 to the extensionless ones). Catalog only — no prices; TCGplayer product ids are included and used as the deterministic identifier (tcgplayer_id) so TCGCSV prices attach to the same assets. Images from images.digimoncard.io/images/cards/<id>.jpg. Primary set derived from the card number prefix (BT5 → 'BT-05' set entry).",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {}
30 +}
added connectors/api/grand-archive/index.test.ts +25 −0
@@ -0,0 +1,25 @@
1 +import { describe, expect, it } from 'vitest';
2 +import meta from './meta.json' with { type: 'json' };
3 +import { localMeta } from '../_lib/local-meta.js';
4 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector from './index.js';
6 +
7 +const connector = createConnector(localMeta(meta));
8 +
9 +describe('grand-archive', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits one catalog item per edition/finish with publisher populations', async () => {
13 + const [name] = listFixtures('grand-archive');
14 + const out = await connector.normalize(loadFixture('grand-archive', name!).raw);
15 + expect(out.length).toBeGreaterThanOrEqual(1);
16 + for (const c of out) {
17 + if (c.kind !== 'catalog_item') throw new Error('expected catalog item');
18 + expect(c.attributes.categorySlug).toBe('other_tcg');
19 + expect(c.attributes.franchise).toBe('Grand Archive');
20 + expect(c.attributes.identifiers.gatcg_edition_id).toBeTruthy();
21 + // production quantity only when the publisher states an exact count
22 + if (c.attributes.productionQuantity !== null) expect(c.attributes.metadata.population_exact).toBe(true);
23 + }
24 + });
25 +});
added connectors/api/grand-archive/index.ts +123 −0
@@ -0,0 +1,123 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, withRetries } from '../_lib/shared.js';
5 +import { UA_HEADERS } from '../_lib/tcg-shared.js';
6 +
7 +/** Grand Archive official index — editions with publisher-declared circulations (print runs). */
8 +const API = 'https://api.gatcg.com';
9 +const PARSER_VERSION = '1.0.0';
10 +
11 +const CirculationSchema = z.object({ uuid: z.string().optional(), kind: z.string().nullable().optional(), foil: z.boolean().optional(), population: z.number().nullable().optional(), population_operator: z.string().nullable().optional(), printing: z.boolean().optional() });
12 +const EditionSchema = z
13 + .object({
14 + uuid: z.string(),
15 + slug: z.string().nullable().optional(),
16 + collector_number: z.string().nullable().optional(),
17 + rarity: z.number().nullable().optional(),
18 + illustrator: z.string().nullable().optional(),
19 + image: z.string().nullable().optional(),
20 + configuration: z.string().nullable().optional(),
21 + set: z.object({ id: z.string().optional(), name: z.string(), prefix: z.string().nullable().optional(), release_date: z.string().nullable().optional(), language: z.string().nullable().optional() }),
22 + circulations: z.array(CirculationSchema).default([]),
23 + })
24 + .loose();
25 +const CardSchema = z
26 + .object({
27 + uuid: z.string(),
28 + name: z.string(),
29 + slug: z.string().nullable().optional(),
30 + element: z.string().nullable().optional(),
31 + types: z.array(z.string()).optional(),
32 + classes: z.array(z.string()).optional(),
33 + editions: z.array(EditionSchema).default([]),
34 + })
35 + .loose();
36 +export type GaCard = z.infer<typeof CardSchema>;
37 +
38 +const RARITY: Record<number, string> = { 1: 'Common', 2: 'Uncommon', 3: 'Rare', 4: 'Super Rare', 5: 'Ultra Rare', 6: 'Promotional', 7: 'Collector Super Rare', 8: 'Collector Ultra Rare', 9: 'Collector Promo' };
39 +
40 +export function trimCard(raw: Record<string, unknown>): GaCard {
41 + const editions = Array.isArray(raw.editions) ? (raw.editions as Record<string, unknown>[]).map((e) => ({
42 + uuid: e.uuid, slug: e.slug, collector_number: e.collector_number, rarity: e.rarity, illustrator: e.illustrator, image: e.image, configuration: e.configuration,
43 + set: e.set && typeof e.set === 'object' ? { id: (e.set as Record<string, unknown>).id, name: (e.set as Record<string, unknown>).name, prefix: (e.set as Record<string, unknown>).prefix, release_date: (e.set as Record<string, unknown>).release_date, language: (e.set as Record<string, unknown>).language } : undefined,
44 + circulations: Array.isArray(e.circulations) ? (e.circulations as Record<string, unknown>[]).map((c) => ({ uuid: c.uuid, kind: c.kind, foil: c.foil, population: c.population, population_operator: c.population_operator, printing: c.printing })) : [],
45 + })) : [];
46 + return CardSchema.parse({ uuid: raw.uuid, name: raw.name, slug: raw.slug, element: raw.element, types: raw.types, classes: raw.classes, editions });
47 +}
48 +
49 +export class GrandArchiveConnector extends BaseConnector {
50 + readonly version = '1.0.0';
51 + readonly parserVersion = PARSER_VERSION;
52 + protected override minIntervalMs = 600;
53 +
54 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
55 + const pageSize = Number(this.meta.config.pageSize ?? 50);
56 + let page = Number(ctx.options.cursor?.page ?? 1);
57 + let count = 0;
58 + for (; page < 400; page++) {
59 + if (ctx.signal?.aborted) return;
60 + await this.throttle();
61 + const res = await withRetries(() => ctx.fetch(`${API}/cards/search?page=${page}&page_size=${pageSize}`, { engines: ['api'], headers: UA_HEADERS }), (r) => r.success && r.json !== null, 3, 2000);
62 + const body = res.json as { data?: unknown[]; has_more?: boolean } | null;
63 + if (!res.success || !Array.isArray(body?.data)) {
64 + ctx.anomaly('page_fetch_failed', `page ${page}: ${res.error ?? res.httpStatus}`);
65 + break;
66 + }
67 + for (const raw of body.data) {
68 + let card: GaCard;
69 + try {
70 + card = trimCard(raw as Record<string, unknown>);
71 + } catch (err) {
72 + ctx.anomaly('parse_failure_card', `page ${page}: ${err instanceof Error ? err.message : String(err)}`);
73 + continue;
74 + }
75 + if (this.reached(ctx, count)) return;
76 + count++;
77 + yield { url: `https://index.gatcg.com/card/${card.slug ?? card.uuid}`, externalId: card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card }, fetchedAt: res.fetchedAt };
78 + }
79 + await ctx.setCursor({ page: page + 1, updatedAt: new Date().toISOString() });
80 + if (!body.has_more) break;
81 + }
82 + await ctx.setCursor({ page: 1, updatedAt: new Date().toISOString() });
83 + }
84 +
85 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
86 + const { card } = z.object({ card: CardSchema }).parse(raw.payload);
87 + const out: NormalizedRecord[] = [];
88 + for (const ed of card.editions) {
89 + const y = ed.set.release_date ? Number(ed.set.release_date.slice(0, 4)) : NaN;
90 + const year = Number.isFinite(y) && y >= 1990 ? y : null; // epoch placeholders (1970) are not release years
91 + const images = ed.image ? [`${API}${ed.image.startsWith('/') ? '' : '/'}${ed.image}`] : [];
92 + const circulations = ed.circulations.length ? ed.circulations : [{ kind: 'NONFOIL', foil: false, population: null, population_operator: null }];
93 + const seen = new Set<string>();
94 + for (const c of circulations) {
95 + const variant = c.foil || c.kind === 'FOIL' ? 'Foil' : null;
96 + const key = variant ?? '';
97 + if (seen.has(key)) continue;
98 + seen.add(key);
99 + const exact = c.population_operator === '=' || c.population_operator === 'EXACT';
100 + const a = attrs({
101 + categorySlug: 'other_tcg',
102 + franchise: 'Grand Archive',
103 + brand: 'Weebs of the Shore',
104 + set: ed.set.name,
105 + setCode: ed.set.prefix ?? null,
106 + name: card.name,
107 + number: ed.collector_number ?? null,
108 + year,
109 + variant,
110 + language: ed.set.language === 'EN' || !ed.set.language ? 'English' : ed.set.language,
111 + rarity: ed.rarity ? (RARITY[ed.rarity] ?? String(ed.rarity)) : null,
112 + productionQuantity: exact && c.population ? c.population : null,
113 + identifiers: { gatcg_edition_id: ed.uuid, gatcg_card_id: card.uuid, ...(ed.slug ? { gatcg_slug: ed.slug } : {}) },
114 + metadata: { element: card.element, types: card.types ?? [], classes: card.classes ?? [], illustrator: ed.illustrator, configuration: ed.configuration, population: c.population ?? null, population_operator: c.population_operator ?? null, population_exact: exact },
115 + });
116 + out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${ed.uuid}${variant ? `:${variant}` : ''}`, rawTitle: makeTitle({ name: card.name, set: ed.set.name, number: ed.collector_number ?? null, year, variant }), imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: year && ed.set.release_date ? new Date(ed.set.release_date) : null }));
117 + }
118 + }
119 + return out;
120 + }
121 +}
122 +
123 +export default (meta: ConnectorMeta) => new GrandArchiveConnector(meta);
added connectors/api/grand-archive/meta.json +30 −0
@@ -0,0 +1,30 @@
1 +{
2 + "id": "grand-archive",
3 + "displayName": "Grand Archive Index (official card API)",
4 + "sourceId": "grand-archive",
5 + "sourceName": "Grand Archive TCG Index",
6 + "sourceType": "manufacturer",
7 + "sourceUrl": "https://index.gatcg.com",
8 + "module": "api/grand-archive",
9 + "enginePriority": ["api"],
10 + "categories": ["other_tcg"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": [],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": true,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 10080,
22 + "priority": "low",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://index.gatcg.com",
26 + "accessNotes": "Official Grand Archive card index API (https://api.gatcg.com/cards/search, paginated, no key). Emits one catalog_item per edition × circulation (non-foil / foil) with the publisher's declared print run in `productionQuantity` when the operator is exact ('=') and in metadata when approximate ('≈') — a rare case of manufacturer-published production counts. No prices (TCGCSV supplies Grand Archive prices via tcgplayer_id when available).",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": { "pageSize": 50 }
30 +}
added connectors/api/mtggoldfish/index.test.ts +39 −0
@@ -0,0 +1,39 @@
1 +import { describe, expect, it } from 'vitest';
2 +import meta from './meta.json' with { type: 'json' };
3 +import { localMeta } from '../_lib/local-meta.js';
4 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseHistory, parseSetPage } from './index.js';
6 +
7 +const connector = createConnector(localMeta(meta));
8 +
9 +describe('mtggoldfish', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('parses the embedded set JSON and history series', () => {
13 + const html = `<title>Limited Edition Alpha (LEA) Price Guide | MTGGoldfish</title><div data-x="{&quot;cards&quot;:[{&quot;name&quot;:&quot;Black Lotus [LEA]&quot;,&quot;set&quot;:&quot;LEA&quot;,&quot;display_name&quot;:&quot;Black Lotus&quot;,&quot;rarity&quot;:&quot;Rare&quot;,&quot;foil&quot;:false,&quot;card_num&quot;:232,&quot;finish&quot;:&quot;regular&quot;,&quot;card_uuid&quot;:&quot;u1&quot;,&quot;prices&quot;:{&quot;paper&quot;:{&quot;current_price&quot;:&quot;12345.5&quot;},&quot;online&quot;:{&quot;current_price&quot;:null}},&quot;links&quot;:{&quot;default&quot;:&quot;/price/limited-edition-alpha/232/black-lotus&quot;},&quot;source_image_variants&quot;:{&quot;672x938&quot;:&quot;https://img/x.webp&quot;}}],&quot;view_type&quot;:&quot;grid&quot;}"></div>`;
14 + const { setName, cards } = parseSetPage(html);
15 + expect(setName).toBe('Limited Edition Alpha');
16 + expect(cards).toHaveLength(1);
17 + expect(cards[0]).toMatchObject({ set: 'LEA', display_name: 'Black Lotus', card_num: 232, paper: 12345.5, card_uuid: 'u1' });
18 + expect(parseHistory('var d = "";\nd += "2010-11-02, 3102.99\\n";\nd += "2010-11-03, 3110\\n";')).toEqual([['2010-11-02', 3102.99], ['2010-11-03', 3110]]);
19 + });
20 +
21 + it('emits magic catalog items keyed like Scryfall (set code + collector number) with USD observations', async () => {
22 + const names = listFixtures('mtggoldfish');
23 + const card = names.find((n) => !n.includes('history')) ?? names[0]!;
24 + const out = await connector.normalize(loadFixture('mtggoldfish', card).raw);
25 + const c = out.find((r) => r.kind === 'catalog_item');
26 + if (!c || c.kind !== 'catalog_item') throw new Error('expected catalog item');
27 + expect(c.attributes.categorySlug).toBe('magic_the_gathering');
28 + expect(c.attributes.setCode).toMatch(/^[A-Z0-9]{2,6}$/);
29 + expect(c.attributes.number).toBeTruthy();
30 + for (const o of out) if (o.kind === 'price_observation') expect(o.currency).toBe('USD');
31 + const history = names.find((n) => n.includes('history'));
32 + if (history) {
33 + const h = await connector.normalize(loadFixture('mtggoldfish', history).raw);
34 + expect(h.length).toBeGreaterThan(10);
35 + const dates = h.map((r) => (r.kind === 'price_observation' ? r.observationDate.toISOString().slice(0, 10) : ''));
36 + expect(new Set(dates).size).toBe(dates.length);
37 + }
38 + });
39 +});
added connectors/api/mtggoldfish/index.ts +214 −0
@@ -0,0 +1,214 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as htmlUtil, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';
5 +import { dayOf, isoDay } from '../_lib/tcg-shared.js';
6 +
7 +/**
8 + * MTGGoldfish — set pages embed a JSON card list with current paper/online prices; the public
9 + * price-history component returns a daily paper price series (2010 →) per printing.
10 + */
11 +const BASE = 'https://www.mtggoldfish.com';
12 +const PARSER_VERSION = '1.0.0';
13 +const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'text/html,application/xhtml+xml' };
14 +
15 +const CardSchema = z.object({
16 + name: z.string(), // "Black Lotus [LEA]" — MTGGoldfish card id
17 + set: z.string(),
18 + display_name: z.string(),
19 + rarity: z.string().nullable().optional(),
20 + foil: z.boolean().optional(),
21 + card_num: z.union([z.number(), z.string()]).nullable().optional(),
22 + finish: z.string().nullable().optional(),
23 + card_uuid: z.string().nullable().optional(),
24 + paper: z.number().nullable(),
25 + online: z.number().nullable(),
26 + link: z.string().nullable(),
27 + image: z.string().nullable(),
28 +});
29 +type Card = z.infer<typeof CardSchema>;
30 +
31 +const SetPayloadSchema = z.object({ kind: z.literal('card'), setName: z.string().nullable(), setSlug: z.string(), card: CardSchema });
32 +const HistoryPayloadSchema = z.object({ kind: z.literal('history'), setName: z.string().nullable(), card: CardSchema, series: z.array(z.tuple([z.string(), z.number()])) });
33 +
34 +/** Parse the escaped JSON card list embedded in a set page. */
35 +export function parseSetPage(html: string): { setName: string | null; cards: Card[] } {
36 + const idx = html.indexOf('&quot;cards&quot;:[');
37 + if (idx < 0) return { setName: titleOf(html), cards: [] };
38 + const start = html.lastIndexOf('="', idx) + 2;
39 + const end = html.indexOf('"', start);
40 + const raw = html.slice(start, end);
41 + const decoded = raw.replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>');
42 + let json: { cards?: Array<Record<string, unknown>> };
43 + try {
44 + json = JSON.parse(decoded) as { cards?: Array<Record<string, unknown>> };
45 + } catch {
46 + return { setName: titleOf(html), cards: [] };
47 + }
48 + const cards: Card[] = [];
49 + for (const c of json.cards ?? []) {
50 + const prices = (c.prices ?? {}) as Record<string, { current_price?: string | null } | null>;
51 + const images = (c.source_image_variants ?? {}) as Record<string, string>;
52 + const links = (c.links ?? {}) as Record<string, string>;
53 + const parsed = CardSchema.safeParse({
54 + name: c.name,
55 + set: c.set,
56 + display_name: c.display_name,
57 + rarity: c.rarity,
58 + foil: c.foil,
59 + card_num: c.card_num,
60 + finish: c.finish,
61 + card_uuid: c.card_uuid,
62 + paper: num(prices.paper?.current_price),
63 + online: num(prices.online?.current_price),
64 + link: links.default ?? null,
65 + image: images['672x938'] ?? images['265x370'] ?? null,
66 + });
67 + if (parsed.success) cards.push(parsed.data);
68 + }
69 + return { setName: titleOf(html), cards };
70 +}
71 +
72 +function titleOf(html: string): string | null {
73 + const m = html.match(/<title>([^<]+)<\/title>/);
74 + if (!m) return null;
75 + return m[1]!.replace(/\s*[|·-]\s*MTGGoldfish.*$/i, '').replace(/\s*Price(s| Guide).*$/i, '').replace(/\s*\([A-Z0-9]{2,6}\)\s*$/, '').trim() || null;
76 +}
77 +
78 +/** "d += '2010-11-02, 3102.99\n'" lines → [[date, price], …] */
79 +export function parseHistory(js: string): Array<[string, number]> {
80 + const out: Array<[string, number]> = [];
81 + for (const m of js.matchAll(/(\d{4}-\d{2}-\d{2}),\s*([0-9.]+)/g)) {
82 + const price = Number(m[2]);
83 + if (Number.isFinite(price) && price > 0) out.push([m[1]!, price]);
84 + }
85 + return out;
86 +}
87 +
88 +export class MtgGoldfishConnector extends BaseConnector {
89 + readonly version = '1.0.0';
90 + readonly parserVersion = PARSER_VERSION;
91 + protected override minIntervalMs = 1500;
92 + override readonly urlPatterns = [/mtggoldfish\.com\/price\/[^/]+\/\d+\//i, /mtggoldfish\.com\/sets\/[^/?#]+/i];
93 +
94 + private async page(ctx: CrawlContext, url: string) {
95 + await this.throttle();
96 + return withRetries(() => ctx.fetch(url, { engines: ['api', 'firecrawl'], headers: HEADERS, responseType: 'text', minQuality: 0.3 }), (r) => r.success && Boolean(r.html || r.markdown), 3, 2500);
97 + }
98 +
99 + private async discoverSets(ctx: CrawlContext): Promise<string[]> {
100 + const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
101 + if (seeds.length) return seeds;
102 + const res = await this.page(ctx, `${BASE}/sets`);
103 + const text = res.html ?? '';
104 + const slugs = [...new Set([...text.matchAll(/href="\/sets\/([^"#?]+)"/g)].map((m) => m[1]!))];
105 + return slugs;
106 + }
107 +
108 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
109 + const sets = await this.discoverSets(ctx);
110 + const maxSets = ctx.options.mode === 'backfill' ? Infinity : Number(this.meta.config.maxSetsPerRun ?? 60);
111 + const hist = (this.meta.config.history ?? {}) as { enabled?: boolean; minPrice?: number; maxPerRun?: number; days?: number };
112 + let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);
113 + if (setIdx >= sets.length) setIdx = 0;
114 + let done = 0;
115 + let count = 0;
116 + let historyFetched = 0;
117 + for (; setIdx < sets.length && done < maxSets; setIdx++, done++) {
118 + const slug = sets[setIdx]!;
119 + const res = await this.page(ctx, `${BASE}/sets/${slug}`);
120 + if (!res.success || !res.html) {
121 + ctx.anomaly('page_fetch_failed', `${slug}: ${res.error ?? res.httpStatus}`);
122 + continue;
123 + }
124 + const { setName, cards } = parseSetPage(res.html);
125 + if (!cards.length) ctx.anomaly('empty_page', `set ${slug}`);
126 + for (const card of cards) {
127 + if (this.reached(ctx, count)) return;
128 + count++;
129 + const url = card.link ? `${BASE}${card.link}` : `${BASE}/sets/${slug}`;
130 + yield { url, externalId: card.card_uuid ?? card.name, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'card', setName, setSlug: slug, card }, fetchedAt: res.fetchedAt };
131 + if (hist.enabled !== false && card.paper !== null && card.paper >= Number(hist.minPrice ?? 100) && historyFetched < Number(hist.maxPerRun ?? 150) && !card.foil) {
132 + historyFetched++;
133 + const q = new URLSearchParams({ card_id: card.name, selector: '#tab-paper', type: 'paper', price_type: '' });
134 + const h = await this.page(ctx, `${BASE}/price_history_component?${q.toString()}`);
135 + if (h.success && h.html) {
136 + const cutoff = Date.now() - Number(hist.days ?? 1095) * 86_400_000;
137 + const series = parseHistory(h.html).filter(([d]) => new Date(`${d}T00:00:00Z`).getTime() >= cutoff);
138 + if (series.length) yield { url, externalId: `${card.card_uuid ?? card.name}:history`, kind: 'price_observation', engine: h.engine, httpStatus: h.httpStatus, payload: { kind: 'history', setName, card, series }, fetchedAt: h.fetchedAt };
139 + } else ctx.anomaly('page_fetch_failed', `history ${card.name}: ${h.error ?? h.httpStatus}`);
140 + }
141 + }
142 + await ctx.setCursor({ setIdx: setIdx + 1, updatedAt: new Date().toISOString() });
143 + }
144 + if (setIdx >= sets.length) await ctx.setCursor({ setIdx: 0, updatedAt: new Date().toISOString() });
145 + }
146 +
147 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
148 + const m = url.match(/mtggoldfish\.com\/price\/([^/]+)\/(\d+)\//i);
149 + if (!m) return [];
150 + // Card pages do not embed the JSON list; resolve through the set page.
151 + const res = await this.page(ctx, `${BASE}/sets/${m[1]}`);
152 + if (!res.success || !res.html) return [];
153 + const { setName, cards } = parseSetPage(res.html);
154 + const card = cards.find((c) => String(c.card_num) === m[2] && !c.foil) ?? cards.find((c) => String(c.card_num) === m[2]);
155 + if (!card) return [];
156 + return [{ url, externalId: card.card_uuid ?? card.name, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'card', setName, setSlug: m[1]!, card }, fetchedAt: res.fetchedAt }];
157 + }
158 +
159 + private attrsFor(card: Card, setName: string | null) {
160 + const variant = card.foil || /foil/i.test(card.finish ?? '') ? (/etched/i.test(card.finish ?? '') ? 'Etched Foil' : 'Foil') : null;
161 + return {
162 + variant,
163 + a: attrs({
164 + categorySlug: 'magic_the_gathering',
165 + franchise: 'Magic: The Gathering',
166 + brand: 'Wizards of the Coast',
167 + set: setName ?? card.set,
168 + setCode: card.set.toUpperCase(),
169 + name: card.display_name,
170 + number: card.card_num !== null && card.card_num !== undefined ? String(card.card_num) : null,
171 + year: null,
172 + variant,
173 + language: 'English',
174 + rarity: card.rarity ?? null,
175 + identifiers: { mtggoldfish_card_id: card.name, ...(card.card_uuid ? { mtggoldfish_uuid: card.card_uuid } : {}) },
176 + metadata: { finish: card.finish ?? null, online_price: card.online },
177 + }),
178 + };
179 + }
180 +
181 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
182 + const payload = raw.payload as { kind?: string };
183 + if (payload.kind === 'history') {
184 + const { card, setName, series } = HistoryPayloadSchema.parse(raw.payload);
185 + const { a, variant } = this.attrsFor(card, setName);
186 + const rawTitle = makeTitle({ name: card.display_name, set: setName ?? card.set, number: card.card_num !== null && card.card_num !== undefined ? String(card.card_num) : null, variant });
187 + const out: NormalizedRecord[] = [];
188 + const seen = new Set<string>();
189 + for (const [date, price] of series) {
190 + if (seen.has(date)) continue;
191 + seen.add(date);
192 + const day = isoDay(date);
193 + if (!day) continue;
194 + out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.card_uuid ?? card.name}:paper:${date}`, rawTitle, imageUrls: card.image ? [card.image] : [], attributes: a, observedAt: raw.fetchedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency: 'USD', observationDate: day, sampleSize: null }));
195 + }
196 + return out;
197 + }
198 + const { card, setName } = SetPayloadSchema.parse(raw.payload);
199 + const { a, variant } = this.attrsFor(card, setName);
200 + const number = card.card_num !== null && card.card_num !== undefined ? String(card.card_num) : null;
201 + const rawTitle = makeTitle({ name: card.display_name, set: setName ?? card.set, number, variant });
202 + const images = card.image ? [card.image] : [];
203 + const out: NormalizedRecord[] = [
204 + catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: card.card_uuid ?? card.name, rawTitle, imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null }),
205 + ];
206 + if (card.paper) {
207 + out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.card_uuid ?? card.name}:paper:current`, rawTitle, imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.75, parserVersion: PARSER_VERSION, priceKind: 'market', price: card.paper, currency: 'USD', observationDate: dayOf(raw.fetchedAt), sampleSize: null }));
208 + }
209 + return out;
210 + }
211 +}
212 +
213 +export default (meta: ConnectorMeta) => new MtgGoldfishConnector(meta);
214 +export { htmlUtil as _html };
added connectors/api/mtggoldfish/meta.json +34 −0
@@ -0,0 +1,34 @@
1 +{
2 + "id": "mtggoldfish",
3 + "displayName": "MTGGoldfish (paper prices & history)",
4 + "sourceId": "mtggoldfish",
5 + "sourceName": "MTGGoldfish",
6 + "sourceType": "pricing_guide",
7 + "sourceUrl": "https://www.mtggoldfish.com",
8 + "module": "api/mtggoldfish",
9 + "enginePriority": ["api", "firecrawl"],
10 + "categories": ["magic_the_gathering"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.75,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.mtggoldfish.com/robots.txt",
26 + "accessNotes": "Public set pages (https://www.mtggoldfish.com/sets/<Set+Name>) embed a JSON payload with every printing (card_uuid, set code, collector number, finish, current paper/online price, images). robots.txt allows '/' for generic agents (only widgets/embeds are disallowed; Content-Signal ai-train=no is respected — data is used as market reference, not for training). Paper price = TCGplayer-derived market price → price_observation 'market' dated by fetch day (confidence 0.75). For cards above `history.minPrice` the public price-history component (/price_history_component, daily series since 2010) is fetched, capped per run → dated observations that give assets a real multi-year guide-price history. Politeness 1.5 s/page; ~340 sets.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [],
31 + "maxSetsPerRun": 60,
32 + "history": { "enabled": true, "minPrice": 100, "maxPerRun": 150, "days": 1095 }
33 + }
34 +}
added connectors/api/pokemonprice/index.test.ts +41 −0
@@ -0,0 +1,41 @@
1 +import { describe, expect, it } from 'vitest';
2 +import meta from './meta.json' with { type: 'json' };
3 +import { localMeta } from '../_lib/local-meta.js';
4 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { extractJson, setAndVariant } from './index.js';
6 +import { parseCompactGrade } from '../_lib/tcg-shared.js';
7 +
8 +const connector = createConnector(localMeta(meta));
9 +
10 +describe('pokemonprice', () => {
11 + runFixtureSuite(connector, it, expect);
12 +
13 + it('aligns set names and variants with the pokemontcg vocabulary', () => {
14 + expect(setAndVariant('Base Set 1st Edition', 'Holo 1st edition')).toEqual({ set: 'Base Set', variant: '1st Edition Holo' });
15 + expect(setAndVariant('Base Set Unlimited', 'Holo')).toEqual({ set: 'Base Set', variant: 'Holo' });
16 + expect(setAndVariant('Base Set Shadowless', null)).toEqual({ set: 'Base Set', variant: 'Shadowless' });
17 + expect(setAndVariant('Evolving Skies', null)).toEqual({ set: 'Evolving Skies', variant: null });
18 + expect(parseCompactGrade('PSA9')).toEqual({ grader: 'psa', grade: '9' });
19 + expect(parseCompactGrade('Raw')).toEqual({ grader: 'raw', grade: null });
20 + expect(extractJson('x "rows":[{"a":[1,2]},{"b":"]"}] y', 'rows')).toEqual([{ a: [1, 2] }, { b: ']' }]);
21 + });
22 +
23 + it('emits per-grade guide values with grader/grade and transaction counts', async () => {
24 + const [name] = listFixtures('pokemonprice');
25 + const out = await connector.normalize(loadFixture('pokemonprice', name!).raw);
26 + const cat = out.find((r) => r.kind === 'catalog_item');
27 + if (!cat || cat.kind !== 'catalog_item') throw new Error('expected catalog item');
28 + expect(cat.attributes.categorySlug).toBe('pokemon');
29 + expect(cat.attributes.set).toBeTruthy();
30 + expect(cat.attributes.number).toMatch(/^\d+/);
31 + const obs = out.filter((r) => r.kind === 'price_observation');
32 + expect(obs.length).toBeGreaterThan(3);
33 + const graded = obs.filter((o) => o.kind === 'price_observation' && o.grade.grader && o.grade.grader !== 'raw');
34 + expect(graded.length).toBeGreaterThan(0);
35 + for (const o of obs) {
36 + if (o.kind !== 'price_observation') continue;
37 + expect(o.currency).toBe('USD');
38 + expect(o.confidence).toBeLessThanOrEqual(0.8);
39 + }
40 + });
41 +});
added connectors/api/pokemonprice/index.ts +262 −0
@@ -0,0 +1,262 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';
5 +import { dayOf, isoDay, parseCompactGrade } from '../_lib/tcg-shared.js';
6 +
7 +/** PokemonPrice.com — per-grade fair-value model for Pokémon cards, embedded in server-rendered pages. */
8 +const BASE = 'https://www.pokemonprice.com';
9 +const PARSER_VERSION = '1.0.0';
10 +const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'text/html,application/xhtml+xml' };
11 +
12 +const GradeRowSchema = z.object({
13 + grade: z.string(),
14 + fair_price: z.union([z.string(), z.number()]).nullable().optional(),
15 + low_price: z.union([z.string(), z.number()]).nullable().optional(),
16 + high_price: z.union([z.string(), z.number()]).nullable().optional(),
17 + confidence: z.union([z.string(), z.number()]).nullable().optional(),
18 + last_sale_date: z.string().nullable().optional(),
19 +});
20 +const TxSchema = z.object({ month: z.string(), grade: z.string(), count: z.number() });
21 +const PayloadSchema = z.object({
22 + slug: z.string(),
23 + setSlug: z.string(),
24 + name: z.string(),
25 + tag: z.string().nullable(),
26 + number: z.string().nullable(),
27 + total: z.string().nullable(),
28 + setName: z.string().nullable(),
29 + image: z.string().nullable(),
30 + rows: z.array(GradeRowSchema),
31 + transactions: z.array(TxSchema),
32 +});
33 +export type PokemonPricePayload = z.infer<typeof PayloadSchema>;
34 +
35 +/** Decode the RSC flight strings of a Next.js app-router page into one searchable string. */
36 +export function flightText(html: string): string {
37 + const parts: string[] = [];
38 + for (const m of html.matchAll(/self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)/g)) {
39 + try {
40 + parts.push(JSON.parse(`"${m[1]}"`) as string);
41 + } catch {
42 + /* skip undecodable chunk */
43 + }
44 + }
45 + return parts.join('');
46 +}
47 +
48 +/** Extract a balanced JSON array/object that starts right after `key":`. */
49 +export function extractJson(text: string, key: string): unknown | null {
50 + const idx = text.indexOf(`"${key}":`);
51 + if (idx < 0) return null;
52 + let i = text.indexOf(':', idx) + 1;
53 + while (text[i] === ' ') i++;
54 + const open = text[i];
55 + if (open !== '[' && open !== '{') return null;
56 + const close = open === '[' ? ']' : '}';
57 + let depth = 0;
58 + let inStr = false;
59 + for (let j = i; j < text.length; j++) {
60 + const ch = text[j]!;
61 + if (inStr) {
62 + if (ch === '\\') j++;
63 + else if (ch === '"') inStr = false;
64 + continue;
65 + }
66 + if (ch === '"') inStr = true;
67 + else if (ch === open) depth++;
68 + else if (ch === close) {
69 + depth--;
70 + if (depth === 0) {
71 + try {
72 + return JSON.parse(text.slice(i, j + 1));
73 + } catch {
74 + return null;
75 + }
76 + }
77 + }
78 + }
79 + return null;
80 +}
81 +
82 +export function parseCardPage(html: string, url: string): PokemonPricePayload | null {
83 + const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/);
84 + const spans = h1 ? [...h1[1]!.matchAll(/<span[^>]*>([^<]*)<\/span>/g)].map((m) => m[1]!.trim()).filter(Boolean) : [];
85 + const titleSpan = spans[0] ?? '';
86 + const nm = titleSpan.match(/^(.*?)\s*(?:\[(.+?)\])?$/);
87 + const name = (nm?.[1] ?? titleSpan).trim();
88 + const tag = nm?.[2]?.trim() ?? null;
89 + const numSpan = spans.find((s) => /^\d+[A-Za-z]?\s*\/\s*\w+$/.test(s) || /^[A-Za-z0-9]+\s*\/\s*[A-Za-z0-9]+$/.test(s));
90 + const [number, total] = numSpan ? numSpan.split('/').map((s) => s.trim()) : [null, null];
91 + const setName = spans.filter((s) => s !== titleSpan && s !== numSpan).pop() ?? null;
92 + const image = html.match(/https:\/\/cdn\.pokemonprice\.com\/cards\/[^"\\\s]+/)?.[0] ?? null;
93 + const text = flightText(html);
94 + const rowsRaw = extractJson(text, 'rows');
95 + const rows = Array.isArray(rowsRaw) ? rowsRaw.filter((r) => r && typeof r === 'object' && 'grade' in (r as object) && 'fair_price' in (r as object)).map((r) => {
96 + const o = r as Record<string, unknown>;
97 + return { grade: String(o.grade), fair_price: o.fair_price as string, low_price: o.low_price as string, high_price: o.high_price as string, confidence: o.confidence as string, last_sale_date: (o.last_sale_date as string) ?? null };
98 + }) : [];
99 + const txRaw = extractJson(text, 'transactions');
100 + const transactions = Array.isArray(txRaw) ? txRaw.filter((t) => t && typeof t === 'object' && 'month' in (t as object)).map((t) => ({ month: String((t as Record<string, unknown>).month), grade: String((t as Record<string, unknown>).grade), count: Number((t as Record<string, unknown>).count) || 0 })) : [];
101 + if (!name) return null;
102 + const parts = new URL(url).pathname.split('/').filter(Boolean);
103 + const parsed = PayloadSchema.safeParse({ slug: parts.slice(-1)[0] ?? '', setSlug: parts[0] ?? '', name, tag, number: number ?? null, total: total ?? null, setName, image, rows, transactions });
104 + return parsed.success ? parsed.data : null;
105 +}
106 +
107 +/** "Base Set 1st Edition" + tag "Holo 1st edition" → { set: "Base Set", variant: "1st Edition Holo" } */
108 +export function setAndVariant(setName: string | null, tag: string | null): { set: string | null; variant: string | null } {
109 + let set = setName?.trim() ?? null;
110 + const flags = new Set<string>();
111 + if (set) {
112 + const m = set.match(/^(.*?)\s+(1st Edition|Shadowless|Unlimited)$/i);
113 + if (m) {
114 + set = m[1]!.trim();
115 + const f = m[2]!.toLowerCase();
116 + if (f === '1st edition') flags.add('1st Edition');
117 + if (f === 'shadowless') flags.add('Shadowless');
118 + }
119 + }
120 + for (const t of (tag ?? '').split(/[,·]/).map((s) => s.trim().toLowerCase()).filter(Boolean)) {
121 + if (t.includes('reverse')) flags.add('Reverse Holo');
122 + else if (t.includes('1st') && t.includes('holo')) {
123 + flags.add('1st Edition');
124 + flags.add('Holo');
125 + } else if (t.includes('1st')) flags.add('1st Edition');
126 + else if (t.includes('holo')) flags.add('Holo');
127 + else if (t.includes('shadowless')) flags.add('Shadowless');
128 + else if (t !== 'unlimited') flags.add(t.replace(/\b\w/g, (c) => c.toUpperCase()));
129 + }
130 + const ordered: string[] = [];
131 + if (flags.has('1st Edition') && flags.has('Holo')) ordered.push('1st Edition Holo');
132 + else {
133 + if (flags.has('1st Edition')) ordered.push('1st Edition');
134 + if (flags.has('Holo')) ordered.push('Holo');
135 + }
136 + if (flags.has('Shadowless')) ordered.push('Shadowless');
137 + if (flags.has('Reverse Holo')) ordered.push('Reverse Holo');
138 + for (const f of flags) if (!['1st Edition', 'Holo', 'Shadowless', 'Reverse Holo'].includes(f)) ordered.push(f);
139 + return { set, variant: ordered.length ? ordered.join(' · ') : null };
140 +}
141 +
142 +export class PokemonPriceConnector extends BaseConnector {
143 + readonly version = '1.0.0';
144 + readonly parserVersion = PARSER_VERSION;
145 + protected override minIntervalMs = 1500;
146 + override readonly urlPatterns = [/pokemonprice\.com\/[a-z0-9-]+\/[a-z0-9-]+$/i];
147 +
148 + private async page(ctx: CrawlContext, url: string) {
149 + await this.throttle();
150 + return withRetries(() => ctx.fetch(url, { engines: ['api', 'firecrawl'], headers: HEADERS, responseType: 'text', minQuality: 0.3 }), (r) => r.success && Boolean(r.html), 3, 2500);
151 + }
152 +
153 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
154 + const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
155 + let sets = seeds;
156 + if (!sets.length) {
157 + const idx = await this.page(ctx, `${BASE}/card-lists`);
158 + sets = [...new Set([...(idx.html ?? '').matchAll(/href="\/card-lists\/([a-z0-9-]+)"/g)].map((m) => m[1]!))];
159 + }
160 + const maxCards = ctx.options.mode === 'backfill' ? Infinity : Number(this.meta.config.maxCardsPerRun ?? 300);
161 + let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);
162 + if (setIdx >= sets.length) setIdx = 0;
163 + let count = 0;
164 + for (; setIdx < sets.length; setIdx++) {
165 + const setSlug = sets[setIdx]!;
166 + const cardSlugs: string[] = [];
167 + for (let offset = 0; offset < 1200; offset += 30) {
168 + const list = await this.page(ctx, `${BASE}/card-lists/${setSlug}${offset ? `?offset=${offset}` : ''}`);
169 + if (!list.success || !list.html) {
170 + ctx.anomaly('page_fetch_failed', `list ${setSlug}@${offset}: ${list.error ?? list.httpStatus}`);
171 + break;
172 + }
173 + const found = [...new Set([...list.html.matchAll(new RegExp(`href="/${setSlug}/([a-z0-9-]+)"`, 'g'))].map((m) => m[1]!))];
174 + const fresh = found.filter((s) => !cardSlugs.includes(s));
175 + if (!fresh.length) break;
176 + cardSlugs.push(...fresh);
177 + if (!list.html.includes(`offset=${offset + 30}`)) break;
178 + }
179 + for (const cardSlug of cardSlugs) {
180 + if (ctx.signal?.aborted) return;
181 + if (this.reached(ctx, count) || count >= maxCards) {
182 + await ctx.setCursor({ setIdx, updatedAt: new Date().toISOString() });
183 + return;
184 + }
185 + const url = `${BASE}/${setSlug}/${cardSlug}`;
186 + const res = await this.page(ctx, url);
187 + if (!res.success || !res.html) {
188 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
189 + continue;
190 + }
191 + const payload = parseCardPage(res.html, url);
192 + if (!payload) {
193 + ctx.anomaly('parse_failure_card', url);
194 + continue;
195 + }
196 + count++;
197 + yield { url, externalId: `${setSlug}/${cardSlug}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
198 + }
199 + await ctx.setCursor({ setIdx: setIdx + 1, updatedAt: new Date().toISOString() });
200 + }
201 + await ctx.setCursor({ setIdx: 0, updatedAt: new Date().toISOString() });
202 + }
203 +
204 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
205 + if (!this.urlPatterns[0]!.test(url)) return [];
206 + const res = await this.page(ctx, url);
207 + if (!res.success || !res.html) return [];
208 + const payload = parseCardPage(res.html, url);
209 + if (!payload) return [];
210 + return [{ url, externalId: `${payload.setSlug}/${payload.slug}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
211 + }
212 +
213 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
214 + const p = PayloadSchema.parse(raw.payload);
215 + const { set, variant } = setAndVariant(p.setName, p.tag);
216 + const a = attrs({
217 + categorySlug: 'pokemon',
218 + franchise: 'Pokémon',
219 + brand: 'The Pokémon Company',
220 + set,
221 + name: p.name,
222 + number: p.number,
223 + year: null,
224 + variant,
225 + language: 'English',
226 + identifiers: { pokemonprice_slug: `${p.setSlug}/${p.slug}` },
227 + metadata: { total: p.total, set_label: p.setName, tag: p.tag },
228 + });
229 + const images = p.image ? [p.image] : [];
230 + const rawTitle = makeTitle({ name: p.name, set, number: p.number, total: p.total, variant });
231 + const observedAt = raw.fetchedAt;
232 + const out: NormalizedRecord[] = [
233 + catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${p.setSlug}/${p.slug}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION, releaseDate: null }),
234 + ];
235 + const kinds = new Set(((this.meta.config.priceKinds as string[] | undefined) ?? ['guide_value', 'low', 'high']));
236 + const txByGrade = new Map<string, number>();
237 + for (const t of p.transactions) txByGrade.set(t.grade, (txByGrade.get(t.grade) ?? 0) + t.count);
238 + for (const row of p.rows) {
239 + const { grader, grade } = parseCompactGrade(row.grade);
240 + if (!grader && !/^raw$/i.test(row.grade)) continue;
241 + const fair = num(row.fair_price);
242 + if (!fair) continue;
243 + const conf = Math.min(0.8, Math.max(0.2, (Number(row.confidence) || 40) / 100));
244 + // The fair price is the model's current estimate → dated by fetch day; the last sale date is kept in the title context only.
245 + const obsDate = dayOf(observedAt);
246 + const lastSale = isoDay(row.last_sale_date);
247 + const sample = txByGrade.get(row.grade) ?? null;
248 + const gradeLabel = grader === 'raw' ? 'Raw' : `${grader!.toUpperCase()} ${grade}`;
249 + const title = `${rawTitle} — ${gradeLabel}${lastSale ? ` (last sale ${lastSale.toISOString().slice(0, 10)})` : ''}`;
250 + const emit = (kind: 'guide_value' | 'low' | 'high', price: number | null) => {
251 + if (!price || !kinds.has(kind)) return;
252 + out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${p.setSlug}/${p.slug}:${row.grade}:${kind}`, rawTitle: title, imageUrls: images, attributes: a, grade: { grader: grader === 'raw' ? 'raw' : grader, grade, qualifier: null, certificationNumber: null }, observedAt, confidence: conf, parserVersion: PARSER_VERSION, priceKind: kind, price, currency: 'USD', observationDate: obsDate, sampleSize: sample }));
253 + };
254 + emit('guide_value', fair);
255 + emit('low', num(row.low_price));
256 + emit('high', num(row.high_price));
257 + }
258 + return out;
259 + }
260 +}
261 +
262 +export default (meta: ConnectorMeta) => new PokemonPriceConnector(meta);
added connectors/api/pokemonprice/meta.json +30 −0
@@ -0,0 +1,30 @@
1 +{
2 + "id": "pokemonprice",
3 + "displayName": "PokemonPrice (graded Pokémon price guide)",
4 + "sourceId": "pokemonprice",
5 + "sourceName": "PokemonPrice.com",
6 + "sourceType": "pricing_guide",
7 + "sourceUrl": "https://www.pokemonprice.com",
8 + "module": "api/pokemonprice",
9 + "enginePriority": ["api", "firecrawl"],
10 + "categories": ["pokemon"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 2880,
22 + "priority": "medium",
23 + "trustScore": 0.7,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.pokemonprice.com/about-us",
26 + "accessNotes": "Public server-rendered card pages (robots.txt allows '/' for generic agents; Content-Signal ai-train=no respected). Each card page embeds, in its React Server Components payload, a per-grade price model (Raw, PSA 1–10, BGS, CGC: fair/low/high price, model confidence, last sale date) derived from eBay and other public sold listings, plus monthly transaction counts. Emitted as price_observations 'guide_value' (+ low/high) per grade with the model confidence (capped 0.8) and the monthly transaction count as sampleSize — not transactions. Set naming follows pokemontcg ('Base Set', 'Jungle'…); edition suffixes (1st Edition, Shadowless) become the variant so assets merge with the API catalogs. ~167 set lists, ~25k card pages; politeness 1.5 s, `maxCardsPerRun` caps a run and the cursor rotates through sets.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": { "seeds": [], "maxCardsPerRun": 300, "priceKinds": ["guide_value", "low", "high"] }
30 +}
added connectors/api/sorcery-tcg/index.test.ts +21 −0
@@ -0,0 +1,21 @@
1 +import { describe, expect, it } from 'vitest';
2 +import meta from './meta.json' with { type: 'json' };
3 +import { localMeta } from '../_lib/local-meta.js';
4 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector from './index.js';
6 +
7 +const connector = createConnector(localMeta(meta));
8 +
9 +describe('sorcery-tcg', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits one catalog item per printing with finish-derived variants', async () => {
13 + const [name] = listFixtures('sorcery-tcg');
14 + const out = await connector.normalize(loadFixture('sorcery-tcg', name!).raw);
15 + expect(out.length).toBeGreaterThanOrEqual(2);
16 + const variants = out.map((c) => (c.kind === 'catalog_item' ? c.attributes.variant : undefined));
17 + expect(variants).toContain(null);
18 + expect(variants.some((v) => v && v.includes('Foil'))).toBe(true);
19 + for (const c of out) if (c.kind === 'catalog_item') expect(c.attributes.identifiers.sorcery_printing_id).toBeTruthy();
20 + });
21 +});
added connectors/api/sorcery-tcg/index.ts +94 −0
@@ -0,0 +1,94 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, withRetries } from '../_lib/shared.js';
5 +import { UA_HEADERS } from '../_lib/tcg-shared.js';
6 +
7 +/** Sorcery: Contested Realm — official catalog with per-printing finish/product/set data. */
8 +const API = 'https://api.sorcerytcg.com/api';
9 +const PARSER_VERSION = '1.0.0';
10 +
11 +const PrintingSchema = z.object({
12 + id: z.string(),
13 + slug: z.string(),
14 + printedAt: z.string().nullable().optional(),
15 + set: z.object({ name: z.string(), code: z.string().nullable().optional(), releasedAt: z.string().nullable().optional() }),
16 + meta: z.object({ finish: z.string().nullable().optional(), product: z.string().nullable().optional(), artist: z.object({ name: z.string().nullable().optional() }).nullable().optional(), typeline: z.string().nullable().optional() }).loose(),
17 +});
18 +const CardSchema = z.object({
19 + id: z.string(),
20 + name: z.string(),
21 + slug: z.string().nullable().optional(),
22 + engine: z.object({ type: z.string().nullable().optional(), category: z.string().nullable().optional(), rarity: z.string().nullable().optional(), elements: z.array(z.string()).optional(), subtypes: z.array(z.string()).optional() }).loose().optional(),
23 + printings: z.array(PrintingSchema).default([]),
24 +});
25 +export type SorceryCard = z.infer<typeof CardSchema>;
26 +
27 +export function trimCard(raw: Record<string, unknown>): SorceryCard {
28 + const engine = (raw.engine ?? {}) as Record<string, unknown>;
29 + const printings = Array.isArray(raw.printings) ? (raw.printings as Record<string, unknown>[]).map((p) => {
30 + const meta = (p.meta ?? {}) as Record<string, unknown>;
31 + return { id: p.id, slug: p.slug, printedAt: p.printedAt, set: p.set, meta: { finish: meta.finish, product: meta.product, artist: meta.artist, typeline: meta.typeline } };
32 + }) : [];
33 + return CardSchema.parse({ id: raw.id, name: raw.name, slug: raw.slug, engine: { type: engine.type, category: engine.category, rarity: engine.rarity, elements: engine.elements, subtypes: engine.subtypes }, printings });
34 +}
35 +
36 +export class SorceryConnector extends BaseConnector {
37 + readonly version = '1.0.0';
38 + readonly parserVersion = PARSER_VERSION;
39 + protected override minIntervalMs = 1000;
40 +
41 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
42 + await this.throttle();
43 + const res = await withRetries(() => ctx.fetch(`${API}/cards`, { engines: ['api'], headers: UA_HEADERS, timeoutMs: 90_000 }), (r) => r.success && Array.isArray(r.json), 3, 3000);
44 + if (!res.success || !Array.isArray(res.json)) throw new Error(`sorcery-tcg: catalog unavailable (${res.error ?? res.httpStatus})`);
45 + let count = 0;
46 + for (const raw of res.json as Record<string, unknown>[]) {
47 + let card: SorceryCard;
48 + try {
49 + card = trimCard(raw);
50 + } catch (err) {
51 + ctx.anomaly('parse_failure_card', err instanceof Error ? err.message : String(err));
52 + continue;
53 + }
54 + if (ctx.options.seeds?.length && !card.printings.some((p) => ctx.options.seeds!.includes(p.set.code ?? p.set.name))) continue;
55 + if (this.reached(ctx, count)) return;
56 + count++;
57 + yield { url: `https://curiosa.io/cards/${card.slug ?? card.id}`, externalId: card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card }, fetchedAt: res.fetchedAt };
58 + }
59 + await ctx.setCursor({ updatedAt: new Date().toISOString(), cards: count });
60 + }
61 +
62 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
63 + const { card } = z.object({ card: CardSchema }).parse(raw.payload);
64 + const out: NormalizedRecord[] = [];
65 + for (const p of card.printings) {
66 + const finish = (p.meta.finish ?? 'Standard').trim();
67 + const product = (p.meta.product ?? 'Booster').trim();
68 + const variantParts: string[] = [];
69 + if (/foil/i.test(finish)) variantParts.push('Foil');
70 + if (product && !/^booster$/i.test(product)) variantParts.push(product);
71 + const variant = variantParts.length ? variantParts.join(' · ') : null;
72 + const year = p.printedAt ? Number(p.printedAt.slice(0, 4)) || null : null;
73 + const a = attrs({
74 + categorySlug: 'other_tcg',
75 + franchise: 'Sorcery: Contested Realm',
76 + brand: "Erik's Curiosa",
77 + set: p.set.name,
78 + setCode: p.set.code ?? null,
79 + name: card.name,
80 + number: null,
81 + year,
82 + variant,
83 + language: 'English',
84 + rarity: card.engine?.rarity ?? null,
85 + identifiers: { sorcery_printing_id: p.id, sorcery_card_id: card.id, sorcery_printing_slug: p.slug },
86 + metadata: { type: card.engine?.type ?? null, category: card.engine?.category ?? null, elements: card.engine?.elements ?? [], subtypes: card.engine?.subtypes ?? [], finish, product, artist: p.meta.artist?.name ?? null, typeline: p.meta.typeline ?? null },
87 + });
88 + out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: p.id, rawTitle: makeTitle({ name: card.name, set: p.set.name, year, variant }), imageUrls: [], attributes: a, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: p.printedAt ? new Date(p.printedAt) : null }));
89 + }
90 + return out;
91 + }
92 +}
93 +
94 +export default (meta: ConnectorMeta) => new SorceryConnector(meta);
added connectors/api/sorcery-tcg/meta.json +30 −0
@@ -0,0 +1,30 @@
1 +{
2 + "id": "sorcery-tcg",
3 + "displayName": "Sorcery: Contested Realm (official card API)",
4 + "sourceId": "sorcery-tcg",
5 + "sourceName": "Sorcery TCG (Erik's Curiosa)",
6 + "sourceType": "manufacturer",
7 + "sourceUrl": "https://sorcerytcg.com",
8 + "module": "api/sorcery-tcg",
9 + "enginePriority": ["api"],
10 + "categories": ["other_tcg"],
11 + "regions": ["US", "EU"],
12 + "languages": ["en"],
13 + "currency": [],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": false,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 10080,
22 + "priority": "low",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://sorcerytcg.com",
26 + "accessNotes": "Official public JSON (https://api.sorcerytcg.com/api/cards, one call ≈1,100 cards with every printing: set code/name, finish Standard/Foil, product Booster/Precon/Promo, print date). Catalog only, no prices or images; TCGCSV supplies Sorcery prices via TCGplayer product names. One catalog_item per printing (finish → 'Foil' variant; non-booster product noted in metadata).",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {}
30 +}
added connectors/api/swu-db/index.test.ts +25 −0
@@ -0,0 +1,25 @@
1 +import { describe, expect, it } from 'vitest';
2 +import meta from './meta.json' with { type: 'json' };
3 +import { localMeta } from '../_lib/local-meta.js';
4 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector from './index.js';
6 +
7 +const connector = createConnector(localMeta(meta));
8 +
9 +describe('swu-db', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits star_wars_tcg catalog items with tcgplayer ids and USD observations', async () => {
13 + const [name] = listFixtures('swu-db');
14 + const out = await connector.normalize(loadFixture('swu-db', name!).raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + expect(cats.length).toBeGreaterThanOrEqual(1);
17 + for (const c of cats) {
18 + if (c.kind !== 'catalog_item') continue;
19 + expect(c.attributes.categorySlug).toBe('star_wars_tcg');
20 + expect(c.attributes.setCode).toMatch(/^[A-Z]{3}$/);
21 + expect(c.attributes.number).toMatch(/^\d+$/);
22 + }
23 + for (const o of out) if (o.kind === 'price_observation') expect(o.currency).toBe('USD');
24 + });
25 +});
added connectors/api/swu-db/index.ts +138 −0
@@ -0,0 +1,138 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';
5 +import { UA_HEADERS, dayOf } from '../_lib/tcg-shared.js';
6 +
7 +/** SWU-DB — Star Wars: Unlimited catalog with relayed TCGplayer market prices. */
8 +const API = 'https://api.swu-db.com';
9 +const PARSER_VERSION = '1.0.0';
10 +
11 +const CardSchema = z
12 + .object({
13 + Set: z.string(),
14 + Number: z.string(),
15 + Name: z.string(),
16 + Subtitle: z.string().nullable().optional(),
17 + Type: z.string().nullable().optional(),
18 + Rarity: z.string().nullable().optional(),
19 + VariantType: z.string().nullable().optional(),
20 + Unique: z.boolean().optional(),
21 + Artist: z.string().nullable().optional(),
22 + cid: z.string().nullable().optional(),
23 + tcgplayerId: z.string().nullable().optional(),
24 + MarketPrice: z.string().nullable().optional(),
25 + LowPrice: z.string().nullable().optional(),
26 + FoilPrice: z.string().nullable().optional(),
27 + LowFoilPrice: z.string().nullable().optional(),
28 + FrontArt: z.string().nullable().optional(),
29 + Aspects: z.array(z.union([z.string(), z.record(z.string(), z.string())])).optional(),
30 + })
31 + .loose();
32 +export type SwuCard = z.infer<typeof CardSchema>;
33 +
34 +export function trimCard(raw: Record<string, unknown>): SwuCard {
35 + const keep = ['Set', 'Number', 'Name', 'Subtitle', 'Type', 'Rarity', 'VariantType', 'Unique', 'Artist', 'cid', 'tcgplayerId', 'MarketPrice', 'LowPrice', 'FoilPrice', 'LowFoilPrice', 'FrontArt', 'Aspects'];
36 + const out: Record<string, unknown> = {};
37 + for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];
38 + return CardSchema.parse(out);
39 +}
40 +
41 +const RawPayloadSchema = z.object({ card: CardSchema, setName: z.string().nullable() });
42 +
43 +export class SwuDbConnector extends BaseConnector {
44 + readonly version = '1.0.0';
45 + readonly parserVersion = PARSER_VERSION;
46 + protected override minIntervalMs = 500;
47 + override readonly urlPatterns = [/swu-db\.com\/cards?\/[a-z]{3}\/\d+/i];
48 +
49 + private sets(): Record<string, string> {
50 + return (this.meta.config.sets ?? {}) as Record<string, string>;
51 + }
52 +
53 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
54 + const sets = Object.entries(this.sets()).filter(([code]) => !ctx.options.seeds?.length || ctx.options.seeds.map((s) => s.toUpperCase()).includes(code));
55 + let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);
56 + let count = 0;
57 + for (; setIdx < sets.length; setIdx++) {
58 + const [code, setName] = sets[setIdx]!;
59 + await this.throttle();
60 + const res = await withRetries(() => ctx.fetch(`${API}/cards/${code.toLowerCase()}`, { engines: ['api'], headers: UA_HEADERS, timeoutMs: 60_000 }), (r) => r.success && r.json !== null, 3, 2000);
61 + const data = (res.json as { data?: unknown[] } | null)?.data;
62 + if (!res.success || !Array.isArray(data)) {
63 + ctx.anomaly('page_fetch_failed', `${code}: ${res.error ?? res.httpStatus}`);
64 + continue;
65 + }
66 + for (const raw of data) {
67 + let card: SwuCard;
68 + try {
69 + card = trimCard(raw as Record<string, unknown>);
70 + } catch (err) {
71 + ctx.anomaly('parse_failure_card', `${code}: ${err instanceof Error ? err.message : String(err)}`);
72 + continue;
73 + }
74 + if (this.reached(ctx, count)) return;
75 + count++;
76 + yield { url: `https://www.swu-db.com/card/${card.Set}/${card.Number}`, externalId: `${card.Set}-${card.Number}`, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, setName }, fetchedAt: res.fetchedAt };
77 + }
78 + await ctx.setCursor({ setIdx: setIdx + 1, updatedAt: new Date().toISOString() });
79 + }
80 + await ctx.setCursor({ setIdx: 0, updatedAt: new Date().toISOString() });
81 + }
82 +
83 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
84 + const m = url.match(/swu-db\.com\/cards?\/([a-z]{3})\/(\d+)/i);
85 + if (!m) return [];
86 + const res = await ctx.fetch(`${API}/cards/${m[1]!.toLowerCase()}/${m[2]}`, { engines: ['api'], headers: UA_HEADERS });
87 + if (!res.success || !res.json) return [];
88 + const card = trimCard(res.json as Record<string, unknown>);
89 + return [{ url, externalId: `${card.Set}-${card.Number}`, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, setName: this.sets()[card.Set] ?? null }, fetchedAt: res.fetchedAt }];
90 + }
91 +
92 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
93 + const { card, setName } = RawPayloadSchema.parse(raw.payload);
94 + const identifiers: Record<string, string> = { swudb_id: `${card.Set}-${card.Number}` };
95 + if (card.cid) identifiers.swudb_cid = card.cid;
96 + if (card.tcgplayerId) identifiers.tcgplayer_id = card.tcgplayerId;
97 + const name = card.Subtitle ? `${card.Name} - ${card.Subtitle}` : card.Name;
98 + const foilEntry = /F$/i.test(card.Number);
99 + const number = foilEntry ? card.Number.replace(/F$/i, '') : card.Number;
100 + const typeVariant = card.VariantType && card.VariantType !== 'Normal' && card.VariantType !== 'Foil' ? card.VariantType : null;
101 + const baseVariant = foilEntry || card.VariantType === 'Foil' ? (typeVariant ? `${typeVariant} Foil` : 'Foil') : typeVariant;
102 + const images = card.FrontArt ? [card.FrontArt] : [];
103 + const observedAt = raw.fetchedAt;
104 + const obsDate = dayOf(observedAt);
105 + const build = (variant: string | null) =>
106 + attrs({
107 + categorySlug: 'star_wars_tcg',
108 + franchise: 'Star Wars: Unlimited',
109 + brand: 'Fantasy Flight Games',
110 + set: setName ?? card.Set,
111 + setCode: card.Set,
112 + name,
113 + number,
114 + year: null,
115 + variant,
116 + language: 'English',
117 + rarity: card.Rarity ?? null,
118 + identifiers,
119 + metadata: { type: card.Type, unique: card.Unique ?? null, artist: card.Artist, variant_type: card.VariantType ?? 'Normal', foil_price_hint: num(card.FoilPrice) },
120 + });
121 + const out: NormalizedRecord[] = [];
122 + const emit = (variant: string | null, market: number | null, low: number | null) => {
123 + const a = build(variant);
124 + const rawTitle = makeTitle({ name, set: setName ?? card.Set, number, variant });
125 + out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.Set}-${card.Number}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null }));
126 + for (const [kind, price] of [['market', market], ['low', low]] as const) {
127 + if (!price) continue;
128 + // Prices are relayed from TCGplayer without a timestamp → fetch-day observation, moderate confidence.
129 + out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.Set}-${card.Number}:${variant ?? 'normal'}:${kind}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: kind, price, currency: 'USD', observationDate: obsDate, sampleSize: null }));
130 + }
131 + };
132 + // Foil printings are separate API entries (Number "059F"); FoilPrice on the base entry is informational only.
133 + emit(baseVariant, num(card.MarketPrice), num(card.LowPrice));
134 + return out;
135 + }
136 +}
137 +
138 +export default (meta: ConnectorMeta) => new SwuDbConnector(meta);
added connectors/api/swu-db/meta.json +32 −0
@@ -0,0 +1,32 @@
1 +{
2 + "id": "swu-db",
3 + "displayName": "SWU-DB (Star Wars: Unlimited)",
4 + "sourceId": "swu-db",
5 + "sourceName": "SWU-DB",
6 + "sourceType": "catalog",
7 + "sourceUrl": "https://www.swu-db.com",
8 + "module": "api/swu-db",
9 + "enginePriority": ["api"],
10 + "categories": ["star_wars_tcg"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.7,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.swu-db.com/api",
26 + "accessNotes": "Free community API (https://api.swu-db.com/cards/<set>, no key). Each card carries TCGplayer ids and USD MarketPrice/LowPrice (+ FoilPrice/LowFoilPrice) relayed from TCGplayer without a timestamp → observations dated by the fetch day, confidence 0.7. Set codes are configured (the /catalog/sets endpoint needs an API key); VariantType (Normal, Hyperspace, Showcase…) becomes the variant, foil prices create a 'Foil' variant.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "sets": { "SOR": "Spark of Rebellion", "SHD": "Shadows of the Galaxy", "TWI": "Twilight of the Republic", "JTL": "Jump to Lightspeed", "LOF": "Legends of the Force", "SEC": "Secrets of Power" }
31 + }
32 +}
added connectors/api/tcgcsv/_capture.ts +22 −0
Large diff (22 lines) is not loaded by default to keep the page fast.Load diff
added connectors/api/tcgcsv/index.test.ts +44 −0
Large diff (44 lines) is not loaded by default to keep the page fast.Load diff
added connectors/api/tcgcsv/index.ts +189 −0
Large diff (189 lines) is not loaded by default to keep the page fast.Load diff
added connectors/api/tcgcsv/meta.json +60 −0
Large diff (60 lines) is not loaded by default to keep the page fast.Load diff
modified connectors/registry.json +2762 −355
Large diff (3,407 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/digimoncard/bt5-first.json +46 −0
Large diff (47 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/digimoncard/bt5-sixth.json +49 −0
Large diff (50 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/grand-archive/page1-first.json +64 −0
Large diff (65 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/grand-archive/page1-third.json +64 −0
Large diff (65 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/mtggoldfish/lea-air-elemental.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/mtggoldfish/lea-black-lotus.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/pokemonprice/base-set-1st-edition-first.json +1360 −0
Large diff (1,361 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/pokemonprice/base-set-1st-edition-third.json +915 −0
Large diff (916 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/sorcery-tcg/first.json +78 −0
Large diff (79 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/sorcery-tcg/sixth.json +114 −0
Large diff (115 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/swu-db/sor-first.json +48 −0
Large diff (49 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/swu-db/sor-fourth.json +47 −0
Large diff (48 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/tcgcsv/pokemon-base-charizard.json +112 −0
Large diff (113 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/tcgcsv/pokemon-base-energy.json +77 −0
Large diff (78 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/tcgcsv/pokemon-first-product.json +67 −0
Large diff (68 lines) is not loaded by default to keep the page fast.Load diff
modified workers/entity-resolution/canonical-key.ts +2 −1
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere