SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%

Connectors wave 2: PriceCharting cards + SportsCardsPro, TCGdex, Lorcast, OPTCG, Discogs, Brickset, aucfree, PCGS, Novelship; resolver variant-aware identifier match (agents H, J)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 17 days ago (Sep 7, 2026) parent f423c61

63 changed files +9,681 −464

added connectors/api/_lib/capture-pc.ts +45 −0
@@ -0,0 +1,45 @@
1 +/**
2 + * Capture live PriceCharting / SportsCardsPro card pages as compact fixtures (sales trimmed to 40 rows).
3 + * Usage: pnpm tsx connectors/api/_lib/capture-pc.ts
4 + */
5 +import { createCrawlContext, createRouter, loadConnector } from '@rareindex/connectors';
6 +import { saveFixture } from '@rareindex/connectors/testing';
7 +import { childLogger } from '@rareindex/shared';
8 +import type { ProductPayload } from './pc-core.js';
9 +
10 +const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
11 +const targets: Array<{ connector: string; url: string; name: string; expect: Record<string, unknown> }> = [
12 + { connector: 'pricecharting', url: 'https://www.pricecharting.com/game/pokemon-base-set/charizard-4', name: 'pokemon-base-set__charizard-4', expect: { minCount: 10, kinds: ['catalog_item', 'price_observation', 'sale'], requiredFields: ['attributes.setCode', 'attributes.number'] } },
13 + { connector: 'pricecharting', url: 'https://www.pricecharting.com/game/pokemon-base-set/charizard-1st-edition-4', name: 'pokemon-base-set__charizard-1st-edition-4', expect: { minCount: 5, kinds: ['catalog_item', 'price_observation', 'sale'] } },
14 + { connector: 'pricecharting', url: 'https://www.pricecharting.com/game/magic-alpha/black-lotus', name: 'magic-alpha__black-lotus', expect: { minCount: 3, kinds: ['catalog_item', 'price_observation', 'sale'] } },
15 + { connector: 'pricecharting', url: 'https://www.pricecharting.com/game/yugioh-legend-of-blue-eyes-white-dragon/blue-eyes-white-dragon-1st-edition-lob-001', name: 'yugioh-lob__blue-eyes-1st-edition', expect: { minCount: 3, kinds: ['catalog_item', 'price_observation', 'sale'] } },
16 + { connector: 'sportscardspro', url: 'https://www.sportscardspro.com/game/basketball-cards-1986-fleer/michael-jordan-57', name: 'basketball-1986-fleer__michael-jordan-57', expect: { minCount: 10, kinds: ['catalog_item', 'price_observation', 'sale'], requiredFields: ['attributes.set', 'attributes.number'] } },
17 +];
18 +
19 +for (const t of targets) {
20 + const connector = await loadConnector(t.connector);
21 + const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: 1 }, log: childLogger({ connector: t.connector, level: 'warn' }) });
22 + const [rec] = (await connector.lookup?.(t.url, ctx)) ?? [];
23 + if (!rec) {
24 + console.error(`✗ ${t.name}: no record (${ctx.anomalies.join('; ')})`);
25 + continue;
26 + }
27 + const p = rec.payload as ProductPayload;
28 + // keep at most 3 rows per tab to stay small
29 + const perTab = new Map<string, number>();
30 + p.sales = p.sales.filter((s) => {
31 + const n = (perTab.get(s.tab) ?? 0) + 1;
32 + perTab.set(s.tab, n);
33 + return n <= 3;
34 + });
35 + saveFixture(t.connector, t.name, {
36 + raw: { url: rec.url, externalId: rec.externalId ?? null, kind: rec.kind, engine: rec.engine, fetchedAt: rec.fetchedAt ?? new Date(), payload: p },
37 + expect: t.expect,
38 + note: `Live capture of ${t.url} (sales trimmed to 3 rows per grade tab).`,
39 + });
40 + const out = await connector.normalize({ ...rec, externalId: rec.externalId ?? null, fetchedAt: rec.fetchedAt ?? new Date() });
41 + const cat = out.find((r) => r.kind === 'catalog_item');
42 + console.log(`✓ ${t.name}: ${out.length} records; setRef=${JSON.stringify(p.setRef)} cardRef=${JSON.stringify(p.cardRef)} attrs=${cat && 'attributes' in cat ? JSON.stringify({ set: cat.attributes.set, setCode: cat.attributes.setCode, number: cat.attributes.number, name: cat.attributes.name, variant: cat.attributes.variant, ids: cat.attributes.identifiers }) : ''}`);
43 + for (const r of out.filter((x) => x.kind === 'sale').slice(0, 3)) if (r.kind === 'sale') console.log(` sale ${r.saleDate.toISOString().slice(0, 10)} ${r.price} ${r.currency} grader=${r.grade.grader} grade=${r.grade.grade} :: ${r.rawTitle.slice(0, 70)}`);
44 + for (const r of out.filter((x) => x.kind === 'price_observation').slice(0, 4)) if (r.kind === 'price_observation') console.log(` guide ${r.condition.conditionRaw} = ${r.price}`);
45 +}
added connectors/api/_lib/pc-core.ts +763 −0
@@ -0,0 +1,763 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { parseGradeFromTitle } from '@rareindex/taxonomy';
4 +import { extractYear, parsePrice, parseSourceDate, type NormalizedCatalogItem, type NormalizedPriceObservation, type NormalizedRecord, type NormalizedSale, type AssetAttributes, type Grade } from '@rareindex/shared';
5 +
6 +/**
7 + * Shared core for the PriceCharting family of sites (pricecharting.com, sportscardspro.com).
8 + *
9 + * A product page carries (a) guide values per condition/grade column and (b) tables of recently
10 + * completed eBay sales per condition/grade tab (date · title · price · eBay id). The page is
11 + * stored compactly as one raw record and normalised into one catalog_item, one price_observation
12 + * per non-empty guide cell ('guide_value') and one sale per completed-sale row.
13 + *
14 + * Families: video_games · lego · funko · comics (PriceCharting) · cards (Pokémon / Magic / Yu-Gi-Oh!
15 + * on PriceCharting) · sports (SportsCardsPro). Card pages use grade columns instead of conditions.
16 + */
17 +
18 +export const PARSER_VERSION = '2.0.0';
19 +
20 +export const PriceCellSchema = z.object({ key: z.string(), value: z.number().nullable(), raw: z.string() });
21 +export const SaleRowSchema = z.object({ tab: z.string(), date: z.string(), title: z.string(), price: z.number(), ebayId: z.string().nullable(), listedPrice: z.number().nullable() });
22 +export const SetRefSchema = z.object({ id: z.string().nullable(), code: z.string().nullable(), name: z.string().nullable(), source: z.string() });
23 +export const CardRefSchema = z.object({ scryfall_id: z.string().optional(), number: z.string().optional(), pokemontcg_id: z.string().optional(), tcgplayer_id: z.string().optional() });
24 +export const ProductPayloadSchema = z.object({
25 + kind: z.literal('product'),
26 + url: z.string(),
27 + productId: z.string().nullable(),
28 + consoleUri: z.string(),
29 + consoleName: z.string(),
30 + title: z.string(),
31 + flags: z.object({ isComic: z.boolean(), isLegoSet: z.boolean(), isFunkoPop: z.boolean(), isCard: z.boolean(), isCoin: z.boolean(), isSystem: z.boolean() }),
32 + columnLabels: z.array(z.string()),
33 + prices: z.array(PriceCellSchema),
34 + /** card pages: every row of the "full prices" table (Ungraded, Grade 1 … PSA 10, BGS 10 Black…) */
35 + fullPrices: z.array(z.object({ label: z.string(), value: z.number().nullable() })).default([]),
36 + /** tab id → human label ("grade-seventeen" → "CGC 10"), parsed from the page's tab selector */
37 + tabLabels: z.record(z.string(), z.string()).default({}),
38 + sales: z.array(SaleRowSchema),
39 + details: z.record(z.string(), z.string()),
40 + images: z.array(z.string()),
41 + /** set resolved against a reference catalog at crawl time (TCGdex / pokemontcg / Scryfall) */
42 + setRef: SetRefSchema.nullable().default(null),
43 + /** card resolved against a reference catalog at crawl time (Scryfall exact-name lookup) */
44 + cardRef: CardRefSchema.nullable().default(null),
45 + site: z.enum(['pricecharting', 'sportscardspro']).default('pricecharting'),
46 +});
47 +export type ProductPayload = z.infer<typeof ProductPayloadSchema>;
48 +
49 +export type Family = 'video_games' | 'lego' | 'funko' | 'comics' | 'cards' | 'sports';
50 +
51 +interface ColumnMeaning {
52 + condition: string | null;
53 + completeness: string | null;
54 + conditionRaw: string;
55 + grade?: string | null;
56 +}
57 +
58 +export const PRICE_KEYS = ['used_price', 'complete_price', 'new_price', 'graded_price', 'box_only_price', 'manual_only_price'] as const;
59 +export const TAB_TO_KEY: Record<string, (typeof PRICE_KEYS)[number]> = {
60 + used: 'used_price',
61 + cib: 'complete_price',
62 + new: 'new_price',
63 + graded: 'graded_price',
64 + 'box-only': 'box_only_price',
65 + 'manual-only': 'manual_only_price',
66 + 'loose-and-manual': 'graded_price',
67 +};
68 +
69 +/** Column semantics per family (the site reuses the same six cells with different labels). */
70 +export const COLUMN_MEANING: Record<Exclude<Family, 'cards' | 'sports'>, Partial<Record<(typeof PRICE_KEYS)[number], ColumnMeaning>>> = {
71 + video_games: {
72 + used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Loose' },
73 + complete_price: { condition: 'cib', completeness: 'cib', conditionRaw: 'Complete in box' },
74 + new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },
75 + graded_price: { condition: null, completeness: null, conditionRaw: 'Graded (any grader)' },
76 + box_only_price: { condition: null, completeness: 'box_only', conditionRaw: 'Box only' },
77 + manual_only_price: { condition: null, completeness: 'manual_only', conditionRaw: 'Manual only' },
78 + },
79 + lego: {
80 + used_price: { condition: 'used_complete', completeness: 'used_complete', conditionRaw: 'Pieces only (used, complete pieces)' },
81 + complete_price: { condition: 'opened_complete', completeness: 'opened_complete', conditionRaw: 'Complete (pieces, box, manual)' },
82 + new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },
83 + manual_only_price: { condition: null, completeness: 'instructions_only', conditionRaw: 'Manual only' },
84 + },
85 + funko: {
86 + used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Out of box' },
87 + complete_price: { condition: 'boxed', completeness: 'boxed', conditionRaw: 'In damaged box' },
88 + new_price: { condition: 'mint_in_box', completeness: 'boxed', conditionRaw: 'New (mint in box)' },
89 + },
90 + comics: {
91 + used_price: { condition: null, completeness: null, conditionRaw: 'Ungraded (raw)', grade: null },
92 + complete_price: { condition: 'very_good', completeness: null, conditionRaw: 'Graded 4.0 / VG (any grader)', grade: '4.0' },
93 + new_price: { condition: 'fine', completeness: null, conditionRaw: 'Graded 6.0 / Fine (any grader)', grade: '6.0' },
94 + graded_price: { condition: 'very_fine', completeness: null, conditionRaw: 'Graded 8.0 / VF (any grader)', grade: '8.0' },
95 + box_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.2 / NM- (any grader)', grade: '9.2' },
96 + manual_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.8 (any grader)', grade: '9.8' },
97 + },
98 +};
99 +
100 +/** Card pages: the six main cells map to Ungraded / Grade 7 / Grade 8 / Grade 9 / Grade 9.5 / PSA 10. */
101 +export const CARD_CELL_LABEL: Record<(typeof PRICE_KEYS)[number], string> = {
102 + used_price: 'Ungraded',
103 + complete_price: 'Grade 7',
104 + new_price: 'Grade 8',
105 + graded_price: 'Grade 9',
106 + box_only_price: 'Grade 9.5',
107 + manual_only_price: 'PSA 10',
108 +};
109 +const CARD_TAB_LABEL: Record<string, string> = { used: 'Ungraded', cib: 'Grade 7', new: 'Grade 8', graded: 'Grade 9', 'box-only': 'Grade 9.5', 'manual-only': 'PSA 10' };
110 +
111 +/** Generic grader used when the site reports a grade without a grading company ("Grade 9"). */
112 +export const GENERIC_GRADER = 'graded';
113 +
114 +export interface CardGrade {
115 + grader: string | null;
116 + grade: string | null;
117 + qualifier: string | null;
118 + conditionRaw: string;
119 +}
120 +
121 +/** "PSA 10" | "BGS 10 Black" | "CGC 10 Pristine" | "Grade 9.5" | "Ungraded" → structured grade. */
122 +export function gradeFromLabel(label: string): CardGrade | null {
123 + const l = label.replace(/\s*\(\d+\)\s*$/, '').trim();
124 + if (!l) return null;
125 + if (/^ungraded$/i.test(l)) return { grader: null, grade: null, qualifier: null, conditionRaw: 'Ungraded (raw)' };
126 + let m = l.match(/^grade\s+(\d+(?:\.\d)?)$/i);
127 + if (m) return { grader: GENERIC_GRADER, grade: m[1]!, qualifier: null, conditionRaw: `Grade ${m[1]} (any grader)` };
128 + m = l.match(/^(PSA|BGS|CGC|SGC|TAG|ACE)\s+(\d+(?:\.\d)?)\s*(Black|Pristine|Black Label)?$/i);
129 + if (m) {
130 + const q = m[3] ? (/black/i.test(m[3]) ? 'Black Label' : 'Pristine') : null;
131 + return { grader: m[1]!.toLowerCase(), grade: m[2]!, qualifier: q, conditionRaw: l };
132 + }
133 + return null;
134 +}
135 +
136 +/** Refine a generic grade with the grader named in the eBay row title when it agrees on the grade. */
137 +export function refineGradeFromTitle(base: CardGrade, rowTitle: string): CardGrade {
138 + if (base.grader !== GENERIC_GRADER || !base.grade) return base;
139 + const t = parseGradeFromTitle(rowTitle);
140 + if (t.grader && t.grader !== 'raw' && t.grade && Number(t.grade) === Number(base.grade)) {
141 + return { grader: t.grader, grade: base.grade, qualifier: t.qualifier ?? null, conditionRaw: `${t.grader.toUpperCase()} ${base.grade}` };
142 + }
143 + return base;
144 +}
145 +
146 +const NINTENDO = /^(nes|famicom|super-nintendo|super-famicom|nintendo-64|gamecube|wii|wii-u|nintendo-switch|gameboy|gameboy-color|gameboy-advance|nintendo-ds|nintendo-3ds|virtual-boy|jp-|pal-)/;
147 +const SEGA = /^(sega-|pal-sega|jp-sega)/;
148 +const PLAYSTATION = /^(playstation|psp|playstation-vita|jp-playstation|pal-playstation)/;
149 +const XBOX = /^xbox/;
150 +const RETRO = /^(atari|intellivision|colecovision|neo-geo|turbografx|pc-engine|commodore|amiga|vectrex|3do|jaguar|philips-cd-i|magnavox|odyssey|msx|sharp|wonderswan|n-gage|game-com|tiger|evercade|super-cassette|fairchild|bally|arcadia|action-max|amiga-cd32)/;
151 +const SPORTS = ['basketball', 'baseball', 'football', 'hockey', 'soccer', 'wrestling', 'golf', 'racing', 'boxing', 'tennis', 'ufc', 'mma', 'formula', 'multi-sport', 'non-sport'] as const;
152 +
153 +export function familyOf(p: ProductPayload): Family | null {
154 + if (p.site === 'sportscardspro') return 'sports';
155 + if (p.flags.isComic || p.consoleUri.startsWith('comic-books')) return 'comics';
156 + if (p.flags.isLegoSet || p.consoleUri.startsWith('lego')) return 'lego';
157 + if (p.flags.isFunkoPop || p.consoleUri.startsWith('funko')) return 'funko';
158 + if (p.flags.isCoin) return null; // coins handled elsewhere
159 + if (p.flags.isCard || /^(pokemon|magic|yugioh)-/.test(p.consoleUri)) {
160 + return /^(pokemon|magic|yugioh)-/.test(p.consoleUri) ? 'cards' : null; // other card games not mapped yet
161 + }
162 + return 'video_games';
163 +}
164 +
165 +export function categorySlug(p: ProductPayload, family: Family): string | null {
166 + const c = p.consoleUri;
167 + switch (family) {
168 + case 'lego':
169 + return 'lego_sets';
170 + case 'funko':
171 + return 'funko';
172 + case 'comics': {
173 + const pub = (p.details['Publisher'] ?? '').toLowerCase();
174 + if (/marvel/.test(pub)) return 'marvel_comics';
175 + if (/\bdc\b|dc comics|vertigo|wildstorm/.test(pub)) return 'dc_comics';
176 + return 'independent_comics';
177 + }
178 + case 'cards':
179 + if (c.startsWith('pokemon-')) return 'pokemon';
180 + if (c.startsWith('magic-')) return 'magic_the_gathering';
181 + if (c.startsWith('yugioh-')) return 'yugioh';
182 + return null;
183 + case 'sports': {
184 + const sport = SPORTS.find((s) => c.startsWith(s));
185 + switch (sport) {
186 + case 'basketball':
187 + return 'basketball_cards';
188 + case 'baseball':
189 + return 'baseball_cards';
190 + case 'football':
191 + return 'football_cards';
192 + case 'hockey':
193 + return 'hockey_cards';
194 + case 'soccer':
195 + return 'soccer_cards';
196 + case 'racing':
197 + case 'formula':
198 + return 'f1_cards';
199 + case 'non-sport':
200 + return 'non_sport_cards';
201 + default:
202 + return 'other_sports_cards';
203 + }
204 + }
205 + case 'video_games':
206 + if (c === 'pc-games') return 'pc_games';
207 + if (NINTENDO.test(c)) return 'nintendo_games';
208 + if (SEGA.test(c)) return 'sega_games';
209 + if (PLAYSTATION.test(c)) return 'playstation_games';
210 + if (XBOX.test(c)) return 'xbox_games';
211 + if (RETRO.test(c)) return 'atari_retro_games';
212 + return 'video_games';
213 + }
214 +}
215 +
216 +const YGO_CODE = /\b([A-Z0-9]{2,6}-[A-Z]{0,3}\d{2,4})\b/;
217 +/** Tags that describe the card's status, not a distinct printing (kept in metadata, not in variant). */
218 +const NON_VARIANT_TAGS = /^(rookie|rc|key issue|hall of fame|hof|error|misprint)$/i;
219 +
220 +/** "Super Mario 64 [Player's Choice]" → { name, variant } ; "Cloud City #10123" → { name, number } ; "Blue-Eyes White Dragon [1st Edition] LOB-001" → number LOB-001 */
221 +export function splitTitle(title: string, family?: Family | null): { name: string; variant: string | null; number: string | null; tags: string[] } {
222 + let t = title.trim();
223 + const variants: string[] = [];
224 + const tags: string[] = [];
225 + t = t.replace(/\[([^\]]+)\]/g, (_m, v: string) => {
226 + const tag = v.trim();
227 + if (NON_VARIANT_TAGS.test(tag)) tags.push(tag);
228 + else variants.push(tag);
229 + return ' ';
230 + });
231 + let number: string | null = null;
232 + const num = t.match(/#\s*([A-Za-z0-9.\-/]+)/);
233 + if (num) {
234 + number = num[1]!;
235 + t = t.replace(num[0], ' ');
236 + } else if (family === 'cards') {
237 + const code = t.match(YGO_CODE);
238 + if (code) {
239 + number = code[1]!;
240 + t = t.replace(code[0], ' ');
241 + }
242 + }
243 + return { name: t.replace(/\s+/g, ' ').trim(), variant: variants.length ? variants.join(' · ') : null, number, tags };
244 +}
245 +
246 +/** Pokémon/Magic set name from a console name: "Pokemon Base Set" → "Base Set"; "Magic Alpha" → "Alpha". */
247 +export function setNameFromConsole(consoleName: string, family: Family): string {
248 + if (family === 'cards') return consoleName.replace(/^(Pokemon|Pokémon|Magic|YuGiOh|Yu-Gi-Oh!?)\s+/i, '').trim();
249 + return consoleName.trim();
250 +}
251 +
252 +export function parseProductPage(htmlText: string, url: string, site: ProductPayload['site'] = 'pricecharting'): ProductPayload {
253 + const $ = H.load(htmlText);
254 + const h1 = $('h1#product_name');
255 + const consoleName = H.text(h1.find('a')) ?? '';
256 + const consoleUri = (h1.find('a').attr('href') ?? '').replace(/^(https?:\/\/[^/]+)?\/console\//, '') || (url.match(/\/game\/([^/]+)\//)?.[1] ?? '');
257 + const title = h1
258 + .clone()
259 + .children('a')
260 + .remove()
261 + .end()
262 + .text()
263 + .replace(/\s+/g, ' ')
264 + .trim();
265 + const flagsBlock = htmlText.match(/VGPC\.product\s*=\s*\{([\s\S]*?)\}/)?.[1] ?? '';
266 + const flag = (k: string) => new RegExp(`${k}\\s*:\\s*true`).test(flagsBlock);
267 + let productId = flagsBlock.match(/id\s*:\s*(\d+)/)?.[1] ?? null;
268 +
269 + const prices: z.infer<typeof PriceCellSchema>[] = [];
270 + const seen = new Set<string>();
271 + for (const key of PRICE_KEYS) {
272 + if (seen.has(key)) continue;
273 + const cell = $(`td#${key}`).first();
274 + if (!cell.length) continue;
275 + seen.add(key);
276 + const raw = H.text(cell.find('.price').first()) ?? H.text(cell) ?? '';
277 + const parsed = parsePrice(raw, 'USD');
278 + prices.push({ key, value: parsed && parsed.amount > 0 ? parsed.amount : null, raw });
279 + }
280 + const columnLabels: string[] = [];
281 + $('#price_data thead th, table.price_data thead th').each((_, el) => {
282 + const t = H.text($(el));
283 + if (t) columnLabels.push(t);
284 + });
285 + const fullPrices: Array<{ label: string; value: number | null }> = [];
286 + $('#full-prices tr').each((_, tr) => {
287 + const tds = $(tr).find('td');
288 + if (tds.length < 2) return;
289 + const label = H.text(tds.eq(0)) ?? '';
290 + const v = parsePrice(H.text(tds.eq(1)) ?? '', 'USD');
291 + if (label) fullPrices.push({ label, value: v && v.amount > 0 ? v.amount : null });
292 + });
293 + const tabLabels: Record<string, string> = {};
294 + $('option[value^="completed-auctions-"]').each((_, el) => {
295 + const tab = ($(el).attr('value') ?? '').replace('completed-auctions-', '');
296 + const label = (H.text($(el)) ?? '').replace(/\s*\(\d+\)\s*$/, '').trim();
297 + if (tab && label) tabLabels[tab] = label;
298 + });
299 +
300 + const sales: z.infer<typeof SaleRowSchema>[] = [];
301 + $('div[class^="completed-auctions-"]').each((_, sec) => {
302 + const cls = ($(sec).attr('class') ?? '').split(/\s+/).find((c) => c.startsWith('completed-auctions-')) ?? '';
303 + const tab = cls.replace('completed-auctions-', '');
304 + if (!tab || tab === 'condition' || !$(sec).find('table').length) return;
305 + $(sec)
306 + .find('tbody tr')
307 + .each((__, tr) => {
308 + const $tr = $(tr);
309 + const date = H.text($tr.find('td.date')) ?? '';
310 + const titleEl = $tr.find('td.title a').first();
311 + const rowTitle = H.text(titleEl) ?? H.text($tr.find('td.title')) ?? '';
312 + const priceTxt = H.text($tr.find('td.numeric .js-price').first()) ?? H.text($tr.find('td.numeric').first()) ?? '';
313 + const price = parsePrice(priceTxt, 'USD');
314 + const listedTxt = H.text($tr.find('td.listed-price'));
315 + const listed = listedTxt ? parsePrice(listedTxt, 'USD') : null;
316 + const ebayId = ($tr.attr('id') ?? '').match(/ebay-(\d+)/)?.[1] ?? titleEl.attr('href')?.match(/\/itm\/(\d+)/)?.[1] ?? null;
317 + if (!date || !rowTitle || !price || price.amount <= 0) return;
318 + sales.push({ tab, date, title: rowTitle, price: price.amount, ebayId, listedPrice: listed && listed.amount > 0 ? listed.amount : null });
319 + });
320 + });
321 +
322 + const details: Record<string, string> = {};
323 + $('td.title').each((_, el) => {
324 + const k = (H.text($(el)) ?? '').replace(/:$/, '').trim();
325 + const v = H.text($(el).next('td.details'));
326 + if (k && v && v !== 'none' && v !== 'n/a') details[k] = v;
327 + });
328 + if (!productId && details['PriceCharting ID']) productId = details['PriceCharting ID'].trim();
329 + const images: string[] = [];
330 + $('img[src*="images.pricecharting.com"]').each((_, el) => {
331 + const src = $(el).attr('src');
332 + if (src && /\/(240|1600|400)\.jpg$/.test(src) && !images.includes(src)) images.push(src);
333 + });
334 +
335 + return {
336 + kind: 'product',
337 + url,
338 + productId,
339 + consoleUri,
340 + consoleName,
341 + title,
342 + flags: { isComic: flag('is_comic'), isLegoSet: flag('is_lego_set'), isFunkoPop: flag('is_funko_pop'), isCard: flag('is_card'), isCoin: flag('is_coin'), isSystem: flag('is_system') },
343 + columnLabels,
344 + prices,
345 + fullPrices,
346 + tabLabels,
347 + sales,
348 + details,
349 + images: images.slice(0, 4),
350 + setRef: null,
351 + cardRef: null,
352 + site,
353 + };
354 +}
355 +
356 +export interface ConsoleProduct {
357 + id: string;
358 + productUri: string;
359 + productName: string;
360 + consoleUri: string;
361 +}
362 +
363 +/** Parse a JSON body that may arrive wrapped in <html><body>…</body></html> by a rendering engine. */
364 +export function parseJsonBody<T>(res: { json: unknown; html: string | null }): T | null {
365 + if (res.json && typeof res.json === 'object') return res.json as T;
366 + const text = res.html ?? '';
367 + if (!text) return null;
368 + const stripped = text.replace(/^[\s\S]*?<body[^>]*>/i, '').replace(/<\/body>[\s\S]*$/i, '').replace(/<[^>]+>/g, '').trim();
369 + const candidate = stripped.startsWith('{') || stripped.startsWith('[') ? stripped : text.trim();
370 + try {
371 + return JSON.parse(candidate) as T;
372 + } catch {
373 + return null;
374 + }
375 +}
376 +
377 +export interface SiteOptions {
378 + base: string;
379 + site: ProductPayload['site'];
380 + /** category slugs (site "/category/<slug>") whose console lists are discovered automatically */
381 + categoryPrefix: (categorySlug: string) => string;
382 +}
383 +
384 +/**
385 + * Base connector for PriceCharting-like sites. Subclasses provide the site and may enrich a
386 + * parsed product with reference-catalog lookups (setRef / cardRef) before it is yielded.
387 + */
388 +export abstract class PriceChartingLikeConnector extends BaseConnector {
389 + readonly version = '2.0.0';
390 + readonly parserVersion = PARSER_VERSION;
391 + protected override minIntervalMs = 1500;
392 + protected abstract readonly siteOptions: SiteOptions;
393 +
394 + protected get seeds(): string[] {
395 + const s = this.meta.config.seeds;
396 + return Array.isArray(s) ? (s as string[]) : [];
397 + }
398 + protected get cardCategories(): string[] {
399 + const s = this.meta.config.cardCategories;
400 + return Array.isArray(s) ? (s as string[]) : [];
401 + }
402 +
403 + /** Hook: enrich a product payload with reference-catalog data (set code, card number, ids). */
404 + protected async enrich(_ctx: CrawlContext, payload: ProductPayload): Promise<ProductPayload> {
405 + return payload;
406 + }
407 + /** Hook: called once per crawl before iterating consoles (load reference maps). */
408 + protected async prepare(_ctx: CrawlContext): Promise<void> {}
409 +
410 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
411 + const perConsole = Number(this.meta.config.productsPerConsole ?? 200);
412 + const sort = String(this.meta.config.sort ?? 'popularity');
413 + const maxConsoles = Number(this.meta.config.maxConsolesPerCategory ?? 40);
414 + const cursor = ctx.options.cursor ?? {};
415 + const doneConsoles = new Set<string>((cursor.doneConsoles as string[] | undefined) ?? []);
416 + let count = 0;
417 +
418 + await this.prepare(ctx);
419 + let seeds = ctx.options.seeds?.length ? [...ctx.options.seeds] : [...this.seeds];
420 + if (!ctx.options.seeds?.length) {
421 + for (const cat of this.cardCategories) {
422 + const discovered = await this.discoverConsoles(ctx, cat, maxConsoles);
423 + for (const c of discovered) if (!seeds.includes(c)) seeds.push(c);
424 + }
425 + }
426 + seeds = [...new Set(seeds)];
427 +
428 + for (const consoleUri of seeds) {
429 + if (doneConsoles.has(consoleUri) && ctx.options.mode !== 'backfill') continue;
430 + if (ctx.signal?.aborted) return;
431 + const products = await this.listConsole(ctx, consoleUri, perConsole, sort);
432 + if (products.length === 0) ctx.anomaly('empty_console', consoleUri);
433 + for (const p of products) {
434 + if (this.reached(ctx, count)) return;
435 + const url = `${this.siteOptions.base}/game/${p.consoleUri}/${p.productUri}`;
436 + if (!(await ctx.shouldFetch(url))) continue;
437 + await this.throttle();
438 + const rec = await this.fetchProduct(ctx, url);
439 + if (rec) {
440 + count++;
441 + yield rec;
442 + }
443 + }
444 + doneConsoles.add(consoleUri);
445 + await ctx.setCursor({ doneConsoles: [...doneConsoles], updatedAt: new Date().toISOString() });
446 + }
447 + // Full pass complete: reset so the next incremental run starts again.
448 + await ctx.setCursor({ doneConsoles: [], updatedAt: new Date().toISOString() });
449 + }
450 +
451 + /** Console (set) slugs listed on a category page, in page order (most popular first). */
452 + protected async discoverConsoles(ctx: CrawlContext, category: string, max: number): Promise<string[]> {
453 + await this.throttle();
454 + const url = `${this.siteOptions.base}/category/${category}`;
455 + const res = await ctx.fetch(url, { responseType: 'text', minQuality: 0 });
456 + if (!res.success || !res.html) {
457 + ctx.anomaly('category_discovery_failed', `${category}: ${res.error ?? res.httpStatus}`);
458 + return [];
459 + }
460 + const prefix = this.siteOptions.categoryPrefix(category);
461 + const out: string[] = [];
462 + for (const m of res.html.matchAll(/href="(?:https?:\/\/[^/"]+)?\/console\/([a-z0-9-]+)"/g)) {
463 + const slug = m[1]!;
464 + if (slug.startsWith(prefix) && !out.includes(slug)) out.push(slug);
465 + if (out.length >= max) break;
466 + }
467 + return out;
468 + }
469 +
470 + protected async listConsole(ctx: CrawlContext, consoleUri: string, max: number, sort: string): Promise<ConsoleProduct[]> {
471 + const out: ConsoleProduct[] = [];
472 + let cursorPos = 0;
473 + while (out.length < max) {
474 + await this.throttle();
475 + const url = `${this.siteOptions.base}/console/${consoleUri}?sort=${encodeURIComponent(sort)}&cursor=${cursorPos}&format=json`;
476 + const res = await ctx.fetch(url, { responseType: 'json', minQuality: 0 });
477 + const json = parseJsonBody<{ products?: Array<Record<string, unknown>>; cursor?: string }>(res);
478 + if (!res.success || !json?.products) {
479 + if (cursorPos === 0) ctx.anomaly('console_list_failed', `${consoleUri}: ${res.error ?? 'no products'}`);
480 + break;
481 + }
482 + for (const p of json.products) {
483 + if (typeof p.productUri === 'string' && typeof p.consoleUri === 'string') {
484 + out.push({ id: String(p.id ?? ''), productUri: p.productUri, productName: String(p.productName ?? ''), consoleUri: p.consoleUri });
485 + }
486 + }
487 + const next = Number(json.cursor);
488 + if (!Number.isFinite(next) || next <= cursorPos || json.products.length === 0) break;
489 + cursorPos = next;
490 + }
491 + return out.slice(0, max);
492 + }
493 +
494 + protected async fetchProduct(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {
495 + const site = this.siteOptions.site;
496 + const res = await ctx.fetch(url, {
497 + responseType: 'text',
498 + expect: ['title', 'price', 'identifiers', 'category'],
499 + parse: (r) => {
500 + if (!r.html) return null;
501 + const p = parseProductPage(r.html, url, site);
502 + return { title: p.title, price: p.prices.find((x) => x.value !== null)?.value ?? p.fullPrices.find((x) => x.value !== null)?.value ?? null, identifiers: p.productId ? { id: p.productId } : null, category: p.consoleUri };
503 + },
504 + });
505 + if (!res.success || !res.html) {
506 + ctx.anomaly('product_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
507 + return null;
508 + }
509 + let payload = parseProductPage(res.html, url, site);
510 + if (!payload.title) {
511 + ctx.anomaly('parse_failure_title', url);
512 + return null;
513 + }
514 + try {
515 + payload = await this.enrich(ctx, payload);
516 + } catch (err) {
517 + ctx.anomaly('enrich_failed', `${url}: ${err instanceof Error ? err.message : String(err)}`);
518 + }
519 + return { url, externalId: payload.productId, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
520 + }
521 +
522 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
523 + const clean = url.split('?')[0]!;
524 + await this.prepare(ctx);
525 + const rec = await this.fetchProduct(ctx, clean);
526 + return rec ? [rec] : [];
527 + }
528 +
529 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
530 + const p = ProductPayloadSchema.parse(raw.payload);
531 + return normalizeProduct(p, { connectorId: this.meta.id, sourceId: this.meta.sourceId, fetchedAt: raw.fetchedAt });
532 + }
533 +}
534 +
535 +// ---------------------------------------------------------------------------------------------
536 +// Normalisation
537 +// ---------------------------------------------------------------------------------------------
538 +
539 +export interface NormalizeCtx {
540 + connectorId: string;
541 + sourceId: string;
542 + fetchedAt: Date;
543 +}
544 +
545 +const RAW_GRADE: Grade = { grader: null, grade: null, qualifier: null, certificationNumber: null };
546 +
547 +export function normalizeProduct(p: ProductPayload, nctx: NormalizeCtx): NormalizedRecord[] {
548 + const family = familyOf(p);
549 + if (!family) return [];
550 + const slug = categorySlug(p, family);
551 + if (!slug) return [];
552 + const { name, variant, number, tags } = splitTitle(p.title, family);
553 + const isCardFamily = family === 'cards' || family === 'sports';
554 + const setName = isCardFamily ? (p.setRef?.name ?? setNameFromConsole(p.consoleName, family)) : null;
555 + const year = isCardFamily ? (extractYear(p.consoleName) ?? extractYear(p.details['Release Date'] ?? '') ?? null) : (extractYear(p.details['Release Date'] ?? '') ?? extractYear(p.title) ?? null);
556 +
557 + const identifiers: Record<string, string> = {};
558 + if (p.productId) identifiers.pricecharting_id = p.site === 'sportscardspro' ? `scp:${p.productId}` : p.productId;
559 + if (p.details['UPC']) identifiers.upc = p.details['UPC'].replace(/\s+/g, '');
560 + if (p.details['ASIN (Amazon)']) identifiers.asin = p.details['ASIN (Amazon)'];
561 + if (p.details['ePID (eBay)']) identifiers.ebay_epid = p.details['ePID (eBay)'];
562 + if (p.details['Comic.org ID']) identifiers.comics_org_id = p.details['Comic.org ID'];
563 + if (p.details['TCGPlayer ID']) identifiers.tcgplayer_id = p.details['TCGPlayer ID'];
564 + if (family === 'lego' && number) identifiers.lego_set_number = number;
565 + if (p.details['Model Number']) identifiers.model_number = p.details['Model Number'];
566 + if (family === 'funko' && p.details['Box Number']) identifiers.funko_box_number = p.details['Box Number'];
567 + if (p.cardRef?.scryfall_id) identifiers.scryfall_id = p.cardRef.scryfall_id;
568 + if (p.cardRef?.pokemontcg_id) identifiers.pokemontcg_id = p.cardRef.pokemontcg_id;
569 + if (p.cardRef?.tcgplayer_id) identifiers.tcgplayer_id = p.cardRef.tcgplayer_id;
570 +
571 + let cardNumber = number;
572 + if (isCardFamily && !cardNumber && p.cardRef?.number) cardNumber = p.cardRef.number;
573 + const language = /japanese/i.test(p.consoleName) ? 'Japanese' : /korean/i.test(p.consoleName) ? 'Korean' : /chinese/i.test(p.consoleName) ? 'Chinese' : 'English';
574 + const cardVariant = isCardFamily ? normalizeCardVariant(variant, slug) : variant;
575 + const setCode = isCardFamily ? (p.setRef?.code ?? (slug === 'yugioh' && cardNumber ? cardNumber.split('-')[0]! : null)) : null;
576 +
577 + const attributes: AssetAttributes = {
578 + categorySlug: slug,
579 + subcategorySlug: null,
580 + franchise: family === 'funko' || family === 'lego' ? p.consoleName.replace(/^(Funko POP|LEGO)\s*/i, '') || null : slug === 'pokemon' ? 'Pokémon' : slug === 'magic_the_gathering' ? 'Magic: The Gathering' : slug === 'yugioh' ? 'Yu-Gi-Oh!' : null,
581 + brand: family === 'lego' ? 'LEGO' : family === 'funko' ? 'Funko' : family === 'sports' ? brandFromSet(setName ?? '') : slug === 'pokemon' ? 'The Pokémon Company' : slug === 'magic_the_gathering' ? 'Wizards of the Coast' : slug === 'yugioh' ? 'Konami' : (p.details['Publisher'] ?? null),
582 + series: family === 'funko' ? (p.details['Series'] ?? p.consoleName) : null,
583 + set: isCardFamily ? setName : family === 'comics' ? p.consoleName : family === 'lego' ? p.consoleName.replace(/^LEGO\s*/i, '') : p.consoleName,
584 + setCode,
585 + name: family === 'comics' ? p.consoleName : name,
586 + model: null,
587 + reference: null,
588 + number: cardNumber ?? (family === 'funko' ? (p.details['Box Number'] ?? null) : null),
589 + year,
590 + edition: null,
591 + variant: cardVariant,
592 + language: isCardFamily ? language : 'English',
593 + region: isCardFamily ? null : /^(jp|pal)-/.test(p.consoleUri) ? p.consoleUri.split('-')[0]!.toUpperCase() : 'NTSC-U',
594 + country: null,
595 + material: null,
596 + size: null,
597 + color: null,
598 + rarity: null,
599 + productionQuantity: null,
600 + originalMsrp: null,
601 + originalMsrpCurrency: null,
602 + identifiers,
603 + metadata: {
604 + pricecharting_console: p.consoleUri,
605 + pricecharting_site: p.site,
606 + key_issue: p.details['Is Key Issue'] === 'Yes' ? true : undefined,
607 + rookie: family === 'sports' && (p.details['Is Rookie Card'] === 'Yes' || tags.some((t) => /rookie|^rc$/i.test(t))) ? true : undefined,
608 + tags: tags.length ? tags : undefined,
609 + genre: p.details['Genre'],
610 + notes: p.details['Notes'],
611 + set_ref_source: p.setRef?.source,
612 + },
613 + };
614 + if (family === 'comics') {
615 + attributes.name = p.consoleName;
616 + attributes.year = extractYear(p.title) ?? year;
617 + }
618 +
619 + const observedAt = nctx.fetchedAt;
620 + const base = {
621 + connectorId: nctx.connectorId,
622 + sourceId: nctx.sourceId,
623 + sourceUrl: p.url,
624 + externalId: p.productId ? (p.site === 'sportscardspro' ? `scp:${p.productId}` : p.productId) : null,
625 + rawTitle: family === 'comics' ? p.title : `${p.title} (${p.consoleName})`,
626 + description: p.details['Description'] ?? null,
627 + imageUrls: p.images,
628 + attributes,
629 + observedAt,
630 + confidence: 0.9,
631 + parserVersion: PARSER_VERSION,
632 + };
633 + const out: NormalizedRecord[] = [];
634 + const catalog: NormalizedCatalogItem = { kind: 'catalog_item', ...base, grade: RAW_GRADE, condition: { condition: null, conditionRaw: null, completeness: null }, releaseDate: parseSourceDate(p.details['Release Date']) ?? null };
635 + out.push(catalog);
636 +
637 + const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));
638 +
639 + if (isCardFamily) {
640 + // Guide values: prefer the full price table (all grades); fall back to the six main cells.
641 + const cells: Array<{ label: string; value: number | null }> = p.fullPrices.length ? p.fullPrices : p.prices.map((c) => ({ label: CARD_CELL_LABEL[c.key as (typeof PRICE_KEYS)[number]] ?? c.key, value: c.value }));
642 + const seenLabel = new Set<string>();
643 + for (const cell of cells) {
644 + const g = gradeFromLabel(cell.label);
645 + if (!g || cell.value === null || seenLabel.has(cell.label)) continue;
646 + seenLabel.add(cell.label);
647 + out.push({
648 + kind: 'price_observation',
649 + ...base,
650 + externalId: `${base.externalId ?? p.url}:guide:${cell.label.toLowerCase().replace(/[^a-z0-9.]+/g, '-')}`,
651 + grade: { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: null },
652 + condition: { condition: null, conditionRaw: g.conditionRaw, completeness: null },
653 + priceKind: 'guide_value',
654 + price: cell.value,
655 + currency: 'USD',
656 + observationDate: obsDate,
657 + sampleSize: null,
658 + confidence: 0.85,
659 + } satisfies NormalizedPriceObservation);
660 + }
661 + const seenSale = new Set<string>();
662 + for (const row of p.sales) {
663 + const label = p.tabLabels[row.tab] ?? CARD_TAB_LABEL[row.tab];
664 + if (!label) continue;
665 + const g0 = gradeFromLabel(label);
666 + if (!g0) continue;
667 + const g = refineGradeFromTitle(g0, row.title);
668 + const saleDate = parseSourceDate(row.date);
669 + if (!saleDate) continue;
670 + const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;
671 + if (seenSale.has(dedupe)) continue;
672 + seenSale.add(dedupe);
673 + out.push(makeSale(base, p, row, saleDate, { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, { condition: null, conditionRaw: g.conditionRaw, completeness: null }));
674 + }
675 + return out;
676 + }
677 +
678 + const meanings = COLUMN_MEANING[family];
679 + for (const cell of p.prices) {
680 + const m = meanings[cell.key as (typeof PRICE_KEYS)[number]];
681 + if (!m || cell.value === null) continue;
682 + out.push({
683 + kind: 'price_observation',
684 + ...base,
685 + externalId: `${base.externalId ?? p.url}:guide:${cell.key}`,
686 + grade: { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null },
687 + condition: { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness },
688 + priceKind: 'guide_value',
689 + price: cell.value,
690 + currency: 'USD',
691 + observationDate: obsDate,
692 + sampleSize: null,
693 + confidence: 0.85,
694 + } satisfies NormalizedPriceObservation);
695 + }
696 + const seenSale = new Set<string>();
697 + for (const row of p.sales) {
698 + const key = TAB_TO_KEY[row.tab];
699 + const m = key ? meanings[key] : undefined;
700 + if (!m) continue;
701 + const saleDate = parseSourceDate(row.date);
702 + if (!saleDate) continue;
703 + const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;
704 + if (seenSale.has(dedupe)) continue;
705 + seenSale.add(dedupe);
706 + out.push(makeSale(base, p, row, saleDate, { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null }, { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness }));
707 + }
708 + return out;
709 +}
710 +
711 +function makeSale(base: Omit<NormalizedCatalogItem, 'kind' | 'grade' | 'condition' | 'releaseDate'>, p: ProductPayload, row: z.infer<typeof SaleRowSchema>, saleDate: Date, grade: Grade, condition: NormalizedSale['condition']): NormalizedSale {
712 + return {
713 + kind: 'sale',
714 + ...base,
715 + externalId: row.ebayId ? `ebay:${row.ebayId}` : `${p.productId ?? p.url}:${row.date}:${row.price}`,
716 + rawTitle: row.title,
717 + description: null,
718 + grade,
719 + condition,
720 + saleType: 'unknown',
721 + saleDate,
722 + price: row.price,
723 + currency: 'USD',
724 + buyerPremiumIncluded: false,
725 + quantity: 1,
726 + isBundle: /\b(lot|bundle)\b/i.test(row.title),
727 + location: null,
728 + auctionHouse: null,
729 + lotNumber: null,
730 + confidence: 0.75,
731 + };
732 +}
733 +
734 +/** Align bracket tags with the API catalogs' variant vocabulary (pokemontcg: 'Holo', 'Reverse Holo', '1st Edition', '1st Edition Holo'; scryfall: 'Foil'). */
735 +export function normalizeCardVariant(variant: string | null, slug: string): string | null {
736 + if (!variant) return null;
737 + const parts = variant.split(' · ').map((v) => v.trim()).filter(Boolean);
738 + const mapped = parts.map((v) => {
739 + const l = v.toLowerCase();
740 + if (slug === 'pokemon') {
741 + if (l === 'reverse holo' || l === 'reverse holofoil') return 'Reverse Holo';
742 + if (l === 'holo' || l === 'holofoil') return 'Holo';
743 + if (l === '1st edition') return '1st Edition';
744 + if (l === 'shadowless') return 'Shadowless';
745 + }
746 + if (slug === 'magic_the_gathering') {
747 + if (l === 'foil') return 'Foil';
748 + if (l === 'etched foil' || l === 'foil etched') return 'Etched Foil';
749 + }
750 + return v;
751 + });
752 + // Pokémon: "1st Edition · Holo" → "1st Edition Holo" (pokemontcg vocabulary)
753 + if (slug === 'pokemon' && mapped.includes('1st Edition') && mapped.includes('Holo')) {
754 + return ['1st Edition Holo', ...mapped.filter((v) => v !== '1st Edition' && v !== 'Holo')].join(' · ');
755 + }
756 + return mapped.join(' · ');
757 +}
758 +
759 +const CARD_BRANDS = ['Topps', 'Panini', 'Upper Deck', 'Bowman', 'Fleer', 'Donruss', 'Score', 'Leaf', 'Futera', 'O-Pee-Chee', 'Skybox', 'Hoops', 'Pinnacle', 'Pro Set', 'Goudey', 'Playoff', 'Pacific', 'Select', 'Prizm', 'Mosaic', 'Optic'];
760 +export function brandFromSet(setName: string): string | null {
761 + for (const b of CARD_BRANDS) if (new RegExp(`\\b${b.replace('-', '\\-')}\\b`, 'i').test(setName)) return b === 'Prizm' || b === 'Select' || b === 'Mosaic' || b === 'Optic' ? 'Panini' : b;
762 + return null;
763 +}
added connectors/api/brickset/_smoke.ts +22 −0
@@ -0,0 +1,22 @@
1 +import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';
2 +import { saveFixture } from '@rareindex/connectors/testing';
3 +import createConnector from './index.js';
4 +
5 +const save = process.argv.includes('--save');
6 +const meta = getConnectorMeta('brickset');
7 +const connector = createConnector(meta);
8 +const ctx = createCrawlContext({ router: createRouter({}), meta, options: { mode: 'probe', limit: 3, seeds: ['theme-Icons'] } });
9 +for await (const raw of connector.crawl(ctx)) {
10 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
11 + console.log(`raw ${raw.externalId} → ${out.length}`);
12 + for (const r of out) console.log(' ', JSON.stringify(r).slice(0, 320));
13 +}
14 +const lk = await connector.lookup('https://brickset.com/sets/10179-1/Ultimate-Collectors-Millennium-Falcon', ctx);
15 +if (lk[0]) {
16 + const out = await connector.normalize({ ...lk[0], externalId: lk[0].externalId ?? null, fetchedAt: lk[0].fetchedAt ?? new Date() });
17 + console.log('lookup 10179 →', out.length, JSON.stringify(out[0]).slice(0, 400));
18 + if (save) saveFixture('brickset', '10179-1-millennium-falcon', { raw: { url: lk[0].url, externalId: lk[0].externalId ?? null, kind: lk[0].kind, engine: lk[0].engine, fetchedAt: lk[0].fetchedAt ?? new Date(), payload: lk[0].payload }, expect: { count: 3, kinds: ['catalog_item', 'price_observation'], first: { 'attributes.number': '10179', 'attributes.year': 2007 } }, note: 'Captured live from brickset.com' });
19 +}
20 +const lk2 = await connector.lookup('https://brickset.com/sets/75192-1', ctx);
21 +if (lk2[0] && save) saveFixture('brickset', '75192-1-millennium-falcon', { raw: { url: lk2[0].url, externalId: lk2[0].externalId ?? null, kind: lk2[0].kind, engine: lk2[0].engine, fetchedAt: lk2[0].fetchedAt ?? new Date(), payload: lk2[0].payload }, expect: { minCount: 1, kinds: ['catalog_item', 'price_observation'] }, note: 'Captured live from brickset.com' });
22 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/api/brickset/index.test.ts +41 −0
@@ -0,0 +1,41 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector, { parseCurrentValue, parseLaunchExit, parseRrp, setLinks } from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('brickset'));
7 +
8 +describe('brickset', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('maps 10179 to catalog + new/used guide values with barcodes', async () => {
12 + const fx = loadFixture('brickset', '10179-1-millennium-falcon');
13 + const out = await connector.normalize(fx.raw);
14 + const cat = out.find((r) => r.kind === 'catalog_item');
15 + if (cat?.kind !== 'catalog_item') throw new Error();
16 + expect(cat.attributes.categorySlug).toBe('lego_sets');
17 + expect(cat.attributes.identifiers.lego_set_number).toBe('10179');
18 + expect(cat.attributes.identifiers.upc).toBe('673419079419');
19 + expect(cat.attributes.franchise).toBe('Star Wars');
20 + expect(cat.attributes.originalMsrp).toBe(499.99);
21 + expect(cat.attributes.originalMsrpCurrency).toBe('USD');
22 + expect(cat.attributes.metadata.pieces).toBe(5197);
23 + expect(cat.attributes.metadata.retired).toBe(true);
24 + const sealed = out.find((r) => r.kind === 'price_observation' && r.condition.completeness === 'sealed');
25 + const used = out.find((r) => r.kind === 'price_observation' && r.condition.completeness === 'used_complete');
26 + if (sealed?.kind !== 'price_observation' || used?.kind !== 'price_observation') throw new Error();
27 + expect(sealed.price).toBeGreaterThan(used.price);
28 + expect(sealed.priceKind).toBe('guide_value');
29 + expect(sealed.currency).toBe('USD');
30 + });
31 +
32 + it('helpers', () => {
33 + expect(parseRrp('£342.49, $499.99')).toEqual({ gbp: 342.49, usd: 499.99, eur: null });
34 + expect(parseCurrentValue('New: ~$2858 | Used: ~$1172')).toEqual({ newUsd: 2858, usedUsd: 1172 });
35 + expect(parseCurrentValue(undefined)).toEqual({ newUsd: null, usedUsd: null });
36 + const le = parseLaunchExit('01 Oct 07 - 31 Dec 09');
37 + expect(le.launch?.toISOString().slice(0, 10)).toBe('2007-10-01');
38 + expect(le.exit?.toISOString().slice(0, 10)).toBe('2009-12-31');
39 + expect(setLinks('<a href="/sets/10179-1/Millennium-Falcon">x</a><a href="/sets/10179-1/Millennium-Falcon#tab">y</a><a href="/sets/theme-Icons">z</a>')).toEqual(['https://brickset.com/sets/10179-1/Millennium-Falcon']);
40 + });
41 +});
added connectors/api/brickset/index.ts +208 −0
@@ -0,0 +1,208 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, parsePrice, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +/**
6 + * Brickset — LEGO set facts (pieces, minifigs, RRP, launch/exit, barcodes) + Brickset's
7 + * "Current value" New/Used estimates. Raw payload = the parsed definition-list of a set page.
8 + */
9 +
10 +const BASE = 'https://brickset.com';
11 +const PARSER_VERSION = '1.0.0';
12 +
13 +export const SetPayloadSchema = z.object({
14 + kind: z.literal('set_page'),
15 + url: z.string(),
16 + setNumber: z.string(),
17 + title: z.string(),
18 + image: z.string().nullable(),
19 + fields: z.record(z.string(), z.string()),
20 +});
21 +export type SetPayload = z.infer<typeof SetPayloadSchema>;
22 +
23 +/** Parse the <dl> facts of a set page into a flat label → text map (links flattened). */
24 +export function parseSetPage(htmlText: string, url: string): SetPayload | null {
25 + const $ = H.load(htmlText);
26 + const fields: Record<string, string> = {};
27 + $('dl').each((_, dl) => {
28 + const dts = $(dl).find('dt');
29 + dts.each((__, dt) => {
30 + const label = H.text($(dt));
31 + const dd = $(dt).next('dd');
32 + if (!label || !dd.length) return;
33 + const clone = dd.clone();
34 + clone.find('br').replaceWith(' | ');
35 + const value = H.text(clone);
36 + if (value && !(label in fields)) fields[label] = value;
37 + });
38 + });
39 + const setNumber = fields['Set number'] ?? url.match(/\/sets\/(\d+-\d+)/)?.[1] ?? null;
40 + const title = ($('title').text().split('|')[0] ?? '').trim();
41 + if (!setNumber || !title) return null;
42 + const image = $('meta[property="og:image"]').attr('content') ?? $('img[src*="/sets/images/"]').attr('src') ?? null;
43 + return { kind: 'set_page', url, setNumber, title, image, fields };
44 +}
45 +
46 +export function setLinks(htmlText: string): string[] {
47 + const out = new Set<string>();
48 + for (const m of htmlText.matchAll(/href="(\/sets\/\d+-\d+\/[^"#?]*)"/g)) out.add(BASE + m[1]!);
49 + return [...out];
50 +}
51 +
52 +/** "£342.49, $499.99" → { gbp: 342.49, usd: 499.99 } */
53 +export function parseRrp(s: string | undefined): { gbp: number | null; usd: number | null; eur: number | null } {
54 + const grab = (re: RegExp) => {
55 + const m = s?.match(re);
56 + return m ? Number(m[1]!.replace(/,/g, '')) : null;
57 + };
58 + return { gbp: grab(/£\s?([\d,]+(?:\.\d+)?)/), usd: grab(/\$\s?([\d,]+(?:\.\d+)?)/), eur: grab(/€\s?([\d,]+(?:\.\d+)?)/) };
59 +}
60 +
61 +/** "New: ~$2858 | Used: ~$1172" → { newUsd, usedUsd } */
62 +export function parseCurrentValue(s: string | undefined): { newUsd: number | null; usedUsd: number | null } {
63 + const n = s?.match(/New:\s*~?\$([\d,]+(?:\.\d+)?)/);
64 + const u = s?.match(/Used:\s*~?\$([\d,]+(?:\.\d+)?)/);
65 + return { newUsd: n ? Number(n[1]!.replace(/,/g, '')) : null, usedUsd: u ? Number(u[1]!.replace(/,/g, '')) : null };
66 +}
67 +
68 +/** "01 Oct 07 - 31 Dec 09" → { launch: '2007-10-01', exit: '2009-12-31' } */
69 +export function parseLaunchExit(s: string | undefined): { launch: Date | null; exit: Date | null } {
70 + if (!s) return { launch: null, exit: null };
71 + const [a, b] = s.split(/\s+-\s+/);
72 + const fix = (x?: string) => {
73 + if (!x) return null;
74 + const m = x.trim().match(/^(\d{1,2}) ([A-Za-z]{3}) (\d{2})$/);
75 + if (!m) return parseSourceDate(x);
76 + const yy = Number(m[3]);
77 + return parseSourceDate(`${m[1]} ${m[2]} ${yy >= 50 ? 1900 + yy : 2000 + yy}`);
78 + };
79 + return { launch: fix(a), exit: fix(b) };
80 +}
81 +
82 +export class BricksetConnector extends BaseConnector {
83 + readonly version = '1.0.0';
84 + readonly parserVersion = PARSER_VERSION;
85 + protected override minIntervalMs = 1500;
86 + override readonly urlPatterns = [/^https?:\/\/(www\.)?brickset\.com\/sets\/(\d+-\d+)/i];
87 +
88 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
89 + const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
90 + const pages = Number(this.meta.config.pagesPerSeed ?? 2);
91 + const cap = ctx.options.limit ?? Number(this.meta.config.setsPerRun ?? 120);
92 + const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
93 + let count = 0;
94 + for (let i = startSeed; i < seeds.length && count < cap; i++) {
95 + const seed = seeds[i]!;
96 + for (let page = 1; page <= pages && count < cap; page++) {
97 + if (ctx.signal?.aborted) return;
98 + const listUrl = `${BASE}/sets/${seed.replace(/^\//, '')}${page > 1 ? `/page-${page}` : ''}`;
99 + await this.throttle();
100 + const lres = await ctx.fetch(listUrl, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0.2 });
101 + const links = lres.success && lres.html ? setLinks(lres.html) : [];
102 + if (!links.length) {
103 + ctx.anomaly('discover_failed', `${listUrl}: ${lres.error ?? lres.httpStatus ?? 'no set links'}`);
104 + break;
105 + }
106 + for (const url of links) {
107 + if (ctx.signal?.aborted || count >= cap) break;
108 + if (!(await ctx.shouldFetch(url))) continue;
109 + const rec = await this.fetchSet(url, ctx);
110 + if (rec) {
111 + count++;
112 + yield rec;
113 + }
114 + }
115 + }
116 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
117 + }
118 + }
119 +
120 + private async fetchSet(url: string, ctx: CrawlContext): Promise<RawRecordInput | null> {
121 + await this.throttle();
122 + const res = await ctx.fetch(url, {
123 + engines: ['api', 'firecrawl'],
124 + responseType: 'text',
125 + expect: ['title', 'identifiers', 'price'],
126 + parse: (r) => {
127 + const p = r.html ? parseSetPage(r.html, url) : null;
128 + return p ? { title: p.title, identifiers: { set: p.setNumber }, price: p.fields['RRP'] ?? p.fields['Current value'] ?? null } : null;
129 + },
130 + });
131 + const payload = res.success && res.html ? parseSetPage(res.html, url) : null;
132 + if (!payload) {
133 + ctx.anomaly('set_parse_failed', `${url}: ${res.error ?? res.httpStatus}`);
134 + return null;
135 + }
136 + return { url, externalId: payload.setNumber, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
137 + }
138 +
139 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
140 + const m = url.match(this.urlPatterns[0]!);
141 + if (!m) return [];
142 + const rec = await this.fetchSet(`${BASE}/sets/${m[2]}`, ctx);
143 + return rec ? [rec] : [];
144 + }
145 +
146 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
147 + const p = SetPayloadSchema.parse(raw.payload);
148 + const f = p.fields;
149 + const number = p.setNumber.replace(/-\d+$/, '');
150 + const name = (f['Name'] ?? p.title.replace(/^LEGO\s+\d+(?:-\d+)?\s+/i, '')).trim();
151 + const year = f['Year released'] ? Number(f['Year released'].match(/\d{4}/)?.[0]) || null : null;
152 + const rrp = parseRrp(f['RRP']);
153 + const value = parseCurrentValue(f['Current value']);
154 + const { launch, exit } = parseLaunchExit(f['Launch/exit']);
155 + const pieces = f['Pieces'] ? Number(f['Pieces'].replace(/,/g, '')) || null : null;
156 + const minifigs = f['Minifigs'] ? Number(f['Minifigs'].match(/\d+/)?.[0]) || null : null;
157 + const upc = f['Barcodes']?.match(/UPC:\s*(\d+)/)?.[1] ?? null;
158 + const ean = f['Barcodes']?.match(/EAN:\s*(\d+)/)?.[1] ?? null;
159 + const theme = f['Theme'] ?? null;
160 + const subtheme = f['Subtheme'] ?? null;
161 + const rawTitle = `LEGO ${number} ${name}${theme ? ` · ${theme}` : ''}${year ? ` (${year})` : ''}`;
162 + const attributes = AssetAttributesSchema.parse({
163 + categorySlug: 'lego_sets',
164 + brand: 'LEGO',
165 + franchise: theme,
166 + series: subtheme,
167 + set: theme,
168 + name,
169 + number,
170 + year,
171 + productionQuantity: null,
172 + originalMsrp: rrp.usd ?? rrp.gbp ?? rrp.eur,
173 + originalMsrpCurrency: rrp.usd ? 'USD' : rrp.gbp ? 'GBP' : rrp.eur ? 'EUR' : null,
174 + identifiers: { lego_set_number: number, brickset_set: p.setNumber, ...(upc ? { upc } : {}), ...(ean ? { ean } : {}) },
175 + metadata: {
176 + pieces,
177 + minifigs,
178 + rrp_gbp: rrp.gbp,
179 + rrp_usd: rrp.usd,
180 + availability: f['Availability'] ?? null,
181 + packaging: f['Packaging'] ?? null,
182 + launch_date: launch ? launch.toISOString().slice(0, 10) : null,
183 + exit_date: exit ? exit.toISOString().slice(0, 10) : null,
184 + retired: exit ? exit.getTime() < Date.now() : null,
185 + age_range: f['Age range'] ?? null,
186 + rating: f['Rating'] ? Number(f['Rating'].match(/(\d\.\d)/)?.[1]) || null : null,
187 + notes: f['Notes'] ?? null,
188 + },
189 + });
190 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: p.url, rawTitle, imageUrls: p.image ? [p.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
191 + const out: NormalizedRecord[] = [NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: p.setNumber, confidence: 0.95, releaseDate: launch })];
192 + const obsDate = raw.fetchedAt;
193 + if (value.newUsd) {
194 + out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `${p.setNumber}:new`, confidence: 0.65, condition: { condition: 'sealed', conditionRaw: 'New (Brickset current value, BrickLink-derived)', completeness: 'sealed' }, priceKind: 'guide_value', price: value.newUsd, currency: 'USD', observationDate: obsDate, sampleSize: null }));
195 + }
196 + if (value.usedUsd) {
197 + out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `${p.setNumber}:used`, confidence: 0.65, condition: { condition: 'used_complete', conditionRaw: 'Used (Brickset current value, BrickLink-derived)', completeness: 'used_complete' }, priceKind: 'guide_value', price: value.usedUsd, currency: 'USD', observationDate: obsDate, sampleSize: null }));
198 + }
199 + return out;
200 + }
201 +}
202 +
203 +export default function createConnector(meta: ConnectorMeta) {
204 + return new BricksetConnector(meta);
205 +}
206 +
207 +// keep parsePrice referenced for future currency-specific fields
208 +void parsePrice;
added connectors/api/brickset/meta.json +34 −0
@@ -0,0 +1,34 @@
1 +{
2 + "id": "brickset",
3 + "displayName": "Brickset (LEGO set catalog + current values)",
4 + "sourceId": "brickset",
5 + "sourceName": "Brickset",
6 + "sourceType": "catalog",
7 + "sourceUrl": "https://brickset.com",
8 + "module": "api/brickset",
9 + "enginePriority": ["api", "firecrawl"],
10 + "categories": ["lego_sets", "lego"],
11 + "regions": ["global"],
12 + "languages": ["en"],
13 + "currency": ["USD", "GBP"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://brickset.com/about",
26 + "accessNotes": "Public set pages (brickset.com/sets/<number>-1/...) and theme listings (/sets/theme-<Theme>/page-N, 25 sets per page) fetched over plain HTTPS with the RareIndex user agent; robots.txt disallows /admin, /export, /ajax, /profile, /webservices, /buy, /news, /reviews… — none of which are used (the Brickset API/webservices need a personal key: https://brickset.com/tools/webservices/requestkey). Fields parsed from the set page definition list: pieces, minifigs, RRP (GBP + USD), launch/exit dates, availability, packaging, barcodes (UPC/EAN), theme/subtheme and the 'Current value' New/Used estimates that Brickset derives from BrickLink — stored as guide_value observations (USD, confidence 0.65). 1.5 s politeness delay.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": ["theme-Star-Wars", "theme-Icons", "theme-Creator-Expert", "theme-Ideas", "theme-Technic", "theme-Harry-Potter", "theme-Marvel-Super-Heroes", "theme-Architecture", "theme-Ninjago", "theme-Castle", "theme-Space", "theme-Pirates", "theme-Indiana-Jones", "theme-The-Lord-of-the-Rings", "theme-DC-Comics-Super-Heroes"],
31 + "pagesPerSeed": 2,
32 + "setsPerRun": 120
33 + }
34 +}
added connectors/api/discogs/_smoke.ts +30 −0
@@ -0,0 +1,30 @@
1 +/**
2 + * Live probe: pnpm tsx connectors/api/discogs/_smoke.ts [--save]
3 + * Crawls a few records through the real router and prints normalized output; --save writes fixtures.
4 + */
5 +import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';
6 +import { saveFixture } from '@rareindex/connectors/testing';
7 +import createConnector from './index.js';
8 +
9 +const save = process.argv.includes('--save');
10 +const meta = getConnectorMeta('discogs');
11 +const connector = createConnector(meta);
12 +const router = createRouter({});
13 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 4, seeds: ['Nirvana Nevermind'] } });
14 +let n = 0;
15 +for await (const raw of connector.crawl(ctx)) {
16 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
17 + console.log(`raw ${raw.kind} ${raw.externalId} → ${out.length} records`);
18 + for (const r of out.slice(0, 2)) console.log(' ', JSON.stringify(r).slice(0, 400));
19 + if (save && n < 2) {
20 + saveFixture('discogs', raw.kind === 'catalog_item' ? 'nevermind-versions' : 'nevermind-release-stats', {
21 + raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload },
22 + expect: raw.kind === 'catalog_item' ? { minCount: 5, kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.discogs_release_id', 'attributes.name'] } : { minCount: 1, kinds: ['catalog_item', 'price_observation'] },
23 + note: 'Captured live from api.discogs.com',
24 + });
25 + }
26 + n++;
27 +}
28 +const lk = await connector.lookup('https://www.discogs.com/release/367084-Nirvana-Nevermind', ctx);
29 +console.log('lookup →', lk.length, lk[0] ? JSON.stringify((await connector.normalize({ ...lk[0], externalId: lk[0].externalId ?? null, fetchedAt: lk[0].fetchedAt ?? new Date() }))[0]).slice(0, 300) : '');
30 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/api/discogs/index.test.ts +45 −0
@@ -0,0 +1,45 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector, { splitMasterTitle, variantFromFormat, yearFromReleased } from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('discogs'));
7 +
8 +describe('discogs', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('maps versions to music catalog items with deterministic identifiers', async () => {
12 + const fx = loadFixture('discogs', 'nevermind-versions');
13 + const out = await connector.normalize(fx.raw);
14 + expect(out.length).toBeGreaterThan(5);
15 + for (const r of out) {
16 + if (r.kind !== 'catalog_item') throw new Error('expected catalog_item');
17 + expect(r.attributes.categorySlug).toBe('music');
18 + expect(r.attributes.identifiers.discogs_release_id).toMatch(/^\d+$/);
19 + expect(r.attributes.identifiers.discogs_master_id).toBe('13814');
20 + expect(r.attributes.name).toBe('Nirvana – Nevermind');
21 + expect(r.sourceUrl).toMatch(/discogs\.com\/release\/\d+$/);
22 + }
23 + });
24 +
25 + it('maps marketplace stats to a low observation with sample size', async () => {
26 + const fx = loadFixture('discogs', 'nevermind-release-stats');
27 + const out = await connector.normalize(fx.raw);
28 + const obs = out.find((r) => r.kind === 'price_observation');
29 + if (obs?.kind !== 'price_observation') throw new Error('expected observation');
30 + expect(obs.priceKind).toBe('low');
31 + expect(obs.price).toBeGreaterThan(0);
32 + expect(obs.currency).toBe('USD');
33 + expect(obs.sampleSize).toBeGreaterThan(0);
34 + expect(obs.confidence).toBeLessThanOrEqual(0.7);
35 + });
36 +
37 + it('helpers', () => {
38 + expect(splitMasterTitle('Nirvana - Nevermind')).toEqual({ artist: 'Nirvana', title: 'Nevermind' });
39 + expect(splitMasterTitle('Homogenic')).toEqual({ artist: null, title: 'Homogenic' });
40 + expect(variantFromFormat('Album, Reissue, 180 Gram')).toBe('Reissue, 180 Gram');
41 + expect(variantFromFormat('Album, LP')).toBeNull();
42 + expect(yearFromReleased('1991-09-24')).toBe(1991);
43 + expect(yearFromReleased(null)).toBeNull();
44 + });
45 +});
added connectors/api/discogs/index.ts +234 −0
@@ -0,0 +1,234 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type CurrencyCode, type NormalizedRecord, SUPPORTED_CURRENCIES } from '@rareindex/shared';
4 +
5 +/**
6 + * Discogs — music catalog (pressings of iconic masters) + marketplace "lowest ask" observations.
7 + * Raw records: `master_versions` (one page of versions for a master) and `release_stats`
8 + * (one release + its marketplace stats). Everything comes from the official public API.
9 + */
10 +
11 +const API = 'https://api.discogs.com';
12 +const WEB = 'https://www.discogs.com';
13 +const PARSER_VERSION = '1.0.0';
14 +
15 +const VersionSchema = z.object({
16 + id: z.number(),
17 + label: z.string().nullable().optional(),
18 + country: z.string().nullable().optional(),
19 + title: z.string(),
20 + major_formats: z.array(z.string()).default([]),
21 + format: z.string().nullable().optional(),
22 + catno: z.string().nullable().optional(),
23 + released: z.string().nullable().optional(),
24 + thumb: z.string().nullable().optional(),
25 + stats: z.object({ community: z.object({ in_wantlist: z.number().optional(), in_collection: z.number().optional() }).optional() }).optional(),
26 +});
27 +export type Version = z.infer<typeof VersionSchema>;
28 +
29 +const MasterSchema = z.object({ id: z.number(), title: z.string(), year: z.union([z.number(), z.string()]).nullable().optional(), genre: z.array(z.string()).optional(), style: z.array(z.string()).optional(), cover_image: z.string().nullable().optional() });
30 +
31 +export const VersionsPayloadSchema = z.object({ kind: z.literal('master_versions'), master: MasterSchema, versions: z.array(VersionSchema), page: z.number() });
32 +export const StatsPayloadSchema = z.object({
33 + kind: z.literal('release_stats'),
34 + master: MasterSchema.nullable(),
35 + version: VersionSchema,
36 + stats: z.object({ num_for_sale: z.number().nullable().optional(), lowest_price: z.object({ value: z.number(), currency: z.string() }).nullable().optional(), blocked_from_sale: z.boolean().optional() }),
37 + fetchedAt: z.string(),
38 +});
39 +export type VersionsPayload = z.infer<typeof VersionsPayloadSchema>;
40 +export type StatsPayload = z.infer<typeof StatsPayloadSchema>;
41 +
42 +/** "Nirvana - Nevermind" → { artist: 'Nirvana', title: 'Nevermind' } */
43 +export function splitMasterTitle(title: string): { artist: string | null; title: string } {
44 + const m = title.match(/^(.*?)\s+[-–]\s+(.*)$/);
45 + return m ? { artist: m[1]!.trim(), title: m[2]!.trim() } : { artist: null, title: title.trim() };
46 +}
47 +
48 +/** "Album, Reissue, 180 Gram" → variant string without the plain "Album"/"LP" noise; null when nothing notable. */
49 +export function variantFromFormat(format: string | null | undefined): string | null {
50 + if (!format) return null;
51 + const parts = format.split(',').map((s) => s.trim()).filter((s) => s && !/^(album|lp|12"|7"|10"|single|ep|stereo|mono)$/i.test(s));
52 + return parts.length ? parts.join(', ') : null;
53 +}
54 +
55 +export function yearFromReleased(released: string | null | undefined): number | null {
56 + const m = released?.match(/\b(19|20)\d{2}\b/);
57 + return m ? Number(m[0]) : null;
58 +}
59 +
60 +export class DiscogsConnector extends BaseConnector {
61 + readonly version = '1.0.0';
62 + readonly parserVersion = PARSER_VERSION;
63 + protected override minIntervalMs = 2600; // 25 req/min unauthenticated
64 + override readonly urlPatterns = [/^https?:\/\/(www\.)?discogs\.com\/(?:[a-z]{2}\/)?release\/(\d+)/i];
65 +
66 + private headers() {
67 + return { accept: 'application/vnd.discogs.v2.discogs+json' };
68 + }
69 +
70 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
71 + const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
72 + const formats = ((this.meta.config.formats as string[] | undefined) ?? ['Vinyl']).map((f) => f.toLowerCase());
73 + const perMaster = Number(this.meta.config.versionsPerMaster ?? 100);
74 + const statsPer = Number(this.meta.config.statsPerMaster ?? 6);
75 + const startIdx = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
76 + let count = 0;
77 + for (let i = startIdx; i < seeds.length; i++) {
78 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
79 + const q = seeds[i]!;
80 + await this.throttle();
81 + const search = await ctx.fetch(`${API}/database/search?q=${encodeURIComponent(q)}&type=master&per_page=3`, { engines: ['api'], headers: this.headers() });
82 + const results = (search.json as { results?: Array<{ id: number; title: string; year?: string; genre?: string[]; style?: string[]; cover_image?: string }> } | null)?.results ?? [];
83 + if (!search.success || results.length === 0) {
84 + ctx.anomaly('search_failed', `${q}: ${search.error ?? 'no master'}`);
85 + continue;
86 + }
87 + const master = results[0]!;
88 + await this.throttle();
89 + const vres = await ctx.fetch(`${API}/masters/${master.id}/versions?per_page=${Math.min(perMaster, 100)}&sort=released&sort_order=asc`, { engines: ['api'], headers: this.headers() });
90 + const all = (vres.json as { versions?: unknown[] } | null)?.versions ?? [];
91 + const versions = all.map((v) => VersionSchema.safeParse(v)).filter((r) => r.success).map((r) => r.data).filter((v) => v.major_formats.some((f) => formats.includes(f.toLowerCase())));
92 + if (!vres.success || versions.length === 0) {
93 + ctx.anomaly('versions_failed', `${master.title}: ${vres.error ?? 'no versions'}`);
94 + continue;
95 + }
96 + const m = { id: master.id, title: master.title, year: master.year ?? null, genre: master.genre, style: master.style, cover_image: master.cover_image ?? null };
97 + const payload: VersionsPayload = { kind: 'master_versions', master: m, versions, page: 1 };
98 + count++;
99 + yield { url: `${WEB}/master/${master.id}`, externalId: `master:${master.id}:p1`, kind: 'catalog_item', engine: 'api', httpStatus: vres.httpStatus, payload, fetchedAt: vres.fetchedAt };
100 + // marketplace stats for the most collected pressings
101 + const top = [...versions].sort((a, b) => (b.stats?.community?.in_collection ?? 0) - (a.stats?.community?.in_collection ?? 0)).slice(0, statsPer);
102 + for (const v of top) {
103 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
104 + await this.throttle();
105 + const sres = await ctx.fetch(`${API}/marketplace/stats/${v.id}`, { engines: ['api'], headers: this.headers(), failOnHttpError: false });
106 + const stats = sres.json as StatsPayload['stats'] | null;
107 + if (!sres.success || !stats) {
108 + ctx.anomaly('stats_failed', `${v.id}: ${sres.error ?? sres.httpStatus}`);
109 + continue;
110 + }
111 + count++;
112 + const sp: StatsPayload = { kind: 'release_stats', master: m, version: v, stats, fetchedAt: sres.fetchedAt.toISOString() };
113 + yield { url: `${WEB}/release/${v.id}`, externalId: `release:${v.id}:stats`, kind: 'price_observation', engine: 'api', httpStatus: sres.httpStatus, payload: sp, fetchedAt: sres.fetchedAt };
114 + }
115 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
116 + }
117 + }
118 +
119 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
120 + const id = url.match(this.urlPatterns[0]!)?.[2];
121 + if (!id) return [];
122 + await this.throttle();
123 + const rres = await ctx.fetch(`${API}/releases/${id}`, { engines: ['api'], headers: this.headers() });
124 + const rel = rres.json as { id: number; title: string; artists?: Array<{ name: string }>; labels?: Array<{ name: string; catno?: string }>; country?: string; year?: number; released?: string; formats?: Array<{ name: string; descriptions?: string[] }>; master_id?: number; thumb?: string; images?: Array<{ uri: string }>; community?: { have?: number; want?: number } } | null;
125 + if (!rres.success || !rel) return [];
126 + await this.throttle();
127 + const sres = await ctx.fetch(`${API}/marketplace/stats/${id}`, { engines: ['api'], headers: this.headers(), failOnHttpError: false });
128 + const artist = rel.artists?.map((a) => a.name.replace(/\s\(\d+\)$/, '')).join(', ') ?? '';
129 + const version: Version = {
130 + id: rel.id,
131 + label: rel.labels?.[0]?.name ?? null,
132 + country: rel.country ?? null,
133 + title: rel.title,
134 + major_formats: rel.formats?.map((f) => f.name) ?? [],
135 + format: rel.formats?.[0]?.descriptions?.join(', ') ?? null,
136 + catno: rel.labels?.[0]?.catno ?? null,
137 + released: rel.released ?? (rel.year ? String(rel.year) : null),
138 + thumb: rel.images?.[0]?.uri ?? rel.thumb ?? null,
139 + stats: { community: { in_collection: rel.community?.have, in_wantlist: rel.community?.want } },
140 + };
141 + const master = { id: rel.master_id ?? rel.id, title: `${artist} - ${rel.title}`, year: rel.year ?? null, cover_image: version.thumb };
142 + const payload: StatsPayload = { kind: 'release_stats', master, version, stats: (sres.json as StatsPayload['stats'] | null) ?? {}, fetchedAt: rres.fetchedAt.toISOString() };
143 + return [{ url: `${WEB}/release/${id}`, externalId: `release:${id}:stats`, kind: 'price_observation', engine: 'api', httpStatus: rres.httpStatus, payload, fetchedAt: rres.fetchedAt }];
144 + }
145 +
146 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
147 + const p = raw.payload as { kind?: string };
148 + if (p?.kind === 'master_versions') {
149 + const v = VersionsPayloadSchema.parse(raw.payload);
150 + return v.versions.map((ver) => this.catalogFor(v.master, ver, raw.fetchedAt));
151 + }
152 + if (p?.kind === 'release_stats') {
153 + const s = StatsPayloadSchema.parse(raw.payload);
154 + const out: NormalizedRecord[] = [this.catalogFor(s.master ?? { id: s.version.id, title: s.version.title }, s.version, raw.fetchedAt)];
155 + const low = s.stats.lowest_price;
156 + const cur = low?.currency?.toUpperCase();
157 + if (low && low.value > 0 && cur && (SUPPORTED_CURRENCIES as readonly string[]).includes(cur)) {
158 + const cat = out[0] as Extract<NormalizedRecord, { kind: 'catalog_item' }>;
159 + out.push(
160 + NormalizedPriceObservationSchema.parse({
161 + kind: 'price_observation',
162 + connectorId: this.meta.id,
163 + sourceId: this.meta.sourceId,
164 + sourceUrl: cat.sourceUrl,
165 + externalId: `release:${s.version.id}:low`,
166 + rawTitle: cat.rawTitle,
167 + imageUrls: cat.imageUrls,
168 + attributes: cat.attributes,
169 + condition: { condition: null, conditionRaw: 'marketplace lowest ask (any condition)', completeness: null },
170 + observedAt: raw.fetchedAt,
171 + confidence: 0.7,
172 + parserVersion: PARSER_VERSION,
173 + priceKind: 'low',
174 + price: low.value,
175 + currency: cur as CurrencyCode,
176 + observationDate: new Date(s.fetchedAt),
177 + sampleSize: s.stats.num_for_sale ?? null,
178 + }),
179 + );
180 + }
181 + return out;
182 + }
183 + throw new Error(`discogs: unknown payload kind ${String(p?.kind)}`);
184 + }
185 +
186 + private catalogFor(master: { id: number; title: string; year?: number | string | null; genre?: string[]; style?: string[]; cover_image?: string | null }, v: Version, fetchedAt: Date) {
187 + const { artist, title } = splitMasterTitle(master.title);
188 + const year = yearFromReleased(v.released) ?? (master.year ? Number(master.year) || null : null);
189 + const variant = variantFromFormat(v.format);
190 + const fmt = v.major_formats[0] ?? null;
191 + const name = artist ? `${artist} – ${title}` : title;
192 + const rawTitle = `${name} · ${[v.label, v.catno].filter(Boolean).join(' ')}${fmt ? ` ${fmt}` : ''}${variant ? ` ${variant}` : ''}${year ? ` (${year}${v.country ? `, ${v.country}` : ''})` : v.country ? ` (${v.country})` : ''}`;
193 + const attributes = AssetAttributesSchema.parse({
194 + categorySlug: 'music',
195 + brand: v.label ?? null,
196 + series: fmt,
197 + set: v.label ?? null,
198 + name,
199 + number: v.catno ?? null,
200 + year,
201 + variant,
202 + country: v.country ?? null,
203 + identifiers: { discogs_release_id: String(v.id), discogs_master_id: String(master.id), ...(v.catno ? { catalog_number: v.catno } : {}) },
204 + metadata: {
205 + artist,
206 + album: title,
207 + format: v.format ?? null,
208 + major_formats: v.major_formats,
209 + genres: master.genre ?? [],
210 + styles: master.style ?? [],
211 + community_have: v.stats?.community?.in_collection ?? null,
212 + community_want: v.stats?.community?.in_wantlist ?? null,
213 + },
214 + });
215 + return NormalizedCatalogItemSchema.parse({
216 + kind: 'catalog_item',
217 + connectorId: this.meta.id,
218 + sourceId: this.meta.sourceId,
219 + sourceUrl: `${WEB}/release/${v.id}`,
220 + externalId: `release:${v.id}`,
221 + rawTitle,
222 + imageUrls: [v.thumb, master.cover_image].filter((x): x is string => Boolean(x)),
223 + attributes,
224 + observedAt: fetchedAt,
225 + confidence: 0.95,
226 + parserVersion: PARSER_VERSION,
227 + releaseDate: v.released && /^\d{4}-\d{2}-\d{2}$/.test(v.released) ? new Date(`${v.released}T00:00:00Z`) : null,
228 + });
229 + }
230 +}
231 +
232 +export default function createConnector(meta: ConnectorMeta) {
233 + return new DiscogsConnector(meta);
234 +}
added connectors/api/discogs/meta.json +37 −0
@@ -0,0 +1,37 @@
1 +{
2 + "id": "discogs",
3 + "displayName": "Discogs (releases + marketplace lows)",
4 + "sourceId": "discogs",
5 + "sourceName": "Discogs",
6 + "sourceType": "collector_database",
7 + "sourceUrl": "https://www.discogs.com",
8 + "module": "api/discogs",
9 + "enginePriority": ["api"],
10 + "categories": ["music"],
11 + "regions": ["global"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://support.discogs.com/hc/en-us/articles/360009334593-API-Terms-of-Use",
26 + "accessNotes": "Official public Discogs API (api.discogs.com) used unauthenticated with the RareIndex user agent: database/search (type=master), masters/{id}/versions (pressings with label, catno, country, year, format and community have/want counts) and marketplace/stats/{release_id} (lowest asking price + number for sale). Unauthenticated limit is 25 requests/minute → 2.6 s throttle; price_suggestions requires a personal token and is not used. Marketplace stats have no timestamp: observationDate = fetch date (confidence 0.7). Catalog data is CC0; images are hot-linked thumbnails and attributed to Discogs. Sold-price history is not exposed publicly.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + "Nirvana Nevermind", "The Beatles Abbey Road", "Pink Floyd The Dark Side Of The Moon", "Led Zeppelin IV", "Miles Davis Kind Of Blue", "Michael Jackson Thriller", "Radiohead OK Computer", "Fleetwood Mac Rumours", "Daft Punk Random Access Memories", "Kendrick Lamar To Pimp A Butterfly", "The Velvet Underground & Nico", "Joy Division Unknown Pleasures", "Prince Purple Rain", "David Bowie The Rise And Fall Of Ziggy Stardust", "Amy Winehouse Back To Black", "The Beatles Sgt. Pepper's Lonely Hearts Club Band", "Bob Dylan Blonde On Blonde", "John Coltrane A Love Supreme", "The Rolling Stones Exile On Main St.", "Wu-Tang Clan Enter The Wu-Tang (36 Chambers)", "Taylor Swift 1989", "Tyler, The Creator Igor", "Frank Ocean Blonde", "Arctic Monkeys AM", "Björk Homogenic", "Kraftwerk Autobahn", "Sex Pistols Never Mind The Bollocks", "Bruce Springsteen Born To Run", "Nas Illmatic", "Massive Attack Mezzanine"
32 + ],
33 + "formats": ["Vinyl"],
34 + "versionsPerMaster": 100,
35 + "statsPerMaster": 6
36 + }
37 +}
added connectors/api/lorcast/_smoke.ts +3 −0
@@ -0,0 +1,3 @@
1 +// Live smoke: pnpm tsx connectors/api/lorcast/_smoke.ts
2 +process.argv.splice(2, process.argv.length, "lorcast");
3 +await import("../_lib/smoke.js");
added connectors/api/lorcast/index.test.ts +29 −0
@@ -0,0 +1,29 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('lorcast'));
7 +
8 +describe('lorcast', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('emits normal + foil variants with TCGplayer market prices', async () => {
12 + const [name] = listFixtures('lorcast');
13 + const out = await connector.normalize(loadFixture('lorcast', name!).raw);
14 + const cats = out.filter((r) => r.kind === 'catalog_item');
15 + expect(cats.length).toBe(2);
16 + for (const c of cats) {
17 + if (c.kind !== 'catalog_item') continue;
18 + expect(c.attributes.categorySlug).toBe('disney_lorcana');
19 + expect(c.attributes.setCode).toBe('1');
20 + expect(c.attributes.set).toBe('The First Chapter');
21 + expect(c.attributes.number).toMatch(/^\d+$/);
22 + expect(c.attributes.identifiers.lorcast_id).toMatch(/^crd_/);
23 + }
24 + expect(cats.map((c) => c.kind === 'catalog_item' && c.attributes.variant)).toEqual([null, 'Foil']);
25 + const obs = out.filter((r) => r.kind === 'price_observation');
26 + expect(obs.length).toBeGreaterThanOrEqual(1);
27 + for (const o of obs) if (o.kind === 'price_observation') expect(o.currency).toBe('USD');
28 + });
29 +});
added connectors/api/lorcast/index.ts +144 −0
@@ -0,0 +1,144 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';
5 +
6 +/**
7 + * Lorcast connector — Disney Lorcana catalog (all sets, promos) with TCGplayer market prices
8 + * (usd / usd_foil). Open API, no key: https://lorcast.com/docs/api
9 + */
10 +const API = 'https://api.lorcast.com/v0';
11 +const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };
12 +const PARSER_VERSION = '1.0.0';
13 +
14 +const SetSchema = z.object({ id: z.string(), name: z.string(), code: z.string(), released_at: z.string().nullable().optional(), prereleased_at: z.string().nullable().optional() });
15 +const CardSchema = z
16 + .object({
17 + id: z.string(),
18 + name: z.string(),
19 + version: z.string().nullable().optional(),
20 + layout: z.string().optional(),
21 + released_at: z.string().nullable().optional(),
22 + image_uris: z.record(z.string(), z.record(z.string(), z.string())).optional(),
23 + ink: z.string().nullable().optional(),
24 + type: z.array(z.string()).optional(),
25 + rarity: z.string().optional(),
26 + illustrators: z.array(z.string()).optional(),
27 + collector_number: z.string(),
28 + lang: z.string().optional(),
29 + set: z.object({ id: z.string(), code: z.string(), name: z.string() }),
30 + tcgplayer_id: z.number().nullable().optional(),
31 + prices: z.object({ usd: z.number().nullable().optional(), usd_foil: z.number().nullable().optional() }).partial().nullable().optional(),
32 + })
33 + .loose();
34 +type LorCard = z.infer<typeof CardSchema>;
35 +
36 +export function trimCard(raw: Record<string, unknown>): LorCard {
37 + const keep = ['id', 'name', 'version', 'layout', 'released_at', 'image_uris', 'ink', 'type', 'rarity', 'illustrators', 'collector_number', 'lang', 'set', 'tcgplayer_id', 'prices'];
38 + const out: Record<string, unknown> = {};
39 + for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];
40 + return CardSchema.parse(out);
41 +}
42 +
43 +const RawPayloadSchema = z.object({ card: CardSchema, set: SetSchema });
44 +
45 +export class LorcastConnector extends BaseConnector {
46 + readonly version = '1.0.0';
47 + readonly parserVersion = PARSER_VERSION;
48 + protected override minIntervalMs = 250;
49 +
50 + private async get(ctx: CrawlContext, url: string) {
51 + await this.throttle();
52 + return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS }), (r) => r.success && r.json !== null, 4, 1500);
53 + }
54 +
55 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
56 + const setsRes = await this.get(ctx, `${API}/sets`);
57 + const setsJson = (setsRes.json as { results?: unknown[] } | null)?.results;
58 + if (!setsRes.success || !Array.isArray(setsJson)) throw new Error(`lorcast: sets unavailable (${setsRes.error ?? setsRes.httpStatus})`);
59 + const sets = z.array(SetSchema.loose()).parse(setsJson).filter((s) => !ctx.options.seeds?.length || ctx.options.seeds.includes(s.code));
60 + let setIndex = Number(ctx.options.cursor?.setIndex ?? 0);
61 + let count = 0;
62 + for (; setIndex < sets.length; setIndex++) {
63 + const set = sets[setIndex]!;
64 + if (ctx.signal?.aborted) return;
65 + const seenIds = new Set<string>();
66 + for (let page = 1; page < 50; page++) {
67 + const res = await this.get(ctx, `${API}/cards/search?q=${encodeURIComponent(`set:${set.code}`)}&page=${page}`);
68 + const results = (res.json as { results?: unknown[] } | null)?.results;
69 + if (!res.success || !Array.isArray(results) || results.length === 0) break;
70 + let fresh = 0;
71 + for (const raw of results) {
72 + let card: LorCard;
73 + try {
74 + card = trimCard(raw as Record<string, unknown>);
75 + } catch (err) {
76 + ctx.anomaly('parse_failure_card', `${set.code}: ${err instanceof Error ? err.message : String(err)}`);
77 + continue;
78 + }
79 + if (seenIds.has(card.id)) continue;
80 + seenIds.add(card.id);
81 + fresh++;
82 + if (this.reached(ctx, count)) return;
83 + count++;
84 + yield { url: `https://lorcast.com/cards/${card.id}`, externalId: card.id, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, set }, fetchedAt: res.fetchedAt };
85 + }
86 + if (fresh === 0) break; // API returned the same page again → no pagination
87 + }
88 + await ctx.setCursor({ setIndex: setIndex + 1, updatedAt: new Date().toISOString() });
89 + }
90 + await ctx.setCursor({ setIndex: 0, updatedAt: new Date().toISOString() });
91 + }
92 +
93 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
94 + const { card, set } = RawPayloadSchema.parse(raw.payload);
95 + const year = set.released_at ? Number(set.released_at.slice(0, 4)) || null : card.released_at ? Number(card.released_at.slice(0, 4)) || null : null;
96 + const identifiers: Record<string, string> = { lorcast_id: card.id };
97 + if (card.tcgplayer_id) identifiers.tcgplayer_id = String(card.tcgplayer_id);
98 + const digital = card.image_uris?.digital;
99 + const images = digital ? [digital.large ?? digital.normal ?? digital.small].filter((x): x is string => Boolean(x)) : [];
100 + const rarity = card.rarity ? card.rarity.replace(/_/g, ' ') : null;
101 + const name = card.version ? `${card.name} - ${card.version}` : card.name;
102 + const build = (variant: string | null) =>
103 + attrs({
104 + categorySlug: 'disney_lorcana',
105 + franchise: 'Disney Lorcana',
106 + brand: 'Ravensburger',
107 + set: set.name,
108 + setCode: set.code,
109 + name,
110 + number: card.collector_number,
111 + year,
112 + variant,
113 + language: card.lang === 'en' || !card.lang ? 'English' : card.lang,
114 + rarity,
115 + identifiers,
116 + metadata: { ink: card.ink, type: card.type, illustrators: card.illustrators, layout: card.layout, lorcast_set: set.id },
117 + });
118 + const out: NormalizedRecord[] = [];
119 + const observedAt = raw.fetchedAt;
120 + const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));
121 + const isEnchanted = /enchanted/i.test(card.rarity ?? '');
122 + const variantsToEmit: Array<[string | null, number | null]> = [
123 + [isEnchanted ? 'Enchanted' : null, num(card.prices?.usd)],
124 + [isEnchanted ? 'Enchanted' : 'Foil', num(card.prices?.usd_foil)],
125 + ];
126 + const seen = new Set<string>();
127 + for (const [variant, price] of variantsToEmit) {
128 + const key = variant ?? '';
129 + const a = build(variant);
130 + const rawTitle = makeTitle({ name, set: set.name, number: card.collector_number, year, variant });
131 + if (!seen.has(key)) {
132 + seen.add(key);
133 + out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.id}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: set.released_at ? new Date(set.released_at) : null }));
134 + }
135 + if (price) {
136 + // Lorcast relays TCGplayer market prices without a timestamp → dated by fetch day, moderate confidence.
137 + out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${card.id}:${variant ?? 'normal'}:market`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency: 'USD', observationDate: obsDate, sampleSize: null }));
138 + }
139 + }
140 + return out;
141 + }
142 +}
143 +
144 +export default (meta: ConnectorMeta) => new LorcastConnector(meta);
added connectors/api/lorcast/meta.json +30 −0
@@ -0,0 +1,30 @@
1 +{
2 + "id": "lorcast",
3 + "displayName": "Lorcast (Disney Lorcana)",
4 + "sourceId": "lorcast",
5 + "sourceName": "Lorcast",
6 + "sourceType": "catalog",
7 + "sourceUrl": "https://lorcast.com",
8 + "module": "api/lorcast",
9 + "enginePriority": ["api"],
10 + "categories": ["disney_lorcana"],
11 + "regions": ["US", "EU"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.75,
24 + "attributionRequired": true,
25 + "termsUrl": "https://lorcast.com/docs/api",
26 + "accessNotes": "Open Lorcast REST API (v0, no key): /sets then /cards/search?q=set:<code>&page=N. Cards carry TCGplayer ids and USD market prices (normal / foil) without a timestamp, so observations are dated by the fetch day with confidence 0.7. Enchanted cards are emitted as a single 'Enchanted' variant. ~4 req/s self-throttled.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {}
30 +}
added connectors/api/novelship/_smoke.ts +24 −0
@@ -0,0 +1,24 @@
1 +import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';
2 +import { saveFixture } from '@rareindex/connectors/testing';
3 +import createConnector from './index.js';
4 +
5 +const save = process.argv.includes('--save');
6 +const meta = getConnectorMeta('novelship');
7 +const connector = createConnector(meta);
8 +const ctx = createCrawlContext({ router: createRouter({}), meta, options: { mode: 'probe', limit: 2, seeds: ['jordan'] } });
9 +let i = 0;
10 +for await (const raw of connector.crawl(ctx)) {
11 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
12 + const n = (raw.payload as { products: unknown[] }).products.length;
13 + console.log(`raw ${raw.externalId} products=${n} → ${out.length} records`);
14 + for (const r of out.slice(0, 3)) console.log(' ', JSON.stringify(r).slice(0, 360));
15 + if (save && i === 0) saveFixture('novelship', 'jordan-browse-page1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 30, kinds: ['catalog_item', 'price_observation', 'listing'], requiredFields: ['attributes.identifiers.style_code'] }, note: 'Captured live from novelship.com/sneakers/jordan' });
16 + i++;
17 +}
18 +const lk = await connector.lookup('https://novelship.com/air-jordan-1-low-midnight-navy-university-blue-553558-404', ctx);
19 +if (lk[0]) {
20 + const out = await connector.normalize({ ...lk[0], externalId: lk[0].externalId ?? null, fetchedAt: lk[0].fetchedAt ?? new Date() });
21 + console.log('lookup →', out.length, JSON.stringify(out).slice(0, 500));
22 + if (save) saveFixture('novelship', 'aj1-low-midnight-navy-product', { raw: { url: lk[0].url, externalId: lk[0].externalId ?? null, kind: lk[0].kind, engine: lk[0].engine, fetchedAt: lk[0].fetchedAt ?? new Date(), payload: lk[0].payload }, expect: { minCount: 1, first: { 'attributes.identifiers.style_code': '553558-404', 'attributes.categorySlug': 'nike_jordan' } }, note: 'Captured live from a novelship.com product page' });
23 +}
24 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/api/novelship/index.test.ts +48 −0
@@ -0,0 +1,48 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector, { brandCategory, parseBrowsePage } from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('novelship'));
7 +
8 +describe('novelship', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('maps browse products to catalog + last sale + lowest ask', async () => {
12 + const fx = loadFixture('novelship', 'jordan-browse-page1');
13 + const out = await connector.normalize(fx.raw);
14 + const cats = out.filter((r) => r.kind === 'catalog_item');
15 + expect(cats.length).toBeGreaterThanOrEqual(30);
16 + for (const c of cats) {
17 + if (c.kind !== 'catalog_item') throw new Error();
18 + expect(c.attributes.categorySlug).toBe('nike_jordan');
19 + expect(c.attributes.identifiers.style_code).toMatch(/^[A-Z0-9][A-Z0-9\/ -]{3,}$/i);
20 + expect(c.sourceUrl).toMatch(/^https:\/\/novelship\.com\/[a-z0-9-]+$/);
21 + }
22 + const sales = out.filter((r) => r.kind === 'price_observation' && r.priceKind === 'last_sale_reported');
23 + const asks = out.filter((r) => r.kind === 'listing' && r.listingType === 'ask');
24 + expect(sales.length).toBeGreaterThan(10);
25 + expect(asks.length).toBeGreaterThan(10);
26 + for (const s of sales) if (s.kind === 'price_observation') expect(s.currency).toBe('USD');
27 + });
28 +
29 + it('parses a product page and keeps only that product on lookup', async () => {
30 + const fx = loadFixture('novelship', 'aj1-low-midnight-navy-product');
31 + const out = await connector.normalize(fx.raw);
32 + const cat = out.find((r) => r.kind === 'catalog_item');
33 + if (cat?.kind !== 'catalog_item') throw new Error();
34 + expect(cat.attributes.identifiers.style_code).toBe('553558-404');
35 + expect(cat.attributes.color).toContain('Midnight Navy');
36 + expect(cat.attributes.originalMsrp).toBe(120);
37 + });
38 +
39 + it('helpers', () => {
40 + expect(brandCategory('Jordan', 'Air Jordan 1')).toBe('nike_jordan');
41 + expect(brandCategory('adidas', 'Yeezy Boost 350')).toBe('adidas_yeezy');
42 + expect(brandCategory('New Balance', '550')).toBe('new_balance_asics_other');
43 + const html = `x {"cost_retail":100,"drop_date":"2024-01-02","id":7,"image":"https://i/x.jpg","last_sale_price":150.5,"lowest_listing_price":"120.000000","main_brand":"Nike","name":"Nike Dunk Low Panda DD1391-100","name_slug":"nike-dunk-low-panda-dd1391-100","sales_count_180":12,"sku":"DD1391-100","sub_brand":""} y`;
44 + const p = parseBrowsePage(html, 'u', 'nike', 1);
45 + expect(p.products).toHaveLength(1);
46 + expect(p.products[0]).toMatchObject({ id: 7, sku: 'DD1391-100', lastSalePrice: 150.5, lowestListingPrice: 120, costRetail: 100, salesCount180: 12, mainBrand: 'Nike' });
47 + });
48 +});
added connectors/api/novelship/index.ts +190 −0
@@ -0,0 +1,190 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +/**
6 + * Novelship — sneaker catalog with last sale / lowest ask read from the server-rendered
7 + * (React Server Components) payload of public browse pages. One raw record per browse page.
8 + */
9 +
10 +const BASE = 'https://novelship.com';
11 +const PARSER_VERSION = '1.0.0';
12 +
13 +export const ProductSchema = z.object({
14 + id: z.number(),
15 + name: z.string(),
16 + nameSlug: z.string(),
17 + sku: z.string().nullable(),
18 + mainBrand: z.string().nullable(),
19 + subBrand: z.string().nullable(),
20 + colorway: z.string().nullable(),
21 + category: z.string().nullable(),
22 + gender: z.string().nullable(),
23 + dropDate: z.string().nullable(),
24 + costRetail: z.number().nullable(),
25 + lastSalePrice: z.number().nullable(),
26 + lowestListingPrice: z.number().nullable(),
27 + salesCount180: z.number().nullable(),
28 + image: z.string().nullable(),
29 +});
30 +export type Product = z.infer<typeof ProductSchema>;
31 +export const PagePayloadSchema = z.object({ kind: z.literal('browse_page'), url: z.string(), seed: z.string(), page: z.number(), products: z.array(ProductSchema) });
32 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
33 +
34 +const VALUE_RE = '("(?:[^"\\\\]|\\\\.)*"|-?[0-9.]+|null|true|false)';
35 +function decode(v: string | undefined): string | null {
36 + if (v === undefined || v === 'null') return null;
37 + if (v.startsWith('"')) return JSON.parse(v.replace(/\\u0026/g, '&')) as string;
38 + return v;
39 +}
40 +/** Last occurrence of `"key":value` in `seg` (used for keys serialised before name_slug). */
41 +function pickLast(seg: string, key: string): string | null {
42 + const re = new RegExp(`"${key}":${VALUE_RE}`, 'g');
43 + let last: string | undefined;
44 + for (const m of seg.matchAll(re)) last = m[1];
45 + return decode(last);
46 +}
47 +/** First occurrence of `"key":value` in `seg` (used for keys serialised after name_slug). */
48 +function pickFirst(seg: string, key: string): string | null {
49 + const m = seg.match(new RegExp(`"${key}":${VALUE_RE}`));
50 + return decode(m?.[1]);
51 +}
52 +function numOf(v: string | null): number | null {
53 + if (v === null) return null;
54 + const n = Number(v);
55 + return Number.isFinite(n) && n > 0 ? n : null;
56 +}
57 +
58 +/**
59 + * Extract product objects from the RSC payload embedded in the HTML. Browse pages serialise
60 + * product keys alphabetically (keys < "name_slug" precede it, keys > follow it), product pages
61 + * keep insertion order; both are covered by looking on the matching side first.
62 + */
63 +export function parseBrowsePage(htmlText: string, url: string, seed: string, page: number): PagePayload {
64 + const s = htmlText.replace(/\\"/g, '"').replace(/\\\\/g, '\\');
65 + const products = new Map<string, Product>();
66 + const hits = [...s.matchAll(/"name_slug":"([a-z0-9-]+)"/g)];
67 + for (let i = 0; i < hits.length; i++) {
68 + const h = hits[i]!;
69 + const slug = h[1]!;
70 + const at = h.index!;
71 + const prevEnd = i > 0 ? hits[i - 1]!.index! + hits[i - 1]![0].length : Math.max(0, at - 12000);
72 + const nextStart = i + 1 < hits.length ? hits[i + 1]!.index! : Math.min(s.length, at + 12000);
73 + const before = s.slice(Math.max(prevEnd, at - 12000), at);
74 + const after = s.slice(at + h[0].length, Math.min(nextStart, at + 12000));
75 + const get = (key: string) => (key < 'name_slug' ? (pickLast(before, key) ?? pickFirst(after, key)) : (pickFirst(after, key) ?? pickLast(before, key)));
76 + const id = Number(get('id'));
77 + const name = get('name');
78 + if (!Number.isFinite(id) || !name || products.has(slug)) continue;
79 + const sales = get('sales_count_180');
80 + const p: Product = {
81 + id,
82 + name,
83 + nameSlug: slug,
84 + sku: get('sku'),
85 + mainBrand: get('main_brand'),
86 + subBrand: get('sub_brand') || null,
87 + colorway: get('colorway'),
88 + category: get('category'),
89 + gender: get('gender'),
90 + dropDate: get('drop_date'),
91 + costRetail: numOf(get('cost_retail')),
92 + lastSalePrice: numOf(get('last_sale_price')),
93 + lowestListingPrice: numOf(get('lowest_listing_price')),
94 + salesCount180: sales === null ? null : Number(sales),
95 + image: get('image'),
96 + };
97 + if (p.sku || p.lastSalePrice || p.lowestListingPrice) products.set(slug, p);
98 + }
99 + return { kind: 'browse_page', url, seed, page, products: [...products.values()] };
100 +}
101 +
102 +export function brandCategory(brand: string | null, name: string): string {
103 + const b = `${brand ?? ''} ${name}`.toLowerCase();
104 + if (/jordan|nike/.test(b)) return 'nike_jordan';
105 + if (/adidas|yeezy/.test(b)) return 'adidas_yeezy';
106 + return 'new_balance_asics_other';
107 +}
108 +
109 +export class NovelshipConnector extends BaseConnector {
110 + readonly version = '1.0.0';
111 + readonly parserVersion = PARSER_VERSION;
112 + protected override minIntervalMs = 1500;
113 + override readonly urlPatterns = [/^https?:\/\/(www\.)?novelship\.com\/([a-z0-9-]+)$/i];
114 +
115 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
116 + const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
117 + const pages = Number(this.meta.config.pagesPerSeed ?? 3);
118 + let count = 0;
119 + for (const seed of seeds) {
120 + for (let page = 1; page <= pages; page++) {
121 + if (ctx.signal?.aborted || this.reached(ctx, count)) return;
122 + const url = `${BASE}/sneakers/${seed}${page > 1 ? `?page=${page}` : ''}`;
123 + await this.throttle();
124 + const res = await ctx.fetch(url, {
125 + responseType: 'text',
126 + expect: ['title', 'price', 'identifiers'],
127 + parse: (r) => {
128 + const p = r.html ? parseBrowsePage(r.html, url, seed, page) : null;
129 + const first = p?.products[0];
130 + return first ? { title: first.name, price: first.lastSalePrice ?? first.lowestListingPrice, identifiers: first.sku ? { sku: first.sku } : null } : null;
131 + },
132 + });
133 + const payload = res.success && res.html ? parseBrowsePage(res.html, url, seed, page) : null;
134 + if (!payload || payload.products.length === 0) {
135 + ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
136 + break;
137 + }
138 + count++;
139 + yield { url, externalId: `browse:${seed}:${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
140 + }
141 + }
142 + }
143 +
144 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
145 + const slug = url.match(this.urlPatterns[0]!)?.[2];
146 + if (!slug || ['sneakers', 'apparel', 'collectibles', 'browse'].includes(slug)) return [];
147 + await this.throttle();
148 + const res = await ctx.fetch(`${BASE}/${slug}`, { responseType: 'text', minQuality: 0.2 });
149 + if (!res.success || !res.html) return [];
150 + const payload = parseBrowsePage(res.html, `${BASE}/${slug}`, `product:${slug}`, 1);
151 + payload.products = payload.products.filter((p) => p.nameSlug === slug);
152 + if (!payload.products.length) return [];
153 + return [{ url: `${BASE}/${slug}`, externalId: `product:${slug}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
154 + }
155 +
156 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
157 + const p = PagePayloadSchema.parse(raw.payload);
158 + const out: NormalizedRecord[] = [];
159 + for (const pr of p.products) {
160 + const year = pr.dropDate?.match(/^(\d{4})/)?.[1];
161 + const attributes = AssetAttributesSchema.parse({
162 + categorySlug: brandCategory(pr.mainBrand, pr.name),
163 + brand: pr.mainBrand,
164 + series: pr.subBrand,
165 + name: pr.name.replace(/\s+[A-Z0-9]{2,}-?[A-Z0-9]{2,}$/i, (m0) => (pr.sku && m0.trim() === pr.sku ? '' : m0)).trim(),
166 + color: pr.colorway,
167 + year: year ? Number(year) : null,
168 + originalMsrp: pr.costRetail,
169 + originalMsrpCurrency: pr.costRetail ? 'USD' : null,
170 + identifiers: { ...(pr.sku ? { style_code: pr.sku } : {}), novelship_id: String(pr.id) },
171 + metadata: { gender: pr.gender, category: pr.category, drop_date: pr.dropDate, sales_count_180: pr.salesCount180 },
172 + });
173 + const sourceUrl = `${BASE}/${pr.nameSlug}`;
174 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle: pr.name, imageUrls: pr.image ? [pr.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
175 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${pr.id}`, confidence: 0.9, releaseDate: pr.dropDate && /^\d{4}-\d{2}-\d{2}/.test(pr.dropDate) ? new Date(`${pr.dropDate.slice(0, 10)}T00:00:00Z`) : null }));
176 + const cond = { condition: 'new', conditionRaw: 'Brand new (marketplace standard)', completeness: 'with_box' };
177 + if (pr.lastSalePrice) {
178 + out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `product:${pr.id}:last_sale`, confidence: 0.7, condition: cond, priceKind: 'last_sale_reported', price: pr.lastSalePrice, currency: 'USD', observationDate: raw.fetchedAt, sampleSize: pr.salesCount180 }));
179 + }
180 + if (pr.lowestListingPrice) {
181 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `product:${pr.id}:lowest_ask`, confidence: 0.7, condition: cond, listingType: 'ask', price: pr.lowestListingPrice, currency: 'USD', seller: null, availability: 'available', listedAt: null }));
182 + }
183 + }
184 + return out;
185 + }
186 +}
187 +
188 +export default function createConnector(meta: ConnectorMeta) {
189 + return new NovelshipConnector(meta);
190 +}
added connectors/api/novelship/meta.json +33 −0
@@ -0,0 +1,33 @@
1 +{
2 + "id": "novelship",
3 + "displayName": "Novelship (sneaker catalog, last sale & lowest ask)",
4 + "sourceId": "novelship",
5 + "sourceName": "Novelship",
6 + "sourceType": "marketplace",
7 + "sourceUrl": "https://novelship.com",
8 + "module": "api/novelship",
9 + "enginePriority": ["api", "firecrawl", "scrapfly"],
10 + "categories": ["sneakers", "nike_jordan", "adidas_yeezy", "new_balance_asics_other"],
11 + "regions": ["SG", "AU", "NZ", "TW", "HK", "MY", "JP", "US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.7,
24 + "attributionRequired": true,
25 + "termsUrl": "https://novelship.com/terms",
26 + "accessNotes": "Public Novelship browse pages (novelship.com/sneakers/<brand>?page=N) and product pages fetched over plain HTTPS with the RareIndex user agent; robots.txt only disallows sell/auth/dashboard/pay paths. The server-rendered payload embeds each product's SKU (style code), colorway, retail cost, release date, last sale price, lowest listing price and 180-day sales count — no login or private endpoint is used. Prices are read as USD (the anonymous international storefront prices in US$; cost_retail matches US retail) with confidence 0.7; per-size asks are not exposed on listing pages. Sizes/stock are not fetched. 1.5 s politeness delay.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": ["jordan", "nike", "adidas", "new-balance", "asics", "yeezy"],
31 + "pagesPerSeed": 3
32 + }
33 +}
added connectors/api/optcg/_smoke.ts +3 −0
@@ -0,0 +1,3 @@
1 +// Live smoke: pnpm tsx connectors/api/optcg/_smoke.ts
2 +process.argv.splice(2, process.argv.length, "optcg");
3 +await import("../_lib/smoke.js");
added connectors/api/optcg/index.test.ts +30 −0
@@ -0,0 +1,30 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('optcg'));
7 +
8 +describe('optcg', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('maps a Romance Dawn card with market/low prices dated by date_scraped', async () => {
12 + const [name] = listFixtures('optcg');
13 + const fx = loadFixture('optcg', name!);
14 + const out = await connector.normalize(fx.raw);
15 + const cat = out.find((r) => r.kind === 'catalog_item');
16 + if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
17 + expect(cat.attributes.categorySlug).toBe('one_piece_card_game');
18 + expect(cat.attributes.setCode).toBe('OP-01');
19 + expect(cat.attributes.set).toBe('Romance Dawn');
20 + expect(cat.attributes.number).toMatch(/^OP01-\d{3}$/);
21 + expect(cat.attributes.identifiers.optcg_id).toBe(cat.attributes.number);
22 + const obs = out.filter((r) => r.kind === 'price_observation');
23 + expect(obs.length).toBeGreaterThanOrEqual(1);
24 + const market = obs.find((o) => o.kind === 'price_observation' && o.priceKind === 'market');
25 + if (market?.kind === 'price_observation') {
26 + expect(market.currency).toBe('USD');
27 + expect(market.observationDate.getTime()).toBeLessThanOrEqual(fx.raw.fetchedAt.getTime());
28 + }
29 + });
30 +});
added connectors/api/optcg/index.ts +130 −0
@@ -0,0 +1,130 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, num, parseSlashDate, priceObservation, withRetries } from '../_lib/shared.js';
5 +
6 +/**
7 + * OPTCG API connector — One Piece Card Game (English) catalog per set with TCGplayer-derived
8 + * market / inventory prices and the date they were scraped. Open API, no key (https://optcgapi.com).
9 + */
10 +const API = 'https://optcgapi.com/api';
11 +const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };
12 +const PARSER_VERSION = '1.0.0';
13 +
14 +const SetSchema = z.object({ set_name: z.string(), set_id: z.string() });
15 +const CardSchema = z
16 + .object({
17 + card_set_id: z.string(),
18 + card_name: z.string(),
19 + set_name: z.string(),
20 + set_id: z.string(),
21 + rarity: z.string().nullable().optional(),
22 + card_color: z.string().nullable().optional(),
23 + card_type: z.string().nullable().optional(),
24 + card_cost: z.union([z.string(), z.number()]).nullable().optional(),
25 + card_power: z.union([z.string(), z.number()]).nullable().optional(),
26 + sub_types: z.string().nullable().optional(),
27 + attribute: z.string().nullable().optional(),
28 + card_image: z.string().nullable().optional(),
29 + market_price: z.number().nullable().optional(),
30 + inventory_price: z.number().nullable().optional(),
31 + date_scraped: z.string().nullable().optional(),
32 + })
33 + .loose();
34 +type OpCard = z.infer<typeof CardSchema>;
35 +
36 +export function trimCard(raw: Record<string, unknown>): OpCard {
37 + const keep = ['card_set_id', 'card_name', 'set_name', 'set_id', 'rarity', 'card_color', 'card_type', 'card_cost', 'card_power', 'sub_types', 'attribute', 'card_image', 'market_price', 'inventory_price', 'date_scraped'];
38 + const out: Record<string, unknown> = {};
39 + for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];
40 + return CardSchema.parse(out);
41 +}
42 +
43 +const RawPayloadSchema = z.object({ card: CardSchema, set: SetSchema });
44 +
45 +const RARITY: Record<string, string> = { C: 'Common', UC: 'Uncommon', R: 'Rare', SR: 'Super Rare', SEC: 'Secret Rare', L: 'Leader', SP: 'Special', P: 'Promo', TR: 'Treasure Rare' };
46 +
47 +export class OptcgConnector extends BaseConnector {
48 + readonly version = '1.0.0';
49 + readonly parserVersion = PARSER_VERSION;
50 + protected override minIntervalMs = 500;
51 +
52 + private async get(ctx: CrawlContext, url: string) {
53 + await this.throttle();
54 + return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS }), (r) => r.success && r.json !== null, 4, 1500);
55 + }
56 +
57 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
58 + const setsRes = await this.get(ctx, `${API}/allSets/`);
59 + if (!setsRes.success || !Array.isArray(setsRes.json)) throw new Error(`optcg: sets unavailable (${setsRes.error ?? setsRes.httpStatus})`);
60 + const sets = z.array(SetSchema.loose()).parse(setsRes.json).filter((s) => !ctx.options.seeds?.length || ctx.options.seeds.includes(s.set_id));
61 + let setIndex = Number(ctx.options.cursor?.setIndex ?? 0);
62 + let count = 0;
63 + for (; setIndex < sets.length; setIndex++) {
64 + const set = sets[setIndex]!;
65 + if (ctx.signal?.aborted) return;
66 + const res = await this.get(ctx, `${API}/sets/${encodeURIComponent(set.set_id)}/`);
67 + if (!res.success || !Array.isArray(res.json)) {
68 + ctx.anomaly('set_unavailable', `${set.set_id}: ${res.error ?? res.httpStatus}`);
69 + continue;
70 + }
71 + const seen = new Set<string>();
72 + for (const raw of res.json as unknown[]) {
73 + let card: OpCard;
74 + try {
75 + card = trimCard(raw as Record<string, unknown>);
76 + } catch (err) {
77 + ctx.anomaly('parse_failure_card', `${set.set_id}: ${err instanceof Error ? err.message : String(err)}`);
78 + continue;
79 + }
80 + // The API repeats alternate arts with the same card_set_id; keep them distinct by image id.
81 + const imageId = card.card_image?.match(/\/([^/]+)\.(?:jpg|png|webp)$/i)?.[1] ?? null;
82 + const key = imageId && imageId !== card.card_set_id ? `${card.card_set_id}:${imageId}` : card.card_set_id;
83 + if (seen.has(key)) continue;
84 + seen.add(key);
85 + if (this.reached(ctx, count)) return;
86 + count++;
87 + yield { url: `https://optcgapi.com/cards/${encodeURIComponent(card.card_set_id)}/`, externalId: key, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload: { card, set }, fetchedAt: res.fetchedAt };
88 + }
89 + await ctx.setCursor({ setIndex: setIndex + 1, updatedAt: new Date().toISOString() });
90 + }
91 + await ctx.setCursor({ setIndex: 0, updatedAt: new Date().toISOString() });
92 + }
93 +
94 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
95 + const { card, set } = RawPayloadSchema.parse(raw.payload);
96 + const imageId = card.card_image?.match(/\/([^/]+)\.(?:jpg|png|webp)$/i)?.[1] ?? null;
97 + const isAlt = imageId ? imageId !== card.card_set_id : false;
98 + const altSuffix = imageId && isAlt ? imageId.replace(card.card_set_id, '').replace(/^[_-]/, '') : null;
99 + const variant = isAlt ? (/p\d*$/i.test(altSuffix ?? '') || /_p/i.test(altSuffix ?? '') ? `Alternate Art${altSuffix ? ` ${altSuffix.toUpperCase()}` : ''}` : `Alternate Art ${altSuffix}`) : null;
100 + const identifiers: Record<string, string> = { optcg_id: card.card_set_id };
101 + if (imageId) identifiers.optcg_image_id = imageId;
102 + const a = attrs({
103 + categorySlug: 'one_piece_card_game',
104 + franchise: 'One Piece',
105 + brand: 'Bandai',
106 + set: card.set_name || set.set_name,
107 + setCode: card.set_id || set.set_id,
108 + name: card.card_name,
109 + number: card.card_set_id,
110 + year: null,
111 + variant,
112 + language: 'English',
113 + rarity: card.rarity ? (RARITY[card.rarity] ?? card.rarity) : null,
114 + identifiers,
115 + metadata: { color: card.card_color, type: card.card_type, cost: card.card_cost, power: card.card_power, subTypes: card.sub_types, attribute: card.attribute, rarityCode: card.rarity },
116 + });
117 + const rawTitle = makeTitle({ name: card.card_name, set: a.set, number: card.card_set_id, variant });
118 + const images = card.card_image ? [card.card_image] : [];
119 + const out: NormalizedRecord[] = [];
120 + out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: raw.externalId ?? card.card_set_id, rawTitle, imageUrls: images, attributes: a, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null }));
121 + const scraped = parseSlashDate(card.date_scraped) ?? new Date(Date.UTC(raw.fetchedAt.getUTCFullYear(), raw.fetchedAt.getUTCMonth(), raw.fetchedAt.getUTCDate()));
122 + const market = num(card.market_price);
123 + if (market) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${raw.externalId ?? card.card_set_id}:market`, rawTitle, imageUrls: images, attributes: a, observedAt: scraped, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: market, currency: 'USD', observationDate: scraped, sampleSize: null }));
124 + const low = num(card.inventory_price);
125 + if (low) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${raw.externalId ?? card.card_set_id}:low`, rawTitle, imageUrls: images, attributes: a, observedAt: scraped, confidence: 0.65, parserVersion: PARSER_VERSION, priceKind: 'low', price: low, currency: 'USD', observationDate: scraped, sampleSize: null }));
126 + return out;
127 + }
128 +}
129 +
130 +export default (meta: ConnectorMeta) => new OptcgConnector(meta);
added connectors/api/optcg/meta.json +30 −0
@@ -0,0 +1,30 @@
1 +{
2 + "id": "optcg",
3 + "displayName": "OPTCG API (One Piece Card Game)",
4 + "sourceId": "optcgapi",
5 + "sourceName": "OPTCG API",
6 + "sourceType": "catalog",
7 + "sourceUrl": "https://optcgapi.com",
8 + "module": "api/optcg",
9 + "enginePriority": ["api"],
10 + "categories": ["one_piece_card_game"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.65,
24 + "attributionRequired": true,
25 + "termsUrl": "https://optcgapi.com/",
26 + "accessNotes": "Open community API (no key): /api/allSets/ then /api/sets/<set_id>/ returning every card of the set with TCGplayer-derived market_price / inventory_price and a date_scraped, used as the observation date (confidence 0.7 / 0.65). Alternate arts share the card code and are distinguished by image id. One request per set, 0.5 s politeness.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {}
30 +}
modified connectors/api/pricecharting/index.test.ts +52 −0
@@ -73,3 +73,55 @@ describe('pricecharting', () => {
73 73 expect(p.details['UPC']).toBe('045496870010');
74 74 });
75 75 });
76 +
77 +describe('pricecharting cards', () => {
78 + it('maps a Pokémon Base Set card to the pokemontcg set code and per-grade sales', async () => {
79 + const out = await connector.normalize(loadFixture('pricecharting', 'pokemon-base-set__charizard-4').raw);
80 + const cat = out.find((r) => r.kind === 'catalog_item');
81 + if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
82 + expect(cat.attributes.categorySlug).toBe('pokemon');
83 + expect(cat.attributes.setCode).toBe('BS');
84 + expect(cat.attributes.number).toBe('4');
85 + expect(cat.attributes.name).toBe('Charizard');
86 + expect(cat.attributes.identifiers.pokemontcg_id).toBe('base1-4');
87 + expect(cat.attributes.identifiers.tcgplayer_id).toBe('42382');
88 + const guides = out.filter((r) => r.kind === 'price_observation');
89 + expect(guides.some((g) => g.kind === 'price_observation' && g.grade.grader === 'psa' && g.grade.grade === '10')).toBe(true);
90 + expect(guides.some((g) => g.kind === 'price_observation' && g.grade.grader === 'bgs' && g.grade.qualifier === 'Black Label')).toBe(true);
91 + const sales = out.filter((r) => r.kind === 'sale');
92 + expect(sales.length).toBeGreaterThan(10);
93 + expect(sales.some((s) => s.kind === 'sale' && s.grade.grader === 'psa' && s.grade.grade === '10')).toBe(true);
94 + expect(sales.some((s) => s.kind === 'sale' && s.grade.grader === null && s.grade.grade === null)).toBe(true);
95 + });
96 +
97 + it('keeps 1st Edition printings separate from the unlimited pokemontcg id', async () => {
98 + const out = await connector.normalize(loadFixture('pricecharting', 'pokemon-base-set__charizard-1st-edition-4').raw);
99 + const cat = out.find((r) => r.kind === 'catalog_item');
100 + if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
101 + expect(cat.attributes.variant).toBe('1st Edition');
102 + expect(cat.attributes.identifiers.pokemontcg_id).toBeUndefined();
103 + expect(cat.attributes.setCode).toBe('BS');
104 + });
105 +
106 + it('resolves a Magic card to its Scryfall id and collector number', async () => {
107 + const out = await connector.normalize(loadFixture('pricecharting', 'magic-alpha__black-lotus').raw);
108 + const cat = out.find((r) => r.kind === 'catalog_item');
109 + if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
110 + expect(cat.attributes.categorySlug).toBe('magic_the_gathering');
111 + expect(cat.attributes.setCode).toBe('LEA');
112 + expect(cat.attributes.set).toBe('Limited Edition Alpha');
113 + expect(cat.attributes.number).toBe('232');
114 + expect(cat.attributes.identifiers.scryfall_id).toMatch(/^[0-9a-f-]{36}$/);
115 + });
116 +
117 + it('parses Yu-Gi-Oh! printed codes into set code + number', async () => {
118 + const out = await connector.normalize(loadFixture('pricecharting', 'yugioh-lob__blue-eyes-1st-edition').raw);
119 + const cat = out.find((r) => r.kind === 'catalog_item');
120 + if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
121 + expect(cat.attributes.categorySlug).toBe('yugioh');
122 + expect(cat.attributes.setCode).toBe('LOB');
123 + expect(cat.attributes.number).toBe('LOB-001');
124 + expect(cat.attributes.name).toBe('Blue-Eyes White Dragon');
125 + expect(cat.attributes.variant).toBe('1st Edition');
126 + });
127 +});
modified connectors/api/pricecharting/index.ts +160 −427
@@ -1,461 +1,194 @@
1 1 import { z } from 'zod';
2 −import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 −import { extractYear, parsePrice, parseSourceDate, type NormalizedCatalogItem, type NormalizedPriceObservation, type NormalizedRecord, type NormalizedSale, type AssetAttributes } from '@rareindex/shared';
2 +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors';
3 +import { PriceChartingLikeConnector, parseProductPage as parsePage, splitTitle, setNameFromConsole, type ProductPayload, type SiteOptions } from '../_lib/pc-core.js';
4 4
5 5 /**
6 − * PriceCharting connector — video games, LEGO sets, Funko Pops and comic books.
6 + * PriceCharting connector — video games, LEGO, Funko, comics AND trading cards (Pokémon, Magic,
7 + * Yu-Gi-Oh!) with per-grade eBay completed sales (Ungraded, Grade 1–9.5, PSA/BGS/CGC/SGC/TAG/ACE 10).
7 8 *
8 − * A product page carries (a) PriceCharting's own guide value per condition column and
9 − * (b) a table of recently completed eBay sales per condition tab (date · title · price · eBay id).
10 − * We store the parsed page compactly as one raw record and normalise it into:
11 − * - one catalog_item (canonical identity + identifiers),
12 − * - one price_observation per non-empty guide column ('guide_value'),
13 − * - one sale per completed-sale row (sale_type unknown: eBay auction/BIN not distinguished).
9 + * Card products are enriched at crawl time against reference catalogs so their sales attach to the
10 + * canonical assets created by the API connectors:
11 + * - Pokémon: set name → pokemontcg set (id + ptcgoCode) via the maintainers' GitHub mirror;
12 + * `pokemontcg_id` = `${setId}-${number}` for unlimited printings.
13 + * - Magic: set name → Scryfall set code; exact-name lookup → scryfall_id + collector number.
14 + * - Yu-Gi-Oh!: set code is the prefix of the printed card code (LOB-001 → LOB).
14 15 */
15 16
16 17 const BASE = 'https://www.pricecharting.com';
17 −const PARSER_VERSION = '1.0.0';
18 −
19 −export const PriceCellSchema = z.object({ key: z.string(), value: z.number().nullable(), raw: z.string() });
20 −export const SaleRowSchema = z.object({ tab: z.string(), date: z.string(), title: z.string(), price: z.number(), ebayId: z.string().nullable(), listedPrice: z.number().nullable() });
21 −export const ProductPayloadSchema = z.object({
22 − kind: z.literal('product'),
23 − url: z.string(),
24 − productId: z.string().nullable(),
25 − consoleUri: z.string(),
26 − consoleName: z.string(),
27 − title: z.string(),
28 − flags: z.object({ isComic: z.boolean(), isLegoSet: z.boolean(), isFunkoPop: z.boolean(), isCard: z.boolean(), isCoin: z.boolean(), isSystem: z.boolean() }),
29 − columnLabels: z.array(z.string()),
30 − prices: z.array(PriceCellSchema),
31 − sales: z.array(SaleRowSchema),
32 − details: z.record(z.string(), z.string()),
33 − images: z.array(z.string()),
34 −});
35 −export type ProductPayload = z.infer<typeof ProductPayloadSchema>;
36 −
37 −type Family = 'video_games' | 'lego' | 'funko' | 'comics';
38 −
39 −interface ColumnMeaning {
40 − condition: string | null;
41 − completeness: string | null;
42 − conditionRaw: string;
43 − grade?: string | null;
44 −}
45 −
46 −const PRICE_KEYS = ['used_price', 'complete_price', 'new_price', 'graded_price', 'box_only_price', 'manual_only_price'] as const;
47 −const TAB_TO_KEY: Record<string, (typeof PRICE_KEYS)[number]> = {
48 − used: 'used_price',
49 − cib: 'complete_price',
50 − new: 'new_price',
51 − graded: 'graded_price',
52 − 'box-only': 'box_only_price',
53 − 'manual-only': 'manual_only_price',
54 − 'loose-and-manual': 'graded_price',
18 +const POKEMON_SETS_MIRROR = 'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master/sets/en.json';
19 +const SCRYFALL_SETS = 'https://api.scryfall.com/sets';
20 +const SCRYFALL_NAMED = 'https://api.scryfall.com/cards/named';
21 +const SCRYFALL_HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };
22 +
23 +const PkSetSchema = z.object({ id: z.string(), name: z.string(), ptcgoCode: z.string().optional(), releaseDate: z.string().optional(), series: z.string().optional() });
24 +const ScrySetSchema = z.object({ code: z.string(), name: z.string(), set_type: z.string(), released_at: z.string().optional(), digital: z.boolean().optional() });
25 +
26 +const norm = (s: string) => s.toLowerCase().replace(/&/g, 'and').replace(/[^a-z0-9]+/g, ' ').trim();
27 +const normNoSet = (s: string) => norm(s).replace(/\bset\b/g, '').replace(/\s+/g, ' ').trim();
28 +
29 +/** Classic Magic sets whose PriceCharting console names differ from Scryfall's. */
30 +const MAGIC_OVERRIDES: Record<string, string> = {
31 + alpha: 'lea',
32 + beta: 'leb',
33 + unlimited: '2ed',
34 + revised: '3ed',
35 + 'fourth edition': '4ed',
36 + 'fifth edition': '5ed',
37 + 'sixth edition': '6ed',
38 + 'seventh edition': '7ed',
39 + 'eighth edition': '8ed',
40 + 'ninth edition': '9ed',
41 + 'tenth edition': '10e',
42 + 'collectors edition': 'ced',
43 + 'international collectors edition': 'cei',
44 + 'the dark': 'drk',
45 + 'urzas saga': 'usg',
46 + 'urzas legacy': 'ulg',
47 + 'urzas destiny': 'uds',
48 + 'ravnica city of guilds': 'rav',
49 + 'time spiral timeshifted': 'tsb',
50 + 'commander legends': 'cmr',
51 + 'secret lair drop': 'sld',
52 + 'lord of the rings': 'ltr',
53 + 'marvel spider man': 'spm',
54 + 'avatar the last airbender': 'tla',
55 + 'teenage mutant ninja turtles': 'tmn',
56 + 'the hobbit': 'hbt',
55 57 };
56 −
57 −/** Column semantics per family (PriceCharting reuses the same six cells with different labels). */
58 −const COLUMN_MEANING: Record<Family, Partial<Record<(typeof PRICE_KEYS)[number], ColumnMeaning>>> = {
59 − video_games: {
60 − used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Loose' },
61 − complete_price: { condition: 'cib', completeness: 'cib', conditionRaw: 'Complete in box' },
62 − new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },
63 − graded_price: { condition: null, completeness: null, conditionRaw: 'Graded (any grader)' },
64 − box_only_price: { condition: null, completeness: 'box_only', conditionRaw: 'Box only' },
65 − manual_only_price: { condition: null, completeness: 'manual_only', conditionRaw: 'Manual only' },
66 − },
67 − lego: {
68 − used_price: { condition: 'used_complete', completeness: 'used_complete', conditionRaw: 'Pieces only (used, complete pieces)' },
69 − complete_price: { condition: 'opened_complete', completeness: 'opened_complete', conditionRaw: 'Complete (pieces, box, manual)' },
70 − new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },
71 − manual_only_price: { condition: null, completeness: 'instructions_only', conditionRaw: 'Manual only' },
72 − },
73 − funko: {
74 − used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Out of box' },
75 − complete_price: { condition: 'boxed', completeness: 'boxed', conditionRaw: 'In damaged box' },
76 − new_price: { condition: 'mint_in_box', completeness: 'boxed', conditionRaw: 'New (mint in box)' },
77 − },
78 − comics: {
79 − used_price: { condition: null, completeness: null, conditionRaw: 'Ungraded (raw)', grade: null },
80 − complete_price: { condition: 'very_good', completeness: null, conditionRaw: 'Graded 4.0 / VG (any grader)', grade: '4.0' },
81 − new_price: { condition: 'fine', completeness: null, conditionRaw: 'Graded 6.0 / Fine (any grader)', grade: '6.0' },
82 − graded_price: { condition: 'very_fine', completeness: null, conditionRaw: 'Graded 8.0 / VF (any grader)', grade: '8.0' },
83 − box_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.2 / NM- (any grader)', grade: '9.2' },
84 − manual_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.8 (any grader)', grade: '9.8' },
85 − },
58 +const POKEMON_OVERRIDES: Record<string, string> = {
59 + base: 'base1',
60 + 'base 2': 'base4',
61 + promo: 'basep',
62 + 'wizards black star promos': 'basep',
63 + 'black star promo': 'basep',
64 + 'legendary collection': 'base6',
65 + expedition: 'ecard1',
66 + aquapolis: 'ecard2',
67 + skyridge: 'ecard3',
68 + 'xy evolutions': 'xy12',
69 + evolutions: 'xy12',
70 + 'pokemon go': 'pgo',
71 + 'mcdonalds 25th anniversary': 'mcd21',
72 + 'mcdonalds collection 2019': 'mcd19',
73 + 'celebrations classic collection': 'cel25c',
74 + 'shining legends': 'sm35',
75 + 'dragon majesty': 'sm75',
76 + 'detective pikachu': 'det1',
77 + 'generations': 'g1',
78 + 'double crisis': 'dc1',
86 79 };
87 80
88 −const NINTENDO = /^(nes|famicom|super-nintendo|super-famicom|nintendo-64|gamecube|wii|wii-u|nintendo-switch|gameboy|gameboy-color|gameboy-advance|nintendo-ds|nintendo-3ds|virtual-boy|jp-|pal-)/;
89 −const SEGA = /^(sega-|pal-sega|jp-sega)/;
90 −const PLAYSTATION = /^(playstation|psp|playstation-vita|jp-playstation|pal-playstation)/;
91 −const XBOX = /^xbox/;
92 −const RETRO = /^(atari|intellivision|colecovision|neo-geo|turbografx|pc-engine|commodore|amiga|vectrex|3do|jaguar|philips-cd-i|magnavox|odyssey|msx|sharp|wonderswan|n-gage|game-com|tiger|evercade|super-cassette|fairchild|bally|arcadia|action-max|amiga-cd32)/;
93 −
94 −function familyOf(p: ProductPayload): Family | null {
95 − if (p.flags.isComic || p.consoleUri.startsWith('comic-books')) return 'comics';
96 − if (p.flags.isLegoSet || p.consoleUri.startsWith('lego')) return 'lego';
97 − if (p.flags.isFunkoPop || p.consoleUri.startsWith('funko')) return 'funko';
98 − if (p.flags.isCard || p.flags.isCoin) return null; // cards & coins are handled by dedicated connectors
99 − return 'video_games';
100 −}
101 −
102 −function categorySlug(p: ProductPayload, family: Family): string | null {
103 − const c = p.consoleUri;
104 − switch (family) {
105 − case 'lego':
106 − return 'lego_sets';
107 − case 'funko':
108 − return 'funko';
109 − case 'comics': {
110 − const pub = (p.details['Publisher'] ?? '').toLowerCase();
111 − if (/marvel/.test(pub)) return 'marvel_comics';
112 − if (/\bdc\b|dc comics|vertigo|wildstorm/.test(pub)) return 'dc_comics';
113 − return 'independent_comics';
114 − }
115 − case 'video_games':
116 − if (c === 'pc-games') return 'pc_games';
117 − if (NINTENDO.test(c)) return 'nintendo_games';
118 − if (SEGA.test(c)) return 'sega_games';
119 − if (PLAYSTATION.test(c)) return 'playstation_games';
120 − if (XBOX.test(c)) return 'xbox_games';
121 − if (RETRO.test(c)) return 'atari_retro_games';
122 − return 'video_games';
123 − }
124 −}
125 −
126 −/** "Super Mario 64 [Player's Choice]" → { name, variant } ; "Cloud City #10123" → { name, number } */
127 −function splitTitle(title: string): { name: string; variant: string | null; number: string | null } {
128 − let t = title.trim();
129 − const variants: string[] = [];
130 − t = t.replace(/\[([^\]]+)\]/g, (_m, v: string) => {
131 − variants.push(v.trim());
132 − return ' ';
133 − });
134 − let number: string | null = null;
135 − const num = t.match(/#\s*([A-Za-z0-9.\-/]+)/);
136 − if (num) {
137 − number = num[1]!;
138 − t = t.replace(num[0], ' ');
139 − }
140 − return { name: t.replace(/\s+/g, ' ').trim(), variant: variants.length ? variants.join(' · ') : null, number };
141 −}
142 −
143 −function parseProductPage(htmlText: string, url: string): ProductPayload {
144 − const $ = H.load(htmlText);
145 − const h1 = $('h1#product_name');
146 − const consoleName = H.text(h1.find('a')) ?? '';
147 − const consoleUri = (h1.find('a').attr('href') ?? '').replace(/^\/console\//, '') || (url.match(/\/game\/([^/]+)\//)?.[1] ?? '');
148 − const title = h1
149 − .clone()
150 − .children('a')
151 − .remove()
152 − .end()
153 − .text()
154 − .replace(/\s+/g, ' ')
155 − .trim();
156 − const flagsBlock = htmlText.match(/VGPC\.product\s*=\s*\{([\s\S]*?)\}/)?.[1] ?? '';
157 − const flag = (k: string) => new RegExp(`${k}\\s*:\\s*true`).test(flagsBlock);
158 − const productId = flagsBlock.match(/id\s*:\s*(\d+)/)?.[1] ?? null;
159 −
160 − const prices: z.infer<typeof PriceCellSchema>[] = [];
161 − const seen = new Set<string>();
162 − for (const key of PRICE_KEYS) {
163 − if (seen.has(key)) continue;
164 − const cell = $(`td#${key}`).first();
165 − if (!cell.length) continue;
166 − seen.add(key);
167 − const raw = H.text(cell.find('.price').first()) ?? H.text(cell) ?? '';
168 − const parsed = parsePrice(raw, 'USD');
169 − prices.push({ key, value: parsed && parsed.amount > 0 ? parsed.amount : null, raw });
170 − }
171 − const columnLabels: string[] = [];
172 − $('#price_data thead th, table.price_data thead th').each((_, el) => {
173 − const t = H.text($(el));
174 − if (t) columnLabels.push(t);
175 − });
176 −
177 − const sales: z.infer<typeof SaleRowSchema>[] = [];
178 − $('div[class^="completed-auctions-"]').each((_, sec) => {
179 − const cls = ($(sec).attr('class') ?? '').split(/\s+/).find((c) => c.startsWith('completed-auctions-')) ?? '';
180 − const tab = cls.replace('completed-auctions-', '');
181 − if (!tab || tab === 'condition' || !$(sec).find('table').length) return;
182 − $(sec)
183 − .find('tbody tr')
184 − .each((__, tr) => {
185 − const $tr = $(tr);
186 − const date = H.text($tr.find('td.date')) ?? '';
187 − const titleEl = $tr.find('td.title a').first();
188 − const rowTitle = H.text(titleEl) ?? H.text($tr.find('td.title')) ?? '';
189 − const priceTxt = H.text($tr.find('td.numeric .js-price').first()) ?? H.text($tr.find('td.numeric').first()) ?? '';
190 − const price = parsePrice(priceTxt, 'USD');
191 − const listedTxt = H.text($tr.find('td.listed-price'));
192 − const listed = listedTxt ? parsePrice(listedTxt, 'USD') : null;
193 − const ebayId = ($tr.attr('id') ?? '').match(/ebay-(\d+)/)?.[1] ?? titleEl.attr('href')?.match(/\/itm\/(\d+)/)?.[1] ?? null;
194 − if (!date || !rowTitle || !price || price.amount <= 0) return;
195 − sales.push({ tab, date, title: rowTitle, price: price.amount, ebayId, listedPrice: listed && listed.amount > 0 ? listed.amount : null });
196 − });
197 − });
198 −
199 − const details: Record<string, string> = {};
200 − $('td.title').each((_, el) => {
201 − const k = (H.text($(el)) ?? '').replace(/:$/, '').trim();
202 − const v = H.text($(el).next('td.details'));
203 − if (k && v && v !== 'none' && v !== 'n/a') details[k] = v;
204 − });
205 − const images: string[] = [];
206 − $('img[src*="images.pricecharting.com"]').each((_, el) => {
207 − const src = $(el).attr('src');
208 − if (src && /\/(240|1600|400)\.jpg$/.test(src) && !images.includes(src)) images.push(src);
209 − });
210 −
211 − return {
212 − kind: 'product',
213 − url,
214 − productId,
215 − consoleUri,
216 − consoleName,
217 − title,
218 − flags: { isComic: flag('is_comic'), isLegoSet: flag('is_lego_set'), isFunkoPop: flag('is_funko_pop'), isCard: flag('is_card'), isCoin: flag('is_coin'), isSystem: flag('is_system') },
219 − columnLabels,
220 − prices,
221 − sales,
222 − details,
223 − images: images.slice(0, 4),
81 +export class PriceChartingConnector extends PriceChartingLikeConnector {
82 + protected readonly siteOptions: SiteOptions = {
83 + base: BASE,
84 + site: 'pricecharting',
85 + categoryPrefix: (cat) => ({ 'pokemon-cards': 'pokemon-', 'magic-cards': 'magic-', 'yugioh-cards': 'yugioh-' })[cat] ?? cat.replace(/-cards$/, '-'),
224 86 };
225 −}
226 −
227 −interface ConsoleProduct {
228 − id: string;
229 − productUri: string;
230 − productName: string;
231 − consoleUri: string;
232 −}
233 −
234 −export class PriceChartingConnector extends BaseConnector {
235 − readonly version = '1.0.0';
236 − readonly parserVersion = PARSER_VERSION;
237 87 override readonly urlPatterns = [/^https?:\/\/(www\.)?pricecharting\.com\/game\/[^/]+\/[^/?#]+/i];
238 − protected override minIntervalMs = 1500;
239 −
240 − private get seeds(): string[] {
241 − const s = this.meta.config.seeds;
242 − return Array.isArray(s) ? (s as string[]) : [];
243 − }
244 88
245 − async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
246 − const perConsole = Number(this.meta.config.productsPerConsole ?? 200);
247 − const sort = String(this.meta.config.sort ?? 'popularity');
248 − const seeds = ctx.options.seeds?.length ? ctx.options.seeds : this.seeds;
249 − const cursor = ctx.options.cursor ?? {};
250 − const doneConsoles = new Set<string>((cursor.doneConsoles as string[] | undefined) ?? []);
251 − let count = 0;
252 −
253 − for (const consoleUri of seeds) {
254 − if (doneConsoles.has(consoleUri) && ctx.options.mode !== 'backfill') continue;
255 − if (ctx.signal?.aborted) return;
256 − const products = await this.listConsole(ctx, consoleUri, perConsole, sort);
257 − if (products.length === 0) ctx.anomaly('empty_console', consoleUri);
258 − for (const p of products) {
259 − if (this.reached(ctx, count)) return;
260 − const url = `${BASE}/game/${p.consoleUri}/${p.productUri}`;
261 − if (!(await ctx.shouldFetch(url))) continue;
262 − await this.throttle();
263 − const rec = await this.fetchProduct(ctx, url);
264 − if (rec) {
265 − count++;
266 − yield rec;
267 − }
268 − }
269 − doneConsoles.add(consoleUri);
270 − await ctx.setCursor({ doneConsoles: [...doneConsoles], updatedAt: new Date().toISOString() });
271 − }
272 − // Full pass complete: reset so the next incremental run starts again.
273 − await ctx.setCursor({ doneConsoles: [], updatedAt: new Date().toISOString() });
274 − }
89 + private pokemonSets: z.infer<typeof PkSetSchema>[] | null = null;
90 + private scrySets: z.infer<typeof ScrySetSchema>[] | null = null;
91 + private namedCache = new Map<string, { id: string; number: string; tcgplayer_id?: string } | null>();
275 92
276 − private async listConsole(ctx: CrawlContext, consoleUri: string, max: number, sort: string): Promise<ConsoleProduct[]> {
277 − const out: ConsoleProduct[] = [];
278 − let cursorPos = 0;
279 − while (out.length < max) {
280 − await this.throttle();
281 − const url = `${BASE}/console/${consoleUri}?sort=${encodeURIComponent(sort)}&cursor=${cursorPos}&format=json`;
282 − const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0 });
283 − let json = res.json as { products?: Array<Record<string, unknown>>; cursor?: string } | null;
284 − if (!json && res.html) {
93 + protected override async prepare(ctx: CrawlContext): Promise<void> {
94 + if (!this.pokemonSets) {
95 + const r = await ctx.fetch(POKEMON_SETS_MIRROR, { engines: ['api'], force: true, responseType: 'text' });
96 + let data: unknown = r.json;
97 + if (!data && r.html) {
285 98 try {
286 − json = JSON.parse(res.html) as typeof json;
99 + data = JSON.parse(r.html); // GitHub raw serves text/plain
287 100 } catch {
288 − json = null;
101 + data = null;
289 102 }
290 103 }
291 − if (!res.success || !json?.products) {
292 − if (cursorPos === 0) ctx.anomaly('console_list_failed', `${consoleUri}: ${res.error ?? 'no products'}`);
293 − break;
294 − }
295 − for (const p of json.products) {
296 − if (typeof p.productUri === 'string' && typeof p.consoleUri === 'string') {
297 − out.push({ id: String(p.id ?? ''), productUri: p.productUri, productName: String(p.productName ?? ''), consoleUri: p.consoleUri });
298 − }
299 − }
300 − const next = Number(json.cursor);
301 − if (!Number.isFinite(next) || next <= cursorPos || json.products.length === 0) break;
302 − cursorPos = next;
104 + if (r.success && Array.isArray(data)) this.pokemonSets = z.array(PkSetSchema.loose()).parse(data);
105 + else ctx.anomaly('reference_unavailable', `pokemon sets mirror: ${r.error ?? r.httpStatus}`);
106 + }
107 + if (!this.scrySets) {
108 + const r = await ctx.fetch(SCRYFALL_SETS, { engines: ['api'], headers: SCRYFALL_HEADERS, force: true });
109 + const data = (r.json as { data?: unknown[] } | null)?.data;
110 + if (r.success && Array.isArray(data)) this.scrySets = z.array(ScrySetSchema.loose()).parse(data).filter((s) => !s.digital);
111 + else ctx.anomaly('reference_unavailable', `scryfall sets: ${r.error ?? r.httpStatus}`);
303 112 }
304 − return out.slice(0, max);
305 113 }
306 114
307 − private async fetchProduct(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {
308 − const res = await ctx.fetch(url, {
309 − responseType: 'text',
310 − expect: ['title', 'price', 'identifiers', 'category'],
311 − parse: (r) => {
312 − if (!r.html) return null;
313 − const p = parseProductPage(r.html, url);
314 − return { title: p.title, price: p.prices.find((x) => x.value !== null)?.value ?? null, identifiers: p.productId ? { id: p.productId } : null, category: p.consoleUri };
315 − },
316 − });
317 − if (!res.success || !res.html) {
318 − ctx.anomaly('product_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
115 + /** Pokémon console name → pokemontcg set. */
116 + private pokemonSet(consoleName: string): { id: string; code: string; name: string } | null {
117 + if (!this.pokemonSets) return null;
118 + const raw = setNameFromConsole(consoleName, 'cards');
119 + if (/japanese|korean|chinese/i.test(raw)) return null; // non-English printings are separate catalogs
120 + const key = normNoSet(raw);
121 + const byId = (id: string) => this.pokemonSets!.find((s) => s.id === id);
122 + const override = POKEMON_OVERRIDES[key];
123 + const stripped = key.replace(/^(xy|sm|swsh|sv|bw|hgss|dp|ex|me)\s+/, '');
124 + const hit =
125 + (override ? byId(override) : undefined) ??
126 + this.pokemonSets.find((s) => norm(s.name) === norm(raw)) ??
127 + this.pokemonSets.find((s) => normNoSet(s.name) === key) ??
128 + this.pokemonSets.find((s) => normNoSet(s.name) === stripped) ??
129 + this.pokemonSets.find((s) => normNoSet(s.name).replace(/^(xy|sm|swsh|sv|bw|hgss|dp|ex|me)\s+/, '') === stripped);
130 + if (!hit) {
131 + const contains = this.pokemonSets.filter((s) => normNoSet(s.name) === key || (key.length > 6 && normNoSet(s.name).startsWith(key)));
132 + if (contains.length === 1) return toPk(contains[0]!);
319 133 return null;
320 134 }
321 − const payload = parseProductPage(res.html, url);
322 − if (!payload.title) {
323 − ctx.anomaly('parse_failure_title', url);
324 − return null;
135 + return toPk(hit);
136 + function toPk(s: z.infer<typeof PkSetSchema>) {
137 + return { id: s.id, code: (s.ptcgoCode ?? s.id).toUpperCase(), name: s.name };
325 138 }
326 − return { url, externalId: payload.productId, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
327 139 }
328 140
329 − async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
330 − const clean = url.split('?')[0]!;
331 − const rec = await this.fetchProduct(ctx, clean);
332 − return rec ? [rec] : [];
141 + /** Magic console name → Scryfall set. */
142 + private magicSet(consoleName: string): { code: string; name: string } | null {
143 + if (!this.scrySets) return null;
144 + const raw = setNameFromConsole(consoleName, 'cards');
145 + const key = norm(raw);
146 + const override = MAGIC_OVERRIDES[key];
147 + const byCode = (c: string) => this.scrySets!.find((s) => s.code === c);
148 + const hit = (override ? byCode(override) : undefined) ?? this.scrySets.find((s) => norm(s.name) === key) ?? this.scrySets.find((s) => norm(s.name) === `limited edition ${key}`) ?? this.scrySets.find((s) => norm(s.name).replace(/\bedition\b/g, '').trim() === key.replace(/\bedition\b/g, '').trim());
149 + if (hit) return { code: hit.code, name: hit.name };
150 + const TYPES = ['expansion', 'core', 'masters', 'draft_innovation', 'commander', 'masterpiece', 'promo', 'box', 'funny', 'starter'];
151 + const candidates = this.scrySets.filter((s) => (norm(s.name).startsWith(key) || norm(s.name).includes(key)) && TYPES.includes(s.set_type));
152 + if (candidates.length === 1) return { code: candidates[0]!.code, name: candidates[0]!.name };
153 + const main = candidates.filter((s) => s.set_type === 'expansion' || s.set_type === 'core');
154 + if (main.length === 1) return { code: main[0]!.code, name: main[0]!.name };
155 + return null;
333 156 }
334 157
335 − async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
336 − const p = ProductPayloadSchema.parse(raw.payload);
337 − const family = familyOf(p);
338 − if (!family) return [];
339 − const slug = categorySlug(p, family);
340 − if (!slug) return [];
341 − const { name, variant, number } = splitTitle(p.title);
342 − const year = extractYear(p.details['Release Date'] ?? '') ?? extractYear(p.title) ?? null;
343 − const identifiers: Record<string, string> = {};
344 − if (p.productId) identifiers.pricecharting_id = p.productId;
345 − if (p.details['UPC']) identifiers.upc = p.details['UPC'].replace(/\s+/g, '');
346 − if (p.details['ASIN (Amazon)']) identifiers.asin = p.details['ASIN (Amazon)'];
347 − if (p.details['ePID (eBay)']) identifiers.ebay_epid = p.details['ePID (eBay)'];
348 − if (p.details['Comic.org ID']) identifiers.comics_org_id = p.details['Comic.org ID'];
349 − if (family === 'lego' && number) identifiers.lego_set_number = number;
350 − if (p.details['Model Number']) identifiers.model_number = p.details['Model Number'];
351 − if (family === 'funko' && p.details['Box Number']) identifiers.funko_box_number = p.details['Box Number'];
352 −
353 − const attributes: AssetAttributes = {
354 − categorySlug: slug,
355 − subcategorySlug: null,
356 − franchise: family === 'funko' || family === 'lego' ? p.consoleName.replace(/^(Funko POP|LEGO)\s*/i, '') || null : null,
357 − brand: family === 'lego' ? 'LEGO' : family === 'funko' ? 'Funko' : p.details['Publisher'] ?? null,
358 − series: family === 'funko' ? p.details['Series'] ?? p.consoleName : null,
359 − set: family === 'comics' ? p.consoleName : family === 'lego' ? p.consoleName.replace(/^LEGO\s*/i, '') : p.consoleName,
360 − setCode: null,
361 − name: family === 'comics' ? p.consoleName : name,
362 − model: null,
363 − reference: null,
364 − number: number ?? (family === 'funko' ? p.details['Box Number'] ?? null : null),
365 − year,
366 − edition: null,
367 − variant,
368 − language: 'English',
369 − region: /^(jp|pal)-/.test(p.consoleUri) ? p.consoleUri.split('-')[0]!.toUpperCase() : 'NTSC-U',
370 − country: null,
371 − material: null,
372 − size: null,
373 − color: null,
374 − rarity: null,
375 − productionQuantity: null,
376 − originalMsrp: null,
377 − originalMsrpCurrency: null,
378 − identifiers,
379 − metadata: { pricecharting_console: p.consoleUri, key_issue: p.details['Is Key Issue'] === 'Yes' ? true : undefined, genre: p.details['Genre'], notes: p.details['Notes'] },
380 − };
381 − if (family === 'comics') {
382 − // "Amazing Spider-Man #1 (1963)" → name = series (from console), number = issue, year from title
383 − attributes.name = p.consoleName;
384 − attributes.year = extractYear(p.title) ?? year;
158 + protected override async enrich(ctx: CrawlContext, payload: ProductPayload): Promise<ProductPayload> {
159 + const c = payload.consoleUri;
160 + if (c.startsWith('pokemon-')) {
161 + const set = this.pokemonSet(payload.consoleName);
162 + if (!set) return payload;
163 + const { number, variant } = splitTitle(payload.title, 'cards');
164 + const isBasePrinting = !variant || !/1st edition|shadowless/i.test(variant);
165 + const pokemontcgId = number && isBasePrinting ? `${set.id}-${number.split('/')[0]}` : undefined;
166 + return { ...payload, setRef: { id: set.id, code: set.code, name: set.name, source: 'pokemontcg-mirror' }, cardRef: pokemontcgId ? { pokemontcg_id: pokemontcgId } : null };
385 167 }
386 −
387 − const observedAt = raw.fetchedAt;
388 − const base = {
389 − connectorId: this.meta.id,
390 − sourceId: this.meta.sourceId,
391 − sourceUrl: p.url,
392 − externalId: p.productId,
393 − rawTitle: family === 'comics' ? p.title : `${p.title} (${p.consoleName})`,
394 − description: p.details['Description'] ?? null,
395 − imageUrls: p.images,
396 − attributes,
397 − observedAt,
398 − confidence: 0.9,
399 − parserVersion: PARSER_VERSION,
400 − };
401 − const meanings = COLUMN_MEANING[family];
402 − const out: NormalizedRecord[] = [];
403 − const catalog: NormalizedCatalogItem = { kind: 'catalog_item', ...base, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, releaseDate: parseSourceDate(p.details['Release Date']) ?? null };
404 − out.push(catalog);
405 −
406 − const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));
407 − for (const cell of p.prices) {
408 − const m = meanings[cell.key as (typeof PRICE_KEYS)[number]];
409 − if (!m || cell.value === null) continue;
410 − const obs: NormalizedPriceObservation = {
411 − kind: 'price_observation',
412 − ...base,
413 − grade: { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null },
414 − condition: { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness },
415 − priceKind: 'guide_value',
416 − price: cell.value,
417 − currency: 'USD',
418 − observationDate: obsDate,
419 − sampleSize: null,
420 − confidence: 0.85,
421 − };
422 − out.push(obs);
423 − }
424 − const seenSale = new Set<string>();
425 − for (const row of p.sales) {
426 − const key = TAB_TO_KEY[row.tab];
427 − const m = key ? meanings[key] : undefined;
428 − if (!m) continue;
429 − const saleDate = parseSourceDate(row.date);
430 − if (!saleDate) continue;
431 − const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;
432 − if (seenSale.has(dedupe)) continue;
433 − seenSale.add(dedupe);
434 − const sale: NormalizedSale = {
435 − kind: 'sale',
436 − ...base,
437 − externalId: row.ebayId ? `ebay:${row.ebayId}` : `${p.productId ?? p.url}:${row.date}:${row.price}`,
438 − rawTitle: row.title,
439 − description: null,
440 − grade: { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null },
441 − condition: { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness },
442 − saleType: 'unknown',
443 − saleDate,
444 − price: row.price,
445 − currency: 'USD',
446 − buyerPremiumIncluded: false,
447 − quantity: 1,
448 − isBundle: /\b(lot|bundle)\b/i.test(row.title),
449 − location: null,
450 − auctionHouse: null,
451 − lotNumber: null,
452 − confidence: 0.75,
453 − };
454 − out.push(sale);
168 + if (c.startsWith('magic-')) {
169 + const set = this.magicSet(payload.consoleName);
170 + if (!set) return payload;
171 + const { name, number } = splitTitle(payload.title, 'cards');
172 + let cardRef: ProductPayload['cardRef'] = null;
173 + const cacheKey = `${set.code}|${name.toLowerCase()}|${number ?? ''}`;
174 + if (!this.namedCache.has(cacheKey)) {
175 + await new Promise((r) => setTimeout(r, 120)); // Scryfall politeness (≤ 10 req/s)
176 + const url = number
177 + ? `https://api.scryfall.com/cards/${encodeURIComponent(set.code)}/${encodeURIComponent(number)}`
178 + : `${SCRYFALL_NAMED}?exact=${encodeURIComponent(name)}&set=${encodeURIComponent(set.code)}`;
179 + const r = await ctx.fetch(url, { engines: ['api'], headers: SCRYFALL_HEADERS, force: true, failOnHttpError: false });
180 + const j = r.json as { id?: string; collector_number?: string; tcgplayer_id?: number; object?: string } | null;
181 + this.namedCache.set(cacheKey, r.success && j?.object === 'card' && j.id && j.collector_number ? { id: j.id, number: j.collector_number, ...(j.tcgplayer_id ? { tcgplayer_id: String(j.tcgplayer_id) } : {}) } : null);
182 + if (this.namedCache.size > 5000) this.namedCache.clear();
183 + }
184 + const hit = this.namedCache.get(cacheKey) ?? null;
185 + if (hit) cardRef = { scryfall_id: hit.id, number: hit.number, ...(hit.tcgplayer_id ? { tcgplayer_id: hit.tcgplayer_id } : {}) };
186 + return { ...payload, setRef: { id: set.code, code: set.code.toUpperCase(), name: set.name, source: 'scryfall' }, cardRef };
455 187 }
456 − return out;
188 + return payload;
457 189 }
458 190 }
459 191
460 192 export default (meta: ConnectorMeta) => new PriceChartingConnector(meta);
461 −export { parseProductPage };
193 +export const parseProductPage = (html: string, url: string) => parsePage(html, url, 'pricecharting');
194 +export { ProductPayloadSchema, type ProductPayload } from '../_lib/pc-core.js';
modified connectors/api/pricecharting/meta.json +121 −17
@@ -6,11 +6,39 @@
6 6 "sourceType": "pricing_guide",
7 7 "sourceUrl": "https://www.pricecharting.com",
8 8 "module": "api/pricecharting",
9 − "enginePriority": ["api", "firecrawl", "scrapfly"],
10 − "categories": ["video_games", "nintendo_games", "sega_games", "playstation_games", "xbox_games", "atari_retro_games", "pc_games", "lego_sets", "funko", "comics", "marvel_comics", "dc_comics", "independent_comics"],
11 − "regions": ["US"],
12 − "languages": ["en"],
13 − "currency": ["USD"],
9 + "enginePriority": [
10 + "api",
11 + "firecrawl",
12 + "scrapfly"
13 + ],
14 + "categories": [
15 + "video_games",
16 + "nintendo_games",
17 + "sega_games",
18 + "playstation_games",
19 + "xbox_games",
20 + "atari_retro_games",
21 + "pc_games",
22 + "lego_sets",
23 + "funko",
24 + "comics",
25 + "marvel_comics",
26 + "dc_comics",
27 + "independent_comics",
28 + "trading_cards",
29 + "pokemon",
30 + "magic_the_gathering",
31 + "yugioh"
32 + ],
33 + "regions": [
34 + "US"
35 + ],
36 + "languages": [
37 + "en"
38 + ],
39 + "currency": [
40 + "USD"
41 + ],
14 42 "supportsListings": false,
15 43 "supportsSold": true,
16 44 "supportsAuctions": false,
@@ -23,21 +51,97 @@
23 51 "trustScore": 0.7,
24 52 "attributionRequired": true,
25 53 "termsUrl": "https://www.pricecharting.com/page/terms-of-service",
26 − "accessNotes": "Public product pages fetched over plain HTTPS with the RareIndex user agent (robots.txt only disallows /buy, /publish-offer, /stripe-connect). Each product page exposes PriceCharting guide values per condition and a table of recently completed eBay sales (date, title, price, eBay item id) — we store those as sales with the PriceCharting page as source URL. No account, no API key, no photos (TimeWarp is paid and not used). Console listings paginate through the public ?format=json cursor endpoint. ~1 request per product, 1.5 s politeness delay.",
54 + "accessNotes": "Public product pages fetched over plain HTTPS with the RareIndex user agent (robots.txt only disallows /buy, /publish-offer, /stripe-connect). Each product page exposes PriceCharting guide values per condition and a table of recently completed eBay sales (date, title, price, eBay item id) — we store those as sales with the PriceCharting page as source URL. No account, no API key, no photos (TimeWarp is paid and not used). Console listings paginate through the public ?format=json cursor endpoint. ~1 request per product, 1.5 s politeness delay. Trading-card consoles (Pokémon, Magic, Yu-Gi-Oh!) are discovered from the public /category/<game>-cards pages; card pages expose guide values and eBay sales per grade (Ungraded, Grade 1–9.5, TAG/ACE/SGC/CGC/PSA/BGS 10, BGS 10 Black, CGC 10 Pristine). Graded rows without a named company use the generic grader 'graded' unless the eBay title names it. Pokémon sets are mapped to pokemontcg ids/codes through the maintainers' GitHub mirror and Magic sets/cards to Scryfall (one exact-name API lookup per Magic product, ≤ 10 req/s) so sales attach to the API catalogs' canonical assets.",
27 55 "enabled": true,
28 − "schemaVersion": "1.0",
56 + "schemaVersion": "2.0",
29 57 "config": {
30 58 "seeds": [
31 − "nintendo-64", "nes", "super-nintendo", "gamecube", "gameboy", "gameboy-color", "gameboy-advance", "nintendo-ds", "nintendo-switch", "wii",
32 − "playstation", "playstation-2", "playstation-3", "playstation-4", "playstation-5", "psp",
33 − "xbox", "xbox-360", "xbox-one", "xbox-series-x",
34 − "sega-genesis", "sega-dreamcast", "sega-saturn", "sega-master-system", "sega-game-gear",
35 − "atari-2600", "neo-geo-aes", "turbografx-16", "pc-games",
36 − "lego-star-wars", "lego-creator", "lego-technic", "lego-icons", "lego-harry-potter", "lego-ideas", "lego-marvel-super-heroes", "lego-ninjago", "lego-castle", "lego-space", "lego-modular-buildings",
37 − "funko-pop-marvel", "funko-pop-star-wars", "funko-pop-disney", "funko-pop-animation", "funko-pop-television", "funko-pop-movies", "funko-pop-games", "funko-pop-rocks", "funko-pop-comics",
38 − "comic-books-amazing-spider-man", "comic-books-batman", "comic-books-action-comics", "comic-books-detective-comics", "comic-books-x-men", "comic-books-incredible-hulk", "comic-books-fantastic-four", "comic-books-avengers", "comic-books-spawn", "comic-books-walking-dead"
59 + "nintendo-64",
60 + "nes",
61 + "super-nintendo",
62 + "gamecube",
63 + "gameboy",
64 + "gameboy-color",
65 + "gameboy-advance",
66 + "nintendo-ds",
67 + "nintendo-switch",
68 + "wii",
69 + "playstation",
70 + "playstation-2",
71 + "playstation-3",
72 + "playstation-4",
73 + "playstation-5",
74 + "psp",
75 + "xbox",
76 + "xbox-360",
77 + "xbox-one",
78 + "xbox-series-x",
79 + "sega-genesis",
80 + "sega-dreamcast",
81 + "sega-saturn",
82 + "sega-master-system",
83 + "sega-game-gear",
84 + "atari-2600",
85 + "neo-geo-aes",
86 + "turbografx-16",
87 + "pc-games",
88 + "lego-star-wars",
89 + "lego-creator",
90 + "lego-technic",
91 + "lego-icons",
92 + "lego-harry-potter",
93 + "lego-ideas",
94 + "lego-marvel-super-heroes",
95 + "lego-ninjago",
96 + "lego-castle",
97 + "lego-space",
98 + "lego-modular-buildings",
99 + "funko-pop-marvel",
100 + "funko-pop-star-wars",
101 + "funko-pop-disney",
102 + "funko-pop-animation",
103 + "funko-pop-television",
104 + "funko-pop-movies",
105 + "funko-pop-games",
106 + "funko-pop-rocks",
107 + "funko-pop-comics",
108 + "comic-books-amazing-spider-man",
109 + "comic-books-batman",
110 + "comic-books-action-comics",
111 + "comic-books-detective-comics",
112 + "comic-books-x-men",
113 + "comic-books-incredible-hulk",
114 + "comic-books-fantastic-four",
115 + "comic-books-avengers",
116 + "comic-books-spawn",
117 + "comic-books-walking-dead",
118 + "pokemon-base-set",
119 + "pokemon-jungle",
120 + "pokemon-fossil",
121 + "pokemon-team-rocket",
122 + "pokemon-neo-genesis",
123 + "pokemon-evolving-skies",
124 + "pokemon-151",
125 + "pokemon-promo",
126 + "magic-alpha",
127 + "magic-beta",
128 + "magic-unlimited",
129 + "magic-revised",
130 + "magic-arabian-nights",
131 + "magic-legends",
132 + "magic-modern-horizons-3",
133 + "yugioh-legend-of-blue-eyes-white-dragon",
134 + "yugioh-metal-raiders",
135 + "yugioh-spell-ruler",
136 + "yugioh-pharaohs-servant"
39 137 ],
40 − "productsPerConsole": 200,
41 − "sort": "popularity"
138 + "productsPerConsole": 150,
139 + "sort": "popularity",
140 + "cardCategories": [
141 + "pokemon-cards",
142 + "magic-cards",
143 + "yugioh-cards"
144 + ],
145 + "maxConsolesPerCategory": 30
42 146 }
43 147 }
added connectors/api/sportscardspro/_smoke.ts +3 −0
@@ -0,0 +1,3 @@
1 +// Live smoke: pnpm tsx connectors/api/sportscardspro/_smoke.ts
2 +process.argv.splice(2, process.argv.length, "sportscardspro");
3 +await import("../_lib/smoke.js");
added connectors/api/sportscardspro/index.test.ts +35 −0
@@ -0,0 +1,35 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('sportscardspro'));
7 +
8 +describe('sportscardspro', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('maps a 1986 Fleer Jordan page to a sports card asset with graded and raw sales', async () => {
12 + const fx = loadFixture('sportscardspro', 'basketball-1986-fleer__michael-jordan-57');
13 + const out = await connector.normalize(fx.raw);
14 + const cat = out.find((r) => r.kind === 'catalog_item');
15 + if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
16 + expect(cat.attributes.categorySlug).toBe('basketball_cards');
17 + expect(cat.attributes.set).toBe('1986 Fleer');
18 + expect(cat.attributes.year).toBe(1986);
19 + expect(cat.attributes.number).toBe('57');
20 + expect(cat.attributes.name).toBe('Michael Jordan');
21 + expect(cat.attributes.brand).toBe('Fleer');
22 + expect(cat.attributes.variant).toBeNull(); // "[Rookie]" is a status tag, not a printing
23 + expect(cat.attributes.metadata.rookie).toBe(true);
24 + expect(cat.attributes.identifiers.pricecharting_id).toMatch(/^scp:\d+$/);
25 + const guides = out.filter((r) => r.kind === 'price_observation');
26 + expect(guides.length).toBeGreaterThanOrEqual(10);
27 + const psa10 = guides.find((g) => g.kind === 'price_observation' && g.grade.grader === 'psa' && g.grade.grade === '10');
28 + expect(psa10).toBeTruthy();
29 + const sales = out.filter((r) => r.kind === 'sale');
30 + expect(sales.length).toBeGreaterThan(10);
31 + const graded = sales.filter((s) => s.kind === 'sale' && s.grade.grade !== null);
32 + expect(graded.length).toBeGreaterThan(0);
33 + for (const s of sales) if (s.kind === 'sale') expect(s.currency).toBe('USD');
34 + });
35 +});
added connectors/api/sportscardspro/index.ts +23 −0
@@ -0,0 +1,23 @@
1 +import type { ConnectorMeta } from '@rareindex/connectors';
2 +import { PriceChartingLikeConnector, parseProductPage as parsePage, type SiteOptions } from '../_lib/pc-core.js';
3 +
4 +/**
5 + * SportsCardsPro connector (PriceCharting's sports-card site): per-set product pages with guide
6 + * values per grade and eBay completed sales per grade tab (Ungraded, Grade 1–9.5, PSA/BGS/CGC/SGC 10).
7 + * Plain HTTP receives a Cloudflare interstitial, so pages are fetched through Firecrawl (robots.txt
8 + * allows everything except /buy, /publish-offer, /stripe-connect). Same parser as PriceCharting.
9 + */
10 +const BASE = 'https://www.sportscardspro.com';
11 +
12 +export class SportsCardsProConnector extends PriceChartingLikeConnector {
13 + protected readonly siteOptions: SiteOptions = {
14 + base: BASE,
15 + site: 'sportscardspro',
16 + categoryPrefix: (cat) => `${cat}-`, // "basketball-cards" → consoles "basketball-cards-1986-fleer"
17 + };
18 + override readonly urlPatterns = [/^https?:\/\/(www\.)?sportscardspro\.com\/game\/[^/]+\/[^/?#]+/i];
19 + protected override minIntervalMs = 1200;
20 +}
21 +
22 +export default (meta: ConnectorMeta) => new SportsCardsProConnector(meta);
23 +export const parseProductPage = (html: string, url: string) => parsePage(html, url, 'sportscardspro');
added connectors/api/sportscardspro/meta.json +42 −0
@@ -0,0 +1,42 @@
1 +{
2 + "id": "sportscardspro",
3 + "displayName": "SportsCardsPro",
4 + "sourceId": "sportscardspro",
5 + "sourceName": "SportsCardsPro (PriceCharting)",
6 + "sourceType": "pricing_guide",
7 + "sourceUrl": "https://www.sportscardspro.com",
8 + "module": "api/sportscardspro",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["sports_cards", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "f1_cards", "other_sports_cards"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "high",
23 + "trustScore": 0.7,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.sportscardspro.com/page/terms-of-service",
26 + "accessNotes": "Public product pages of PriceCharting's sports-card site. robots.txt only disallows /buy, /publish-offer, /stripe-connect. Plain HTTP with our user agent gets a Cloudflare interstitial (HTTP 403), so pages are fetched through Firecrawl (1 credit per page, ~1 MB each) — no login, no CAPTCHA solving, no account. Each page exposes guide values per grade (Ungraded, Grade 1–9.5, TAG/ACE/SGC/CGC/PSA/BGS 10, BGS 10 Black, CGC 10 Pristine) and recently completed eBay sales per grade tab; graded rows without a named grading company are stored with the generic grader 'graded' unless the eBay title names the company. Set lists come from the public /category/<sport>-cards pages; console listings use the ?format=json cursor endpoint.",
27 + "enabled": true,
28 + "schemaVersion": "2.0",
29 + "config": {
30 + "seeds": [
31 + "basketball-cards-1986-fleer", "basketball-cards-2003-topps-chrome", "basketball-cards-2018-panini-prizm", "basketball-cards-2019-panini-prizm", "basketball-cards-1996-topps-chrome",
32 + "baseball-cards-1952-topps", "baseball-cards-1989-upper-deck", "baseball-cards-2011-topps-update", "baseball-cards-2018-topps-update", "baseball-cards-1993-sp",
33 + "football-cards-2000-playoff-contenders", "football-cards-2017-panini-prizm", "football-cards-2020-panini-prizm", "football-cards-1957-topps",
34 + "hockey-cards-1979-o-pee-chee", "hockey-cards-2005-upper-deck", "hockey-cards-2015-upper-deck",
35 + "soccer-cards-2018-panini-prizm-world-cup", "soccer-cards-2004-panini-mega-cracks"
36 + ],
37 + "cardCategories": ["basketball-cards", "baseball-cards", "football-cards", "hockey-cards", "soccer-cards"],
38 + "maxConsolesPerCategory": 25,
39 + "productsPerConsole": 120,
40 + "sort": "popularity"
41 + }
42 +}
added connectors/api/tcgdex/_smoke.ts +3 −0
@@ -0,0 +1,3 @@
1 +// Live smoke: pnpm tsx connectors/api/tcgdex/_smoke.ts
2 +process.argv.splice(2, process.argv.length, "tcgdex");
3 +await import("../_lib/smoke.js");
added connectors/api/tcgdex/index.test.ts +31 −0
@@ -0,0 +1,31 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('tcgdex'));
7 +
8 +describe('tcgdex', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('aligns Base Set cards with the pokemontcg vocabulary and dates prices by their update time', async () => {
12 + const out = await connector.normalize(loadFixture('tcgdex', 'en-base1-1').raw);
13 + const cat = out.find((r) => r.kind === 'catalog_item');
14 + if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
15 + expect(cat.attributes.categorySlug).toBe('pokemon');
16 + expect(cat.attributes.setCode).toBe('BS');
17 + expect(cat.attributes.number).toBe('1');
18 + expect(cat.attributes.name).toBe('Alakazam');
19 + expect(cat.attributes.variant).toBe('Holo');
20 + expect(cat.attributes.language).toBe('English');
21 + expect(cat.attributes.identifiers.pokemontcg_id).toBe('base1-1');
22 + expect(cat.attributes.identifiers.tcgplayer_id).toMatch(/^\d+$/);
23 + const obs = out.filter((r) => r.kind === 'price_observation');
24 + expect(obs.length).toBeGreaterThan(3);
25 + const usd = obs.find((o) => o.kind === 'price_observation' && o.currency === 'USD' && o.priceKind === 'market');
26 + const eur = obs.find((o) => o.kind === 'price_observation' && o.currency === 'EUR' && o.priceKind === 'trend');
27 + expect(usd).toBeTruthy();
28 + expect(eur).toBeTruthy();
29 + if (usd?.kind === 'price_observation') expect(usd.observationDate.getTime()).toBeLessThan(usd.observedAt.getTime() + 1);
30 + });
31 +});
added connectors/api/tcgdex/index.ts +230 −0
@@ -0,0 +1,230 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { type NormalizedRecord } from '@rareindex/shared';
4 +import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';
5 +
6 +/**
7 + * TCGdex connector — Pokémon TCG catalog for every language (en, ja, fr, de, es, it, pt, ko, zh…)
8 + * with TCGplayer / Cardmarket pricing per variant on English cards. Card ids share pokemontcg.io's
9 + * scheme for English sets (base1-4, swsh1-1), so `pokemontcg_id` lines up with the pokemontcg
10 + * connector and set codes use the same PTCGO abbreviation (tcgOnline) as pokemontcg's ptcgoCode.
11 + */
12 +const API = 'https://api.tcgdex.net/v2';
13 +const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' };
14 +const PARSER_VERSION = '1.0.0';
15 +
16 +const SetBriefSchema = z.object({ id: z.string(), name: z.string(), cardCount: z.object({ total: z.number(), official: z.number() }).partial().optional() });
17 +const SetSchema = z.object({
18 + id: z.string(),
19 + name: z.string(),
20 + releaseDate: z.string().optional(),
21 + tcgOnline: z.string().optional(),
22 + abbreviation: z.object({ official: z.string().optional(), localized: z.string().optional() }).partial().optional(),
23 + serie: z.object({ id: z.string(), name: z.string() }).optional(),
24 + cardCount: z.object({ total: z.number().optional(), official: z.number().optional() }).partial().optional(),
25 + cards: z.array(z.object({ id: z.string(), localId: z.string(), name: z.string(), image: z.string().optional() })),
26 +});
27 +const TcgplayerPricesSchema = z.record(z.string(), z.unknown());
28 +const CardSchema = z.object({
29 + id: z.string(),
30 + localId: z.string(),
31 + name: z.string(),
32 + rarity: z.string().optional(),
33 + category: z.string().optional(),
34 + illustrator: z.string().optional(),
35 + image: z.string().optional(),
36 + hp: z.number().optional(),
37 + types: z.array(z.string()).optional(),
38 + dexId: z.array(z.number()).optional(),
39 + regulationMark: z.string().optional(),
40 + updated: z.string().optional(),
41 + set: z.object({ id: z.string(), name: z.string() }),
42 + variants: z.object({ firstEdition: z.boolean().optional(), holo: z.boolean().optional(), normal: z.boolean().optional(), reverse: z.boolean().optional(), wPromo: z.boolean().optional() }).partial().optional(),
43 + variants_detailed: z
44 + .array(
45 + z
46 + .object({
47 + type: z.string(),
48 + subtype: z.string().optional(),
49 + stamp: z.array(z.string()).optional(),
50 + thirdParty: z.object({ cardmarket: z.number().optional(), tcgplayer: z.number().optional() }).partial().optional(),
51 + pricing: z.object({ cardmarket: z.record(z.string(), z.unknown()).nullable().optional(), tcgplayer: z.record(z.string(), z.unknown()).nullable().optional() }).partial().nullable().optional(),
52 + })
53 + .loose(),
54 + )
55 + .optional(),
56 + pricing: z.object({ cardmarket: z.record(z.string(), z.unknown()).nullable().optional(), tcgplayer: z.record(z.string(), z.unknown()).nullable().optional() }).partial().nullable().optional(),
57 +});
58 +type TcgCard = z.infer<typeof CardSchema>;
59 +
60 +const RawPayloadSchema = z.object({
61 + card: CardSchema,
62 + set: z.object({ id: z.string(), name: z.string(), code: z.string().nullable(), releaseDate: z.string().nullable(), serie: z.string().nullable(), total: z.number().nullable() }),
63 + language: z.string(),
64 +});
65 +
66 +/** Keep the payload compact: drop attacks/abilities/legal text. */
67 +export function trimCard(raw: Record<string, unknown>): TcgCard {
68 + const keep = ['id', 'localId', 'name', 'rarity', 'category', 'illustrator', 'image', 'hp', 'types', 'dexId', 'regulationMark', 'updated', 'set', 'variants', 'variants_detailed', 'pricing'];
69 + const out: Record<string, unknown> = {};
70 + for (const k of keep) if (raw[k] !== undefined) out[k] = raw[k];
71 + if (out.set && typeof out.set === 'object') out.set = { id: (out.set as { id: string }).id, name: (out.set as { name: string }).name };
72 + return CardSchema.parse(out);
73 +}
74 +
75 +const LANG_NAME: Record<string, string> = { en: 'English', ja: 'Japanese', fr: 'French', de: 'German', es: 'Spanish', it: 'Italian', pt: 'Portuguese', ko: 'Korean', zh: 'Chinese', 'zh-tw': 'Chinese (Traditional)', 'zh-cn': 'Chinese (Simplified)', nl: 'Dutch', pl: 'Polish', ru: 'Russian', th: 'Thai', id: 'Indonesian' };
76 +
77 +/** Variant vocabulary aligned with the pokemontcg connector. */
78 +function variantLabel(type: string, stamp?: string[]): string | null {
79 + const first = stamp?.some((s) => /1st/i.test(s));
80 + if (type === 'reverse') return first ? '1st Edition Reverse Holo' : 'Reverse Holo';
81 + if (type === 'holo') return first ? '1st Edition Holo' : 'Holo';
82 + if (type === 'firstEdition') return '1st Edition';
83 + return first ? '1st Edition' : null;
84 +}
85 +
86 +export class TcgdexConnector extends BaseConnector {
87 + readonly version = '1.0.0';
88 + readonly parserVersion = PARSER_VERSION;
89 + protected override minIntervalMs = 120;
90 + override readonly urlPatterns = [/^https?:\/\/(www\.)?tcgdex\.net\/(?:[a-z]{2}\/)?(?:database|cards?)\//i];
91 +
92 + private async get(ctx: CrawlContext, url: string) {
93 + await this.throttle();
94 + return withRetries(() => ctx.fetch(url, { engines: ['api'], headers: HEADERS, timeoutMs: 60_000 }), (r) => r.success && r.json !== null, 4, 1500);
95 + }
96 +
97 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
98 + const languages = ctx.options.seeds?.length ? ctx.options.seeds.filter((s) => /^[a-z]{2}(-[a-z]{2})?$/.test(s)) : ((this.meta.config.languages as string[] | undefined) ?? ['en']);
99 + const seedSets = ctx.options.seeds?.filter((s) => !/^[a-z]{2}(-[a-z]{2})?$/.test(s)) ?? [];
100 + const cursor = ctx.options.cursor ?? {};
101 + let langIndex = Number(cursor.langIndex ?? 0);
102 + let setIndex = Number(cursor.setIndex ?? 0);
103 + let count = 0;
104 + for (; langIndex < languages.length; langIndex++, setIndex = 0) {
105 + const lang = languages[langIndex]!;
106 + const setsRes = await this.get(ctx, `${API}/${lang}/sets`);
107 + if (!setsRes.success || !Array.isArray(setsRes.json)) {
108 + ctx.anomaly('sets_unavailable', `${lang}: ${setsRes.error ?? setsRes.httpStatus}`);
109 + continue;
110 + }
111 + const sets = z.array(SetBriefSchema.loose()).parse(setsRes.json).filter((s) => !seedSets.length || seedSets.includes(s.id));
112 + for (; setIndex < sets.length; setIndex++) {
113 + const brief = sets[setIndex]!;
114 + if (ctx.signal?.aborted) return;
115 + const setRes = await this.get(ctx, `${API}/${lang}/sets/${encodeURIComponent(brief.id)}`);
116 + if (!setRes.success || !setRes.json) {
117 + ctx.anomaly('set_unavailable', `${lang}/${brief.id}`);
118 + continue;
119 + }
120 + const parsedSet = SetSchema.loose().safeParse(setRes.json);
121 + if (!parsedSet.success) {
122 + ctx.anomaly('parse_failure_set', `${lang}/${brief.id}: ${parsedSet.error.issues[0]?.message}`);
123 + continue;
124 + }
125 + const set = parsedSet.data;
126 + const setMeta = { id: set.id, name: set.name, code: set.tcgOnline ?? set.abbreviation?.official ?? null, releaseDate: set.releaseDate ?? null, serie: set.serie?.name ?? null, total: set.cardCount?.official ?? set.cardCount?.total ?? null };
127 + for (const c of set.cards) {
128 + if (this.reached(ctx, count)) return;
129 + const url = `${API}/${lang}/cards/${encodeURIComponent(c.id)}`;
130 + const cardRes = await this.get(ctx, url);
131 + if (!cardRes.success || !cardRes.json) {
132 + ctx.anomaly('card_unavailable', `${lang}/${c.id}`);
133 + continue;
134 + }
135 + let card: TcgCard;
136 + try {
137 + card = trimCard(cardRes.json as Record<string, unknown>);
138 + } catch (err) {
139 + ctx.anomaly('parse_failure_card', `${lang}/${c.id}: ${err instanceof Error ? err.message : String(err)}`);
140 + continue;
141 + }
142 + count++;
143 + yield { url: `https://tcgdex.net/${lang}/database/${set.id}/${card.localId}`, externalId: `${lang}:${card.id}`, kind: 'catalog_item', engine: 'api', httpStatus: cardRes.httpStatus, payload: { card, set: setMeta, language: lang }, fetchedAt: cardRes.fetchedAt };
144 + }
145 + await ctx.setCursor({ langIndex, setIndex: setIndex + 1, updatedAt: new Date().toISOString() });
146 + }
147 + }
148 + await ctx.setCursor({ langIndex: 0, setIndex: 0, updatedAt: new Date().toISOString() });
149 + }
150 +
151 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
152 + const { card, set, language } = RawPayloadSchema.parse(raw.payload);
153 + const year = set.releaseDate ? Number(set.releaseDate.slice(0, 4)) || null : null;
154 + const setCode = set.code ?? set.id.toUpperCase();
155 + const langName = LANG_NAME[language] ?? language;
156 + const sourceUrl = raw.url;
157 + const images = card.image ? [`${card.image}/high.webp`] : [];
158 + const variants = card.variants_detailed?.length
159 + ? card.variants_detailed
160 + : Object.entries(card.variants ?? {})
161 + .filter(([, v]) => v)
162 + .map(([k]) => ({ type: k === 'firstEdition' ? 'firstEdition' : k, subtype: undefined, stamp: undefined, thirdParty: undefined, pricing: undefined }))
163 + .filter((v) => v.type !== 'wPromo');
164 + if (!variants.length) variants.push({ type: 'normal', subtype: undefined, stamp: undefined, thirdParty: undefined, pricing: undefined });
165 + const out: NormalizedRecord[] = [];
166 + const seenVariant = new Set<string>();
167 + for (const v of variants) {
168 + const variant = variantLabel(v.type, v.stamp) ?? (v.subtype && v.subtype !== 'unlimited' && v.subtype !== 'standard' ? cap(v.subtype) : null);
169 + const vkey = variant ?? '';
170 + if (seenVariant.has(vkey)) continue;
171 + seenVariant.add(vkey);
172 + const identifiers: Record<string, string> = { tcgdex_id: `${language}:${card.id}` };
173 + if (language === 'en' && !variant?.includes('1st Edition')) identifiers.pokemontcg_id = card.id;
174 + if (v.thirdParty?.tcgplayer) identifiers.tcgplayer_id = String(v.thirdParty.tcgplayer);
175 + if (v.thirdParty?.cardmarket) identifiers.cardmarket_id = String(v.thirdParty.cardmarket);
176 + const a = attrs({
177 + categorySlug: 'pokemon',
178 + franchise: 'Pokémon',
179 + brand: 'The Pokémon Company',
180 + series: set.serie,
181 + set: set.name,
182 + setCode,
183 + name: card.name,
184 + number: card.localId.replace(/^0+(?=\d)/, ''),
185 + year,
186 + variant,
187 + language: langName,
188 + rarity: card.rarity ?? null,
189 + identifiers,
190 + metadata: { hp: card.hp, types: card.types, illustrator: card.illustrator, dexIds: card.dexId, regulationMark: card.regulationMark, category: card.category, tcgdex_set: set.id },
191 + });
192 + const rawTitle = makeTitle({ name: card.name, set: set.name, number: a.number, total: set.total, year, variant });
193 + const observedAt = card.updated ? new Date(card.updated) : raw.fetchedAt;
194 + out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}${variant ? `:${variant}` : ''}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.9, parserVersion: PARSER_VERSION, releaseDate: set.releaseDate ? new Date(set.releaseDate) : null }));
195 +
196 + const pricing = (v.pricing ?? (variants.length === 1 ? card.pricing : null)) ?? null;
197 + const tcg = pricing?.tcgplayer as Record<string, unknown> | null | undefined;
198 + if (tcg) {
199 + const updated = typeof tcg.updated === 'string' ? new Date(tcg.updated) : null;
200 + for (const [k, val] of Object.entries(tcg)) {
201 + if (!val || typeof val !== 'object') continue;
202 + const bucket = val as Record<string, unknown>;
203 + const kinds: Array<[string, 'market' | 'low' | 'mid' | 'high']> = [['marketPrice', 'market'], ['lowPrice', 'low'], ['midPrice', 'mid'], ['highPrice', 'high']];
204 + for (const [field, priceKind] of kinds) {
205 + const price = num(bucket[field]);
206 + if (!price || !updated) continue;
207 + out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}:${k}:${priceKind}`, rawTitle, imageUrls: images, attributes: a, observedAt: updated, confidence: 0.8, parserVersion: PARSER_VERSION, priceKind, price, currency: 'USD', observationDate: updated, sampleSize: null }));
208 + }
209 + }
210 + }
211 + const cm = pricing?.cardmarket as Record<string, unknown> | null | undefined;
212 + if (cm) {
213 + const updated = typeof cm.updated === 'string' ? new Date(cm.updated) : null;
214 + const kinds: Array<[string, 'market' | 'low' | 'trend' | 'average_7d' | 'average_30d']> = [['avg', 'market'], ['low', 'low'], ['trend', 'trend'], ['avg7', 'average_7d'], ['avg30', 'average_30d']];
215 + for (const [field, priceKind] of kinds) {
216 + const price = num(cm[field]);
217 + if (!price || !updated) continue;
218 + out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${language}:${card.id}:cm:${priceKind}`, rawTitle, imageUrls: images, attributes: a, observedAt: updated, confidence: 0.75, parserVersion: PARSER_VERSION, priceKind, price, currency: 'EUR', observationDate: updated, sampleSize: null }));
219 + }
220 + }
221 + }
222 + return out;
223 + }
224 +}
225 +
226 +function cap(s: string): string {
227 + return s.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
228 +}
229 +
230 +export default (meta: ConnectorMeta) => new TcgdexConnector(meta);
added connectors/api/tcgdex/meta.json +32 −0
@@ -0,0 +1,32 @@
1 +{
2 + "id": "tcgdex",
3 + "displayName": "TCGdex (Pokémon, multilingual)",
4 + "sourceId": "tcgdex",
5 + "sourceName": "TCGdex",
6 + "sourceType": "catalog",
7 + "sourceUrl": "https://tcgdex.dev",
8 + "module": "api/tcgdex",
9 + "enginePriority": ["api"],
10 + "categories": ["pokemon"],
11 + "regions": ["global"],
12 + "languages": ["en", "ja", "fr", "de", "es", "it", "pt", "ko", "zh"],
13 + "currency": ["USD", "EUR"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "high",
23 + "trustScore": 0.8,
24 + "attributionRequired": true,
25 + "termsUrl": "https://tcgdex.dev/",
26 + "accessNotes": "Open REST API (https://api.tcgdex.net/v2, no key, MIT-licensed data) covering every Pokémon TCG language including Japanese; English cards carry TCGplayer (USD) and Cardmarket (EUR) prices per variant with their own update timestamps, which we store as price observations dated by the price's `updated` field. One request per set and per card at ≈8 req/s (self-throttled 120 ms). English ids match pokemontcg.io (base1-4) so `pokemontcg_id` aligns; set codes use the PTCGO abbreviation like pokemontcg's ptcgoCode.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "languages": ["en", "ja"]
31 + }
32 +}
added connectors/firecrawl/aucfree/_smoke.ts +18 −0
@@ -0,0 +1,18 @@
1 +import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';
2 +import { saveFixture } from '@rareindex/connectors/testing';
3 +import createConnector from './index.js';
4 +
5 +const save = process.argv.includes('--save');
6 +const meta = getConnectorMeta('aucfree');
7 +const connector = createConnector(meta);
8 +const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
9 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2, categories: ['pokemon', 'gundam'] } });
10 +let i = 0;
11 +for await (const raw of connector.crawl(ctx)) {
12 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
13 + console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} sales`);
14 + for (const r of out.slice(0, 3)) console.log(' ', JSON.stringify(r).slice(0, 360));
15 + if (save && i < 2) saveFixture('aucfree', i === 0 ? 'pokemon-psa10-page1' : 'seed-page-2', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 10, kinds: ['sale'], requiredFields: ['saleDate', 'price', 'currency'] }, note: 'Captured live via Firecrawl from aucfree.com' });
16 + i++;
17 +}
18 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/aucfree/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector, { normaliseGradeText, parseJapaneseDate, parseSearchPage } from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('aucfree'));
7 +
8 +describe('aucfree', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('maps closed lots to JPY auction sales with parsed PSA grades', async () => {
12 + const fx = loadFixture('aucfree', 'pokemon-psa10-page1');
13 + const out = await connector.normalize(fx.raw);
14 + expect(out.length).toBeGreaterThanOrEqual(40);
15 + const graded = out.filter((r) => r.kind === 'sale' && r.grade.grader === 'psa' && r.grade.grade === '10');
16 + expect(graded.length).toBeGreaterThan(20);
17 + for (const r of out) {
18 + if (r.kind !== 'sale') throw new Error();
19 + expect(r.currency).toBe('JPY');
20 + expect(r.saleType).toBe('auction');
21 + expect(r.buyerPremiumIncluded).toBe(false);
22 + expect(r.attributes.categorySlug).toBe('pokemon');
23 + expect(r.attributes.language).toBe('Japanese');
24 + expect(r.sourceUrl).toMatch(/^https:\/\/aucfree\.com\/items\/[a-z0-9]+$/);
25 + expect(r.saleDate.getUTCHours()).toBe(0);
26 + }
27 + });
28 +
29 + it('helpers', () => {
30 + expect(parseJapaneseDate('2026年9月6日')?.toISOString()).toBe('2026-09-06T00:00:00.000Z');
31 + expect(parseJapaneseDate('nope')).toBeNull();
32 + expect(normaliseGradeText('【PSA10】ポケモンカード')).toContain('PSA 10');
33 + expect(normaliseGradeText('BGS9.5 カード')).toContain('BGS 9.5');
34 + const html = `<table><tr class="results_bid"><td><a class="item_title" href="https://aucfree.com/items/x123">テスト カード PSA10</a><div class="results_bid-image"><img data-src="https://img.aucfree.com/x123.1.jpg"></div></td><td class="results-price"><a class="item_price">12,345円</a></td><td class="results-bid">3件</td><td class="results-limit">2026年9月1日</td></tr></table>`;
35 + const p = parseSearchPage(html, 'u', { q: 'q', category: 'pokemon', language: 'Japanese' }, 1);
36 + expect(p.rows).toEqual([{ id: 'x123', url: 'https://aucfree.com/items/x123', title: 'テスト カード PSA10', priceJpy: 12345, bids: 3, endedOn: '2026年9月1日', image: 'https://img.aucfree.com/x123.1.jpg' }]);
37 + });
38 +});
added connectors/firecrawl/aucfree/index.ts +146 −0
@@ -0,0 +1,146 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { parseGradeFromTitle } from '@rareindex/taxonomy';
4 +import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';
5 +
6 +/**
7 + * aucfree — closed Yahoo! Auctions Japan lots (JPY hammer prices) for Japanese collectibles.
8 + * One raw record per search-result page; normalise → one sale per row.
9 + */
10 +
11 +const BASE = 'https://aucfree.com';
12 +const PARSER_VERSION = '1.0.0';
13 +
14 +const SeedSchema = z.object({ q: z.string(), category: z.string(), language: z.string().nullable().default(null) });
15 +type Seed = z.infer<typeof SeedSchema>;
16 +
17 +export const RowSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), priceJpy: z.number(), bids: z.number().nullable(), endedOn: z.string(), image: z.string().nullable() });
18 +export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), seed: SeedSchema, page: z.number(), rows: z.array(RowSchema) });
19 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
20 +
21 +export function parseSearchPage(htmlText: string, url: string, seed: Seed, page: number): PagePayload {
22 + const $ = H.load(htmlText);
23 + const rows: z.infer<typeof RowSchema>[] = [];
24 + $('tr.results_bid').each((_, tr) => {
25 + const $tr = $(tr);
26 + const a = $tr.find('a.item_title').first();
27 + const href = a.attr('href') ?? '';
28 + const id = href.match(/\/items\/([a-z0-9]+)/i)?.[1];
29 + const title = H.text(a);
30 + const priceTxt = H.text($tr.find('.item_price').first()) ?? '';
31 + const price = Number(priceTxt.replace(/[^\d]/g, ''));
32 + const bidsTxt = H.text($tr.find('td.results-bid').first());
33 + const bids = bidsTxt ? Number(bidsTxt.replace(/[^\d]/g, '')) : null;
34 + const endedOn = H.text($tr.find('td.results-limit').first()) ?? '';
35 + const img = $tr.find('.results_bid-image img').attr('data-src') ?? $tr.find('.results_bid-image img').attr('src') ?? null;
36 + if (!id || !title || !price || !endedOn) return;
37 + rows.push({ id, url: `${BASE}/items/${id}`, title, priceJpy: price, bids: Number.isFinite(bids as number) ? bids : null, endedOn, image: img });
38 + });
39 + return { kind: 'search_page', url, seed, page, rows };
40 +}
41 +
42 +/** "2026年9月6日" → UTC date */
43 +export function parseJapaneseDate(s: string): Date | null {
44 + const m = s.match(/(\d{4})年\s*(\d{1,2})月\s*(\d{1,2})日/);
45 + if (!m) return null;
46 + return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])));
47 +}
48 +
49 +/** Normalise "PSA10" / "PSA10" / "BGS9.5" so the shared grade parser can read them. */
50 +export function normaliseGradeText(title: string): string {
51 + return title
52 + .normalize('NFKC')
53 + .replace(/\b(PSA|BGS|CGC|SGC|ARS|ACE)\s*(\d{1,2}(?:\.\d)?)/gi, '$1 $2')
54 + .replace(/【|】|\[|\]/g, ' ');
55 +}
56 +
57 +const BUNDLE_RE = /まとめ|セット売り|大量|\d+\s*枚セット|\d+\s*点セット|おまとめ|引退品/;
58 +
59 +export class AucfreeConnector extends BaseConnector {
60 + readonly version = '1.0.0';
61 + readonly parserVersion = PARSER_VERSION;
62 + protected override minIntervalMs = 2000;
63 +
64 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
65 + const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []);
66 + const pages = Number(this.meta.config.pagesPerSeed ?? 2);
67 + const filter = ctx.options.categories;
68 + let count = 0;
69 + for (const seed of seeds) {
70 + if (filter?.length && !filter.includes(seed.category)) continue;
71 + for (let page = 1; page <= pages; page++) {
72 + if (ctx.signal?.aborted || this.reached(ctx, count)) return;
73 + const url = `${BASE}/search?o=t2&q=${encodeURIComponent(seed.q)}${page > 1 ? `&p=${page}` : ''}`;
74 + await this.throttle();
75 + const res = await ctx.fetch(url, {
76 + engines: ['firecrawl', 'scrapfly'],
77 + expect: ['title', 'price', 'date', 'status'],
78 + parse: (r) => {
79 + const p = r.html ? parseSearchPage(r.html, url, seed, page) : null;
80 + const row = p?.rows[0];
81 + return row ? { title: row.title, price: row.priceJpy, date: row.endedOn, status: 'sold' } : null;
82 + },
83 + });
84 + const payload = res.success && res.html ? parseSearchPage(res.html, url, seed, page) : null;
85 + if (!payload || payload.rows.length === 0) {
86 + ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
87 + break;
88 + }
89 + count++;
90 + yield { url, externalId: `search:${seed.q}:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
91 + }
92 + }
93 + }
94 +
95 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
96 + const p = PagePayloadSchema.parse(raw.payload);
97 + const out: NormalizedRecord[] = [];
98 + for (const r of p.rows) {
99 + const saleDate = parseJapaneseDate(r.endedOn);
100 + if (!saleDate) continue;
101 + const cleaned = normaliseGradeText(r.title);
102 + const g = parseGradeFromTitle(cleaned);
103 + const isBundle = BUNDLE_RE.test(r.title);
104 + const attributes = AssetAttributesSchema.parse({
105 + categorySlug: p.seed.category,
106 + name: r.title.normalize('NFKC').replace(/\s+/g, ' ').trim(),
107 + language: p.seed.language,
108 + country: 'JP',
109 + identifiers: { yahoo_auction_id: r.id },
110 + metadata: { seed_query: p.seed.q, bids: r.bids },
111 + });
112 + out.push(
113 + NormalizedSaleSchema.parse({
114 + kind: 'sale',
115 + connectorId: this.meta.id,
116 + sourceId: this.meta.sourceId,
117 + sourceUrl: r.url,
118 + externalId: r.id,
119 + rawTitle: r.title,
120 + imageUrls: r.image ? [r.image] : [],
121 + attributes,
122 + grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier, certificationNumber: null },
123 + condition: { condition: null, conditionRaw: null, completeness: null },
124 + observedAt: raw.fetchedAt,
125 + confidence: 0.7,
126 + parserVersion: PARSER_VERSION,
127 + saleType: 'auction',
128 + saleDate,
129 + price: r.priceJpy,
130 + currency: 'JPY',
131 + buyerPremiumIncluded: false,
132 + quantity: 1,
133 + isBundle,
134 + location: 'Japan',
135 + auctionHouse: 'Yahoo! Auctions Japan',
136 + lotNumber: r.id,
137 + }),
138 + );
139 + }
140 + return out;
141 + }
142 +}
143 +
144 +export default function createConnector(meta: ConnectorMeta) {
145 + return new AucfreeConnector(meta);
146 +}
added connectors/firecrawl/aucfree/meta.json +42 −0
@@ -0,0 +1,42 @@
1 +{
2 + "id": "aucfree",
3 + "displayName": "aucfree (Yahoo! Auctions Japan closed lots)",
4 + "sourceId": "aucfree",
5 + "sourceName": "aucfree — オークフリー",
6 + "sourceType": "analytics_provider",
7 + "sourceUrl": "https://aucfree.com",
8 + "module": "firecrawl/aucfree",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["pokemon", "yugioh", "one_piece_card_game", "gundam", "action_figures", "nintendo_games", "designer_toys"],
11 + "regions": ["JP"],
12 + "languages": ["ja"],
13 + "currency": ["JPY"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": true,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.75,
24 + "attributionRequired": true,
25 + "termsUrl": "https://aucfree.com/about",
26 + "accessNotes": "aucfree.com publishes closed Yahoo! Auctions Japan lots (end price, bid count, end date) free of charge; no robots.txt is served (404) so no path is disallowed. Plain HTTPS returns 403 to non-browser clients, so pages are fetched through Firecrawl (1 credit per search page of 50 lots ≈ 0.02 credit/sale) with Scrapfly as fallback. Search pages per keyword seed (`/search?o=t2&q=<kw>&p=N`, sorted by end date). Prices are JPY hammer prices without buyer premium; dates are the lot end dates (Japanese '2026年9月6日' format). Identification is title-only (Japanese titles) → confidence 0.7; PSA/BGS grades parsed from the title. Only market data is stored; item pages, seller and bidder data are not fetched.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "q": "ポケモンカード PSA10", "category": "pokemon", "language": "Japanese" },
32 + { "q": "ポケモンカード 旧裏 PSA", "category": "pokemon", "language": "Japanese" },
33 + { "q": "遊戯王 PSA10", "category": "yugioh", "language": "Japanese" },
34 + { "q": "ワンピースカード PSA10", "category": "one_piece_card_game", "language": "Japanese" },
35 + { "q": "ガンプラ MG 未組立", "category": "gundam", "language": null },
36 + { "q": "figma 新品", "category": "action_figures", "language": null },
37 + { "q": "ファミコン 未開封", "category": "nintendo_games", "language": null },
38 + { "q": "BE@RBRICK 1000%", "category": "designer_toys", "language": null }
39 + ],
40 + "pagesPerSeed": 2
41 + }
42 +}
added connectors/firecrawl/pcgs-priceguide/_smoke.ts +21 −0
@@ -0,0 +1,21 @@
1 +import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';
2 +import { saveFixture } from '@rareindex/connectors/testing';
3 +import createConnector from './index.js';
4 +
5 +const save = process.argv.includes('--save');
6 +const meta = getConnectorMeta('pcgs-priceguide');
7 +const connector = createConnector(meta);
8 +const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
9 +const ctx = createCrawlContext({ router, meta, options: { mode: 'backfill', limit: 1, seeds: ['morgan-dollar/744'] } });
10 +for await (const raw of connector.crawl(ctx)) {
11 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
12 + const p = raw.payload as { rows: unknown[]; grades: string[]; lastUpdate: string | null };
13 + console.log(`raw ${raw.externalId} (${raw.engine}) rows=${p.rows.length} grades=${p.grades.join(',')} lastUpdate=${p.lastUpdate} → ${out.length} records`);
14 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 330));
15 + if (save) {
16 + // keep the fixture small: first 12 rows
17 + const small = { ...(raw.payload as object), rows: p.rows.slice(0, 12) };
18 + saveFixture('pcgs-priceguide', 'morgan-dollar-ms', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: small }, expect: { minCount: 20, kinds: ['catalog_item', 'price_observation'], first: { 'attributes.categorySlug': 'coins', 'attributes.series': 'Morgan Dollar' } }, note: 'Captured live via Firecrawl; rows truncated to 12' });
19 + }
20 +}
21 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/pcgs-priceguide/index.test.ts +43 −0
@@ -0,0 +1,43 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { getConnectorMeta } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import createConnector, { lastUpdateDate, parseDescription } from './index.js';
5 +
6 +const connector = createConnector(getConnectorMeta('pcgs-priceguide'));
7 +
8 +describe('pcgs-priceguide', () => {
9 + runFixtureSuite(connector, it, expect);
10 +
11 + it('maps rows to coin catalog items + per-grade guide values', async () => {
12 + const fx = loadFixture('pcgs-priceguide', 'morgan-dollar-ms');
13 + const out = await connector.normalize(fx.raw);
14 + const cats = out.filter((r) => r.kind === 'catalog_item');
15 + expect(cats.length).toBe(12);
16 + const first = cats[0];
17 + if (first?.kind !== 'catalog_item') throw new Error();
18 + expect(first.attributes.identifiers.pcgs_number).toMatch(/^\d+$/);
19 + expect(first.attributes.year).toBeGreaterThanOrEqual(1878);
20 + expect(first.attributes.country).toBe('US');
21 + const obs = out.filter((r) => r.kind === 'price_observation');
22 + expect(obs.length).toBeGreaterThan(50);
23 + for (const o of obs) {
24 + if (o.kind !== 'price_observation') throw new Error();
25 + expect(o.grade.grader).toBe('pcgs');
26 + expect(o.grade.grade).toMatch(/^[A-Z]{2,4}\d{1,2}\+?$/);
27 + expect(o.priceKind).toBe('guide_value');
28 + expect(o.currency).toBe('USD');
29 + expect(o.observationDate.toISOString().slice(0, 10)).toBe('2026-09-06');
30 + }
31 + // plus grades are separate observations
32 + expect(obs.some((o) => o.kind === 'price_observation' && o.grade.grade?.endsWith('+'))).toBe(true);
33 + });
34 +
35 + it('helpers', () => {
36 + expect(parseDescription('1878 8TF')).toEqual({ year: 1878, mintMark: null, variety: '8TF' });
37 + expect(parseDescription('1893-S')).toEqual({ year: 1893, mintMark: 'S', variety: null });
38 + expect(parseDescription('1921-D')).toEqual({ year: 1921, mintMark: 'D', variety: null });
39 + expect(lastUpdateDate('09-06 10:59 PM EST', new Date('2026-09-07T05:00:00Z')).toISOString().slice(0, 10)).toBe('2026-09-06');
40 + expect(lastUpdateDate('12-30 10:59 PM EST', new Date('2026-01-02T05:00:00Z')).toISOString().slice(0, 10)).toBe('2025-12-30');
41 + expect(lastUpdateDate(null, new Date('2026-01-02T05:00:00Z')).toISOString()).toBe('2026-01-02T05:00:00.000Z');
42 + });
43 +});
added connectors/firecrawl/pcgs-priceguide/index.ts +199 −0
@@ -0,0 +1,199 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +/**
6 + * PCGS Price Guide — US coin series pages: one row per coin (PCGS #, description, designation)
7 + * with guide values per grade column. Raw payload = parsed table; normalise → catalog item +
8 + * one guide_value observation per (grade, plus-grade).
9 + */
10 +
11 +const BASE = 'https://www.pcgs.com';
12 +const PARSER_VERSION = '1.0.0';
13 +
14 +export const RowSchema = z.object({
15 + pcgsNumber: z.string(),
16 + description: z.string(),
17 + designation: z.string().nullable(),
18 + /** grade column label (e.g. "65") → { value, plus } */
19 + prices: z.record(z.string(), z.object({ value: z.number().nullable(), plus: z.number().nullable() })),
20 +});
21 +export const PagePayloadSchema = z.object({
22 + kind: z.literal('price_guide_page'),
23 + url: z.string(),
24 + seriesName: z.string(),
25 + seriesSlug: z.string(),
26 + seriesId: z.string(),
27 + lastUpdate: z.string().nullable(),
28 + grades: z.array(z.string()),
29 + rows: z.array(RowSchema),
30 +});
31 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
32 +
33 +function money(s: string | null): number | null {
34 + if (!s) return null;
35 + const n = Number(s.replace(/[^\d.]/g, ''));
36 + return Number.isFinite(n) && n > 0 ? n : null;
37 +}
38 +
39 +export function parsePriceGuidePage(htmlText: string, url: string): PagePayload | null {
40 + const $ = H.load(htmlText);
41 + const h1 = H.text($('h1').first()) ?? '';
42 + const seriesName = h1.replace(/\s*Price Guide$/i, '').trim();
43 + const m = url.match(/\/prices\/detail\/([^/]+)\/(\d+)/);
44 + if (!m || !seriesName) return null;
45 + const lastUpdate = $('main').text().match(/Last Update:\s*([0-9]{2}-[0-9]{2}\s+[0-9:]+\s*[AP]M\s*[A-Z]{3})/)?.[1] ?? null;
46 + // header: find the row containing "PCGS #" and numeric grade cells
47 + let grades: string[] = [];
48 + let table: ReturnType<typeof $> | null = null;
49 + $('table').each((_, t) => {
50 + const txt = $(t).text();
51 + if (/PCGS\s*#/.test(txt) && /Desig/.test(txt)) {
52 + table = $(t);
53 + return false;
54 + }
55 + return undefined;
56 + });
57 + if (!table) return null;
58 + const tbl = table as ReturnType<typeof $>;
59 + tbl.find('tr').each((_, tr) => {
60 + const cells = $(tr).find('th,td').map((__, c) => H.text($(c)) ?? '').get();
61 + if (cells.some((c) => /PCGS\s*#/.test(c))) {
62 + grades = cells.filter((c) => /^\d{1,2}$/.test(c));
63 + return false;
64 + }
65 + return undefined;
66 + });
67 + if (!grades.length) return null;
68 + const rows: z.infer<typeof RowSchema>[] = [];
69 + tbl.find('tr').each((_, tr) => {
70 + const $tr = $(tr);
71 + const numLink = $tr.find('a[href*="/coinfacts/coin/detail/"]').first();
72 + const pcgsNumber = (H.text(numLink) ?? '').trim();
73 + if (!/^\d+$/.test(pcgsNumber)) return;
74 + const tds = $tr.children('td');
75 + const descCell = tds.eq(1).clone();
76 + descCell.find('.hidden-print, a[data-func]').remove();
77 + const description = H.text(descCell) ?? '';
78 + const desigCell = tds.eq(2).clone();
79 + desigCell.find('br').replaceWith(' ');
80 + const designation = (H.text(desigCell) ?? '').replace(/\s*\+\s*$/, '').trim() || null;
81 + const prices: Record<string, { value: number | null; plus: number | null }> = {};
82 + const priceCells = tds.slice(3);
83 + grades.forEach((g, i) => {
84 + const cell = priceCells.eq(i);
85 + if (!cell.length) return;
86 + const anchors = cell.find('a');
87 + const value = money(H.text(anchors.eq(0)));
88 + const plus = anchors.length > 1 ? money(H.text(anchors.eq(1))) : null;
89 + if (value !== null || plus !== null) prices[g] = { value, plus };
90 + });
91 + if (description && Object.keys(prices).length) rows.push({ pcgsNumber, description, designation, prices });
92 + });
93 + return { kind: 'price_guide_page', url, seriesName, seriesSlug: m[1]!, seriesId: m[2]!, lastUpdate, grades, rows };
94 +}
95 +
96 +/** "09-06 10:59 PM EST" + fetch date → Date (year inferred from the fetch date, never after it). */
97 +export function lastUpdateDate(stamp: string | null, fetchedAt: Date): Date {
98 + const m = stamp?.match(/^(\d{2})-(\d{2})/);
99 + if (!m) return fetchedAt;
100 + let year = fetchedAt.getUTCFullYear();
101 + let d = new Date(Date.UTC(year, Number(m[1]) - 1, Number(m[2])));
102 + if (d.getTime() > fetchedAt.getTime() + 86_400_000) d = new Date(Date.UTC(--year, Number(m[1]) - 1, Number(m[2])));
103 + return d;
104 +}
105 +
106 +/** "1878 8TF" → { year: 1878, mintMark: null, variety: '8TF' }; "1893-S" → { year: 1893, mintMark: 'S' } */
107 +export function parseDescription(desc: string): { year: number | null; mintMark: string | null; variety: string | null } {
108 + const m = desc.match(/^(\d{4})(?:-([A-Z]{1,2}))?\s*(.*)$/);
109 + if (!m) return { year: null, mintMark: null, variety: desc || null };
110 + return { year: Number(m[1]), mintMark: m[2] ?? null, variety: m[3]?.trim() || null };
111 +}
112 +
113 +export class PcgsPriceGuideConnector extends BaseConnector {
114 + readonly version = '1.0.0';
115 + readonly parserVersion = PARSER_VERSION;
116 + protected override minIntervalMs = 2000;
117 +
118 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
119 + const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
120 + const desigs = (this.meta.config.designations as string[] | undefined) ?? ['ms'];
121 + let count = 0;
122 + for (const seed of seeds) {
123 + for (const d of desigs) {
124 + if (ctx.signal?.aborted || this.reached(ctx, count)) return;
125 + const url = `${BASE}/prices/detail/${seed}/most-active/${d}`;
126 + if (ctx.options.mode !== 'backfill' && !(await ctx.shouldFetch(url))) continue;
127 + await this.throttle();
128 + const res = await ctx.fetch(url, {
129 + engines: ['firecrawl', 'scrapfly'],
130 + expect: ['title', 'price', 'identifiers'],
131 + parse: (r) => {
132 + const p = r.html ? parsePriceGuidePage(r.html, url) : null;
133 + const row = p?.rows[0];
134 + return p && row ? { title: p.seriesName, price: Object.values(row.prices)[0]?.value ?? null, identifiers: { pcgs: row.pcgsNumber } } : null;
135 + },
136 + });
137 + const payload = res.success && res.html ? parsePriceGuidePage(res.html, url) : null;
138 + if (!payload || payload.rows.length === 0) {
139 + ctx.anomaly(payload ? 'empty_page' : 'page_parse_failed', `${url}: ${res.error ?? res.httpStatus}`);
140 + continue;
141 + }
142 + count++;
143 + yield { url, externalId: `series:${payload.seriesId}:${d}`, kind: 'price_observation', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
144 + }
145 + }
146 + }
147 +
148 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
149 + const p = PagePayloadSchema.parse(raw.payload);
150 + const obsDate = lastUpdateDate(p.lastUpdate, raw.fetchedAt);
151 + const out: NormalizedRecord[] = [];
152 + for (const row of p.rows) {
153 + const { year, mintMark, variety } = parseDescription(row.description);
154 + const desig = row.designation?.split(/\s+/)[0]?.toUpperCase() ?? null; // MS | PR | SP
155 + const name = `${row.description} ${p.seriesName}`.trim();
156 + const rawTitle = `${row.description} ${p.seriesName}${desig ? ` (${desig})` : ''} · PCGS #${row.pcgsNumber}`;
157 + const attributes = AssetAttributesSchema.parse({
158 + categorySlug: 'coins',
159 + brand: 'United States Mint',
160 + series: p.seriesName,
161 + set: p.seriesName,
162 + name,
163 + number: row.pcgsNumber,
164 + year,
165 + variant: [mintMark ? `${mintMark} mint` : null, variety, desig && desig !== 'MS' ? desig : null].filter(Boolean).join(' · ') || null,
166 + country: 'US',
167 + identifiers: { pcgs_number: row.pcgsNumber },
168 + metadata: { mint_mark: mintMark, variety, designation: desig, pcgs_series_id: p.seriesId, pcgs_series_slug: p.seriesSlug },
169 + });
170 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${BASE}/coinfacts/coin/detail/${row.pcgsNumber}`, rawTitle, imageUrls: [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
171 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `coin:${row.pcgsNumber}`, confidence: 0.95 }));
172 + for (const [g, pr] of Object.entries(row.prices)) {
173 + for (const [suffix, value] of [['', pr.value], ['+', pr.plus]] as const) {
174 + if (!value) continue;
175 + const grade = `${desig ?? 'MS'}${g}${suffix}`;
176 + out.push(
177 + NormalizedPriceObservationSchema.parse({
178 + kind: 'price_observation',
179 + ...base,
180 + externalId: `coin:${row.pcgsNumber}:${grade}`,
181 + confidence: 0.85,
182 + grade: { grader: 'pcgs', grade, qualifier: null, certificationNumber: null },
183 + priceKind: 'guide_value',
184 + price: value,
185 + currency: 'USD',
186 + observationDate: obsDate,
187 + sampleSize: null,
188 + }),
189 + );
190 + }
191 + }
192 + }
193 + return out;
194 + }
195 +}
196 +
197 +export default function createConnector(meta: ConnectorMeta) {
198 + return new PcgsPriceGuideConnector(meta);
199 +}
added connectors/firecrawl/pcgs-priceguide/meta.json +35 −0
@@ -0,0 +1,35 @@
1 +{
2 + "id": "pcgs-priceguide",
3 + "displayName": "PCGS Price Guide (US coins)",
4 + "sourceId": "pcgs",
5 + "sourceName": "PCGS",
6 + "sourceType": "grading_company",
7 + "sourceUrl": "https://www.pcgs.com",
8 + "module": "firecrawl/pcgs-priceguide",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["coins"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": false,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 10080,
22 + "priority": "low",
23 + "trustScore": 0.9,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.pcgs.com/legal",
26 + "accessNotes": "Public PCGS Price Guide category pages (pcgs.com/prices/detail/<series>/<id>/most-active) — no login required; robots.txt does not restrict /prices. Plain HTTPS is refused by the CDN (403) so pages are rendered through Firecrawl (1 credit per category page ≈ 100 coins × 10 grades). Values are PCGS retail guide values in USD per grade (columns 4…70 and '+' grades), dated with the page's 'Last Update' stamp. Stored as guide_value observations with grader 'pcgs' and grade like MS65 / PR65 / MS65+; PCGS coin numbers are kept as identifiers (pcgs_number). Auction Prices Realized and the Population Report on pcgs.com require a Collectors account and are not fetched.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + "morgan-dollar/744", "peace-dollar/26", "lincoln-cent-wheat-reverse/46", "indian-cent/44", "buffalo-nickel/83", "mercury-dime/703", "walking-liberty-half-dollar/733", "standing-liberty-quarter/111", "franklin-half-dollar/734", "kennedy-half-dollar/125", "washington-quarter/112", "trade-dollar/743", "flying-eagle-cent/664", "barber-half-dollar/732", "silver-eagles/939", "liberty-seated-dollar/29"
32 + ],
33 + "designations": ["ms"]
34 + }
35 +}
modified connectors/registry.json +617 −16
@@ -1,6 +1,97 @@
1 1 {
2 2 "version": "1.0",
3 3 "connectors": [
4 + {
5 + "id": "aucfree",
6 + "displayName": "aucfree (Yahoo! Auctions Japan closed lots)",
7 + "sourceId": "aucfree",
8 + "sourceName": "aucfree \u2014 \u30aa\u30fc\u30af\u30d5\u30ea\u30fc",
9 + "sourceType": "analytics_provider",
10 + "sourceUrl": "https://aucfree.com",
11 + "module": "firecrawl/aucfree",
12 + "enginePriority": [
13 + "firecrawl",
14 + "scrapfly"
15 + ],
16 + "categories": [
17 + "pokemon",
18 + "yugioh",
19 + "one_piece_card_game",
20 + "gundam",
21 + "action_figures",
22 + "nintendo_games",
23 + "designer_toys"
24 + ],
25 + "regions": [
26 + "JP"
27 + ],
28 + "languages": [
29 + "ja"
30 + ],
31 + "currency": [
32 + "JPY"
33 + ],
34 + "supportsListings": false,
35 + "supportsSold": true,
36 + "supportsAuctions": true,
37 + "supportsImages": true,
38 + "supportsCatalog": false,
39 + "supportsPopulation": false,
40 + "supportsLookup": false,
41 + "refreshFrequencyMinutes": 720,
42 + "priority": "medium",
43 + "trustScore": 0.75,
44 + "attributionRequired": true,
45 + "termsUrl": "https://aucfree.com/about",
46 + "accessNotes": "aucfree.com publishes closed Yahoo! Auctions Japan lots (end price, bid count, end date) free of charge; no robots.txt is served (404) so no path is disallowed. Plain HTTPS returns 403 to non-browser clients, so pages are fetched through Firecrawl (1 credit per search page of 50 lots \u2248 0.02 credit/sale) with Scrapfly as fallback. Search pages per keyword seed (`/search?o=t2&q=<kw>&p=N`, sorted by end date). Prices are JPY hammer prices without buyer premium; dates are the lot end dates (Japanese '2026\u5e749\u67086\u65e5' format). Identification is title-only (Japanese titles) \u2192 confidence 0.7; PSA/BGS grades parsed from the title. Only market data is stored; item pages, seller and bidder data are not fetched.",
47 + "enabled": true,
48 + "schemaVersion": "1.0",
49 + "config": {
50 + "seeds": [
51 + {
52 + "q": "\u30dd\u30b1\u30e2\u30f3\u30ab\u30fc\u30c9 PSA10",
53 + "category": "pokemon",
54 + "language": "Japanese"
55 + },
56 + {
57 + "q": "\u30dd\u30b1\u30e2\u30f3\u30ab\u30fc\u30c9 \u65e7\u88cf PSA",
58 + "category": "pokemon",
59 + "language": "Japanese"
60 + },
61 + {
62 + "q": "\u904a\u622f\u738b PSA10",
63 + "category": "yugioh",
64 + "language": "Japanese"
65 + },
66 + {
67 + "q": "\u30ef\u30f3\u30d4\u30fc\u30b9\u30ab\u30fc\u30c9 PSA10",
68 + "category": "one_piece_card_game",
69 + "language": "Japanese"
70 + },
71 + {
72 + "q": "\u30ac\u30f3\u30d7\u30e9 MG \u672a\u7d44\u7acb",
73 + "category": "gundam",
74 + "language": null
75 + },
76 + {
77 + "q": "figma \u65b0\u54c1",
78 + "category": "action_figures",
79 + "language": null
80 + },
81 + {
82 + "q": "\u30d5\u30a1\u30df\u30b3\u30f3 \u672a\u958b\u5c01",
83 + "category": "nintendo_games",
84 + "language": null
85 + },
86 + {
87 + "q": "BE@RBRICK 1000%",
88 + "category": "designer_toys",
89 + "language": null
90 + }
91 + ],
92 + "pagesPerSeed": 2
93 + }
94 + },
4 95 {
5 96 "id": "brickeconomy",
6 97 "displayName": "BrickEconomy",
@@ -38,7 +129,7 @@
38 129 "trustScore": 0.65,
39 130 "attributionRequired": true,
40 131 "termsUrl": "https://www.brickeconomy.com/legal-terms",
41 − "accessNotes": "Public set pages fetched through Firecrawl (markdown; ~1 credit/page). robots.txt allows all crawlers (explicitly including AI bots). Plain HTTPS returns 403 for non-browser agents, so Firecrawl is the primary engine and Scrapfly the fallback. We capture set facts (number, name, theme, subtheme, year, release/retire dates, pieces, minifigs), retail price, and BrickEconomy's estimated New/Sealed and Used values as guide observations — never as sales. Discovery via theme/top lists in config.seeds.",
132 + "accessNotes": "Public set pages fetched through Firecrawl (markdown; ~1 credit/page). robots.txt allows all crawlers (explicitly including AI bots). Plain HTTPS returns 403 for non-browser agents, so Firecrawl is the primary engine and Scrapfly the fallback. We capture set facts (number, name, theme, subtheme, year, release/retire dates, pieces, minifigs), retail price, and BrickEconomy's estimated New/Sealed and Used values as guide observations \u2014 never as sales. Discovery via theme/top lists in config.seeds.",
42 133 "enabled": true,
43 134 "schemaVersion": "1.0",
44 135 "config": {
@@ -55,6 +146,69 @@
55 146 "setsPerSeed": 60
56 147 }
57 148 },
149 + {
150 + "id": "brickset",
151 + "displayName": "Brickset (LEGO set catalog + current values)",
152 + "sourceId": "brickset",
153 + "sourceName": "Brickset",
154 + "sourceType": "catalog",
155 + "sourceUrl": "https://brickset.com",
156 + "module": "api/brickset",
157 + "enginePriority": [
158 + "api",
159 + "firecrawl"
160 + ],
161 + "categories": [
162 + "lego_sets",
163 + "lego"
164 + ],
165 + "regions": [
166 + "global"
167 + ],
168 + "languages": [
169 + "en"
170 + ],
171 + "currency": [
172 + "USD",
173 + "GBP"
174 + ],
175 + "supportsListings": false,
176 + "supportsSold": false,
177 + "supportsAuctions": false,
178 + "supportsImages": true,
179 + "supportsCatalog": true,
180 + "supportsPopulation": false,
181 + "supportsLookup": true,
182 + "refreshFrequencyMinutes": 1440,
183 + "priority": "medium",
184 + "trustScore": 0.85,
185 + "attributionRequired": true,
186 + "termsUrl": "https://brickset.com/about",
187 + "accessNotes": "Public set pages (brickset.com/sets/<number>-1/...) and theme listings (/sets/theme-<Theme>/page-N, 25 sets per page) fetched over plain HTTPS with the RareIndex user agent; robots.txt disallows /admin, /export, /ajax, /profile, /webservices, /buy, /news, /reviews\u2026 \u2014 none of which are used (the Brickset API/webservices need a personal key: https://brickset.com/tools/webservices/requestkey). Fields parsed from the set page definition list: pieces, minifigs, RRP (GBP + USD), launch/exit dates, availability, packaging, barcodes (UPC/EAN), theme/subtheme and the 'Current value' New/Used estimates that Brickset derives from BrickLink \u2014 stored as guide_value observations (USD, confidence 0.65). 1.5 s politeness delay.",
188 + "enabled": true,
189 + "schemaVersion": "1.0",
190 + "config": {
191 + "seeds": [
192 + "theme-Star-Wars",
193 + "theme-Icons",
194 + "theme-Creator-Expert",
195 + "theme-Ideas",
196 + "theme-Technic",
197 + "theme-Harry-Potter",
198 + "theme-Marvel-Super-Heroes",
199 + "theme-Architecture",
200 + "theme-Ninjago",
201 + "theme-Castle",
202 + "theme-Space",
203 + "theme-Pirates",
204 + "theme-Indiana-Jones",
205 + "theme-The-Lord-of-the-Rings",
206 + "theme-DC-Comics-Super-Heroes"
207 + ],
208 + "pagesPerSeed": 2,
209 + "setsPerRun": 120
210 + }
211 + },
58 212 {
59 213 "id": "chrono24",
60 214 "displayName": "Chrono24",
@@ -96,7 +250,7 @@
96 250 "trustScore": 0.7,
97 251 "attributionRequired": true,
98 252 "termsUrl": "https://www.chrono24.com/info/terms-of-use.htm",
99 − "accessNotes": "Public model listing pages (e.g. /rolex/daytona--mod2.htm) fetched through Scrapfly without JS rendering (Cloudflare shield → ~40 credits/page); robots.txt allows these paths (Crawl-delay 0.1). Each page embeds a schema.org ItemList of Offers (name, price in the site currency, image, listing URL) — we store asking prices as listings only, never as sales. No login, no per-listing detail fetch (seller details stay on Chrono24). Model pages are configured in config.seeds; pagination via --modN-P.htm.",
253 + "accessNotes": "Public model listing pages (e.g. /rolex/daytona--mod2.htm) fetched through Scrapfly without JS rendering (Cloudflare shield \u2192 ~40 credits/page); robots.txt allows these paths (Crawl-delay 0.1). Each page embeds a schema.org ItemList of Offers (name, price in the site currency, image, listing URL) \u2014 we store asking prices as listings only, never as sales. No login, no per-listing detail fetch (seller details stay on Chrono24). Model pages are configured in config.seeds; pagination via --modN-P.htm.",
100 254 "enabled": true,
101 255 "schemaVersion": "1.0",
102 256 "config": {
@@ -162,7 +316,7 @@
162 316 "trustScore": 0.9,
163 317 "attributionRequired": true,
164 318 "termsUrl": "https://www.comicconnect.com/terms",
165 − "accessNotes": "Public sold archive (/browse/comics/?filtertype=Sold, ~479k results, 20 per page) fetched over plain HTTPS with the RareIndex user agent. robots.txt: Allow / with Content-Signal search=yes, ai-train=no (we only index sale facts and link back). Each card exposes title, publisher + grade, sold date/time, 'Sold For' price and whether a 15% buyer's premium applies (bp attribute) — we store hammer price with buyer_premium_included=false and keep the premium note in metadata. No login, no bidder data. 1.5 s politeness delay.",
319 + "accessNotes": "Public sold archive (/browse/comics/?filtertype=Sold, ~479k results, 20 per page) fetched over plain HTTPS with the RareIndex user agent. robots.txt: Allow / with Content-Signal search=yes, ai-train=no (we only index sale facts and link back). Each card exposes title, publisher + grade, sold date/time, 'Sold For' price and whether a 15% buyer's premium applies (bp attribute) \u2014 we store hammer price with buyer_premium_included=false and keep the premium note in metadata. No login, no bidder data. 1.5 s politeness delay.",
166 320 "enabled": true,
167 321 "schemaVersion": "1.0",
168 322 "config": {
@@ -171,6 +325,84 @@
171 325 "sortType": "ended_desc"
172 326 }
173 327 },
328 + {
329 + "id": "discogs",
330 + "displayName": "Discogs (releases + marketplace lows)",
331 + "sourceId": "discogs",
332 + "sourceName": "Discogs",
333 + "sourceType": "collector_database",
334 + "sourceUrl": "https://www.discogs.com",
335 + "module": "api/discogs",
336 + "enginePriority": [
337 + "api"
338 + ],
339 + "categories": [
340 + "music"
341 + ],
342 + "regions": [
343 + "global"
344 + ],
345 + "languages": [
346 + "en"
347 + ],
348 + "currency": [
349 + "USD"
350 + ],
351 + "supportsListings": false,
352 + "supportsSold": false,
353 + "supportsAuctions": false,
354 + "supportsImages": true,
355 + "supportsCatalog": true,
356 + "supportsPopulation": false,
357 + "supportsLookup": true,
358 + "refreshFrequencyMinutes": 1440,
359 + "priority": "medium",
360 + "trustScore": 0.85,
361 + "attributionRequired": true,
362 + "termsUrl": "https://support.discogs.com/hc/en-us/articles/360009334593-API-Terms-of-Use",
363 + "accessNotes": "Official public Discogs API (api.discogs.com) used unauthenticated with the RareIndex user agent: database/search (type=master), masters/{id}/versions (pressings with label, catno, country, year, format and community have/want counts) and marketplace/stats/{release_id} (lowest asking price + number for sale). Unauthenticated limit is 25 requests/minute \u2192 2.6 s throttle; price_suggestions requires a personal token and is not used. Marketplace stats have no timestamp: observationDate = fetch date (confidence 0.7). Catalog data is CC0; images are hot-linked thumbnails and attributed to Discogs. Sold-price history is not exposed publicly.",
364 + "enabled": true,
365 + "schemaVersion": "1.0",
366 + "config": {
367 + "seeds": [
368 + "Nirvana Nevermind",
369 + "The Beatles Abbey Road",
370 + "Pink Floyd The Dark Side Of The Moon",
371 + "Led Zeppelin IV",
372 + "Miles Davis Kind Of Blue",
373 + "Michael Jackson Thriller",
374 + "Radiohead OK Computer",
375 + "Fleetwood Mac Rumours",
376 + "Daft Punk Random Access Memories",
377 + "Kendrick Lamar To Pimp A Butterfly",
378 + "The Velvet Underground & Nico",
379 + "Joy Division Unknown Pleasures",
380 + "Prince Purple Rain",
381 + "David Bowie The Rise And Fall Of Ziggy Stardust",
382 + "Amy Winehouse Back To Black",
383 + "The Beatles Sgt. Pepper's Lonely Hearts Club Band",
384 + "Bob Dylan Blonde On Blonde",
385 + "John Coltrane A Love Supreme",
386 + "The Rolling Stones Exile On Main St.",
387 + "Wu-Tang Clan Enter The Wu-Tang (36 Chambers)",
388 + "Taylor Swift 1989",
389 + "Tyler, The Creator Igor",
390 + "Frank Ocean Blonde",
391 + "Arctic Monkeys AM",
392 + "Bj\u00f6rk Homogenic",
393 + "Kraftwerk Autobahn",
394 + "Sex Pistols Never Mind The Bollocks",
395 + "Bruce Springsteen Born To Run",
396 + "Nas Illmatic",
397 + "Massive Attack Mezzanine"
398 + ],
399 + "formats": [
400 + "Vinyl"
401 + ],
402 + "versionsPerMaster": 100,
403 + "statsPerMaster": 6
404 + }
405 + },
174 406 {
175 407 "id": "goldin",
176 408 "displayName": "Goldin (sold auction results)",
@@ -219,7 +451,7 @@
219 451 "trustScore": 0.9,
220 452 "attributionRequired": true,
221 453 "termsUrl": "https://goldin.co/useragreement",
222 − "accessNotes": "Public 'Sold Items' result grids (https://goldin.co/buy/sc/<subcategory>?show_only=Sold%20Items&sort=Most_Recent_Bids&number_of_lots=240) rendered through Scrapfly (asp + JS rendering, ~6 credits per page of 240 sold lots). goldin.co/robots.txt allows /buy/ and /item/ and disallows /api/; we never call goldin.co/api ourselves — the lot data is read from the XHR (lots_v2 on their CloudFront search endpoint) that the public page performs during rendering, exactly what a browser does. Plain HTTP and Firecrawl return the empty SPA shell / an Akamai challenge. Prices: the grid shows the final price INCLUDING buyer's premium (current_price × (1 + buyer_premium%)); we store that as the sale price with buyerPremiumIncluded=true and keep the hammer price and BP % in metadata. Sale date = lot end_timestamp; lots whose end_timestamp is in the future (placeholder private-sale dates like 2235-06-04) are skipped and counted as anomalies. Cert numbers appear only in item-page descriptions (lookup).",
454 + "accessNotes": "Public 'Sold Items' result grids (https://goldin.co/buy/sc/<subcategory>?show_only=Sold%20Items&sort=Most_Recent_Bids&number_of_lots=240) rendered through Scrapfly (asp + JS rendering, ~6 credits per page of 240 sold lots). goldin.co/robots.txt allows /buy/ and /item/ and disallows /api/; we never call goldin.co/api ourselves \u2014 the lot data is read from the XHR (lots_v2 on their CloudFront search endpoint) that the public page performs during rendering, exactly what a browser does. Plain HTTP and Firecrawl return the empty SPA shell / an Akamai challenge. Prices: the grid shows the final price INCLUDING buyer's premium (current_price \u00d7 (1 + buyer_premium%)); we store that as the sale price with buyerPremiumIncluded=true and keep the hammer price and BP % in metadata. Sale date = lot end_timestamp; lots whose end_timestamp is in the future (placeholder private-sale dates like 2235-06-04) are skipped and counted as anomalies. Cert numbers appear only in item-page descriptions (lookup).",
223 455 "enabled": true,
224 456 "schemaVersion": "1.0",
225 457 "config": {
@@ -284,6 +516,212 @@
284 516 "renderingWaitMs": 6000
285 517 }
286 518 },
519 + {
520 + "id": "lorcast",
521 + "displayName": "Lorcast (Disney Lorcana)",
522 + "sourceId": "lorcast",
523 + "sourceName": "Lorcast",
524 + "sourceType": "catalog",
525 + "sourceUrl": "https://lorcast.com",
526 + "module": "api/lorcast",
527 + "enginePriority": [
528 + "api"
529 + ],
530 + "categories": [
531 + "disney_lorcana"
532 + ],
533 + "regions": [
534 + "US",
535 + "EU"
536 + ],
537 + "languages": [
538 + "en"
539 + ],
540 + "currency": [
541 + "USD"
542 + ],
543 + "supportsListings": false,
544 + "supportsSold": false,
545 + "supportsAuctions": false,
546 + "supportsImages": true,
547 + "supportsCatalog": true,
548 + "supportsPopulation": false,
549 + "supportsLookup": false,
550 + "refreshFrequencyMinutes": 1440,
551 + "priority": "medium",
552 + "trustScore": 0.75,
553 + "attributionRequired": true,
554 + "termsUrl": "https://lorcast.com/docs/api",
555 + "accessNotes": "Open Lorcast REST API (v0, no key): /sets then /cards/search?q=set:<code>&page=N. Cards carry TCGplayer ids and USD market prices (normal / foil) without a timestamp, so observations are dated by the fetch day with confidence 0.7. Enchanted cards are emitted as a single 'Enchanted' variant. ~4 req/s self-throttled.",
556 + "enabled": true,
557 + "schemaVersion": "1.0",
558 + "config": {}
559 + },
560 + {
561 + "id": "novelship",
562 + "displayName": "Novelship (sneaker catalog, last sale & lowest ask)",
563 + "sourceId": "novelship",
564 + "sourceName": "Novelship",
565 + "sourceType": "marketplace",
566 + "sourceUrl": "https://novelship.com",
567 + "module": "api/novelship",
568 + "enginePriority": [
569 + "api",
570 + "firecrawl",
571 + "scrapfly"
572 + ],
573 + "categories": [
574 + "sneakers",
575 + "nike_jordan",
576 + "adidas_yeezy",
577 + "new_balance_asics_other"
578 + ],
579 + "regions": [
580 + "SG",
581 + "AU",
582 + "NZ",
583 + "TW",
584 + "HK",
585 + "MY",
586 + "JP",
587 + "US"
588 + ],
589 + "languages": [
590 + "en"
591 + ],
592 + "currency": [
593 + "USD"
594 + ],
595 + "supportsListings": true,
596 + "supportsSold": false,
597 + "supportsAuctions": false,
598 + "supportsImages": true,
599 + "supportsCatalog": true,
600 + "supportsPopulation": false,
601 + "supportsLookup": true,
602 + "refreshFrequencyMinutes": 720,
603 + "priority": "medium",
604 + "trustScore": 0.7,
605 + "attributionRequired": true,
606 + "termsUrl": "https://novelship.com/terms",
607 + "accessNotes": "Public Novelship browse pages (novelship.com/sneakers/<brand>?page=N) and product pages fetched over plain HTTPS with the RareIndex user agent; robots.txt only disallows sell/auth/dashboard/pay paths. The server-rendered payload embeds each product's SKU (style code), colorway, retail cost, release date, last sale price, lowest listing price and 180-day sales count \u2014 no login or private endpoint is used. Prices are read as USD (the anonymous international storefront prices in US$; cost_retail matches US retail) with confidence 0.7; per-size asks are not exposed on listing pages. Sizes/stock are not fetched. 1.5 s politeness delay.",
608 + "enabled": true,
609 + "schemaVersion": "1.0",
610 + "config": {
611 + "seeds": [
612 + "jordan",
613 + "nike",
614 + "adidas",
615 + "new-balance",
616 + "asics",
617 + "yeezy"
618 + ],
619 + "pagesPerSeed": 3
620 + }
621 + },
622 + {
623 + "id": "optcg",
624 + "displayName": "OPTCG API (One Piece Card Game)",
625 + "sourceId": "optcgapi",
626 + "sourceName": "OPTCG API",
627 + "sourceType": "catalog",
628 + "sourceUrl": "https://optcgapi.com",
629 + "module": "api/optcg",
630 + "enginePriority": [
631 + "api"
632 + ],
633 + "categories": [
634 + "one_piece_card_game"
635 + ],
636 + "regions": [
637 + "US"
638 + ],
639 + "languages": [
640 + "en"
641 + ],
642 + "currency": [
643 + "USD"
644 + ],
645 + "supportsListings": false,
646 + "supportsSold": false,
647 + "supportsAuctions": false,
648 + "supportsImages": true,
649 + "supportsCatalog": true,
650 + "supportsPopulation": false,
651 + "supportsLookup": false,
652 + "refreshFrequencyMinutes": 1440,
653 + "priority": "medium",
654 + "trustScore": 0.65,
655 + "attributionRequired": true,
656 + "termsUrl": "https://optcgapi.com/",
657 + "accessNotes": "Open community API (no key): /api/allSets/ then /api/sets/<set_id>/ returning every card of the set with TCGplayer-derived market_price / inventory_price and a date_scraped, used as the observation date (confidence 0.7 / 0.65). Alternate arts share the card code and are distinguished by image id. One request per set, 0.5 s politeness.",
658 + "enabled": true,
659 + "schemaVersion": "1.0",
660 + "config": {}
661 + },
662 + {
663 + "id": "pcgs-priceguide",
664 + "displayName": "PCGS Price Guide (US coins)",
665 + "sourceId": "pcgs",
666 + "sourceName": "PCGS",
667 + "sourceType": "grading_company",
668 + "sourceUrl": "https://www.pcgs.com",
669 + "module": "firecrawl/pcgs-priceguide",
670 + "enginePriority": [
671 + "firecrawl",
672 + "scrapfly"
673 + ],
674 + "categories": [
675 + "coins"
676 + ],
677 + "regions": [
678 + "US"
679 + ],
680 + "languages": [
681 + "en"
682 + ],
683 + "currency": [
684 + "USD"
685 + ],
686 + "supportsListings": false,
687 + "supportsSold": false,
688 + "supportsAuctions": false,
689 + "supportsImages": false,
690 + "supportsCatalog": true,
691 + "supportsPopulation": false,
692 + "supportsLookup": false,
693 + "refreshFrequencyMinutes": 10080,
694 + "priority": "low",
695 + "trustScore": 0.9,
696 + "attributionRequired": true,
697 + "termsUrl": "https://www.pcgs.com/legal",
698 + "accessNotes": "Public PCGS Price Guide category pages (pcgs.com/prices/detail/<series>/<id>/most-active) \u2014 no login required; robots.txt does not restrict /prices. Plain HTTPS is refused by the CDN (403) so pages are rendered through Firecrawl (1 credit per category page \u2248 100 coins \u00d7 10 grades). Values are PCGS retail guide values in USD per grade (columns 4\u202670 and '+' grades), dated with the page's 'Last Update' stamp. Stored as guide_value observations with grader 'pcgs' and grade like MS65 / PR65 / MS65+; PCGS coin numbers are kept as identifiers (pcgs_number). Auction Prices Realized and the Population Report on pcgs.com require a Collectors account and are not fetched.",
699 + "enabled": true,
700 + "schemaVersion": "1.0",
701 + "config": {
702 + "seeds": [
703 + "morgan-dollar/744",
704 + "peace-dollar/26",
705 + "lincoln-cent-wheat-reverse/46",
706 + "indian-cent/44",
707 + "buffalo-nickel/83",
708 + "mercury-dime/703",
709 + "walking-liberty-half-dollar/733",
710 + "standing-liberty-quarter/111",
711 + "franklin-half-dollar/734",
712 + "kennedy-half-dollar/125",
713 + "washington-quarter/112",
714 + "trade-dollar/743",
715 + "flying-eagle-cent/664",
716 + "barber-half-dollar/732",
717 + "silver-eagles/939",
718 + "liberty-seated-dollar/29"
719 + ],
720 + "designations": [
721 + "ms"
722 + ]
723 + }
724 + },
287 725 {
288 726 "id": "phillips-watches",
289 727 "displayName": "Phillips Watches (auction results)",
@@ -332,7 +770,7 @@
332 770 "trustScore": 0.95,
333 771 "attributionRequired": true,
334 772 "termsUrl": "https://www.phillips.com/about/terms",
335 − "accessNotes": "Past watch sales are discovered from the public /auctions/past page (plain HTTPS; embedded JSON with saleNumber, title, end date, location). Each sale page is rendered through Firecrawl (JS wait ~6 s, 1 credit) to read the public lot list: lot number, maker, reference, model, estimate and 'Sold For' (Phillips publishes prices realised including buyer's premium). robots.txt disallows only /search and filter paths. No login, no bidder data. Large sales may lazy-load beyond the first ~70 lots — coverage is recorded per run.",
773 + "accessNotes": "Past watch sales are discovered from the public /auctions/past page (plain HTTPS; embedded JSON with saleNumber, title, end date, location). Each sale page is rendered through Firecrawl (JS wait ~6 s, 1 credit) to read the public lot list: lot number, maker, reference, model, estimate and 'Sold For' (Phillips publishes prices realised including buyer's premium). robots.txt disallows only /search and filter paths. No login, no bidder data. Large sales may lazy-load beyond the first ~70 lots \u2014 coverage is recorded per run.",
336 774 "enabled": true,
337 775 "schemaVersion": "1.0",
338 776 "config": {
@@ -343,9 +781,9 @@
343 781 },
344 782 {
345 783 "id": "pokemontcg",
346 − "displayName": "Pokémon TCG API (pokemontcg.io)",
784 + "displayName": "Pok\u00e9mon TCG API (pokemontcg.io)",
347 785 "sourceId": "pokemontcg",
348 − "sourceName": "Pokémon TCG API",
786 + "sourceName": "Pok\u00e9mon TCG API",
349 787 "sourceType": "catalog",
350 788 "sourceUrl": "https://pokemontcg.io",
351 789 "module": "api/pokemontcg",
@@ -379,7 +817,7 @@
379 817 "trustScore": 0.8,
380 818 "attributionRequired": true,
381 819 "termsUrl": "https://docs.pokemontcg.io/",
382 − "accessNotes": "Public REST API v2 (https://api.pokemontcg.io/v2). Optional X-Api-Key (POKEMONTCG_API_KEY) raises limits (20k/day) — without a key ~1k/day and 30/min. Crawl = sets, then cards per set (pageSize 250, cursor {setIndex,page}). The API is intermittently unavailable (5xx); the connector retries with backoff and falls back to the maintainers' public GitHub mirror (PokemonTCG/pokemon-tcg-data, same card schema, no prices) for the catalog when the API keeps failing. Prices are TCGplayer (USD, per printing variant: normal/holofoil/reverseHolofoil/1stEdition…) and Cardmarket (EUR) daily aggregates with their own updatedAt → stored as price_observations, never as sales. Pokémon © Nintendo/Creatures/GAME FREAK; data attributed to pokemontcg.io.",
820 + "accessNotes": "Public REST API v2 (https://api.pokemontcg.io/v2). Optional X-Api-Key (POKEMONTCG_API_KEY) raises limits (20k/day) \u2014 without a key ~1k/day and 30/min. Crawl = sets, then cards per set (pageSize 250, cursor {setIndex,page}). The API is intermittently unavailable (5xx); the connector retries with backoff and falls back to the maintainers' public GitHub mirror (PokemonTCG/pokemon-tcg-data, same card schema, no prices) for the catalog when the API keeps failing. Prices are TCGplayer (USD, per printing variant: normal/holofoil/reverseHolofoil/1stEdition\u2026) and Cardmarket (EUR) daily aggregates with their own updatedAt \u2192 stored as price_observations, never as sales. Pok\u00e9mon \u00a9 Nintendo/Creatures/GAME FREAK; data attributed to pokemontcg.io.",
383 821 "enabled": true,
384 822 "schemaVersion": "1.0",
385 823 "config": {
@@ -414,7 +852,11 @@
414 852 "comics",
415 853 "marvel_comics",
416 854 "dc_comics",
417 − "independent_comics"
855 + "independent_comics",
856 + "trading_cards",
857 + "pokemon",
858 + "magic_the_gathering",
859 + "yugioh"
418 860 ],
419 861 "regions": [
420 862 "US"
@@ -437,9 +879,9 @@
437 879 "trustScore": 0.7,
438 880 "attributionRequired": true,
439 881 "termsUrl": "https://www.pricecharting.com/page/terms-of-service",
440 − "accessNotes": "Public product pages fetched over plain HTTPS with the RareIndex user agent (robots.txt only disallows /buy, /publish-offer, /stripe-connect). Each product page exposes PriceCharting guide values per condition and a table of recently completed eBay sales (date, title, price, eBay item id) — we store those as sales with the PriceCharting page as source URL. No account, no API key, no photos (TimeWarp is paid and not used). Console listings paginate through the public ?format=json cursor endpoint. ~1 request per product, 1.5 s politeness delay.",
882 + "accessNotes": "Public product pages fetched over plain HTTPS with the RareIndex user agent (robots.txt only disallows /buy, /publish-offer, /stripe-connect). Each product page exposes PriceCharting guide values per condition and a table of recently completed eBay sales (date, title, price, eBay item id) \u2014 we store those as sales with the PriceCharting page as source URL. No account, no API key, no photos (TimeWarp is paid and not used). Console listings paginate through the public ?format=json cursor endpoint. ~1 request per product, 1.5 s politeness delay. Trading-card consoles (Pok\u00e9mon, Magic, Yu-Gi-Oh!) are discovered from the public /category/<game>-cards pages; card pages expose guide values and eBay sales per grade (Ungraded, Grade 1\u20139.5, TAG/ACE/SGC/CGC/PSA/BGS 10, BGS 10 Black, CGC 10 Pristine). Graded rows without a named company use the generic grader 'graded' unless the eBay title names it. Pok\u00e9mon sets are mapped to pokemontcg ids/codes through the maintainers' GitHub mirror and Magic sets/cards to Scryfall (one exact-name API lookup per Magic product, \u2264 10 req/s) so sales attach to the API catalogs' canonical assets.",
441 883 "enabled": true,
442 − "schemaVersion": "1.0",
884 + "schemaVersion": "2.0",
443 885 "config": {
444 886 "seeds": [
445 887 "nintendo-64",
@@ -500,10 +942,35 @@
500 942 "comic-books-fantastic-four",
501 943 "comic-books-avengers",
502 944 "comic-books-spawn",
503 − "comic-books-walking-dead"
945 + "comic-books-walking-dead",
946 + "pokemon-base-set",
947 + "pokemon-jungle",
948 + "pokemon-fossil",
949 + "pokemon-team-rocket",
950 + "pokemon-neo-genesis",
951 + "pokemon-evolving-skies",
952 + "pokemon-151",
953 + "pokemon-promo",
954 + "magic-alpha",
955 + "magic-beta",
956 + "magic-unlimited",
957 + "magic-revised",
958 + "magic-arabian-nights",
959 + "magic-legends",
960 + "magic-modern-horizons-3",
961 + "yugioh-legend-of-blue-eyes-white-dragon",
962 + "yugioh-metal-raiders",
963 + "yugioh-spell-ruler",
964 + "yugioh-pharaohs-servant"
504 965 ],
505 − "productsPerConsole": 200,
506 − "sort": "popularity"
966 + "productsPerConsole": 150,
967 + "sort": "popularity",
968 + "cardCategories": [
969 + "pokemon-cards",
970 + "magic-cards",
971 + "yugioh-cards"
972 + ],
973 + "maxConsolesPerCategory": 30
507 974 }
508 975 },
509 976 {
@@ -543,7 +1010,7 @@
543 1010 "trustScore": 0.85,
544 1011 "attributionRequired": true,
545 1012 "termsUrl": "https://scryfall.com/docs/api",
546 − "accessNotes": "Official public API, no key. Bulk 'default_cards' JSONL (gzip, ~78 MB, refreshed daily) is streamed for the full catalog; single-card lookups use /cards/<set>/<number>. Scryfall asks for ≤10 req/s (we pace 100 ms), an identifying User-Agent and Accept headers. Prices (usd/usd_foil/usd_etched/eur/eur_foil) are Scryfall's daily market aggregates from TCGplayer/Cardmarket → stored as price_observations, never as sales. Card data © Wizards of the Coast; Scryfall requests attribution and no implication of endorsement.",
1013 + "accessNotes": "Official public API, no key. Bulk 'default_cards' JSONL (gzip, ~78 MB, refreshed daily) is streamed for the full catalog; single-card lookups use /cards/<set>/<number>. Scryfall asks for \u226410 req/s (we pace 100 ms), an identifying User-Agent and Accept headers. Prices (usd/usd_foil/usd_etched/eur/eur_foil) are Scryfall's daily market aggregates from TCGplayer/Cardmarket \u2192 stored as price_observations, never as sales. Card data \u00a9 Wizards of the Coast; Scryfall requests attribution and no implication of endorsement.",
547 1014 "enabled": true,
548 1015 "schemaVersion": "1.0",
549 1016 "config": {
@@ -551,6 +1018,140 @@
551 1018 "requestIntervalMs": 100
552 1019 }
553 1020 },
1021 + {
1022 + "id": "sportscardspro",
1023 + "displayName": "SportsCardsPro",
1024 + "sourceId": "sportscardspro",
1025 + "sourceName": "SportsCardsPro (PriceCharting)",
1026 + "sourceType": "pricing_guide",
1027 + "sourceUrl": "https://www.sportscardspro.com",
1028 + "module": "api/sportscardspro",
1029 + "enginePriority": [
1030 + "firecrawl",
1031 + "scrapfly"
1032 + ],
1033 + "categories": [
1034 + "sports_cards",
1035 + "baseball_cards",
1036 + "basketball_cards",
1037 + "football_cards",
1038 + "hockey_cards",
1039 + "soccer_cards",
1040 + "f1_cards",
1041 + "other_sports_cards"
1042 + ],
1043 + "regions": [
1044 + "US"
1045 + ],
1046 + "languages": [
1047 + "en"
1048 + ],
1049 + "currency": [
1050 + "USD"
1051 + ],
1052 + "supportsListings": false,
1053 + "supportsSold": true,
1054 + "supportsAuctions": false,
1055 + "supportsImages": true,
1056 + "supportsCatalog": true,
1057 + "supportsPopulation": false,
1058 + "supportsLookup": true,
1059 + "refreshFrequencyMinutes": 1440,
1060 + "priority": "high",
1061 + "trustScore": 0.7,
1062 + "attributionRequired": true,
1063 + "termsUrl": "https://www.sportscardspro.com/page/terms-of-service",
1064 + "accessNotes": "Public product pages of PriceCharting's sports-card site. robots.txt only disallows /buy, /publish-offer, /stripe-connect. Plain HTTP with our user agent gets a Cloudflare interstitial (HTTP 403), so pages are fetched through Firecrawl (1 credit per page, ~1 MB each) \u2014 no login, no CAPTCHA solving, no account. Each page exposes guide values per grade (Ungraded, Grade 1\u20139.5, TAG/ACE/SGC/CGC/PSA/BGS 10, BGS 10 Black, CGC 10 Pristine) and recently completed eBay sales per grade tab; graded rows without a named grading company are stored with the generic grader 'graded' unless the eBay title names the company. Set lists come from the public /category/<sport>-cards pages; console listings use the ?format=json cursor endpoint.",
1065 + "enabled": true,
1066 + "schemaVersion": "2.0",
1067 + "config": {
1068 + "seeds": [
1069 + "basketball-cards-1986-fleer",
1070 + "basketball-cards-2003-topps-chrome",
1071 + "basketball-cards-2018-panini-prizm",
1072 + "basketball-cards-2019-panini-prizm",
1073 + "basketball-cards-1996-topps-chrome",
1074 + "baseball-cards-1952-topps",
1075 + "baseball-cards-1989-upper-deck",
1076 + "baseball-cards-2011-topps-update",
1077 + "baseball-cards-2018-topps-update",
1078 + "baseball-cards-1993-sp",
1079 + "football-cards-2000-playoff-contenders",
1080 + "football-cards-2017-panini-prizm",
1081 + "football-cards-2020-panini-prizm",
1082 + "football-cards-1957-topps",
1083 + "hockey-cards-1979-o-pee-chee",
1084 + "hockey-cards-2005-upper-deck",
1085 + "hockey-cards-2015-upper-deck",
1086 + "soccer-cards-2018-panini-prizm-world-cup",
1087 + "soccer-cards-2004-panini-mega-cracks"
1088 + ],
1089 + "cardCategories": [
1090 + "basketball-cards",
1091 + "baseball-cards",
1092 + "football-cards",
1093 + "hockey-cards",
1094 + "soccer-cards"
1095 + ],
1096 + "maxConsolesPerCategory": 25,
1097 + "productsPerConsole": 120,
1098 + "sort": "popularity"
1099 + }
1100 + },
1101 + {
1102 + "id": "tcgdex",
1103 + "displayName": "TCGdex (Pok\u00e9mon, multilingual)",
1104 + "sourceId": "tcgdex",
1105 + "sourceName": "TCGdex",
1106 + "sourceType": "catalog",
1107 + "sourceUrl": "https://tcgdex.dev",
1108 + "module": "api/tcgdex",
1109 + "enginePriority": [
1110 + "api"
1111 + ],
1112 + "categories": [
1113 + "pokemon"
1114 + ],
1115 + "regions": [
1116 + "global"
1117 + ],
1118 + "languages": [
1119 + "en",
1120 + "ja",
1121 + "fr",
1122 + "de",
1123 + "es",
1124 + "it",
1125 + "pt",
1126 + "ko",
1127 + "zh"
1128 + ],
1129 + "currency": [
1130 + "USD",
1131 + "EUR"
1132 + ],
1133 + "supportsListings": false,
1134 + "supportsSold": false,
1135 + "supportsAuctions": false,
1136 + "supportsImages": true,
1137 + "supportsCatalog": true,
1138 + "supportsPopulation": false,
1139 + "supportsLookup": false,
1140 + "refreshFrequencyMinutes": 1440,
1141 + "priority": "high",
1142 + "trustScore": 0.8,
1143 + "attributionRequired": true,
1144 + "termsUrl": "https://tcgdex.dev/",
1145 + "accessNotes": "Open REST API (https://api.tcgdex.net/v2, no key, MIT-licensed data) covering every Pok\u00e9mon TCG language including Japanese; English cards carry TCGplayer (USD) and Cardmarket (EUR) prices per variant with their own update timestamps, which we store as price observations dated by the price's `updated` field. One request per set and per card at \u22488 req/s (self-throttled 120 ms). English ids match pokemontcg.io (base1-4) so `pokemontcg_id` aligns; set codes use the PTCGO abbreviation like pokemontcg's ptcgoCode.",
1146 + "enabled": true,
1147 + "schemaVersion": "1.0",
1148 + "config": {
1149 + "languages": [
1150 + "en",
1151 + "ja"
1152 + ]
1153 + }
1154 + },
554 1155 {
555 1156 "id": "ygoprodeck",
556 1157 "displayName": "YGOPRODeck (Yu-Gi-Oh!)",
@@ -588,7 +1189,7 @@
588 1189 "trustScore": 0.75,
589 1190 "attributionRequired": true,
590 1191 "termsUrl": "https://ygoprodeck.com/api-guide/",
591 − "accessNotes": "Public API v7 (https://db.ygoprodeck.com/api/v7/cardinfo.php), no key; rate limit 20 req/s — a full crawl is ONE request (~21 MB, ~14.5k cards / ~44k printings) plus optional misc=yes. The API carries no price timestamp: card_sets[].set_price (USD, per printing) and card_prices (cardmarket EUR / tcgplayer / ebay / amazon / coolstuffinc USD, card-level) are recorded as price_observations dated at fetch time with confidence 0.7 and metadata.price_scope. Images may not be hot-linked in bulk per the API guide (we store URLs, never mirror). Yu-Gi-Oh! © Konami.",
1192 + "accessNotes": "Public API v7 (https://db.ygoprodeck.com/api/v7/cardinfo.php), no key; rate limit 20 req/s \u2014 a full crawl is ONE request (~21 MB, ~14.5k cards / ~44k printings) plus optional misc=yes. The API carries no price timestamp: card_sets[].set_price (USD, per printing) and card_prices (cardmarket EUR / tcgplayer / ebay / amazon / coolstuffinc USD, card-level) are recorded as price_observations dated at fetch time with confidence 0.7 and metadata.price_scope. Images may not be hot-linked in bulk per the API guide (we store URLs, never mirror). Yu-Gi-Oh! \u00a9 Konami.",
592 1193 "enabled": true,
593 1194 "schemaVersion": "1.0",
594 1195 "config": {
added data/fixtures/aucfree/pokemon-psa10-page1.json +484 −0
@@ -0,0 +1,484 @@
1 +{
2 + "raw": {
3 + "url": "https://aucfree.com/search?o=t2&q=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA10",
4 + "externalId": "search:ポケモンカード PSA10:1",
5 + "kind": "sale",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T05:45:38.424Z",
8 + "payload": {
9 + "kind": "search_page",
10 + "url": "https://aucfree.com/search?o=t2&q=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA10",
11 + "seed": {
12 + "q": "ポケモンカード PSA10",
13 + "category": "pokemon",
14 + "language": "Japanese"
15 + },
16 + "page": 1,
17 + "rows": [
18 + {
19 + "id": "c1243517639",
20 + "url": "https://aucfree.com/items/c1243517639",
21 + "title": "【PSA10】 ポケモンカード リーフィア マスターボールミラー ポケカ",
22 + "priceJpy": 11000,
23 + "bids": 1,
24 + "endedOn": "2026年9月6日",
25 + "image": "https://img.aucfree.com/c1243517639.1.jpg"
26 + },
27 + {
28 + "id": "w1243294390",
29 + "url": "https://aucfree.com/items/w1243294390",
30 + "title": "【PSA10】コダック マスターボールミラー 054/165 ポケモンカード151",
31 + "priceJpy": 38000,
32 + "bids": 17,
33 + "endedOn": "2026年9月6日",
34 + "image": "https://img.aucfree.com/w1243294390.1.jpg"
35 + },
36 + {
37 + "id": "j1243177837",
38 + "url": "https://aucfree.com/items/j1243177837",
39 + "title": "メガカイリューex PSA10 SAR ポケモンカード MEGAドリームex",
40 + "priceJpy": 37000,
41 + "bids": 42,
42 + "endedOn": "2026年9月6日",
43 + "image": "https://img.aucfree.com/j1243177837.1.jpg"
44 + },
45 + {
46 + "id": "b1242954632",
47 + "url": "https://aucfree.com/items/b1242954632",
48 + "title": "ポケモンカード 引退品 まとめ売り PSA10 マリィのプライド エーフィex 他",
49 + "priceJpy": 200000,
50 + "bids": 1,
51 + "endedOn": "2026年9月6日",
52 + "image": "https://img.aucfree.com/b1242954632.1.jpg"
53 + },
54 + {
55 + "id": "f1242149782",
56 + "url": "https://aucfree.com/items/f1242149782",
57 + "title": "MレックウザEX [プロモカードパック 25thANNIVERSARY edition] S8...",
58 + "priceJpy": 27091,
59 + "bids": 2,
60 + "endedOn": "2026年9月6日",
61 + "image": "https://img.aucfree.com/f1242149782.1.jpg"
62 + },
63 + {
64 + "id": "e1243398299",
65 + "url": "https://aucfree.com/items/e1243398299",
66 + "title": "【PSA10】ピカチュウ プロモ 120/SV-P ピッピ AR 086/080 ポケモンカー...",
67 + "priceJpy": 10500,
68 + "bids": 28,
69 + "endedOn": "2026年9月6日",
70 + "image": "https://img.aucfree.com/e1243398299.1.jpg"
71 + },
72 + {
73 + "id": "w1242151031",
74 + "url": "https://aucfree.com/items/w1242151031",
75 + "title": "サナ SR [蒼空ストリーム] S7R 077/067 (PSA10) ポケモンカード ポケカ",
76 + "priceJpy": 14546,
77 + "bids": 2,
78 + "endedOn": "2026年9月6日",
79 + "image": "https://img.aucfree.com/w1242151031.1.jpg"
80 + },
81 + {
82 + "id": "b1242871779",
83 + "url": "https://aucfree.com/items/b1242871779",
84 + "title": "ポケモンカード ピカチュウ マクドナルド プロモ 020/M-P PSA10 トレカ ポケカ",
85 + "priceJpy": 11000,
86 + "bids": 19,
87 + "endedOn": "2026年9月6日",
88 + "image": "https://img.aucfree.com/b1242871779.1.jpg"
89 + },
90 + {
91 + "id": "c1242935437",
92 + "url": "https://aucfree.com/items/c1242935437",
93 + "title": "PSA10 ピカチュウex SAR 234/193 M2a ポケモンカード",
94 + "priceJpy": 57000,
95 + "bids": 6,
96 + "endedOn": "2026年9月6日",
97 + "image": "https://img.aucfree.com/c1242935437.1.jpg"
98 + },
99 + {
100 + "id": "j1243429895",
101 + "url": "https://aucfree.com/items/j1243429895",
102 + "title": "【PSA10】 ポケモンカード シブヤのピカチュウ ポケカ",
103 + "priceJpy": 130000,
104 + "bids": 1,
105 + "endedOn": "2026年9月6日",
106 + "image": "https://img.aucfree.com/j1243429895.1.jpg"
107 + },
108 + {
109 + "id": "c1242515105",
110 + "url": "https://aucfree.com/items/c1242515105",
111 + "title": "PSA10 ポケモンカード 25th ピカチュウ 001/028 鑑定品 10枚セット",
112 + "priceJpy": 37000,
113 + "bids": 43,
114 + "endedOn": "2026年9月6日",
115 + "image": "https://img.aucfree.com/c1242515105.1.jpg"
116 + },
117 + {
118 + "id": "f1243125302",
119 + "url": "https://aucfree.com/items/f1243125302",
120 + "title": "ポケモンカード ギャラドスごっこピカチュウ プロモ PSA10",
121 + "priceJpy": 750000,
122 + "bids": 189,
123 + "endedOn": "2026年9月6日",
124 + "image": "https://img.aucfree.com/f1243125302.1.jpg"
125 + },
126 + {
127 + "id": "h1243005150",
128 + "url": "https://aucfree.com/items/h1243005150",
129 + "title": "美品 PSA鑑定品 PSA10 ポケモンカード ミュウex SV2a 195/165 SR",
130 + "priceJpy": 9250,
131 + "bids": 12,
132 + "endedOn": "2026年9月5日",
133 + "image": "https://img.aucfree.com/h1243005150.1.jpg"
134 + },
135 + {
136 + "id": "w1242841055",
137 + "url": "https://aucfree.com/items/w1242841055",
138 + "title": "ポケモンカード コイキング AR PSA10 トリプレットビート",
139 + "priceJpy": 34000,
140 + "bids": 29,
141 + "endedOn": "2026年9月5日",
142 + "image": "https://img.aucfree.com/w1242841055.1.jpg"
143 + },
144 + {
145 + "id": "o1242587483",
146 + "url": "https://aucfree.com/items/o1242587483",
147 + "title": "【PSA10】ポケモンカード ソード&シールド ハイクラスパック VMAXクライマックス ピカ...",
148 + "priceJpy": 31500,
149 + "bids": 27,
150 + "endedOn": "2026年9月5日",
151 + "image": "https://img.aucfree.com/o1242587483.1.jpg"
152 + },
153 + {
154 + "id": "1242992598",
155 + "url": "https://aucfree.com/items/1242992598",
156 + "title": "美品 PSA鑑定品 PSA10 ポケモンカード メガリザードンXex M2a 223/193 MA",
157 + "priceJpy": 11500,
158 + "bids": 11,
159 + "endedOn": "2026年9月5日",
160 + "image": "https://img.aucfree.com/1242992598.1.jpg"
161 + },
162 + {
163 + "id": "b1242992232",
164 + "url": "https://aucfree.com/items/b1242992232",
165 + "title": "美品 PSA鑑定品 PSA10 ポケモンカード メガゲンガーex M2a 230/193 MA...",
166 + "priceJpy": 5850,
167 + "bids": 14,
168 + "endedOn": "2026年9月5日",
169 + "image": "https://img.aucfree.com/b1242992232.1.jpg"
170 + },
171 + {
172 + "id": "p1242605655",
173 + "url": "https://aucfree.com/items/p1242605655",
174 + "title": "【PSA10】 ポケモンカード そらをとぶピカチュウV [RR] (s8a_023/028) ...",
175 + "priceJpy": 11500,
176 + "bids": 16,
177 + "endedOn": "2026年9月5日",
178 + "image": "https://img.aucfree.com/p1242605655.1.jpg"
179 + },
180 + {
181 + "id": "f1243004000",
182 + "url": "https://aucfree.com/items/f1243004000",
183 + "title": "美品 PSA鑑定品 PSA10 ポケモンカード メガジガルデex M3 113/080 SAR",
184 + "priceJpy": 5200,
185 + "bids": 12,
186 + "endedOn": "2026年9月5日",
187 + "image": "https://img.aucfree.com/f1243004000.1.jpg"
188 + },
189 + {
190 + "id": "k1242727507",
191 + "url": "https://aucfree.com/items/k1242727507",
192 + "title": "J001★⑧★同梱不可★ポケモンカード PSA10 鑑定済 P ピカチュウ YU NAGABA...",
193 + "priceJpy": 41500,
194 + "bids": 66,
195 + "endedOn": "2026年9月5日",
196 + "image": "https://img.aucfree.com/k1242727507.1.jpg"
197 + },
198 + {
199 + "id": "b1242879170",
200 + "url": "https://aucfree.com/items/b1242879170",
201 + "title": "マリィ PSA10 シャイニースターV SR ポケモンカード s4a 198/190 SR ...",
202 + "priceJpy": 28000,
203 + "bids": 1,
204 + "endedOn": "2026年9月5日",
205 + "image": "https://img.aucfree.com/b1242879170.1.jpg"
206 + },
207 + {
208 + "id": "k1241468233",
209 + "url": "https://aucfree.com/items/k1241468233",
210 + "title": "PSA10 ビクティニ 288/SV-P BWR争奪戦 プロモ ポケモンカード",
211 + "priceJpy": 298000,
212 + "bids": 9,
213 + "endedOn": "2026年9月5日",
214 + "image": "https://img.aucfree.com/k1241468233.1.jpg"
215 + },
216 + {
217 + "id": "h1242598759",
218 + "url": "https://aucfree.com/items/h1242598759",
219 + "title": "【PSA10】ポケモンカード ハイクラスパック VSTARユニバース リザードンVSTAR S...",
220 + "priceJpy": 25800,
221 + "bids": 23,
222 + "endedOn": "2026年9月5日",
223 + "image": "https://img.aucfree.com/h1242598759.1.jpg"
224 + },
225 + {
226 + "id": "w1242993891",
227 + "url": "https://aucfree.com/items/w1242993891",
228 + "title": "美品 PSA鑑定品 PSA10 ポケモンカード オドリドリex M2 111/080 SAR",
229 + "priceJpy": 8750,
230 + "bids": 11,
231 + "endedOn": "2026年9月5日",
232 + "image": "https://img.aucfree.com/w1242993891.1.jpg"
233 + },
234 + {
235 + "id": "q1242588665",
236 + "url": "https://aucfree.com/items/q1242588665",
237 + "title": "【PSA10】ポケモンカード ロケット団の栄光 ロケット団のファイヤーex SAR 124/0...",
238 + "priceJpy": 30500,
239 + "bids": 22,
240 + "endedOn": "2026年9月5日",
241 + "image": "https://img.aucfree.com/q1242588665.1.jpg"
242 + },
243 + {
244 + "id": "g1243184146",
245 + "url": "https://aucfree.com/items/g1243184146",
246 + "title": "一円スタート 鑑定品 美品 ピカチュウ プロモ PSA10 S-P ピカピカ キャンペーン ...",
247 + "priceJpy": 21000,
248 + "bids": 33,
249 + "endedOn": "2026年9月5日",
250 + "image": "https://img.aucfree.com/g1243184146.1.jpg"
251 + },
252 + {
253 + "id": "x1243186762",
254 + "url": "https://aucfree.com/items/x1243186762",
255 + "title": "一円スタート 鑑定品 美品 PSA10 ポケモンカード S-P ピカチュウVMAX コロコロコ...",
256 + "priceJpy": 12500,
257 + "bids": 30,
258 + "endedOn": "2026年9月5日",
259 + "image": "https://img.aucfree.com/x1243186762.1.jpg"
260 + },
261 + {
262 + "id": "l1242905407",
263 + "url": "https://aucfree.com/items/l1242905407",
264 + "title": "ポケモンカード メガリザードンXex MA PSA10 メガドリーム GEM MT 鑑定品",
265 + "priceJpy": 13501,
266 + "bids": 9,
267 + "endedOn": "2026年9月5日",
268 + "image": "https://img.aucfree.com/l1242905407.1.jpg"
269 + },
270 + {
271 + "id": "l1243011828",
272 + "url": "https://aucfree.com/items/l1243011828",
273 + "title": "美品 PSA鑑定品 PSA10 ポケモンカード メガリザードンXex M2a 223/193 MA",
274 + "priceJpy": 12000,
275 + "bids": 15,
276 + "endedOn": "2026年9月5日",
277 + "image": "https://img.aucfree.com/l1243011828.1.jpg"
278 + },
279 + {
280 + "id": "w1243282838",
281 + "url": "https://aucfree.com/items/w1243282838",
282 + "title": "ポケモンカード サンダース YU NAGABA プロモ 064/SV-P PSA10",
283 + "priceJpy": 27000,
284 + "bids": 12,
285 + "endedOn": "2026年9月5日",
286 + "image": "https://img.aucfree.com/w1243282838.1.jpg"
287 + },
288 + {
289 + "id": "k1241472466",
290 + "url": "https://aucfree.com/items/k1241472466",
291 + "title": "PSA10 ミュウex 25th ANNIVERSARY プロモ 014/025 ポケモンカード",
292 + "priceJpy": 23000,
293 + "bids": 1,
294 + "endedOn": "2026年9月5日",
295 + "image": "https://img.aucfree.com/k1241472466.1.jpg"
296 + },
297 + {
298 + "id": "x1243176198",
299 + "url": "https://aucfree.com/items/x1243176198",
300 + "title": "ポケモンカード カヒリ 065/060 SR PSA10 PSA鑑定品",
301 + "priceJpy": 20500,
302 + "bids": 16,
303 + "endedOn": "2026年9月5日",
304 + "image": "https://img.aucfree.com/x1243176198.1.jpg"
305 + },
306 + {
307 + "id": "e1242972950",
308 + "url": "https://aucfree.com/items/e1242972950",
309 + "title": "PSA10 オトシドリ AR 089/078 ポケモンカード SV1V",
310 + "priceJpy": 3500,
311 + "bids": 7,
312 + "endedOn": "2026年9月5日",
313 + "image": "https://img.aucfree.com/e1242972950.1.jpg"
314 + },
315 + {
316 + "id": "r1243193724",
317 + "url": "https://aucfree.com/items/r1243193724",
318 + "title": "ポケモンカード ピカチュウ 242/SV-P PSA10 PSA鑑定品",
319 + "priceJpy": 11000,
320 + "bids": 35,
321 + "endedOn": "2026年9月5日",
322 + "image": "https://img.aucfree.com/r1243193724.1.jpg"
323 + },
324 + {
325 + "id": "h1243105410",
326 + "url": "https://aucfree.com/items/h1243105410",
327 + "title": "ポケモンカード PSA10 リザードンGX RR SMP2 007/024 ムービースペシャル...",
328 + "priceJpy": 10091,
329 + "bids": 5,
330 + "endedOn": "2026年9月5日",
331 + "image": "https://img.aucfree.com/h1243105410.1.jpg"
332 + },
333 + {
334 + "id": "m1243121082",
335 + "url": "https://aucfree.com/items/m1243121082",
336 + "title": "1円~ PSA10 ポケモンカード ポケカ SM11b 067/049 SR メイ",
337 + "priceJpy": 73000,
338 + "bids": 25,
339 + "endedOn": "2026年9月5日",
340 + "image": "https://img.aucfree.com/m1243121082.1.jpg"
341 + },
342 + {
343 + "id": "n1243185509",
344 + "url": "https://aucfree.com/items/n1243185509",
345 + "title": "ポケモンカード ピカチュウ 197/SV-P PSA10 PSA鑑定品",
346 + "priceJpy": 8027,
347 + "bids": 35,
348 + "endedOn": "2026年9月5日",
349 + "image": "https://img.aucfree.com/n1243185509.1.jpg"
350 + },
351 + {
352 + "id": "u1243193825",
353 + "url": "https://aucfree.com/items/u1243193825",
354 + "title": "ポケモンカード カメックス 202/165 SAR PSA10 PSA鑑定品",
355 + "priceJpy": 29100,
356 + "bids": 42,
357 + "endedOn": "2026年9月5日",
358 + "image": "https://img.aucfree.com/u1243193825.1.jpg"
359 + },
360 + {
361 + "id": "u1243192922",
362 + "url": "https://aucfree.com/items/u1243192922",
363 + "title": "ポケモンカード ピカチュウ 272/S-P PSA10 PSA鑑定品",
364 + "priceJpy": 14900,
365 + "bids": 22,
366 + "endedOn": "2026年9月5日",
367 + "image": "https://img.aucfree.com/u1243192922.1.jpg"
368 + },
369 + {
370 + "id": "n1243429081",
371 + "url": "https://aucfree.com/items/n1243429081",
372 + "title": "PSA10 メガカイリューex RR 126/193 ポケモンカード PSA10",
373 + "priceJpy": 11000,
374 + "bids": 1,
375 + "endedOn": "2026年9月5日",
376 + "image": "https://img.aucfree.com/n1243429081.1.jpg"
377 + },
378 + {
379 + "id": "j1243184891",
380 + "url": "https://aucfree.com/items/j1243184891",
381 + "title": "ポケモンカード ピカチュウ 120/SV-P PSA10 PSA鑑定品",
382 + "priceJpy": 8750,
383 + "bids": 28,
384 + "endedOn": "2026年9月5日",
385 + "image": "https://img.aucfree.com/j1243184891.1.jpg"
386 + },
387 + {
388 + "id": "d1243196858",
389 + "url": "https://aucfree.com/items/d1243196858",
390 + "title": "ポケモンカード メガラティアスex 088/063 SAR PSA10 PSA鑑定品",
391 + "priceJpy": 8680,
392 + "bids": 18,
393 + "endedOn": "2026年9月5日",
394 + "image": "https://img.aucfree.com/d1243196858.1.jpg"
395 + },
396 + {
397 + "id": "l1243332941",
398 + "url": "https://aucfree.com/items/l1243332941",
399 + "title": "名探偵ピカチュウ PSA10 ポケモンカード ポケカ プロモ",
400 + "priceJpy": 53000,
401 + "bids": 44,
402 + "endedOn": "2026年9月5日",
403 + "image": "https://img.aucfree.com/l1243332941.1.jpg"
404 + },
405 + {
406 + "id": "j1243182686",
407 + "url": "https://aucfree.com/items/j1243182686",
408 + "title": "ポケモンカード ピカチュウvmax 114/100 HR PSA10 PSA鑑定品",
409 + "priceJpy": 103005,
410 + "bids": 43,
411 + "endedOn": "2026年9月5日",
412 + "image": "https://img.aucfree.com/j1243182686.1.jpg"
413 + },
414 + {
415 + "id": "l1243195522",
416 + "url": "https://aucfree.com/items/l1243195522",
417 + "title": "ポケモンカード シャワーズ 063/SV-P ナガバ PSA10 PSA鑑定品",
418 + "priceJpy": 28002,
419 + "bids": 18,
420 + "endedOn": "2026年9月5日",
421 + "image": "https://img.aucfree.com/l1243195522.1.jpg"
422 + },
423 + {
424 + "id": "j1243188643",
425 + "url": "https://aucfree.com/items/j1243188643",
426 + "title": "ポケモンカード オドリドリex 111/080 SAR PSA10 PSA鑑定品",
427 + "priceJpy": 8750,
428 + "bids": 14,
429 + "endedOn": "2026年9月5日",
430 + "image": "https://img.aucfree.com/j1243188643.1.jpg"
431 + },
432 + {
433 + "id": "r1243199662",
434 + "url": "https://aucfree.com/items/r1243199662",
435 + "title": "ポケモンカード ロケット団のファイヤーex 124/098 SAR PSA10 PSA鑑定品",
436 + "priceJpy": 34500,
437 + "bids": 42,
438 + "endedOn": "2026年9月5日",
439 + "image": "https://img.aucfree.com/r1243199662.1.jpg"
440 + },
441 + {
442 + "id": "j1243184838",
443 + "url": "https://aucfree.com/items/j1243184838",
444 + "title": "ポケモンカード サンダース 064/SV-P ナガバ PSA10 PSA鑑定品",
445 + "priceJpy": 27002,
446 + "bids": 15,
447 + "endedOn": "2026年9月5日",
448 + "image": "https://img.aucfree.com/j1243184838.1.jpg"
449 + },
450 + {
451 + "id": "s1243192682",
452 + "url": "https://aucfree.com/items/s1243192682",
453 + "title": "ポケモンカード ピカチュウ 291/SV-P PSA10 PSA鑑定品",
454 + "priceJpy": 6900,
455 + "bids": 16,
456 + "endedOn": "2026年9月5日",
457 + "image": "https://img.aucfree.com/s1243192682.1.jpg"
458 + },
459 + {
460 + "id": "l1243192728",
461 + "url": "https://aucfree.com/items/l1243192728",
462 + "title": "ポケモンカード リーフィア 068/SV-P ナガバ PSA10 PSA鑑定品",
463 + "priceJpy": 27000,
464 + "bids": 13,
465 + "endedOn": "2026年9月5日",
466 + "image": "https://img.aucfree.com/l1243192728.1.jpg"
467 + }
468 + ]
469 + }
470 + },
471 + "expect": {
472 + "minCount": 10,
473 + "kinds": [
474 + "sale"
475 + ],
476 + "requiredFields": [
477 + "saleDate",
478 + "price",
479 + "currency"
480 + ]
481 + },
482 + "note": "Captured live via Firecrawl from aucfree.com",
483 + "capturedAt": "2026-09-07T05:45:38.461Z"
484 +}
\ No newline at end of file
added data/fixtures/aucfree/seed-page-2.json +484 −0
@@ -0,0 +1,484 @@
1 +{
2 + "raw": {
3 + "url": "https://aucfree.com/search?o=t2&q=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA10&p=2",
4 + "externalId": "search:ポケモンカード PSA10:2",
5 + "kind": "sale",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T05:45:40.486Z",
8 + "payload": {
9 + "kind": "search_page",
10 + "url": "https://aucfree.com/search?o=t2&q=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA10&p=2",
11 + "seed": {
12 + "q": "ポケモンカード PSA10",
13 + "category": "pokemon",
14 + "language": "Japanese"
15 + },
16 + "page": 2,
17 + "rows": [
18 + {
19 + "id": "p1243194672",
20 + "url": "https://aucfree.com/items/p1243194672",
21 + "title": "ポケモンカード メガアブソルex 089/063 SAR PSA10 PSA鑑定品",
22 + "priceJpy": 8510,
23 + "bids": 12,
24 + "endedOn": "2026年9月5日",
25 + "image": "https://img.aucfree.com/p1243194672.1.jpg"
26 + },
27 + {
28 + "id": "p1243154420",
29 + "url": "https://aucfree.com/items/p1243154420",
30 + "title": "1円~ PSA10 ポケモンカード ポケカ SV2D 083/071 AR リキキリン",
31 + "priceJpy": 3491,
32 + "bids": 8,
33 + "endedOn": "2026年9月5日",
34 + "image": "https://img.aucfree.com/p1243154420.1.jpg"
35 + },
36 + {
37 + "id": "q1241203514",
38 + "url": "https://aucfree.com/items/q1241203514",
39 + "title": "ポケモンカード 【PSA10】ニンフィアVMAX CSR 232/184+楽園ドラゴーナ BO...",
40 + "priceJpy": 36000,
41 + "bids": 32,
42 + "endedOn": "2026年9月5日",
43 + "image": "https://img.aucfree.com/q1241203514.1.jpg"
44 + },
45 + {
46 + "id": "g1243172895",
47 + "url": "https://aucfree.com/items/g1243172895",
48 + "title": "ポケモンカード ピカチュウ 208/S-P ナガバ PSA10 PSA鑑定品",
49 + "priceJpy": 44002,
50 + "bids": 31,
51 + "endedOn": "2026年9月5日",
52 + "image": "https://img.aucfree.com/g1243172895.1.jpg"
53 + },
54 + {
55 + "id": "1243108914",
56 + "url": "https://aucfree.com/items/1243108914",
57 + "title": "1円~ PSA10 ポケモンカード ポケカ プロモ 067/SV-P ブラッキー",
58 + "priceJpy": 38001,
59 + "bids": 17,
60 + "endedOn": "2026年9月5日",
61 + "image": "https://img.aucfree.com/1243108914.1.jpg"
62 + },
63 + {
64 + "id": "c1243187086",
65 + "url": "https://aucfree.com/items/c1243187086",
66 + "title": "ポケモンカード ナンジャモのハラバリーex 125/100 SAR PSA10 PSA鑑定品",
67 + "priceJpy": 16000,
68 + "bids": 16,
69 + "endedOn": "2026年9月5日",
70 + "image": "https://img.aucfree.com/c1243187086.1.jpg"
71 + },
72 + {
73 + "id": "v1243192410",
74 + "url": "https://aucfree.com/items/v1243192410",
75 + "title": "ポケモンカード イーブイ 062/SV-P ナガバ PSA10 PSA鑑定品",
76 + "priceJpy": 35000,
77 + "bids": 24,
78 + "endedOn": "2026年9月5日",
79 + "image": "https://img.aucfree.com/v1243192410.1.jpg"
80 + },
81 + {
82 + "id": "j1243183585",
83 + "url": "https://aucfree.com/items/j1243183585",
84 + "title": "ポケモンカード ニンフィア 070/SV-P ナガバ PSA10 PSA鑑定品",
85 + "priceJpy": 27500,
86 + "bids": 14,
87 + "endedOn": "2026年9月5日",
88 + "image": "https://img.aucfree.com/j1243183585.1.jpg"
89 + },
90 + {
91 + "id": "l1243162202",
92 + "url": "https://aucfree.com/items/l1243162202",
93 + "title": "1円~ PSA10 ポケモンカード ポケカ SV2a 101/165 R マルマイン マスター...",
94 + "priceJpy": 6484,
95 + "bids": 14,
96 + "endedOn": "2026年9月5日",
97 + "image": "https://img.aucfree.com/l1243162202.1.jpg"
98 + },
99 + {
100 + "id": "r1242616508",
101 + "url": "https://aucfree.com/items/r1242616508",
102 + "title": "【PSA10・美品】BWR ゼクロムex 174/086 ポケモンカード ポケカ 送料無料!!",
103 + "priceJpy": 121000,
104 + "bids": 38,
105 + "endedOn": "2026年9月5日",
106 + "image": "https://img.aucfree.com/r1242616508.1.jpg"
107 + },
108 + {
109 + "id": "1243178502",
110 + "url": "https://aucfree.com/items/1243178502",
111 + "title": "ポケモンカード メガゲンガー 240/193 SAR PSA10 PSA鑑定品",
112 + "priceJpy": 72500,
113 + "bids": 45,
114 + "endedOn": "2026年9月5日",
115 + "image": "https://img.aucfree.com/1243178502.1.jpg"
116 + },
117 + {
118 + "id": "r1243195681",
119 + "url": "https://aucfree.com/items/r1243195681",
120 + "title": "ポケモンカード ゴース 080/071 AR PSA10 PSA鑑定品",
121 + "priceJpy": 12500,
122 + "bids": 16,
123 + "endedOn": "2026年9月5日",
124 + "image": "https://img.aucfree.com/r1243195681.1.jpg"
125 + },
126 + {
127 + "id": "r1241776184",
128 + "url": "https://aucfree.com/items/r1241776184",
129 + "title": "【PSA10】 リザードンex SAR 201/165 ポケモンカード151 ポケカ",
130 + "priceJpy": 104000,
131 + "bids": 9,
132 + "endedOn": "2026年9月5日",
133 + "image": "https://img.aucfree.com/r1241776184.1.jpg"
134 + },
135 + {
136 + "id": "f1243183502",
137 + "url": "https://aucfree.com/items/f1243183502",
138 + "title": "ポケモンカード メガダークライ 114/081 SAR PSA10 PSA鑑定品",
139 + "priceJpy": 42600,
140 + "bids": 60,
141 + "endedOn": "2026年9月5日",
142 + "image": "https://img.aucfree.com/f1243183502.1.jpg"
143 + },
144 + {
145 + "id": "s1243160230",
146 + "url": "https://aucfree.com/items/s1243160230",
147 + "title": "1円~ PSA10 ポケモンカード ポケカ SV2a 011/165 C トランセル マスター...",
148 + "priceJpy": 11002,
149 + "bids": 23,
150 + "endedOn": "2026年9月5日",
151 + "image": "https://img.aucfree.com/s1243160230.1.jpg"
152 + },
153 + {
154 + "id": "p1243193019",
155 + "url": "https://aucfree.com/items/p1243193019",
156 + "title": "ポケモンカード ブラッキー 067/SV-P ナガバ PSA10 PSA鑑定品",
157 + "priceJpy": 43400,
158 + "bids": 27,
159 + "endedOn": "2026年9月5日",
160 + "image": "https://img.aucfree.com/p1243193019.1.jpg"
161 + },
162 + {
163 + "id": "x1243173898",
164 + "url": "https://aucfree.com/items/x1243173898",
165 + "title": "ポケモンカード ゲンガー 074/071 CHR PSA10 PSA鑑定品",
166 + "priceJpy": 13500,
167 + "bids": 30,
168 + "endedOn": "2026年9月5日",
169 + "image": "https://img.aucfree.com/x1243173898.1.jpg"
170 + },
171 + {
172 + "id": "r1242998525",
173 + "url": "https://aucfree.com/items/r1242998525",
174 + "title": "ポケモンカード モンスターボール PSA10 ゴールデンボックス プロモ GOLDE...",
175 + "priceJpy": 1,
176 + "bids": 26,
177 + "endedOn": "2026年9月5日",
178 + "image": "https://img.aucfree.com/r1242998525.1.jpg"
179 + },
180 + {
181 + "id": "1243182361",
182 + "url": "https://aucfree.com/items/1243182361",
183 + "title": "ポケモンカード ヒトカゲ 051/049 PSA10 PSA鑑定品",
184 + "priceJpy": 34700,
185 + "bids": 16,
186 + "endedOn": "2026年9月5日",
187 + "image": "https://img.aucfree.com/1243182361.1.jpg"
188 + },
189 + {
190 + "id": "f1239597965",
191 + "url": "https://aucfree.com/items/f1239597965",
192 + "title": "■ポケモンカード PSA10 メガカイリューex M2a 246/193 SAR【中古】ポケカ...",
193 + "priceJpy": 40000,
194 + "bids": 1,
195 + "endedOn": "2026年9月5日",
196 + "image": "https://img.aucfree.com/f1239597965.1.jpg"
197 + },
198 + {
199 + "id": "m1243150237",
200 + "url": "https://aucfree.com/items/m1243150237",
201 + "title": "1円~ PSA10 ポケモンカード ポケカ SV10 099/098 AR ロケット団のワナイダー",
202 + "priceJpy": 3000,
203 + "bids": 7,
204 + "endedOn": "2026年9月5日",
205 + "image": "https://img.aucfree.com/m1243150237.1.jpg"
206 + },
207 + {
208 + "id": "n1243180386",
209 + "url": "https://aucfree.com/items/n1243180386",
210 + "title": "ポケモンカード リーフィア 002/187 マスターボールミラー PSA10 PSA鑑定品",
211 + "priceJpy": 9500,
212 + "bids": 12,
213 + "endedOn": "2026年9月5日",
214 + "image": "https://img.aucfree.com/n1243180386.1.jpg"
215 + },
216 + {
217 + "id": "e1243194727",
218 + "url": "https://aucfree.com/items/e1243194727",
219 + "title": "ポケモンカード ガマゲロゲ 109/086 AR PSA10 PSA鑑定品",
220 + "priceJpy": 17000,
221 + "bids": 14,
222 + "endedOn": "2026年9月5日",
223 + "image": "https://img.aucfree.com/e1243194727.1.jpg"
224 + },
225 + {
226 + "id": "m1243190475",
227 + "url": "https://aucfree.com/items/m1243190475",
228 + "title": "ポケモンカード Nのレシラム 109/100 AR PSA10 PSA鑑定品",
229 + "priceJpy": 8500,
230 + "bids": 18,
231 + "endedOn": "2026年9月5日",
232 + "image": "https://img.aucfree.com/m1243190475.1.jpg"
233 + },
234 + {
235 + "id": "v1243119399",
236 + "url": "https://aucfree.com/items/v1243119399",
237 + "title": "1円~ PSA10 ポケモンカード ポケカ s4a 198/190 SR マリィ",
238 + "priceJpy": 26001,
239 + "bids": 31,
240 + "endedOn": "2026年9月5日",
241 + "image": "https://img.aucfree.com/v1243119399.1.jpg"
242 + },
243 + {
244 + "id": "d1243193864",
245 + "url": "https://aucfree.com/items/d1243193864",
246 + "title": "ポケモンカード Nのゼクロム 210/193 AR PSA10 PSA鑑定品",
247 + "priceJpy": 5500,
248 + "bids": 28,
249 + "endedOn": "2026年9月5日",
250 + "image": "https://img.aucfree.com/d1243193864.1.jpg"
251 + },
252 + {
253 + "id": "x1242974051",
254 + "url": "https://aucfree.com/items/x1242974051",
255 + "title": "ポケモンカード PSA10 ナンジャモ SAR",
256 + "priceJpy": 87000,
257 + "bids": 39,
258 + "endedOn": "2026年9月5日",
259 + "image": "https://img.aucfree.com/x1242974051.1.jpg"
260 + },
261 + {
262 + "id": "c1243022305",
263 + "url": "https://aucfree.com/items/c1243022305",
264 + "title": "ポケモンカード リザードン VMAX SSR 308 / 190 S4a シャイニースターV ...",
265 + "priceJpy": 31000,
266 + "bids": 25,
267 + "endedOn": "2026年9月5日",
268 + "image": "https://img.aucfree.com/c1243022305.1.jpg"
269 + },
270 + {
271 + "id": "t1242927971",
272 + "url": "https://aucfree.com/items/t1242927971",
273 + "title": "ガブリアスV CSR PSA10 ポケモンカード S9a 084/067",
274 + "priceJpy": 19000,
275 + "bids": 16,
276 + "endedOn": "2026年9月5日",
277 + "image": "https://img.aucfree.com/t1242927971.1.jpg"
278 + },
279 + {
280 + "id": "m1243188622",
281 + "url": "https://aucfree.com/items/m1243188622",
282 + "title": "ポケモンカード イーブイ 125/187 マスターボールミラー PSA10 PSA鑑定品",
283 + "priceJpy": 11500,
284 + "bids": 38,
285 + "endedOn": "2026年9月5日",
286 + "image": "https://img.aucfree.com/m1243188622.1.jpg"
287 + },
288 + {
289 + "id": "h1243183746",
290 + "url": "https://aucfree.com/items/h1243183746",
291 + "title": "ポケモンカード レアコイル 112/106 AR PSA10 PSA鑑定品",
292 + "priceJpy": 6750,
293 + "bids": 19,
294 + "endedOn": "2026年9月5日",
295 + "image": "https://img.aucfree.com/h1243183746.1.jpg"
296 + },
297 + {
298 + "id": "r1243121723",
299 + "url": "https://aucfree.com/items/r1243121723",
300 + "title": "1円~ PSA10 ポケモンカード ポケカ s8a-G 005/015 ピカチュウV",
301 + "priceJpy": 6300,
302 + "bids": 12,
303 + "endedOn": "2026年9月5日",
304 + "image": "https://img.aucfree.com/r1243121723.1.jpg"
305 + },
306 + {
307 + "id": "c1243190457",
308 + "url": "https://aucfree.com/items/c1243190457",
309 + "title": "ポケモンカード グレイシア 040/187 マスターボールミラー PSA10 PSA鑑定品",
310 + "priceJpy": 9750,
311 + "bids": 41,
312 + "endedOn": "2026年9月5日",
313 + "image": "https://img.aucfree.com/c1243190457.1.jpg"
314 + },
315 + {
316 + "id": "w1243178684",
317 + "url": "https://aucfree.com/items/w1243178684",
318 + "title": "ポケモンカード ブラッキー 092/187 マスターボールミラー PSA10 PSA鑑定品",
319 + "priceJpy": 27510,
320 + "bids": 45,
321 + "endedOn": "2026年9月5日",
322 + "image": "https://img.aucfree.com/w1243178684.1.jpg"
323 + },
324 + {
325 + "id": "l1243192822",
326 + "url": "https://aucfree.com/items/l1243192822",
327 + "title": "ポケモンカード ニンフィア 068/187 マスターボールミラー PSA10 PSA鑑定品",
328 + "priceJpy": 16500,
329 + "bids": 19,
330 + "endedOn": "2026年9月5日",
331 + "image": "https://img.aucfree.com/l1243192822.1.jpg"
332 + },
333 + {
334 + "id": "l1243153943",
335 + "url": "https://aucfree.com/items/l1243153943",
336 + "title": "1円~ PSA10 ポケモンカード ポケカ SV10 106/098 AR ロケット団のヤミカラス",
337 + "priceJpy": 3500,
338 + "bids": 12,
339 + "endedOn": "2026年9月5日",
340 + "image": "https://img.aucfree.com/l1243153943.1.jpg"
341 + },
342 + {
343 + "id": "1241761635",
344 + "url": "https://aucfree.com/items/1241761635",
345 + "title": "【PSA10】 カメックス 25th ANNIVERSARY プロモ S8a-P 003/02...",
346 + "priceJpy": 24500,
347 + "bids": 6,
348 + "endedOn": "2026年9月5日",
349 + "image": "https://img.aucfree.com/1241761635.1.jpg"
350 + },
351 + {
352 + "id": "e1243156115",
353 + "url": "https://aucfree.com/items/e1243156115",
354 + "title": "1円~ PSA10 ポケモンカード ポケカ SV1S 096/078 SR パフュートンex",
355 + "priceJpy": 2200,
356 + "bids": 7,
357 + "endedOn": "2026年9月5日",
358 + "image": "https://img.aucfree.com/e1243156115.1.jpg"
359 + },
360 + {
361 + "id": "n1243182585",
362 + "url": "https://aucfree.com/items/n1243182585",
363 + "title": "ポケモンカード ピカチュウ 126/S-P PSA10 PSA鑑定品",
364 + "priceJpy": 14100,
365 + "bids": 30,
366 + "endedOn": "2026年9月5日",
367 + "image": "https://img.aucfree.com/n1243182585.1.jpg"
368 + },
369 + {
370 + "id": "c1243188386",
371 + "url": "https://aucfree.com/items/c1243188386",
372 + "title": "ポケモンカード ピカチュウ 020/M-P PSA10 PSA鑑定品",
373 + "priceJpy": 11100,
374 + "bids": 47,
375 + "endedOn": "2026年9月5日",
376 + "image": "https://img.aucfree.com/c1243188386.1.jpg"
377 + },
378 + {
379 + "id": "n1242633404",
380 + "url": "https://aucfree.com/items/n1242633404",
381 + "title": "【PSA10・美品】BWR レシラムex 174/086 ポケモンカード ポケカ 送料無料!!",
382 + "priceJpy": 119000,
383 + "bids": 37,
384 + "endedOn": "2026年9月5日",
385 + "image": "https://img.aucfree.com/n1242633404.1.jpg"
386 + },
387 + {
388 + "id": "w1243010286",
389 + "url": "https://aucfree.com/items/w1243010286",
390 + "title": "ポケモンカード ゲンシグラードン EX RR 040 / 070 XY5 ガイアボルケーノ P...",
391 + "priceJpy": 25000,
392 + "bids": 22,
393 + "endedOn": "2026年9月5日",
394 + "image": "https://img.aucfree.com/w1243010286.1.jpg"
395 + },
396 + {
397 + "id": "r1241776581",
398 + "url": "https://aucfree.com/items/r1241776581",
399 + "title": "【PSA10】 なみのりピカチュウV RR 25th ANNIVERSARY COLLECTI...",
400 + "priceJpy": 12000,
401 + "bids": 2,
402 + "endedOn": "2026年9月5日",
403 + "image": "https://img.aucfree.com/r1241776581.1.jpg"
404 + },
405 + {
406 + "id": "x1243007587",
407 + "url": "https://aucfree.com/items/x1243007587",
408 + "title": "ポケモンカード ナンジャモのカイデン 232 / SV-P PROMO プロモ プロモーション...",
409 + "priceJpy": 5500,
410 + "bids": 13,
411 + "endedOn": "2026年9月5日",
412 + "image": "https://img.aucfree.com/x1243007587.1.jpg"
413 + },
414 + {
415 + "id": "x1242706507",
416 + "url": "https://aucfree.com/items/x1242706507",
417 + "title": "2025 POKEMON M1L JP GARGANACL ART RARE GEM MT 1...",
418 + "priceJpy": 3100,
419 + "bids": 8,
420 + "endedOn": "2026年9月5日",
421 + "image": "https://img.aucfree.com/x1242706507.1.jpg"
422 + },
423 + {
424 + "id": "e1242717037",
425 + "url": "https://aucfree.com/items/e1242717037",
426 + "title": "PSA10 ヒンバス 035/187 マスボ REVERSE HOLO 2024 POKEMO...",
427 + "priceJpy": 3100,
428 + "bids": 7,
429 + "endedOn": "2026年9月5日",
430 + "image": "https://img.aucfree.com/e1242717037.1.jpg"
431 + },
432 + {
433 + "id": "r1242877065",
434 + "url": "https://aucfree.com/items/r1242877065",
435 + "title": "ジャンク●ポケモンカード シブヤのピカチュウ プロモ 002/S-P PSA10●703X",
436 + "priceJpy": 91455,
437 + "bids": 9,
438 + "endedOn": "2026年9月5日",
439 + "image": "https://img.aucfree.com/r1242877065.1.jpg"
440 + },
441 + {
442 + "id": "e1242746405",
443 + "url": "https://aucfree.com/items/e1242746405",
444 + "title": "PSA10 ダークライVSTAR VSTARユニバース SAR ポケモンカード sr ur ピ...",
445 + "priceJpy": 13500,
446 + "bids": 27,
447 + "endedOn": "2026年9月5日",
448 + "image": "https://img.aucfree.com/e1242746405.1.jpg"
449 + },
450 + {
451 + "id": "o1242718467",
452 + "url": "https://aucfree.com/items/o1242718467",
453 + "title": "2022 POKEMON JPN.SWSH FA BIBAREL VSTAR UNIVERSE...",
454 + "priceJpy": 3600,
455 + "bids": 8,
456 + "endedOn": "2026年9月5日",
457 + "image": "https://img.aucfree.com/o1242718467.1.jpg"
458 + },
459 + {
460 + "id": "m1242499921",
461 + "url": "https://aucfree.com/items/m1242499921",
462 + "title": "PSA10 レアコイル AR 112/106 超電ブレイカー ポケモンカード",
463 + "priceJpy": 6501,
464 + "bids": 15,
465 + "endedOn": "2026年9月5日",
466 + "image": "https://img.aucfree.com/m1242499921.1.jpg"
467 + }
468 + ]
469 + }
470 + },
471 + "expect": {
472 + "minCount": 10,
473 + "kinds": [
474 + "sale"
475 + ],
476 + "requiredFields": [
477 + "saleDate",
478 + "price",
479 + "currency"
480 + ]
481 + },
482 + "note": "Captured live via Firecrawl from aucfree.com",
483 + "capturedAt": "2026-09-07T05:45:40.515Z"
484 +}
\ No newline at end of file
added data/fixtures/brickset/10179-1-millennium-falcon.json +54 −0
@@ -0,0 +1,54 @@
1 +{
2 + "raw": {
3 + "url": "https://brickset.com/sets/10179-1",
4 + "externalId": "10179-1",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:45:40.061Z",
8 + "payload": {
9 + "kind": "set_page",
10 + "url": "https://brickset.com/sets/10179-1",
11 + "setNumber": "10179-1",
12 + "title": "LEGO 10179 Ultimate Collector's Millennium Falcon",
13 + "image": "https://images.brickset.com/sets/images/10179-1.jpg",
14 + "fields": {
15 + "Number": "10179-1",
16 + "Name": "Ultimate Collector's Millennium Falcon",
17 + "Category": "Normal",
18 + "Theme group": "Licensed",
19 + "Theme": "Star Wars",
20 + "Subtheme": "Ultimate Collector Series",
21 + "Year released": "2007",
22 + "Launch/exit": "01 Oct 07 - 31 Dec 09",
23 + "Tags": "View tags »Chewbacca | Han Solo | Luke Skywalker | Obi-Wan Kenobi | Princess Leia | A New Hope | D2C | Freighter | Millennium Falcon YT-1300 Light Freighter | Original Trilogy | Rebel Alliance | Smuggler | Starfighter | Wookiees",
24 + "Pieces": "5197",
25 + "Minifigs": "5, 1 unique to this set",
26 + "RRP": "£342.49, $499.99",
27 + "RRP (inflated)": "£545, $760",
28 + "Current value": "New: ~$2858 | Used: ~$1172",
29 + "Price per piece": "6.6p, 9.6c",
30 + "Age range": "16+",
31 + "Packaging": "Box",
32 + "Packaging size": "65.2 x 48 x 19.2 cm | (25⅝\" x 18⅞\" x 7½\")",
33 + "Barcodes": "UPC: 673419079419 | EAN: 5702014499874",
34 + "LEGO item numbers": "NA: 4566078 | EU: 4495738",
35 + "Availability": "LEGO exclusive",
36 + "Notes": "Re-imagined as 75192-1",
37 + "Rating": "✭✭✭✭✭ 4.6 | 246 ratings, 14 reviews"
38 + }
39 + }
40 + },
41 + "expect": {
42 + "count": 3,
43 + "kinds": [
44 + "catalog_item",
45 + "price_observation"
46 + ],
47 + "first": {
48 + "attributes.number": "10179",
49 + "attributes.year": 2007
50 + }
51 + },
52 + "note": "Captured live from brickset.com",
53 + "capturedAt": "2026-09-07T05:45:40.067Z"
54 +}
\ No newline at end of file
added data/fixtures/brickset/75192-1-millennium-falcon.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "raw": {
3 + "url": "https://brickset.com/sets/75192-1",
4 + "externalId": "75192-1",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:45:41.475Z",
8 + "payload": {
9 + "kind": "set_page",
10 + "url": "https://brickset.com/sets/75192-1",
11 + "setNumber": "75192-1",
12 + "title": "LEGO 75192 Millennium Falcon",
13 + "image": "https://images.brickset.com/sets/images/75192-1.jpg",
14 + "fields": {
15 + "Number": "75192-1",
16 + "Name": "Millennium Falcon",
17 + "Category": "Normal",
18 + "Theme group": "Licensed",
19 + "Theme": "Star Wars",
20 + "Subtheme": "Ultimate Collector Series",
21 + "Year released": "2017",
22 + "Launch/exit": "01 Oct 17 - 31 Dec 26",
23 + "Tags": "View tags »BB-8 | C-3PO | Chewbacca | Finn | Han Solo | Princess Leia | Rey | Anniversary Set | Award Winning Product | Birds | Board Game | Brick Built Animals | Brick Built Figure | Brick Separator | Brickset Bouts Finalist | Converts | D2C | Dejarik | Disney | Droids | Freighter | Info Plaque | Millennium Falcon YT-1300 Light Freighter | Mynock | Nameplate | Original Trilogy | Porg | Rebel Alliance | Remake Sets | Resistance | Sequel Trilogy | Starfighter | The Empire Strikes Back | The Force Awakens | The Last Jedi | Wookiees",
24 + "Pieces": "7541",
25 + "Minifigs": "8, 2 unique to this set",
26 + "Model size": "21 x 84 x 56 cm | (8¼\" x 33⅛\" x 22\")",
27 + "Designer": "Hans Burkhard Schlömer",
28 + "RRP": "£734.99, $849.99, €849.99",
29 + "Price per piece": "9.7p, 11.3c, 11.3c",
30 + "Age range": "16+",
31 + "Packaging": "Box",
32 + "Packaging size": "53.3 x 46 x 38.5 cm | (20\" x 18⅛\" x 15⅛\")",
33 + "Barcodes": "UPC: 673419267656 | EAN: 5702015869935",
34 + "LEGO item numbers": "NA: 6175771 | EU: 6175770",
35 + "Availability": "LEGO exclusive",
36 + "Notes": "Re-imagined version of 10179-1",
37 + "Rating": "✭✭✭✭✭ 4.4 | 1477 ratings, 10 reviews | Brickset review"
38 + }
39 + }
40 + },
41 + "expect": {
42 + "minCount": 1,
43 + "kinds": [
44 + "catalog_item",
45 + "price_observation"
46 + ]
47 + },
48 + "note": "Captured live from brickset.com",
49 + "capturedAt": "2026-09-07T05:45:41.482Z"
50 +}
\ No newline at end of file
added data/fixtures/discogs/nevermind-release-stats.json +62 −0
@@ -0,0 +1,62 @@
1 +{
2 + "raw": {
3 + "url": "https://www.discogs.com/release/1813006",
4 + "externalId": "release:1813006:stats",
5 + "kind": "price_observation",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:43:18.000Z",
8 + "payload": {
9 + "kind": "release_stats",
10 + "master": {
11 + "id": 13814,
12 + "title": "Nirvana - Nevermind",
13 + "year": "1991",
14 + "genre": [
15 + "Rock"
16 + ],
17 + "style": [
18 + "Grunge",
19 + "Alternative Rock"
20 + ],
21 + "cover_image": ""
22 + },
23 + "version": {
24 + "id": 1813006,
25 + "label": "DGC",
26 + "country": "US",
27 + "title": "Nevermind",
28 + "major_formats": [
29 + "Vinyl"
30 + ],
31 + "format": "LP, Album",
32 + "catno": "DGC-24425",
33 + "released": "1991",
34 + "thumb": "https://i.discogs.com/TcLhjpdmx3MmaXiqlQZbPOskOdPkxDcfwtS8TIp9tvo/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTE4MTMw/MDYtMTQzNjgxNDc2/Mi05NjgxLmpwZWc.jpeg",
35 + "stats": {
36 + "community": {
37 + "in_wantlist": 14023,
38 + "in_collection": 10951
39 + }
40 + }
41 + },
42 + "stats": {
43 + "num_for_sale": 11,
44 + "lowest_price": {
45 + "value": 1162.79,
46 + "currency": "USD"
47 + },
48 + "blocked_from_sale": false
49 + },
50 + "fetchedAt": "2026-09-07T05:43:18.000Z"
51 + }
52 + },
53 + "expect": {
54 + "minCount": 1,
55 + "kinds": [
56 + "catalog_item",
57 + "price_observation"
58 + ]
59 + },
60 + "note": "Captured live from api.discogs.com",
61 + "capturedAt": "2026-09-07T05:43:18.003Z"
62 +}
\ No newline at end of file
added data/fixtures/discogs/nevermind-versions.json +306 −0
@@ -0,0 +1,306 @@
1 +{
2 + "raw": {
3 + "url": "https://www.discogs.com/master/13814",
4 + "externalId": "master:13814:p1",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:43:15.710Z",
8 + "payload": {
9 + "kind": "master_versions",
10 + "master": {
11 + "id": 13814,
12 + "title": "Nirvana - Nevermind",
13 + "year": "1991",
14 + "genre": [
15 + "Rock"
16 + ],
17 + "style": [
18 + "Grunge",
19 + "Alternative Rock"
20 + ],
21 + "cover_image": ""
22 + },
23 + "versions": [
24 + {
25 + "id": 2082547,
26 + "label": "DGC",
27 + "country": "Argentina",
28 + "title": "Nevermind",
29 + "major_formats": [
30 + "Vinyl"
31 + ],
32 + "format": "LP, Album",
33 + "catno": "DGC TLP 24425, TLP 24425, TLP-24425",
34 + "released": "1991",
35 + "thumb": "https://i.discogs.com/3AK0BftPzDjRME31-ScSU1Nx7yI839KyDtKWx9fK0Eg/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTIwODI1/NDctMTU3MjY5NjEx/Ny00NTg2LmpwZWc.jpeg",
36 + "stats": {
37 + "community": {
38 + "in_wantlist": 1816,
39 + "in_collection": 460
40 + }
41 + }
42 + },
43 + {
44 + "id": 6865492,
45 + "label": "DGC",
46 + "country": "Mexico",
47 + "title": "Nevermind",
48 + "major_formats": [
49 + "Vinyl"
50 + ],
51 + "format": "LP, Album",
52 + "catno": "DGC2-4425",
53 + "released": "1991",
54 + "thumb": "https://i.discogs.com/7tGTg0bv4ljzj3w8h75DhFqHi1ATRttMbJfpjMJiIfI/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTY4NjU0/OTItMTcwMjIzMjEx/Mi03MTg4LmpwZWc.jpeg",
55 + "stats": {
56 + "community": {
57 + "in_wantlist": 1521,
58 + "in_collection": 199
59 + }
60 + }
61 + },
62 + {
63 + "id": 1813006,
64 + "label": "DGC",
65 + "country": "US",
66 + "title": "Nevermind",
67 + "major_formats": [
68 + "Vinyl"
69 + ],
70 + "format": "LP, Album",
71 + "catno": "DGC-24425",
72 + "released": "1991",
73 + "thumb": "https://i.discogs.com/TcLhjpdmx3MmaXiqlQZbPOskOdPkxDcfwtS8TIp9tvo/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTE4MTMw/MDYtMTQzNjgxNDc2/Mi05NjgxLmpwZWc.jpeg",
74 + "stats": {
75 + "community": {
76 + "in_wantlist": 14023,
77 + "in_collection": 10951
78 + }
79 + }
80 + },
81 + {
82 + "id": 22183597,
83 + "label": "Geffen Records",
84 + "country": "Mexico",
85 + "title": "Nevermind",
86 + "major_formats": [
87 + "Vinyl"
88 + ],
89 + "format": "LP, Album, Stereo",
90 + "catno": "DGC2-4425",
91 + "released": "1991",
92 + "thumb": "https://i.discogs.com/6r27an7rtqK8fNlFjSBc2o_sC3TVUBK_esYqKhY71lw/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTIyMTgz/NTk3LTE2NDUwMzcx/OTgtNDAzNy5qcGVn.jpeg",
93 + "stats": {
94 + "community": {
95 + "in_wantlist": 598,
96 + "in_collection": 22
97 + }
98 + }
99 + },
100 + {
101 + "id": 19132174,
102 + "label": "DGC",
103 + "country": "Argentina",
104 + "title": "Nevermind",
105 + "major_formats": [
106 + "Vinyl"
107 + ],
108 + "format": "LP, Album, Promo, Stereo",
109 + "catno": "TLP-24425, TLP 24425, DGC TLP 24425",
110 + "released": "1991",
111 + "thumb": "https://i.discogs.com/edB0zxxL4SFZ8qlSkIcrop_BqIT4UIUT_10U5f7Bn4g/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTE5MTMy/MTc0LTE2OTk3Njk4/MTEtODcwOC5qcGVn.jpeg",
112 + "stats": {
113 + "community": {
114 + "in_wantlist": 694,
115 + "in_collection": 14
116 + }
117 + }
118 + },
119 + {
120 + "id": 8638781,
121 + "label": "DGC",
122 + "country": "Europe",
123 + "title": "Nevermind",
124 + "major_formats": [
125 + "Vinyl"
126 + ],
127 + "format": "LP, Album, Misprint, Stereo",
128 + "catno": "GEF 24425, DGC 24425, GEF 24425-8",
129 + "released": "1991",
130 + "thumb": "https://i.discogs.com/LxP2ezlnvCN_aoq4Cc5pFc5mvSJvLtJpDj09Itz2DAQ/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTg2Mzg3/ODEtMTQ2NTY3NzU1/My04MjA5LmpwZWc.jpeg",
131 + "stats": {
132 + "community": {
133 + "in_wantlist": 2526,
134 + "in_collection": 5549
135 + }
136 + }
137 + },
138 + {
139 + "id": 7178280,
140 + "label": "DGC",
141 + "country": "France",
142 + "title": "Nevermind",
143 + "major_formats": [
144 + "Vinyl"
145 + ],
146 + "format": "LP, Album",
147 + "catno": "GEF 24425, GEF 24425 (5C), DGC 24425",
148 + "released": "1991",
149 + "thumb": "https://i.discogs.com/M3fhXq_-ViLnUkn7rk1xs0UOD1NbZwMq8zVTrJBw8VM/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTcxNzgy/ODAtMTQzNTQ4ODEz/NC03MzA4LmpwZWc.jpeg",
150 + "stats": {
151 + "community": {
152 + "in_wantlist": 1947,
153 + "in_collection": 964
154 + }
155 + }
156 + },
157 + {
158 + "id": 4090390,
159 + "label": "Geffen Records",
160 + "country": "Venezuela",
161 + "title": "Nevermind",
162 + "major_formats": [
163 + "Vinyl"
164 + ],
165 + "format": "LP, Album, Stereo",
166 + "catno": "GEFFEN-70.214",
167 + "released": "1991",
168 + "thumb": "https://i.discogs.com/Pu5XpL2mSbcmxKA2TnELlkuDsSRuXKItGsuj-RZmbQU/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTQwOTAz/OTAtMTQzNDIyNTcy/Ny02MDA3LmpwZWc.jpeg",
169 + "stats": {
170 + "community": {
171 + "in_wantlist": 1483,
172 + "in_collection": 185
173 + }
174 + }
175 + },
176 + {
177 + "id": 9255316,
178 + "label": "DGC",
179 + "country": "Brazil",
180 + "title": "Nevermind",
181 + "major_formats": [
182 + "Vinyl"
183 + ],
184 + "format": "LP, Album",
185 + "catno": "170.8040",
186 + "released": "1991",
187 + "thumb": "https://i.discogs.com/mIN3GVXpqZJfqz7EDTI1-R37EksISiX8o33fRVNICng/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTkyNTUz/MTYtMTQ3NzQ1MTM5/My02NzYyLmpwZWc.jpeg",
188 + "stats": {
189 + "community": {
190 + "in_wantlist": 1177,
191 + "in_collection": 152
192 + }
193 + }
194 + },
195 + {
196 + "id": 4485245,
197 + "label": "DGC",
198 + "country": "Spain",
199 + "title": "Nevermind",
200 + "major_formats": [
201 + "Vinyl"
202 + ],
203 + "format": "LP, Album",
204 + "catno": "5C GEF 24425, GEF 24425 (5C)",
205 + "released": "1991",
206 + "thumb": "https://i.discogs.com/ULj3uGgXVSfXCY-17geaMALwBxduteACq-YXwWuk-_M/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTQ0ODUy/NDUtMTQ1MDQ4NTkz/OC0xMDMwLmpwZWc.jpeg",
207 + "stats": {
208 + "community": {
209 + "in_wantlist": 1656,
210 + "in_collection": 611
211 + }
212 + }
213 + },
214 + {
215 + "id": 380426,
216 + "label": "DGC",
217 + "country": "Europe",
218 + "title": "Nevermind",
219 + "major_formats": [
220 + "Vinyl"
221 + ],
222 + "format": "LP, Album, Repress",
223 + "catno": "GEF 24425, DGC 24425, DGC-24425",
224 + "released": "1991",
225 + "thumb": "https://i.discogs.com/_seSApCcxZSVU9OyHDWwCrEUZL09RKH1p8lHK6jkv3c/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTM4MDQy/Ni0xMjYyOTUwODcy/LmpwZWc.jpeg",
226 + "stats": {
227 + "community": {
228 + "in_wantlist": 4305,
229 + "in_collection": 6678
230 + }
231 + }
232 + },
233 + {
234 + "id": 3269994,
235 + "label": "DGC",
236 + "country": "US",
237 + "title": "Nevermind",
238 + "major_formats": [
239 + "Vinyl"
240 + ],
241 + "format": "LP, Album, Club Edition",
242 + "catno": "DGC-24425",
243 + "released": "1991",
244 + "thumb": "https://i.discogs.com/0AHdu4HQ8ReEWX_x9pNdYUf31j2KAKvved3MvG_GwLQ/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTMyNjk5/OTQtMTQyODY3ODQz/Ny03NTMzLnBuZw.jpeg",
245 + "stats": {
246 + "community": {
247 + "in_wantlist": 2826,
248 + "in_collection": 760
249 + }
250 + }
251 + },
252 + {
253 + "id": 33756957,
254 + "label": "DGC (2)",
255 + "country": "Germany",
256 + "title": "Nevermind",
257 + "major_formats": [
258 + "Vinyl"
259 + ],
260 + "format": "LP, Album, Limited Edition, Picture Disc, Unofficial Release, Stereo",
261 + "catno": "GEF 24425, DGC 24425",
262 + "released": "1991",
263 + "thumb": "https://i.discogs.com/aCJl5x5lY2jHYGd3U-n2zzMOID157Ln24bZ6tnHmZbM/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTMzNzU2/OTU3LTE3NDUxNDkw/MzQtODkxNy5qcGVn.jpeg",
264 + "stats": {
265 + "community": {
266 + "in_wantlist": 213,
267 + "in_collection": 73
268 + }
269 + }
270 + },
271 + {
272 + "id": 2661648,
273 + "label": "DGC",
274 + "country": "Spain",
275 + "title": "Nevermind",
276 + "major_formats": [
277 + "Vinyl"
278 + ],
279 + "format": "LP, Album",
280 + "catno": "GEF 24425, GEF 24425 (5C), DGC-24425",
281 + "released": "1991",
282 + "thumb": "https://i.discogs.com/lGV61f9w9Bj12eIm0jfC5WBbDat0shfBTB7R7UVa7Xk/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTI2NjE2/NDgtMTM4MDgwMjQy/Mi05MzUyLmpwZWc.jpeg",
283 + "stats": {
284 + "community": {
285 + "in_wantlist": 2041,
286 + "in_collection": 2458
287 + }
288 + }
289 + }
290 + ],
291 + "page": 1
292 + }
293 + },
294 + "expect": {
295 + "minCount": 5,
296 + "kinds": [
297 + "catalog_item"
298 + ],
299 + "requiredFields": [
300 + "attributes.identifiers.discogs_release_id",
301 + "attributes.name"
302 + ]
303 + },
304 + "note": "Captured live from api.discogs.com",
305 + "capturedAt": "2026-09-07T05:43:15.721Z"
306 +}
\ No newline at end of file
added data/fixtures/lorcast/crd-3a299da6bf864690a188f07aeb55ffdf.json +65 −0
@@ -0,0 +1,65 @@
1 +{
2 + "raw": {
3 + "url": "https://lorcast.com/cards/crd_3a299da6bf864690a188f07aeb55ffdf",
4 + "externalId": "crd_3a299da6bf864690a188f07aeb55ffdf",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:40.718Z",
8 + "payload": {
9 + "card": {
10 + "id": "crd_3a299da6bf864690a188f07aeb55ffdf",
11 + "name": "A Whole New World",
12 + "layout": "normal",
13 + "released_at": "2023-08-18",
14 + "image_uris": {
15 + "digital": {
16 + "small": "https://cards.lorcast.io/card/digital/small/crd_3a299da6bf864690a188f07aeb55ffdf.avif?1709690747",
17 + "normal": "https://cards.lorcast.io/card/digital/normal/crd_3a299da6bf864690a188f07aeb55ffdf.avif?1709690747",
18 + "large": "https://cards.lorcast.io/card/digital/large/crd_3a299da6bf864690a188f07aeb55ffdf.avif?1709690747"
19 + }
20 + },
21 + "ink": "Steel",
22 + "type": [
23 + "Action",
24 + "Song"
25 + ],
26 + "rarity": "Super_rare",
27 + "illustrators": [
28 + "Koni"
29 + ],
30 + "collector_number": "195",
31 + "lang": "en",
32 + "set": {
33 + "id": "set_7ecb0e0c71af496a9e0110e23824e0a5",
34 + "code": "1",
35 + "name": "The First Chapter"
36 + },
37 + "tcgplayer_id": 506088,
38 + "prices": {
39 + "usd": 1.88,
40 + "usd_foil": 12.14
41 + }
42 + },
43 + "set": {
44 + "id": "set_7ecb0e0c71af496a9e0110e23824e0a5",
45 + "name": "The First Chapter",
46 + "code": "1",
47 + "released_at": "2023-08-18",
48 + "prereleased_at": "2023-08-18"
49 + }
50 + }
51 + },
52 + "expect": {
53 + "minCount": 1,
54 + "kinds": [
55 + "catalog_item",
56 + "price_observation"
57 + ],
58 + "requiredFields": [
59 + "attributes.setCode",
60 + "attributes.number"
61 + ]
62 + },
63 + "note": "Live capture https://lorcast.com/cards/crd_3a299da6bf864690a188f07aeb55ffdf",
64 + "capturedAt": "2026-09-07T05:50:40.721Z"
65 +}
\ No newline at end of file
added data/fixtures/lorcast/crd-e080b948d1bd4c87b40c050f56b2d50f.json +65 −0
@@ -0,0 +1,65 @@
1 +{
2 + "raw": {
3 + "url": "https://lorcast.com/cards/crd_e080b948d1bd4c87b40c050f56b2d50f",
4 + "externalId": "crd_e080b948d1bd4c87b40c050f56b2d50f",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:40.718Z",
8 + "payload": {
9 + "card": {
10 + "id": "crd_e080b948d1bd4c87b40c050f56b2d50f",
11 + "name": "Abu",
12 + "version": "Mischievous Monkey",
13 + "layout": "normal",
14 + "released_at": "2023-08-18",
15 + "image_uris": {
16 + "digital": {
17 + "small": "https://cards.lorcast.io/card/digital/small/crd_e080b948d1bd4c87b40c050f56b2d50f.avif?1709690747",
18 + "normal": "https://cards.lorcast.io/card/digital/normal/crd_e080b948d1bd4c87b40c050f56b2d50f.avif?1709690747",
19 + "large": "https://cards.lorcast.io/card/digital/large/crd_e080b948d1bd4c87b40c050f56b2d50f.avif?1709690747"
20 + }
21 + },
22 + "ink": "Ruby",
23 + "type": [
24 + "Character"
25 + ],
26 + "rarity": "Common",
27 + "illustrators": [
28 + "Oleg Yurkov"
29 + ],
30 + "collector_number": "103",
31 + "lang": "en",
32 + "set": {
33 + "id": "set_7ecb0e0c71af496a9e0110e23824e0a5",
34 + "code": "1",
35 + "name": "The First Chapter"
36 + },
37 + "tcgplayer_id": 507461,
38 + "prices": {
39 + "usd": 0.07,
40 + "usd_foil": 0.2
41 + }
42 + },
43 + "set": {
44 + "id": "set_7ecb0e0c71af496a9e0110e23824e0a5",
45 + "name": "The First Chapter",
46 + "code": "1",
47 + "released_at": "2023-08-18",
48 + "prereleased_at": "2023-08-18"
49 + }
50 + }
51 + },
52 + "expect": {
53 + "minCount": 1,
54 + "kinds": [
55 + "catalog_item",
56 + "price_observation"
57 + ],
58 + "requiredFields": [
59 + "attributes.setCode",
60 + "attributes.number"
61 + ]
62 + },
63 + "note": "Live capture https://lorcast.com/cards/crd_e080b948d1bd4c87b40c050f56b2d50f",
64 + "capturedAt": "2026-09-07T05:50:40.722Z"
65 +}
\ No newline at end of file
added data/fixtures/novelship/aj1-low-midnight-navy-product.json +43 −0
@@ -0,0 +1,43 @@
1 +{
2 + "raw": {
3 + "url": "https://novelship.com/air-jordan-1-low-midnight-navy-university-blue-553558-404",
4 + "externalId": "product:air-jordan-1-low-midnight-navy-university-blue-553558-404",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:48:37.969Z",
8 + "payload": {
9 + "kind": "browse_page",
10 + "url": "https://novelship.com/air-jordan-1-low-midnight-navy-university-blue-553558-404",
11 + "seed": "product:air-jordan-1-low-midnight-navy-university-blue-553558-404",
12 + "page": 1,
13 + "products": [
14 + {
15 + "id": 835272,
16 + "name": "Air Jordan 1 Low 'Midnight Navy University Blue' 553558-404",
17 + "nameSlug": "air-jordan-1-low-midnight-navy-university-blue-553558-404",
18 + "sku": "553558-404",
19 + "mainBrand": "Jordan",
20 + "subBrand": null,
21 + "colorway": "Midnight Navy/White/University Blue",
22 + "category": "Lifestyle Shoes (Basketball-Inspired)",
23 + "gender": "men",
24 + "dropDate": "2026-03-20",
25 + "costRetail": 120,
26 + "lastSalePrice": 200.9,
27 + "lowestListingPrice": 88,
28 + "salesCount180": 6,
29 + "image": "https://images.novelship.com/product/air_jordan_1_low__midnight_navy_university_blue__5_0_44983.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
30 + }
31 + ]
32 + }
33 + },
34 + "expect": {
35 + "minCount": 1,
36 + "first": {
37 + "attributes.identifiers.style_code": "553558-404",
38 + "attributes.categorySlug": "nike_jordan"
39 + }
40 + },
41 + "note": "Captured live from a novelship.com product page",
42 + "capturedAt": "2026-09-07T05:48:37.973Z"
43 +}
\ No newline at end of file
added data/fixtures/novelship/jordan-browse-page1.json +710 −0
@@ -0,0 +1,710 @@
1 +{
2 + "raw": {
3 + "url": "https://novelship.com/sneakers/jordan",
4 + "externalId": "browse:jordan:1",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:48:35.107Z",
8 + "payload": {
9 + "kind": "browse_page",
10 + "url": "https://novelship.com/sneakers/jordan",
11 + "seed": "jordan",
12 + "page": 1,
13 + "products": [
14 + {
15 + "id": 835272,
16 + "name": "Air Jordan 1 Low 'Midnight Navy University Blue' 553558-404",
17 + "nameSlug": "air-jordan-1-low-midnight-navy-university-blue-553558-404",
18 + "sku": "553558-404",
19 + "mainBrand": "Jordan",
20 + "subBrand": null,
21 + "colorway": null,
22 + "category": "Lifestyle Shoes (Basketball-Inspired)",
23 + "gender": "men",
24 + "dropDate": "2026-03-20",
25 + "costRetail": 120,
26 + "lastSalePrice": 200.9,
27 + "lowestListingPrice": 88,
28 + "salesCount180": null,
29 + "image": "https://images.novelship.com/product/air_jordan_1_low__midnight_navy_university_blue__5_0_44983.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
30 + },
31 + {
32 + "id": 159765,
33 + "name": "Air Jordan 1 Low SE 'Vivid Orange' FN7308-008",
34 + "nameSlug": "air-jordan-1-low-se-vivid-orange",
35 + "sku": "FN7308-008",
36 + "mainBrand": "Jordan",
37 + "subBrand": null,
38 + "colorway": null,
39 + "category": "Lifestyle Shoes (Basketball-Inspired)",
40 + "gender": "men",
41 + "dropDate": "2023-08-29",
42 + "costRetail": 120,
43 + "lastSalePrice": 211.7,
44 + "lowestListingPrice": 206,
45 + "salesCount180": null,
46 + "image": "https://images.novelship.com/product/air_jordan_1_low_se__vivid_orange__fn7308-008_0_18293.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
47 + },
48 + {
49 + "id": 81873,
50 + "name": "Air Jordan 1 Low 'Triple White' (2022) 553558-136",
51 + "nameSlug": "air-jordan-1-low-triple-white-2022",
52 + "sku": "553558-136",
53 + "mainBrand": "Jordan",
54 + "subBrand": null,
55 + "colorway": null,
56 + "category": "Lifestyle Shoes (Basketball-Inspire",
57 + "gender": "men",
58 + "dropDate": "2022-09-01",
59 + "costRetail": 110,
60 + "lastSalePrice": 152.2982,
61 + "lowestListingPrice": 118.715344,
62 + "salesCount180": null,
63 + "image": "https://images.novelship.com/product/air_jordan_1_low__triple_white___2022__553558-136_0_70762.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
64 + },
65 + {
66 + "id": 28177,
67 + "name": "Air Jordan 4 Retro 'Tattoo' BQ0897-006",
68 + "nameSlug": "air-jordan-4-retro-tattoo-2018",
69 + "sku": "BQ0897-006",
70 + "mainBrand": "Jordan",
71 + "subBrand": null,
72 + "colorway": null,
73 + "category": "Lifestyle Shoes (Basketball-Inspired)",
74 + "gender": "men",
75 + "dropDate": "2018-12-29",
76 + "costRetail": 225,
77 + "lastSalePrice": 516.375,
78 + "lowestListingPrice": 325.007582,
79 + "salesCount180": null,
80 + "image": "https://images.novelship.com/product/air_jordan_4_retro__tattoo__bq0897_006_0_77866.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
81 + },
82 + {
83 + "id": 70272,
84 + "name": "(Toddler) Travis Scott x Air Jordan 1 Low OG 'Reverse Mocha' DO5441-162",
85 + "nameSlug": "travis-scott-x-air-jordan-1-low-og-reverse-mocha-td",
86 + "sku": "DO5441-162",
87 + "mainBrand": "Jordan",
88 + "subBrand": null,
89 + "colorway": null,
90 + "category": "Toddler Shoes",
91 + "gender": "td",
92 + "dropDate": "2022-07-21",
93 + "costRetail": 50,
94 + "lastSalePrice": 351.9262,
95 + "lowestListingPrice": 259.487091,
96 + "salesCount180": null,
97 + "image": "https://images.novelship.com/product/_toddler__travis_scott_x_air_jordan_1_low_og__reve_0_23373.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
98 + },
99 + {
100 + "id": 676082,
101 + "name": "Air Jordan CMFT Era 'Black Mineral' HJ6777-002",
102 + "nameSlug": "air-jordan-cmft-era-black-mineral-hj-6777-002",
103 + "sku": "HJ6777-002",
104 + "mainBrand": "Jordan",
105 + "subBrand": null,
106 + "colorway": null,
107 + "category": "Lifestyle Shoes (Basketball-Inspired)",
108 + "gender": "men",
109 + "dropDate": "2025-04-01",
110 + "costRetail": 115,
111 + "lastSalePrice": 200.7074,
112 + "lowestListingPrice": 115.796115,
113 + "salesCount180": null,
114 + "image": "https://images.novelship.com/product/air_jordan_cmft_era__black_mineral__hj6777-002_0_88329.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
115 + },
116 + {
117 + "id": 90876,
118 + "name": "(Grade School) Air Jordan Hydro 11 Retro Slides 'Bred' AJ0022-006",
119 + "nameSlug": "jordan-hydro-11-retro-slides-bred-gs",
120 + "sku": "AJ0022-006",
121 + "mainBrand": "Jordan",
122 + "subBrand": null,
123 + "colorway": null,
124 + "category": "Slides",
125 + "gender": "youth",
126 + "dropDate": "2020-06-29",
127 + "costRetail": 224,
128 + "lastSalePrice": 59.7,
129 + "lowestListingPrice": 39,
130 + "salesCount180": null,
131 + "image": "https://images.novelship.com/product/_grade_school__air_jordan_hydro_11_retro_slides__b_0_7157.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
132 + },
133 + {
134 + "id": 1009428,
135 + "name": "Travis Scott x Air Jordan Air 1 Retro Low OG 'Sail Tropical Pink' IQ7604-101",
136 + "nameSlug": "travis-scott-x-air-jordan-air-1-retro-low-og-sail-tropical-pink-iq-7604-101",
137 + "sku": "IQ7604-101",
138 + "mainBrand": "Jordan",
139 + "subBrand": null,
140 + "colorway": null,
141 + "category": "Lifestyle Shoes (Basketball-Inspired)",
142 + "gender": "men",
143 + "dropDate": "2026-05-22",
144 + "costRetail": 155,
145 + "lastSalePrice": 647.1,
146 + "lowestListingPrice": 438,
147 + "salesCount180": null,
148 + "image": "https://images.novelship.com/product/travis_scott_x_air_jordan_air_1_retro_low_og__sail_0_64519.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
149 + },
150 + {
151 + "id": 701927,
152 + "name": "Nigel Sylvester x Air Jordan Air 4 Retro OG 'Sail' IQ8055-100",
153 + "nameSlug": "nigel-sylvester-x-air-jordan-air-4-retro-og-sail-iq-8055-100",
154 + "sku": "IQ8055-100",
155 + "mainBrand": "Jordan",
156 + "subBrand": null,
157 + "colorway": null,
158 + "category": "Lifestyle Shoes (Basketball-Inspired)",
159 + "gender": "men",
160 + "dropDate": "2026-06-10",
161 + "costRetail": 225,
162 + "lastSalePrice": 382.2505,
163 + "lowestListingPrice": 267,
164 + "salesCount180": null,
165 + "image": "https://images.novelship.com/product/nigel_sylvester_x_air_jordan_air_4_retro_og__sail__0_16538.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
166 + },
167 + {
168 + "id": 593408,
169 + "name": "Nike SB x Air Jordan Air 4 Retro SP 'Navy' DR5415-100",
170 + "nameSlug": "nike-sb-x-air-jordan-air-4-retro-sp-navy-dr-5415-100",
171 + "sku": "DR5415-100",
172 + "mainBrand": "Jordan",
173 + "subBrand": null,
174 + "colorway": null,
175 + "category": "Lifestyle Shoes (Skateboarding-Inspired)",
176 + "gender": "men",
177 + "dropDate": "2025-03-18",
178 + "costRetail": 225,
179 + "lastSalePrice": 284.1367,
180 + "lowestListingPrice": 202.88647,
181 + "salesCount180": null,
182 + "image": "https://images.novelship.com/product/nike_sb_x_air_jordan_air_4_retro_sp__navy__dr5415-_0_88700.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
183 + },
184 + {
185 + "id": 674988,
186 + "name": "Air Jordan 1 Low 'Sail College Grey' 553558-169",
187 + "nameSlug": "air-jordan-1-low-sail-college-grey-rattan-553558-169",
188 + "sku": "553558-169",
189 + "mainBrand": "Jordan",
190 + "subBrand": null,
191 + "colorway": null,
192 + "category": "Lifestyle Shoes (Basketball-Inspired)",
193 + "gender": "men",
194 + "dropDate": "2025-06-20",
195 + "costRetail": 120,
196 + "lastSalePrice": 159.1224,
197 + "lowestListingPrice": 105.361131,
198 + "salesCount180": null,
199 + "image": "https://images.novelship.com/product/air_jordan_1_low__sail_college_grey__553558-169_0_11039.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
200 + },
201 + {
202 + "id": 1018612,
203 + "name": "Air Jordan 1 Low OG 'Laser' IV6750-001",
204 + "nameSlug": "air-jordan-1-low-og-laser-iv-6750-001",
205 + "sku": "IV6750-001",
206 + "mainBrand": "Jordan",
207 + "subBrand": null,
208 + "colorway": null,
209 + "category": "Lifestyle Shoes (Basketball-Inspired)",
210 + "gender": "men",
211 + "dropDate": "2026-08-06",
212 + "costRetail": 165,
213 + "lastSalePrice": 190.3538,
214 + "lowestListingPrice": 124.715983,
215 + "salesCount180": null,
216 + "image": "https://images.novelship.com/product/air_jordan_1_low_og__laser__iv6750-001_0_50837.png?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
217 + },
218 + {
219 + "id": 567297,
220 + "name": "(Toddler) Travis Scott x Air Jordan Air 1 Low OG SP 'Medium Olive' DO5441-200",
221 + "nameSlug": "toddler-travis-scott-x-air-jordan-air-1-low-og-sp-medium-olive-do-5441-200",
222 + "sku": "DO5441-200",
223 + "mainBrand": "Jordan",
224 + "subBrand": null,
225 + "colorway": null,
226 + "category": "Kids Sneakers",
227 + "gender": "td",
228 + "dropDate": "2024-09-09",
229 + "costRetail": 50,
230 + "lastSalePrice": 198.6,
231 + "lowestListingPrice": 176,
232 + "salesCount180": null,
233 + "image": "https://images.novelship.com/product/_toddler__travis_scott_x_air_jordan_air_1_low_og_s_0_15971.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
234 + },
235 + {
236 + "id": 90743,
237 + "name": "Air Jordan 1 Retro Low OG 'Black Toe' CZ0790-106",
238 + "nameSlug": "air-jordan-1-low-og-varsity-red",
239 + "sku": "CZ0790-106",
240 + "mainBrand": "Jordan",
241 + "subBrand": null,
242 + "colorway": null,
243 + "category": "Lifestyle Shoes (Basketball-Inspired)",
244 + "gender": "men",
245 + "dropDate": "2023-08-04",
246 + "costRetail": 140,
247 + "lastSalePrice": 159.2,
248 + "lowestListingPrice": 105,
249 + "salesCount180": null,
250 + "image": "https://images.novelship.com/product/air_jordan_1_retro_low_og__black_toe__cz0790-106_0_14037.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
251 + },
252 + {
253 + "id": 1016393,
254 + "name": "Air Jordan 1 Retro Low OG 'Sail' HQ6998-100",
255 + "nameSlug": "air-jordan-1-retro-low-og-sail-hq-6998-100",
256 + "sku": "HQ6998-100",
257 + "mainBrand": "Jordan",
258 + "subBrand": null,
259 + "colorway": null,
260 + "category": "Lifestyle Shoes (Basketball-Inspired)",
261 + "gender": "men",
262 + "dropDate": "2026-08-01",
263 + "costRetail": 145,
264 + "lastSalePrice": 196.9,
265 + "lowestListingPrice": 175.802504,
266 + "salesCount180": null,
267 + "image": "https://images.novelship.com/product/air_jordan_1_retro_low_og__sail__hq6998-100_0_7313.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
268 + },
269 + {
270 + "id": 28417,
271 + "name": "Air Jordan 6 Retro 'Doernbecher' 2019 CI6293-416",
272 + "nameSlug": "air-jordan-6-retro-doernbecher-15th-anniversary",
273 + "sku": "CI6293-416",
274 + "mainBrand": "Jordan",
275 + "subBrand": null,
276 + "colorway": null,
277 + "category": "Lifestyle Shoes (Basketball-Inspired)",
278 + "gender": "men",
279 + "dropDate": "2019-02-23",
280 + "costRetail": 190,
281 + "lastSalePrice": 2154,
282 + "lowestListingPrice": 609.794665,
283 + "salesCount180": null,
284 + "image": "https://images.novelship.com/product/air_jordan_6_retro__doernbecher__2019_c"
285 + },
286 + {
287 + "id": 65764,
288 + "name": "Air Jordan 1 Low OG 'Bleached Coral' CZ0790-061",
289 + "nameSlug": "air-jordan-1-low-og-bleached-coral",
290 + "sku": "CZ0790-061",
291 + "mainBrand": "Jordan",
292 + "subBrand": null,
293 + "colorway": null,
294 + "category": "Lifestyle Shoes (Basketball-Inspired)",
295 + "gender": "men",
296 + "dropDate": "2022-07-02",
297 + "costRetail": 130,
298 + "lastSalePrice": 217,
299 + "lowestListingPrice": 136.572963,
300 + "salesCount180": null,
301 + "image": "https://images.novelship.com/product/air_jordan_1_low_og__bleached_coral__cz0790_061_0_3039.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
302 + },
303 + {
304 + "id": 676883,
305 + "name": "Air Jordan Stadium 90 'White University Blue' DX4397-141",
306 + "nameSlug": "air-jordan-stadium-90-white-university-blue-dx-4397-141",
307 + "sku": "DX4397-141",
308 + "mainBrand": "Jordan",
309 + "subBrand": null,
310 + "colorway": null,
311 + "category": "Lifestyle Shoes (Basketball-Inspired)",
312 + "gender": "men",
313 + "dropDate": "2024-12-20",
314 + "costRetail": 120,
315 + "lastSalePrice": null,
316 + "lowestListingPrice": 103.632657,
317 + "salesCount180": null,
318 + "image": "https://images.novelship.com/product/air_jordan_stadium_90__white_university_blue__dx43_0_16034.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
319 + },
320 + {
321 + "id": 131176,
322 + "name": "(Women) Air Jordan Nola Slide 'Pearl White' CZ8027-201",
323 + "nameSlug": "air-jordan-nola-slide-pearl-white-wmns",
324 + "sku": "CZ8027-201",
325 + "mainBrand": "Jordan",
326 + "subBrand": null,
327 + "colorway": null,
328 + "category": "Slides",
329 + "gender": "women",
330 + "dropDate": "2021-10-05",
331 + "costRetail": 40,
332 + "lastSalePrice": 92.2,
333 + "lowestListingPrice": 43,
334 + "salesCount180": null,
335 + "image": "https://images.novelship.com/product/_women__air_jordan_nola_slide__pearl_white__cz8027_0_98706.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
336 + },
337 + {
338 + "id": 676151,
339 + "name": "(Preschool) Air Jordan Spizike Low 'UNC' FQ3951-141",
340 + "nameSlug": "youth-air-jordan-spizike-low-unc-fq-3951-141",
341 + "sku": "FQ3951-141",
342 + "mainBrand": "Jordan",
343 + "subBrand": null,
344 + "colorway": null,
345 + "category": "Kids Sneakers",
346 + "gender": "ps",
347 + "dropDate": "2024-10-01",
348 + "costRetail": 80,
349 + "lastSalePrice": 92.2,
350 + "lowestListingPrice": 56,
351 + "salesCount180": null,
352 + "image": "https://images.novelship.com/product/_preschool__air_jordan_spizike_low__unc__fq3951-14_0_34338.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
353 + },
354 + {
355 + "id": 77182,
356 + "name": "(Grade School) Air Jordan 1 Mid 'Team Red' DQ8423-615",
357 + "nameSlug": "air-jordan-1-mid-team-red-gs",
358 + "sku": "DQ8423-615",
359 + "mainBrand": "Jordan",
360 + "subBrand": null,
361 + "colorway": null,
362 + "category": "Kids Sneakers",
363 + "gender": "youth",
364 + "dropDate": "2022-10-28",
365 + "costRetail": 110,
366 + "lastSalePrice": 185.53072,
367 + "lowestListingPrice": 86.286268,
368 + "salesCount180": null,
369 + "image": "https://images.novelship.com/product/_grade_school__air_jordan_1_mid__team_red__dq8423__0_16958.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
370 + },
371 + {
372 + "id": 422740,
373 + "name": "Air Jordan Luka 2 'Bright Mango' DX8733-800/DX9012-800",
374 + "nameSlug": "air-jordan-luka-2-bright-mango-dx-8733-800-dx-9012-800",
375 + "sku": "DX8733-800/DX9012-800",
376 + "mainBrand": "Jordan",
377 + "subBrand": null,
378 + "colorway": null,
379 + "category": "Basketball Shoes",
380 + "gender": "men",
381 + "dropDate": "2024-05-02",
382 + "costRetail": 130,
383 + "lastSalePrice": 78.4,
384 + "lowestListingPrice": 77,
385 + "salesCount180": null,
386 + "image": "https://images.novelship.com/product/air_jordan_luka_2__bright_mango__dx8733-800_dx9012_0_50014.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
387 + },
388 + {
389 + "id": 42000,
390 + "name": "Air Jordan 1 Mid 'Chicago' 554724-173",
391 + "nameSlug": "air-jordan-1-mid-chicago-2020",
392 + "sku": "554724-173",
393 + "mainBrand": "Jordan",
394 + "subBrand": null,
395 + "colorway": null,
396 + "category": "Lifestyle Shoes (Basketball-Inspired)",
397 + "gender": "men",
398 + "dropDate": "2020-09-14",
399 + "costRetail": 115,
400 + "lastSalePrice": 236.2,
401 + "lowestListingPrice": 160,
402 + "salesCount180": null,
403 + "image": "https://images.novelship.com/product/air_jordan_1_mid__chicago___also_worn_by______5547_0_59250.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
404 + },
405 + {
406 + "id": 676088,
407 + "name": "Air Jordan 1 Low 'Denim' IH0648-141",
408 + "nameSlug": "air-jordan-1-low-denim-ih-0648-141",
409 + "sku": "IH0648-141",
410 + "mainBrand": "Jordan",
411 + "subBrand": null,
412 + "colorway": null,
413 + "category": "Lifestyle Shoes (Basketball-Inspired)",
414 + "gender": "men",
415 + "dropDate": "2025-04-01",
416 + "costRetail": 125,
417 + "lastSalePrice": 149.8038,
418 + "lowestListingPrice": 146.448027,
419 + "salesCount180": null,
420 + "image": "https://images.novelship.com/product/air_jordan_1_low__denim__ih0648-141_0_4222.jpeg?fit=fi"
421 + },
422 + {
423 + "id": 86192,
424 + "name": "Air Jordan 1 High OG 'Vibrations of Naija' FD8631-100",
425 + "nameSlug": "air-jordan-1-high-og-vibrations-of-naija",
426 + "sku": "FD8631-100",
427 + "mainBrand": "Jordan",
428 + "subBrand": null,
429 + "colorway": null,
430 + "category": "Lifestyle Shoes (Basketball-Inspired)",
431 + "gender": "men",
432 + "dropDate": "2023-05-27",
433 + "costRetail": 180,
434 + "lastSalePrice": 252.14963,
435 + "lowestListingPrice": 127,
436 + "salesCount180": null,
437 + "image": "https://images.novelship.com/product/air_jordan_1_high_og__vibrations_of_naija__fd8631__0_80089.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
438 + },
439 + {
440 + "id": 75933,
441 + "name": "Air Jordan 2 Low 'Craft' DV9956-118",
442 + "nameSlug": "air-jordan-2-low-craft",
443 + "sku": "DV9956-118",
444 + "mainBrand": "Jordan",
445 + "subBrand": null,
446 + "colorway": null,
447 + "category": "Lifestyle Shoes (Basketball-Inspired)",
448 + "gender": "men",
449 + "dropDate": "2023-03-15",
450 + "costRetail": 150,
451 + "lastSalePrice": 112,
452 + "lowestListingPrice": 97,
453 + "salesCount180": null,
454 + "image": "https://images.novelship.com/product/air_jordan_2_low__craft__dv9956_118_0_55246.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
455 + },
456 + {
457 + "id": 802178,
458 + "name": "(Women) Air Jordan 1 Mid SE 'Valentine's Day' IB7018-600",
459 + "nameSlug": "women-air-jordan-1-mid-se-valentine-s-day-ib-7018-600",
460 + "sku": "IB7018-600",
461 + "mainBrand": "Jordan",
462 + "subBrand": null,
463 + "colorway": null,
464 + "category": "Lifestyle Shoes (Basketball-Inspired)",
465 + "gender": "women",
466 + "dropDate": "2026-02-01",
467 + "costRetail": 140,
468 + "lastSalePrice": 155,
469 + "lowestListingPrice": 143.436322,
470 + "salesCount180": null,
471 + "image": "https://images.novelship.com/product/_women__air_jordan_1_mid_se"
472 + },
473 + {
474 + "id": 57356,
475 + "name": "(Women) Air Jordan 1 Mid 'Dark Pony' DO7440-821",
476 + "nameSlug": "air-jordan-1-mid-dark-pony-wmns",
477 + "sku": "DO7440-821",
478 + "mainBrand": "Jordan",
479 + "subBrand": null,
480 + "colorway": null,
481 + "category": "Lifestyle Shoes (Basketball-Inspired)",
482 + "gender": "women",
483 + "dropDate": "2021-10-01",
484 + "costRetail": 115,
485 + "lastSalePrice": 155,
486 + "lowestListingPrice": 166.233918,
487 + "salesCount180": null,
488 + "image": "https://images.novelship.com/product/_women__air_jordan_1_mid__dark_pony__do7440_821_0_85700.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
489 + },
490 + {
491 + "id": 134307,
492 + "name": "Air Jordan Why Not Zer0.3 PF 'Triple White' CD3002-103",
493 + "nameSlug": "air-jordan-why-not-zer0-3-pf-triple-white",
494 + "sku": "CD3002-103",
495 + "mainBrand": "Jordan",
496 + "subBrand": null,
497 + "colorway": null,
498 + "category": "Basketball Shoes",
499 + "gender": "men",
500 + "dropDate": "2020-04-03",
501 + "costRetail": 130,
502 + "lastSalePrice": 143.9054,
503 + "lowestListingPrice": 201.102496,
504 + "salesCount180": null,
505 + "image": "https://images.novelship.com/product/1683556083610_AirJordanW0.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
506 + },
507 + {
508 + "id": 780760,
509 + "name": "Air Jordan Ultra 'Phantom' II3794-001",
510 + "nameSlug": "air-jordan-3-rm-phantom-summit-white-ii-3794-001",
511 + "sku": "II3794-001",
512 + "mainBrand": "Jordan",
513 + "subBrand": null,
514 + "colorway": null,
515 + "category": "Lifestyle Shoes (Basketball-Inspired)",
516 + "gender": "men",
517 + "dropDate": "2026-03-01",
518 + "costRetail": 145,
519 + "lastSalePrice": 340.5,
520 + "lowestListingPrice": 141.582644,
521 + "salesCount180": null,
522 + "image": "https://images.novelship.com/product/air_jordan_3_rm__phantom_summit_white__ii3794-001_0_20744.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
523 + },
524 + {
525 + "id": 75537,
526 + "name": "(Women) Air Jordan 1 Mid 'Multi-Color' DN3738-400",
527 + "nameSlug": "air-jordan-1-mid-multi-color",
528 + "sku": "DN3738-400",
529 + "mainBrand": "Jordan",
530 + "subBrand": null,
531 + "colorway": null,
532 + "category": "Lifestyle Shoes (Basketball-Inspired)",
533 + "gender": "women",
534 + "dropDate": "2022-10-12",
535 + "costRetail": 135,
536 + "lastSalePrice": 340.5,
537 + "lowestListingPrice": 255.270426,
538 + "salesCount180": null,
539 + "image": "https://images.novelship.com/product/_women__air_jordan_1_mid__multi-color__dn3738-400_0_89641.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
540 + },
541 + {
542 + "id": 35072,
543 + "name": "(Women) Air Jordan 1 Mid 'Digital Pink' CW5379-600",
544 + "nameSlug": "air-jordan-1-mid-digital-pink-wmns",
545 + "sku": "CW5379-600",
546 + "mainBrand": "Jordan",
547 + "subBrand": null,
548 + "colorway": null,
549 + "category": "Lifestyle Shoes (Basketball-Inspired)",
550 + "gender": "women",
551 + "dropDate": "2020-07-22",
552 + "costRetail": 125,
553 + "lastSalePrice": 153.12161,
554 + "lowestListingPrice": 222.83454,
555 + "salesCount180": null,
556 + "image": "https://images.novelship.com/product/_women__air_jordan_1_mid__digital_pink__cw5379_600_0_45377.jpeg?fit=fill&bg=FFFFFF&trim"
557 + },
558 + {
559 + "id": 336275,
560 + "name": "Air Jordan Air 200E 'Black Ash Green' DC9836-061",
561 + "nameSlug": "air-jordan-air-200-e-black-ash-green-dc-9836-061",
562 + "sku": "DC9836-061",
563 + "mainBrand": "Jordan",
564 + "subBrand": null,
565 + "colorway": null,
566 + "category": "Lifestyle Shoes (Basketball-Inspired)",
567 + "gender": "men",
568 + "dropDate": "2023-09-24",
569 + "costRetail": 135,
570 + "lastSalePrice": 329.5,
571 + "lowestListingPrice": 219,
572 + "salesCount180": null,
573 + "image": "https://images.novelship.com/product/air_jordan_air_200e__black_ash_green__dc9836-061_0_21650.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
574 + },
575 + {
576 + "id": 49720,
577 + "name": "(Women) Air Jordan 1 Mid 'Apricot' DH4270-800",
578 + "nameSlug": "air-jordan-1-mid-apricot-wmns",
579 + "sku": "DH4270-800",
580 + "mainBrand": "Jordan",
581 + "subBrand": null,
582 + "colorway": null,
583 + "category": "Lifestyle Shoes (Basketball-Inspired)",
584 + "gender": "women",
585 + "dropDate": "2021-03-04",
586 + "costRetail": 140,
587 + "lastSalePrice": 329.5,
588 + "lowestListingPrice": 149.853795,
589 + "salesCount180": null,
590 + "image": "https://images.novelship.com/product/_women__air_jordan_1_mid__apricot__dh4270_800_0_52103.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
591 + },
592 + {
593 + "id": 64878,
594 + "name": "(Grade School) Air Jordan 13 Retro 'Barons') 414574-115",
595 + "nameSlug": "air-jordan-13-retro-gs-barons",
596 + "sku": "414574-115",
597 + "mainBrand": "Jordan",
598 + "subBrand": null,
599 + "colorway": null,
600 + "category": "Kids Basketball Shoes",
601 + "gender": "youth",
602 + "dropDate": "2014-10-25",
603 + "costRetail": 140,
604 + "lastSalePrice": 243.90244,
605 + "lowestListingPrice": 134.608929,
606 + "salesCount180": null,
607 + "image": "https://images.novelship.com/product/_grade_school__air_jordan_13_retro__barons___41457_0_96833.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
608 + },
609 + {
610 + "id": 782374,
611 + "name": "(Toddler) Air Jordan 4 Retro 'Valentine's Day' IB7070-100",
612 + "nameSlug": "toddler-air-jordan-4-retro-valentine-s-day-2026-ib-7070-100",
613 + "sku": "IB7070-100",
614 + "mainBrand": "Jordan",
615 + "subBrand": null,
616 + "colorway": null,
617 + "category": "Toddler Shoes",
618 + "gender": "td",
619 + "dropDate": "2026-02-22",
620 + "costRetail": 85,
621 + "lastSalePrice": 200.8,
622 + "lowestListingPrice": 66.411341,
623 + "salesCount180": null,
624 + "image": "https://images.novelship.com/product/_toddler__air_jordan_4_retro__valentine_s_day__ib7_0_42920.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
625 + },
626 + {
627 + "id": 835401,
628 + "name": "Air Jordan 1 Low 'Olive Grey' 553558-046",
629 + "nameSlug": "air-jordan-1-low-olive-grey-553558-046",
630 + "sku": "553558-046",
631 + "mainBrand": "Jordan",
632 + "subBrand": null,
633 + "colorway": null,
634 + "category": "Lifestyle Shoes (Basketball-Inspired)",
635 + "gender": "men",
636 + "dropDate": "2026-03-20",
637 + "costRetail": 120,
638 + "lastSalePrice": 144.8,
639 + "lowestListingPrice": 107.038425,
640 + "salesCount180": null,
641 + "image": "https://images.novelship.com/product/air_jordan_1_low__olive_grey__553558-046_0_32805.png?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
642 + },
643 + {
644 + "id": 675047,
645 + "name": "Air Jordan 1 Mule Golf 'White Midnight Navy' FJ1214-103",
646 + "nameSlug": "air-jordan-1-mule-golf-white-midnight-navy-fj-1214-103",
647 + "sku": "FJ1214-103",
648 + "mainBrand": "Jordan",
649 + "subBrand": null,
650 + "colorway": null,
651 + "category": "Golf Shoes",
652 + "gender": "men",
653 + "dropDate": "2025-06-20",
654 + "costRetail": 115,
655 + "lastSalePrice": 123.7391,
656 + "lowestListingPrice": 95.011846,
657 + "salesCount180": null,
658 + "image": "https://images.novelship.com/product/air_jordan_1_mule_golf__white_midnight_navy__fj121_0_90838.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
659 + },
660 + {
661 + "id": 1016080,
662 + "name": "(Women) Air Jordan Mule Black/Metallic Silver HJ4292-001",
663 + "nameSlug": "women-air-jordan-mule-black-metallic-silver-hj-4292-001",
664 + "sku": "HJ4292-001",
665 + "mainBrand": "Jordan",
666 + "subBrand": null,
667 + "colorway": null,
668 + "category": "Lifestyle Shoes (Basketball-Inspired)",
669 + "gender": "women",
670 + "dropDate": "2025-07-14",
671 + "costRetail": 145,
672 + "lastSalePrice": 151.8649,
673 + "lowestListingPrice": 118.553165,
674 + "salesCount180": null,
675 + "image": "https://images.novelship.com/product/_women__air_jordan_mule_black_metallic_silver_hj42_0_80981.png?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
676 + },
677 + {
678 + "id": 86178,
679 + "name": "Air Jordan Legacy 312 'Exploration Unit' FB1875-141",
680 + "nameSlug": "jordan-legacy-312-exploration-unit",
681 + "sku": "FB1875-141",
682 + "mainBrand": "Jordan",
683 + "subBrand": null,
684 + "colorway": null,
685 + "category": "Lifestyle Shoes (Basketball-Inspired)",
686 + "gender": "men",
687 + "dropDate": "2022-01-01",
688 + "costRetail": 150,
689 + "lastSalePrice": 193.3225,
690 + "lowestListingPrice": 165.74738,
691 + "salesCount180": null,
692 + "image": "https://images.novelship.com/product/air_jordan_legacy_312__exploration_unit__fb1875-14_0_59907.jpeg?fit=fill&bg=FFFFFF&trim=color&auto=format,compress&q=75"
693 + }
694 + ]
695 + }
696 + },
697 + "expect": {
698 + "minCount": 30,
699 + "kinds": [
700 + "catalog_item",
701 + "price_observation",
702 + "listing"
703 + ],
704 + "requiredFields": [
705 + "attributes.identifiers.style_code"
706 + ]
707 + },
708 + "note": "Captured live from novelship.com/sneakers/jordan",
709 + "capturedAt": "2026-09-07T05:48:35.129Z"
710 +}
\ No newline at end of file
added data/fixtures/optcg/op01-015.json +45 −0
@@ -0,0 +1,45 @@
1 +{
2 + "raw": {
3 + "url": "https://optcgapi.com/cards/OP01-015/",
4 + "externalId": "OP01-015",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:41.375Z",
8 + "payload": {
9 + "card": {
10 + "card_set_id": "OP01-015",
11 + "card_name": "Tony Tony.Chopper",
12 + "set_name": "Romance Dawn",
13 + "set_id": "OP-01",
14 + "rarity": "UC",
15 + "card_color": "Red",
16 + "card_type": "Character",
17 + "card_cost": "3",
18 + "card_power": "4000",
19 + "sub_types": "Animal Straw Hat Crew",
20 + "attribute": "Wisdom",
21 + "card_image": "https://optcgapi.com/media/static/Card_Images/OP01-015.jpg",
22 + "market_price": 0.89,
23 + "inventory_price": 0.6,
24 + "date_scraped": "2026-09-06"
25 + },
26 + "set": {
27 + "set_name": "Romance Dawn",
28 + "set_id": "OP-01"
29 + }
30 + }
31 + },
32 + "expect": {
33 + "minCount": 1,
34 + "kinds": [
35 + "catalog_item",
36 + "price_observation"
37 + ],
38 + "requiredFields": [
39 + "attributes.setCode",
40 + "attributes.number"
41 + ]
42 + },
43 + "note": "Live capture https://optcgapi.com/cards/OP01-015/",
44 + "capturedAt": "2026-09-07T05:50:41.379Z"
45 +}
\ No newline at end of file
added data/fixtures/optcg/op01-077.json +45 −0
@@ -0,0 +1,45 @@
1 +{
2 + "raw": {
3 + "url": "https://optcgapi.com/cards/OP01-077/",
4 + "externalId": "OP01-077",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:41.375Z",
8 + "payload": {
9 + "card": {
10 + "card_set_id": "OP01-077",
11 + "card_name": "Perona",
12 + "set_name": "Romance Dawn",
13 + "set_id": "OP-01",
14 + "rarity": "UC",
15 + "card_color": "Blue",
16 + "card_type": "Character",
17 + "card_cost": "1",
18 + "card_power": "2000",
19 + "sub_types": "Thriller Bark Pirates",
20 + "attribute": "Special",
21 + "card_image": "https://optcgapi.com/media/static/Card_Images/OP01-077.jpg",
22 + "market_price": 0.76,
23 + "inventory_price": 0.56,
24 + "date_scraped": "2026-09-06"
25 + },
26 + "set": {
27 + "set_name": "Romance Dawn",
28 + "set_id": "OP-01"
29 + }
30 + }
31 + },
32 + "expect": {
33 + "minCount": 1,
34 + "kinds": [
35 + "catalog_item",
36 + "price_observation"
37 + ],
38 + "requiredFields": [
39 + "attributes.setCode",
40 + "attributes.number"
41 + ]
42 + },
43 + "note": "Live capture https://optcgapi.com/cards/OP01-077/",
44 + "capturedAt": "2026-09-07T05:50:41.378Z"
45 +}
\ No newline at end of file
added data/fixtures/pcgs-priceguide/morgan-dollar-ms.json +384 −0
@@ -0,0 +1,384 @@
1 +{
2 + "raw": {
3 + "url": "https://www.pcgs.com/prices/detail/morgan-dollar/744/most-active/ms",
4 + "externalId": "series:744:ms",
5 + "kind": "price_observation",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T05:47:45.128Z",
8 + "payload": {
9 + "kind": "price_guide_page",
10 + "url": "https://www.pcgs.com/prices/detail/morgan-dollar/744/most-active/ms",
11 + "seriesName": "Morgan Dollar",
12 + "seriesSlug": "morgan-dollar",
13 + "seriesId": "744",
14 + "lastUpdate": "09-06 10:59 PM EST",
15 + "grades": [
16 + "4",
17 + "12",
18 + "40",
19 + "55",
20 + "63",
21 + "64",
22 + "65",
23 + "66",
24 + "67",
25 + "68"
26 + ],
27 + "rows": [
28 + {
29 + "pcgsNumber": "7072",
30 + "description": "1878 8TF",
31 + "designation": "MS",
32 + "prices": {
33 + "4": {
34 + "value": 92,
35 + "plus": null
36 + },
37 + "12": {
38 + "value": 100,
39 + "plus": null
40 + },
41 + "40": {
42 + "value": 160,
43 + "plus": null
44 + },
45 + "55": {
46 + "value": 205,
47 + "plus": 220
48 + },
49 + "63": {
50 + "value": 525,
51 + "plus": 575
52 + },
53 + "64": {
54 + "value": 675,
55 + "plus": 950
56 + },
57 + "65": {
58 + "value": 2300,
59 + "plus": 3200
60 + },
61 + "66": {
62 + "value": 6000,
63 + "plus": 22000
64 + },
65 + "67": {
66 + "value": 72500,
67 + "plus": 85000
68 + },
69 + "68": {
70 + "value": 110000,
71 + "plus": null
72 + }
73 + }
74 + },
75 + {
76 + "pcgsNumber": "7073",
77 + "description": "1878 8TF",
78 + "designation": "PL",
79 + "prices": {
80 + "63": {
81 + "value": 625,
82 + "plus": 775
83 + },
84 + "64": {
85 + "value": 1400,
86 + "plus": 1550
87 + },
88 + "65": {
89 + "value": 5750,
90 + "plus": 10000
91 + }
92 + }
93 + },
94 + {
95 + "pcgsNumber": "97073",
96 + "description": "1878 8TF",
97 + "designation": "DMPL",
98 + "prices": {
99 + "63": {
100 + "value": 2150,
101 + "plus": 2350
102 + },
103 + "64": {
104 + "value": 3500,
105 + "plus": 7000
106 + },
107 + "65": {
108 + "value": 23500,
109 + "plus": 28500
110 + },
111 + "66": {
112 + "value": 52500,
113 + "plus": null
114 + }
115 + }
116 + },
117 + {
118 + "pcgsNumber": "7078",
119 + "description": "1878 7/8TF Strong",
120 + "designation": "MS",
121 + "prices": {
122 + "4": {
123 + "value": 92,
124 + "plus": null
125 + },
126 + "12": {
127 + "value": 92,
128 + "plus": null
129 + },
130 + "40": {
131 + "value": 110,
132 + "plus": null
133 + },
134 + "55": {
135 + "value": 175,
136 + "plus": 185
137 + },
138 + "63": {
139 + "value": 450,
140 + "plus": 500
141 + },
142 + "64": {
143 + "value": 650,
144 + "plus": 900
145 + },
146 + "65": {
147 + "value": 2500,
148 + "plus": 4250
149 + },
150 + "66": {
151 + "value": 10500,
152 + "plus": 60000
153 + }
154 + }
155 + },
156 + {
157 + "pcgsNumber": "7079",
158 + "description": "1878 7/8TF Strong",
159 + "designation": "PL",
160 + "prices": {
161 + "63": {
162 + "value": 575,
163 + "plus": 775
164 + },
165 + "64": {
166 + "value": 1400,
167 + "plus": 1500
168 + },
169 + "65": {
170 + "value": 5500,
171 + "plus": 7000
172 + }
173 + }
174 + },
175 + {
176 + "pcgsNumber": "97079",
177 + "description": "1878 7/8TF Strong",
178 + "designation": "DMPL",
179 + "prices": {
180 + "63": {
181 + "value": 2500,
182 + "plus": 3500
183 + },
184 + "64": {
185 + "value": 5000,
186 + "plus": 6000
187 + },
188 + "65": {
189 + "value": 15500,
190 + "plus": null
191 + }
192 + }
193 + },
194 + {
195 + "pcgsNumber": "7074",
196 + "description": "1878 7TF Reverse of 1878",
197 + "designation": "MS",
198 + "prices": {
199 + "4": {
200 + "value": 92,
201 + "plus": null
202 + },
203 + "12": {
204 + "value": 92,
205 + "plus": null
206 + },
207 + "40": {
208 + "value": 92,
209 + "plus": null
210 + },
211 + "55": {
212 + "value": 125,
213 + "plus": 125
214 + },
215 + "63": {
216 + "value": 225,
217 + "plus": 275
218 + },
219 + "64": {
220 + "value": 300,
221 + "plus": 525
222 + },
223 + "65": {
224 + "value": 750,
225 + "plus": 2000
226 + },
227 + "66": {
228 + "value": 2650,
229 + "plus": 23500
230 + },
231 + "67": {
232 + "value": 35000,
233 + "plus": null
234 + }
235 + }
236 + },
237 + {
238 + "pcgsNumber": "7075",
239 + "description": "1878 7TF Reverse of 1878",
240 + "designation": "PL",
241 + "prices": {
242 + "63": {
243 + "value": 400,
244 + "plus": 450
245 + },
246 + "64": {
247 + "value": 650,
248 + "plus": 700
249 + },
250 + "65": {
251 + "value": 2650,
252 + "plus": 3250
253 + },
254 + "66": {
255 + "value": 6500,
256 + "plus": null
257 + }
258 + }
259 + },
260 + {
261 + "pcgsNumber": "97075",
262 + "description": "1878 7TF Reverse of 1878",
263 + "designation": "DMPL",
264 + "prices": {
265 + "63": {
266 + "value": 850,
267 + "plus": 1000
268 + },
269 + "64": {
270 + "value": 1850,
271 + "plus": 6500
272 + },
273 + "65": {
274 + "value": 7500,
275 + "plus": 16500
276 + },
277 + "66": {
278 + "value": 45000,
279 + "plus": null
280 + }
281 + }
282 + },
283 + {
284 + "pcgsNumber": "7076",
285 + "description": "1878 7TF Reverse of 1879",
286 + "designation": "MS",
287 + "prices": {
288 + "4": {
289 + "value": 92,
290 + "plus": null
291 + },
292 + "12": {
293 + "value": 92,
294 + "plus": null
295 + },
296 + "40": {
297 + "value": 92,
298 + "plus": null
299 + },
300 + "55": {
301 + "value": 150,
302 + "plus": 160
303 + },
304 + "63": {
305 + "value": 450,
306 + "plus": 550
307 + },
308 + "64": {
309 + "value": 800,
310 + "plus": 1150
311 + },
312 + "65": {
313 + "value": 2150,
314 + "plus": 2750
315 + },
316 + "66": {
317 + "value": 6500,
318 + "plus": 8250
319 + },
320 + "67": {
321 + "value": 60000,
322 + "plus": null
323 + }
324 + }
325 + },
326 + {
327 + "pcgsNumber": "7077",
328 + "description": "1878 7TF Reverse of 1879",
329 + "designation": "PL",
330 + "prices": {
331 + "63": {
332 + "value": 625,
333 + "plus": 900
334 + },
335 + "64": {
336 + "value": 1550,
337 + "plus": 1650
338 + },
339 + "65": {
340 + "value": 4750,
341 + "plus": 6750
342 + },
343 + "66": {
344 + "value": 20000,
345 + "plus": null
346 + }
347 + }
348 + },
349 + {
350 + "pcgsNumber": "97077",
351 + "description": "1878 7TF Reverse of 1879",
352 + "designation": "DMPL",
353 + "prices": {
354 + "63": {
355 + "value": 1650,
356 + "plus": 1900
357 + },
358 + "64": {
359 + "value": 4500,
360 + "plus": 5000
361 + },
362 + "65": {
363 + "value": 16000,
364 + "plus": 30000
365 + }
366 + }
367 + }
368 + ]
369 + }
370 + },
371 + "expect": {
372 + "minCount": 20,
373 + "kinds": [
374 + "catalog_item",
375 + "price_observation"
376 + ],
377 + "first": {
378 + "attributes.categorySlug": "coins",
379 + "attributes.series": "Morgan Dollar"
380 + }
381 + },
382 + "note": "Captured live via Firecrawl; rows truncated to 12",
383 + "capturedAt": "2026-09-07T05:47:45.221Z"
384 +}
\ No newline at end of file
added data/fixtures/pricecharting/magic-alpha__black-lotus.json +345 −0
@@ -0,0 +1,345 @@
1 +{
2 + "raw": {
3 + "url": "https://www.pricecharting.com/game/magic-alpha/black-lotus",
4 + "externalId": "2244625",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:37.206Z",
8 + "payload": {
9 + "kind": "product",
10 + "url": "https://www.pricecharting.com/game/magic-alpha/black-lotus",
11 + "productId": "2244625",
12 + "consoleUri": "magic-alpha",
13 + "consoleName": "Magic Alpha",
14 + "title": "Black Lotus",
15 + "flags": {
16 + "isComic": false,
17 + "isLegoSet": false,
18 + "isFunkoPop": false,
19 + "isCard": true,
20 + "isCoin": false,
21 + "isSystem": false
22 + },
23 + "columnLabels": [
24 + "Ungraded",
25 + "Grade 7",
26 + "Grade 8",
27 + "Grade 9",
28 + "Grade 9.5",
29 + "PSA 10",
30 + "Grade 9",
31 + "Grade 9.5",
32 + "PSA 10"
33 + ],
34 + "prices": [
35 + {
36 + "key": "used_price",
37 + "value": 64098.91,
38 + "raw": "$64,098.91"
39 + },
40 + {
41 + "key": "complete_price",
42 + "value": 56000,
43 + "raw": "$56,000.00"
44 + },
45 + {
46 + "key": "new_price",
47 + "value": 70000,
48 + "raw": "$70,000.00"
49 + },
50 + {
51 + "key": "graded_price",
52 + "value": 75684.34,
53 + "raw": "$75,684.34"
54 + },
55 + {
56 + "key": "box_only_price",
57 + "value": 83253,
58 + "raw": "$83,253.00"
59 + },
60 + {
61 + "key": "manual_only_price",
62 + "value": 90821,
63 + "raw": "$90,821.00"
64 + }
65 + ],
66 + "fullPrices": [
67 + {
68 + "label": "Ungraded",
69 + "value": 64098.91
70 + },
71 + {
72 + "label": "Grade 1",
73 + "value": null
74 + },
75 + {
76 + "label": "Grade 2",
77 + "value": 165
78 + },
79 + {
80 + "label": "Grade 3",
81 + "value": 183
82 + },
83 + {
84 + "label": "Grade 4",
85 + "value": 215
86 + },
87 + {
88 + "label": "Grade 5",
89 + "value": 253
90 + },
91 + {
92 + "label": "Grade 6",
93 + "value": 316
94 + },
95 + {
96 + "label": "Grade 7",
97 + "value": 56000
98 + },
99 + {
100 + "label": "Grade 8",
101 + "value": 70000
102 + },
103 + {
104 + "label": "Grade 9",
105 + "value": 75684.34
106 + },
107 + {
108 + "label": "Grade 9.5",
109 + "value": 83253
110 + },
111 + {
112 + "label": "TAG 10",
113 + "value": null
114 + },
115 + {
116 + "label": "ACE 10",
117 + "value": null
118 + },
119 + {
120 + "label": "SGC 10",
121 + "value": 54493
122 + },
123 + {
124 + "label": "CGC 10",
125 + "value": 54493
126 + },
127 + {
128 + "label": "PSA 10",
129 + "value": 90821
130 + },
131 + {
132 + "label": "BGS 10",
133 + "value": 118067
134 + },
135 + {
136 + "label": "BGS 10 Black",
137 + "value": 590335
138 + },
139 + {
140 + "label": "CGC 10 Pristine",
141 + "value": 98087
142 + }
143 + ],
144 + "tabLabels": {
145 + "used": "Ungraded",
146 + "grade-twenty": "BGS 10 Black",
147 + "grade-nineteen": "CGC 10 Prist.",
148 + "manual-only": "PSA 10",
149 + "loose-and-box": "BGS 10",
150 + "grade-seventeen": "CGC 10",
151 + "grade-eighteen": "SGC 10",
152 + "grade-twenty-one": "TAG 10",
153 + "grade-twenty-two": "ACE 10",
154 + "box-only": "Grade 9.5",
155 + "graded": "Grade 9",
156 + "new": "Grade 8",
157 + "cib": "Grade 7",
158 + "grade-six": "Grade 6",
159 + "grade-five": "Grade 5",
160 + "grade-four": "Grade 4",
161 + "grade-three": "Grade 3",
162 + "box-and-manual": "Grade 2",
163 + "loose-and-manual": "Grade 1"
164 + },
165 + "sales": [
166 + {
167 + "tab": "used",
168 + "date": "2026-07-02",
169 + "title": "mtg black lotus alpha magic the gathering - lightly played but good condition",
170 + "price": 33767.5,
171 + "ebayId": "127949551212",
172 + "listedPrice": null
173 + },
174 + {
175 + "tab": "used",
176 + "date": "2026-05-12",
177 + "title": "MTG ALPHA BLACK LOTUS - Magic the Gathering - Get it while you can!",
178 + "price": 73999,
179 + "ebayId": "267494594619",
180 + "listedPrice": null
181 + },
182 + {
183 + "tab": "used",
184 + "date": "2026-05-11",
185 + "title": "***Alpha Black Lotus (Edge Ding/Small Indent)** MTG Alpha Magic Kid Icarus",
186 + "price": 57469,
187 + "ebayId": "198331003446",
188 + "listedPrice": null
189 + },
190 + {
191 + "tab": "cib",
192 + "date": "2022-06-11",
193 + "title": "Magic: The Gathering Black Lotus Limited Edition (Alpha) BGS Trading Card Game 7",
194 + "price": 87000,
195 + "ebayId": null,
196 + "listedPrice": null
197 + },
198 + {
199 + "tab": "cib",
200 + "date": "2021-07-27",
201 + "title": "1993 Magic The Gathering MTG Alpha Black Lotus R A BGS 7.5 NRMT+",
202 + "price": 76210,
203 + "ebayId": "402996163183",
204 + "listedPrice": null
205 + },
206 + {
207 + "tab": "new",
208 + "date": "2024-07-13",
209 + "title": "1993 Magic The Gathering MTG Alpha Black Lotus BGS 8 NM-MT",
210 + "price": 70000,
211 + "ebayId": null,
212 + "listedPrice": null
213 + },
214 + {
215 + "tab": "new",
216 + "date": "2024-05-20",
217 + "title": "***BGS 8.5 Alpha Black Lotus*** MTG Alpha Magic Kid Icarus",
218 + "price": 66800,
219 + "ebayId": "196375557903",
220 + "listedPrice": null
221 + },
222 + {
223 + "tab": "new",
224 + "date": "2024-01-13",
225 + "title": "1993 Magic The Gathering MTG Alpha Black Lotus BGS 8 NM-MT",
226 + "price": 88800,
227 + "ebayId": null,
228 + "listedPrice": null
229 + },
230 + {
231 + "tab": "graded",
232 + "date": "2025-08-03",
233 + "title": "1993 Magic: The Gathering Limited Edition Alpha Rare Black Lotus - CGC MINT 9",
234 + "price": 113444,
235 + "ebayId": null,
236 + "listedPrice": null
237 + },
238 + {
239 + "tab": "graded",
240 + "date": "2025-04-22",
241 + "title": "Magic MTG Black Ward - BETA CGC 9 -ALPHA LOTUS STARDER BREAK (school pre-modern)",
242 + "price": 38,
243 + "ebayId": "236051350066",
244 + "listedPrice": null
245 + },
246 + {
247 + "tab": "graded",
248 + "date": "2022-12-02",
249 + "title": "Magic: The Gathering Black Lotus Limited Edition (Alpha) BGS Trading Card Game MINT 9",
250 + "price": 174000,
251 + "ebayId": null,
252 + "listedPrice": null
253 + },
254 + {
255 + "tab": "manual-only",
256 + "date": "2026-03-09",
257 + "title": "Magic: The Gathering Alpha Black Lotus PSA 10 #40 1993 English Artifact Card #40",
258 + "price": 16,
259 + "ebayId": "236676023914",
260 + "listedPrice": null
261 + },
262 + {
263 + "tab": "grade-three",
264 + "date": "2026-08-14",
265 + "title": "1993 MTG LIMITED EDITION ALPHA BLACK LOTUS PSA 3",
266 + "price": 55000,
267 + "ebayId": "307126685877",
268 + "listedPrice": null
269 + },
270 + {
271 + "tab": "grade-three",
272 + "date": "2026-04-01",
273 + "title": "***PSA 3 Alpha Black Lotus*** MTG Alpha Magic Power 9 Kid Icarus",
274 + "price": 54999.99,
275 + "ebayId": "198142754837",
276 + "listedPrice": null
277 + },
278 + {
279 + "tab": "grade-three",
280 + "date": "2026-01-07",
281 + "title": "MTG ALPHA BLACK LOTUS Beckett BGS Graded 3 MAGIC THE GATHERING (8.5 Centering)",
282 + "price": 48900,
283 + "ebayId": "226303746780",
284 + "listedPrice": 53795
285 + },
286 + {
287 + "tab": "grade-four",
288 + "date": "2025-09-24",
289 + "title": "1993 MAGIC THE GATHERING MTG ALPHA BLACK LOTUS PSA 4",
290 + "price": 57400,
291 + "ebayId": "116781594842",
292 + "listedPrice": null
293 + },
294 + {
295 + "tab": "grade-five",
296 + "date": "2024-07-01",
297 + "title": "MTG Magic the Gathering Alpha Black Lotus BGS 5.5",
298 + "price": 37000,
299 + "ebayId": "176432852068",
300 + "listedPrice": null
301 + },
302 + {
303 + "tab": "grade-six",
304 + "date": "2022-02-22",
305 + "title": "1993 Magic The Gathering MTG Alpha Channel U G BGS 6.5 EX-MT+",
306 + "price": 316,
307 + "ebayId": "363728217497",
308 + "listedPrice": null
309 + }
310 + ],
311 + "details": {
312 + "See": "Population Report",
313 + "Genre": "Magic Card",
314 + "Release Date": "August 5, 1993",
315 + "PriceCharting ID": "2244625"
316 + },
317 + "images": [
318 + "https://storage.googleapis.com/images.pricecharting.com/38b84ac4d2f774578e261b1a49980b6e2ea289de22ff7b804ae9d7d24888a410/240.jpg",
319 + "https://storage.googleapis.com/images.pricecharting.com/38b84ac4d2f774578e261b1a49980b6e2ea289de22ff7b804ae9d7d24888a410/1600.jpg"
320 + ],
321 + "setRef": {
322 + "id": "lea",
323 + "code": "LEA",
324 + "name": "Limited Edition Alpha",
325 + "source": "scryfall"
326 + },
327 + "cardRef": {
328 + "scryfall_id": "b0faa7f2-b547-42c4-a810-839da50dadfe",
329 + "number": "232",
330 + "tcgplayer_id": "1042"
331 + },
332 + "site": "pricecharting"
333 + }
334 + },
335 + "expect": {
336 + "minCount": 3,
337 + "kinds": [
338 + "catalog_item",
339 + "price_observation",
340 + "sale"
341 + ]
342 + },
343 + "note": "Live capture of https://www.pricecharting.com/game/magic-alpha/black-lotus (sales trimmed to 3 rows per grade tab).",
344 + "capturedAt": "2026-09-07T05:50:37.389Z"
345 +}
\ No newline at end of file
added data/fixtures/pricecharting/pokemon-base-set__charizard-1st-edition-4.json +523 −0
@@ -0,0 +1,523 @@
1 +{
2 + "raw": {
3 + "url": "https://www.pricecharting.com/game/pokemon-base-set/charizard-1st-edition-4",
4 + "externalId": "715593",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:36.348Z",
8 + "payload": {
9 + "kind": "product",
10 + "url": "https://www.pricecharting.com/game/pokemon-base-set/charizard-1st-edition-4",
11 + "productId": "715593",
12 + "consoleUri": "pokemon-base-set",
13 + "consoleName": "Pokemon Base Set",
14 + "title": "Charizard [1st Edition] #4",
15 + "flags": {
16 + "isComic": false,
17 + "isLegoSet": false,
18 + "isFunkoPop": false,
19 + "isCard": true,
20 + "isCoin": false,
21 + "isSystem": false
22 + },
23 + "columnLabels": [
24 + "Ungraded",
25 + "Grade 7",
26 + "Grade 8",
27 + "Grade 9",
28 + "Grade 9.5",
29 + "PSA 10",
30 + "Grade 9",
31 + "Grade 9.5",
32 + "PSA 10"
33 + ],
34 + "prices": [
35 + {
36 + "key": "used_price",
37 + "value": 10100,
38 + "raw": "$10,100.00"
39 + },
40 + {
41 + "key": "complete_price",
42 + "value": 18605,
43 + "raw": "$18,605.00"
44 + },
45 + {
46 + "key": "new_price",
47 + "value": 29012.9,
48 + "raw": "$29,012.90"
49 + },
50 + {
51 + "key": "graded_price",
52 + "value": 46693.05,
53 + "raw": "$46,693.05"
54 + },
55 + {
56 + "key": "box_only_price",
57 + "value": 75434.08,
58 + "raw": "$75,434.08"
59 + },
60 + {
61 + "key": "manual_only_price",
62 + "value": 343098,
63 + "raw": "$343,098.00"
64 + }
65 + ],
66 + "fullPrices": [
67 + {
68 + "label": "Ungraded",
69 + "value": 10100
70 + },
71 + {
72 + "label": "Grade 1",
73 + "value": 9051.56
74 + },
75 + {
76 + "label": "Grade 2",
77 + "value": 9000
78 + },
79 + {
80 + "label": "Grade 3",
81 + "value": 9650
82 + },
83 + {
84 + "label": "Grade 4",
85 + "value": 10913.49
86 + },
87 + {
88 + "label": "Grade 5",
89 + "value": 13485
90 + },
91 + {
92 + "label": "Grade 6",
93 + "value": 16268.4
94 + },
95 + {
96 + "label": "Grade 7",
97 + "value": 18605
98 + },
99 + {
100 + "label": "Grade 8",
101 + "value": 29012.9
102 + },
103 + {
104 + "label": "Grade 9",
105 + "value": 46693.05
106 + },
107 + {
108 + "label": "Grade 9.5",
109 + "value": 75434.08
110 + },
111 + {
112 + "label": "TAG 10",
113 + "value": null
114 + },
115 + {
116 + "label": "ACE 10",
117 + "value": null
118 + },
119 + {
120 + "label": "SGC 10",
121 + "value": 205859
122 + },
123 + {
124 + "label": "CGC 10",
125 + "value": 63885.1
126 + },
127 + {
128 + "label": "PSA 10",
129 + "value": 343098
130 + },
131 + {
132 + "label": "BGS 10",
133 + "value": 446027
134 + },
135 + {
136 + "label": "BGS 10 Black",
137 + "value": 2230135
138 + },
139 + {
140 + "label": "CGC 10 Pristine",
141 + "value": 95340
142 + }
143 + ],
144 + "tabLabels": {
145 + "used": "Ungraded",
146 + "grade-twenty": "BGS 10 Black",
147 + "grade-nineteen": "CGC 10 Prist.",
148 + "manual-only": "PSA 10",
149 + "loose-and-box": "BGS 10",
150 + "grade-seventeen": "CGC 10",
151 + "grade-eighteen": "SGC 10",
152 + "grade-twenty-one": "TAG 10",
153 + "grade-twenty-two": "ACE 10",
154 + "box-only": "Grade 9.5",
155 + "graded": "Grade 9",
156 + "new": "Grade 8",
157 + "cib": "Grade 7",
158 + "grade-six": "Grade 6",
159 + "grade-five": "Grade 5",
160 + "grade-four": "Grade 4",
161 + "grade-three": "Grade 3",
162 + "box-and-manual": "Grade 2",
163 + "loose-and-manual": "Grade 1"
164 + },
165 + "sales": [
166 + {
167 + "tab": "used",
168 + "date": "2026-09-04",
169 + "title": "1999 Pokemon Base Set Shadowless 1st Edition #4/102 Charizard Holo",
170 + "price": 12700,
171 + "ebayId": "377451975143",
172 + "listedPrice": null
173 + },
174 + {
175 + "tab": "used",
176 + "date": "2026-08-31",
177 + "title": "1999 Pokémon Base Set 1st Edition Shadowless Charizard #4/102, 4/102",
178 + "price": 7500,
179 + "ebayId": "318801328311",
180 + "listedPrice": null
181 + },
182 + {
183 + "tab": "used",
184 + "date": "2026-08-20",
185 + "title": "1st Edition Shadowless Charizard 4/102 Base Holo Vintage 1999 Pokemon Card #102",
186 + "price": 12706,
187 + "ebayId": "137614550471",
188 + "listedPrice": null
189 + },
190 + {
191 + "tab": "cib",
192 + "date": "2026-08-16",
193 + "title": "1999 POKEMON BASE SET 1ST EDITION #4 CHARIZARD-HOLO PSA 7 #4",
194 + "price": 20200,
195 + "ebayId": "298567992825",
196 + "listedPrice": null
197 + },
198 + {
199 + "tab": "cib",
200 + "date": "2026-08-12",
201 + "title": "Pokémon Charizard 1st Edition Shadowless Base Set BGS 7 THICK Stamp 4/102 4/102",
202 + "price": 17999,
203 + "ebayId": "287363487315",
204 + "listedPrice": null
205 + },
206 + {
207 + "tab": "cib",
208 + "date": "2026-08-10",
209 + "title": "1999 Pokemon Base Set 1st Edition Holo #4 Charizard - CGC NM 7",
210 + "price": 13847,
211 + "ebayId": null,
212 + "listedPrice": null
213 + },
214 + {
215 + "tab": "new",
216 + "date": "2026-07-31",
217 + "title": "1999 Pokemon Base Set 1st Edition Charizard Holo 4/102 BGS 8.5 QUAD SUBGRADES 4/102",
218 + "price": 23700,
219 + "ebayId": "267737894662",
220 + "listedPrice": null
221 + },
222 + {
223 + "tab": "new",
224 + "date": "2026-07-30",
225 + "title": "Pokemon 1st Ed Base Set Shadowless Thick Stamp Charizard Holo BGS 8.5 MBA SILVER 4/102",
226 + "price": 28100,
227 + "ebayId": "800391207928",
228 + "listedPrice": null
229 + },
230 + {
231 + "tab": "new",
232 + "date": "2026-07-19",
233 + "title": "1999 POKEMON BASE SET 1ST EDITION #4 CHARIZARD-HOLO PSA 8 #4",
234 + "price": 35000,
235 + "ebayId": "298506859622",
236 + "listedPrice": null
237 + },
238 + {
239 + "tab": "graded",
240 + "date": "2026-08-31",
241 + "title": "1999 Pokemon Base Set 1st Edition Holo #4 Charizard - PSA NM-MT+ 8.5",
242 + "price": 46360,
243 + "ebayId": null,
244 + "listedPrice": null
245 + },
246 + {
247 + "tab": "graded",
248 + "date": "2026-08-20",
249 + "title": "1999 Pokemon Base Set 1st Edition Shadowless Charizard HOLO R (Thin Stamp) BGS 9 4/102",
250 + "price": 40100,
251 + "ebayId": "800499471429",
252 + "listedPrice": null
253 + },
254 + {
255 + "tab": "graded",
256 + "date": "2026-08-06",
257 + "title": "1999 Pokemon Base Set 1st Edition Shadowless Charizard HOLO R (Thin Stamp) BGS 9 4/102",
258 + "price": 39078,
259 + "ebayId": "800429683647",
260 + "listedPrice": null
261 + },
262 + {
263 + "tab": "box-only",
264 + "date": "2026-05-28",
265 + "title": "1999 Pokemon Base Set 1st Edition Shadowless Charizard Holo Thick Stamp BGS 9.5",
266 + "price": 70000,
267 + "ebayId": "287349668112",
268 + "listedPrice": null
269 + },
270 + {
271 + "tab": "box-only",
272 + "date": "2026-05-21",
273 + "title": "1999 Pokemon 1st Edition Base Set Shadowless Charizard #4 Holo R BGS 9.5 4/102",
274 + "price": 70401,
275 + "ebayId": "800015034294",
276 + "listedPrice": null
277 + },
278 + {
279 + "tab": "box-only",
280 + "date": "2026-05-07",
281 + "title": "1999 Pokemon Base Set 1st Edition Shadowless Charizard Holo Thick Stamp BGS 9.5",
282 + "price": 80605,
283 + "ebayId": "389954046683",
284 + "listedPrice": null
285 + },
286 + {
287 + "tab": "manual-only",
288 + "date": "2026-06-23",
289 + "title": "1999 Pokemon PSA 10 Base Set 1st Edition Charizard Holo #Pokemon TCG",
290 + "price": 343098,
291 + "ebayId": "397970894297",
292 + "listedPrice": null
293 + },
294 + {
295 + "tab": "manual-only",
296 + "date": "2026-05-27",
297 + "title": "Pokémon Charizard 1st Edition Shadowless Holo Base Set 4/102 PSA 10 4/102",
298 + "price": 325700,
299 + "ebayId": "306951160281",
300 + "listedPrice": null
301 + },
302 + {
303 + "tab": "manual-only",
304 + "date": "2026-05-13",
305 + "title": "1999 Pokemon PSA 10 Base Set 1st Edition Charizard Holo #Pokemon TCG",
306 + "price": 362900,
307 + "ebayId": "397849981038",
308 + "listedPrice": 616930
309 + },
310 + {
311 + "tab": "loose-and-manual",
312 + "date": "2026-07-12",
313 + "title": "1999 POKEMON BASE SET 1ST EDITION #4 CHARIZARD-HOLO PSA 1 #4",
314 + "price": 9100,
315 + "ebayId": "307041271557",
316 + "listedPrice": null
317 + },
318 + {
319 + "tab": "loose-and-manual",
320 + "date": "2026-06-17",
321 + "title": "1999 POKEMON BASE SET 1ST EDITION #4 CHARIZARD-HOLO PSA 1 #4",
322 + "price": 9800,
323 + "ebayId": "298396624001",
324 + "listedPrice": null
325 + },
326 + {
327 + "tab": "loose-and-manual",
328 + "date": "2026-06-13",
329 + "title": "1999 Pokemon 1st Edition Charizard Holo Shadowless #4 PSA 1 Base Set LOW POP 004/102",
330 + "price": 9000,
331 + "ebayId": "227372769830",
332 + "listedPrice": 9999
333 + },
334 + {
335 + "tab": "box-and-manual",
336 + "date": "2026-06-22",
337 + "title": "1999 POKEMON GAME 1ST EDITION #4 CHARIZARD-HOLO PSA 2 #4",
338 + "price": 9000,
339 + "ebayId": "298438998580",
340 + "listedPrice": 9999
341 + },
342 + {
343 + "tab": "box-and-manual",
344 + "date": "2026-06-18",
345 + "title": "1999 Pokemon Game 1st Edition Shadowless #4 Charizard Holo PSA 2 GD",
346 + "price": 9876,
347 + "ebayId": "377251658999",
348 + "listedPrice": null
349 + },
350 + {
351 + "tab": "box-and-manual",
352 + "date": "2026-05-27",
353 + "title": "Pokemon Charizard Base Set Shadowless 1st Edition Holo Rare #4/102 PSA 2 GOOD 4/102",
354 + "price": 9750,
355 + "ebayId": "318199587824",
356 + "listedPrice": null
357 + },
358 + {
359 + "tab": "grade-three",
360 + "date": "2026-09-05",
361 + "title": "1999 POKEMON BASE SET 1ST EDITION #4 CHARIZARD-HOLO PSA 3 #4",
362 + "price": 9700,
363 + "ebayId": "117380478483",
364 + "listedPrice": null
365 + },
366 + {
367 + "tab": "grade-three",
368 + "date": "2026-09-02",
369 + "title": "Pokemon Charizard Base Set 1999 1st Edition Holo 4/102 BGS 3 4/102",
370 + "price": 8800,
371 + "ebayId": "407186234783",
372 + "listedPrice": 10000
373 + },
374 + {
375 + "tab": "grade-three",
376 + "date": "2026-08-30",
377 + "title": "1999 POKEMON BASE SET 1ST EDITION #4 CHARIZARD-HOLO PSA 3 #4",
378 + "price": 10500,
379 + "ebayId": "307133776447",
380 + "listedPrice": null
381 + },
382 + {
383 + "tab": "grade-four",
384 + "date": "2026-08-30",
385 + "title": "1999 POKEMON BASE SET 1ST EDITION #4 CHARIZARD-HOLO PSA 4 #4",
386 + "price": 11100,
387 + "ebayId": "117372271652",
388 + "listedPrice": null
389 + },
390 + {
391 + "tab": "grade-four",
392 + "date": "2026-07-16",
393 + "title": "1999 POKEMON GAME CHARIZARD-HOLO 1ST EDITION PSA 4 4/102",
394 + "price": 11700,
395 + "ebayId": "178274171514",
396 + "listedPrice": 14000
397 + },
398 + {
399 + "tab": "grade-four",
400 + "date": "2026-07-10",
401 + "title": "1999 POKEMON GAME 1ST EDITION #4 CHARIZARD-HOLO PSA 4 #4",
402 + "price": 11250,
403 + "ebayId": "117294170914",
404 + "listedPrice": null
405 + },
406 + {
407 + "tab": "grade-five",
408 + "date": "2026-08-27",
409 + "title": "CGC 5 Charizard 4/102 Base Set 1st Edition Shadowless Holo Pokemon Card 4/102",
410 + "price": 11799.99,
411 + "ebayId": "128042207431",
412 + "listedPrice": null
413 + },
414 + {
415 + "tab": "grade-five",
416 + "date": "2026-08-06",
417 + "title": "1999 Pokemon Base Set 1st Edition Shadowless Charizard Holo #4 PSA 5 #4",
418 + "price": 13300,
419 + "ebayId": "800430294342",
420 + "listedPrice": null
421 + },
422 + {
423 + "tab": "grade-five",
424 + "date": "2026-07-23",
425 + "title": "1999 Pokemon Base Set 1st Edition Shadowless Charizard Holo #4 PSA 5 #4",
426 + "price": 14700,
427 + "ebayId": "800347620164",
428 + "listedPrice": null
429 + },
430 + {
431 + "tab": "grade-six",
432 + "date": "2026-09-03",
433 + "title": "Pokemon 1999 1st Edition Base Set Shadowless Charizard Holo PSA 6 #4",
434 + "price": 16310,
435 + "ebayId": "800568611855",
436 + "listedPrice": null
437 + },
438 + {
439 + "tab": "grade-six",
440 + "date": "2026-08-13",
441 + "title": "1999 Pokemon Base 1st Edition Thin Stamp Charizard Holo #4 BGS 6 EX-MT 004/102",
442 + "price": 14995,
443 + "ebayId": "318449518718",
444 + "listedPrice": null
445 + },
446 + {
447 + "tab": "grade-six",
448 + "date": "2026-08-05",
449 + "title": "1999 Pokemon Base Set 1st Edition Thin Stamp 4 Charizard Holo Shadowless BGS 6.5",
450 + "price": 12200,
451 + "ebayId": "377376921681",
452 + "listedPrice": null
453 + },
454 + {
455 + "tab": "grade-seventeen",
456 + "date": "2026-07-26",
457 + "title": "Private Sale",
458 + "price": 114000,
459 + "ebayId": null,
460 + "listedPrice": null
461 + },
462 + {
463 + "tab": "grade-seventeen",
464 + "date": "2026-07-19",
465 + "title": "Pokemon Glurak Charizard 1st Edition Base Set Holo 4/102 CGC 10 Gem Mint",
466 + "price": 11439,
467 + "ebayId": "298493955813",
468 + "listedPrice": null
469 + },
470 + {
471 + "tab": "grade-seventeen",
472 + "date": "2026-07-15",
473 + "title": "Charizard 1999 Pokemon Base Set 1st Edition Holo CGC 10 Graded 4/102 4/102",
474 + "price": 10000,
475 + "ebayId": "307064438900",
476 + "listedPrice": null
477 + },
478 + {
479 + "tab": "grade-nineteen",
480 + "date": "2025-02-13",
481 + "title": "1999 POKEMON 1st EDITION BASE SET CHARIZARD HOLO - CGC 10 PRISTINE",
482 + "price": 95340,
483 + "ebayId": "296980967673",
484 + "listedPrice": null
485 + }
486 + ],
487 + "details": {
488 + "See": "Population Report",
489 + "Genre": "Pokemon Card",
490 + "Release Date": "January 9, 1999",
491 + "Publisher": "Wizards of the Coast (WOTC)",
492 + "Card Number": "#4",
493 + "Notes": "Charizard",
494 + "ePID (eBay)": "715593",
495 + "TCGPlayer ID": "106999",
496 + "PriceCharting ID": "715593"
497 + },
498 + "images": [
499 + "https://storage.googleapis.com/images.pricecharting.com/gmkd37jg7haprczl/240.jpg",
500 + "https://storage.googleapis.com/images.pricecharting.com/rzkrh7rvfn5m7veb/240.jpg",
501 + "https://storage.googleapis.com/images.pricecharting.com/gmkd37jg7haprczl/1600.jpg"
502 + ],
503 + "setRef": {
504 + "id": "base1",
505 + "code": "BS",
506 + "name": "Base",
507 + "source": "pokemontcg-mirror"
508 + },
509 + "cardRef": null,
510 + "site": "pricecharting"
511 + }
512 + },
513 + "expect": {
514 + "minCount": 5,
515 + "kinds": [
516 + "catalog_item",
517 + "price_observation",
518 + "sale"
519 + ]
520 + },
521 + "note": "Live capture of https://www.pricecharting.com/game/pokemon-base-set/charizard-1st-edition-4 (sales trimmed to 3 rows per grade tab).",
522 + "capturedAt": "2026-09-07T05:50:36.471Z"
523 +}
\ No newline at end of file
added data/fixtures/pricecharting/pokemon-base-set__charizard-4.json +545 −0
@@ -0,0 +1,545 @@
1 +{
2 + "raw": {
3 + "url": "https://www.pricecharting.com/game/pokemon-base-set/charizard-4",
4 + "externalId": "630417",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:35.667Z",
8 + "payload": {
9 + "kind": "product",
10 + "url": "https://www.pricecharting.com/game/pokemon-base-set/charizard-4",
11 + "productId": "630417",
12 + "consoleUri": "pokemon-base-set",
13 + "consoleName": "Pokemon Base Set",
14 + "title": "Charizard #4",
15 + "flags": {
16 + "isComic": false,
17 + "isLegoSet": false,
18 + "isFunkoPop": false,
19 + "isCard": true,
20 + "isCoin": false,
21 + "isSystem": false
22 + },
23 + "columnLabels": [
24 + "Ungraded",
25 + "Grade 7",
26 + "Grade 8",
27 + "Grade 9",
28 + "Grade 9.5",
29 + "PSA 10",
30 + "Grade 9",
31 + "Grade 9.5",
32 + "PSA 10"
33 + ],
34 + "prices": [
35 + {
36 + "key": "used_price",
37 + "value": 345.2,
38 + "raw": "$345.20"
39 + },
40 + {
41 + "key": "complete_price",
42 + "value": 732.65,
43 + "raw": "$732.65"
44 + },
45 + {
46 + "key": "new_price",
47 + "value": 1405,
48 + "raw": "$1,405.00"
49 + },
50 + {
51 + "key": "graded_price",
52 + "value": 3200,
53 + "raw": "$3,200.00"
54 + },
55 + {
56 + "key": "box_only_price",
57 + "value": 7276.55,
58 + "raw": "$7,276.55"
59 + },
60 + {
61 + "key": "manual_only_price",
62 + "value": 12531.01,
63 + "raw": "$12,531.01"
64 + }
65 + ],
66 + "fullPrices": [
67 + {
68 + "label": "Ungraded",
69 + "value": 345.2
70 + },
71 + {
72 + "label": "Grade 1",
73 + "value": 350
74 + },
75 + {
76 + "label": "Grade 2",
77 + "value": 280
78 + },
79 + {
80 + "label": "Grade 3",
81 + "value": 325.89
82 + },
83 + {
84 + "label": "Grade 4",
85 + "value": 359.5
86 + },
87 + {
88 + "label": "Grade 5",
89 + "value": 417.93
90 + },
91 + {
92 + "label": "Grade 6",
93 + "value": 545
94 + },
95 + {
96 + "label": "Grade 7",
97 + "value": 732.65
98 + },
99 + {
100 + "label": "Grade 8",
101 + "value": 1405
102 + },
103 + {
104 + "label": "Grade 9",
105 + "value": 3200
106 + },
107 + {
108 + "label": "Grade 9.5",
109 + "value": 7276.55
110 + },
111 + {
112 + "label": "TAG 10",
113 + "value": 5694.44
114 + },
115 + {
116 + "label": "ACE 10",
117 + "value": null
118 + },
119 + {
120 + "label": "SGC 10",
121 + "value": 7519
122 + },
123 + {
124 + "label": "CGC 10",
125 + "value": 3891.98
126 + },
127 + {
128 + "label": "PSA 10",
129 + "value": 12531.01
130 + },
131 + {
132 + "label": "BGS 10",
133 + "value": 16290
134 + },
135 + {
136 + "label": "BGS 10 Black",
137 + "value": 81450
138 + },
139 + {
140 + "label": "CGC 10 Pristine",
141 + "value": 20800
142 + }
143 + ],
144 + "tabLabels": {
145 + "used": "Ungraded",
146 + "grade-twenty": "BGS 10 Black",
147 + "grade-nineteen": "CGC 10 Prist.",
148 + "manual-only": "PSA 10",
149 + "loose-and-box": "BGS 10",
150 + "grade-seventeen": "CGC 10",
151 + "grade-eighteen": "SGC 10",
152 + "grade-twenty-one": "TAG 10",
153 + "grade-twenty-two": "ACE 10",
154 + "box-only": "Grade 9.5",
155 + "graded": "Grade 9",
156 + "new": "Grade 8",
157 + "cib": "Grade 7",
158 + "grade-six": "Grade 6",
159 + "grade-five": "Grade 5",
160 + "grade-four": "Grade 4",
161 + "grade-three": "Grade 3",
162 + "box-and-manual": "Grade 2",
163 + "loose-and-manual": "Grade 1"
164 + },
165 + "sales": [
166 + {
167 + "tab": "used",
168 + "date": "2026-09-06",
169 + "title": "Pokémon TCG Charizard Base Set Unlimited Holo Rare Card LP 4/102-Free shipping 4/102",
170 + "price": 265,
171 + "ebayId": "398338804115",
172 + "listedPrice": null
173 + },
174 + {
175 + "tab": "used",
176 + "date": "2026-09-05",
177 + "title": "Wizards of the Coast Pokémon TCG Charizard Base Set Holo Rare 4/102 120 HP",
178 + "price": 151.49,
179 + "ebayId": "237037830596",
180 + "listedPrice": null
181 + },
182 + {
183 + "tab": "used",
184 + "date": "2026-09-05",
185 + "title": "Pokémon TCG Charizard Base Set Holographic Rare Card 4/102 4/102",
186 + "price": 299,
187 + "ebayId": "206523328704",
188 + "listedPrice": null
189 + },
190 + {
191 + "tab": "cib",
192 + "date": "2026-09-04",
193 + "title": "Pokemon 1999 Base Set Charizard 4/102 Holo Rare PSA 7 English 4/102",
194 + "price": 635,
195 + "ebayId": "820056719747",
196 + "listedPrice": null
197 + },
198 + {
199 + "tab": "cib",
200 + "date": "2026-09-04",
201 + "title": "1999 POKEMON BASE SET UNLIMITED #4 CHARIZARD-HOLO PSA 7 #4",
202 + "price": 689,
203 + "ebayId": "307143753156",
204 + "listedPrice": null
205 + },
206 + {
207 + "tab": "cib",
208 + "date": "2026-09-04",
209 + "title": "1999 POKEMON BASE SET UNLIMITED #4 CHARIZARD-HOLO PSA 7 #4",
210 + "price": 709,
211 + "ebayId": "117375091191",
212 + "listedPrice": null
213 + },
214 + {
215 + "tab": "new",
216 + "date": "2026-09-05",
217 + "title": "1999 POKEMON BASE SET #4 CHARIZARD-HOLO PSA 8 #4",
218 + "price": 1314.18,
219 + "ebayId": "307146414408",
220 + "listedPrice": null
221 + },
222 + {
223 + "tab": "new",
224 + "date": "2026-09-05",
225 + "title": "1999 POKEMON BASE SET UNLIMITED #4 CHARIZARD-HOLO PSA 8 #4",
226 + "price": 1625,
227 + "ebayId": "298614747684",
228 + "listedPrice": null
229 + },
230 + {
231 + "tab": "new",
232 + "date": "2026-09-05",
233 + "title": "1999 Pokemon Charizard Base Set Holo 4/102 PSA 8 4/102",
234 + "price": 1225,
235 + "ebayId": "168647068271",
236 + "listedPrice": null
237 + },
238 + {
239 + "tab": "graded",
240 + "date": "2026-09-05",
241 + "title": "1999 Pokemon Base Set #4 Charizard Holo R BGS 9 MINT (9,9,9,9.5) Great PSA 9",
242 + "price": 2850,
243 + "ebayId": "307159469969",
244 + "listedPrice": null
245 + },
246 + {
247 + "tab": "graded",
248 + "date": "2026-09-05",
249 + "title": "1999 Pokemon Game Base Set Unlimited #4 CHARIZARD - HOLO PSA MINT 9 4/102",
250 + "price": 3600,
251 + "ebayId": "158254443469",
252 + "listedPrice": null
253 + },
254 + {
255 + "tab": "graded",
256 + "date": "2026-09-05",
257 + "title": "1999 POKEMON BASE SET #4 CHARIZARD-HOLO PSA 9 #4",
258 + "price": 3096.43,
259 + "ebayId": "307146414372",
260 + "listedPrice": null
261 + },
262 + {
263 + "tab": "box-only",
264 + "date": "2026-08-24",
265 + "title": "Pokemon Charizard Holo Base Set Unlimited 4 BGS 9.5 #4",
266 + "price": 8988.69,
267 + "ebayId": "287524903899",
268 + "listedPrice": null
269 + },
270 + {
271 + "tab": "box-only",
272 + "date": "2026-08-07",
273 + "title": "1999 Pokemon Charizard HOLO R Base Unlimited BGS 9.5",
274 + "price": 9100,
275 + "ebayId": "800434392147",
276 + "listedPrice": null
277 + },
278 + {
279 + "tab": "box-only",
280 + "date": "2026-08-03",
281 + "title": "Pokemon Charizard Base Set Unlimited 4/102 Holo CGC 9.5 4/102",
282 + "price": 3072,
283 + "ebayId": "287481273454",
284 + "listedPrice": null
285 + },
286 + {
287 + "tab": "manual-only",
288 + "date": "2026-08-16",
289 + "title": "Charizard Base Set 4/102 Holo PSA 10 Gem Mint 1999 English Pokemon TCG 4/102",
290 + "price": 8000,
291 + "ebayId": "206494266821",
292 + "listedPrice": 9629
293 + },
294 + {
295 + "tab": "manual-only",
296 + "date": "2026-08-14",
297 + "title": "Charizard Base Set Holo 4/102 PSA 10 Gem Mint 1999 English Pokemon TCG 4/102",
298 + "price": 10300,
299 + "ebayId": "287509235115",
300 + "listedPrice": null
301 + },
302 + {
303 + "tab": "manual-only",
304 + "date": "2026-05-04",
305 + "title": "PSA 10 - Pokemon Charizard Holo #4/102 Base Set Unlimited Base Set 4/102",
306 + "price": 30100,
307 + "ebayId": "188320306747",
308 + "listedPrice": null
309 + },
310 + {
311 + "tab": "loose-and-manual",
312 + "date": "2026-09-03",
313 + "title": "1999 POKEMON BASE SET UNLIMITED #4 CHARIZARD-HOLO PSA 1 #4",
314 + "price": 350,
315 + "ebayId": "117377832693",
316 + "listedPrice": null
317 + },
318 + {
319 + "tab": "loose-and-manual",
320 + "date": "2026-09-02",
321 + "title": "Pokémon TCG Charizard 04/102 Base Set Holo Rare WOTC 1999 Vintage Ace 1",
322 + "price": 219,
323 + "ebayId": "336742130846",
324 + "listedPrice": 246.02
325 + },
326 + {
327 + "tab": "loose-and-manual",
328 + "date": "2026-08-30",
329 + "title": "Pokemon Charizard Holo Base Set Unlimited 4/102 PSA 1 4/102",
330 + "price": 314,
331 + "ebayId": "287541277658",
332 + "listedPrice": null
333 + },
334 + {
335 + "tab": "box-and-manual",
336 + "date": "2026-08-23",
337 + "title": "(PSA) CGC 2.5 Charizard Base Set Holo Pokemon Card #4/102 4/102",
338 + "price": 232.5,
339 + "ebayId": "800515061808",
340 + "listedPrice": null
341 + },
342 + {
343 + "tab": "box-and-manual",
344 + "date": "2026-08-09",
345 + "title": "(PSA) CGC 2.5 Charizard Base Set Holo Pokemon Card #4/102 4/102",
346 + "price": 318,
347 + "ebayId": "800444156593",
348 + "listedPrice": null
349 + },
350 + {
351 + "tab": "box-and-manual",
352 + "date": "2026-07-20",
353 + "title": "Pokémon Charizard 4/102 1999 Base Set Holo Card Unlimited Rare, CGC 2.5 4/102",
354 + "price": 300,
355 + "ebayId": "188643748242",
356 + "listedPrice": null
357 + },
358 + {
359 + "tab": "grade-three",
360 + "date": "2026-08-31",
361 + "title": "Wizards of the Coast 1999 Pokemon TCG Charizard 4/102 Base Set Holo PSA 3 4/102",
362 + "price": 350,
363 + "ebayId": "800584018446",
364 + "listedPrice": null
365 + },
366 + {
367 + "tab": "grade-three",
368 + "date": "2026-08-31",
369 + "title": "Pokemon Charizard Holo Base Set II 4 PSA 3 #4",
370 + "price": 194.5,
371 + "ebayId": "287541277634",
372 + "listedPrice": null
373 + },
374 + {
375 + "tab": "grade-three",
376 + "date": "2026-08-24",
377 + "title": "WOTC Pokemon TCG Charizard Base Set 1999 Holo Rare 4/102 120HP TAG 3 4/102",
378 + "price": 248.5,
379 + "ebayId": "137632245926",
380 + "listedPrice": null
381 + },
382 + {
383 + "tab": "grade-four",
384 + "date": "2026-09-02",
385 + "title": "Wizards of the Coast Pokemon TCG Charizard 4/102 Base Set Holo CGC 4.5 1999 4/102",
386 + "price": 375,
387 + "ebayId": "137661611651",
388 + "listedPrice": null
389 + },
390 + {
391 + "tab": "grade-four",
392 + "date": "2026-08-30",
393 + "title": "Pokemon Charizard Holo Base Set Unlimited 4/102 PSA 4 4/102",
394 + "price": 310,
395 + "ebayId": "287538725771",
396 + "listedPrice": null
397 + },
398 + {
399 + "tab": "grade-four",
400 + "date": "2026-08-30",
401 + "title": "1999 POKEMON BASE SET UNLIMITED #4 CHARIZARD-HOLO PSA 4 #4",
402 + "price": 350,
403 + "ebayId": "117364501305",
404 + "listedPrice": null
405 + },
406 + {
407 + "tab": "grade-five",
408 + "date": "2026-09-05",
409 + "title": "1999 POKEMON BASE SET UNLIMITED #4 CHARIZARD-HOLO PSA 5 #4",
410 + "price": 405,
411 + "ebayId": "117375573653",
412 + "listedPrice": null
413 + },
414 + {
415 + "tab": "grade-five",
416 + "date": "2026-09-04",
417 + "title": "CHARIZARD 1999 POKEMON GAME BASE SET 4/102 RARE HOLO PSA 5 B Q5677 4/102",
418 + "price": 408.08,
419 + "ebayId": "278317133910",
420 + "listedPrice": null
421 + },
422 + {
423 + "tab": "grade-five",
424 + "date": "2026-08-31",
425 + "title": "Charizard Holo PSA 5 Base Set 4/102 1999 Pokemon TCG 4/102",
426 + "price": 445.99,
427 + "ebayId": "137650865905",
428 + "listedPrice": null
429 + },
430 + {
431 + "tab": "grade-six",
432 + "date": "2026-09-04",
433 + "title": "Pokémon Card Charizard 4/102 Base Set 1999 Holo Rare Unlimited Vintage PSA 6 4/102",
434 + "price": 580,
435 + "ebayId": "318788779540",
436 + "listedPrice": null
437 + },
438 + {
439 + "tab": "grade-six",
440 + "date": "2026-09-04",
441 + "title": "1999 Pokemon game Charizard Holo 4/102 Base Set PSA 6 WOTC 4/102",
442 + "price": 480,
443 + "ebayId": "820056385895",
444 + "listedPrice": null
445 + },
446 + {
447 + "tab": "grade-six",
448 + "date": "2026-09-04",
449 + "title": "1999 POKEMON BASE SET UNLIMITED #4 CHARIZARD-HOLO PSA 6 #4",
450 + "price": 515,
451 + "ebayId": "307143753159",
452 + "listedPrice": null
453 + },
454 + {
455 + "tab": "grade-seventeen",
456 + "date": "2026-08-17",
457 + "title": "2021 Pokemon Celebrations Classic Coll. Gold Metal UPC #4 Charizard CGC 10 #4",
458 + "price": 2100,
459 + "ebayId": "147431108520",
460 + "listedPrice": 2400
461 + },
462 + {
463 + "tab": "grade-seventeen",
464 + "date": "2026-08-16",
465 + "title": "CHARIZARD POKEMON CELEB CLSC. COLL. BASE SET HOLO 4/102 2021 GER CGC 10",
466 + "price": 303.17,
467 + "ebayId": "137595385904",
468 + "listedPrice": null
469 + },
470 + {
471 + "tab": "grade-seventeen",
472 + "date": "2026-08-14",
473 + "title": "2021 Pokemon Celebrations Classic Ultra Premium #4 Charizard Gold Metal CGC 10",
474 + "price": 2325,
475 + "ebayId": "377399959704",
476 + "listedPrice": null
477 + },
478 + {
479 + "tab": "grade-nineteen",
480 + "date": "2026-01-26",
481 + "title": "1999 Pokemon Base Set Charizard Holo 4/102 CGC PRISTINE 10 *POP 11 !!!*",
482 + "price": 20800,
483 + "ebayId": "389510851948",
484 + "listedPrice": null
485 + },
486 + {
487 + "tab": "grade-twenty-one",
488 + "date": "2026-07-30",
489 + "title": "WOTC POKÉMON CHARIZARD 2002 #39/165 EXP BASE SET REV HOLO TAG 10 39/165",
490 + "price": 5100,
491 + "ebayId": "800391674311",
492 + "listedPrice": null
493 + },
494 + {
495 + "tab": "grade-twenty-one",
496 + "date": "2026-07-30",
497 + "title": "WOTC POKÉMON CHARIZARD 2002 #40/165 EXP BASE SET REV HOLO TAG 10 40/165",
498 + "price": 6288.88,
499 + "ebayId": "800391674262",
500 + "listedPrice": null
501 + }
502 + ],
503 + "details": {
504 + "See": "Population Report",
505 + "Genre": "Pokemon Card",
506 + "Release Date": "January 9, 1999",
507 + "Publisher": "Wizards of the Coast",
508 + "Card Number": "#4",
509 + "ePID (eBay)": "630417",
510 + "TCGPlayer ID": "42382",
511 + "PriceCharting ID": "630417",
512 + "Description": "One of the most popular cards of all time."
513 + },
514 + "images": [
515 + "https://storage.googleapis.com/images.pricecharting.com/hpgpcpsd42huitud/240.jpg",
516 + "https://storage.googleapis.com/images.pricecharting.com/kmwn5qjyipwzbuwm/240.jpg",
517 + "https://storage.googleapis.com/images.pricecharting.com/hpgpcpsd42huitud/1600.jpg"
518 + ],
519 + "setRef": {
520 + "id": "base1",
521 + "code": "BS",
522 + "name": "Base",
523 + "source": "pokemontcg-mirror"
524 + },
525 + "cardRef": {
526 + "pokemontcg_id": "base1-4"
527 + },
528 + "site": "pricecharting"
529 + }
530 + },
531 + "expect": {
532 + "minCount": 10,
533 + "kinds": [
534 + "catalog_item",
535 + "price_observation",
536 + "sale"
537 + ],
538 + "requiredFields": [
539 + "attributes.setCode",
540 + "attributes.number"
541 + ]
542 + },
543 + "note": "Live capture of https://www.pricecharting.com/game/pokemon-base-set/charizard-4 (sales trimmed to 3 rows per grade tab).",
544 + "capturedAt": "2026-09-07T05:50:35.825Z"
545 +}
\ No newline at end of file
added data/fixtures/pricecharting/yugioh-lob__blue-eyes-1st-edition.json +507 −0
@@ -0,0 +1,507 @@
1 +{
2 + "raw": {
3 + "url": "https://www.pricecharting.com/game/yugioh-legend-of-blue-eyes-white-dragon/blue-eyes-white-dragon-1st-edition-lob-001",
4 + "externalId": "2530687",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:38.032Z",
8 + "payload": {
9 + "kind": "product",
10 + "url": "https://www.pricecharting.com/game/yugioh-legend-of-blue-eyes-white-dragon/blue-eyes-white-dragon-1st-edition-lob-001",
11 + "productId": "2530687",
12 + "consoleUri": "yugioh-legend-of-blue-eyes-white-dragon",
13 + "consoleName": "YuGiOh Legend of Blue Eyes White Dragon",
14 + "title": "Blue-Eyes White Dragon [1st Edition] LOB-001",
15 + "flags": {
16 + "isComic": false,
17 + "isLegoSet": false,
18 + "isFunkoPop": false,
19 + "isCard": true,
20 + "isCoin": false,
21 + "isSystem": false
22 + },
23 + "columnLabels": [
24 + "Ungraded",
25 + "Grade 7",
26 + "Grade 8",
27 + "Grade 9",
28 + "Grade 9.5",
29 + "PSA 10",
30 + "Grade 9",
31 + "Grade 9.5",
32 + "PSA 10"
33 + ],
34 + "prices": [
35 + {
36 + "key": "used_price",
37 + "value": 1400,
38 + "raw": "$1,400.00"
39 + },
40 + {
41 + "key": "complete_price",
42 + "value": 3333.14,
43 + "raw": "$3,333.14"
44 + },
45 + {
46 + "key": "new_price",
47 + "value": 4000,
48 + "raw": "$4,000.00"
49 + },
50 + {
51 + "key": "graded_price",
52 + "value": 8162.25,
53 + "raw": "$8,162.25"
54 + },
55 + {
56 + "key": "box_only_price",
57 + "value": 15700,
58 + "raw": "$15,700.00"
59 + },
60 + {
61 + "key": "manual_only_price",
62 + "value": 45000,
63 + "raw": "$45,000.00"
64 + }
65 + ],
66 + "fullPrices": [
67 + {
68 + "label": "Ungraded",
69 + "value": 1400
70 + },
71 + {
72 + "label": "Grade 1",
73 + "value": 669.04
74 + },
75 + {
76 + "label": "Grade 2",
77 + "value": 700
78 + },
79 + {
80 + "label": "Grade 3",
81 + "value": 1233.4
82 + },
83 + {
84 + "label": "Grade 4",
85 + "value": 1463
86 + },
87 + {
88 + "label": "Grade 5",
89 + "value": 1721.66
90 + },
91 + {
92 + "label": "Grade 6",
93 + "value": 2719.22
94 + },
95 + {
96 + "label": "Grade 7",
97 + "value": 3333.14
98 + },
99 + {
100 + "label": "Grade 8",
101 + "value": 4000
102 + },
103 + {
104 + "label": "Grade 9",
105 + "value": 8162.25
106 + },
107 + {
108 + "label": "Grade 9.5",
109 + "value": 15700
110 + },
111 + {
112 + "label": "TAG 10",
113 + "value": null
114 + },
115 + {
116 + "label": "ACE 10",
117 + "value": null
118 + },
119 + {
120 + "label": "SGC 10",
121 + "value": 27000
122 + },
123 + {
124 + "label": "CGC 10",
125 + "value": 19466.74
126 + },
127 + {
128 + "label": "PSA 10",
129 + "value": 45000
130 + },
131 + {
132 + "label": "BGS 10",
133 + "value": 58500
134 + },
135 + {
136 + "label": "BGS 10 Black",
137 + "value": 292500
138 + },
139 + {
140 + "label": "CGC 10 Pristine",
141 + "value": 35040
142 + }
143 + ],
144 + "tabLabels": {
145 + "used": "Ungraded",
146 + "grade-twenty": "BGS 10 Black",
147 + "grade-nineteen": "CGC 10 Prist.",
148 + "manual-only": "PSA 10",
149 + "loose-and-box": "BGS 10",
150 + "grade-seventeen": "CGC 10",
151 + "grade-eighteen": "SGC 10",
152 + "grade-twenty-one": "TAG 10",
153 + "grade-twenty-two": "ACE 10",
154 + "box-only": "Grade 9.5",
155 + "graded": "Grade 9",
156 + "new": "Grade 8",
157 + "cib": "Grade 7",
158 + "grade-six": "Grade 6",
159 + "grade-five": "Grade 5",
160 + "grade-four": "Grade 4",
161 + "grade-three": "Grade 3",
162 + "box-and-manual": "Grade 2",
163 + "loose-and-manual": "Grade 1"
164 + },
165 + "sales": [
166 + {
167 + "tab": "used",
168 + "date": "2026-08-31",
169 + "title": "Yu-Gi-Oh! Blue-Eyes White Dragon LOB-001 Ultra Rare Holo 1st Edition English #LOB-001",
170 + "price": 1400,
171 + "ebayId": "800285469015",
172 + "listedPrice": 2000
173 + },
174 + {
175 + "tab": "used",
176 + "date": "2026-08-26",
177 + "title": "YUGIOH 2002 | BLUE-EYES WHITE DRAGON | LOB-001 | 1ST EDITION (AE) | NM-MINT",
178 + "price": 926.79,
179 + "ebayId": "158132639554",
180 + "listedPrice": null
181 + },
182 + {
183 + "tab": "used",
184 + "date": "2026-08-24",
185 + "title": "Blue-Eyes White Dragon LOB-001 GLOSSY Ultra Rare 1st Edition YU-GI-OH #LOB-001",
186 + "price": 2324,
187 + "ebayId": "278274336236",
188 + "listedPrice": null
189 + },
190 + {
191 + "tab": "cib",
192 + "date": "2026-06-05",
193 + "title": "2002 YU-GI-OH! BLUE EYES WHITE DRAGON LOB-001 1ST EDITION CARD PSA 7 NM",
194 + "price": 3356,
195 + "ebayId": "227315484139",
196 + "listedPrice": null
197 + },
198 + {
199 + "tab": "cib",
200 + "date": "2026-04-20",
201 + "title": "2002 YUGIOH! BLUE EYES WHITE DRAGON 1ST EDITION LOB-001 PSA 7 ENGLISH - SHARP -",
202 + "price": 3376,
203 + "ebayId": "298209419395",
204 + "listedPrice": null
205 + },
206 + {
207 + "tab": "cib",
208 + "date": "2026-02-07",
209 + "title": "Wavy 1st Edition Blue-Eyes White Dragon PSA 7 LOB-001 Yu-Gi-Oh BEWD #LOB-001",
210 + "price": 4100,
211 + "ebayId": "227196676186",
212 + "listedPrice": null
213 + },
214 + {
215 + "tab": "new",
216 + "date": "2026-08-31",
217 + "title": "2002 YU-GI-OH! LOB-001 Blue-Eyes White Dragon 1st Edition PSA 8 NM-MT #1",
218 + "price": 4199.99,
219 + "ebayId": "198580316202",
220 + "listedPrice": 5999.99
221 + },
222 + {
223 + "tab": "new",
224 + "date": "2026-07-31",
225 + "title": "YU-GI-OH! yugioh Blue-Eyes White Dragon LOB-001 Ultra Rare 1st Edition GLSY PSA8 LOB-001",
226 + "price": 4980,
227 + "ebayId": "198361025766",
228 + "listedPrice": null
229 + },
230 + {
231 + "tab": "new",
232 + "date": "2026-06-22",
233 + "title": "Yu-Gi-Oh! Blue-Eyes White Dragon Legend of Blue Eyes 1st Edition LOB-001 PSA 8",
234 + "price": 4000,
235 + "ebayId": "287386167693",
236 + "listedPrice": null
237 + },
238 + {
239 + "tab": "graded",
240 + "date": "2026-09-01",
241 + "title": "Yugioh Blue-Eyes White Dragon Legend Of Blue Eyes 1st Edition LOB-001 BGS 9 Mint #LOB001",
242 + "price": 6250,
243 + "ebayId": "287480062874",
244 + "listedPrice": null
245 + },
246 + {
247 + "tab": "graded",
248 + "date": "2026-08-28",
249 + "title": "Yugioh Blue Eyes White Dragon Legend Of Blue Eyes LOB-001 1st Edition PSA 9 #1",
250 + "price": 9000,
251 + "ebayId": "287520946541",
252 + "listedPrice": null
253 + },
254 + {
255 + "tab": "graded",
256 + "date": "2026-08-03",
257 + "title": "Blue Eyes White Dragon LOB-001 WAVY Ultra Rare 1st Edition Yugioh PSA 9 *544 #001-97",
258 + "price": 9100,
259 + "ebayId": "287475714115",
260 + "listedPrice": null
261 + },
262 + {
263 + "tab": "box-only",
264 + "date": "2026-08-09",
265 + "title": "Blue-eyes White Dragon LOB-001 Ultra Rare Wavy 1st Edition Yugioh BGS 9.5 *852 #LOB1",
266 + "price": 18000,
267 + "ebayId": "287511630805",
268 + "listedPrice": 25000
269 + },
270 + {
271 + "tab": "box-only",
272 + "date": "2026-03-30",
273 + "title": "Yu-Gi-Oh! Blue-Eyes White Dragon Legend of Blue Eyes 1st Ed LOB-001 WAVY BGS 9.5",
274 + "price": 15700,
275 + "ebayId": "287220077962",
276 + "listedPrice": null
277 + },
278 + {
279 + "tab": "box-only",
280 + "date": "2026-03-18",
281 + "title": "BGS 9.5 - Blue-Eyes White Dragon LOB-001 1st Edition Yugioh Gem Mint #LOB-001",
282 + "price": 15102,
283 + "ebayId": "127732201039",
284 + "listedPrice": null
285 + },
286 + {
287 + "tab": "manual-only",
288 + "date": "2026-03-08",
289 + "title": "Yu-Gi-Oh PSA 10 Blue Eyes White Dragon 1st Edition LOB 001 Gem MINT GLOSSY #LOB-001",
290 + "price": 50000,
291 + "ebayId": "267578896591",
292 + "listedPrice": 64999
293 + },
294 + {
295 + "tab": "manual-only",
296 + "date": "2026-01-25",
297 + "title": "PSA 10 - Blue-Eyes White Dragon LOB-001 Ultra Rare 1st Ed. Yugioh #LOB-001",
298 + "price": 40000,
299 + "ebayId": "236599921586",
300 + "listedPrice": 45000
301 + },
302 + {
303 + "tab": "manual-only",
304 + "date": "2026-01-25",
305 + "title": "Blue Eyes White Dragon LOB-001 Ultra Rare WAVY 1st Edition Yugioh PSA 10 *151 #001-97",
306 + "price": 49000,
307 + "ebayId": "287091584321",
308 + "listedPrice": 99999.99
309 + },
310 + {
311 + "tab": "loose-and-manual",
312 + "date": "2026-07-08",
313 + "title": "Yu-Gi-Oh Blue Eyes White Dragon LOB-001 1st Edition PSA 1 Trading Card *POP 40* #LOB-001",
314 + "price": 1100,
315 + "ebayId": "336673738178",
316 + "listedPrice": null
317 + },
318 + {
319 + "tab": "loose-and-manual",
320 + "date": "2026-06-08",
321 + "title": "Yu-Gi-Oh Blue Eyes White Dragon LOB-001 1st Edition PSA 1 Trading Card *POP 40* #LOB-001",
322 + "price": 1000,
323 + "ebayId": "127907965980",
324 + "listedPrice": null
325 + },
326 + {
327 + "tab": "loose-and-manual",
328 + "date": "2026-04-28",
329 + "title": "Good Looking 1! AE Blue Eyes White Dragon 1st edition lob-001",
330 + "price": 183.46,
331 + "ebayId": "298234284304",
332 + "listedPrice": 366.92
333 + },
334 + {
335 + "tab": "box-and-manual",
336 + "date": "2025-11-20",
337 + "title": "Blue-Eyes White Dragon LOB-001 1st Edition Glossy CGC 2 North American #LOB-001",
338 + "price": 700,
339 + "ebayId": "227064380933",
340 + "listedPrice": null
341 + },
342 + {
343 + "tab": "box-and-manual",
344 + "date": "2025-10-09",
345 + "title": "Blue-Eyes White Dragon LOB-001 1st Edition Glossy CGC 2 North American #LOB-001",
346 + "price": 700,
347 + "ebayId": "306195401642",
348 + "listedPrice": null
349 + },
350 + {
351 + "tab": "box-and-manual",
352 + "date": "2025-07-15",
353 + "title": "Blue-Eyes White Dragon LOB-001 1st Edition PSA 2 #LOB-001",
354 + "price": 500,
355 + "ebayId": "127215934892",
356 + "listedPrice": null
357 + },
358 + {
359 + "tab": "grade-three",
360 + "date": "2026-08-03",
361 + "title": "Yu-Gi-Oh! Blue-Eyes White Dragon Legend 1st Edition LOB-001 CGC 3 ENGLISH",
362 + "price": 1565,
363 + "ebayId": "377384866758",
364 + "listedPrice": null
365 + },
366 + {
367 + "tab": "grade-three",
368 + "date": "2026-07-31",
369 + "title": "Yu-Gi-Oh! Blue-Eyes White Dragon Legend 1st Edition LOB-001 CGC 3 ENGLISH",
370 + "price": 1200,
371 + "ebayId": "377345288984",
372 + "listedPrice": 1565
373 + },
374 + {
375 + "tab": "grade-three",
376 + "date": "2025-07-22",
377 + "title": "Blue-Eyes White Dragon LOB-001 The Legend of Blue Eyes 1st Edition PSA 3 VG #LOB-001",
378 + "price": 750,
379 + "ebayId": "297484041477",
380 + "listedPrice": null
381 + },
382 + {
383 + "tab": "grade-four",
384 + "date": "2026-04-06",
385 + "title": "2002 Yu-Gi-Oh! LOB 1st Ed Legend/Blue Eyes #001 Blue Eyes White Dragon PSA 4",
386 + "price": 5100,
387 + "ebayId": "377071362082",
388 + "listedPrice": null
389 + },
390 + {
391 + "tab": "grade-four",
392 + "date": "2026-02-14",
393 + "title": "1st Ed Blue Eyes White Dragon LOB-001 PSA 4 #LOB-001",
394 + "price": 911,
395 + "ebayId": "397582972514",
396 + "listedPrice": null
397 + },
398 + {
399 + "tab": "grade-four",
400 + "date": "2025-11-05",
401 + "title": "WAVY 1st Ed Blue Eyes White Dragon PSA 4 LOB-001 The Legend Of Blue Eyes BEWD #LOB-001",
402 + "price": 2550,
403 + "ebayId": "336253890460",
404 + "listedPrice": null
405 + },
406 + {
407 + "tab": "grade-five",
408 + "date": "2026-06-28",
409 + "title": "Blue-Eyes White Dragon LOB-001 1st Edition - PSA 5, Pop. 86 - USA, English, 2002 #LOB-001",
410 + "price": 2399.99,
411 + "ebayId": "177876531442",
412 + "listedPrice": null
413 + },
414 + {
415 + "tab": "grade-five",
416 + "date": "2026-06-17",
417 + "title": "2002 YU-GI-OH! LOB-LEGEND OF BLUE EYES WHITE DRAGON BLUE-EYES WHITE DRAGON PSA 5 #001",
418 + "price": 2000,
419 + "ebayId": "307006892045",
420 + "listedPrice": null
421 + },
422 + {
423 + "tab": "grade-five",
424 + "date": "2026-06-07",
425 + "title": "Blue Eyes White Dragon LOB-001 Ultra Rare 1st Edition GLOSSY Yugioh PSA 5 *590 #001-97",
426 + "price": 1600,
427 + "ebayId": "287354441154",
428 + "listedPrice": null
429 + },
430 + {
431 + "tab": "grade-six",
432 + "date": "2026-07-29",
433 + "title": "Yugioh! Blue-Eyes White Dragon Legend of Blue Eyes 1st Edition LOB-001 PSA 6",
434 + "price": 2449,
435 + "ebayId": "267736554599",
436 + "listedPrice": null
437 + },
438 + {
439 + "tab": "grade-six",
440 + "date": "2026-05-08",
441 + "title": "2002 YU-GI-OH! LOB-LEGEND OF BLUE EYES WHITE DRAGON BLUE-EYES WHITE DRAGON PSA 6 #001",
442 + "price": 8670,
443 + "ebayId": "117169038313",
444 + "listedPrice": null
445 + },
446 + {
447 + "tab": "grade-six",
448 + "date": "2026-03-26",
449 + "title": "2002 Yugioh Blue-Eyes White Dragon 1st Edition Ultra Rare LOB-001 BGS 6 EX-MT #001",
450 + "price": 2800,
451 + "ebayId": "227166297437",
452 + "listedPrice": 3000
453 + },
454 + {
455 + "tab": "grade-seventeen",
456 + "date": "2026-09-04",
457 + "title": "Yugioh Legend Of Blue-Eyes White Dragon LOB-001 1st Edition Ultra Rare CGC 10 North american #LOB-001",
458 + "price": 20000,
459 + "ebayId": "178433880960",
460 + "listedPrice": 24995
461 + },
462 + {
463 + "tab": "grade-seventeen",
464 + "date": "2026-08-16",
465 + "title": "Blue-eyes White Dragon LOB-001 1st Edition Wavy Ultra Rare Yugioh CGC 10 *002 North american #LOB-001",
466 + "price": 18988,
467 + "ebayId": "278246414325",
468 + "listedPrice": null
469 + },
470 + {
471 + "tab": "grade-seventeen",
472 + "date": "2026-05-04",
473 + "title": "Blue-eyes White Dragon LOB-001 1st Edition Glossy Ultra Rare Yugioh CGC 10 *001 North american #LOB-001",
474 + "price": 12105,
475 + "ebayId": "287282362594",
476 + "listedPrice": null
477 + }
478 + ],
479 + "details": {
480 + "See": "Population Report",
481 + "Genre": "YuGiOh Card",
482 + "Release Date": "March 8, 2002",
483 + "Card Number": "#LOB-001",
484 + "Notes": "Ultra Rare",
485 + "TCGPlayer ID": "21792",
486 + "PriceCharting ID": "2530687"
487 + },
488 + "images": [
489 + "https://storage.googleapis.com/images.pricecharting.com/ec1170f04e0b02ada3978417b4cfd7838ac257f24722c5c07344c28479fa23f1/240.jpg",
490 + "https://storage.googleapis.com/images.pricecharting.com/ec1170f04e0b02ada3978417b4cfd7838ac257f24722c5c07344c28479fa23f1/1600.jpg"
491 + ],
492 + "setRef": null,
493 + "cardRef": null,
494 + "site": "pricecharting"
495 + }
496 + },
497 + "expect": {
498 + "minCount": 3,
499 + "kinds": [
500 + "catalog_item",
501 + "price_observation",
502 + "sale"
503 + ]
504 + },
505 + "note": "Live capture of https://www.pricecharting.com/game/yugioh-legend-of-blue-eyes-white-dragon/blue-eyes-white-dragon-1st-edition-lob-001 (sales trimmed to 3 rows per grade tab).",
506 + "capturedAt": "2026-09-07T05:50:38.124Z"
507 +}
\ No newline at end of file
added data/fixtures/sportscardspro/basketball-1986-fleer__michael-jordan-57.json +503 −0
@@ -0,0 +1,503 @@
1 +{
2 + "raw": {
3 + "url": "https://www.sportscardspro.com/game/basketball-cards-1986-fleer/michael-jordan-57",
4 + "externalId": "72584",
5 + "kind": "catalog_item",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T05:50:38.794Z",
8 + "payload": {
9 + "kind": "product",
10 + "url": "https://www.sportscardspro.com/game/basketball-cards-1986-fleer/michael-jordan-57",
11 + "productId": "72584",
12 + "consoleUri": "basketball-cards-1986-fleer",
13 + "consoleName": "1986 Fleer",
14 + "title": "Michael Jordan #57 [Rookie]",
15 + "flags": {
16 + "isComic": false,
17 + "isLegoSet": false,
18 + "isFunkoPop": false,
19 + "isCard": false,
20 + "isCoin": false,
21 + "isSystem": false
22 + },
23 + "columnLabels": [
24 + "Ungraded",
25 + "Grade 7",
26 + "Grade 8",
27 + "Grade 9",
28 + "Grade 9.5",
29 + "PSA 10",
30 + "Grade 9",
31 + "Grade 9.5",
32 + "PSA 10"
33 + ],
34 + "prices": [
35 + {
36 + "key": "used_price",
37 + "value": 4387.5,
38 + "raw": "$4,387.50"
39 + },
40 + {
41 + "key": "complete_price",
42 + "value": 13110.48,
43 + "raw": "$13,110.48"
44 + },
45 + {
46 + "key": "new_price",
47 + "value": 17250,
48 + "raw": "$17,250.00"
49 + },
50 + {
51 + "key": "graded_price",
52 + "value": 38291.75,
53 + "raw": "$38,291.75"
54 + },
55 + {
56 + "key": "box_only_price",
57 + "value": 45760,
58 + "raw": "$45,760.00"
59 + },
60 + {
61 + "key": "manual_only_price",
62 + "value": 329400,
63 + "raw": "$329,400.00"
64 + }
65 + ],
66 + "fullPrices": [
67 + {
68 + "label": "Ungraded",
69 + "value": 4387.5
70 + },
71 + {
72 + "label": "Grade 1",
73 + "value": 5300
74 + },
75 + {
76 + "label": "Grade 2",
77 + "value": 6000
78 + },
79 + {
80 + "label": "Grade 3",
81 + "value": 6850
82 + },
83 + {
84 + "label": "Grade 4",
85 + "value": 7830.41
86 + },
87 + {
88 + "label": "Grade 5",
89 + "value": 8874.04
90 + },
91 + {
92 + "label": "Grade 6",
93 + "value": 9751.52
94 + },
95 + {
96 + "label": "Grade 7",
97 + "value": 13110.48
98 + },
99 + {
100 + "label": "Grade 8",
101 + "value": 17250
102 + },
103 + {
104 + "label": "Grade 9",
105 + "value": 38291.75
106 + },
107 + {
108 + "label": "Grade 9.5",
109 + "value": 45760
110 + },
111 + {
112 + "label": "TAG 10",
113 + "value": null
114 + },
115 + {
116 + "label": "ACE 10",
117 + "value": null
118 + },
119 + {
120 + "label": "SGC 10",
121 + "value": 111975
122 + },
123 + {
124 + "label": "CGC 10",
125 + "value": 197640
126 + },
127 + {
128 + "label": "PSA 10",
129 + "value": 329400
130 + },
131 + {
132 + "label": "BGS 10",
133 + "value": 428220
134 + },
135 + {
136 + "label": "BGS 10 Black",
137 + "value": 2141100
138 + },
139 + {
140 + "label": "CGC 10 Pristine",
141 + "value": 355752
142 + }
143 + ],
144 + "tabLabels": {
145 + "used": "Ungraded",
146 + "grade-twenty": "BGS 10 Black",
147 + "grade-nineteen": "CGC 10 Prist.",
148 + "manual-only": "PSA 10",
149 + "loose-and-box": "BGS 10",
150 + "grade-seventeen": "CGC 10",
151 + "grade-eighteen": "SGC 10",
152 + "grade-twenty-one": "TAG 10",
153 + "grade-twenty-two": "ACE 10",
154 + "box-only": "Grade 9.5",
155 + "graded": "Grade 9",
156 + "new": "Grade 8",
157 + "cib": "Grade 7",
158 + "grade-six": "Grade 6",
159 + "grade-five": "Grade 5",
160 + "grade-four": "Grade 4",
161 + "grade-three": "Grade 3",
162 + "box-and-manual": "Grade 2",
163 + "loose-and-manual": "Grade 1"
164 + },
165 + "sales": [
166 + {
167 + "tab": "used",
168 + "date": "2026-09-05",
169 + "title": "1986-87 Fleer #57 MICHAEL JORDAN RC Rookie Card - SGC Authentic #57",
170 + "price": 4103.33,
171 + "ebayId": "237036449409",
172 + "listedPrice": null
173 + },
174 + {
175 + "tab": "used",
176 + "date": "2026-09-04",
177 + "title": "#57 MICHAEL JORDAN rc 1986 FLEER SGC A rookie #57",
178 + "price": 4325,
179 + "ebayId": "158244187319",
180 + "listedPrice": 4695
181 + },
182 + {
183 + "tab": "used",
184 + "date": "2026-08-31",
185 + "title": "#57 MICHAEL JORDAN rc 1986 FLEER SGC A rookie #57",
186 + "price": 4425,
187 + "ebayId": "800587891478",
188 + "listedPrice": 4695
189 + },
190 + {
191 + "tab": "cib",
192 + "date": "2026-09-04",
193 + "title": "1986 Fleer Michael Jordan Rookie #57 PSA NM 7....",
194 + "price": 14640,
195 + "ebayId": null,
196 + "listedPrice": null
197 + },
198 + {
199 + "tab": "cib",
200 + "date": "2026-09-04",
201 + "title": "1986 FLEER #57 MICHAEL JORDAN PSA 7.5 #57",
202 + "price": 14700,
203 + "ebayId": "117395803522",
204 + "listedPrice": null
205 + },
206 + {
207 + "tab": "cib",
208 + "date": "2026-09-02",
209 + "title": "1986 Fleer #57 Michael Jordan Rookie (RC) PSA 7 NM #57",
210 + "price": 12689.99,
211 + "ebayId": "398319363158",
212 + "listedPrice": null
213 + },
214 + {
215 + "tab": "new",
216 + "date": "2026-09-05",
217 + "title": "1986 FLEER #57 MICHAEL JORDAN ROOKIE RC PSA 8 PD #57",
218 + "price": 13800,
219 + "ebayId": "117381662174",
220 + "listedPrice": null
221 + },
222 + {
223 + "tab": "new",
224 + "date": "2026-09-04",
225 + "title": "1986 Fleer Michael Jordan Rookie #57 PSA NM-MT+ 8.5 - MBA Gold!...",
226 + "price": 45140,
227 + "ebayId": null,
228 + "listedPrice": null
229 + },
230 + {
231 + "tab": "new",
232 + "date": "2026-09-04",
233 + "title": "1986 Fleer Michael Jordan Rookie #57 BGS NM-MT 8....",
234 + "price": 12810,
235 + "ebayId": null,
236 + "listedPrice": null
237 + },
238 + {
239 + "tab": "graded",
240 + "date": "2026-09-05",
241 + "title": "1986 FLEER #57 MICHAEL JORDAN ROOKIE RC PSA 9 #57",
242 + "price": 55600,
243 + "ebayId": "307149786913",
244 + "listedPrice": null
245 + },
246 + {
247 + "tab": "graded",
248 + "date": "2026-09-04",
249 + "title": "1986-87 Fleer #57 Michael Jordan Rookie Card - BGS MINT 9",
250 + "price": 29341,
251 + "ebayId": null,
252 + "listedPrice": null
253 + },
254 + {
255 + "tab": "graded",
256 + "date": "2026-08-30",
257 + "title": "1986-87 Fleer - Michael Jordan #57 (RC) Bgs 9 #57",
258 + "price": 28000,
259 + "ebayId": "147526618124",
260 + "listedPrice": null
261 + },
262 + {
263 + "tab": "box-only",
264 + "date": "2026-09-04",
265 + "title": "1986 Fleer Michael Jordan Rookie #57 BGS Gem Mint 9.5....",
266 + "price": 64050,
267 + "ebayId": null,
268 + "listedPrice": null
269 + },
270 + {
271 + "tab": "box-only",
272 + "date": "2026-07-26",
273 + "title": "1986 Fleer Michael Jordan #57 Rookie RC BGS 9.5 GEM MINT INVESTMENT GRADE! #57",
274 + "price": 45000,
275 + "ebayId": "318632307846",
276 + "listedPrice": 63750
277 + },
278 + {
279 + "tab": "box-only",
280 + "date": "2026-07-24",
281 + "title": "1986 Fleer Michael Jordan #57 Rookie RC BGS 9.5 GEM MINT INVESTMENT GRADE! #57",
282 + "price": 45760,
283 + "ebayId": "318564835550",
284 + "listedPrice": 63023
285 + },
286 + {
287 + "tab": "manual-only",
288 + "date": "2026-09-04",
289 + "title": "1986 Fleer Michael Jordan Rookie #57 PSA Gem Mint 10....",
290 + "price": 414800,
291 + "ebayId": null,
292 + "listedPrice": null
293 + },
294 + {
295 + "tab": "manual-only",
296 + "date": "2026-06-29",
297 + "title": "1986-87 Fleer #57 Michael Jordan Rookie Card - PSA GEM MT 10",
298 + "price": 396500,
299 + "ebayId": null,
300 + "listedPrice": null
301 + },
302 + {
303 + "tab": "manual-only",
304 + "date": "2026-05-26",
305 + "title": "Fleer 1986-87 Michael Jordan Rookie #57 PSA 10 Chicago Bulls NBA #57",
306 + "price": 2000,
307 + "ebayId": "358602801220",
308 + "listedPrice": null
309 + },
310 + {
311 + "tab": "loose-and-manual",
312 + "date": "2026-08-25",
313 + "title": "1986-87 Fleer Michael Jordan Rookie RC #57 Bulls PSA 1",
314 + "price": 5600,
315 + "ebayId": "128034969640",
316 + "listedPrice": null
317 + },
318 + {
319 + "tab": "loose-and-manual",
320 + "date": "2026-08-02",
321 + "title": "1986 FLEER #57 MICHAEL JORDAN Rookie Graded - PSA 1 #57",
322 + "price": 5300,
323 + "ebayId": "407113279087",
324 + "listedPrice": null
325 + },
326 + {
327 + "tab": "loose-and-manual",
328 + "date": "2026-07-22",
329 + "title": "1986 FLEER #57 MICHAEL JORDAN BULLS HOF CGC 1",
330 + "price": 4223,
331 + "ebayId": "267729608653",
332 + "listedPrice": null
333 + },
334 + {
335 + "tab": "box-and-manual",
336 + "date": "2026-08-30",
337 + "title": "MICHAEL JORDAN BGS 2 Centering 9 1986-87 FLEER BASKETBALL #57 ROOKIE RC BULLS #57",
338 + "price": 5000,
339 + "ebayId": "298627586162",
340 + "listedPrice": 5555
341 + },
342 + {
343 + "tab": "box-and-manual",
344 + "date": "2026-08-27",
345 + "title": "1986 FLEER #57 MICHAEL JORDAN PSA 2 #57",
346 + "price": 6500,
347 + "ebayId": "278299146438",
348 + "listedPrice": null
349 + },
350 + {
351 + "tab": "box-and-manual",
352 + "date": "2026-08-18",
353 + "title": "MICHAEL JORDAN PSA 2.5 1986-87 FLEER #57 ROOKIE RC BULLS 8774 #57",
354 + "price": 6600,
355 + "ebayId": "178400864396",
356 + "listedPrice": null
357 + },
358 + {
359 + "tab": "grade-three",
360 + "date": "2026-09-04",
361 + "title": "1986-87 Fleer #57 Michael Jordan Rookie Card - PSA VG 3",
362 + "price": 7991,
363 + "ebayId": null,
364 + "listedPrice": null
365 + },
366 + {
367 + "tab": "grade-three",
368 + "date": "2026-09-04",
369 + "title": "1986 Fleer #57 Michael Jordan RC Rookie PSA 3 VG Bulls Centered Looks nicer! #57",
370 + "price": 6878,
371 + "ebayId": "278312114498",
372 + "listedPrice": null
373 + },
374 + {
375 + "tab": "grade-three",
376 + "date": "2026-09-03",
377 + "title": "1986-87 Fleer Michael Jordan Rookie RC #57 Bulls BGS 3.5",
378 + "price": 5450,
379 + "ebayId": "128049266712",
380 + "listedPrice": null
381 + },
382 + {
383 + "tab": "grade-four",
384 + "date": "2026-09-02",
385 + "title": "1986-87 Fleer Set-Break # 57 Michael Jordan SGC 4 VG EX #57",
386 + "price": 5600,
387 + "ebayId": "327321185642",
388 + "listedPrice": null
389 + },
390 + {
391 + "tab": "grade-four",
392 + "date": "2026-08-28",
393 + "title": "1986 FLEER #57 MICHAEL JORDAN ROOKIE RC PSA 4 #57",
394 + "price": 8100,
395 + "ebayId": "307135191539",
396 + "listedPrice": null
397 + },
398 + {
399 + "tab": "grade-four",
400 + "date": "2026-08-26",
401 + "title": "MICHAEL JORDAN BGS 4.5 1986-87 FLEER BASKETBALL #57 ROOKIE RC BULLS 0161 #57",
402 + "price": 6900,
403 + "ebayId": "188811675022",
404 + "listedPrice": null
405 + },
406 + {
407 + "tab": "grade-five",
408 + "date": "2026-09-01",
409 + "title": "1986-87 Fleer #57 Michael Jordan RC Rookie BGS 5.5 EX+ BULLS HOF",
410 + "price": 7100,
411 + "ebayId": "147520678067",
412 + "listedPrice": null
413 + },
414 + {
415 + "tab": "grade-five",
416 + "date": "2026-08-31",
417 + "title": "1986 FLEER MICHAEL JORDAN ROOKIE PSA 5 EX! RC #57 CHICAGO BULLS VINTAGE!! #57",
418 + "price": 9660.99,
419 + "ebayId": "318761198005",
420 + "listedPrice": null
421 + },
422 + {
423 + "tab": "grade-five",
424 + "date": "2026-08-28",
425 + "title": "1986-87 Fleer #57 Michael Jordan Rookie Card - PSA EX 5",
426 + "price": 9951.54,
427 + "ebayId": null,
428 + "listedPrice": null
429 + },
430 + {
431 + "tab": "grade-six",
432 + "date": "2026-09-04",
433 + "title": "1986-87 Fleer - Michael Jordan #57 (RC) BGS 6.5 #57",
434 + "price": 7650,
435 + "ebayId": "237049503492",
436 + "listedPrice": null
437 + },
438 + {
439 + "tab": "grade-six",
440 + "date": "2026-09-02",
441 + "title": "1986 Fleer Michael Jordan Rookie #57 Beckett 6 P1454 #57",
442 + "price": 9925,
443 + "ebayId": "366606015198",
444 + "listedPrice": 10500
445 + },
446 + {
447 + "tab": "grade-six",
448 + "date": "2026-09-01",
449 + "title": "1986 Fleer Basketball - #57 Michael Jordan - PSA 6 #57",
450 + "price": 11600,
451 + "ebayId": "407171042383",
452 + "listedPrice": null
453 + },
454 + {
455 + "tab": "grade-eighteen",
456 + "date": "2025-07-25",
457 + "title": "1986 Fleer Michael Jordan Rookie #57 SGC Gem Mint 10....",
458 + "price": 118950,
459 + "ebayId": null,
460 + "listedPrice": null
461 + },
462 + {
463 + "tab": "grade-eighteen",
464 + "date": "2025-05-08",
465 + "title": "1986 Fleer Michael Jordan Rookie #57 SGC Gem Mint 10 RAREST GEM 10 JORDAN CARD #57",
466 + "price": 105000,
467 + "ebayId": "256409730922",
468 + "listedPrice": null
469 + }
470 + ],
471 + "details": {
472 + "See": "Population Report",
473 + "Is Rookie Card": "Yes",
474 + "Genre": "Basketball Cards",
475 + "Publisher": "Fleer",
476 + "Card Number": "#57",
477 + "ePID (eBay)": "23056508814",
478 + "PriceCharting ID": "72584"
479 + },
480 + "images": [
481 + "https://storage.googleapis.com/images.pricecharting.com/cxhob7pcogyxq3yu/240.jpg",
482 + "https://storage.googleapis.com/images.pricecharting.com/cxhob7pcogyxq3yu/1600.jpg"
483 + ],
484 + "setRef": null,
485 + "cardRef": null,
486 + "site": "sportscardspro"
487 + }
488 + },
489 + "expect": {
490 + "minCount": 10,
491 + "kinds": [
492 + "catalog_item",
493 + "price_observation",
494 + "sale"
495 + ],
496 + "requiredFields": [
497 + "attributes.set",
498 + "attributes.number"
499 + ]
500 + },
501 + "note": "Live capture of https://www.sportscardspro.com/game/basketball-cards-1986-fleer/michael-jordan-57 (sales trimmed to 3 rows per grade tab).",
502 + "capturedAt": "2026-09-07T05:50:38.932Z"
503 +}
\ No newline at end of file
added data/fixtures/tcgdex/en-base1-1.json +204 −0
@@ -0,0 +1,204 @@
1 +{
2 + "raw": {
3 + "url": "https://tcgdex.net/en/database/base1/1",
4 + "externalId": "en:base1-1",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:39.646Z",
8 + "payload": {
9 + "card": {
10 + "id": "base1-1",
11 + "localId": "1",
12 + "name": "Alakazam",
13 + "rarity": "Rare",
14 + "category": "Pokemon",
15 + "illustrator": "Ken Sugimori",
16 + "image": "https://assets.tcgdex.net/en/base/base1/1",
17 + "hp": 80,
18 + "types": [
19 + "Psychic"
20 + ],
21 + "dexId": [
22 + 65
23 + ],
24 + "updated": "2026-08-20T08:25:49+01:00",
25 + "set": {
26 + "id": "base1",
27 + "name": "Base Set"
28 + },
29 + "variants": {
30 + "firstEdition": true,
31 + "holo": true,
32 + "normal": false,
33 + "reverse": false,
34 + "wPromo": false
35 + },
36 + "variants_detailed": [
37 + {
38 + "type": "holo",
39 + "subtype": "unlimited",
40 + "thirdParty": {
41 + "cardmarket": 273696,
42 + "tcgplayer": 42346
43 + },
44 + "pricing": {
45 + "cardmarket": {
46 + "updated": "2026-09-06T15:09:28.340Z",
47 + "unit": "EUR",
48 + "idProduct": 273696,
49 + "avg": 62.83,
50 + "low": 10,
51 + "trend": 60.75,
52 + "avg1": 30.85,
53 + "avg7": 59.51,
54 + "avg30": 71.81,
55 + "avg-holo": null,
56 + "low-holo": null,
57 + "trend-holo": 19.66,
58 + "avg1-holo": 7.99,
59 + "avg7-holo": 14.29,
60 + "avg30-holo": 25.6
61 + },
62 + "tcgplayer": {
63 + "unit": "USD",
64 + "updated": "2026-09-06T15:09:24.320Z",
65 + "holofoil": {
66 + "productId": 42346,
67 + "lowPrice": 43.99,
68 + "midPrice": 60,
69 + "highPrice": 9999,
70 + "marketPrice": 69.34,
71 + "directLowPrice": 385
72 + }
73 + }
74 + },
75 + "size": "standard",
76 + "variantId": "4ffrmhcfiaejakhepqdkx7o"
77 + },
78 + {
79 + "type": "holo",
80 + "subtype": "shadowless",
81 + "stamp": [
82 + "1st-edition"
83 + ],
84 + "thirdParty": {
85 + "cardmarket": 660227,
86 + "tcgplayer": 106996
87 + },
88 + "pricing": {
89 + "cardmarket": {
90 + "updated": "2026-09-06T15:09:28.340Z",
91 + "unit": "EUR",
92 + "idProduct": 660227,
93 + "avg": 163.25,
94 + "low": 65,
95 + "trend": 245.24,
96 + "avg1": 70,
97 + "avg7": 159.71,
98 + "avg30": 190.17,
99 + "avg-holo": null,
100 + "low-holo": null,
101 + "trend-holo": 0,
102 + "avg1-holo": null,
103 + "avg7-holo": null,
104 + "avg30-holo": null
105 + },
106 + "tcgplayer": null
107 + },
108 + "size": "standard",
109 + "variantId": "mtltux8qtgdu4exu903oasum21juxbvx6lx"
110 + },
111 + {
112 + "type": "holo",
113 + "subtype": "shadowless",
114 + "thirdParty": {
115 + "cardmarket": 660227,
116 + "tcgplayer": 106996
117 + },
118 + "pricing": {
119 + "cardmarket": {
120 + "updated": "2026-09-06T15:09:28.340Z",
121 + "unit": "EUR",
122 + "idProduct": 660227,
123 + "avg": 163.25,
124 + "low": 65,
125 + "trend": 245.24,
126 + "avg1": 70,
127 + "avg7": 159.71,
128 + "avg30": 190.17,
129 + "avg-holo": null,
130 + "low-holo": null,
131 + "trend-holo": 0,
132 + "avg1-holo": null,
133 + "avg7-holo": null,
134 + "avg30-holo": null
135 + },
136 + "tcgplayer": null
137 + },
138 + "size": "standard",
139 + "variantId": "3takscxpcqoqcfnxk1ivs2y6"
140 + },
141 + {
142 + "type": "holo",
143 + "subtype": "1999-2000-copyright",
144 + "size": "standard",
145 + "variantId": "zqq5g2u9n0st0gren5bssktmac2ywqaw"
146 + }
147 + ],
148 + "pricing": {
149 + "cardmarket": {
150 + "updated": "2026-09-06T15:09:28.340Z",
151 + "unit": "EUR",
152 + "idProduct": 273696,
153 + "avg": 62.83,
154 + "low": 10,
155 + "trend": 60.75,
156 + "avg1": 30.85,
157 + "avg7": 59.51,
158 + "avg30": 71.81,
159 + "avg-holo": null,
160 + "low-holo": null,
161 + "trend-holo": 19.66,
162 + "avg1-holo": 7.99,
163 + "avg7-holo": 14.29,
164 + "avg30-holo": 25.6
165 + },
166 + "tcgplayer": {
167 + "unit": "USD",
168 + "updated": "2026-09-06T15:09:24.320Z",
169 + "holofoil": {
170 + "productId": 42346,
171 + "lowPrice": 43.99,
172 + "midPrice": 60,
173 + "highPrice": 9999,
174 + "marketPrice": 69.34,
175 + "directLowPrice": 385
176 + }
177 + }
178 + }
179 + },
180 + "set": {
181 + "id": "base1",
182 + "name": "Base Set",
183 + "code": "BS",
184 + "releaseDate": "1999-01-09",
185 + "serie": "Base",
186 + "total": 102
187 + },
188 + "language": "en"
189 + }
190 + },
191 + "expect": {
192 + "minCount": 1,
193 + "kinds": [
194 + "catalog_item",
195 + "price_observation"
196 + ],
197 + "requiredFields": [
198 + "attributes.setCode",
199 + "attributes.number"
200 + ]
201 + },
202 + "note": "Live capture https://tcgdex.net/en/database/base1/1",
203 + "capturedAt": "2026-09-07T05:50:39.652Z"
204 +}
\ No newline at end of file
added data/fixtures/tcgdex/en-base1-2.json +204 −0
@@ -0,0 +1,204 @@
1 +{
2 + "raw": {
3 + "url": "https://tcgdex.net/en/database/base1/2",
4 + "externalId": "en:base1-2",
5 + "kind": "catalog_item",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T05:50:39.768Z",
8 + "payload": {
9 + "card": {
10 + "id": "base1-2",
11 + "localId": "2",
12 + "name": "Blastoise",
13 + "rarity": "Rare",
14 + "category": "Pokemon",
15 + "illustrator": "Ken Sugimori",
16 + "image": "https://assets.tcgdex.net/en/base/base1/2",
17 + "hp": 100,
18 + "types": [
19 + "Water"
20 + ],
21 + "dexId": [
22 + 9
23 + ],
24 + "updated": "2026-09-04T23:45:25+01:00",
25 + "set": {
26 + "id": "base1",
27 + "name": "Base Set"
28 + },
29 + "variants": {
30 + "firstEdition": true,
31 + "holo": true,
32 + "normal": false,
33 + "reverse": false,
34 + "wPromo": false
35 + },
36 + "variants_detailed": [
37 + {
38 + "type": "holo",
39 + "subtype": "unlimited",
40 + "thirdParty": {
41 + "cardmarket": 273697,
42 + "tcgplayer": 42360
43 + },
44 + "pricing": {
45 + "cardmarket": {
46 + "updated": "2026-09-06T15:09:28.340Z",
47 + "unit": "EUR",
48 + "idProduct": 273697,
49 + "avg": 131.15,
50 + "low": 23,
51 + "trend": 290.94,
52 + "avg1": 127.48,
53 + "avg7": 393.81,
54 + "avg30": 227.49,
55 + "avg-holo": null,
56 + "low-holo": null,
57 + "trend-holo": 73.51,
58 + "avg1-holo": 100,
59 + "avg7-holo": 50.43,
60 + "avg30-holo": 64.46
61 + },
62 + "tcgplayer": {
63 + "unit": "USD",
64 + "updated": "2026-09-06T15:09:24.320Z",
65 + "holofoil": {
66 + "productId": 42360,
67 + "lowPrice": 143.99,
68 + "midPrice": 192.49,
69 + "highPrice": 1581.3,
70 + "marketPrice": 228.33,
71 + "directLowPrice": 143.99
72 + }
73 + }
74 + },
75 + "size": "standard",
76 + "variantId": "4ffrmhcfiaejakhepqdkx7o"
77 + },
78 + {
79 + "type": "holo",
80 + "subtype": "shadowless",
81 + "stamp": [
82 + "1st-edition"
83 + ],
84 + "thirdParty": {
85 + "cardmarket": 660226,
86 + "tcgplayer": 106997
87 + },
88 + "pricing": {
89 + "cardmarket": {
90 + "updated": "2026-09-06T15:09:28.340Z",
91 + "unit": "EUR",
92 + "idProduct": 660226,
93 + "avg": 480,
94 + "low": 159.94,
95 + "trend": 914.17,
96 + "avg1": 340,
97 + "avg7": 910.85,
98 + "avg30": 635.22,
99 + "avg-holo": null,
100 + "low-holo": null,
101 + "trend-holo": 0,
102 + "avg1-holo": null,
103 + "avg7-holo": null,
104 + "avg30-holo": null
105 + },
106 + "tcgplayer": null
107 + },
108 + "size": "standard",
109 + "variantId": "mtltux8qtgdu4exu903oasum21juxbvx6lx"
110 + },
111 + {
112 + "type": "holo",
113 + "subtype": "shadowless",
114 + "thirdParty": {
115 + "cardmarket": 660226,
116 + "tcgplayer": 106997
117 + },
118 + "pricing": {
119 + "cardmarket": {
120 + "updated": "2026-09-06T15:09:28.340Z",
121 + "unit": "EUR",
122 + "idProduct": 660226,
123 + "avg": 480,
124 + "low": 159.94,
125 + "trend": 914.17,
126 + "avg1": 340,
127 + "avg7": 910.85,
128 + "avg30": 635.22,
129 + "avg-holo": null,
130 + "low-holo": null,
131 + "trend-holo": 0,
132 + "avg1-holo": null,
133 + "avg7-holo": null,
134 + "avg30-holo": null
135 + },
136 + "tcgplayer": null
137 + },
138 + "size": "standard",
139 + "variantId": "3takscxpcqoqcfnxk1ivs2y6"
140 + },
141 + {
142 + "type": "holo",
143 + "subtype": "1999-2000-copyright",
144 + "size": "standard",
145 + "variantId": "zqq5g2u9n0st0gren5bssktmac2ywqaw"
146 + }
147 + ],
148 + "pricing": {
149 + "cardmarket": {
150 + "updated": "2026-09-06T15:09:28.340Z",
151 + "unit": "EUR",
152 + "idProduct": 273697,
153 + "avg": 131.15,
154 + "low": 23,
155 + "trend": 290.94,
156 + "avg1": 127.48,
157 + "avg7": 393.81,
158 + "avg30": 227.49,
159 + "avg-holo": null,
160 + "low-holo": null,
161 + "trend-holo": 73.51,
162 + "avg1-holo": 100,
163 + "avg7-holo": 50.43,
164 + "avg30-holo": 64.46
165 + },
166 + "tcgplayer": {
167 + "unit": "USD",
168 + "updated": "2026-09-06T15:09:24.320Z",
169 + "holofoil": {
170 + "productId": 42360,
171 + "lowPrice": 143.99,
172 + "midPrice": 192.49,
173 + "highPrice": 1581.3,
174 + "marketPrice": 228.33,
175 + "directLowPrice": 143.99
176 + }
177 + }
178 + }
179 + },
180 + "set": {
181 + "id": "base1",
182 + "name": "Base Set",
183 + "code": "BS",
184 + "releaseDate": "1999-01-09",
185 + "serie": "Base",
186 + "total": 102
187 + },
188 + "language": "en"
189 + }
190 + },
191 + "expect": {
192 + "minCount": 1,
193 + "kinds": [
194 + "catalog_item",
195 + "price_observation"
196 + ],
197 + "requiredFields": [
198 + "attributes.setCode",
199 + "attributes.number"
200 + ]
201 + },
202 + "note": "Live capture https://tcgdex.net/en/database/base1/2",
203 + "capturedAt": "2026-09-07T05:50:39.769Z"
204 +}
\ No newline at end of file
modified workers/entity-resolution/resolver.ts +9 −4
@@ -73,13 +73,18 @@ export async function resolveAsset(a: AssetAttributes, opts: { enrich?: boolean;
73 73 return { assetId: cached, method: 'canonical_key', confidence: 0.97, created: false, canonicalKey };
74 74 }
75 75
76 − // 1. deterministic identifiers
76 + // 1. deterministic identifiers. Some ids are card-level, not printing-level (pokemontcg_id, ygo_id share
77 + // one id across Holo / 1st Edition / Reverse variants), so a hit must also agree on the variant when both
78 + // sides state one; otherwise fall through to the canonical key (which includes the variant).
79 + const normVariant = (x: string | null | undefined) => (x ? x.toLowerCase().replace(/[\s.\-/]+/g, '') : '');
80 + const incomingVariant = normVariant(a.variant);
77 81 for (const [k, v] of Object.entries(ids)) {
78 − const [row] = await db()
79 − .select({ id: assets.id })
82 + const rows = await db()
83 + .select({ id: assets.id, variant: assets.variant })
80 84 .from(assets)
81 85 .where(and(sql`${assets.identifiers} @> ${JSON.stringify({ [k]: v })}::jsonb`, eq(assets.familySlug, categoryFamily(a.categorySlug))))
82 − .limit(1);
86 + .limit(5);
87 + const row = rows.find((r) => !incomingVariant || !normVariant(r.variant) || normVariant(r.variant) === incomingVariant);
83 88 if (row) {
84 89 remember(canonicalKey, row.id);
85 90 if (opts.enrich) await enrichAsset(row.id, a, opts);
86 91