TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, priceObservation } from '../_lib/shared.js';5import { HTML_HEADERS, JSON_HEADERS, cleanText, dayOf, usdEur } from '../_g1-cards-eu-jp-lib/index.js';67/**8 * Limitless TCG Pokémon card database: set index → per-set list tables (number, name, rarity, image,9 * relayed TCGplayer USD / Cardmarket EUR prices) and, in backfill/lookup, the per-card price history10 * JSON the public card page itself loads (/api/cards/<id>/prices).11 */12const SITE = 'https://limitlesstcg.com';13const PARSER_VERSION = '1.0.0';1415export const LangSchema = z.enum(['en', 'jp']);16export type Lang = z.infer<typeof LangSchema>;1718export const SetSchema = z.object({ code: z.string(), name: z.string(), lang: LangSchema, releaseDate: z.string().nullable(), cardCount: z.number().int().nullable() });19export type LimitlessSet = z.infer<typeof SetSchema>;2021const PriceSchema = z.object({ amount: z.number(), currency: z.enum(['USD', 'EUR']), url: z.string().nullable(), tcgplayerId: z.string().nullable() });22export const CardRowSchema = z.object({23 number: z.string(),24 name: z.string(),25 url: z.string(),26 type: z.string().nullable(),27 rarity: z.string().nullable(),28 image: z.string().nullable(),29 usd: PriceSchema.nullable(),30 eur: PriceSchema.nullable(),31});32export type CardRow = z.infer<typeof CardRowSchema>;3334const SetPagePayloadSchema = z.object({ set: SetSchema, cards: z.array(CardRowSchema) });35const HistoryPayloadSchema = z.object({36 set: SetSchema,37 card: z.object({ number: z.string(), name: z.string(), url: z.string(), cardId: z.number().int(), rarity: z.string().nullable(), image: z.string().nullable(), tcgplayerId: z.string().nullable() }),38 history: z.object({ tcgplayer: z.array(z.tuple([z.number(), z.number()])).default([]), cardmarket: z.array(z.tuple([z.number(), z.number()])).default([]) }),39});40const RawPayloadSchema = z.union([SetPagePayloadSchema.extend({ type: z.literal('set') }), HistoryPayloadSchema.extend({ type: z.literal('history') })]);41export type LimitlessPayload = z.infer<typeof RawPayloadSchema>;4243const MONTHS: Record<string, number> = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11 };4445/** "26 Sep 25" | "17 Jul 26" → "2025-09-26" (Limitless prints 2-digit years). */46export function parseShortDate(s: string | null | undefined): string | null {47 if (!s) return null;48 const m = s.trim().match(/^(\d{1,2})\s+([A-Za-z]{3})\s+(\d{2}|\d{4})$/);49 if (!m) return null;50 const mo = MONTHS[m[2]!.toLowerCase()];51 if (mo === undefined) return null;52 const y = m[3]!.length === 2 ? 2000 + Number(m[3]) : Number(m[3]);53 const d = new Date(Date.UTC(y, mo, Number(m[1])));54 return Number.isNaN(d.getTime()) ? null : d.toISOString().slice(0, 10);55}5657export function tcgplayerIdFromUrl(href: string | null | undefined): string | null {58 if (!href) return null;59 const dec = decodeURIComponent(href);60 return dec.match(/tcgplayer\.com\/product\/(\d+)/)?.[1] ?? null;61}6263/** Parse the set index tables (/cards and /cards/jp). */64export function parseSetIndex(doc: string, lang: Lang): LimitlessSet[] {65 const $ = html.load(doc);66 const out: LimitlessSet[] = [];67 const prefix = lang === 'jp' ? '/cards/jp/' : '/cards/';68 $('table.sets-table tr').each((_, tr) => {69 const tds = $(tr).find('td');70 if (tds.length < 2) return;71 const a = $(tds[0]).find('a').first();72 const href = a.attr('href') ?? '';73 if (!href.startsWith(prefix)) return;74 const code = href.slice(prefix.length).split(/[/?#]/)[0]!;75 if (!code || (lang === 'en' && code.startsWith('jp'))) return;76 const codeSpan = a.find('span.code').text().trim();77 const name = cleanText(a.clone().children('img, span').remove().end().text()) ?? code;78 const releaseDate = parseShortDate($(tds[1]).text());79 const countText = tds.length > 2 ? $(tds[2]).find('a').clone().children().remove().end().text() : '';80 const cardCount = Number.parseInt(countText.replace(/[^0-9]/g, ''), 10);81 out.push({ code: codeSpan || code, name, lang, releaseDate, cardCount: Number.isFinite(cardCount) ? cardCount : null });82 });83 return out;84}8586function priceCell($: html.CheerioRoot, td: unknown): z.infer<typeof PriceSchema> | null {87 const a = $(td as never).find('a.card-price').first();88 if (!a.length) return null;89 const p = usdEur(a.text());90 if (!p) return null;91 const url = a.attr('href') ?? null;92 return { amount: p.amount, currency: p.currency, url, tcgplayerId: tcgplayerIdFromUrl(url) };93}9495/** Parse a set list page (?display=list): one row per card. */96export function parseSetList(doc: string): { setName: string | null; cards: CardRow[] } {97 const $ = html.load(doc);98 const cards: CardRow[] = [];99 let setName: string | null = null;100 $('table.card-list tr').each((_, tr) => {101 const tds = $(tr).find('td');102 if (tds.length < 3) return;103 const tooltip = $(tds[0]).find('.card-set').attr('data-tooltip');104 if (tooltip && !setName) setName = tooltip.trim();105 const numA = $(tds[1]).find('a').first();106 const nameA = $(tds[2]).find('a').first();107 const number = cleanText(numA.text());108 const name = cleanText(nameA.text());109 const href = nameA.attr('href') ?? numA.attr('href');110 if (!number || !name || !href) return;111 const type = tds.length > 3 ? cleanText($(tds[3]).text()) : null;112 const rarity = tds.length > 4 ? cleanText($(tds[4]).text()) : null;113 const hover = $(tr).attr('data-hover') ?? null;114 const image = hover ? hover.replace(/_XS\.png$/, '.png') : null;115 const usd = tds.length > 5 ? priceCell($, tds[5]) : null;116 const eur = tds.length > 6 ? priceCell($, tds[6]) : null;117 cards.push({ number, name, url: href.startsWith('http') ? href : `${SITE}${href}`, type, rarity, image, usd, eur });118 });119 return { setName, cards };120}121122/** Card page: numeric cardId (drives /api/cards/<id>/prices), name, number/rarity line, image. */123export function parseCardPage(doc: string): { cardId: number | null; name: string | null; number: string | null; rarity: string | null; setName: string | null; image: string | null; tcgplayerId: string | null } {124 const $ = html.load(doc);125 const cardId = Number(doc.match(/var\s+cardId\s*=\s*(\d+)/)?.[1] ?? NaN);126 const name = cleanText($('.card-text-name').first().text());127 const details = cleanText($('.prints-current-details').text()) ?? '';128 const m = details.match(/#\s*([A-Za-z0-9]+)\s*·\s*(.+)$/);129 const setName = cleanText($('.prints-current-details .text-lg').text())?.replace(/\s*\([A-Z0-9-]+\)\s*$/, '') ?? null;130 const image = $('.card-image img').attr('data-src') ?? $('.card-image img').attr('src') ?? null;131 const tcgplayerId = tcgplayerIdFromUrl($('a.card-buy-button.usd, a.card-price.usd').first().attr('href'));132 return { cardId: Number.isFinite(cardId) ? cardId : null, name, number: m?.[1] ?? null, rarity: m?.[2]?.trim() ?? null, setName, image, tcgplayerId };133}134135const HISTORY_RESPONSE = z.object({ tcgplayer: z.array(z.tuple([z.number(), z.number()])).default([]), cardmarket: z.array(z.tuple([z.number(), z.number()])).default([]) });136137export class LimitlessTcgConnector extends BaseConnector {138 readonly version = '1.0.0';139 readonly parserVersion = PARSER_VERSION;140 protected override minIntervalMs = 1500;141 override readonly urlPatterns = [/^https?:\/\/(www\.)?limitlesstcg\.com\/cards\/(jp\/)?[A-Za-z0-9-]+\/[A-Za-z0-9]+\/?$/];142143 private async page(ctx: CrawlContext, url: string) {144 await this.throttle(url);145 return ctx.fetch(url, { engines: ['api'], headers: HTML_HEADERS, responseType: 'text', minQuality: 0 });146 }147148 private async sets(ctx: CrawlContext, lang: Lang): Promise<LimitlessSet[] | null> {149 const url = lang === 'jp' ? `${SITE}/cards/jp` : `${SITE}/cards`;150 const res = await this.page(ctx, url);151 if (!res.success || !res.html) {152 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);153 return null;154 }155 const sets = parseSetIndex(res.html, lang);156 if (!sets.length) ctx.anomaly('selector_missing', `${url}: no sets-table rows`);157 return sets;158 }159160 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {161 const langs = z.array(LangSchema).parse(this.meta.config.languages ?? ['en', 'jp']);162 const backfill = ctx.options.mode === 'backfill';163 const maxSets = Number(this.meta.config.maxSetsPerRun ?? 0) || Infinity;164 let count = 0;165 let langIdx = Number(ctx.options.cursor?.langIdx ?? 0);166 let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);167 const phase = String(ctx.options.cursor?.phase ?? 'sets');168 if (phase === 'sets') {169 for (; langIdx < langs.length; langIdx++, setIdx = 0) {170 const lang = langs[langIdx]!;171 let sets = await this.sets(ctx, lang);172 if (!sets) continue;173 if (ctx.options.seeds?.length) sets = sets.filter((s) => ctx.options.seeds!.includes(s.code) || ctx.options.seeds!.includes(`${lang}:${s.code}`));174 sets = sets.slice(0, Number.isFinite(maxSets) ? maxSets : sets.length);175 for (; setIdx < sets.length; setIdx++) {176 if (ctx.signal?.aborted) return;177 if (this.reached(ctx, count)) {178 await ctx.setCursor({ phase: 'sets', langIdx, setIdx });179 return;180 }181 const set = sets[setIdx]!;182 const url = lang === 'jp' ? `${SITE}/cards/jp/${set.code}?display=list` : `${SITE}/cards/${set.code}?display=list`;183 const res = await this.page(ctx, url);184 if (!res.success || !res.html) {185 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);186 continue;187 }188 const { setName, cards } = parseSetList(res.html);189 if (!cards.length) {190 ctx.anomaly('parse_failure_page', url);191 continue;192 }193 count++;194 const payload: LimitlessPayload = { type: 'set', set: { ...set, name: setName ?? set.name }, cards };195 yield { url, externalId: `set:${lang}:${set.code}`, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };196 await ctx.setCursor({ phase: 'sets', langIdx, setIdx: setIdx + 1 });197 await ctx.progress({ page: setIdx + 1, totalPages: sets.length, itemsProcessed: count });198 }199 }200 }201 if (backfill) {202 // Price history for English cards of the newest sets (the XHR the public card page performs).203 const maxCards = Number(this.meta.config.historyCardsPerRun ?? 60);204 const newest = Number(this.meta.config.historySetsNewestFirst ?? 6);205 const sets = (await this.sets(ctx, 'en'))?.slice(0, newest) ?? [];206 let hSet = Number(ctx.options.cursor?.hSet ?? 0);207 let hCard = Number(ctx.options.cursor?.hCard ?? 0);208 let done = 0;209 for (; hSet < sets.length && done < maxCards; hSet++, hCard = 0) {210 const set = sets[hSet]!;211 const listRes = await this.page(ctx, `${SITE}/cards/${set.code}?display=list`);212 if (!listRes.success || !listRes.html) continue;213 const { cards } = parseSetList(listRes.html);214 for (; hCard < cards.length && done < maxCards; hCard++) {215 if (ctx.signal?.aborted) return;216 if (this.reached(ctx, count)) {217 await ctx.setCursor({ phase: 'history', hSet, hCard });218 return;219 }220 const card = cards[hCard]!;221 const rec = await this.history(ctx, set, card);222 done++;223 if (!rec) continue;224 count++;225 yield rec;226 await ctx.setCursor({ phase: 'history', hSet, hCard: hCard + 1 });227 }228 }229 if (hSet >= sets.length) await ctx.setCursor({ phase: 'sets', langIdx: 0, setIdx: 0, done: true, completedAt: new Date().toISOString() });230 else await ctx.setCursor({ phase: 'history', hSet, hCard });231 return;232 }233 await ctx.setCursor({ phase: 'sets', langIdx: 0, setIdx: 0, completedAt: new Date().toISOString() });234 }235236 /** Card page → cardId → /api/cards/<id>/prices (dated history). */237 private async history(ctx: CrawlContext, set: LimitlessSet, card: Pick<CardRow, 'number' | 'name' | 'url' | 'rarity' | 'image'> & { usd?: CardRow['usd'] }): Promise<RawRecordInput | null> {238 const pageRes = await this.page(ctx, card.url);239 if (!pageRes.success || !pageRes.html) {240 ctx.anomaly('page_fetch_failed', `${card.url}: ${pageRes.error ?? pageRes.httpStatus}`);241 return null;242 }243 const cp = parseCardPage(pageRes.html);244 if (!cp.cardId) {245 ctx.anomaly('selector_missing', `${card.url}: cardId not found`);246 return null;247 }248 const apiUrl = `${SITE}/api/cards/${cp.cardId}/prices`;249 await this.throttle(apiUrl);250 const res = await ctx.fetch(apiUrl, { engines: ['api'], headers: { ...JSON_HEADERS, referer: card.url }, minQuality: 0 });251 const parsed = HISTORY_RESPONSE.safeParse(res.json);252 if (!res.success || !parsed.success) {253 ctx.anomaly('page_fetch_failed', `${apiUrl}: ${res.error ?? 'unexpected history shape'}`);254 return null;255 }256 const payload: LimitlessPayload = {257 type: 'history',258 set,259 card: { number: card.number, name: cp.name ?? card.name, url: card.url, cardId: cp.cardId, rarity: cp.rarity ?? card.rarity ?? null, image: cp.image ?? card.image ?? null, tcgplayerId: cp.tcgplayerId ?? card.usd?.tcgplayerId ?? null },260 history: parsed.data,261 };262 return { url: card.url, externalId: `history:${set.lang}:${set.code}/${card.number}`, kind: 'price_observation', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };263 }264265 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {266 const m = url.match(/limitlesstcg\.com\/cards\/(jp\/)?([A-Za-z0-9-]+)\/([A-Za-z0-9]+)/);267 if (!m) return [];268 const lang: Lang = m[1] ? 'jp' : 'en';269 const pageRes = await this.page(ctx, url);270 if (!pageRes.success || !pageRes.html) return [];271 const cp = parseCardPage(pageRes.html);272 const set: LimitlessSet = { code: m[2]!, name: cp.setName ?? m[2]!, lang, releaseDate: null, cardCount: null };273 const rec = await this.history(ctx, set, { number: cp.number ?? m[3]!, name: cp.name ?? m[3]!, url, rarity: cp.rarity, image: cp.image });274 return rec ? [rec] : [];275 }276277 private attrsFor(set: LimitlessSet, card: { number: string; name: string; rarity: string | null; type?: string | null; tcgplayerId: string | null }) {278 const ids: Record<string, string> = { limitless_card: `${set.lang}/${set.code}/${card.number}` };279 if (card.tcgplayerId) ids.tcgplayer_id = card.tcgplayerId;280 return attrs({281 categorySlug: 'pokemon',282 franchise: 'Pokémon',283 brand: 'The Pokémon Company',284 set: set.name,285 setCode: set.code,286 name: card.name,287 number: card.number,288 year: set.releaseDate ? Number(set.releaseDate.slice(0, 4)) || null : null,289 language: set.lang === 'jp' ? 'Japanese' : 'English',290 rarity: card.rarity,291 identifiers: ids,292 metadata: { card_type: card.type ?? null, set_card_count: set.cardCount, release_date: set.releaseDate },293 });294 }295296 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {297 const p = RawPayloadSchema.parse(raw.payload);298 const observedAt = raw.fetchedAt;299 const out: NormalizedRecord[] = [];300 const releaseDate = p.set.releaseDate ? new Date(p.set.releaseDate) : null;301 if (p.type === 'set') {302 for (const c of p.cards) {303 const a = this.attrsFor(p.set, { ...c, tcgplayerId: c.usd?.tcgplayerId ?? null });304 const rawTitle = makeTitle({ name: c.name, set: p.set.name, number: c.number, total: p.set.cardCount, year: a.year });305 const images = c.image ? [c.image] : [];306 out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, externalId: `${p.set.lang}:${p.set.code}/${c.number}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate }));307 for (const [provider, price] of [['tcgplayer', c.usd], ['cardmarket', c.eur]] as const) {308 if (!price) continue;309 out.push(310 priceObservation({311 kind: 'price_observation',312 connectorId: this.meta.id,313 sourceId: this.meta.sourceId,314 sourceUrl: c.url,315 externalId: `${p.set.lang}:${p.set.code}/${c.number}:${provider}`,316 rawTitle,317 imageUrls: images,318 attributes: { ...a, metadata: { ...a.metadata, provider, relayed: true, vendor_url: price.url } },319 observedAt,320 confidence: 0.6,321 parserVersion: PARSER_VERSION,322 priceKind: 'market',323 price: price.amount,324 currency: price.currency,325 observationDate: dayOf(observedAt),326 sampleSize: null,327 }),328 );329 }330 }331 return out;332 }333 const a = this.attrsFor(p.set, { number: p.card.number, name: p.card.name, rarity: p.card.rarity, tcgplayerId: p.card.tcgplayerId });334 const rawTitle = makeTitle({ name: p.card.name, set: p.set.name, number: p.card.number, year: a.year });335 const images = p.card.image ? [p.card.image] : [];336 for (const [provider, series, currency] of [['tcgplayer', p.history.tcgplayer, 'USD'], ['cardmarket', p.history.cardmarket, 'EUR']] as const) {337 const seen = new Set<string>();338 for (const [ms, cents] of series) {339 if (!(cents > 0)) continue;340 const day = dayOf(new Date(ms));341 const key = day.toISOString().slice(0, 10);342 if (seen.has(key) || day.getTime() > Date.now()) continue;343 seen.add(key);344 out.push(345 priceObservation({346 kind: 'price_observation',347 connectorId: this.meta.id,348 sourceId: this.meta.sourceId,349 sourceUrl: p.card.url,350 externalId: `${p.set.lang}:${p.set.code}/${p.card.number}:${provider}:${key}`,351 rawTitle,352 imageUrls: images,353 attributes: { ...a, metadata: { ...a.metadata, provider, relayed: true, limitless_card_id: p.card.cardId } },354 observedAt,355 confidence: 0.6,356 parserVersion: PARSER_VERSION,357 priceKind: 'market',358 price: cents / 100,359 currency,360 observationDate: day,361 sampleSize: null,362 }),363 );364 }365 }366 return out;367 }368}369370export default (meta: ConnectorMeta) => new LimitlessTcgConnector(meta);371