TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { attrs } from '../_lib/shared.js';6import { BOT_HEADERS, cardCategorySlug, decodeXml, parseSitemap } from '../_lib/wave4.js';78/** MySlabs — graded slab marketplace; public slab pages expose a Product JSON-LD. */9const PARSER_VERSION = '1.0.0';10const SITE = 'https://myslabs.com';1112const RawPayloadSchema = z.object({13 id: z.string(),14 name: z.string(),15 description: z.string().nullable(),16 images: z.array(z.string()),17 price: z.number().nullable(),18 currency: z.string().nullable(),19 availability: z.string().nullable(),20 category: z.string().nullable(),21 seller: z.string().nullable(),22});23export type MySlabsPayload = z.infer<typeof RawPayloadSchema>;2425const SPORTS_BRAND = /\b(topps|panini|bowman|prizm|fleer|donruss|upper deck|select|mosaic|optic|chrome|leaf|score|o-pee-chee|hoops|skybox|sp authentic|national treasures|immaculate|flawless|futera|stadium club|goudey|play ball|kellogg)\b/i;26const SPORT_WORDS: Array<[RegExp, string]> = [27 [/\b(basketball|nba|wnba|hoops|lebron|jordan|curry|wembanyama|kobe|giannis|luka|jokic)\b/i, 'basketball_cards'],28 [/\b(baseball|mlb|mantle|ohtani|trout|jeter|bowman|goudey|play ball)\b/i, 'baseball_cards'],29 [/\b(football|nfl|mahomes|brady|rookie qb|panini contenders|donruss optic football)\b/i, 'football_cards'],30 [/\b(hockey|nhl|gretzky|mcdavid|crosby|o-pee-chee|young guns|bedard)\b/i, 'hockey_cards'],31 [/\b(soccer|fifa|messi|ronaldo|mbappe|mbappé|haaland|yamal|premier league|champions league)\b/i, 'soccer_cards'],32 [/\b(f1|formula 1|formula one|verstappen|hamilton|leclerc|norris)\b/i, 'f1_cards'],33 [/\b(ufc|wwe|wrestling|boxing|golf|tennis|nascar|olympic|mma)\b/i, 'other_sports_cards'],34];3536/** Title → taxonomy slug for MySlabs inventory (cards/comics); null when unsure. */37export function slabCategory(title: string, categoryHint: string | null): string | null {38 const hinted = cardCategorySlug(categoryHint);39 if (hinted && hinted !== 'trading_cards') return hinted;40 const tcg = cardCategorySlug(title);41 if (tcg && ['pokemon', 'magic_the_gathering', 'yugioh', 'disney_lorcana', 'one_piece_card_game', 'digimon_tcg', 'flesh_and_blood', 'star_wars_tcg', 'dragon_ball_tcg'].includes(tcg)) return tcg;42 if ((/\b(cgc|cbcs)\b/i.test(title) && /#\s?\d+/.test(title) || /\bcomic|\b#\d+\s*\(?\d{4}\)?/i.test(title)) && !SPORTS_BRAND.test(title)) return /\bmarvel|spider-man|x-men|avengers|hulk|iron man|captain america|wolverine|daredevil|fantastic four\b/i.test(title) ? 'marvel_comics' : /\bdc\b|batman|superman|detective comics|action comics|wonder woman|flash|green lantern|joker/i.test(title) ? 'dc_comics' : 'independent_comics';43 for (const [re, slug] of SPORT_WORDS) if (re.test(title)) return slug;44 if (SPORTS_BRAND.test(title)) return 'sports_cards';45 if (hinted) return hinted;46 return null;47}4849function extractPayload(id: string, htmlText: string): MySlabsPayload | null {50 const products = H.jsonLd(htmlText, 'Product');51 const p = products[0];52 if (!p) return null;53 const offers = (Array.isArray(p.offers) ? p.offers[0] : p.offers) as Record<string, unknown> | undefined;54 const images = (Array.isArray(p.image) ? p.image : p.image ? [p.image] : []).map((u) => decodeXml(String(u)));55 const priceRaw = offers?.price;56 const price = priceRaw === undefined || priceRaw === null || priceRaw === '' ? null : Number(priceRaw);57 const availability = typeof offers?.availability === 'string' ? offers.availability.replace(/^https?:\/\/schema\.org\//, '') : null;58 const seller = (offers?.seller as { name?: string } | undefined)?.name ?? null;59 const $ = H.load(htmlText);60 const crumbs = $('nav[aria-label="breadcrumb"] a, .breadcrumb a, ol.breadcrumb li')61 .map((_, el) => $(el).text().trim())62 .get()63 .filter(Boolean);64 const category = crumbs.find((c) => /card|comic|pok|magic|yu-gi|sport/i.test(c)) ?? null;65 return { id, name: String(p.name ?? '').trim(), description: typeof p.description === 'string' ? p.description.trim() : null, images, price: Number.isFinite(price as number) ? (price as number) : null, currency: typeof offers?.priceCurrency === 'string' ? offers.priceCurrency : null, availability, category, seller };66}6768export class MySlabsConnector extends BaseConnector {69 readonly version = '1.0.0';70 readonly parserVersion = PARSER_VERSION;71 override readonly urlPatterns = [/^https?:\/\/(www\.)?myslabs\.com\/slab\/view\/\d+/i];72 protected override minIntervalMs = 1500;7374 private async fetchSlab(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {75 await this.throttle();76 const res = await ctx.fetch(url, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'text/html' }, responseType: 'text' });77 if (!res.success || !res.html) {78 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);79 return null;80 }81 const id = url.match(/slab\/view\/(\d+)/)?.[1] ?? url;82 const payload = extractPayload(id, res.html);83 if (!payload || !payload.name) {84 ctx.anomaly('parse_failure_slab', url);85 return null;86 }87 return { url, externalId: id, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };88 }8990 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {91 const max = ctx.options.mode === 'backfill' ? Number(this.meta.config.backfillSlabs ?? 2000) : Number(this.meta.config.maxSlabsPerRun ?? 250);92 const files = ctx.options.mode === 'backfill' ? ['sitemap-slabs-1.xml', 'sitemap-slabs-2.xml'] : ['sitemap-slabs-1.xml'];93 const urls: Array<{ id: number; loc: string }> = [];94 for (const f of files) {95 const res = await ctx.fetch(`${SITE}/${f}`, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'application/xml,text/xml' }, responseType: 'text', timeoutMs: 120_000 });96 if (!res.success || !res.html) {97 ctx.anomaly('page_fetch_failed', `${f}: ${res.error ?? res.httpStatus}`);98 continue;99 }100 for (const e of parseSitemap(res.html)) {101 const id = Number(e.loc.match(/slab\/view\/(\d+)/)?.[1]);102 if (Number.isFinite(id)) urls.push({ id, loc: e.loc });103 }104 }105 if (ctx.options.seeds?.length) urls.push(...ctx.options.seeds.map((s) => ({ id: Number(s.match(/\d+/)?.[0] ?? 0), loc: s.startsWith('http') ? s : `${SITE}/slab/view/${s}/` })));106 urls.sort((a, b) => b.id - a.id);107 let count = 0;108 for (const u of urls.slice(0, max)) {109 if (ctx.signal?.aborted) return;110 if (this.reached(ctx, count)) return;111 if (!(await ctx.shouldFetch(u.loc))) continue;112 const rec = await this.fetchSlab(ctx, u.loc);113 if (!rec) continue;114 count++;115 yield rec;116 if (count % 25 === 0) await ctx.setCursor({ lastId: u.id, at: new Date().toISOString() });117 }118 await ctx.setCursor({ completedAt: new Date().toISOString(), newestId: urls[0]?.id ?? null });119 }120121 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {122 const rec = await this.fetchSlab(ctx, url);123 return rec ? [rec] : [];124 }125126 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {127 const p = RawPayloadSchema.parse(raw.payload);128 const desc = (p.description ?? '').replace(/<[^>]+>/g, ' ').replace(/&[a-z]+;/g, ' ').slice(0, 400);129 const categorySlug = slabCategory(`${p.name} ${desc}`, p.category);130 if (!categorySlug) return [];131 // Slab pages carry the grade only when the seller typed it (title or description); never guessed.132 const g = parseGradeFromTitle(p.name).grader ? parseGradeFromTitle(p.name) : parseGradeFromTitle(desc);133 const cert = p.description?.match(/\b(?:cert|certification|serial)\s*(?:#|no\.?|number)?\s*[:\-]?\s*(\d{6,12})\b/i)?.[1] ?? null;134 const year = extractYear(p.name);135 const number = p.name.match(/#\s?([A-Za-z0-9-]+)/)?.[1] ?? null;136 const a = attrs({ categorySlug, name: p.name.replace(/\s*\b(PSA|BGS|CGC|SGC|TAG|CSG|CBCS)\b.*$/i, '').trim() || p.name, year, number, identifiers: { myslabs_id: p.id }, metadata: { seller: p.seller, category_label: p.category } });137 const availability = p.availability === 'OutOfStock' || p.availability === 'SoldOut' ? 'sold' : p.availability === 'InStock' || p.availability === 'PreOrder' ? 'available' : 'unknown';138 const currency = (p.currency ?? 'USD').toUpperCase();139 const listing = NormalizedListingSchema.parse({140 kind: 'listing',141 connectorId: this.meta.id,142 sourceId: this.meta.sourceId,143 sourceUrl: raw.url,144 externalId: p.id,145 rawTitle: p.name,146 description: p.description,147 imageUrls: p.images,148 attributes: a,149 grade: { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: cert },150 condition: {},151 observedAt: raw.fetchedAt,152 confidence: 0.75,153 parserVersion: PARSER_VERSION,154 listingType: 'fixed_price',155 price: p.price,156 currency: p.price === null ? null : currency,157 seller: p.seller,158 availability,159 });160 return [listing];161 }162}163164export default (meta: ConnectorMeta) => new MySlabsConnector(meta);165