TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, extractYear, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';56const BASE = 'https://www.ma-shops.com';7const PARSER_VERSION = '1.0.0';89export const CellSchema = z.object({10 shop: z.string(),11 id: z.string(),12 url: z.string(),13 title: z.string(),14 price: z.number().nullable(),15 currency: z.string().nullable(),16 seller: z.string().nullable(),17 image: z.string().nullable(),18});19export type Cell = z.infer<typeof CellSchema>;20export const PagePayloadSchema = z.object({ kind: z.literal('gallery_page'), url: z.string(), categorySlug: z.string(), items: z.array(CellSchema) });21export type PagePayload = z.infer<typeof PagePayloadSchema>;2223/** "642.98 CAN$" | "1,250.00 US$" | "89.00 EUR" → amount + ISO currency. */24export function parseMaPrice(text: string | null | undefined): { price: number | null; currency: CurrencyCode | null } {25 if (!text) return { price: null, currency: null };26 const m = text.replace(/\s+/g, ' ').match(/([\d.,]+)\s*(CAN\$|US\$|EUR|€|GBP|£|CHF)/i);27 if (!m) return { price: null, currency: null };28 const num = Number(m[1]!.replace(/,/g, ''));29 const cur = /CAN/i.test(m[2]!) ? 'CAD' : /US/i.test(m[2]!) || m[2] === '$' ? 'USD' : /EUR|€/i.test(m[2]!) ? 'EUR' : /GBP|£/i.test(m[2]!) ? 'GBP' : 'CHF';30 return { price: Number.isFinite(num) && num > 0 ? num : null, currency: cur };31}3233export function parseGalleryPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {34 const $ = H.load(htmlText);35 const items: Cell[] = [];36 const seen = new Set<string>();37 $('.galleryCell').each((_, el) => {38 const e = $(el);39 const a = e.find('.galleryItemTitle a, a[href*="item.php?id="]').first();40 const href = a.attr('href');41 const m = href?.match(/^\/?([a-z0-9_-]+)\/item\.php\?id=(\d+)/i);42 if (!href || !m) return;43 const key = `${m[1]}/${m[2]}`;44 if (seen.has(key)) return;45 seen.add(key);46 // gallery titles are truncated with an ellipsis; the thumbnail alt/title carries the full title47 const title = ((e.find('img.thumb').attr('title') || e.find('img.thumb').attr('alt') || H.text(e.find('.galleryItemTitle').first())) ?? '').replace(/\s+/g, ' ').trim();48 if (!title) return;49 const { price, currency } = parseMaPrice(H.text(e.find('.itemPrice').first()));50 const seller = H.text(e.find('.gallerySellerName').first());51 const image = e.find('img.thumb').attr('src') ?? null;52 items.push({ shop: m[1]!, id: m[2]!, url: `${BASE}/${m[1]}/item.php?id=${m[2]}`, title, price, currency, seller, image: image ? (image.startsWith('http') ? image : BASE + image) : null });53 });54 return { kind: 'gallery_page', url: pageUrl, categorySlug, items };55}5657export class MaShopsConnector extends BaseConnector {58 readonly version = '1.0.0';59 readonly parserVersion = PARSER_VERSION;60 protected override minIntervalMs = 2000;6162 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {63 const seeds = (this.meta.config.seeds as Array<{ path: string; categorySlug: string }> | undefined) ?? [];64 const pages = Number(this.meta.config.pagesPerSeed ?? 1);65 const cap = ctx.options.limit;66 let count = 0;67 const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;68 for (let i = start; i < seeds.length; i++) {69 const seed = seeds[i]!;70 for (let page = 1; page <= pages; page++) {71 if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;72 const sep = seed.path.includes('?') ? '&' : '?';73 const url = `${BASE}${seed.path}${page > 1 ? `${sep}page=${page}` : ''}`;74 await this.throttle();75 const res = await ctx.fetch(url, {76 engines: ['api', 'firecrawl'],77 responseType: 'text',78 expect: ['title', 'price'],79 parse: (r) => {80 const p = r.html ? parseGalleryPage(r.html, url, seed.categorySlug) : null;81 return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;82 },83 });84 const payload = res.success && res.html ? parseGalleryPage(res.html, url, seed.categorySlug) : null;85 if (!payload?.items.length) {86 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no gallery cells'}`);87 break;88 }89 count++;90 yield { url, externalId: `${seed.path}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };91 }92 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });93 }94 }9596 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {97 const p = PagePayloadSchema.parse(raw.payload);98 const out: NormalizedRecord[] = [];99 for (const c of p.items) {100 const g = parseGradeFromTitle(c.title);101 const attributes = AssetAttributesSchema.parse({102 categorySlug: p.categorySlug,103 name: c.title,104 year: extractYear(c.title),105 identifiers: { ma_shops_item: `${c.shop}/${c.id}` },106 metadata: { dealer: c.seller, shop_slug: c.shop, unique_item: true },107 });108 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };109 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `${c.shop}/${c.id}`, confidence: 0.8, listingType: 'fixed_price', price: c.price, currency: c.currency, seller: c.seller ?? c.shop, location: 'EU', availability: c.price ? 'available' : 'unknown' }));110 }111 return out;112 }113}114115export default function createConnector(meta: ConnectorMeta) {116 return new MaShopsConnector(meta);117}118