import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, extractYear, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; const BASE = 'https://www.ma-shops.com'; const PARSER_VERSION = '1.0.0'; export const CellSchema = z.object({ shop: z.string(), id: z.string(), url: z.string(), title: z.string(), price: z.number().nullable(), currency: z.string().nullable(), seller: z.string().nullable(), image: z.string().nullable(), }); export type Cell = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('gallery_page'), url: z.string(), categorySlug: z.string(), items: z.array(CellSchema) }); export type PagePayload = z.infer; /** "642.98 CAN$" | "1,250.00 US$" | "89.00 EUR" → amount + ISO currency. */ export function parseMaPrice(text: string | null | undefined): { price: number | null; currency: CurrencyCode | null } { if (!text) return { price: null, currency: null }; const m = text.replace(/\s+/g, ' ').match(/([\d.,]+)\s*(CAN\$|US\$|EUR|€|GBP|£|CHF)/i); if (!m) return { price: null, currency: null }; const num = Number(m[1]!.replace(/,/g, '')); const cur = /CAN/i.test(m[2]!) ? 'CAD' : /US/i.test(m[2]!) || m[2] === '$' ? 'USD' : /EUR|€/i.test(m[2]!) ? 'EUR' : /GBP|£/i.test(m[2]!) ? 'GBP' : 'CHF'; return { price: Number.isFinite(num) && num > 0 ? num : null, currency: cur }; } export function parseGalleryPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload { const $ = H.load(htmlText); const items: Cell[] = []; const seen = new Set(); $('.galleryCell').each((_, el) => { const e = $(el); const a = e.find('.galleryItemTitle a, a[href*="item.php?id="]').first(); const href = a.attr('href'); const m = href?.match(/^\/?([a-z0-9_-]+)\/item\.php\?id=(\d+)/i); if (!href || !m) return; const key = `${m[1]}/${m[2]}`; if (seen.has(key)) return; seen.add(key); // gallery titles are truncated with an ellipsis; the thumbnail alt/title carries the full title const title = ((e.find('img.thumb').attr('title') || e.find('img.thumb').attr('alt') || H.text(e.find('.galleryItemTitle').first())) ?? '').replace(/\s+/g, ' ').trim(); if (!title) return; const { price, currency } = parseMaPrice(H.text(e.find('.itemPrice').first())); const seller = H.text(e.find('.gallerySellerName').first()); const image = e.find('img.thumb').attr('src') ?? null; items.push({ shop: m[1]!, id: m[2]!, url: `${BASE}/${m[1]}/item.php?id=${m[2]}`, title, price, currency, seller, image: image ? (image.startsWith('http') ? image : BASE + image) : null }); }); return { kind: 'gallery_page', url: pageUrl, categorySlug, items }; } export class MaShopsConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.seeds as Array<{ path: string; categorySlug: string }> | undefined) ?? []; const pages = Number(this.meta.config.pagesPerSeed ?? 1); const cap = ctx.options.limit; let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; for (let i = start; i < seeds.length; i++) { const seed = seeds[i]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return; const sep = seed.path.includes('?') ? '&' : '?'; const url = `${BASE}${seed.path}${page > 1 ? `${sep}page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', expect: ['title', 'price'], parse: (r) => { const p = r.html ? parseGalleryPage(r.html, url, seed.categorySlug) : null; return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null; }, }); const payload = res.success && res.html ? parseGalleryPage(res.html, url, seed.categorySlug) : null; if (!payload?.items.length) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no gallery cells'}`); break; } count++; yield { url, externalId: `${seed.path}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const c of p.items) { const g = parseGradeFromTitle(c.title); const attributes = AssetAttributesSchema.parse({ categorySlug: p.categorySlug, name: c.title, year: extractYear(c.title), identifiers: { ma_shops_item: `${c.shop}/${c.id}` }, metadata: { dealer: c.seller, shop_slug: c.shop, unique_item: true }, }); const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `${c.shop}/${c.id}`, confidence: 0.8, listingType: 'fixed_price', price: c.price, currency: c.currency, seller: c.seller ?? c.shop, location: 'EU', availability: c.price ? 'available' : 'unknown' })); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new MaShopsConnector(meta); }