import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { designSlug, makerFromTitle, readSeedCursor, yearOrDecade, type DesignVertical } from '../_g10-lib/index.js'; /** * Pamono — Berlin-based vintage design marketplace; the .com storefront prices in USD. Category pages * (/furniture, /lighting, /home-accessories, /jewelry-watches, ?p=N, ~100 cards) are server-rendered * Magento HTML: each
carries the product link, title, the price * (itemprop=price content=…, optional old-price) and the CDN image. Asking prices → listings. */ const BASE = 'https://www.pamono.com'; const PARSER_VERSION = '1.0.0'; export const SeedSchema = z.object({ path: z.string(), vertical: z.enum(['furniture', 'lighting', 'decor', 'art', 'jewelry', 'watches', 'fashion', 'tableware', 'rugs', 'pens', 'unknown']).default('unknown') }); export type Seed = z.infer; export const CardSchema = z.object({ sku: z.string(), productId: z.string().nullable(), url: z.string(), title: z.string(), price: z.number().nullable(), oldPrice: z.number().nullable(), image: z.string().nullable(), currency: z.string().nullable(), boosted: z.boolean().default(false) }); export type Card = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), currency: z.string().nullable(), cards: z.array(CardSchema), snapshot: z.string().optional() }); export type PagePayload = z.infer; const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pagesPerSeed: z.number().int().min(1).default(2), seedsPerRun: z.number().int().min(1).default(2) }); /** Store currency from the inline `add.state = {"currencyCode":"USD",…}` bootstrap. */ export function parseStoreCurrency(htmlText: string): string | null { return htmlText.match(/"currencyCode"\s*:\s*"([A-Z]{3})"/)?.[1] ?? null; } export function parseCategoryHtml(htmlText: string): Card[] { const $ = H.load(htmlText); const out: Card[] = []; const seen = new Set(); $('article.product-card').each((_, el) => { const e = $(el); const sku = e.attr('data-sku') ?? ''; const a = e.find('a.link-wrapper').first(); const url = a.attr('href'); const title = H.text(e.find('p.title').first()) ?? a.attr('title')?.trim() ?? null; if (!sku || !url || !title || seen.has(sku)) return; seen.add(sku); const priceEl = e.find('[itemprop="price"]').first(); const priceContent = priceEl.attr('content'); const priceText = H.text(priceEl); const price = priceContent ? Number.parseFloat(priceContent) : Number.parseFloat((priceText ?? '').replace(/[^0-9.]/g, '')); const oldEl = e.find('[itemprop="old-price"]').first(); const old = oldEl.attr('content') ? Number.parseFloat(oldEl.attr('content')!) : NaN; const symbol = priceText?.match(/[$€£]/)?.[0]; const img = e.find('img.image').first(); const image = img.attr('data-lazy') ?? e.find('noscript img').first().attr('src') ?? (img.attr('src')?.startsWith('http') ? img.attr('src')! : null); out.push({ sku, productId: sku.match(/-(\d+)$/)?.[1] ?? e.find('button.heart-icon').attr('data-product-id') ?? null, url: url.startsWith('http') ? url : `${BASE}${url}`, title, price: Number.isFinite(price) && price > 0 ? price : null, oldPrice: Number.isFinite(old) && old > 0 ? old : null, image, currency: symbol === '$' ? 'USD' : symbol === '€' ? 'EUR' : symbol === '£' ? 'GBP' : null, boosted: e.find('.boosted-item-label').length > 0, }); }); return out; } /** Keep the store bootstrap line and the first `n` product cards (fixture snapshot). */ export function trimCategoryHtml(htmlText: string, n = 3): string { const $ = H.load(htmlText); const cards = $('article.product-card').toArray().slice(0, n).map((el) => $.html(el)); const cur = parseStoreCurrency(htmlText); return `
${cards.join('\n')}
`; } export function pageUrl(seedPath: string, page: number): string { return `${BASE}${seedPath}${page > 1 ? `?p=${page}` : ''}`; } export class PamonoConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 4000; private readonly cfg: z.infer; constructor(meta: ConnectorMeta) { super(meta); this.cfg = ConfigSchema.parse(meta.config); } async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds; const backfill = ctx.options.mode === 'backfill'; const maxPages = backfill ? this.policy.backfillMaxPages : this.cfg.pagesPerSeed; const start = readSeedCursor(ctx.options.cursor, seeds.length); const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun); let count = 0; let items = 0; for (let k = 0; k < seedsThisRun; k++) { const seedIndex = (start.seedIndex + k) % seeds.length; const seed = seeds[seedIndex]!; let page = k === 0 ? start.page : 1; let firstSkuOfPrev: string | null = null; for (; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = pageUrl(seed.path, page); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'currency'], parse: (r) => { const cards = r.html ? parseCategoryHtml(r.html) : []; const priced = cards.find((c) => c.price); return cards.length ? { title: cards[0]!.title, price: priced?.price ?? null, currency: priced?.currency ?? parseStoreCurrency(r.html ?? '') } : null; }, minQuality: 0.3, }); const cards = res.success && res.html ? parseCategoryHtml(res.html) : null; if (!cards) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } // Magento serves the last page again for out-of-range ?p= values → detect the repeat and stop. if (!cards.length || (firstSkuOfPrev && cards[0]!.sku === firstSkuOfPrev)) { if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no product cards`); break; } firstSkuOfPrev = cards[0]!.sku; count++; items += cards.length; const payload: PagePayload = { kind: 'listing_page', url, seed, page, currency: parseStoreCurrency(res.html!), cards }; yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() }); await ctx.progress({ page, itemsProcessed: items }); if (cards.length < 60) break; } const nextSeed = (seedIndex + 1) % seeds.length; await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const vertical: DesignVertical = p.seed.vertical; for (const c of p.cards) { if (!c.price) continue; const categorySlug = designSlug(null, c.title, vertical === 'unknown' ? 'furniture' : vertical); if (!categorySlug) continue; const currency = c.currency ?? p.currency ?? 'USD'; const { year, decade } = yearOrDecade(c.title); const attributes = AssetAttributesSchema.parse({ categorySlug, brand: makerFromTitle(c.title), name: c.title, year, identifiers: { pamono_sku: c.sku, ...(c.productId ? { pamono_product_id: c.productId } : {}) }, metadata: { decade, regular_price: c.oldPrice, on_sale: c.oldPrice !== null && c.oldPrice > c.price, boosted: c.boosted, seed: p.seed.path }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, externalId: c.sku, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.75, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: c.price, currency: /^[A-Z]{3}$/.test(currency) ? currency : 'USD', availability: 'available', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): PamonoConnector { return new PamonoConnector(meta); }