TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { designSlug, makerFromTitle, readSeedCursor, yearOrDecade, type DesignVertical } from '../_g10-lib/index.js';56/**7 * Pamono — Berlin-based vintage design marketplace; the .com storefront prices in USD. Category pages8 * (/furniture, /lighting, /home-accessories, /jewelry-watches, ?p=N, ~100 cards) are server-rendered9 * Magento HTML: each <article class="product-card" data-sku> carries the product link, title, the price10 * (itemprop=price content=…, optional old-price) and the CDN image. Asking prices → listings.11 */12const BASE = 'https://www.pamono.com';13const PARSER_VERSION = '1.0.0';1415export const SeedSchema = z.object({ path: z.string(), vertical: z.enum(['furniture', 'lighting', 'decor', 'art', 'jewelry', 'watches', 'fashion', 'tableware', 'rugs', 'pens', 'unknown']).default('unknown') });16export type Seed = z.infer<typeof SeedSchema>;1718export const CardSchema = z.object({ sku: z.string(), productId: z.string().nullable(), url: z.string(), title: z.string(), price: z.number().nullable(), oldPrice: z.number().nullable(), image: z.string().nullable(), currency: z.string().nullable(), boosted: z.boolean().default(false) });19export type Card = z.infer<typeof CardSchema>;2021export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), currency: z.string().nullable(), cards: z.array(CardSchema), snapshot: z.string().optional() });22export type PagePayload = z.infer<typeof PagePayloadSchema>;2324const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pagesPerSeed: z.number().int().min(1).default(2), seedsPerRun: z.number().int().min(1).default(2) });2526/** Store currency from the inline `add.state = {"currencyCode":"USD",…}` bootstrap. */27export function parseStoreCurrency(htmlText: string): string | null {28 return htmlText.match(/"currencyCode"\s*:\s*"([A-Z]{3})"/)?.[1] ?? null;29}3031export function parseCategoryHtml(htmlText: string): Card[] {32 const $ = H.load(htmlText);33 const out: Card[] = [];34 const seen = new Set<string>();35 $('article.product-card').each((_, el) => {36 const e = $(el);37 const sku = e.attr('data-sku') ?? '';38 const a = e.find('a.link-wrapper').first();39 const url = a.attr('href');40 const title = H.text(e.find('p.title').first()) ?? a.attr('title')?.trim() ?? null;41 if (!sku || !url || !title || seen.has(sku)) return;42 seen.add(sku);43 const priceEl = e.find('[itemprop="price"]').first();44 const priceContent = priceEl.attr('content');45 const priceText = H.text(priceEl);46 const price = priceContent ? Number.parseFloat(priceContent) : Number.parseFloat((priceText ?? '').replace(/[^0-9.]/g, ''));47 const oldEl = e.find('[itemprop="old-price"]').first();48 const old = oldEl.attr('content') ? Number.parseFloat(oldEl.attr('content')!) : NaN;49 const symbol = priceText?.match(/[$€£]/)?.[0];50 const img = e.find('img.image').first();51 const image = img.attr('data-lazy') ?? e.find('noscript img').first().attr('src') ?? (img.attr('src')?.startsWith('http') ? img.attr('src')! : null);52 out.push({53 sku,54 productId: sku.match(/-(\d+)$/)?.[1] ?? e.find('button.heart-icon').attr('data-product-id') ?? null,55 url: url.startsWith('http') ? url : `${BASE}${url}`,56 title,57 price: Number.isFinite(price) && price > 0 ? price : null,58 oldPrice: Number.isFinite(old) && old > 0 ? old : null,59 image,60 currency: symbol === '$' ? 'USD' : symbol === '€' ? 'EUR' : symbol === '£' ? 'GBP' : null,61 boosted: e.find('.boosted-item-label').length > 0,62 });63 });64 return out;65}6667/** Keep the store bootstrap line and the first `n` product cards (fixture snapshot). */68export function trimCategoryHtml(htmlText: string, n = 3): string {69 const $ = H.load(htmlText);70 const cards = $('article.product-card').toArray().slice(0, n).map((el) => $.html(el));71 const cur = parseStoreCurrency(htmlText);72 return `<!doctype html><html><head><script>add.state = {"currencyCode":"${cur ?? 'USD'}","pageType":"category"};</script></head><body><div class="products">${cards.join('\n')}</div></body></html>`;73}7475export function pageUrl(seedPath: string, page: number): string {76 return `${BASE}${seedPath}${page > 1 ? `?p=${page}` : ''}`;77}7879export class PamonoConnector extends BaseConnector {80 readonly version = '1.0.0';81 readonly parserVersion = PARSER_VERSION;82 protected override minIntervalMs = 4000;83 private readonly cfg: z.infer<typeof ConfigSchema>;8485 constructor(meta: ConnectorMeta) {86 super(meta);87 this.cfg = ConfigSchema.parse(meta.config);88 }8990 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {91 const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds;92 const backfill = ctx.options.mode === 'backfill';93 const maxPages = backfill ? this.policy.backfillMaxPages : this.cfg.pagesPerSeed;94 const start = readSeedCursor(ctx.options.cursor, seeds.length);95 const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun);96 let count = 0;97 let items = 0;98 for (let k = 0; k < seedsThisRun; k++) {99 const seedIndex = (start.seedIndex + k) % seeds.length;100 const seed = seeds[seedIndex]!;101 let page = k === 0 ? start.page : 1;102 let firstSkuOfPrev: string | null = null;103 for (; page <= maxPages; page++) {104 if (ctx.signal?.aborted || this.reached(ctx, count)) return;105 const url = pageUrl(seed.path, page);106 await this.throttle(url);107 const res = await ctx.fetch(url, {108 engines: ['api'],109 responseType: 'text',110 expect: ['title', 'price', 'currency'],111 parse: (r) => {112 const cards = r.html ? parseCategoryHtml(r.html) : [];113 const priced = cards.find((c) => c.price);114 return cards.length ? { title: cards[0]!.title, price: priced?.price ?? null, currency: priced?.currency ?? parseStoreCurrency(r.html ?? '') } : null;115 },116 minQuality: 0.3,117 });118 const cards = res.success && res.html ? parseCategoryHtml(res.html) : null;119 if (!cards) {120 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);121 break;122 }123 // Magento serves the last page again for out-of-range ?p= values → detect the repeat and stop.124 if (!cards.length || (firstSkuOfPrev && cards[0]!.sku === firstSkuOfPrev)) {125 if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no product cards`);126 break;127 }128 firstSkuOfPrev = cards[0]!.sku;129 count++;130 items += cards.length;131 const payload: PagePayload = { kind: 'listing_page', url, seed, page, currency: parseStoreCurrency(res.html!), cards };132 yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };133 await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() });134 await ctx.progress({ page, itemsProcessed: items });135 if (cards.length < 60) break;136 }137 const nextSeed = (seedIndex + 1) % seeds.length;138 await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) });139 }140 }141142 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {143 const p = PagePayloadSchema.parse(raw.payload);144 const out: NormalizedRecord[] = [];145 const vertical: DesignVertical = p.seed.vertical;146 for (const c of p.cards) {147 if (!c.price) continue;148 const categorySlug = designSlug(null, c.title, vertical === 'unknown' ? 'furniture' : vertical);149 if (!categorySlug) continue;150 const currency = c.currency ?? p.currency ?? 'USD';151 const { year, decade } = yearOrDecade(c.title);152 const attributes = AssetAttributesSchema.parse({153 categorySlug,154 brand: makerFromTitle(c.title),155 name: c.title,156 year,157 identifiers: { pamono_sku: c.sku, ...(c.productId ? { pamono_product_id: c.productId } : {}) },158 metadata: { decade, regular_price: c.oldPrice, on_sale: c.oldPrice !== null && c.oldPrice > c.price, boosted: c.boosted, seed: p.seed.path },159 });160 out.push(161 NormalizedListingSchema.parse({162 kind: 'listing',163 connectorId: this.meta.id,164 sourceId: this.meta.sourceId,165 sourceUrl: c.url,166 externalId: c.sku,167 rawTitle: c.title,168 imageUrls: c.image ? [c.image] : [],169 attributes,170 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },171 condition: { condition: null, conditionRaw: null, completeness: null },172 observedAt: raw.fetchedAt,173 confidence: 0.75,174 parserVersion: PARSER_VERSION,175 listingType: 'fixed_price',176 price: c.price,177 currency: /^[A-Z]{3}$/.test(currency) ? currency : 'USD',178 availability: 'available',179 }),180 );181 }182 return out;183 }184}185186export default function createConnector(meta: ConnectorMeta): PamonoConnector {187 return new PamonoConnector(meta);188}189