import { z } from 'zod'; import { BaseConnector, adapters, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { designSlug, makerFromTitle, plainText, readSeedCursor, yearOrDecade, type DesignVertical } from '../_g10-lib/index.js'; /** * Chairish — curated vintage/antique furniture, decor, art and jewelry marketplace (US, USD). * Public browse pages (/collection/, /style/, ?page=N, 48 items) embed one schema.org Product * per listing in JSON-LD (name, description, images, brand, color, material, dimensions, Offer with price, * currency, availability, condition, category path and the seller's city/country). We read only that * structured block. Asking prices → listings (never sales). */ const BASE = 'https://www.chairish.com'; const PARSER_VERSION = '1.0.0'; const PAGE_SIZE = 48; export const SeedSchema = z.object({ path: z.string(), slug: z.string().nullable().optional(), vertical: z.enum(['furniture', 'lighting', 'decor', 'art', 'jewelry', 'watches', 'fashion', 'tableware', 'rugs', 'pens', 'unknown']).optional() }); export type Seed = z.infer; export const ItemSchema = z.object({ id: z.string(), url: z.string(), name: z.string(), description: z.string().nullable(), images: z.array(z.string()), brand: z.string().nullable(), color: z.string().nullable(), material: z.string().nullable(), category: z.string().nullable(), price: z.number().nullable(), currency: z.string().nullable(), availability: z.enum(['available', 'sold', 'ended', 'unknown']), condition: z.string().nullable(), sellerCity: z.string().nullable(), sellerCountry: z.string().nullable(), dimensions: z.string().nullable(), }); export type Item = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), items: z.array(ItemSchema), snapshot: z.string().optional() }); export type PagePayload = z.infer; const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pagesPerSeed: z.number().int().min(1).default(2), seedsPerRun: z.number().int().min(1).default(4) }); const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null); const dim = (v: unknown): string | null => { if (!v || typeof v !== 'object') return null; const r = v as { value?: unknown; unitText?: unknown; unitCode?: unknown }; const val = r.value !== undefined ? String(r.value) : null; return val ? `${val}${str(r.unitText) ?? (r.unitCode === 'INH' ? ' in' : '')}` : null; }; /** JSON-LD Product[] → trimmed items. Exported for tests. */ export function parseBrowseHtml(htmlText: string): Item[] { const products = adapters.productsFromHtml(htmlText); const out: Item[] = []; for (const p of products) { const url = p.url ?? ''; const id = url.match(/\/product\/(\d+)\//)?.[1]; if (!id || !p.name) continue; const offer = p.offers[0]; const rawOffer = (Array.isArray(p.raw.offers) ? p.raw.offers[0] : p.raw.offers) as Record | undefined; const seller = rawOffer?.seller as { address?: { addressLocality?: unknown; addressCountry?: { name?: unknown } | string } } | undefined; const country = seller?.address?.addressCountry; const dims = [dim(p.raw.width), dim(p.raw.depth), dim(p.raw.height)].filter(Boolean); out.push({ id, url: url.startsWith('http') ? url : `${BASE}${url}`, name: p.name, description: plainText(p.description, 1500), images: p.images.slice(0, 6), brand: p.brand, color: str(p.raw.color), material: str(p.raw.material), category: str(rawOffer?.category), price: offer?.price ?? null, currency: offer?.currency ?? null, availability: offer?.availability ?? 'unknown', condition: offer?.condition ?? null, sellerCity: str(seller?.address?.addressLocality), sellerCountry: typeof country === 'string' ? country : str(country?.name), dimensions: dims.length === 3 ? `W ${dims[0]} × D ${dims[1]} × H ${dims[2]}` : null, }); } return out; } /** Keep only the JSON-LD scripts (first `n` products) for a compact fixture snapshot. */ export function trimBrowseHtml(htmlText: string, n = 3): string { const $ = H.load(htmlText); const scripts = $('script[type="application/ld+json"]').toArray(); const kept: string[] = []; for (const s of scripts) { const txt = $(s).contents().text(); try { const j = JSON.parse(txt) as unknown; if (Array.isArray(j)) kept.push(JSON.stringify(j.slice(0, n))); else kept.push(txt); } catch { /* skip */ } } const next = $('link[rel="next"], a[rel="next"]').first().attr('href'); return `${$('title').text()}${next ? `` : ''}${kept.map((k) => ``).join('')}`; } /** Chairish marks the following page with (and a[rel=next]); absent on the last page. */ export function hasNextPage(htmlText: string): boolean { const $ = H.load(htmlText); return $('link[rel="next"], a[rel="next"]').length > 0; } export function pageUrl(seedPath: string, page: number): string { return `${BASE}${seedPath}${page > 1 ? `?page=${page}` : ''}`; } function verticalFor(seed: Seed): DesignVertical { if (seed.vertical) return seed.vertical; const p = seed.path; if (/lighting|lamps/.test(p)) return 'lighting'; if (/\/art\b|paintings|prints|photograph/.test(p)) return 'art'; if (/jewelry/.test(p)) return 'jewelry'; if (/watches/.test(p)) return 'watches'; if (/handbag|bags|wallets|fashion/.test(p)) return 'fashion'; if (/tableware|barware|serveware/.test(p)) return 'tableware'; if (/decor|mirrors|accents|vessels|statues/.test(p)) return 'decor'; if (/rugs|textiles|pillows|wallpaper/.test(p)) return 'rugs'; if (/furniture|seating|tables|casegoods|desks|beds|sofas|style\//.test(p)) return 'furniture'; return 'unknown'; } export class ChairishConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; override readonly urlPatterns = [/^https?:\/\/(www\.)?chairish\.com\/product\/(\d+)\//i]; private readonly cfg: z.infer; constructor(meta: ConnectorMeta) { super(meta); this.cfg = ConfigSchema.parse(meta.config); } async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds; const backfill = ctx.options.mode === 'backfill'; const maxPages = backfill ? this.policy.backfillMaxPages : this.cfg.pagesPerSeed; const start = readSeedCursor(ctx.options.cursor, seeds.length); const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun); let count = 0; let items = 0; for (let k = 0; k < seedsThisRun; k++) { const seedIndex = (start.seedIndex + k) % seeds.length; const seed = seeds[seedIndex]!; let page = k === 0 ? start.page : 1; for (; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = pageUrl(seed.path, page); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'currency'], parse: (r) => { const list = r.html ? parseBrowseHtml(r.html) : []; const priced = list.find((i) => i.price); return list.length ? { title: list[0]!.name, price: priced?.price ?? null, currency: priced?.currency ?? null } : null; }, minQuality: 0.3, }); const list = res.success && res.html ? parseBrowseHtml(res.html) : null; if (!list) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } if (!list.length) { if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no JSON-LD products`); break; } count++; items += list.length; const payload: PagePayload = { kind: 'listing_page', url, seed, page, items: list }; yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() }); await ctx.progress({ page, itemsProcessed: items }); if (list.length < PAGE_SIZE || !hasNextPage(res.html!)) break; } const nextSeed = (seedIndex + 1) % seeds.length; await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) }); } } async lookup(url: string, ctx: CrawlContext): Promise { if (!this.urlPatterns[0]!.test(url)) return []; await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 }); const list = res.success && res.html ? parseBrowseHtml(res.html) : []; if (!list.length) return []; const seed: Seed = { path: new URL(url).pathname, slug: null, vertical: 'unknown' }; const payload: PagePayload = { kind: 'listing_page', url, seed, page: 1, items: list.slice(0, 1) }; return [{ url, externalId: `product:${list[0]!.id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const vertical = verticalFor(p.seed); const out: NormalizedRecord[] = []; for (const it of p.items) { const categorySlug = p.seed.slug ?? designSlug(it.category, it.name, vertical); if (!categorySlug) continue; const { year, decade } = yearOrDecade(it.name); const brand = it.brand && !/^(unknown|unbranded|n\/a|none)$/i.test(it.brand) ? it.brand : makerFromTitle(it.name); const attributes = AssetAttributesSchema.parse({ categorySlug, brand, name: it.name, year, material: it.material, color: it.color, size: it.dimensions, identifiers: { chairish_product_id: it.id }, metadata: { source_category: it.category, decade, seller_location: [it.sellerCity, it.sellerCountry].filter(Boolean).join(', ') || null }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, externalId: it.id, rawTitle: it.name, description: it.description, imageUrls: it.images, attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: it.condition, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: it.price, currency: it.currency && /^[A-Z]{3}$/.test(it.currency) ? it.currency : it.price ? 'USD' : null, seller: null, location: [it.sellerCity, it.sellerCountry].filter(Boolean).join(', ') || null, availability: it.availability, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): ChairishConnector { return new ChairishConnector(meta); }