TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, adapters, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { designSlug, makerFromTitle, plainText, readSeedCursor, yearOrDecade, type DesignVertical } from '../_g10-lib/index.js';56/**7 * Chairish — curated vintage/antique furniture, decor, art and jewelry marketplace (US, USD).8 * Public browse pages (/collection/<slug>, /style/<slug>, ?page=N, 48 items) embed one schema.org Product9 * per listing in JSON-LD (name, description, images, brand, color, material, dimensions, Offer with price,10 * currency, availability, condition, category path and the seller's city/country). We read only that11 * structured block. Asking prices → listings (never sales).12 */13const BASE = 'https://www.chairish.com';14const PARSER_VERSION = '1.0.0';15const PAGE_SIZE = 48;1617export const SeedSchema = z.object({ path: z.string(), slug: z.string().nullable().optional(), vertical: z.enum(['furniture', 'lighting', 'decor', 'art', 'jewelry', 'watches', 'fashion', 'tableware', 'rugs', 'pens', 'unknown']).optional() });18export type Seed = z.infer<typeof SeedSchema>;1920export const ItemSchema = z.object({21 id: z.string(),22 url: z.string(),23 name: z.string(),24 description: z.string().nullable(),25 images: z.array(z.string()),26 brand: z.string().nullable(),27 color: z.string().nullable(),28 material: z.string().nullable(),29 category: z.string().nullable(),30 price: z.number().nullable(),31 currency: z.string().nullable(),32 availability: z.enum(['available', 'sold', 'ended', 'unknown']),33 condition: z.string().nullable(),34 sellerCity: z.string().nullable(),35 sellerCountry: z.string().nullable(),36 dimensions: z.string().nullable(),37});38export type Item = z.infer<typeof ItemSchema>;3940export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), items: z.array(ItemSchema), snapshot: z.string().optional() });41export type PagePayload = z.infer<typeof PagePayloadSchema>;4243const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pagesPerSeed: z.number().int().min(1).default(2), seedsPerRun: z.number().int().min(1).default(4) });4445const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null);46const dim = (v: unknown): string | null => {47 if (!v || typeof v !== 'object') return null;48 const r = v as { value?: unknown; unitText?: unknown; unitCode?: unknown };49 const val = r.value !== undefined ? String(r.value) : null;50 return val ? `${val}${str(r.unitText) ?? (r.unitCode === 'INH' ? ' in' : '')}` : null;51};5253/** JSON-LD Product[] → trimmed items. Exported for tests. */54export function parseBrowseHtml(htmlText: string): Item[] {55 const products = adapters.productsFromHtml(htmlText);56 const out: Item[] = [];57 for (const p of products) {58 const url = p.url ?? '';59 const id = url.match(/\/product\/(\d+)\//)?.[1];60 if (!id || !p.name) continue;61 const offer = p.offers[0];62 const rawOffer = (Array.isArray(p.raw.offers) ? p.raw.offers[0] : p.raw.offers) as Record<string, unknown> | undefined;63 const seller = rawOffer?.seller as { address?: { addressLocality?: unknown; addressCountry?: { name?: unknown } | string } } | undefined;64 const country = seller?.address?.addressCountry;65 const dims = [dim(p.raw.width), dim(p.raw.depth), dim(p.raw.height)].filter(Boolean);66 out.push({67 id,68 url: url.startsWith('http') ? url : `${BASE}${url}`,69 name: p.name,70 description: plainText(p.description, 1500),71 images: p.images.slice(0, 6),72 brand: p.brand,73 color: str(p.raw.color),74 material: str(p.raw.material),75 category: str(rawOffer?.category),76 price: offer?.price ?? null,77 currency: offer?.currency ?? null,78 availability: offer?.availability ?? 'unknown',79 condition: offer?.condition ?? null,80 sellerCity: str(seller?.address?.addressLocality),81 sellerCountry: typeof country === 'string' ? country : str(country?.name),82 dimensions: dims.length === 3 ? `W ${dims[0]} × D ${dims[1]} × H ${dims[2]}` : null,83 });84 }85 return out;86}8788/** Keep only the JSON-LD scripts (first `n` products) for a compact fixture snapshot. */89export function trimBrowseHtml(htmlText: string, n = 3): string {90 const $ = H.load(htmlText);91 const scripts = $('script[type="application/ld+json"]').toArray();92 const kept: string[] = [];93 for (const s of scripts) {94 const txt = $(s).contents().text();95 try {96 const j = JSON.parse(txt) as unknown;97 if (Array.isArray(j)) kept.push(JSON.stringify(j.slice(0, n)));98 else kept.push(txt);99 } catch {100 /* skip */101 }102 }103 const next = $('link[rel="next"], a[rel="next"]').first().attr('href');104 return `<!doctype html><html><head><title>${$('title').text()}</title>${next ? `<link rel="next" href="${next}">` : ''}${kept.map((k) => `<script type="application/ld+json">${k}</script>`).join('')}</head><body></body></html>`;105}106107/** Chairish marks the following page with <link rel="next"> (and a[rel=next]); absent on the last page. */108export function hasNextPage(htmlText: string): boolean {109 const $ = H.load(htmlText);110 return $('link[rel="next"], a[rel="next"]').length > 0;111}112113export function pageUrl(seedPath: string, page: number): string {114 return `${BASE}${seedPath}${page > 1 ? `?page=${page}` : ''}`;115}116117function verticalFor(seed: Seed): DesignVertical {118 if (seed.vertical) return seed.vertical;119 const p = seed.path;120 if (/lighting|lamps/.test(p)) return 'lighting';121 if (/\/art\b|paintings|prints|photograph/.test(p)) return 'art';122 if (/jewelry/.test(p)) return 'jewelry';123 if (/watches/.test(p)) return 'watches';124 if (/handbag|bags|wallets|fashion/.test(p)) return 'fashion';125 if (/tableware|barware|serveware/.test(p)) return 'tableware';126 if (/decor|mirrors|accents|vessels|statues/.test(p)) return 'decor';127 if (/rugs|textiles|pillows|wallpaper/.test(p)) return 'rugs';128 if (/furniture|seating|tables|casegoods|desks|beds|sofas|style\//.test(p)) return 'furniture';129 return 'unknown';130}131132export class ChairishConnector extends BaseConnector {133 readonly version = '1.0.0';134 readonly parserVersion = PARSER_VERSION;135 protected override minIntervalMs = 3000;136 override readonly urlPatterns = [/^https?:\/\/(www\.)?chairish\.com\/product\/(\d+)\//i];137 private readonly cfg: z.infer<typeof ConfigSchema>;138139 constructor(meta: ConnectorMeta) {140 super(meta);141 this.cfg = ConfigSchema.parse(meta.config);142 }143144 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {145 const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds;146 const backfill = ctx.options.mode === 'backfill';147 const maxPages = backfill ? this.policy.backfillMaxPages : this.cfg.pagesPerSeed;148 const start = readSeedCursor(ctx.options.cursor, seeds.length);149 const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun);150 let count = 0;151 let items = 0;152 for (let k = 0; k < seedsThisRun; k++) {153 const seedIndex = (start.seedIndex + k) % seeds.length;154 const seed = seeds[seedIndex]!;155 let page = k === 0 ? start.page : 1;156 for (; page <= maxPages; page++) {157 if (ctx.signal?.aborted || this.reached(ctx, count)) return;158 const url = pageUrl(seed.path, page);159 await this.throttle(url);160 const res = await ctx.fetch(url, {161 engines: ['api'],162 responseType: 'text',163 expect: ['title', 'price', 'currency'],164 parse: (r) => {165 const list = r.html ? parseBrowseHtml(r.html) : [];166 const priced = list.find((i) => i.price);167 return list.length ? { title: list[0]!.name, price: priced?.price ?? null, currency: priced?.currency ?? null } : null;168 },169 minQuality: 0.3,170 });171 const list = res.success && res.html ? parseBrowseHtml(res.html) : null;172 if (!list) {173 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);174 break;175 }176 if (!list.length) {177 if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no JSON-LD products`);178 break;179 }180 count++;181 items += list.length;182 const payload: PagePayload = { kind: 'listing_page', url, seed, page, items: list };183 yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };184 await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() });185 await ctx.progress({ page, itemsProcessed: items });186 if (list.length < PAGE_SIZE || !hasNextPage(res.html!)) break;187 }188 const nextSeed = (seedIndex + 1) % seeds.length;189 await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) });190 }191 }192193 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {194 if (!this.urlPatterns[0]!.test(url)) return [];195 await this.throttle(url);196 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 });197 const list = res.success && res.html ? parseBrowseHtml(res.html) : [];198 if (!list.length) return [];199 const seed: Seed = { path: new URL(url).pathname, slug: null, vertical: 'unknown' };200 const payload: PagePayload = { kind: 'listing_page', url, seed, page: 1, items: list.slice(0, 1) };201 return [{ url, externalId: `product:${list[0]!.id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];202 }203204 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {205 const p = PagePayloadSchema.parse(raw.payload);206 const vertical = verticalFor(p.seed);207 const out: NormalizedRecord[] = [];208 for (const it of p.items) {209 const categorySlug = p.seed.slug ?? designSlug(it.category, it.name, vertical);210 if (!categorySlug) continue;211 const { year, decade } = yearOrDecade(it.name);212 const brand = it.brand && !/^(unknown|unbranded|n\/a|none)$/i.test(it.brand) ? it.brand : makerFromTitle(it.name);213 const attributes = AssetAttributesSchema.parse({214 categorySlug,215 brand,216 name: it.name,217 year,218 material: it.material,219 color: it.color,220 size: it.dimensions,221 identifiers: { chairish_product_id: it.id },222 metadata: { source_category: it.category, decade, seller_location: [it.sellerCity, it.sellerCountry].filter(Boolean).join(', ') || null },223 });224 out.push(225 NormalizedListingSchema.parse({226 kind: 'listing',227 connectorId: this.meta.id,228 sourceId: this.meta.sourceId,229 sourceUrl: it.url,230 externalId: it.id,231 rawTitle: it.name,232 description: it.description,233 imageUrls: it.images,234 attributes,235 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },236 condition: { condition: null, conditionRaw: it.condition, completeness: null },237 observedAt: raw.fetchedAt,238 confidence: 0.8,239 parserVersion: PARSER_VERSION,240 listingType: 'fixed_price',241 price: it.price,242 currency: it.currency && /^[A-Z]{3}$/.test(it.currency) ? it.currency : it.price ? 'USD' : null,243 seller: null,244 location: [it.sellerCity, it.sellerCountry].filter(Boolean).join(', ') || null,245 availability: it.availability,246 }),247 );248 }249 return out;250 }251}252253export default function createConnector(meta: ConnectorMeta): ChairishConnector {254 return new ChairishConnector(meta);255}256