TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { designSlug, plainText, readSeedCursor, yearOrDecade, type DesignVertical } from '../_g10-lib/index.js';56/**7 * 1stDibs — the largest online marketplace for vintage/antique furniture, design, jewelry, watches,8 * fashion and art (USD by default). Browse pages (/furniture/seating/, /jewelry/, … ?page=N, 60 items)9 * embed the server-rendered Relay store in <script id="serverVars_data" type="application/json">;10 * we read the `Item` records from it (title, serviceId, prices in ten currencies, seller company,11 * creators/designers, location, period/style attributes, materials, measurements, photos, sold/hold12 * flags) and never call 1stDibs' GraphQL endpoint ourselves. Asking prices → listings.13 */14const BASE = 'https://www.1stdibs.com';15const PARSER_VERSION = '1.0.0';16const PAGE_SIZE = 60;1718export const SeedSchema = z.object({ path: z.string(), slug: z.string().nullable().optional(), vertical: z.enum(['furniture', 'lighting', 'decor', 'art', 'jewelry', 'watches', 'fashion', 'tableware', 'rugs', 'pens', 'unknown']).default('unknown') });19export type Seed = z.infer<typeof SeedSchema>;2021export const ItemSchema = z.object({22 serviceId: z.string(),23 title: z.string(),24 url: z.string(),25 prices: z.record(z.string(), z.number()),26 amountType: z.string().nullable(),27 isSold: z.boolean(),28 isOnHold: z.boolean(),29 isUnavailable: z.boolean(),30 isNewListing: z.boolean().nullable(),31 seller: z.object({ serviceId: z.string().nullable(), company: z.string().nullable() }),32 creators: z.array(z.string()),33 country: z.string().nullable(),34 location: z.string().nullable(),35 attributesText: z.string().nullable(),36 materials: z.string().nullable(),37 description: z.string().nullable(),38 measurement: z.string().nullable(),39 categoryCode: z.string().nullable(),40 browseUrl: z.string().nullable(),41 vertical: z.string().nullable(),42 images: z.array(z.string()),43});44export type Item = z.infer<typeof ItemSchema>;4546export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), totalResults: z.number().int().nullable(), maxPages: z.number().int().nullable(), items: z.array(ItemSchema), snapshot: z.string().optional() });47export type PagePayload = z.infer<typeof PagePayloadSchema>;4849const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pagesPerSeed: z.number().int().min(1).default(1), seedsPerRun: z.number().int().min(1).default(3), maxPagesPerSeed: z.number().int().min(1).default(50) });5051type Rec = Record<string, unknown>;5253/** Relay normalized store → dereferenced view (bounded depth). */54export function makeDeref(store: Record<string, Rec>) {55 const deref = (v: unknown, depth = 0): unknown => {56 if (depth > 4 || v === null || typeof v !== 'object') return v;57 if (Array.isArray(v)) return v.map((x) => deref(x, depth + 1));58 const o = v as Rec;59 if (typeof o.__ref === 'string') return deref(store[o.__ref] ?? null, depth + 1);60 if (Array.isArray(o.__refs)) return o.__refs.map((r) => deref(store[String(r)] ?? null, depth + 1));61 const out: Rec = {};62 for (const [k, x] of Object.entries(o)) if (k !== '__id') out[k] = deref(x, depth + 1);63 return out;64 };65 return deref;66}6768function firstKey(o: Rec, prefix: string): unknown {69 const k = Object.keys(o).find((x) => x === prefix || x.startsWith(`${prefix}(`));70 return k ? o[k] : undefined;71}7273const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null);7475/** Parse the serverVars_data bootstrap of a browse page. Exported for tests. */76export function parseBrowseHtml(htmlText: string): { items: Item[]; totalResults: number | null; maxPages: number | null } | null {77 const m = htmlText.match(/<script id="serverVars_data" type="application\/json">([\s\S]*?)<\/script>/);78 if (!m) return null;79 let root: Rec;80 try {81 root = JSON.parse(m[1]!) as Rec;82 } catch {83 return null;84 }85 const store = ((root.dbl as Rec | undefined)?.relayData ?? (root.relay as Rec | undefined)?.store ?? null) as Record<string, Rec> | null;86 if (!store || typeof store !== 'object') return null;87 const deref = makeDeref(store);88 const items: Item[] = [];89 let totalResults: number | null = null;90 let maxPages: number | null = null;91 for (const rec of Object.values(store)) {92 if (!rec || typeof rec !== 'object') continue;93 if (rec.__typename === 'ItemSearchQueryConnection') {94 if (typeof rec.totalResults === 'number') totalResults = rec.totalResults;95 if (typeof rec.displayMaxNumberOfPages === 'number') maxPages = rec.displayMaxNumberOfPages;96 continue;97 }98 if (rec.__typename !== 'Item' || typeof rec.serviceId !== 'string' || typeof rec.title !== 'string') continue;99 const dp = deref(firstKey(rec, 'displayPrice')) as Array<{ convertedAmountList?: Array<{ amount?: number; currency?: string }>; amountType?: string }> | undefined;100 const prices: Record<string, number> = {};101 let amountType: string | null = null;102 for (const d of dp ?? []) {103 amountType ??= d.amountType ?? null;104 for (const c of d.convertedAmountList ?? []) if (c.currency && typeof c.amount === 'number' && c.amount > 0) prices[c.currency] ??= c.amount;105 }106 const track = rec.ecommerceTrackingParams as { price?: number; convertedAmounts?: Record<string, number> } | undefined;107 if (!Object.keys(prices).length && track?.convertedAmounts) for (const [k, v] of Object.entries(track.convertedAmounts)) if (typeof v === 'number' && v > 0) prices[k] = v;108 const seller = deref(rec.seller) as { serviceId?: string; sellerProfile?: { company?: string } } | null;109 const creators = (deref(rec.creators) as Array<{ creator?: { displayName?: string } }> | null) ?? [];110 const address = deref(rec.address) as { englishCountryName?: string } | null;111 const qv = deref(rec.quickViewDisplay) as { paragraphs?: Array<{ key?: string; text?: string }> } | null;112 const para = (key: string) => str(qv?.paragraphs?.find((p) => p.key === key)?.text);113 const meas = deref(rec.measurement) as Rec | null;114 const measList = meas ? (firstKey(meas, 'display') as Array<{ unit?: string; value?: string }> | undefined) : undefined;115 const photos = (deref(firstKey(rec, 'photos')) as Array<{ masterOrZoomPath?: string; versions?: Array<{ webPath?: string }> }> | null) ?? [];116 const images = photos.map((p) => p.versions?.find((v) => v.webPath?.includes('width=768'))?.webPath ?? p.masterOrZoomPath ?? null).filter((x): x is string => Boolean(x));117 const link = deref(rec.linkData) as { path?: string } | null;118 const path = str(rec.localizedPdpUrl) ?? str(link?.path);119 if (!path) continue;120 items.push({121 serviceId: rec.serviceId,122 title: rec.title,123 url: path.startsWith('http') ? path : `${BASE}${path}`,124 prices,125 amountType,126 isSold: rec.isSold === true,127 isOnHold: rec.isOnHold === true,128 isUnavailable: rec.isUnavailable === true,129 isNewListing: typeof rec.isNewListing === 'boolean' ? rec.isNewListing : null,130 seller: { serviceId: str(seller?.serviceId), company: str(seller?.sellerProfile?.company) },131 creators: creators.map((c) => str(c?.creator?.displayName)).filter((x): x is string => Boolean(x)),132 country: str(address?.englishCountryName),133 location: para('location'),134 attributesText: para('attributes'),135 materials: para('materials'),136 description: plainText(para('description'), 1200),137 measurement: str(measList?.find((x) => x.unit === 'IN')?.value ?? measList?.[0]?.value),138 categoryCode: str(rec.categoryCode),139 browseUrl: str(rec.browseUrl),140 vertical: str(rec.vertical),141 images: [...new Set(images)].slice(0, 4),142 });143 }144 return { items, totalResults, maxPages };145}146147export function pageUrl(seedPath: string, page: number): string {148 const p = seedPath.endsWith('/') ? seedPath : `${seedPath}/`;149 return `${BASE}${p}${page > 1 ? `?page=${page}` : ''}`;150}151152/**153 * Fixture helper: rebuild a minimal page whose serverVars_data store only holds the search connection154 * record plus the first `n` Item records and everything reachable from them through __ref/__refs.155 */156export function trimStoreHtml(htmlText: string, n = 3): string {157 const m = htmlText.match(/<script id="serverVars_data" type="application\/json">([\s\S]*?)<\/script>/);158 if (!m) return '';159 const root = JSON.parse(m[1]!) as Rec;160 const store = ((root.dbl as Rec | undefined)?.relayData ?? {}) as Record<string, Rec>;161 const keep = new Set<string>();162 const queue: string[] = [];163 for (const [k, v] of Object.entries(store)) {164 if (v?.__typename === 'ItemSearchQueryConnection') keep.add(k);165 }166 let picked = 0;167 for (const [k, v] of Object.entries(store)) {168 if (v?.__typename === 'Item' && picked < n) {169 queue.push(k);170 picked++;171 }172 }173 const collect = (v: unknown) => {174 if (!v || typeof v !== 'object') return;175 if (Array.isArray(v)) return v.forEach(collect);176 const o = v as Rec;177 if (typeof o.__ref === 'string') queue.push(o.__ref);178 if (Array.isArray(o.__refs)) for (const r of o.__refs) queue.push(String(r));179 for (const [k, x] of Object.entries(o)) if (k !== '__ref' && k !== '__refs') collect(x);180 };181 while (queue.length) {182 const k = queue.shift()!;183 if (keep.has(k) || !store[k]) continue;184 keep.add(k);185 collect(store[k]);186 }187 const small: Record<string, Rec> = {};188 for (const k of keep) {189 const rec = store[k]!;190 // the connection record references every item on the page; keep only its scalar fields191 small[k] = rec.__typename === 'ItemSearchQueryConnection' ? Object.fromEntries(Object.entries(rec).filter(([, v]) => v === null || typeof v !== 'object')) : rec;192 }193 return `<!doctype html><html><head><script id="serverVars_data" type="application/json">${JSON.stringify({ dbl: { relayData: small } })}</script></head><body></body></html>`;194}195196function verticalFor(seed: Seed, item: Item): DesignVertical {197 if (seed.vertical !== 'unknown') return seed.vertical;198 const v = item.vertical ?? '';199 const code = item.categoryCode ?? '';200 if (v === 'jewelry') return code.startsWith('J_WAT') ? 'watches' : 'jewelry';201 if (v === 'fashion') return 'fashion';202 if (v === 'art') return 'art';203 if (code.startsWith('F_LIG')) return 'lighting';204 if (code.startsWith('F_DEC') || code.startsWith('F_SER')) return 'decor';205 if (code.startsWith('F_RUG') || code.startsWith('F_TEX')) return 'rugs';206 return v === 'furniture' ? 'furniture' : 'unknown';207}208209export class FirstDibsConnector extends BaseConnector {210 readonly version = '1.0.0';211 readonly parserVersion = PARSER_VERSION;212 protected override minIntervalMs = 5000;213 override readonly urlPatterns = [/^https?:\/\/(www\.)?1stdibs\.com\/(?:furniture|jewelry|fashion|art)\/.+\/id-([a-z]_\d+)\/?/i];214 private readonly cfg: z.infer<typeof ConfigSchema>;215216 constructor(meta: ConnectorMeta) {217 super(meta);218 this.cfg = ConfigSchema.parse(meta.config);219 }220221 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {222 const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds;223 const backfill = ctx.options.mode === 'backfill';224 const maxPages = backfill ? Math.min(this.policy.backfillMaxPages, this.cfg.maxPagesPerSeed) : this.cfg.pagesPerSeed;225 const start = readSeedCursor(ctx.options.cursor, seeds.length);226 const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun);227 let count = 0;228 let items = 0;229 for (let k = 0; k < seedsThisRun; k++) {230 const seedIndex = (start.seedIndex + k) % seeds.length;231 const seed = seeds[seedIndex]!;232 let page = k === 0 ? start.page : 1;233 for (; page <= maxPages; page++) {234 if (ctx.signal?.aborted || this.reached(ctx, count)) return;235 const url = pageUrl(seed.path, page);236 await this.throttle(url);237 const res = await ctx.fetch(url, {238 engines: ['api'],239 responseType: 'text',240 timeoutMs: 90_000,241 expect: ['title', 'price', 'currency'],242 parse: (r) => {243 const p = r.html ? parseBrowseHtml(r.html) : null;244 const priced = p?.items.find((i) => i.prices.USD);245 return p?.items.length ? { title: p.items[0]!.title, price: priced?.prices.USD ?? null, currency: priced ? 'USD' : null } : null;246 },247 minQuality: 0.3,248 });249 const parsed = res.success && res.html ? parseBrowseHtml(res.html) : null;250 if (!parsed) {251 ctx.anomaly(res.success ? 'schema_drift' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'serverVars_data missing'}`);252 break;253 }254 if (!parsed.items.length) {255 if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no Item records`);256 break;257 }258 count++;259 items += parsed.items.length;260 const payload: PagePayload = { kind: 'listing_page', url, seed, page, totalResults: parsed.totalResults, maxPages: parsed.maxPages, items: parsed.items };261 yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };262 await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() });263 await ctx.progress({ page, totalPages: parsed.maxPages, itemsProcessed: items });264 if (parsed.items.length < PAGE_SIZE || (parsed.maxPages !== null && page >= parsed.maxPages)) break;265 }266 const nextSeed = (seedIndex + 1) % seeds.length;267 await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) });268 }269 }270271 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {272 const p = PagePayloadSchema.parse(raw.payload);273 const out: NormalizedRecord[] = [];274 for (const it of p.items) {275 const vertical = verticalFor(p.seed, it);276 const categorySlug = p.seed.slug ?? designSlug(it.attributesText ?? it.browseUrl, `${it.title} ${it.attributesText ?? ''}`, vertical);277 if (!categorySlug) continue;278 const currency = it.prices.USD ? 'USD' : Object.keys(it.prices)[0] ?? null;279 const price = currency ? it.prices[currency]! : null;280 const { year, decade } = yearOrDecade(`${it.title} ${it.attributesText ?? ''}`);281 const period = it.attributesText?.match(/\b(1[5-9]th Century|20th Century|21st Century|Antique|Vintage|Mid-Century Modern|Art Deco|Art Nouveau|Victorian|Georgian|Regency|Louis X[VI]+|Bauhaus|Scandinavian Modern|Hollywood Regency|Brutalist|Space Age|Postmodern|Memphis)\b/i)?.[0] ?? null;282 const attributes = AssetAttributesSchema.parse({283 categorySlug,284 brand: it.creators[0] ?? null,285 name: it.title,286 year,287 material: it.materials,288 size: it.measurement,289 country: null,290 identifiers: { firstdibs_item_id: it.serviceId, ...(it.seller.serviceId ? { firstdibs_seller_id: it.seller.serviceId } : {}) },291 metadata: { decade, period, creators: it.creators, attributes_text: it.attributesText, category_code: it.categoryCode, browse_url: it.browseUrl, vertical: it.vertical, converted_prices: it.prices, amount_type: it.amountType, item_country: it.country, is_new_listing: it.isNewListing },292 });293 out.push(294 NormalizedListingSchema.parse({295 kind: 'listing',296 connectorId: this.meta.id,297 sourceId: this.meta.sourceId,298 sourceUrl: it.url,299 externalId: it.serviceId,300 rawTitle: it.title,301 description: it.description,302 imageUrls: it.images,303 attributes,304 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },305 condition: { condition: null, conditionRaw: null, completeness: null },306 observedAt: raw.fetchedAt,307 confidence: price ? 0.82 : 0.6,308 parserVersion: PARSER_VERSION,309 listingType: 'fixed_price',310 price,311 currency,312 seller: it.seller.company,313 location: it.location ?? it.country,314 availability: it.isSold ? 'sold' : it.isOnHold || it.isUnavailable ? 'ended' : 'available',315 }),316 );317 }318 return out;319 }320}321322export default function createConnector(meta: ConnectorMeta): FirstDibsConnector {323 return new FirstDibsConnector(meta);324}325