SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%
3.8 KB · 48 lines json
Raw Blame History
1{2  "id": "1stdibs",3  "displayName": "1stDibs (vintage design, antiques, jewelry, watches, fashion & art listings)",4  "sourceId": "1stdibs",5  "sourceName": "1stDibs",6  "sourceType": "marketplace",7  "sourceUrl": "https://www.1stdibs.com",8  "module": "api/1stdibs",9  "enginePriority": ["api"],10  "categories": ["design_furniture", "antiques", "porcelain", "glass_crystal", "silver", "clocks", "jewelry", "gemstones", "rolex", "patek_philippe", "audemars_piguet", "omega", "other_watches", "luxury_handbags", "art", "contemporary_art", "photography"],11  "regions": ["US", "GB", "FR", "IT", "DE"],12  "languages": ["en"],13  "currency": ["USD"],14  "supportsListings": true,15  "supportsSold": false,16  "supportsAuctions": false,17  "supportsImages": true,18  "supportsCatalog": false,19  "supportsPopulation": false,20  "supportsLookup": false,21  "refreshFrequencyMinutes": 1440,22  "priority": "medium",23  "trustScore": 0.8,24  "attributionRequired": true,25  "termsUrl": "https://www.1stdibs.com/about/user-agreement/",26  "acquisitionMethod": "public HTML — server-rendered Relay store (serverVars_data JSON) on browse pages",27  "historicalDepth": "none",28  "accessNotes": "1stDibs browse pages (/furniture/seating/, /furniture/lighting/, /furniture/decorative-objects/, /jewelry/, /jewelry/watches/wrist-watches/, /fashion/handbags-purses-bags/, /art/prints-works-on-paper/ … with ?page=N, 60 items per page, at most 50 pages per browse path as capped by the site) are served to the honest RareIndexBot UA as ~3 MB server-rendered documents whose <script id=\"serverVars_data\"> JSON contains the Relay store used to hydrate the grid. We read the `Item` records from that store: title, serviceId (f_########), the retail display price converted by 1stDibs into USD/EUR/GBP/CAD/AUD/CHF/MXN/NOK/SEK/DKK (USD kept as the listing price, the others in metadata), seller company + dealer id, designer/creator names, dealer city/country, period-style attribute line, materials, measurements, photos, sold/on-hold flags. We never call 1stDibs' GraphQL (/soa/graphql — reserved to the site, robots.txt allows only that exact path) nor the /search/ paths robots.txt disallows; product detail pages are not fetched. The page carries an F5/Shape client-detection script, but no challenge was served to plain HTTPS requests; if 1stDibs starts blocking, the connector fails loudly (schema_drift/page_fetch_failed) — no evasion engine is configured. User Agreement (checked 2026-09-08) has no clause on automated access to public pages. Dealer asking prices only → listings (never sales); 5 s between requests, 1 in flight, 3 browse paths × 1 page per daily run with seed rotation (≈180 listings/run, ~9 MB), backfill capped at the site's 50-page window. Category mapping: seed vertical + category code + attribute line/title → design_furniture (default), antiques (period evidence), porcelain/glass/silver/clocks (decorative objects), jewelry/gemstones, watch brand slugs, luxury_handbags, art/photography/contemporary_art; rugs & textiles skipped.",29  "enabled": true,30  "schemaVersion": "1.0",31  "config": {32    "pagesPerSeed": 1,33    "seedsPerRun": 3,34    "maxPagesPerSeed": 50,35    "seeds": [36      { "path": "/furniture/seating/", "vertical": "furniture" },37      { "path": "/furniture/tables/", "vertical": "furniture" },38      { "path": "/furniture/storage-case-pieces/", "vertical": "furniture" },39      { "path": "/furniture/lighting/", "vertical": "lighting" },40      { "path": "/furniture/decorative-objects/", "vertical": "decor" },41      { "path": "/jewelry/", "vertical": "jewelry" },42      { "path": "/jewelry/watches/wrist-watches/", "vertical": "watches" },43      { "path": "/fashion/handbags-purses-bags/", "slug": "luxury_handbags", "vertical": "fashion" },44      { "path": "/art/prints-works-on-paper/", "vertical": "art" }45    ]46  }47}48