SPB Git
spboucher.ai

spb/rareindex

Public

Connectors: luxury/watch/sneaker marketplaces (10 sources, agent M); Firecrawl engine keeps rawHtml

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 7, 2026) parent 8c17a9c Browse files

50 changed files +10,431 −2

50 changed files
Load all 50 files Unified Split
added connectors/api/_luxury-lib/capture.ts +59 −0
@@ -0,0 +1,59 @@
1+/**
2+ * Live capture + smoke for the luxury-wave connectors. Runs a probe crawl through the real router,
3+ * prints normalised samples and saves the first raw record(s) as fixtures.
4+ *
5+ * pnpm tsx connectors/api/_luxury-lib/capture.ts <connectorId> [limit=1] [--no-save]
6+ */
7+import { readFileSync, existsSync } from 'node:fs';
8+import path from 'node:path';
9+import { createCrawlContext, createRouter, ConnectorMetaSchema } from '@rareindex/connectors';
10+import { saveFixture } from '@rareindex/connectors/testing';
11+import type { RareIndexConnector, ConnectorMeta } from '@rareindex/connectors';
12+
13+function loadEnv() {
14+ const p = path.resolve(process.cwd(), '.env');
15+ if (!existsSync(p)) return;
16+ for (const line of readFileSync(p, 'utf8').split('\n')) {
17+ const m = line.match(/^([A-Z0-9_]+)=(.*)$/);
18+ if (m && !process.env[m[1]!]) process.env[m[1]!] = m[2]!.replace(/^"(.*)"$/, '$1');
19+ }
20+}
21+
22+async function main() {
23+ loadEnv();
24+ const id = process.argv[2];
25+ if (!id) throw new Error('usage: capture.ts <connectorId> [limit] [--no-save]');
26+ const limit = Number(process.argv[3] ?? 1);
27+ const save = !process.argv.includes('--no-save');
28+ const dirs = ['api', 'firecrawl', 'scrapfly'];
29+ const dir = dirs.map((d) => path.resolve('connectors', d, id)).find((d) => existsSync(path.join(d, 'meta.json')));
30+ if (!dir) throw new Error(`no meta.json for ${id}`);
31+ const metaRaw = JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'));
32+ const meta: ConnectorMeta = ConnectorMetaSchema.parse({ ...metaRaw, module: metaRaw.module ?? `${path.basename(path.dirname(dir))}/${id}` });
33+ const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector };
34+ const connector = mod.default(meta);
35+ const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
36+ const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit } });
37+ let n = 0;
38+ for await (const raw of connector.crawl(ctx)) {
39+ n++;
40+ const records = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
41+ const kinds = records.reduce<Record<string, number>>((acc, r) => ((acc[r.kind] = (acc[r.kind] ?? 0) + 1), acc), {});
42+ console.log(`\n[${id}] raw #${n} ${raw.url} engine=${raw.engine} → ${records.length} records ${JSON.stringify(kinds)}`);
43+ for (const r of records.slice(0, 3)) {
44+ const anyR = r as Record<string, unknown>;
45+ console.log(' -', r.kind, '|', (anyR.rawTitle as string | undefined)?.slice(0, 90) ?? (anyR.title as string | undefined), '|', 'price' in r ? `${(r as { price: unknown }).price} ${(r as { currency?: string }).currency ?? ''}` : '', '|', 'saleDate' in r ? (r as { saleDate: Date }).saleDate.toISOString().slice(0, 10) : '', '|', 'attributes' in r ? JSON.stringify({ cat: r.attributes.categorySlug, brand: r.attributes.brand, model: r.attributes.model, ref: r.attributes.reference, ids: r.attributes.identifiers }).slice(0, 220) : '');
46+ }
47+ if (save) {
48+ const name = `${(raw.externalId ?? `raw-${n}`).replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '').toLowerCase()}`;
49+ saveFixture(id, name, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: Math.min(records.length, 1), kinds: [...new Set(records.map((r) => r.kind))] }, note: 'Captured live by connectors/api/_luxury-lib/capture.ts' });
50+ console.log(` saved fixture ${name}`);
51+ }
52+ }
53+ console.log(`\n[${id}] done: ${n} raw records; engine stats ${JSON.stringify(ctx.engineStats)}; anomalies ${JSON.stringify(ctx.anomalies)}`);
54+}
55+
56+main().catch((e) => {
57+ console.error(e);
58+ process.exit(1);
59+});
added connectors/api/_luxury-lib/index.ts +208 −0
@@ -0,0 +1,208 @@
1+import { z } from 'zod';
2+import type { CrawlContext } from '@rareindex/connectors';
3+import { CurrencySchema, extractYear, type CurrencyCode } from '@rareindex/shared';
4+
5+/**
6+ * Shared helpers for luxury / watch / sneaker marketplace connectors. Conventions match the
7+ * existing chrono24 (watches: brand/model/reference) and novelship (sneakers: style_code) connectors
8+ * so entity resolution merges records across sources.
9+ */
10+
11+// ---------- watches ----------
12+export const WATCH_BRAND_CATEGORY: Record<string, string> = {
13+ rolex: 'rolex',
14+ 'patek philippe': 'patek_philippe',
15+ patek: 'patek_philippe',
16+ 'audemars piguet': 'audemars_piguet',
17+ audemars: 'audemars_piguet',
18+ omega: 'omega',
19+};
20+export function watchCategory(brand: string | null | undefined): string {
21+ const b = (brand ?? '').toLowerCase().trim();
22+ for (const [k, v] of Object.entries(WATCH_BRAND_CATEGORY)) if (b === k || b.startsWith(k)) return v;
23+ return 'other_watches';
24+}
25+export const WATCH_BRANDS = ['Rolex', 'Patek Philippe', 'Audemars Piguet', 'Omega', 'Cartier', 'Tudor', 'Breitling', 'IWC', 'Jaeger-LeCoultre', 'A. Lange & Söhne', 'Vacheron Constantin', 'Richard Mille', 'Grand Seiko', 'Seiko', 'TAG Heuer', 'Panerai', 'Hublot', 'Zenith', 'Breguet', 'Blancpain', 'F.P. Journe', 'MB&F', 'De Bethune', 'Greubel Forsey', 'Urwerk', 'Chopard', 'Piaget', 'Bulgari', 'Bvlgari', 'Franck Muller', 'Girard-Perregaux', 'Glashütte Original', 'H. Moser & Cie', 'Jaquet Droz', 'Laurent Ferrier', 'Parmigiani', 'Roger Dubuis', 'Ulysse Nardin', 'Czapek', 'Longines', 'Tiffany & Co.', 'Hermès', 'Chanel', 'Louis Vuitton', 'Bell & Ross', 'Nomos', 'Oris', 'Sinn'];
26+
27+/** Same heuristics as chrono24: 116500LN, 126610LV, 5711/1A-010, 15400ST.OO.1220ST.01, 311.30.42.30.01.005, RM 011 */
28+export const WATCH_REF_RE = /\b(\d{4,6}[A-Z]{0,3}(?:\/\d[A-Z0-9]*)?(?:-\d{3})?|\d{5}[A-Z]{2}\.[A-Z]{2}\.\d{4}[A-Z]{2}\.\d{2}|\d{3}\.\d{2}\.\d{2}\.\d{2}\.\d{2}\.\d{3}|RM\s?\d{2,3}(?:-\d{2})?|W[A-Z0-9]{6,8}|[A-Z]{1,3}\d{3,5}[A-Z]{0,2}\.[A-Z0-9.]{3,})\b/;
29+export function watchReferenceFromText(text: string): string | null {
30+ return [...text.matchAll(new RegExp(WATCH_REF_RE.source, 'g'))].map((m) => m[1]!.replace(/\s+/g, ' ')).find((r) => !/^(18|19|20)\d{2}$/.test(r) && !/^\d{4}$/.test(r)) ?? null;
31+}
32+export function watchMaterial(text: string): string | null {
33+ const l = text.toLowerCase();
34+ if (/platinum/.test(l)) return 'platinum';
35+ if (/two[- ]tone|rolesor|steel (?:and|&) (?:yellow |rose |everose )?gold/.test(l)) return 'two-tone';
36+ if (/yellow gold|rose gold|everose|white gold|18k|18ct|gold/.test(l)) return 'gold';
37+ if (/titanium/.test(l)) return 'titanium';
38+ if (/ceramic/.test(l)) return 'ceramic';
39+ if (/steel|stainless|\bss\b/.test(l)) return 'steel';
40+ return null;
41+}
42+export function watchCompleteness(text: string): string | null {
43+ const l = text.toLowerCase();
44+ if (/full set|box (?:and|&|\/|\+) papers|box\/papers|complete set|box and card|box & card/.test(l)) return 'full_set';
45+ if (/papers|warranty card|guarantee/.test(l)) return 'papers_only';
46+ if (/\bbox\b/.test(l)) return 'box_only';
47+ return null;
48+}
49+export function watchConditionRaw(text: string): string | null {
50+ const l = text.toLowerCase();
51+ if (/unworn|brand new|\bnew\b|never worn/.test(l)) return 'Unworn';
52+ if (/like new|\bmint\b|excellent|pristine/.test(l)) return 'Excellent';
53+ if (/very good|good/.test(l)) return 'Good';
54+ if (/fair/.test(l)) return 'Fair';
55+ if (/pre-owned|used|preowned/.test(l)) return 'Pre-owned';
56+ return null;
57+}
58+export function caseSize(text: string): string | null {
59+ const m = text.match(/\b(\d{2}(?:\.\d)?)\s?mm\b/i);
60+ return m ? `${m[1]}mm` : null;
61+}
62+
63+// ---------- sneakers ----------
64+export function sneakerCategory(brand: string | null | undefined, name = ''): string {
65+ const b = `${brand ?? ''} ${name}`.toLowerCase();
66+ if (/jordan|nike/.test(b)) return 'nike_jordan';
67+ if (/adidas|yeezy/.test(b)) return 'adidas_yeezy';
68+ return 'new_balance_asics_other';
69+}
70+/** Style codes: FV5029-003, DZ5485-612, IE7264, GX1332, M990GL6, BB550STA, 1201A019-100, CT8012-170 */
71+export const STYLE_CODE_RE = /\b([A-Z]{1,2}\d{4,5}-\d{3}|\d{6}-\d{3}|[A-Z]{2}\d{4}|[A-Z]{1,3}\d{3,4}[A-Z]{1,4}\d{0,2}|\d{4}[A-Z]\d{3}-\d{3}|[A-Z]{2}\d{4}-\d{3})\b/;
72+export function styleCodeFromText(text: string): string | null {
73+ const m = text.toUpperCase().match(STYLE_CODE_RE);
74+ return m ? m[1]! : null;
75+}
76+
77+// ---------- handbags ----------
78+const BAG_MODELS = ['Birkin', 'Kelly', 'Constance', 'Lindy', 'Evelyne', 'Picotin', 'Garden Party', 'Bolide', 'Classic Flap', 'Classic Double Flap', 'Double Flap', 'Single Flap', '2.55', 'Reissue', 'Boy', 'Coco Handle', '19 Flap', 'Deauville', 'Gabrielle', 'Speedy', 'Neverfull', 'Alma', 'Keepall', 'Pochette Métis', 'Pochette Metis', 'Capucines', 'Twist', 'OnTheGo', 'Onthego', 'Multi Pochette', 'Noé', 'Petite Malle', 'Lady Dior', 'Saddle', 'Book Tote', 'Caro', 'Diorama', 'Jackie', 'Dionysus', 'Marmont', 'Bamboo', 'Horsebit', 'Saint Louis', 'Goyardine', 'Artois', 'Anjou', 'Baguette', 'Peekaboo', 'Cassette', 'Jodie', 'Pouch', 'Puzzle', 'Hammock', 'Galleria', 'Re-Edition', 'Cleo', 'Luggage', 'Triomphe', 'Loulou', 'Kate', 'Sac de Jour', 'Le 5 à 7'];
79+export function bagModel(title: string): string | null {
80+ const t = title.toLowerCase();
81+ for (const m of BAG_MODELS) if (t.includes(m.toLowerCase())) return m;
82+ return null;
83+}
84+export function bagSize(title: string): string | null {
85+ const m = title.match(/\b(\d{2})\b(?!\s?mm)/) ?? title.match(/\b(Mini|Small|Medium|Large|Jumbo|Maxi|PM|MM|GM|Nano|Micro)\b/i);
86+ return m ? m[1]! : null;
87+}
88+export const BAG_MATERIALS = ['Togo', 'Epsom', 'Clemence', 'Clémence', 'Swift', 'Box Calf', 'Chevre', 'Chèvre', 'Ostrich', 'Crocodile', 'Alligator', 'Lizard', 'Caviar', 'Lambskin', 'Calfskin', 'Patent', 'Canvas', 'Monogram', 'Damier', 'Epi', 'Vernis', 'Empreinte', 'Suede', 'Nylon', 'Denim', 'Tweed', 'Python', 'Goatskin', 'Goyardine', 'Raffia'];
89+export function bagMaterial(title: string): string | null {
90+ const t = title.toLowerCase();
91+ for (const m of BAG_MATERIALS) if (t.includes(m.toLowerCase())) return m;
92+ return null;
93+}
94+export function bagHardware(title: string): string | null {
95+ const m = title.match(/\b(Gold|Palladium|Silver|Rose Gold|Ruthenium|Brushed Gold|Light Gold|Aged Gold|Antique Gold|Gunmetal|Permabrass)\s+Hardware\b/i) ?? title.match(/\b(GHW|PHW|RGHW|SHW|BGHW)\b/);
96+ return m ? m[1]! : null;
97+}
98+const LUXURY_BAG_BRANDS = ['hermès', 'hermes', 'chanel', 'louis vuitton', 'dior', 'christian dior', 'gucci', 'goyard', 'fendi', 'bottega veneta', 'prada', 'celine', 'céline', 'loewe', 'saint laurent', 'ysl', 'balenciaga', 'givenchy', 'valentino', 'burberry', 'miu miu', 'chloé', 'chloe', 'jacquemus', 'the row'];
99+export function isLuxuryBagBrand(brand: string | null | undefined): boolean {
100+ return LUXURY_BAG_BRANDS.includes((brand ?? '').toLowerCase().trim());
101+}
102+
103+// ---------- Shopify ----------
104+export const ShopifyVariantSchema = z.object({
105+ id: z.number(),
106+ title: z.string(),
107+ price: z.string(),
108+ compare_at_price: z.string().nullable().optional(),
109+ available: z.boolean(),
110+ sku: z.string().nullable().optional(),
111+ option1: z.string().nullable().optional(),
112+ updated_at: z.string().optional(),
113+});
114+export const ShopifyProductSchema = z.object({
115+ id: z.number(),
116+ title: z.string(),
117+ handle: z.string(),
118+ body_html: z.string().nullable().optional(),
119+ published_at: z.string().nullable().optional(),
120+ created_at: z.string().optional(),
121+ updated_at: z.string().optional(),
122+ vendor: z.string().nullable().optional(),
123+ product_type: z.string().nullable().optional(),
124+ tags: z.array(z.string()).optional(),
125+ variants: z.array(ShopifyVariantSchema),
126+ images: z.array(z.object({ src: z.string() })).optional(),
127+ options: z.array(z.object({ name: z.string(), values: z.array(z.string()).optional() })).optional(),
128+});
129+export type ShopifyProduct = z.infer<typeof ShopifyProductSchema>;
130+
131+/** Trim a Shopify product to what normalize() needs (payloads stay compact). */
132+export function trimShopifyProduct(p: ShopifyProduct): ShopifyProduct {
133+ return {
134+ id: p.id,
135+ title: p.title,
136+ handle: p.handle,
137+ body_html: p.body_html ? p.body_html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 1200) : null,
138+ published_at: p.published_at ?? null,
139+ updated_at: p.updated_at,
140+ vendor: p.vendor ?? null,
141+ product_type: p.product_type ?? null,
142+ tags: (p.tags ?? []).filter((t) => !/^(updated-|amazon-|tiktok-)/.test(t)).slice(0, 40),
143+ variants: p.variants.slice(0, 20).map((v) => ({ id: v.id, title: v.title, price: v.price, compare_at_price: v.compare_at_price ?? null, available: v.available, sku: v.sku ?? null, option1: v.option1 ?? null, updated_at: v.updated_at })),
144+ images: (p.images ?? []).slice(0, 2).map((i) => ({ src: i.src })),
145+ options: (p.options ?? []).map((o) => ({ name: o.name })),
146+ };
147+}
148+
149+/** Fetch one page of a public Shopify products.json feed (plain HTTPS, no auth). */
150+export async function fetchShopifyPage(ctx: CrawlContext, base: string, path: string, page: number, limit = 250): Promise<{ products: ShopifyProduct[]; res: Awaited<ReturnType<CrawlContext['fetch']>> }> {
151+ const url = `${base}${path}?limit=${limit}&page=${page}`;
152+ const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => {
153+ const j = r.json as { products?: Array<{ title?: string; variants?: Array<{ price?: string }> }> } | null;
154+ const f = j?.products?.[0];
155+ return f ? { title: f.title, price: f.variants?.[0]?.price } : { title: 'empty-page', price: 0 };
156+ } });
157+ const parsed = z.object({ products: z.array(ShopifyProductSchema) }).safeParse(res.json);
158+ return { products: parsed.success ? parsed.data.products : [], res };
159+}
160+
161+export function currencyOr(code: string | null | undefined, fallback: CurrencyCode): CurrencyCode {
162+ const c = CurrencySchema.safeParse((code ?? '').toUpperCase());
163+ return c.success ? c.data : fallback;
164+}
165+
166+export function yearFrom(text: string | null | undefined): number | null {
167+ return text ? extractYear(text) : null;
168+}
169+
170+export function moneyNumber(s: string | number | null | undefined): number | null {
171+ if (s === null || s === undefined) return null;
172+ const n = typeof s === 'number' ? s : Number(String(s).replace(/[^0-9.]/g, ''));
173+ return Number.isFinite(n) && n > 0 ? n : null;
174+}
175+
176+// ---------- Firecrawl rawHtml (scripts kept) ----------
177+import type { ExtractionResult } from '@rareindex/shared';
178+/**
179+ * The shared Firecrawl engine requests the `html` format, which strips <script> blocks — pages whose
180+ * data lives in __NEXT_DATA__ need `rawHtml`. This helper calls Firecrawl directly for rawHtml and
181+ * books the attempt into ctx.engineStats so cost/health accounting stays correct. Falls back to
182+ * ctx.fetch when no key is configured.
183+ */
184+export async function fetchRawHtml(ctx: CrawlContext, url: string, opts: { waitForMs?: number; timeoutMs?: number } = {}): Promise<ExtractionResult> {
185+ const key = process.env.FIRECRAWL_API_KEY;
186+ if (!key) return ctx.fetch(url, { minQuality: 0.2 });
187+ const started = Date.now();
188+ const stats = (ctx.engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 });
189+ stats.attempts++;
190+ try {
191+ const res = await fetch('https://api.firecrawl.dev/v2/scrape', {
192+ method: 'POST',
193+ headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
194+ body: JSON.stringify({ url, formats: ['rawHtml'], onlyMainContent: false, timeout: opts.timeoutMs ?? 60_000, ...(opts.waitForMs ? { waitFor: opts.waitForMs } : {}) }),
195+ });
196+ const text = await res.text();
197+ const json = JSON.parse(text) as { success?: boolean; data?: { rawHtml?: string; metadata?: { statusCode?: number; url?: string } }; error?: string; creditsUsed?: number };
198+ const status = json.data?.metadata?.statusCode ?? null;
199+ const ok = Boolean(res.ok && json.success && json.data?.rawHtml && (status === null || status < 400));
200+ stats.credits += json.creditsUsed ?? 1;
201+ stats.ms += Date.now() - started;
202+ if (ok) stats.success++;
203+ return { success: ok, engine: 'firecrawl', url, finalUrl: json.data?.metadata?.url ?? null, httpStatus: status, html: json.data?.rawHtml ?? null, markdown: null, json: null, qualityScore: ok ? 0.8 : 0, requiresReview: false, error: ok ? null : (json.error ?? `status ${status ?? res.status}`), costCredits: json.creditsUsed ?? 1, durationMs: Date.now() - started, fetchedAt: new Date() };
204+ } catch (err) {
205+ stats.ms += Date.now() - started;
206+ return { success: false, engine: 'firecrawl', url, finalUrl: null, httpStatus: null, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error: err instanceof Error ? err.message : String(err), costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };
207+ }
208+}
added connectors/api/antiquorum/index.test.ts +37 −0
@@ -0,0 +1,37 @@
1+import { describe, expect, it } from 'vitest';
2+import { ConnectorMetaSchema } from '@rareindex/connectors';
3+import metaJson from './meta.json' with { type: 'json' };
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector from './index.js';
6+
7+// meta.json is read directly so the suite does not depend on connectors/registry.json being regenerated.
8+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
9+
10+describe('antiquorum', () => {
11+ runFixtureSuite(connector, it, expect);
12+
13+ it('normalises the first fixture into sale records with the expected fields', async () => {
14+ const [name] = listFixtures('antiquorum');
15+ const fx = loadFixture('antiquorum', name!);
16+ const out = await connector.normalize(fx.raw);
17+ expect(out.length).toBeGreaterThan(0);
18+ const sale = out.find((r) => r.kind === 'sale');
19+ if (sale?.kind !== 'sale') throw new Error('no sale');
20+ expect(sale.saleType).toBe('auction');
21+ expect(sale.buyerPremiumIncluded).toBe(true);
22+ expect(['CHF', 'HKD', 'EUR', 'USD']).toContain(sale.currency);
23+ expect(sale.auctionHouse).toBe('Antiquorum');
24+ expect(sale.lotNumber).toMatch(/^\d+/);
25+ expect(sale.attributes.brand).not.toMatch(/switzerland/i);
26+ expect(sale.saleDate.getUTCFullYear()).toBeGreaterThanOrEqual(2022);
27+ const lot = out.find((r) => r.kind === 'auction_lot');
28+ if (lot && lot.kind === 'auction_lot') expect(lot.auctionHouse).toBe('Antiquorum');
29+ });
30+ it('parses the catalogue home into auctions with ids and dates', async () => {
31+ const { parseAuctionIndex } = await import('./index.js');
32+ const html = '<div><h3>Important Modern &amp; Vintage Timepieces</h3><p>Hong Kong May 31, 2026</p><a href="/en/auctions/Hong_Kong_May_31_2026/lots">Lots</a> <a href="/en/auctions/387/price-list">Price list</a></div><div><h3>Only Online Auction</h3><p>Sep 18, 2025</p><a href="/en/auctions/hong_kong_september_18_2025/lots">Lots</a><a href="/en/auctions/378/price-list">x</a></div>';
33+ const a = parseAuctionIndex(html);
34+ expect(a[0]).toMatchObject({ slug: 'Hong_Kong_May_31_2026', id: '387', date: 'May 31, 2026', location: 'Hong Kong' });
35+ expect(a[1]).toMatchObject({ slug: 'hong_kong_september_18_2025', id: '378' });
36+ });
37+});
added connectors/api/antiquorum/index.ts +219 −0
@@ -0,0 +1,219 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';
4+import { caseSize, currencyOr, moneyNumber, watchCategory, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js';
5+
6+/** Antiquorum — catalogue lots pages (RDFa Products + 'Sold: CCY amount', premium-inclusive). */
7+const BASE = 'https://catalog.antiquorum.swiss';
8+const PARSER_VERSION = '1.0.0';
9+
10+export const AuctionSchema = z.object({ slug: z.string(), id: z.string().nullable(), title: z.string().nullable(), date: z.string().nullable(), location: z.string().nullable() });
11+export const LotSchema = z.object({
12+ lotNumber: z.string(),
13+ name: z.string(),
14+ url: z.string(),
15+ sku: z.string().nullable(),
16+ image: z.string().nullable(),
17+ brand: z.string().nullable(),
18+ model: z.string().nullable(),
19+ reference: z.string().nullable(),
20+ year: z.string().nullable(),
21+ material: z.string().nullable(),
22+ diameter: z.string().nullable(),
23+ description: z.string().nullable(),
24+ estimateLow: z.number().nullable(),
25+ estimateHigh: z.number().nullable(),
26+ estimateCurrency: z.string().nullable(),
27+ soldPrice: z.number().nullable(),
28+ soldCurrency: z.string().nullable(),
29+ accessories: z.string().nullable(),
30+});
31+export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), url: z.string(), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) });
32+export type PagePayload = z.infer<typeof PagePayloadSchema>;
33+
34+const unesc = (s: string) => s.replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/\s+/g, ' ').trim();
35+
36+/** Auctions listed on the catalogue home: slug (+ numeric id, date, title) per block. */
37+export function parseAuctionIndex(htmlText: string): z.infer<typeof AuctionSchema>[] {
38+ const out: z.infer<typeof AuctionSchema>[] = [];
39+ const seen = new Set<string>();
40+ const anchors = [...htmlText.matchAll(/href="\/en\/auctions\/([A-Za-z0-9_]+)\/lots"/g)];
41+ for (let i = 0; i < anchors.length; i++) {
42+ const slug = anchors[i]![1]!;
43+ if (seen.has(slug)) continue;
44+ seen.add(slug);
45+ const start = anchors[i]!.index!;
46+ const prevEnd = i > 0 ? anchors[i - 1]!.index! + anchors[i - 1]![0].length : 0;
47+ const end = i + 1 < anchors.length ? anchors[i + 1]!.index! : Math.min(htmlText.length, start + 4000);
48+ // title + date precede the lots link inside an auction card; the price-list link follows it
49+ const before = htmlText.slice(Math.max(prevEnd, start - 1500), start);
50+ const after = htmlText.slice(start, end);
51+ const dateRe = /([A-Z][a-z]{2,8} \d{1,2}(?:-\d{1,2})?,? 20\d\d)/g;
52+ const id = after.match(/\/en\/auctions\/(\d+)\/price-list/)?.[1] ?? before.match(/\/en\/auctions\/(\d+)\/price-list/)?.[1] ?? null;
53+ const beforeDates = [...before.matchAll(dateRe)].map((m) => m[1]!);
54+ const date = beforeDates[beforeDates.length - 1] ?? after.match(dateRe)?.[0] ?? null;
55+ const titles = [...before.matchAll(/<h[1-5][^>]*>\s*([^<]{4,90}?)\s*<\/h[1-5]>/g)].map((m) => m[1]!);
56+ const title = titles[titles.length - 1] ?? null;
57+ const loc = slug.match(/hong_kong|geneva|monaco|new_york|dubai/i)?.[0]?.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()) ?? null;
58+ out.push({ slug, id, title: title ? unesc(title) : null, date, location: loc });
59+ }
60+ return out;
61+}
62+
63+export function parseLotsPage(htmlText: string, url: string, auction: z.infer<typeof AuctionSchema>, page: number): PagePayload {
64+ const lots: z.infer<typeof LotSchema>[] = [];
65+ const blocks = htmlText.split(/<h4>\s*LOT\s+/).slice(1);
66+ for (const b of blocks) {
67+ const lotNumber = b.match(/^(\d+[A-Z]?)/)?.[1];
68+ if (!lotNumber) continue;
69+ const name = b.match(/property="schema:name" content="([^"]*)"/)?.[1];
70+ const urlRel = b.match(/rel="schema:url" resource="([^"\s]+)/)?.[1] ?? b.match(/href="(\/en\/lots\/[^"]+)"/)?.[1];
71+ if (!name || !urlRel) continue;
72+ const spec = (label: string) => b.match(new RegExp(`<strong>${label}</strong>&emsp;([^<]{1,120})`))?.[1]?.trim() ?? null;
73+ const est = b.match(/N_lots_estimation'\s*>\s*([A-Z]{3})\s*([\d,]+)\s*-\s*([\d,]+)/);
74+ const sold = b.match(/Sold:\s*([A-Z]{3})\s*([\d,]+)/);
75+ lots.push({
76+ lotNumber,
77+ name: unesc(name),
78+ url: urlRel.startsWith('http') ? urlRel.trim() : `${BASE}${urlRel}`,
79+ sku: b.match(/property="schema:sku" content="([^"]*)"/)?.[1] ?? null,
80+ image: b.match(/rel="schema:image" resource="([^"]+)"/)?.[1] ?? null,
81+ brand: spec('Brand'),
82+ model: spec('Model'),
83+ reference: spec('Reference'),
84+ year: spec('Year'),
85+ material: spec('Material'),
86+ diameter: spec('Diameter'),
87+ description: b.match(/property="schema:description" content="([^"]*)"/)?.[1]?.slice(0, 500) ?? null,
88+ estimateLow: est ? moneyNumber(est[2]) : null,
89+ estimateHigh: est ? moneyNumber(est[3]) : null,
90+ estimateCurrency: est ? est[1]! : null,
91+ soldPrice: sold ? moneyNumber(sold[2]) : null,
92+ soldCurrency: sold ? sold[1]! : null,
93+ accessories: spec('Accessories')?.slice(0, 200) ?? null,
94+ });
95+ }
96+ return { kind: 'lots_page', url, auction, page, lots };
97+}
98+
99+export class AntiquorumConnector extends BaseConnector {
100+ readonly version = '1.0.0';
101+ readonly parserVersion = PARSER_VERSION;
102+ protected override minIntervalMs = 2000;
103+ override readonly urlPatterns = [/^https?:\/\/catalog\.antiquorum\.swiss\/en\/lots\/[a-z0-9-]+-lot-(\d+)-(\d+)/i];
104+
105+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
106+ const perRun = Number(this.meta.config.auctionsPerRun ?? 3);
107+ const maxPages = Number(this.meta.config.maxPagesPerAuction ?? 25);
108+ const done = new Set<string>(((ctx.options.cursor?.doneSlugs as string[] | undefined) ?? []));
109+ await this.throttle();
110+ const home = await ctx.fetch(`${BASE}/en`, { responseType: 'text', minQuality: 0.2 });
111+ if (!home.success || !home.html) {
112+ ctx.anomaly('page_fetch_failed', `home: ${home.error ?? home.httpStatus}`);
113+ return;
114+ }
115+ const auctions = parseAuctionIndex(home.html);
116+ const now = Date.now();
117+ // upcoming/current auctions first (they change), then past auctions not yet crawled (backfill).
118+ const withDate = auctions.map((a) => ({ a, t: a.date ? (parseSourceDate(a.date)?.getTime() ?? 0) : 0 }));
119+ const upcoming = withDate.filter((x) => x.t >= now - 3 * 86_400_000).map((x) => x.a);
120+ const past = withDate.filter((x) => x.t < now - 3 * 86_400_000 && !done.has(x.a.slug)).sort((x, y) => y.t - x.t).map((x) => x.a);
121+ const selected = ctx.options.seeds?.length ? auctions.filter((a) => ctx.options.seeds!.includes(a.slug)) : [...upcoming, ...past].slice(0, ctx.options.mode === 'backfill' ? perRun * 4 : perRun);
122+ let count = 0;
123+ for (const auction of selected) {
124+ for (let page = 1; page <= maxPages; page++) {
125+ if (ctx.signal?.aborted || this.reached(ctx, count)) return;
126+ const url = `${BASE}/en/auctions/${auction.slug}/lots${page > 1 ? `?page=${page}` : ''}`;
127+ await this.throttle();
128+ const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'date'], parse: (r) => {
129+ const p = r.html ? parseLotsPage(r.html, url, auction, page) : null;
130+ const first = p?.lots[0];
131+ return first ? { title: first.name, price: first.soldPrice ?? first.estimateLow, date: auction.date } : null;
132+ } });
133+ const payload = res.success && res.html ? parseLotsPage(res.html, url, auction, page) : null;
134+ if (!payload || payload.lots.length === 0) {
135+ if (page === 1) ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
136+ break;
137+ }
138+ count++;
139+ yield { url, externalId: `${auction.slug}:${page}`, kind: payload.lots.some((l) => l.soldPrice) ? 'sale' : 'auction_lot', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
140+ if (!(res.html ?? '').includes(`lots?page=${page + 1}`)) break;
141+ }
142+ const isPast = auction.date ? (parseSourceDate(auction.date)?.getTime() ?? 0) < now - 3 * 86_400_000 : false;
143+ if (isPast) {
144+ done.add(auction.slug);
145+ await ctx.setCursor({ doneSlugs: [...done].slice(-200) });
146+ }
147+ }
148+ }
149+
150+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
151+ const m = url.match(this.urlPatterns[0]!);
152+ if (!m) return [];
153+ await this.throttle();
154+ const home = await ctx.fetch(`${BASE}/en`, { responseType: 'text', minQuality: 0.2 });
155+ const auction = home.html ? parseAuctionIndex(home.html).find((a) => a.id === m[1]) : undefined;
156+ if (!auction) return [];
157+ // lots pages hold 20 lots each; lot N sits on page ceil(N/20)
158+ const page = Math.max(1, Math.ceil(Number(m[2]) / 20));
159+ const pageUrl = `${BASE}/en/auctions/${auction.slug}/lots?page=${page}`;
160+ await this.throttle();
161+ const res = await ctx.fetch(pageUrl, { responseType: 'text', minQuality: 0.2 });
162+ if (!res.success || !res.html) return [];
163+ const payload = parseLotsPage(res.html, pageUrl, auction, page);
164+ payload.lots = payload.lots.filter((l) => l.url.split('?')[0] === url.split('?')[0]);
165+ return payload.lots.length ? [{ url: pageUrl, externalId: `${auction.slug}:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : [];
166+ }
167+
168+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
169+ const p = PagePayloadSchema.parse(raw.payload);
170+ const saleDate = p.auction.date ? parseSourceDate(p.auction.date.replace(/(\d{1,2})-\d{1,2},/, '$1,')) : null;
171+ const out: NormalizedRecord[] = [];
172+ for (const lot of p.lots) {
173+ const brandRaw = lot.brand ?? lot.name.split(',')[0] ?? null;
174+ const brand = brandRaw ? unesc(brandRaw).replace(/,\s*(switzerland|germany|france|japan|usa|england|u\.?s\.?a\.?)\s*$/i, '').replace(/\b([A-Z])([A-Z]+)\b/g, (_m, a: string, b: string) => a + b.toLowerCase()).trim() : null;
175+ const isJewelry = /jewel|necklace|bracelet|ring\b|earring|brooch|diamond/i.test(lot.name) && !/watch|wristwatch|chronograph/i.test(lot.name + (lot.description ?? ''));
176+ const categorySlug = isJewelry ? 'jewelry' : watchCategory(brand);
177+ const reference = lot.reference ?? watchReferenceFromText(lot.name);
178+ const yearMatch = lot.year?.match(/(19|20)\d{2}/)?.[0];
179+ const attributes = AssetAttributesSchema.parse({
180+ categorySlug,
181+ brand,
182+ name: `${brand ?? ''} ${lot.model ?? ''}`.trim() || lot.name,
183+ model: lot.model,
184+ reference,
185+ year: yearMatch ? Number(yearMatch) : null,
186+ material: lot.material ? (watchMaterial(lot.material) ?? lot.material.toLowerCase()) : watchMaterial(lot.name),
187+ size: lot.diameter ? caseSize(lot.diameter) : null,
188+ identifiers: { ...(reference ? { reference } : {}), antiquorum_lot: lot.sku ?? `${p.auction.id ?? p.auction.slug}-${lot.lotNumber}` },
189+ metadata: { auction: p.auction.title, auction_slug: p.auction.slug, location: p.auction.location, estimate: lot.estimateLow ? { low: lot.estimateLow, high: lot.estimateHigh, currency: lot.estimateCurrency } : null, accessories: lot.accessories },
190+ });
191+ const accessories = (lot.accessories ?? '').toLowerCase();
192+ const completeness = accessories ? (/box/.test(accessories) && /(certificate|papers|warranty|guarantee)/.test(accessories) ? 'full_set' : /(certificate|papers|warranty|guarantee)/.test(accessories) ? 'papers_only' : /box/.test(accessories) ? 'box_only' : null) : null;
193+ const base = {
194+ connectorId: this.meta.id,
195+ sourceId: this.meta.sourceId,
196+ sourceUrl: lot.url,
197+ externalId: lot.sku ?? `${p.auction.slug}-${lot.lotNumber}`,
198+ rawTitle: lot.name,
199+ description: lot.description,
200+ imageUrls: lot.image ? [lot.image] : [],
201+ attributes,
202+ condition: { condition: null, conditionRaw: null, completeness },
203+ observedAt: raw.fetchedAt,
204+ parserVersion: PARSER_VERSION,
205+ };
206+ if (lot.soldPrice && lot.soldCurrency && saleDate) {
207+ out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, confidence: 0.92, saleType: 'auction', saleDate, price: lot.soldPrice, currency: currencyOr(lot.soldCurrency, 'CHF'), buyerPremiumIncluded: true, auctionHouse: 'Antiquorum', lotNumber: lot.lotNumber, location: p.auction.location }));
208+ } else if (!lot.soldPrice) {
209+ const isPast = saleDate ? saleDate.getTime() < Date.now() - 86_400_000 : false;
210+ out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, confidence: 0.85, auctionHouse: 'Antiquorum', auctionName: p.auction.title, lotNumber: lot.lotNumber, endsAt: saleDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currency: lot.estimateCurrency ? currencyOr(lot.estimateCurrency, 'CHF') : null, status: isPast ? 'ended' : 'upcoming', location: p.auction.location }));
211+ }
212+ }
213+ return out;
214+ }
215+}
216+
217+export default function createConnector(meta: ConnectorMeta) {
218+ return new AntiquorumConnector(meta);
219+}
added connectors/api/antiquorum/meta.json +33 −0
@@ -0,0 +1,33 @@
1+{
2+ "id": "antiquorum",
3+ "displayName": "Antiquorum (watch auction results & upcoming lots)",
4+ "sourceId": "antiquorum",
5+ "sourceName": "Antiquorum",
6+ "sourceType": "auction_house",
7+ "sourceUrl": "https://catalog.antiquorum.swiss",
8+ "module": "api/antiquorum",
9+ "enginePriority": ["api", "firecrawl"],
10+ "categories": ["watches", "rolex", "patek_philippe", "audemars_piguet", "omega", "other_watches", "jewelry"],
11+ "regions": ["CH", "HK", "MC"],
12+ "languages": ["en"],
13+ "currency": ["CHF", "HKD", "EUR", "USD"],
14+ "supportsListings": false,
15+ "supportsSold": true,
16+ "supportsAuctions": true,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": true,
21+ "refreshFrequencyMinutes": 1440,
22+ "priority": "high",
23+ "trustScore": 0.9,
24+ "attributionRequired": true,
25+ "termsUrl": "https://www.antiquorum.swiss/conditions-of-sale/",
26+ "accessNotes": "Public auction catalogue (catalog.antiquorum.swiss/en → /en/auctions/<slug>/lots?page=N) fetched over plain HTTPS with the RareIndex user agent; robots.txt only disallows /wp-admin/. Each lot carries schema.org RDFa (name, sku, brand, price, currency, image) and a spec table (Brand, Model, Reference, Year, Material). Sold lots show 'Sold: <CCY> <amount>' which Antiquorum's own price lists confirm is hammer + buyer's premium (e.g. lot 387-1: hammer 230,000 → sold 287,500 HKD), so buyerPremiumIncluded=true. Sale date = the auction date on the catalogue home. Unsold/upcoming lots become auction lots with estimates. 2 s between pages.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "auctionsPerRun": 3,
31+ "maxPagesPerAuction": 25
32+ }
33+}
added connectors/api/bobs-watches/index.test.ts +28 −0
@@ -0,0 +1,28 @@
1+import { describe, expect, it } from 'vitest';
2+import { ConnectorMetaSchema } from '@rareindex/connectors';
3+import metaJson from './meta.json' with { type: 'json' };
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector from './index.js';
6+
7+// meta.json is read directly so the suite does not depend on connectors/registry.json being regenerated.
8+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
9+
10+describe('bobs-watches', () => {
11+ runFixtureSuite(connector, it, expect);
12+
13+ it('normalises the first fixture into listing records with the expected fields', async () => {
14+ const [name] = listFixtures('bobs-watches');
15+ const fx = loadFixture('bobs-watches', name!);
16+ const out = await connector.normalize(fx.raw);
17+ expect(out.length).toBeGreaterThan(0);
18+ const l = out[0];
19+ if (l?.kind !== 'listing') throw new Error('no listing');
20+ expect(l.attributes.categorySlug).toBe('rolex');
21+ expect(l.attributes.brand).toBe('Rolex');
22+ expect(l.attributes.reference).toMatch(/^\d{4,6}/);
23+ expect(l.attributes.identifiers.reference).toBe(l.attributes.reference);
24+ expect(l.currency).toBe('USD');
25+ expect(l.sourceUrl).toMatch(/bobswatches\.com\/.+\.html$/);
26+ });
27+
28+});
added connectors/api/bobs-watches/index.ts +128 −0
@@ -0,0 +1,128 @@
1+import { z } from 'zod';
2+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { normalizeCondition } from '@rareindex/taxonomy';
4+import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
5+import { caseSize, currencyOr, moneyNumber, watchCategory, watchCompleteness, watchConditionRaw, watchMaterial, watchReferenceFromText, yearFrom } from '../_luxury-lib/index.js';
6+
7+/** Bob's Watches — schema.org Product JSON-LD on public model pages (pre-owned Rolex, USD). */
8+const BASE = 'https://www.bobswatches.com';
9+const PARSER_VERSION = '1.0.0';
10+
11+export const ItemSchema = z.object({ name: z.string(), mpn: z.string().nullable(), sku: z.string().nullable(), url: z.string(), color: z.string().nullable(), price: z.number(), currency: z.string(), availability: z.string().nullable(), image: z.string().nullable(), condition: z.string().nullable() });
12+export const PagePayloadSchema = z.object({ kind: z.literal('model_page'), url: z.string(), seed: z.string(), items: z.array(ItemSchema) });
13+export type PagePayload = z.infer<typeof PagePayloadSchema>;
14+
15+export function parseModelPage(htmlText: string, url: string, seed: string): PagePayload {
16+ const items: z.infer<typeof ItemSchema>[] = [];
17+ for (const block of H.jsonLd(htmlText, 'Product')) {
18+ const offers = (block.offers as Record<string, unknown> | undefined) ?? {};
19+ const price = moneyNumber(offers.price as string | number | undefined);
20+ if (!price) continue;
21+ const img = block.image;
22+ items.push({
23+ name: String(block.name ?? '').replace(/\s+/g, ' ').trim(),
24+ mpn: block.mpn ? String(block.mpn) : null,
25+ sku: block.sku ? String(block.sku) : null,
26+ url: String(block.url ?? ''),
27+ color: block.color ? String(block.color) : null,
28+ price,
29+ currency: String(offers.priceCurrency ?? 'USD'),
30+ availability: offers.availability ? String(offers.availability).replace(/^.*\//, '') : null,
31+ image: Array.isArray(img) ? String(img[0] ?? '') || null : typeof img === 'string' ? img : null,
32+ condition: block.itemCondition ? String(block.itemCondition).replace(/^.*\//, '') : null,
33+ });
34+ }
35+ return { kind: 'model_page', url, seed, items };
36+}
37+
38+export class BobsWatchesConnector extends BaseConnector {
39+ readonly version = '1.0.0';
40+ readonly parserVersion = PARSER_VERSION;
41+ protected override minIntervalMs = 1500;
42+ override readonly urlPatterns = [/^https?:\/\/(www\.)?bobswatches\.com\/[a-z0-9-]+\.html$/i];
43+
44+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
45+ const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
46+ let count = 0;
47+ for (const seed of seeds) {
48+ if (ctx.signal?.aborted || this.reached(ctx, count)) return;
49+ const url = `${BASE}/${seed}`;
50+ await this.throttle();
51+ const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => {
52+ const first = r.html ? parseModelPage(r.html, url, seed).items[0] : undefined;
53+ return first ? { title: first.name, price: first.price, identifiers: first.mpn ? { mpn: first.mpn } : null } : null;
54+ } });
55+ const payload = res.success && res.html ? parseModelPage(res.html, url, seed) : null;
56+ if (!payload || payload.items.length === 0) {
57+ ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
58+ continue;
59+ }
60+ count++;
61+ yield { url, externalId: `model:${seed}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
62+ }
63+ }
64+
65+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
66+ await this.throttle();
67+ const res = await ctx.fetch(url, { responseType: 'text', minQuality: 0.2 });
68+ if (!res.success || !res.html) return [];
69+ const payload = parseModelPage(res.html, url, 'lookup');
70+ return payload.items.length ? [{ url, externalId: `product:${payload.items[0]!.sku ?? url}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : [];
71+ }
72+
73+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
74+ const p = PagePayloadSchema.parse(raw.payload);
75+ const out: NormalizedRecord[] = [];
76+ const seen = new Set<string>();
77+ for (const it of p.items) {
78+ const id = it.sku ?? it.url;
79+ if (seen.has(id)) continue;
80+ seen.add(id);
81+ const brandMatch = it.name.match(/\b(Rolex|Omega|Patek Philippe|Audemars Piguet|Cartier|Tudor|Breitling|Panerai|IWC)\b/i);
82+ const brand = brandMatch ? brandMatch[1]! : 'Rolex';
83+ const categorySlug = watchCategory(brand);
84+ const ref = it.mpn ?? watchReferenceFromText(it.name);
85+ const model = it.name.match(/Rolex\s+([A-Z][A-Za-z -]+?)(?:\s+Ref\b|\s+\d|\s+Black|\s+Blue|\s+White|\s+Green|$)/)?.[1]?.trim() ?? p.seed.replace(/^rolex-/, '').replace(/-/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
86+ const conditionRaw = watchConditionRaw(it.name) ?? (it.condition === 'UsedCondition' ? 'Pre-owned' : it.condition === 'NewCondition' ? 'Unworn' : null);
87+ const attributes = AssetAttributesSchema.parse({
88+ categorySlug,
89+ brand,
90+ name: `${brand} ${model}`.trim(),
91+ model,
92+ reference: ref,
93+ year: yearFrom(it.name),
94+ material: it.color ? (watchMaterial(it.color) ?? it.color.toLowerCase()) : watchMaterial(it.name),
95+ size: caseSize(it.name),
96+ identifiers: { ...(ref ? { reference: ref } : {}), bobs_sku: id },
97+ metadata: { model_page: p.url, color: it.color },
98+ });
99+ out.push(
100+ NormalizedListingSchema.parse({
101+ kind: 'listing',
102+ connectorId: this.meta.id,
103+ sourceId: this.meta.sourceId,
104+ sourceUrl: it.url,
105+ externalId: id,
106+ rawTitle: it.name,
107+ imageUrls: it.image ? [it.image] : [],
108+ attributes,
109+ condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: watchCompleteness(it.name) },
110+ observedAt: raw.fetchedAt,
111+ confidence: 0.85,
112+ parserVersion: PARSER_VERSION,
113+ listingType: 'fixed_price',
114+ price: it.price,
115+ currency: currencyOr(it.currency, 'USD'),
116+ seller: "Bob's Watches",
117+ location: 'US',
118+ availability: it.availability === 'InStock' ? 'available' : it.availability === 'SoldOut' || it.availability === 'OutOfStock' ? 'sold' : 'unknown',
119+ }),
120+ );
121+ }
122+ return out;
123+ }
124+}
125+
126+export default function createConnector(meta: ConnectorMeta) {
127+ return new BobsWatchesConnector(meta);
128+}
added connectors/api/bobs-watches/meta.json +32 −0
@@ -0,0 +1,32 @@
1+{
2+ "id": "bobs-watches",
3+ "displayName": "Bob's Watches (pre-owned Rolex listings)",
4+ "sourceId": "bobs-watches",
5+ "sourceName": "Bob's Watches",
6+ "sourceType": "dealer",
7+ "sourceUrl": "https://www.bobswatches.com",
8+ "module": "api/bobs-watches",
9+ "enginePriority": ["api", "firecrawl"],
10+ "categories": ["rolex", "watches", "other_watches"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": true,
15+ "supportsSold": false,
16+ "supportsAuctions": false,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": true,
21+ "refreshFrequencyMinutes": 720,
22+ "priority": "medium",
23+ "trustScore": 0.8,
24+ "attributionRequired": true,
25+ "termsUrl": "https://www.bobswatches.com/terms-and-conditions",
26+ "accessNotes": "Public model-family pages (bobswatches.com/rolex-<model>) fetched over plain HTTPS with the RareIndex user agent; robots.txt allows them (sort/query/search URLs are disallowed and never used). Each page embeds schema.org Product JSON-LD per watch (name, mpn = reference, sku = stock id, colour/material, USD price, availability). Listings only; Bob's does not publish realised prices. 1.5 s between pages.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "seeds": ["rolex-submariner", "rolex-daytona", "rolex-gmt-master-ii", "rolex-datejust", "rolex-day-date", "rolex-explorer", "rolex-sea-dweller", "rolex-yacht-master", "rolex-oyster-perpetual", "rolex-sky-dweller", "rolex-milgauss", "rolex-air-king", "vintage-rolex", "used-rolex-watches"]
31+ }
32+}
added connectors/api/crown-caliber/index.test.ts +30 −0
@@ -0,0 +1,30 @@
1+import { describe, expect, it } from 'vitest';
2+import { ConnectorMetaSchema } from '@rareindex/connectors';
3+import metaJson from './meta.json' with { type: 'json' };
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector from './index.js';
6+
7+// meta.json is read directly so the suite does not depend on connectors/registry.json being regenerated.
8+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
9+
10+describe('crown-caliber', () => {
11+ runFixtureSuite(connector, it, expect);
12+
13+ it('normalises the first fixture into listing records with the expected fields', async () => {
14+ const [name] = listFixtures('crown-caliber');
15+ const fx = loadFixture('crown-caliber', name!);
16+ const out = await connector.normalize(fx.raw);
17+ expect(out.length).toBeGreaterThan(0);
18+ const l = out[0];
19+ if (l?.kind !== 'listing') throw new Error('no listing');
20+ expect(l.attributes.categorySlug).toBe('rolex');
21+ expect(l.attributes.reference).toBeTruthy();
22+ expect(l.currency).toBe('USD');
23+ expect(l.sourceUrl).toMatch(/europeanwatch\.com\/watch\//);
24+ });
25+ it('splits names into reference + model and reads circa years', async () => {
26+ const { splitName } = await import('./index.js');
27+ expect(splitName('Rolex 16710 GMT-Master II Coke Bezel SS Black Dial Circa. 2000', 'Rolex')).toEqual({ reference: '16710', model: 'GMT-Master II Coke Bezel' });
28+ expect(splitName('Patek Philippe 5711/1A-010 Nautilus SS Blue Dial', 'Patek Philippe')).toMatchObject({ reference: '5711/1A-010' });
29+ });
30+});
added connectors/api/crown-caliber/index.ts +158 −0
@@ -0,0 +1,158 @@
1+import { z } from 'zod';
2+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { normalizeCondition } from '@rareindex/taxonomy';
4+import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
5+import { WATCH_BRANDS, caseSize, currencyOr, moneyNumber, watchCategory, watchCompleteness, watchConditionRaw, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js';
6+
7+/** European Watch Company (ex Crown & Caliber domain) — ItemList JSON-LD on brand pages. */
8+const BASE = 'https://www.europeanwatch.com';
9+const PARSER_VERSION = '1.0.0';
10+
11+export const ItemSchema = z.object({ name: z.string(), sku: z.string().nullable(), url: z.string(), price: z.number(), currency: z.string(), availability: z.string().nullable(), image: z.string().nullable(), condition: z.string().nullable() });
12+export const PagePayloadSchema = z.object({ kind: z.literal('brand_page'), url: z.string(), brand: z.string(), page: z.number(), items: z.array(ItemSchema), details: z.record(z.string(), z.string()).optional() });
13+export type PagePayload = z.infer<typeof PagePayloadSchema>;
14+
15+export function parseBrandPage(htmlText: string, url: string, brand: string, page: number): PagePayload {
16+ const items: z.infer<typeof ItemSchema>[] = [];
17+ const push = (prod: Record<string, unknown>) => {
18+ const offers = (prod.offers as Record<string, unknown> | undefined) ?? {};
19+ const price = moneyNumber(offers.price as string | number | undefined);
20+ if (!price) return;
21+ const img = prod.image;
22+ items.push({
23+ name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(),
24+ sku: prod.sku ? String(prod.sku) : null,
25+ url: String(prod.url ?? ''),
26+ price,
27+ currency: String(offers.priceCurrency ?? 'USD'),
28+ availability: offers.availability ? String(offers.availability).replace(/^.*\//, '') : null,
29+ image: Array.isArray(img) ? String(img[0] ?? '') || null : typeof img === 'string' ? img : null,
30+ condition: offers.itemCondition ? String(offers.itemCondition).replace(/^.*\//, '') : null,
31+ });
32+ };
33+ for (const list of H.jsonLd(htmlText, 'ItemList')) {
34+ for (const el of (list.itemListElement as Array<Record<string, unknown>>) ?? []) {
35+ const prod = (el.item as Record<string, unknown> | undefined) ?? el;
36+ if (prod && prod['@type'] === 'Product') push(prod);
37+ }
38+ }
39+ for (const prod of H.jsonLd(htmlText, 'Product')) push(prod);
40+ // product pages: spec table "Condition / Box / Papers / Year / Reference"
41+ const details: Record<string, string> = {};
42+ for (const m of htmlText.matchAll(/(Condition|Box|Papers|Year|Reference|Movement|Case Size)<\/(?:p|dt|span)>\s*<(?:p|dd|span)[^>]*>([^<]{1,40})</g)) details[m[1]!.toLowerCase()] = m[2]!.trim();
43+ return { kind: 'brand_page', url, brand, page, items, ...(Object.keys(details).length ? { details } : {}) };
44+}
45+
46+/** "Rolex 126234 Datejust 36 Jubilee SS Green Palm Dial" → reference 126234, model "Datejust 36" */
47+export function splitName(name: string, brand: string): { reference: string | null; model: string | null } {
48+ const rest = name.replace(new RegExp(`^${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s+`, 'i'), '');
49+ const tokens = rest.split(/\s+/);
50+ const ref = watchReferenceFromText(rest) ?? (tokens[0] && /\d/.test(tokens[0]) ? tokens[0] : null);
51+ const afterRef = ref && tokens.length > 1 && tokens[0] === ref ? tokens.slice(1) : tokens;
52+ const modelTokens: string[] = [];
53+ for (const t of afterRef) {
54+ if (/^(SS|18k|18K|Steel|Gold|Platinum|Titanium|Ceramic|Two|Rose|Yellow|White|Black|Blue|Green|Silver|Grey|Gray|Champagne|Circa|Full|Box|Papers|Dial|Bracelet|Strap|Jubilee|Oyster|Leather|\d{2}mm|\d{4})$/i.test(t)) break;
55+ modelTokens.push(t);
56+ if (modelTokens.length >= 4) break;
57+ }
58+ return { reference: ref, model: modelTokens.length ? modelTokens.join(' ') : null };
59+}
60+
61+export class EuropeanWatchConnector extends BaseConnector {
62+ readonly version = '1.0.0';
63+ readonly parserVersion = PARSER_VERSION;
64+ protected override minIntervalMs = 1500;
65+ override readonly urlPatterns = [/^https?:\/\/(www\.)?europeanwatch\.com\/watch\/[a-z0-9-]+/i, /^https?:\/\/(www\.)?crownandcaliber\.com\/products\/[a-z0-9-]+/i];
66+
67+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
68+ const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
69+ const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2);
70+ let count = 0;
71+ for (const seed of seeds) {
72+ let prevFirst = '';
73+ for (let page = 1; page <= pages; page++) {
74+ if (ctx.signal?.aborted || this.reached(ctx, count)) return;
75+ const url = `${BASE}/brand/${seed}${page > 1 ? `?page=${page}` : ''}`;
76+ await this.throttle();
77+ const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price'], parse: (r) => {
78+ const first = r.html ? parseBrandPage(r.html, url, seed, page).items[0] : undefined;
79+ return first ? { title: first.name, price: first.price } : null;
80+ } });
81+ const payload = res.success && res.html ? parseBrandPage(res.html, url, seed, page) : null;
82+ if (!payload || payload.items.length === 0) {
83+ if (page === 1) ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
84+ break;
85+ }
86+ const firstSku = payload.items[0]!.sku ?? payload.items[0]!.url;
87+ if (firstSku === prevFirst) break; // the site serves the whole brand inventory on one page
88+ prevFirst = firstSku;
89+ count++;
90+ yield { url, externalId: `brand:${seed}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
91+ }
92+ }
93+ }
94+
95+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
96+ await this.throttle();
97+ const res = await ctx.fetch(url, { responseType: 'text', minQuality: 0.2 });
98+ if (!res.success || !res.html) return [];
99+ const payload = parseBrandPage(res.html, url, 'lookup', 1);
100+ return payload.items.length ? [{ url, externalId: `product:${payload.items[0]!.sku ?? url}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : [];
101+ }
102+
103+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
104+ const p = PagePayloadSchema.parse(raw.payload);
105+ const out: NormalizedRecord[] = [];
106+ const seen = new Set<string>();
107+ for (const it of p.items) {
108+ const id = it.sku ?? it.url;
109+ if (seen.has(id)) continue;
110+ seen.add(id);
111+ const brand = WATCH_BRANDS.find((b) => it.name.toLowerCase().startsWith(b.toLowerCase())) ?? it.name.split(' ')[0]!;
112+ const categorySlug = watchCategory(brand);
113+ const { reference, model } = splitName(it.name, brand);
114+ const circa = it.name.match(/Circa\.?\s*(\d{4})/i)?.[1] ?? p.details?.year?.match(/\d{4}/)?.[0];
115+ const conditionRaw = p.details?.condition ?? watchConditionRaw(it.name) ?? (it.condition === 'UsedCondition' ? 'Pre-owned' : it.condition === 'NewCondition' ? 'Unworn' : null);
116+ const completeness = p.details ? (/yes/i.test(p.details.box ?? '') && /yes/i.test(p.details.papers ?? '') ? 'full_set' : /yes/i.test(p.details.papers ?? '') ? 'papers_only' : /yes/i.test(p.details.box ?? '') ? 'box_only' : 'watch_only') : watchCompleteness(it.name);
117+ const attributes = AssetAttributesSchema.parse({
118+ categorySlug,
119+ brand,
120+ name: `${brand} ${model ?? ''}`.trim(),
121+ model,
122+ reference,
123+ year: circa ? Number(circa) : null,
124+ material: watchMaterial(it.name),
125+ size: caseSize(it.name),
126+ identifiers: { ...(reference ? { reference } : {}), europeanwatch_sku: id },
127+ metadata: { brand_page: p.url },
128+ });
129+ out.push(
130+ NormalizedListingSchema.parse({
131+ kind: 'listing',
132+ connectorId: this.meta.id,
133+ sourceId: this.meta.sourceId,
134+ sourceUrl: it.url,
135+ externalId: id,
136+ rawTitle: it.name,
137+ imageUrls: it.image ? [it.image] : [],
138+ attributes,
139+ condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness },
140+ observedAt: raw.fetchedAt,
141+ confidence: 0.85,
142+ parserVersion: PARSER_VERSION,
143+ listingType: 'fixed_price',
144+ price: it.price,
145+ currency: currencyOr(it.currency, 'USD'),
146+ seller: 'European Watch Company',
147+ location: 'Boston, US',
148+ availability: it.availability === 'InStock' ? 'available' : it.availability ? 'sold' : 'unknown',
149+ }),
150+ );
151+ }
152+ return out;
153+ }
154+}
155+
156+export default function createConnector(meta: ConnectorMeta) {
157+ return new EuropeanWatchConnector(meta);
158+}
added connectors/api/crown-caliber/meta.json +65 −0
@@ -0,0 +1,65 @@
1+{
2+ "id": "crown-caliber",
3+ "displayName": "European Watch Company (crownandcaliber.com) \u2014 pre-owned watch listings",
4+ "sourceId": "europeanwatch",
5+ "sourceName": "European Watch Company",
6+ "sourceType": "dealer",
7+ "sourceUrl": "https://www.europeanwatch.com",
8+ "module": "api/crown-caliber",
9+ "enginePriority": [
10+ "api",
11+ "firecrawl"
12+ ],
13+ "categories": [
14+ "watches",
15+ "rolex",
16+ "patek_philippe",
17+ "audemars_piguet",
18+ "omega",
19+ "other_watches"
20+ ],
21+ "regions": [
22+ "US"
23+ ],
24+ "languages": [
25+ "en"
26+ ],
27+ "currency": [
28+ "USD"
29+ ],
30+ "supportsListings": true,
31+ "supportsSold": false,
32+ "supportsAuctions": false,
33+ "supportsImages": true,
34+ "supportsCatalog": false,
35+ "supportsPopulation": false,
36+ "supportsLookup": true,
37+ "refreshFrequencyMinutes": 720,
38+ "priority": "medium",
39+ "trustScore": 0.8,
40+ "attributionRequired": true,
41+ "termsUrl": "https://www.europeanwatch.com/terms",
42+ "accessNotes": "crownandcaliber.com now redirects to europeanwatch.com (Boston dealer). Public brand pages (/brand/<slug>?page=N) embed a schema.org ItemList of Products (name, sku, USD price, availability, condition) \u2014 fetched over plain HTTPS with the RareIndex user agent; robots.txt only disallows /login. References and circa years are parsed from the product name; box/papers/condition details live on product pages and are only fetched by lookup(). 1.5 s between pages, the whole brand inventory is served on one page (~130 watches for Rolex; pagination parameters are ignored, which the crawler detects).",
43+ "enabled": true,
44+ "schemaVersion": "1.0",
45+ "config": {
46+ "seeds": [
47+ "rolex",
48+ "patek-philippe",
49+ "audemars-piguet",
50+ "omega",
51+ "cartier",
52+ "tudor",
53+ "vacheron-constantin",
54+ "a-lange-and-sohne",
55+ "iwc",
56+ "jaeger-lecoultre",
57+ "breitling",
58+ "panerai",
59+ "grand-seiko",
60+ "f-p-journe",
61+ "zenith"
62+ ],
63+ "pagesPerSeed": 1
64+ }
65+}
added connectors/api/fashionphile/index.test.ts +35 −0
@@ -0,0 +1,35 @@
1+import { describe, expect, it } from 'vitest';
2+import { ConnectorMetaSchema } from '@rareindex/connectors';
3+import metaJson from './meta.json' with { type: 'json' };
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector from './index.js';
6+
7+// meta.json is read directly so the suite does not depend on connectors/registry.json being regenerated.
8+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
9+
10+describe('fashionphile', () => {
11+ runFixtureSuite(connector, it, expect);
12+
13+ it('normalises the first fixture into listing records with the expected fields', async () => {
14+ const [name] = listFixtures('fashionphile');
15+ const fx = loadFixture('fashionphile', name!);
16+ const out = await connector.normalize(fx.raw);
17+ expect(out.length).toBeGreaterThan(0);
18+ const l = out.find((r) => r.kind === 'listing');
19+ if (l?.kind !== 'listing') throw new Error('no listing');
20+ expect(l.currency).toBe('USD');
21+ expect(l.price).toBeGreaterThan(0);
22+ expect(l.seller).toBe('FASHIONPHILE');
23+ expect(['luxury_handbags', 'fashion_streetwear', 'jewelry', 'rolex', 'other_watches']).toContain(l.attributes.categorySlug);
24+ expect(l.attributes.brand).toBeTruthy();
25+ expect(l.attributes.identifiers.fashionphile_sku).toBeTruthy();
26+ expect(new Set(out.map((r) => (r as { availability?: string }).availability))).toBeTruthy();
27+ });
28+ it('maps sold-out published products to availability=sold and classifies shoes as fashion', async () => {
29+ const { categoryFor } = await import('./index.js');
30+ expect(categoryFor('Shoes', 'Hermes', 'Calfskin Womens Mage Sandals 39 Black')).toBe('fashion_streetwear');
31+ expect(categoryFor('', 'Hermes', 'Calfskin Womens Mage Sandals 39 Black')).toBe('fashion_streetwear');
32+ expect(categoryFor('Bags', 'Hermes', 'Togo Birkin 30 Gold')).toBe('luxury_handbags');
33+ expect(categoryFor('Watches', 'Rolex', 'Stainless Steel Datejust 36 Watch')).toBe('rolex');
34+ });
35+});
added connectors/api/fashionphile/index.ts +129 −0
@@ -0,0 +1,129 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { normalizeCondition } from '@rareindex/taxonomy';
4+import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
5+import { ShopifyProductSchema, bagHardware, bagMaterial, bagModel, bagSize, caseSize, fetchShopifyPage, moneyNumber, trimShopifyProduct, watchCategory, watchMaterial, watchReferenceFromText, yearFrom } from '../_luxury-lib/index.js';
6+
7+/**
8+ * FASHIONPHILE — one Shopify product = one authenticated pre-owned item. Listings (asking prices in
9+ * USD) plus sold state (`available:false` on a published product = the site shows "SOLD").
10+ */
11+const BASE = 'https://www.fashionphile.com';
12+const PARSER_VERSION = '1.0.0';
13+
14+export const PagePayloadSchema = z.object({ kind: z.literal('shopify_page'), url: z.string(), seed: z.string(), page: z.number(), products: z.array(ShopifyProductSchema) });
15+export type PagePayload = z.infer<typeof PagePayloadSchema>;
16+
17+export function categoryFor(productType: string | null | undefined, vendor: string | null | undefined, title: string): string {
18+ const t = (productType ?? '').toLowerCase();
19+ const l = title.toLowerCase();
20+ if (/watch/.test(t) || /\bwatch\b/.test(l)) return watchCategory(vendor);
21+ if (/jewel|ring|bracelet|necklace|earring|brooch|pendant/.test(t)) return 'jewelry';
22+ if (/bag|wallet|clutch|tote|backpack|pouch|luggage|small leather/.test(t)) return 'luxury_handbags';
23+ if (/shoe|sneaker|boot|sandal|heel|pump|loafer|apparel|clothing|scarf|belt|hat|sunglass|jacket|coat|dress/.test(t)) return 'fashion_streetwear';
24+ if (/\b(sandals?|pumps?|loafers?|sneakers?|boots?|heels?|mules?|flats?|espadrilles?|slides?)\b/.test(l) || /\b(muffler|scarf|shawl|stole|belt|sunglasses|hat|cap|jacket|coat|dress|sweater|cardigan|t-shirt|shirt)\b/.test(l)) return 'fashion_streetwear';
25+ return 'luxury_handbags';
26+}
27+
28+export class FashionphileConnector extends BaseConnector {
29+ readonly version = '1.0.0';
30+ readonly parserVersion = PARSER_VERSION;
31+ protected override minIntervalMs = 1500;
32+ override readonly urlPatterns = [/^https?:\/\/(www\.)?fashionphile\.com\/products\/([a-z0-9-]+)/i];
33+
34+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
35+ const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
36+ const pages = ctx.options.mode === 'backfill' ? 20 : Number(this.meta.config.pagesPerSeed ?? 2);
37+ let count = 0;
38+ for (const seed of seeds) {
39+ for (let page = 1; page <= pages; page++) {
40+ if (ctx.signal?.aborted || this.reached(ctx, count)) return;
41+ await this.throttle();
42+ const { products, res } = await fetchShopifyPage(ctx, BASE, `/collections/${seed}/products.json`, page);
43+ if (!res.success) {
44+ ctx.anomaly('page_fetch_failed', `${seed} p${page}: ${res.error ?? res.httpStatus}`);
45+ break;
46+ }
47+ if (products.length === 0) break;
48+ count++;
49+ const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}/collections/${seed}/products.json?page=${page}`, seed, page, products: products.map(trimShopifyProduct) };
50+ yield { url: payload.url, externalId: `collection:${seed}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
51+ if (products.length < 250) break;
52+ }
53+ }
54+ }
55+
56+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
57+ const handle = url.match(this.urlPatterns[0]!)?.[2];
58+ if (!handle) return [];
59+ await this.throttle();
60+ const res = await ctx.fetch(`${BASE}/products/${handle}.json`, { engines: ['api'], responseType: 'json', minQuality: 0.2 });
61+ const parsed = z.object({ product: ShopifyProductSchema }).safeParse(res.json);
62+ if (!res.success || !parsed.success) return [];
63+ const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}/products/${handle}`, seed: `product:${handle}`, page: 1, products: [trimShopifyProduct(parsed.data.product)] };
64+ return [{ url: payload.url, externalId: `product:${handle}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
65+ }
66+
67+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
68+ const p = PagePayloadSchema.parse(raw.payload);
69+ const out: NormalizedRecord[] = [];
70+ for (const pr of p.products) {
71+ const v = pr.variants[0];
72+ if (!v) continue;
73+ const price = moneyNumber(v.price);
74+ if (!price) continue;
75+ const brand = pr.vendor ?? null;
76+ const categorySlug = categoryFor(pr.product_type, brand, pr.title);
77+ const isWatch = categorySlug.endsWith('watches') || categorySlug === 'rolex' || categorySlug === 'patek_philippe' || categorySlug === 'audemars_piguet' || categorySlug === 'omega';
78+ const title = pr.title.replace(/\s+/g, ' ').trim();
79+ const ref = isWatch ? watchReferenceFromText(title) : null;
80+ const model = isWatch ? null : bagModel(title);
81+ const attributes = AssetAttributesSchema.parse({
82+ categorySlug,
83+ brand,
84+ name: brand ? `${brand} ${title}` : title,
85+ model,
86+ reference: ref,
87+ year: yearFrom(pr.body_html ?? '') ?? null,
88+ material: isWatch ? watchMaterial(title) : bagMaterial(title),
89+ size: isWatch ? caseSize(title) : bagSize(title),
90+ color: null,
91+ identifiers: { fashionphile_sku: v.sku ?? String(pr.id), ...(ref ? { reference: ref } : {}) },
92+ metadata: { product_type: pr.product_type, hardware: bagHardware(title), tags: pr.tags?.slice(0, 12) ?? [], compare_at_price: moneyNumber(v.compare_at_price) },
93+ });
94+ const sourceUrl = `${BASE}/products/${pr.handle}`;
95+ const availability = pr.published_at ? (v.available ? 'available' : 'sold') : 'removed';
96+ const listedAt = pr.published_at ? new Date(pr.published_at) : null;
97+ out.push(
98+ NormalizedListingSchema.parse({
99+ kind: 'listing',
100+ connectorId: this.meta.id,
101+ sourceId: this.meta.sourceId,
102+ sourceUrl,
103+ externalId: String(pr.id),
104+ rawTitle: title,
105+ description: pr.body_html ?? null,
106+ imageUrls: (pr.images ?? []).map((i) => i.src),
107+ attributes,
108+ condition: { condition: normalizeCondition(categorySlug, null), conditionRaw: null, completeness: null },
109+ observedAt: raw.fetchedAt,
110+ confidence: 0.85,
111+ parserVersion: PARSER_VERSION,
112+ listingType: 'fixed_price',
113+ price,
114+ currency: 'USD',
115+ seller: 'FASHIONPHILE',
116+ location: 'US',
117+ quantity: 1,
118+ listedAt: listedAt && !Number.isNaN(listedAt.getTime()) ? listedAt : null,
119+ availability,
120+ }),
121+ );
122+ }
123+ return out;
124+ }
125+}
126+
127+export default function createConnector(meta: ConnectorMeta) {
128+ return new FashionphileConnector(meta);
129+}
added connectors/api/fashionphile/meta.json +33 −0
@@ -0,0 +1,33 @@
1+{
2+ "id": "fashionphile",
3+ "displayName": "FASHIONPHILE (pre-owned luxury listings & sold items)",
4+ "sourceId": "fashionphile",
5+ "sourceName": "FASHIONPHILE",
6+ "sourceType": "marketplace",
7+ "sourceUrl": "https://www.fashionphile.com",
8+ "module": "api/fashionphile",
9+ "enginePriority": ["api"],
10+ "categories": ["luxury_handbags", "watches", "other_watches", "rolex", "jewelry", "fashion_streetwear"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": true,
15+ "supportsSold": false,
16+ "supportsAuctions": false,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": true,
21+ "refreshFrequencyMinutes": 720,
22+ "priority": "medium",
23+ "trustScore": 0.8,
24+ "attributionRequired": true,
25+ "termsUrl": "https://www.fashionphile.com/terms-of-use",
26+ "accessNotes": "Public Shopify storefront feed (www.fashionphile.com/collections/<brand>/products.json) fetched over plain HTTPS with the RareIndex user agent; robots.txt allows /collections/ (only sort/filter permutations, cart, account and checkout paths are disallowed). Each product is one authenticated pre-owned item priced in USD; `available:false` on a still-published product means the item shows SOLD on the site, which is recorded as a listing with availability=sold (the last asking price is not asserted as the transaction price). Condition ratings are only on HTML product pages and are not fetched. 1.5 s between requests, 250 products per page.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "seeds": ["hermes", "chanel", "louis-vuitton", "dior", "gucci", "goyard", "fendi", "bottega-veneta", "prada", "celine", "loewe", "saint-laurent", "rolex", "cartier", "van-cleef-arpels"],
31+ "pagesPerSeed": 2
32+ }
33+}
added connectors/api/laced/index.test.ts +29 −0
@@ -0,0 +1,29 @@
1+import { describe, expect, it } from 'vitest';
2+import { ConnectorMetaSchema } from '@rareindex/connectors';
3+import metaJson from './meta.json' with { type: 'json' };
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector from './index.js';
6+
7+// meta.json is read directly so the suite does not depend on connectors/registry.json being regenerated.
8+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
9+
10+describe('laced', () => {
11+ runFixtureSuite(connector, it, expect);
12+
13+ it('normalises the first fixture into catalog_item records with the expected fields', async () => {
14+ const [name] = listFixtures('laced');
15+ const fx = loadFixture('laced', name!);
16+ const out = await connector.normalize(fx.raw);
17+ expect(out.length).toBeGreaterThan(0);
18+ const cat = out.find((r) => r.kind === 'catalog_item');
19+ if (cat?.kind !== 'catalog_item') throw new Error('no catalog item');
20+ expect(cat.attributes.identifiers.style_code).toMatch(/^[A-Z]{1,2}\d{4,5}-\d{3}$/);
21+ expect(cat.attributes.categorySlug).toBe('nike_jordan');
22+ const asks = out.filter((r) => r.kind === 'listing');
23+ expect(asks.length).toBeGreaterThan(3);
24+ for (const a of asks) if (a.kind === 'listing') { expect(a.currency).toBe('GBP'); expect(a.attributes.size).toMatch(/^UK /); }
25+ const low = out.find((r) => r.kind === 'price_observation');
26+ if (low?.kind === 'price_observation') expect(low.price).toBeLessThanOrEqual(Math.min(...asks.map((a) => (a as { price: number }).price)));
27+ });
28+
29+});
added connectors/api/laced/index.ts +143 −0
@@ -0,0 +1,143 @@
1+import { z } from 'zod';
2+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';
4+import { currencyOr, moneyNumber, sneakerCategory, styleCodeFromText } from '../_luxury-lib/index.js';
5+
6+/** Laced — ProductGroup JSON-LD on product pages: style code + per-size lowest asks (GBP). */
7+const BASE = 'https://www.laced.com';
8+const PARSER_VERSION = '1.0.0';
9+
10+export const SizeOfferSchema = z.object({ size: z.string(), price: z.number(), currency: z.string(), available: z.boolean() });
11+export const ProductPayloadSchema = z.object({
12+ kind: z.literal('product_page'),
13+ url: z.string(),
14+ slug: z.string(),
15+ name: z.string(),
16+ sku: z.string().nullable(),
17+ brand: z.string().nullable(),
18+ images: z.array(z.string()),
19+ description: z.string().nullable(),
20+ lowPrice: z.number().nullable(),
21+ highPrice: z.number().nullable(),
22+ currency: z.string(),
23+ sizes: z.array(SizeOfferSchema),
24+});
25+export type ProductPayload = z.infer<typeof ProductPayloadSchema>;
26+
27+export function parseProductPage(htmlText: string, url: string): ProductPayload | null {
28+ const group = H.jsonLd(htmlText, 'ProductGroup')[0] ?? H.jsonLd(htmlText, 'Product')[0];
29+ if (!group) return null;
30+ const agg = (group.offers as Record<string, unknown> | undefined) ?? {};
31+ const sizes: z.infer<typeof SizeOfferSchema>[] = [];
32+ for (const v of (group.hasVariant as Array<Record<string, unknown>>) ?? []) {
33+ const off = (v.offers as Record<string, unknown> | undefined) ?? {};
34+ const price = moneyNumber(off.price as string | number | undefined);
35+ if (!price) continue;
36+ sizes.push({ size: String(v.size ?? '').trim(), price, currency: String(off.priceCurrency ?? agg.priceCurrency ?? 'GBP'), available: !/OutOfStock|SoldOut/.test(String(off.availability ?? '')) });
37+ }
38+ const brand = group.brand && typeof group.brand === 'object' ? String((group.brand as { name?: string }).name ?? '') : group.brand ? String(group.brand) : null;
39+ const img = group.image;
40+ const slug = url.match(/\/products\/([a-z0-9-]+)/i)?.[1] ?? url;
41+ return {
42+ kind: 'product_page',
43+ url: `${BASE}/products/${slug}`,
44+ slug,
45+ name: String(group.name ?? '').replace(/\s+/g, ' ').trim(),
46+ sku: group.sku ? String(group.sku) : null,
47+ brand: brand || null,
48+ images: Array.isArray(img) ? img.slice(0, 3).map(String) : typeof img === 'string' ? [img] : [],
49+ description: group.description ? String(group.description).slice(0, 500) : null,
50+ lowPrice: moneyNumber(agg.lowPrice as string | number | undefined),
51+ highPrice: moneyNumber(agg.highPrice as string | number | undefined),
52+ currency: String(agg.priceCurrency ?? sizes[0]?.currency ?? 'GBP'),
53+ sizes,
54+ };
55+}
56+
57+export function parseBrandPage(htmlText: string): string[] {
58+ return [...new Set([...htmlText.matchAll(/href="\/products\/([a-z0-9-]+)"/g)].map((m) => m[1]!))];
59+}
60+
61+export class LacedConnector extends BaseConnector {
62+ readonly version = '1.0.0';
63+ readonly parserVersion = PARSER_VERSION;
64+ protected override minIntervalMs = 1500;
65+ override readonly urlPatterns = [/^https?:\/\/(www\.)?laced\.com\/(?:[a-z]{2}\/)?products\/([a-z0-9-]+)/i];
66+
67+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
68+ const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
69+ const perSeed = Number(this.meta.config.productsPerSeed ?? 24);
70+ let count = 0;
71+ for (const seed of seeds) {
72+ await this.throttle();
73+ const list = await ctx.fetch(`${BASE}/${seed}`, { responseType: 'text', minQuality: 0.2 });
74+ if (!list.success || !list.html) {
75+ ctx.anomaly('page_fetch_failed', `${seed}: ${list.error ?? list.httpStatus}`);
76+ continue;
77+ }
78+ const slugs = parseBrandPage(list.html).slice(0, perSeed);
79+ if (!slugs.length) ctx.anomaly('empty_page', seed);
80+ for (const slug of slugs) {
81+ if (ctx.signal?.aborted || this.reached(ctx, count)) return;
82+ const url = `${BASE}/products/${slug}`;
83+ if (!(await ctx.shouldFetch(url))) continue;
84+ const rec = await this.fetchProduct(url, ctx);
85+ if (rec) {
86+ count++;
87+ yield rec;
88+ }
89+ }
90+ }
91+ }
92+
93+ private async fetchProduct(url: string, ctx: CrawlContext): Promise<RawRecordInput | null> {
94+ await this.throttle();
95+ const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => {
96+ const p = r.html ? parseProductPage(r.html, url) : null;
97+ return p ? { title: p.name, price: p.lowPrice ?? p.sizes[0]?.price, identifiers: p.sku ? { sku: p.sku } : null } : null;
98+ } });
99+ const payload = res.success && res.html ? parseProductPage(res.html, url) : null;
100+ if (!payload) {
101+ if (res.httpStatus !== 404) ctx.anomaly(res.success ? 'parse_failure' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
102+ return null;
103+ }
104+ return { url: payload.url, externalId: payload.slug, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
105+ }
106+
107+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
108+ const slug = url.match(this.urlPatterns[0]!)?.[2];
109+ if (!slug) return [];
110+ const rec = await this.fetchProduct(`${BASE}/products/${slug}`, ctx);
111+ return rec ? [rec] : [];
112+ }
113+
114+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
115+ const p = ProductPayloadSchema.parse(raw.payload);
116+ const styleCode = p.sku ?? styleCodeFromText(p.name);
117+ const year = p.name.match(/\((\d{4})\)/)?.[1];
118+ const attributes = AssetAttributesSchema.parse({
119+ categorySlug: sneakerCategory(p.brand, p.name),
120+ brand: p.brand,
121+ name: p.name.replace(/\s*\(\d{4}\)\s*$/, '').trim(),
122+ year: year ? Number(year) : null,
123+ identifiers: { ...(styleCode ? { style_code: styleCode } : {}), laced_slug: p.slug },
124+ metadata: { sizes_listed: p.sizes.length },
125+ });
126+ const currency = currencyOr(p.currency, 'GBP');
127+ const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: p.url, rawTitle: p.name, description: p.description, imageUrls: p.images, attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
128+ const cond = { condition: 'new', conditionRaw: 'Brand new (deadstock marketplace)', completeness: 'with_box' };
129+ const out: NormalizedRecord[] = [NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${p.slug}`, confidence: 0.9 })];
130+ const low = p.lowPrice ?? (p.sizes.length ? Math.min(...p.sizes.map((s) => s.price)) : null);
131+ if (low) out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `product:${p.slug}:low`, confidence: 0.75, condition: cond, priceKind: 'low', price: low, currency, observationDate: raw.fetchedAt, sampleSize: p.sizes.length || null }));
132+ for (const s of p.sizes) {
133+ if (!s.available) continue;
134+ const uk = s.size.match(/UK\s*([\d.]+)/i)?.[1] ?? s.size.split('|')[0]?.trim() ?? s.size;
135+ out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, attributes: { ...attributes, size: `UK ${uk}` }, externalId: `product:${p.slug}:size:${uk}`, confidence: 0.75, condition: cond, listingType: 'ask', price: s.price, currency: currencyOr(s.currency, currency), seller: 'Laced marketplace', location: 'GB', availability: 'available' }));
136+ }
137+ return out;
138+ }
139+}
140+
141+export default function createConnector(meta: ConnectorMeta) {
142+ return new LacedConnector(meta);
143+}
added connectors/api/laced/meta.json +33 −0
@@ -0,0 +1,33 @@
1+{
2+ "id": "laced",
3+ "displayName": "Laced (UK sneaker marketplace — asks per size)",
4+ "sourceId": "laced",
5+ "sourceName": "Laced",
6+ "sourceType": "marketplace",
7+ "sourceUrl": "https://www.laced.com",
8+ "module": "api/laced",
9+ "enginePriority": ["api", "firecrawl"],
10+ "categories": ["sneakers", "nike_jordan", "adidas_yeezy", "new_balance_asics_other"],
11+ "regions": ["GB", "EU"],
12+ "languages": ["en"],
13+ "currency": ["GBP"],
14+ "supportsListings": true,
15+ "supportsSold": false,
16+ "supportsAuctions": false,
17+ "supportsImages": true,
18+ "supportsCatalog": true,
19+ "supportsPopulation": false,
20+ "supportsLookup": true,
21+ "refreshFrequencyMinutes": 720,
22+ "priority": "medium",
23+ "trustScore": 0.75,
24+ "attributionRequired": true,
25+ "termsUrl": "https://www.laced.com/pages/terms-and-conditions",
26+ "accessNotes": "Public brand pages (laced.com/<brand>) list product slugs; each product page embeds a schema.org ProductGroup with the style code (sku), brand and one Product per size carrying the current lowest ask in GBP. Plain HTTPS with the RareIndex user agent; robots.txt allows everything except account/admin. New-condition marketplace (deadstock). 1.5 s between pages; productsPerSeed caps product-page fetches per run.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "seeds": ["air-jordan", "nike", "adidas", "yeezy", "new-balance", "asics"],
31+ "productsPerSeed": 24
32+ }
33+}
added connectors/api/rebag/index.test.ts +31 −0
@@ -0,0 +1,31 @@
1+import { describe, expect, it } from 'vitest';
2+import { ConnectorMetaSchema } from '@rareindex/connectors';
3+import metaJson from './meta.json' with { type: 'json' };
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector from './index.js';
6+
7+// meta.json is read directly so the suite does not depend on connectors/registry.json being regenerated.
8+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
9+
10+describe('rebag', () => {
11+ runFixtureSuite(connector, it, expect);
12+
13+ it('normalises the first fixture into listing records with the expected fields', async () => {
14+ const [name] = listFixtures('rebag');
15+ const fx = loadFixture('rebag', name!);
16+ const out = await connector.normalize(fx.raw);
17+ expect(out.length).toBeGreaterThan(0);
18+ const l = out.find((r) => r.kind === 'listing' && r.attributes.categorySlug === 'luxury_handbags');
19+ if (l?.kind !== 'listing') throw new Error('no handbag listing');
20+ expect(l.currency).toBe('USD');
21+ expect(l.condition.conditionRaw).toBeTruthy();
22+ expect(l.attributes.identifiers.rebag_item).toBeTruthy();
23+ });
24+ it('parses Rebag variant titles and retail price', async () => {
25+ const { parseVariantTitle, retailFromBody, categoryFor } = await import('./index.js');
26+ expect(parseVariantTitle('Great | Item # 417505/4 / Blue')).toEqual({ grade: 'Great', color: 'Blue' });
27+ expect(retailFromBody('<b>Estimated Retail Price:</b> $4,850<br>')).toBe(4850);
28+ expect(categoryFor('Top Handle Bag', 'Hermes', [])).toBe('luxury_handbags');
29+ expect(categoryFor('Dress', 'Hermes', [])).toBe('fashion_streetwear');
30+ });
31+});
added connectors/api/rebag/index.ts +145 −0
@@ -0,0 +1,145 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { normalizeCondition } from '@rareindex/taxonomy';
4+import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
5+import { ShopifyProductSchema, bagHardware, bagMaterial, bagModel, bagSize, caseSize, fetchShopifyPage, moneyNumber, trimShopifyProduct, watchCategory, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js';
6+
7+/** Rebag — Shopify feed of individually graded pre-owned luxury items (USD). */
8+const BASE = 'https://shop.rebag.com';
9+const PARSER_VERSION = '1.0.0';
10+
11+export const PagePayloadSchema = z.object({ kind: z.literal('shopify_page'), url: z.string(), seed: z.string(), page: z.number(), products: z.array(ShopifyProductSchema) });
12+export type PagePayload = z.infer<typeof PagePayloadSchema>;
13+
14+const CONDITION_MAP: Record<string, string> = { pristine: 'pristine', excellent: 'excellent', great: 'very_good', 'very good': 'very_good', good: 'good', fair: 'fair', new: 'new' };
15+
16+export function categoryFor(productType: string | null | undefined, vendor: string | null | undefined, tags: string[]): string {
17+ const t = (productType ?? '').toLowerCase();
18+ const tagStr = tags.join(' ').toLowerCase();
19+ if (/watch/.test(t) || /bc-filter-watches/.test(tagStr)) return watchCategory(vendor);
20+ if (/jewel|ring|bracelet|necklace|earring|brooch|pendant|charm/.test(t)) return 'jewelry';
21+ if (/bag|tote|clutch|satchel|backpack|wallet|pouch|luggage|handle|hobo|crossbody|shoulder|belt bag|case|small leather/.test(t)) return 'luxury_handbags';
22+ if (/shoe|sneaker|boot|sandal|heel|pump|loafer|apparel|clothing|scarf|shawl|belt|hat|sunglass|dress|jacket|coat|\btop\b|pant|skirt|sweater/.test(t)) return 'fashion_streetwear';
23+ return 'luxury_handbags';
24+}
25+
26+/** "Great | Item # 417505/4 / Blue" → { grade: 'Great', color: 'Blue' } */
27+export function parseVariantTitle(title: string): { grade: string | null; color: string | null } {
28+ const parts = title.split('|').map((s) => s.trim());
29+ const grade = parts[0] && !/item/i.test(parts[0]) ? parts[0] : null;
30+ const colorPart = parts[parts.length - 1] ?? '';
31+ const color = colorPart.includes('/') ? (colorPart.split('/').pop()?.trim() ?? null) : null;
32+ return { grade, color: color || null };
33+}
34+
35+export function retailFromBody(body: string | null | undefined): number | null {
36+ const m = (body ?? '').replace(/<[^>]+>/g, ' ').match(/Estimated Retail Price:\s*\$?\s*([0-9,]+(?:\.\d+)?)/i);
37+ return m ? moneyNumber(m[1]) : null;
38+}
39+
40+export class RebagConnector extends BaseConnector {
41+ readonly version = '1.0.0';
42+ readonly parserVersion = PARSER_VERSION;
43+ protected override minIntervalMs = 1500;
44+ override readonly urlPatterns = [/^https?:\/\/(shop\.|www\.)?rebag\.com\/products\/([a-z0-9-]+)/i];
45+
46+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
47+ const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];
48+ const pages = ctx.options.mode === 'backfill' ? 20 : Number(this.meta.config.pagesPerSeed ?? 2);
49+ let count = 0;
50+ for (const seed of seeds) {
51+ for (let page = 1; page <= pages; page++) {
52+ if (ctx.signal?.aborted || this.reached(ctx, count)) return;
53+ await this.throttle();
54+ const { products, res } = await fetchShopifyPage(ctx, BASE, `/collections/${seed}/products.json`, page);
55+ if (!res.success) {
56+ ctx.anomaly('page_fetch_failed', `${seed} p${page}: ${res.error ?? res.httpStatus}`);
57+ break;
58+ }
59+ if (products.length === 0) break;
60+ count++;
61+ const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}/collections/${seed}/products.json?page=${page}`, seed, page, products: products.map(trimShopifyProduct) };
62+ yield { url: payload.url, externalId: `collection:${seed}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
63+ if (products.length < 250) break;
64+ }
65+ }
66+ }
67+
68+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
69+ const handle = url.match(this.urlPatterns[0]!)?.[2];
70+ if (!handle) return [];
71+ await this.throttle();
72+ const res = await ctx.fetch(`${BASE}/products/${handle}.json`, { engines: ['api'], responseType: 'json', minQuality: 0.2 });
73+ const parsed = z.object({ product: ShopifyProductSchema }).safeParse(res.json);
74+ if (!res.success || !parsed.success) return [];
75+ const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}/products/${handle}`, seed: `product:${handle}`, page: 1, products: [trimShopifyProduct(parsed.data.product)] };
76+ return [{ url: payload.url, externalId: `product:${handle}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
77+ }
78+
79+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
80+ const p = PagePayloadSchema.parse(raw.payload);
81+ const out: NormalizedRecord[] = [];
82+ for (const pr of p.products) {
83+ const v = pr.variants[0];
84+ if (!v) continue;
85+ const price = moneyNumber(v.price);
86+ if (!price) continue;
87+ const brand = pr.vendor ?? null;
88+ const tags = pr.tags ?? [];
89+ const categorySlug = categoryFor(pr.product_type, brand, tags);
90+ const isWatch = ['rolex', 'patek_philippe', 'audemars_piguet', 'omega', 'other_watches'].includes(categorySlug);
91+ const title = pr.title.replace(/\s+/g, ' ').trim();
92+ const { grade, color } = parseVariantTitle(v.title);
93+ const conditionRaw = grade;
94+ const condition = grade ? (CONDITION_MAP[grade.toLowerCase()] ?? normalizeCondition(categorySlug, grade)) : null;
95+ const ref = isWatch ? watchReferenceFromText(`${title} ${pr.body_html ?? ''}`) : null;
96+ const retail = retailFromBody(pr.body_html);
97+ const attributes = AssetAttributesSchema.parse({
98+ categorySlug,
99+ brand,
100+ name: brand ? `${brand} ${title}` : title,
101+ model: isWatch ? null : bagModel(title),
102+ reference: ref,
103+ material: isWatch ? watchMaterial(title) : bagMaterial(title),
104+ size: isWatch ? caseSize(title) : bagSize(title),
105+ color,
106+ originalMsrp: retail,
107+ originalMsrpCurrency: retail ? 'USD' : null,
108+ identifiers: { rebag_item: v.sku ?? String(pr.id), ...(ref ? { reference: ref } : {}) },
109+ metadata: { product_type: pr.product_type, hardware: bagHardware(title), filters: tags.filter((t) => t.startsWith('bc-filter-')).map((t) => t.slice(10)).slice(0, 15) },
110+ });
111+ const accessories = (pr.body_html ?? '').match(/Accessories:\s*([^.]{0,80})/i)?.[1]?.trim() ?? null;
112+ const completeness = accessories ? (/no accessories/i.test(accessories) ? 'item_only' : /box/i.test(accessories) && /dust ?bag|card|receipt/i.test(accessories) ? 'full_set' : /dust ?bag/i.test(accessories) ? 'dust_bag' : /box/i.test(accessories) ? 'box' : null) : null;
113+ out.push(
114+ NormalizedListingSchema.parse({
115+ kind: 'listing',
116+ connectorId: this.meta.id,
117+ sourceId: this.meta.sourceId,
118+ sourceUrl: `${BASE}/products/${pr.handle}`,
119+ externalId: String(pr.id),
120+ rawTitle: title,
121+ description: pr.body_html ?? null,
122+ imageUrls: (pr.images ?? []).map((i) => i.src),
123+ attributes,
124+ condition: { condition, conditionRaw, completeness },
125+ observedAt: raw.fetchedAt,
126+ confidence: 0.85,
127+ parserVersion: PARSER_VERSION,
128+ listingType: 'fixed_price',
129+ price,
130+ currency: 'USD',
131+ seller: 'Rebag',
132+ location: 'US',
133+ quantity: 1,
134+ listedAt: pr.published_at ? new Date(pr.published_at) : null,
135+ availability: v.available ? 'available' : 'sold',
136+ }),
137+ );
138+ }
139+ return out;
140+ }
141+}
142+
143+export default function createConnector(meta: ConnectorMeta) {
144+ return new RebagConnector(meta);
145+}
added connectors/api/rebag/meta.json +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added connectors/api/subdial/index.test.ts +27 −0
Large diff (27 lines) is not loaded by default to keep the page fast.Load diff
added connectors/api/subdial/index.ts +160 −0
Large diff (160 lines) is not loaded by default to keep the page fast.Load diff
added connectors/api/subdial/meta.json +32 −0
Large diff (32 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/flight-club/index.test.ts +26 −0
Large diff (26 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/flight-club/index.ts +96 −0
Large diff (96 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/flight-club/meta.json +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/hypeboost/index.test.ts +26 −0
Large diff (26 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/hypeboost/index.ts +106 −0
Large diff (106 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/hypeboost/meta.json +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/watchfinder/index.test.ts +30 −0
Large diff (30 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/watchfinder/index.ts +132 −0
Large diff (132 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/watchfinder/meta.json +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/antiquorum/monaco-june-2026-1.json +432 −0
Large diff (433 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/antiquorum/monaco-june-2026-2.json +432 −0
Large diff (433 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/bobs-watches/model-rolex-daytona.json +564 −0
Large diff (565 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/bobs-watches/model-rolex-submariner.json +564 −0
Large diff (565 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/crown-caliber/brand-rolex-1.json +1335 −0
Large diff (1,336 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/fashionphile/collection-hermes-1.json +865 −0
Large diff (866 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/flight-club/search-air-jordans-1.json +299 −0
Large diff (300 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/hypeboost/grid-air-jordan-1.json +423 −0
Large diff (424 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/hypeboost/grid-nike-1.json +423 −0
Large diff (424 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/laced/air-jordan-1-retro-low-og-sp-travis-scott-velvet-brown.json +164 −0
Large diff (165 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/laced/air-jordan-4-retro-black-cat-2025-gs.json +86 −0
Large diff (87 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/laced/air-jordan-4-retro-black-cat-2025.json +134 −0
Large diff (135 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/rebag/collection-hermes-1.json +1625 −0
Large diff (1,626 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/subdial/sd092678.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/subdial/sd092916.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/subdial/sd093236.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/watchfinder/model-rolex-daytona-1.json +614 −0
Large diff (615 lines) is not loaded by default to keep the page fast.Load diff
modified packages/connectors/src/engines/firecrawl.ts +3 −2
Large diff (18 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere