Connector layer expansion: SDK (capabilities, domain policies, circuit breaker, adapters Shopify/WooCommerce/sitemap/RSS/PDF/schema.org), resumable backfills, schema-drift + health metrics, certificates provenance, multi-currency views, source registry (443 sources), connector:new scaffolder, admin explorer + coverage dashboard, docs; +119 connectors (212 total) across cards EU/JP, grading cert lookups, Shopify fleets NA/intl, numismatics, comics/toys/games, auction houses NA + EU/APAC, Asia marketplaces, watches, gated eBay/Etsy/Trade Me/Rakuten/Yahoo/Numista/CardTrader
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
256 changed files +19,785 −62
modified
CLAUDE.md
+1 −1
@@ -183,7 +183,7 @@ One canonical asset graph containing every collectible, every important variatio | ||
| 183 | 183 | |
| 184 | 184 | - Package manager `pnpm`; `pnpm typecheck`, `pnpm test`, `pnpm build`. Workers run with `tsx`. |
| 185 | 185 | - Secrets only in `.env` (git-ignored) or the `mld` manifest on the cluster. Never in code or fixtures. |
| 186 | −- Add a connector: `connectors/<engine>/<id>/{meta.json,index.ts,index.test.ts}` + `data/fixtures/<id>/*.json`, then `pnpm tsx scripts/build-registry.ts` and `pnpm db:seed`. | |
| 186 | +- Add a connector: `pnpm connector:new <id> --url … [--adapter shopify|woocommerce]` → `connectors/<engine>/<id>/{meta.json,index.ts,index.test.ts,README.md}` + real fixtures in `data/fixtures/<id>/`, source entry in `data/sources/entries/<group>.json`, host policy in `connectors/domains.d/<group>.json`; then `pnpm registry`, `pnpm sources:build`, `pnpm db:seed`. Guide: `docs/connectors/ADDING_A_CONNECTOR.md`; catalogue: `docs/connectors/SOURCES.md`. | |
| 187 | 187 | - DB changes: edit `packages/database/src/schema/*`, run `pnpm db:generate`, commit the SQL migration. |
| 188 | 188 | - UI: dense, neutral, tabular numbers (`num` utility), `<Unavailable/>` for missing evidence, every number with its sample size / confidence / freshness when it is an estimate. |
| 189 | 189 | - Member accounts: email + password, email verification code and multi-factor code delivered through Resend (`RESEND_API_KEY`, `EMAIL_FROM`). |
modified
apps/web/src/app/admin/connectors/[id]/page.tsx
+107 −7
@@ -4,6 +4,7 @@ import { requireAdmin } from '@/lib/admin/auth'; | ||
| 4 | 4 | import { connectorDetail } from '@/lib/admin/queries'; |
| 5 | 5 | import { connectorAction, updateConnectorConfig } from '@/lib/admin/actions'; |
| 6 | 6 | import { AdminShell, ActionButton, StatusPill, Kpi, fmtTs, n } from '@/components/admin/shell'; |
| 7 | +import { HealthLabel, SourceLogo, Progress, Chip, Json } from '@/components/admin/connector-bits'; | |
| 7 | 8 | import { Table, th, td, tdNum } from '@/components/ui/primitives'; |
| 8 | 9 | |
| 9 | 10 | export default async function ConnectorInspectPage({ params }: { params: Promise<{ id: string }> }) { |
@@ -14,32 +15,119 @@ export default async function ConnectorInspectPage({ params }: { params: Promise | ||
| 14 | 15 | const c = d.connector; |
| 15 | 16 | const health = (d.health?.health ?? {}) as Record<string, unknown>; |
| 16 | 17 | const outputs = (d.outputs ?? {}) as Record<string, unknown>; |
| 18 | + const meta = (c.meta ?? {}) as Record<string, unknown>; | |
| 19 | + const activeBackfill = d.backfills.find((b) => b.status === 'running' || b.status === 'paused'); | |
| 20 | + const fieldStats = d.fieldStats as Array<{ field: string; day: string; total: number; nulls: number }>; | |
| 21 | + const today = new Date().toISOString().slice(0, 10); | |
| 22 | + const nullRates = Object.values( | |
| 23 | + fieldStats.reduce<Record<string, { field: string; today: [number, number]; base: [number, number] }>>((acc, r) => { | |
| 24 | + const e = (acc[r.field] ??= { field: r.field, today: [0, 0], base: [0, 0] }); | |
| 25 | + const t = r.day === today ? e.today : e.base; | |
| 26 | + t[0] += Number(r.total); | |
| 27 | + t[1] += Number(r.nulls); | |
| 28 | + return acc; | |
| 29 | + }, {}), | |
| 30 | + ); | |
| 31 | + const actions = [ | |
| 32 | + ['probe', 'Test', 'neutral'], | |
| 33 | + ['run', 'Run now', 'neutral'], | |
| 34 | + ['backfill', activeBackfill ? 'Continue backfill' : 'Backfill', 'neutral'], | |
| 35 | + ...(activeBackfill?.status === 'running' ? [['backfill_pause', 'Pause backfill', 'neutral']] : []), | |
| 36 | + ...(activeBackfill ? [['backfill_reset', 'Reset backfill', 'danger']] : []), | |
| 37 | + ['retry', 'Retry from cursor', 'neutral'], | |
| 38 | + c.status === 'active' ? ['pause', 'Pause', 'danger'] : ['resume', 'Resume', 'primary'], | |
| 39 | + c.status === 'maintenance' ? ['resume', 'End maintenance', 'primary'] : ['maintenance', 'Maintenance', 'neutral'], | |
| 40 | + ] as const; | |
| 17 | 41 | return ( |
| 18 | 42 | <AdminShell |
| 19 | 43 | current="/admin/connectors" |
| 20 | 44 | title={String(c.display_name)} |
| 21 | − subtitle={<>{String(c.id)} · source <span className="font-mono">{String(c.source_id)}</span> · {(c.engine_priority as string[]).join(' → ')} · categories {(c.categories as string[]).join(', ')} · <Link href="/admin/connectors" className="underline">back</Link></>} | |
| 45 | + subtitle={ | |
| 46 | + <> | |
| 47 | + <SourceLogo domain={String(meta.domain ?? '')} name={String(c.display_name)} /> {String(c.id)} · source <span className="font-mono">{String(c.source_id)}</span> · {String(meta.domain ?? '')} · {String(meta.country ?? (c.regions as string[])?.[0] ?? 'global')} · {(c.engine_priority as string[]).join(' → ')} · {String(meta.acquisitionMethod ?? '')} · categories {(c.categories as string[]).join(', ')} · <Link href="/admin/connectors" className="underline">back</Link> | |
| 48 | + </> | |
| 49 | + } | |
| 22 | 50 | actions={ |
| 23 | 51 | <div className="flex flex-wrap gap-1"> |
| 24 | − {(['run', 'probe', 'recrawl', 'retry', c.status === 'paused' ? 'resume' : 'pause'] as const).map((a) => ( | |
| 52 | + {actions.map(([a, label, tone]) => ( | |
| 25 | 53 | <form key={a} action={connectorAction}> |
| 26 | 54 | <input type="hidden" name="id" value={String(c.id)} /> |
| 27 | 55 | <input type="hidden" name="action" value={a} /> |
| 28 | − <ActionButton label={a} small={false} tone={a === 'pause' ? 'danger' : a === 'resume' ? 'primary' : 'neutral'} /> | |
| 56 | + <ActionButton label={label} small={false} tone={tone as 'neutral' | 'danger' | 'primary'} /> | |
| 29 | 57 | </form> |
| 30 | 58 | ))} |
| 31 | 59 | </div> |
| 32 | 60 | } |
| 33 | 61 | > |
| 62 | + <div className="mb-3 flex flex-wrap gap-1"> | |
| 63 | + {((meta.capabilities as string[]) ?? []).map((cap) => ( | |
| 64 | + <Chip key={cap} tone="index">{cap}</Chip> | |
| 65 | + ))} | |
| 66 | + {((meta.requires as string[]) ?? []).map((r) => ( | |
| 67 | + <Chip key={r} tone="alert">requires {r}</Chip> | |
| 68 | + ))} | |
| 69 | + <Chip>refresh {String(meta.refreshClass ?? '')} · {n(c.refresh_frequency_minutes)} min</Chip> | |
| 70 | + <Chip>history {String(meta.historicalDepth ?? 'none')}</Chip> | |
| 71 | + {meta.termsUrl ? ( | |
| 72 | + <a href={String(meta.termsUrl)} target="_blank" rel="noreferrer noopener" className="text-[10px] underline text-subtle">terms</a> | |
| 73 | + ) : null} | |
| 74 | + </div> | |
| 34 | 75 | <div className="grid grid-cols-2 gap-3 md:grid-cols-4 xl:grid-cols-6"> |
| 35 | − <Kpi label="Status" value={<StatusPill status={String(c.status)} />} sub={<>health <StatusPill status={(d.health?.status as string) ?? 'unknown'} /></>} /> | |
| 76 | + <Kpi label="Status" value={<StatusPill status={String(c.status)} />} sub={<>health <HealthLabel status={(d.health?.status as string) ?? 'unknown'} /></>} /> | |
| 36 | 77 | <Kpi label="Success 24h" value={health.success_rate_24h != null ? `${Math.round(Number(health.success_rate_24h) * 100)}%` : '—'} sub={`${n(health.pages_success)}/${n(health.pages_attempted)} pages`} /> |
| 37 | 78 | <Kpi label="Firecrawl / Scrapfly" value={`${health.firecrawl_success_rate != null ? Math.round(Number(health.firecrawl_success_rate) * 100) + '%' : '—'} / ${health.scrapfly_fallback_rate != null ? Math.round(Number(health.scrapfly_fallback_rate) * 100) + '%' : '—'}`} sub="success / fallback share" /> |
| 38 | 79 | <Kpi label="Parse failures" value={health.parse_failure_rate != null ? `${(Number(health.parse_failure_rate) * 100).toFixed(1)}%` : '—'} sub={`${n(health.duplicates_24h)} duplicates 24h`} /> |
| 39 | 80 | <Kpi label="Outputs" value={`${n(outputs.sales)} sales`} sub={`${n(outputs.listings)} listings · ${n(outputs.observations)} observations`} /> |
| 40 | − <Kpi label="Last success" value={fmtTs(c.last_success_at).slice(0, 16)} sub={`next ${fmtTs(c.next_run_at).slice(0, 16)}`} /> | |
| 81 | + <Kpi label="Last success" value={fmtTs(c.last_success_at).slice(0, 16)} sub={`next ${fmtTs(c.next_run_at).slice(0, 16)} · fresh ${health.data_freshness ? fmtTs(health.data_freshness).slice(0, 10) : '—'}`} /> | |
| 82 | + <Kpi label="Requests 24h" value={n(health.requests_24h)} sub={`${health.latency_ms_avg != null ? n(health.latency_ms_avg) + ' ms avg' : '—'} · ${n(health.challenges_24h)} challenges · ${n(health.rate_limited_24h)}× 429`} tone={Number(health.challenges_24h) > 0 ? 'alert' : undefined} /> | |
| 83 | + <Kpi label="HTTP errors 24h" value={Object.entries((health.http_errors as Record<string, number>) ?? {}).map(([k, v]) => `${k}×${v}`).join(' ') || '0'} sub={`${n(health.circuit_refusals_24h)} refused by circuit`} tone={Object.keys((health.http_errors as Record<string, number>) ?? {}).length ? 'loss' : undefined} /> | |
| 84 | + <Kpi label="Volume baseline" value={health.records_7d_daily_avg != null ? `${n(health.records_7d_daily_avg)}/day` : '—'} sub={`${n(health.records_24h)} raw in 24h`} tone={((health.anomalies as string[]) ?? []).some((a) => a.startsWith('result_count_collapse')) ? 'alert' : undefined} /> | |
| 85 | + <Kpi label="Certificates" value={n(outputs.certificates)} sub="cert numbers seen via this source" /> | |
| 41 | 86 | </div> |
| 42 | 87 | |
| 88 | + {activeBackfill || d.backfills.length ? ( | |
| 89 | + <section className="card mt-6"> | |
| 90 | + <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Historical backfill campaigns</h2></header> | |
| 91 | + <Table> | |
| 92 | + <thead><tr><th className={th}>Started</th><th className={th}>Status</th><th className={th}>Progress</th><th className={`${th} text-right`}>Pages</th><th className={`${th} text-right`}>Items</th><th className={th}>Reached date</th><th className={`${th} text-right`}>Runs</th><th className={`${th} text-right`}>Errors</th><th className={th}>Last cursor</th><th className={th}>Last error</th></tr></thead> | |
| 93 | + <tbody> | |
| 94 | + {d.backfills.map((b) => ( | |
| 95 | + <tr key={String(b.id)}> | |
| 96 | + <td className={td}>{fmtTs(b.started_at)}</td> | |
| 97 | + <td className={td}><StatusPill status={String(b.status)} /></td> | |
| 98 | + <td className={td}><Progress percent={b.percent as number | null} status={String(b.status)} /></td> | |
| 99 | + <td className={tdNum}>{n(b.pages_processed)}{b.total_pages ? ` / ${n(b.total_pages)}` : ''}</td> | |
| 100 | + <td className={tdNum}>{n(b.items_processed)}</td> | |
| 101 | + <td className={td}>{b.reached_date ? String(b.reached_date) : '—'}</td> | |
| 102 | + <td className={tdNum}>{n(b.runs)}</td> | |
| 103 | + <td className={tdNum}>{n(b.errors)}</td> | |
| 104 | + <td className={`${td} max-w-[260px] truncate font-mono text-[10px]`} title={JSON.stringify(b.last_cursor)}>{JSON.stringify(b.last_cursor)}</td> | |
| 105 | + <td className={`${td} max-w-[220px] truncate text-loss`} title={String(b.last_error ?? '')}>{String(b.last_error ?? '')}</td> | |
| 106 | + </tr> | |
| 107 | + ))} | |
| 108 | + </tbody> | |
| 109 | + </Table> | |
| 110 | + </section> | |
| 111 | + ) : null} | |
| 112 | + | |
| 113 | + {nullRates.length ? ( | |
| 114 | + <section className="card mt-6"> | |
| 115 | + <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Field presence (schema-drift monitor)</h2></header> | |
| 116 | + <div className="grid grid-cols-2 gap-x-6 gap-y-1 px-4 py-3 text-xs md:grid-cols-4 xl:grid-cols-6"> | |
| 117 | + {nullRates.map((f) => { | |
| 118 | + const tRate = f.today[0] ? f.today[1] / f.today[0] : null; | |
| 119 | + const bRate = f.base[0] ? f.base[1] / f.base[0] : null; | |
| 120 | + const drift = tRate !== null && bRate !== null && tRate - bRate >= 0.3; | |
| 121 | + return ( | |
| 122 | + <div key={f.field} className={drift ? 'text-alert' : ''}> | |
| 123 | + <span className="font-mono">{f.field}</span> <span className="text-subtle">null today {tRate === null ? '—' : `${Math.round(tRate * 100)}%`} · 7d {bRate === null ? '—' : `${Math.round(bRate * 100)}%`}</span> | |
| 124 | + </div> | |
| 125 | + ); | |
| 126 | + })} | |
| 127 | + </div> | |
| 128 | + </section> | |
| 129 | + ) : null} | |
| 130 | + | |
| 43 | 131 | {d.anomalies.length ? ( |
| 44 | 132 | <section className="card mt-6"> |
| 45 | 133 | <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold text-alert">Anomalies (recent runs)</h2></header> |
@@ -81,7 +169,10 @@ export default async function ConnectorInspectPage({ params }: { params: Promise | ||
| 81 | 169 | <li key={String(r.id)} className="px-4 py-2"> |
| 82 | 170 | <div className="flex items-center justify-between gap-2"><span className="font-mono text-[11px]">{String(r.kind)} · {String(r.engine)} · {String(r.http_status ?? '')}</span><span className="text-subtle">{fmtTs(r.fetched_at)}</span></div> |
| 83 | 171 | <a href={String(r.url)} target="_blank" rel="noreferrer noopener" className="block truncate text-muted hover:text-fg">{String(r.url)}</a> |
| 84 | − {r.process_error ? <p className="text-loss">{String(r.process_error)}</p> : <p className="text-[10px] text-subtle">{r.processed_at ? `processed ${fmtTs(r.processed_at)}` : 'unprocessed'}</p>} | |
| 172 | + <div className="flex items-center justify-between"> | |
| 173 | + {r.process_error ? <p className="text-loss">{String(r.process_error)}</p> : <p className="text-[10px] text-subtle">{r.processed_at ? `processed ${fmtTs(r.processed_at)}` : 'unprocessed'}</p>} | |
| 174 | + <Link href={`/admin/connectors/${String(c.id)}/raw/${String(r.id)}`} className="text-[10px] underline">view payload</Link> | |
| 175 | + </div> | |
| 85 | 176 | </li> |
| 86 | 177 | ))} |
| 87 | 178 | </ul> |
@@ -93,7 +184,10 @@ export default async function ConnectorInspectPage({ params }: { params: Promise | ||
| 93 | 184 | {d.normalizedSample.map((r) => ( |
| 94 | 185 | <li key={String(r.id)} className="px-4 py-2"> |
| 95 | 186 | <div className="flex items-center justify-between gap-2"><span className="truncate">{String(r.raw_title ?? '')}</span><StatusPill status={String(r.status)} /></div> |
| 96 | − <p className="text-[10px] text-subtle">{String(r.kind)} · {r.price ? `${String(r.price)} ${String(r.currency ?? '')}` : ''} · {r.match_method ? `${String(r.match_method)} ${r.match_confidence ? Math.round(Number(r.match_confidence) * 100) + '%' : ''}` : ''} {r.reject_reason ? `· ${String(r.reject_reason)}` : ''}</p> | |
| 187 | + <div className="flex items-center justify-between"> | |
| 188 | + <p className="text-[10px] text-subtle">{String(r.kind)} · {r.price ? `${String(r.price)} ${String(r.currency ?? '')}` : ''} · {r.match_method ? `${String(r.match_method)} ${r.match_confidence ? Math.round(Number(r.match_confidence) * 100) + '%' : ''}` : ''} {r.reject_reason ? `· ${String(r.reject_reason)}` : ''}</p> | |
| 189 | + <Link href={`/admin/connectors/${String(c.id)}/record/${String(r.id)}`} className="text-[10px] underline">view record</Link> | |
| 190 | + </div> | |
| 97 | 191 | </li> |
| 98 | 192 | ))} |
| 99 | 193 | </ul> |
@@ -128,6 +222,12 @@ export default async function ConnectorInspectPage({ params }: { params: Promise | ||
| 128 | 222 | </form> |
| 129 | 223 | </section> |
| 130 | 224 | </div> |
| 225 | + | |
| 226 | + <section className="card mt-6"> | |
| 227 | + <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Access & compliance notes (meta.json)</h2></header> | |
| 228 | + <div className="px-4 py-3 text-xs leading-relaxed text-muted">{String(meta.accessNotes ?? 'No accessNotes recorded — add them to meta.json (SPEC §36).')}</div> | |
| 229 | + <details className="border-t border-border px-4 py-2 text-xs"><summary className="cursor-pointer text-subtle">registry entry</summary><Json value={meta} className="mt-2" /></details> | |
| 230 | + </section> | |
| 131 | 231 | </AdminShell> |
| 132 | 232 | ); |
| 133 | 233 | } |
added
apps/web/src/app/admin/connectors/[id]/raw/[rawId]/page.tsx
+33 −0
@@ -0,0 +1,33 @@ | ||
| 1 | +import Link from 'next/link'; | |
| 2 | +import { notFound } from 'next/navigation'; | |
| 3 | +import { requireAdmin } from '@/lib/admin/auth'; | |
| 4 | +import { rawRecordDetail } from '@/lib/admin/queries'; | |
| 5 | +import { AdminShell, fmtTs } from '@/components/admin/shell'; | |
| 6 | +import { Json } from '@/components/admin/connector-bits'; | |
| 7 | + | |
| 8 | +/** "View last payload" (SPEC §29): the immutable raw capture with its provenance (engine, parser/connector versions, snapshot ref). */ | |
| 9 | +export default async function RawRecordPage({ params }: { params: Promise<{ id: string; rawId: string }> }) { | |
| 10 | + await requireAdmin(); | |
| 11 | + const { id, rawId } = await params; | |
| 12 | + const r = await rawRecordDetail(rawId); | |
| 13 | + if (!r || String(r.connector_id) !== id) notFound(); | |
| 14 | + const payload = r.payload as Record<string, unknown> | null; | |
| 15 | + const snapshot = payload && typeof payload === 'object' && typeof payload.snapshot === 'string' ? (payload.snapshot as string) : null; | |
| 16 | + const rest = snapshot && payload ? Object.fromEntries(Object.entries(payload).filter(([k]) => k !== 'snapshot')) : payload; | |
| 17 | + return ( | |
| 18 | + <AdminShell current="/admin/connectors" title={`Raw payload ${rawId}`} subtitle={<><Link href={`/admin/connectors/${id}`} className="underline">{id}</Link> · {String(r.kind)} · engine {String(r.engine)} · HTTP {String(r.http_status ?? '—')} · fetched {fmtTs(r.fetched_at)} · parser v{String(r.parser_version)} · connector v{String(r.connector_version)} · run {String(r.run_id ?? '—')}</>}> | |
| 19 | + <div className="mb-3 text-xs text-muted"> | |
| 20 | + <a href={String(r.url)} target="_blank" rel="noreferrer noopener" className="underline">{String(r.url)}</a> · external id <span className="font-mono">{String(r.external_id ?? '—')}</span> · hash <span className="font-mono">{String(r.content_hash).slice(0, 16)}…</span> | |
| 21 | + {r.snapshot_ref ? <> · snapshot on disk: <span className="font-mono">{String(r.snapshot_ref)}</span></> : null} | |
| 22 | + {r.process_error ? <div className="mt-1 text-loss">process error: {String(r.process_error)}</div> : <div className="mt-1 text-subtle">{r.processed_at ? `normalised ${fmtTs(r.processed_at)}` : 'not yet normalised'}</div>} | |
| 23 | + </div> | |
| 24 | + <Json value={rest} /> | |
| 25 | + {snapshot ? ( | |
| 26 | + <details className="mt-4 text-xs"> | |
| 27 | + <summary className="cursor-pointer text-subtle">inline snapshot ({snapshot.length.toLocaleString()} chars)</summary> | |
| 28 | + <Json value={snapshot.slice(0, 200_000)} className="mt-2" /> | |
| 29 | + </details> | |
| 30 | + ) : null} | |
| 31 | + </AdminShell> | |
| 32 | + ); | |
| 33 | +} | |
added
apps/web/src/app/admin/connectors/[id]/record/[recId]/page.tsx
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +import Link from 'next/link'; | |
| 2 | +import { notFound } from 'next/navigation'; | |
| 3 | +import { requireAdmin } from '@/lib/admin/auth'; | |
| 4 | +import { normalizedRecordDetail } from '@/lib/admin/queries'; | |
| 5 | +import { AdminShell, StatusPill, fmtTs } from '@/components/admin/shell'; | |
| 6 | +import { Json } from '@/components/admin/connector-bits'; | |
| 7 | + | |
| 8 | +/** "View last parsed record" (SPEC §29): the canonical normalised record and its resolution decision. */ | |
| 9 | +export default async function NormalizedRecordPage({ params }: { params: Promise<{ id: string; recId: string }> }) { | |
| 10 | + await requireAdmin(); | |
| 11 | + const { id, recId } = await params; | |
| 12 | + const r = await normalizedRecordDetail(recId); | |
| 13 | + if (!r || String(r.connector_id) !== id) notFound(); | |
| 14 | + return ( | |
| 15 | + <AdminShell current="/admin/connectors" title={`Normalised record ${recId}`} subtitle={<><Link href={`/admin/connectors/${id}`} className="underline">{id}</Link> · {String(r.kind)} · <StatusPill status={String(r.status)} /> · created {fmtTs(r.created_at)} · from raw <Link href={`/admin/connectors/${id}/raw/${String(r.raw_record_id)}`} className="underline">{String(r.raw_record_id)}</Link></>}> | |
| 16 | + <div className="mb-3 grid gap-1 text-xs text-muted md:grid-cols-2"> | |
| 17 | + <div>match: <span className="font-mono">{String(r.match_method ?? '—')}</span> {r.match_confidence ? `(${Math.round(Number(r.match_confidence) * 100)}%)` : ''}</div> | |
| 18 | + <div>asset: {r.asset_id ? <Link href={`/admin/data-quality?asset=${String(r.asset_id)}`} className="font-mono underline">{String(r.asset_id)}</Link> : '—'} · variant <span className="font-mono">{String(r.variant_id ?? '—')}</span></div> | |
| 19 | + <div>target: <span className="font-mono">{String(r.target_id ?? '—')}</span></div> | |
| 20 | + <div className={r.reject_reason ? 'text-loss' : ''}>reason: {String(r.reject_reason ?? '—')}</div> | |
| 21 | + </div> | |
| 22 | + <Json value={r.payload} /> | |
| 23 | + </AdminShell> | |
| 24 | + ); | |
| 25 | +} | |
modified
apps/web/src/app/admin/connectors/page.tsx
+184 −42
@@ -3,75 +3,210 @@ import { requireAdmin } from '@/lib/admin/auth'; | ||
| 3 | 3 | import { connectorsOverview } from '@/lib/admin/queries'; |
| 4 | 4 | import { connectorAction } from '@/lib/admin/actions'; |
| 5 | 5 | import { AdminShell, ActionButton, StatusPill, fmtTs, n } from '@/components/admin/shell'; |
| 6 | +import { HealthLabel, SourceLogo, Pct, Progress, Chip } from '@/components/admin/connector-bits'; | |
| 6 | 7 | import { Table, th, td, tdNum } from '@/components/ui/primitives'; |
| 7 | 8 | |
| 8 | −function pct(v: unknown): string { | |
| 9 | − const x = Number(v); | |
| 10 | − return Number.isFinite(x) ? `${Math.round(x * 100)}%` : '—'; | |
| 9 | +type Row = Record<string, unknown>; | |
| 10 | + | |
| 11 | +function str(v: unknown): string { | |
| 12 | + return v === null || v === undefined ? '' : String(v); | |
| 11 | 13 | } |
| 12 | 14 | |
| 13 | −export default async function ConnectorsPage() { | |
| 15 | +/** Connector explorer (SPEC §29): every connector with source, geography, acquisition, health, usage, backfill and actions. */ | |
| 16 | +export default async function ConnectorsPage({ searchParams }: { searchParams: Promise<Record<string, string | undefined>> }) { | |
| 14 | 17 | await requireAdmin(); |
| 15 | − const rows = await connectorsOverview(); | |
| 18 | + const sp = await searchParams; | |
| 19 | + const rowsAll = (await connectorsOverview()) as Row[]; | |
| 20 | + const q = (sp.q ?? '').toLowerCase(); | |
| 21 | + const status = sp.status ?? ''; | |
| 22 | + const category = sp.category ?? ''; | |
| 23 | + const country = sp.country ?? ''; | |
| 24 | + const rows = rowsAll.filter((r) => { | |
| 25 | + const meta = (r.meta ?? {}) as Row; | |
| 26 | + if (status && String(r.health_status ?? 'unknown') !== status && String(r.status) !== status) return false; | |
| 27 | + if (category && !((r.categories as string[]) ?? []).includes(category)) return false; | |
| 28 | + if (country && String(meta.country ?? (r.regions as string[])?.[0] ?? '') !== country) return false; | |
| 29 | + if (q && !`${str(r.id)} ${str(r.display_name)} ${str(meta.domain)} ${((r.categories as string[]) ?? []).join(' ')}`.toLowerCase().includes(q)) return false; | |
| 30 | + return true; | |
| 31 | + }); | |
| 32 | + const counts = rowsAll.reduce<Record<string, number>>((acc, r) => { | |
| 33 | + const k = String(r.health_status ?? 'unknown'); | |
| 34 | + acc[k] = (acc[k] ?? 0) + 1; | |
| 35 | + return acc; | |
| 36 | + }, {}); | |
| 37 | + const categories = [...new Set(rowsAll.flatMap((r) => (r.categories as string[]) ?? []))].sort(); | |
| 38 | + const countries = [...new Set(rowsAll.map((r) => String(((r.meta ?? {}) as Row).country ?? (r.regions as string[])?.[0] ?? 'global')))].sort(); | |
| 39 | + | |
| 16 | 40 | return ( |
| 17 | − <AdminShell current="/admin/connectors" title="Connector control center" subtitle={`${rows.length} connectors registered · actions enqueue jobs on the crawl.run queue consumed by the worker`}> | |
| 18 | − <div className="card"> | |
| 41 | + <AdminShell | |
| 42 | + current="/admin/connectors" | |
| 43 | + title="Connector explorer" | |
| 44 | + subtitle={ | |
| 45 | + <> | |
| 46 | + {rowsAll.length} connectors · {Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(' · ')} · <Link href="/admin/coverage" className="underline">coverage dashboard</Link> · <Link href="/api/admin/connectors" className="underline">JSON</Link> | |
| 47 | + </> | |
| 48 | + } | |
| 49 | + actions={ | |
| 50 | + <form className="flex flex-wrap items-center gap-1 text-xs" method="get"> | |
| 51 | + <input name="q" defaultValue={sp.q ?? ''} placeholder="search id, domain, category" className="rounded-md border border-border bg-sunken px-2 py-1 text-xs" /> | |
| 52 | + <select name="status" defaultValue={status} className="rounded-md border border-border bg-sunken px-2 py-1 text-xs"> | |
| 53 | + <option value="">any health</option> | |
| 54 | + {['healthy', 'degraded', 'failing', 'disabled', 'paused', 'maintenance', 'unknown'].map((s) => ( | |
| 55 | + <option key={s} value={s}>{s}</option> | |
| 56 | + ))} | |
| 57 | + </select> | |
| 58 | + <select name="category" defaultValue={category} className="rounded-md border border-border bg-sunken px-2 py-1 text-xs"> | |
| 59 | + <option value="">any category</option> | |
| 60 | + {categories.map((c) => ( | |
| 61 | + <option key={c} value={c}>{c}</option> | |
| 62 | + ))} | |
| 63 | + </select> | |
| 64 | + <select name="country" defaultValue={country} className="rounded-md border border-border bg-sunken px-2 py-1 text-xs"> | |
| 65 | + <option value="">any country</option> | |
| 66 | + {countries.map((c) => ( | |
| 67 | + <option key={c} value={c}>{c}</option> | |
| 68 | + ))} | |
| 69 | + </select> | |
| 70 | + <button type="submit" className="rounded-md border border-border px-2 py-1 text-xs hover:bg-inset">Filter</button> | |
| 71 | + </form> | |
| 72 | + } | |
| 73 | + > | |
| 74 | + <div className="card overflow-x-auto"> | |
| 19 | 75 | <Table> |
| 20 | 76 | <thead> |
| 21 | 77 | <tr> |
| 22 | − <th className={th}>Connector</th> | |
| 78 | + <th className={th}>Source</th> | |
| 79 | + <th className={th}>Category · country</th> | |
| 23 | 80 | <th className={th}>Health</th> |
| 24 | 81 | <th className={th}>Status</th> |
| 25 | − <th className={`${th} text-right`}>Pages/min</th> | |
| 82 | + <th className={th}>Acquisition</th> | |
| 83 | + <th className={th}>Last sync</th> | |
| 84 | + <th className={`${th} text-right`}>Items 24h</th> | |
| 26 | 85 | <th className={`${th} text-right`}>Errors</th> |
| 27 | − <th className={th}>Last crawl</th> | |
| 28 | − <th className={`${th} text-right`}>Records 24h</th> | |
| 29 | − <th className={`${th} text-right`}>Dup.</th> | |
| 30 | − <th className={`${th} text-right`}>FC rate</th> | |
| 31 | − <th className={`${th} text-right`}>SF rate</th> | |
| 32 | − <th className={`${th} text-right`}>Parser conf.</th> | |
| 33 | − <th className={th}>Schema</th> | |
| 86 | + <th className={`${th} text-right`}>FC · SF credits</th> | |
| 87 | + <th className={th}>Rate limit</th> | |
| 88 | + <th className={th}>Priority</th> | |
| 89 | + <th className={th}>Backfill</th> | |
| 34 | 90 | <th className={th}>Actions</th> |
| 35 | 91 | </tr> |
| 36 | 92 | </thead> |
| 37 | 93 | <tbody> |
| 38 | − {rows.length === 0 ? <tr><td className={td} colSpan={13}><span className="text-subtle">No connectors in the registry yet. Add meta.json files under connectors/ and run pnpm db:seed.</span></td></tr> : null} | |
| 94 | + {rows.length === 0 ? ( | |
| 95 | + <tr> | |
| 96 | + <td className={td} colSpan={13}> | |
| 97 | + <span className="text-subtle">No connector matches. Add meta.json files under connectors/, run pnpm registry && pnpm db:seed.</span> | |
| 98 | + </td> | |
| 99 | + </tr> | |
| 100 | + ) : null} | |
| 39 | 101 | {rows.map((r) => { |
| 40 | − const health = (r.health ?? {}) as Record<string, unknown>; | |
| 102 | + const meta = (r.meta ?? {}) as Row; | |
| 103 | + const health = (r.health ?? {}) as Row; | |
| 41 | 104 | const anomalies = Array.isArray(r.last_anomalies) ? (r.last_anomalies as string[]) : []; |
| 42 | − const drift = anomalies.some((a) => /schema|selector|missing_field|redesign/.test(a)); | |
| 105 | + const drift = ((health.schema_drift as string[]) ?? []).length > 0 || anomalies.some((a) => /schema_drift|selector|pagination_failure|result_count_collapse/.test(a)); | |
| 106 | + const httpErrors = Object.values((health.http_errors as Record<string, number>) ?? {}).reduce((a, b) => a + b, 0); | |
| 107 | + const cats = (r.categories as string[]) ?? []; | |
| 108 | + const id = str(r.id); | |
| 109 | + const domain = str(meta.domain); | |
| 110 | + const country = str(meta.country ?? (r.regions as string[])?.[0] ?? 'global'); | |
| 111 | + const refresh = Number(r.refresh_frequency_minutes); | |
| 112 | + const refreshClass = str(meta.refreshClass) || (refresh <= 5 ? 'hot' : refresh <= 60 ? 'active' : refresh <= 1440 ? 'normal' : 'archive'); | |
| 113 | + const missing = (health.missing_requirements as string[]) ?? []; | |
| 43 | 114 | return ( |
| 44 | − <tr key={String(r.id)} className="align-top"> | |
| 115 | + <tr key={id} className="align-top"> | |
| 116 | + <td className={td}> | |
| 117 | + <div className="flex items-center gap-2"> | |
| 118 | + <SourceLogo domain={domain} name={str(r.display_name)} logoUrl={str(meta.logoUrl) || null} /> | |
| 119 | + <div> | |
| 120 | + <Link href={`/admin/connectors/${id}`} className="font-medium hover:underline">{str(r.display_name)}</Link> | |
| 121 | + <div className="text-[11px] text-subtle"> | |
| 122 | + {domain ? ( | |
| 123 | + <a href={`https://${domain}`} target="_blank" rel="noreferrer noopener" className="hover:underline">{domain}</a> | |
| 124 | + ) : ( | |
| 125 | + id | |
| 126 | + )} | |
| 127 | + {' · '} | |
| 128 | + {id} | |
| 129 | + </div> | |
| 130 | + </div> | |
| 131 | + </div> | |
| 132 | + </td> | |
| 133 | + <td className={td}> | |
| 134 | + <div className="flex max-w-[220px] flex-wrap gap-1"> | |
| 135 | + {cats.slice(0, 3).map((c) => ( | |
| 136 | + <Chip key={c}>{c}</Chip> | |
| 137 | + ))} | |
| 138 | + {cats.length > 3 ? <Chip>+{cats.length - 3}</Chip> : null} | |
| 139 | + <Chip tone="index">{country}</Chip> | |
| 140 | + </div> | |
| 141 | + </td> | |
| 142 | + <td className={td}> | |
| 143 | + <HealthLabel status={str(r.health_status) || 'unknown'} /> | |
| 144 | + <div className="text-[10px] text-subtle"> | |
| 145 | + {health.success_rate_24h != null ? <>success <Pct value={health.success_rate_24h} /></> : 'no runs 24h'} | |
| 146 | + {r.health_at ? ` · ${fmtTs(r.health_at).slice(5, 16)}` : ''} | |
| 147 | + </div> | |
| 148 | + {drift ? <div className="text-[10px] text-alert">schema drift?</div> : null} | |
| 149 | + {missing.length ? <div className="text-[10px] text-alert">needs {missing.join(', ')}</div> : null} | |
| 150 | + </td> | |
| 151 | + <td className={td}> | |
| 152 | + <StatusPill status={str(r.status)} /> | |
| 153 | + {r.last_run_status ? <div className="text-[10px] text-subtle">run: {str(r.last_run_status)}</div> : null} | |
| 154 | + </td> | |
| 155 | + <td className={td}> | |
| 156 | + <div className="text-xs">{str(meta.acquisitionMethod) || (r.engine_priority as string[]).join(' → ')}</div> | |
| 157 | + <div className="text-[10px] text-subtle">{(r.engine_priority as string[]).join('→')} · {refreshClass} · every {n(refresh)} min</div> | |
| 158 | + </td> | |
| 45 | 159 | <td className={td}> |
| 46 | − <Link href={`/admin/connectors/${String(r.id)}`} className="font-medium hover:underline">{String(r.display_name)}</Link> | |
| 47 | − <div className="text-[11px] text-subtle">{String(r.id)} · {(r.engine_priority as string[]).join('→')} · {String(r.priority)} · every {n(r.refresh_frequency_minutes)} min</div> | |
| 160 | + {fmtTs(r.last_started_at)} | |
| 161 | + <div className="text-[10px] text-subtle">ok: {fmtTs(r.last_success_at).slice(0, 16)}</div> | |
| 162 | + <div className="text-[10px] text-subtle">fresh: {health.data_freshness ? fmtTs(health.data_freshness).slice(0, 10) : '—'}</div> | |
| 163 | + </td> | |
| 164 | + <td className={tdNum}> | |
| 165 | + {n(r.records_24h)} | |
| 166 | + <div className="text-[10px] text-subtle">{n(r.normalized_24h)} norm. · {n(r.duplicates_24h)} dup.</div> | |
| 167 | + <div className="text-[10px] text-subtle">{n(r.pages_per_min, 1)} pages/min</div> | |
| 168 | + </td> | |
| 169 | + <td className={`${tdNum} ${r.last_error || httpErrors ? 'text-loss' : ''}`} title={str(r.last_error)}> | |
| 170 | + {httpErrors || (r.last_error ? 1 : 0)} | |
| 171 | + {anomalies.length ? <div className="text-[10px] text-alert">{anomalies.length} anomal.</div> : null} | |
| 172 | + {Number(health.challenges_24h) > 0 ? <div className="text-[10px] text-alert">{n(health.challenges_24h)} challenges</div> : null} | |
| 173 | + </td> | |
| 174 | + <td className={tdNum}> | |
| 175 | + {n(r.fc_credits_24h)} · {n(r.sf_credits_24h)} | |
| 176 | + <div className="text-[10px] text-subtle"> | |
| 177 | + FC <Pct value={health.firecrawl_success_rate} /> · SF <Pct value={health.scrapfly_fallback_rate} /> | |
| 178 | + </div> | |
| 48 | 179 | </td> |
| 49 | − <td className={td}><StatusPill status={(r.health_status as string) ?? 'unknown'} />{r.health_at ? <div className="text-[10px] text-subtle">{fmtTs(r.health_at)}</div> : null}</td> | |
| 50 | − <td className={td}><StatusPill status={String(r.status)} />{r.last_run_status ? <div className="text-[10px] text-subtle">run: {String(r.last_run_status)}</div> : null}</td> | |
| 51 | − <td className={tdNum}>{n(r.pages_per_min, 1)}</td> | |
| 52 | − <td className={`${tdNum} ${r.last_error ? 'text-loss' : ''}`} title={String(r.last_error ?? '')}>{r.last_error ? '1' : '0'}{anomalies.length ? <div className="text-[10px] text-alert">{anomalies.length} anomal.</div> : null}</td> | |
| 53 | − <td className={td}>{fmtTs(r.last_started_at)}<div className="text-[10px] text-subtle">ok: {fmtTs(r.last_success_at)}</div></td> | |
| 54 | − <td className={tdNum}>{n(r.records_24h)}<div className="text-[10px] text-subtle">{n(r.normalized_24h)} norm.</div></td> | |
| 55 | − <td className={tdNum}>{n(r.duplicates_24h)}</td> | |
| 56 | − <td className={tdNum}>{pct(health.firecrawl_success_rate)}</td> | |
| 57 | − <td className={tdNum}>{pct(health.scrapfly_fallback_rate)}</td> | |
| 58 | − <td className={tdNum}>{pct(r.parser_confidence)}</td> | |
| 59 | − <td className={td}>v{String(r.schema_version)}{drift ? <div className="text-[10px] text-alert">drift?</div> : null}</td> | |
| 60 | 180 | <td className={td}> |
| 61 | − <div className="flex flex-wrap gap-1"> | |
| 62 | − {(['run', 'probe', 'recrawl', 'retry'] as const).map((a) => ( | |
| 181 | + <div className="text-xs">{health.latency_ms_avg != null ? `${n(health.latency_ms_avg)} ms` : '—'}</div> | |
| 182 | + <div className="text-[10px] text-subtle">{Number(health.rate_limited_24h) > 0 ? `${n(health.rate_limited_24h)}× 429` : 'no 429'} · {n(health.requests_24h)} req</div> | |
| 183 | + </td> | |
| 184 | + <td className={td}> | |
| 185 | + <Chip tone={str(r.priority) === 'high' ? 'index' : 'neutral'}>{str(r.priority)}</Chip> | |
| 186 | + <div className="text-[10px] text-subtle">trust {source_trust(meta)}</div> | |
| 187 | + </td> | |
| 188 | + <td className={td}>{r.backfill_id ? <Progress percent={r.backfill_percent as number | null} status={str(r.backfill_status)} /> : <span className="text-[10px] text-subtle">{str(meta.historicalDepth) && str(meta.historicalDepth) !== 'none' ? `history: ${str(meta.historicalDepth)}` : '—'}</span>}</td> | |
| 189 | + <td className={td}> | |
| 190 | + <div className="flex max-w-[210px] flex-wrap gap-1"> | |
| 191 | + {( | |
| 192 | + [ | |
| 193 | + ['probe', 'Test'], | |
| 194 | + ['run', 'Run now'], | |
| 195 | + ['backfill', 'Backfill'], | |
| 196 | + ] as const | |
| 197 | + ).map(([a, label]) => ( | |
| 63 | 198 | <form key={a} action={connectorAction}> |
| 64 | − <input type="hidden" name="id" value={String(r.id)} /> | |
| 199 | + <input type="hidden" name="id" value={id} /> | |
| 65 | 200 | <input type="hidden" name="action" value={a} /> |
| 66 | − <ActionButton label={a} /> | |
| 201 | + <ActionButton label={label} /> | |
| 67 | 202 | </form> |
| 68 | 203 | ))} |
| 69 | 204 | <form action={connectorAction}> |
| 70 | − <input type="hidden" name="id" value={String(r.id)} /> | |
| 71 | − <input type="hidden" name="action" value={r.status === 'paused' ? 'resume' : 'pause'} /> | |
| 72 | − <ActionButton label={r.status === 'paused' ? 'resume' : 'pause'} tone={r.status === 'paused' ? 'primary' : 'danger'} /> | |
| 205 | + <input type="hidden" name="id" value={id} /> | |
| 206 | + <input type="hidden" name="action" value={r.status === 'active' ? 'pause' : 'resume'} /> | |
| 207 | + <ActionButton label={r.status === 'active' ? 'Pause' : 'Resume'} tone={r.status === 'active' ? 'danger' : 'primary'} /> | |
| 73 | 208 | </form> |
| 74 | − <Link href={`/admin/connectors/${String(r.id)}`} className="rounded-md border border-border px-2 py-0.5 text-[11px] font-medium hover:bg-inset">inspect</Link> | |
| 209 | + <Link href={`/admin/connectors/${id}`} className="rounded-md border border-border px-2 py-0.5 text-[11px] font-medium hover:bg-inset">Logs</Link> | |
| 75 | 210 | </div> |
| 76 | 211 | </td> |
| 77 | 212 | </tr> |
@@ -80,7 +215,14 @@ export default async function ConnectorsPage() { | ||
| 80 | 215 | </tbody> |
| 81 | 216 | </Table> |
| 82 | 217 | </div> |
| 83 | − <p className="mt-3 text-[11px] text-subtle">run = incremental crawl · probe = 25-record smoke test · recrawl = backfill mode · retry = re-run from the last cursor. FC = Firecrawl success rate, SF = share of pages that needed Scrapfly (from the 24 h health snapshot).</p> | |
| 218 | + <p className="mt-3 text-[11px] text-subtle"> | |
| 219 | + Test = 25-record probe · Run now = incremental crawl · Backfill = resumable historical campaign (progress in the Backfill column) · Pause/Resume flips scheduling; Maintenance is available on the connector page. FC/SF = Firecrawl/Scrapfly credits spent in 24 h and success/fallback rates. Health labels: UP · DEGRADED · BROKEN · DISABLED · MAINTENANCE. | |
| 220 | + </p> | |
| 84 | 221 | </AdminShell> |
| 85 | 222 | ); |
| 86 | 223 | } |
| 224 | + | |
| 225 | +function source_trust(meta: Row): string { | |
| 226 | + const t = Number(meta.trustScore); | |
| 227 | + return Number.isFinite(t) ? `${Math.round(t * 100)}%` : '—'; | |
| 228 | +} | |
added
apps/web/src/app/api/admin/connectors/route.ts
+58 −0
@@ -0,0 +1,58 @@ | ||
| 1 | +import { NextResponse } from 'next/server'; | |
| 2 | +import { connectorsOverview } from '@/lib/admin/queries'; | |
| 3 | + | |
| 4 | +export const runtime = 'nodejs'; | |
| 5 | +export const dynamic = 'force-dynamic'; | |
| 6 | + | |
| 7 | +const LABEL: Record<string, string> = { healthy: 'UP', degraded: 'DEGRADED', failing: 'BROKEN', paused: 'DISABLED', disabled: 'DISABLED', maintenance: 'MAINTENANCE', unknown: 'UNKNOWN' }; | |
| 8 | + | |
| 9 | +/** Connector health API (SPEC §12): one object per connector with status, usage, errors, freshness and backfill progress. */ | |
| 10 | +export async function GET() { | |
| 11 | + const rows = await connectorsOverview(); | |
| 12 | + return NextResponse.json({ | |
| 13 | + as_of: new Date().toISOString(), | |
| 14 | + connectors: rows.map((r) => { | |
| 15 | + const h = (r.health ?? {}) as Record<string, unknown>; | |
| 16 | + const m = (r.meta ?? {}) as Record<string, unknown>; | |
| 17 | + return { | |
| 18 | + id: r.id, | |
| 19 | + name: r.display_name, | |
| 20 | + source: r.source_id, | |
| 21 | + domain: m.domain ?? null, | |
| 22 | + country: m.country ?? null, | |
| 23 | + categories: r.categories, | |
| 24 | + capabilities: m.capabilities ?? [], | |
| 25 | + acquisition: m.acquisitionMethod ?? (r.engine_priority as string[]).join('>'), | |
| 26 | + engines: r.engine_priority, | |
| 27 | + priority: r.priority, | |
| 28 | + refresh_minutes: r.refresh_frequency_minutes, | |
| 29 | + refresh_class: m.refreshClass ?? null, | |
| 30 | + status: r.status, | |
| 31 | + health: r.health_status ?? 'unknown', | |
| 32 | + health_label: LABEL[String(r.health_status ?? 'unknown')] ?? 'UNKNOWN', | |
| 33 | + health_at: r.health_at, | |
| 34 | + success_rate_24h: h.success_rate_24h ?? null, | |
| 35 | + requests_24h: h.requests_24h ?? null, | |
| 36 | + items_24h: r.records_24h ?? 0, | |
| 37 | + normalized_24h: r.normalized_24h ?? 0, | |
| 38 | + duplicates_24h: r.duplicates_24h ?? 0, | |
| 39 | + latest_success: r.last_success_at, | |
| 40 | + last_run: r.last_started_at, | |
| 41 | + last_run_status: r.last_run_status, | |
| 42 | + last_error: r.last_error, | |
| 43 | + http_errors: h.http_errors ?? {}, | |
| 44 | + parse_failure_rate: h.parse_failure_rate ?? null, | |
| 45 | + schema_drift: h.schema_drift ?? [], | |
| 46 | + latency_ms_avg: h.latency_ms_avg ?? null, | |
| 47 | + rate_limited_24h: h.rate_limited_24h ?? 0, | |
| 48 | + challenges_24h: h.challenges_24h ?? 0, | |
| 49 | + data_freshness: h.data_freshness ?? null, | |
| 50 | + firecrawl: { credits_24h: r.fc_credits_24h ?? 0, success_rate: h.firecrawl_success_rate ?? null }, | |
| 51 | + scrapfly: { credits_24h: r.sf_credits_24h ?? 0, fallback_rate: h.scrapfly_fallback_rate ?? null }, | |
| 52 | + missing_requirements: h.missing_requirements ?? [], | |
| 53 | + backfill: r.backfill_id ? { id: r.backfill_id, status: r.backfill_status, percent: r.backfill_percent, pages: r.backfill_pages, items: r.backfill_items, reached_date: r.backfill_reached_date, updated_at: r.backfill_updated_at } : null, | |
| 54 | + anomalies: r.last_anomalies ?? [], | |
| 55 | + }; | |
| 56 | + }), | |
| 57 | + }); | |
| 58 | +} | |
added
apps/web/src/components/admin/connector-bits.tsx
+52 −0
@@ -0,0 +1,52 @@ | ||
| 1 | +import type { ReactNode } from 'react'; | |
| 2 | +import { cn } from '@/lib/format'; | |
| 3 | + | |
| 4 | +/** Operator labels for health statuses (SPEC §12): UP · DEGRADED · BROKEN · DISABLED · MAINTENANCE. */ | |
| 5 | +const LABELS: Record<string, { label: string; cls: string }> = { | |
| 6 | + healthy: { label: 'UP', cls: 'bg-gain-bg text-gain' }, | |
| 7 | + degraded: { label: 'DEGRADED', cls: 'bg-alert-bg text-alert' }, | |
| 8 | + failing: { label: 'BROKEN', cls: 'bg-loss-bg text-loss' }, | |
| 9 | + paused: { label: 'DISABLED', cls: 'bg-inset text-muted' }, | |
| 10 | + disabled: { label: 'DISABLED', cls: 'bg-inset text-muted' }, | |
| 11 | + maintenance: { label: 'MAINTENANCE', cls: 'bg-index-bg text-index' }, | |
| 12 | + unknown: { label: 'UNKNOWN', cls: 'bg-inset text-subtle' }, | |
| 13 | +}; | |
| 14 | + | |
| 15 | +export function HealthLabel({ status, small = true }: { status: string | null | undefined; small?: boolean }) { | |
| 16 | + const l = LABELS[status ?? 'unknown'] ?? LABELS.unknown!; | |
| 17 | + return <span className={cn('inline-flex rounded-sm font-semibold tracking-wide', small ? 'px-1.5 py-0.5 text-[10px]' : 'px-2 py-1 text-xs', l.cls)}>{l.label}</span>; | |
| 18 | +} | |
| 19 | + | |
| 20 | +/** Source favicon (Google s2 service, falls back to an initial). */ | |
| 21 | +export function SourceLogo({ domain, name, logoUrl, size = 18 }: { domain: string | null | undefined; name: string; logoUrl?: string | null; size?: number }) { | |
| 22 | + const src = logoUrl ?? (domain ? `https://www.google.com/s2/favicons?domain=${encodeURIComponent(domain)}&sz=${size * 2}` : null); | |
| 23 | + if (!src) return <span className="inline-flex h-[18px] w-[18px] items-center justify-center rounded-sm bg-inset text-[10px] font-semibold text-muted">{name.slice(0, 1).toUpperCase()}</span>; | |
| 24 | + // eslint-disable-next-line @next/next/no-img-element | |
| 25 | + return <img src={src} alt="" width={size} height={size} loading="lazy" className="inline-block rounded-sm bg-white/5" />; | |
| 26 | +} | |
| 27 | + | |
| 28 | +export function Pct({ value, digits = 0 }: { value: unknown; digits?: number }) { | |
| 29 | + const x = Number(value); | |
| 30 | + return <>{Number.isFinite(x) ? `${(x * 100).toFixed(digits)}%` : '—'}</>; | |
| 31 | +} | |
| 32 | + | |
| 33 | +/** Thin progress bar for backfill campaigns. */ | |
| 34 | +export function Progress({ percent, status }: { percent: number | null | undefined; status?: string | null }) { | |
| 35 | + const p = percent === null || percent === undefined ? null : Math.max(0, Math.min(100, Number(percent))); | |
| 36 | + return ( | |
| 37 | + <div className="min-w-[72px]"> | |
| 38 | + <div className="h-1.5 w-full overflow-hidden rounded-full bg-inset"> | |
| 39 | + <div className={cn('h-full rounded-full', status === 'completed' ? 'bg-gain' : status === 'failed' ? 'bg-loss' : status === 'paused' ? 'bg-alert' : 'bg-index')} style={{ width: `${p ?? (status === 'running' ? 8 : 0)}%` }} /> | |
| 40 | + </div> | |
| 41 | + <div className="mt-0.5 text-[10px] text-subtle">{p === null ? (status ?? '—') : `${p.toFixed(0)}% · ${status ?? ''}`}</div> | |
| 42 | + </div> | |
| 43 | + ); | |
| 44 | +} | |
| 45 | + | |
| 46 | +export function Json({ value, className }: { value: unknown; className?: string }) { | |
| 47 | + return <pre className={cn('max-h-[70vh] overflow-auto rounded-md border border-border bg-sunken p-3 font-mono text-[11px] leading-relaxed', className)}>{typeof value === 'string' ? value : JSON.stringify(value, null, 2)}</pre>; | |
| 48 | +} | |
| 49 | + | |
| 50 | +export function Chip({ children, tone = 'neutral' }: { children: ReactNode; tone?: 'neutral' | 'index' | 'alert' }) { | |
| 51 | + return <span className={cn('inline-flex rounded-sm px-1.5 py-0.5 text-[10px]', tone === 'index' ? 'bg-index-bg text-index' : tone === 'alert' ? 'bg-alert-bg text-alert' : 'bg-inset text-muted')}>{children}</span>; | |
| 52 | +} | |
modified
apps/web/src/components/admin/shell.tsx
+2 −1
@@ -5,6 +5,7 @@ import { cn } from '@/lib/format'; | ||
| 5 | 5 | export const ADMIN_NAV = [ |
| 6 | 6 | { href: '/admin', label: 'Overview' }, |
| 7 | 7 | { href: '/admin/connectors', label: 'Connectors' }, |
| 8 | + { href: '/admin/coverage', label: 'Coverage' }, | |
| 8 | 9 | { href: '/admin/data-quality', label: 'Data quality' }, |
| 9 | 10 | { href: '/admin/costs', label: 'Costs' }, |
| 10 | 11 | { href: '/admin/taxonomy', label: 'Taxonomy' }, |
@@ -41,7 +42,7 @@ export function AdminShell({ children, title, subtitle, actions, current }: { ch | ||
| 41 | 42 | |
| 42 | 43 | export function StatusPill({ status }: { status: string | null | undefined }) { |
| 43 | 44 | const s = status ?? 'unknown'; |
| 44 | − const tone: Record<string, string> = { healthy: 'bg-gain-bg text-gain', active: 'bg-gain-bg text-gain', success: 'bg-gain-bg text-gain', valid: 'bg-gain-bg text-gain', degraded: 'bg-alert-bg text-alert', partial: 'bg-alert-bg text-alert', paused: 'bg-alert-bg text-alert', flagged: 'bg-alert-bg text-alert', pending: 'bg-alert-bg text-alert', running: 'bg-index-bg text-index', failing: 'bg-loss-bg text-loss', failed: 'bg-loss-bg text-loss', excluded: 'bg-loss-bg text-loss', disabled: 'bg-inset text-muted', unknown: 'bg-inset text-muted' }; | |
| 45 | + const tone: Record<string, string> = { maintenance: 'bg-index-bg text-index', completed: 'bg-gain-bg text-gain', healthy: 'bg-gain-bg text-gain', active: 'bg-gain-bg text-gain', success: 'bg-gain-bg text-gain', valid: 'bg-gain-bg text-gain', degraded: 'bg-alert-bg text-alert', partial: 'bg-alert-bg text-alert', paused: 'bg-alert-bg text-alert', flagged: 'bg-alert-bg text-alert', pending: 'bg-alert-bg text-alert', running: 'bg-index-bg text-index', failing: 'bg-loss-bg text-loss', failed: 'bg-loss-bg text-loss', excluded: 'bg-loss-bg text-loss', disabled: 'bg-inset text-muted', unknown: 'bg-inset text-muted' }; | |
| 45 | 46 | return <span className={cn('inline-flex rounded-sm px-1.5 py-0.5 text-[11px] font-medium', tone[s] ?? 'bg-inset text-muted')}>{s}</span>; |
| 46 | 47 | } |
| 47 | 48 | |
modified
apps/web/src/lib/admin/actions.ts
+15 −6
@@ -4,7 +4,7 @@ import { cookies } from 'next/headers'; | ||
| 4 | 4 | import { redirect } from 'next/navigation'; |
| 5 | 5 | import { revalidatePath } from 'next/cache'; |
| 6 | 6 | import { z } from 'zod'; |
| 7 | −import { getDb, connectors, connectorRuns, sales, normalizedRecords, taxonomyProposals, categories, auditLog, events, eq, sql } from '@rareindex/database'; | |
| 7 | +import { getDb, connectors, connectorRuns, connectorBackfills, sales, normalizedRecords, taxonomyProposals, categories, auditLog, events, eq, and, inArray, sql } from '@rareindex/database'; | |
| 8 | 8 | import { newId } from '@rareindex/shared'; |
| 9 | 9 | import { ADMIN_COOKIE, adminCookieValue, isAdmin, tokenMatches } from './auth'; |
| 10 | 10 | import { enqueue, QUEUES } from './queue'; |
@@ -33,9 +33,12 @@ export async function adminLogout(): Promise<void> { | ||
| 33 | 33 | redirect('/admin/login'); |
| 34 | 34 | } |
| 35 | 35 | |
| 36 | −const ConnectorAction = z.enum(['run', 'probe', 'recrawl', 'retry', 'pause', 'resume']); | |
| 36 | +const ConnectorAction = z.enum(['run', 'probe', 'recrawl', 'backfill', 'backfill_pause', 'backfill_reset', 'retry', 'pause', 'resume', 'maintenance']); | |
| 37 | 37 | |
| 38 | −/** Connector control-center actions (§144): run/recrawl/probe enqueue `crawl.run`; pause/resume flip status. */ | |
| 38 | +/** | |
| 39 | + * Connector control-center actions (§144, SPEC §29): run/probe/backfill enqueue `crawl.run`; | |
| 40 | + * pause/resume/maintenance flip status; backfill_pause/backfill_reset manage the resumable campaign. | |
| 41 | + */ | |
| 39 | 42 | export async function connectorAction(formData: FormData): Promise<void> { |
| 40 | 43 | await guard(); |
| 41 | 44 | const id = String(formData.get('id') ?? ''); |
@@ -43,11 +46,17 @@ export async function connectorAction(formData: FormData): Promise<void> { | ||
| 43 | 46 | const db = getDb(); |
| 44 | 47 | const [c] = await db.select().from(connectors).where(eq(connectors.id, id)).limit(1); |
| 45 | 48 | if (!c) throw new Error(`unknown connector ${id}`); |
| 46 | − if (action === 'pause' || action === 'resume') { | |
| 47 | − await db.update(connectors).set({ status: action === 'pause' ? 'paused' : 'active', updatedAt: new Date() }).where(eq(connectors.id, id)); | |
| 49 | + if (action === 'pause' || action === 'resume' || action === 'maintenance') { | |
| 50 | + await db.update(connectors).set({ status: action === 'pause' ? 'paused' : action === 'maintenance' ? 'maintenance' : 'active', updatedAt: new Date() }).where(eq(connectors.id, id)); | |
| 48 | 51 | await audit('connector', id, action, `admin ${action}`); |
| 52 | + } else if (action === 'backfill_pause') { | |
| 53 | + await db.update(connectorBackfills).set({ status: 'paused', updatedAt: new Date() }).where(and(eq(connectorBackfills.connectorId, id), eq(connectorBackfills.status, 'running'))); | |
| 54 | + await audit('connector', id, action, 'admin paused backfill campaign'); | |
| 55 | + } else if (action === 'backfill_reset') { | |
| 56 | + await db.update(connectorBackfills).set({ status: 'failed', lastError: 'reset by admin', finishedAt: new Date(), updatedAt: new Date() }).where(and(eq(connectorBackfills.connectorId, id), inArray(connectorBackfills.status, ['running', 'paused']))); | |
| 57 | + await audit('connector', id, action, 'admin reset backfill campaign'); | |
| 49 | 58 | } else { |
| 50 | − const mode = action === 'recrawl' ? 'backfill' : action === 'probe' ? 'probe' : 'incremental'; | |
| 59 | + const mode = action === 'recrawl' || action === 'backfill' ? 'backfill' : action === 'probe' ? 'probe' : 'incremental'; | |
| 51 | 60 | const payload: Record<string, unknown> = { connectorId: id, mode, trigger: action === 'retry' ? 'retry' : 'manual', requestedAt: new Date().toISOString() }; |
| 52 | 61 | if (action === 'probe') payload.limit = 25; |
| 53 | 62 | if (action === 'retry') { |
modified
apps/web/src/lib/admin/queries.ts
+63 −5
@@ -43,17 +43,25 @@ export async function connectorsOverview() { | ||
| 43 | 43 | greatest(1, extract(epoch from (max(coalesce(finished_at, now())) - min(started_at))) / 60.0) as minutes |
| 44 | 44 | from connector_runs where started_at >= now() - interval '24 hours' group by connector_id), |
| 45 | 45 | conf as ( |
| 46 | − select connector_id, avg((payload->>'confidence')::float) as parser_confidence, count(*)::int as normalized_24h from normalized_records where created_at >= now() - interval '24 hours' group by connector_id) | |
| 47 | − select c.id, c.display_name, c.source_id, c.status, c.priority, c.engine_priority, c.categories, c.refresh_frequency_minutes, c.schema_version, c.connector_version, c.last_run_at, c.last_success_at, c.next_run_at, | |
| 46 | + select connector_id, avg((payload->>'confidence')::float) as parser_confidence, count(*)::int as normalized_24h from normalized_records where created_at >= now() - interval '24 hours' group by connector_id), | |
| 47 | + spend as ( | |
| 48 | + select connector_id, sum(credits) filter (where kind = 'firecrawl')::float as fc_credits_24h, sum(credits) filter (where kind = 'scrapfly')::float as sf_credits_24h from costs where occurred_at >= now() - interval '24 hours' and connector_id is not null group by connector_id), | |
| 49 | + bf as ( | |
| 50 | + select distinct on (connector_id) connector_id, id as backfill_id, status as backfill_status, percent as backfill_percent, pages_processed as backfill_pages, items_processed as backfill_items, updated_at as backfill_updated_at, reached_date as backfill_reached_date from connector_backfills order by connector_id, started_at desc) | |
| 51 | + select c.id, c.display_name, c.source_id, c.status, c.priority, c.engine_priority, c.categories, c.regions, c.refresh_frequency_minutes, c.schema_version, c.connector_version, c.last_run_at, c.last_success_at, c.next_run_at, c.meta, | |
| 48 | 52 | h.status as health_status, h.computed_at as health_at, h.health, |
| 49 | 53 | lr.id as last_run_id, lr.status as last_run_status, lr.error as last_error, lr.started_at as last_started_at, lr.finished_at as last_finished_at, lr.pages_attempted as last_pages, lr.records_raw as last_records, lr.anomalies as last_anomalies, |
| 50 | 54 | d.pages as pages_24h, d.records as records_24h, d.duplicates as duplicates_24h, d.credits as credits_24h, (coalesce(d.pages,0) / coalesce(d.minutes,1)) as pages_per_min, |
| 51 | − cf.parser_confidence, cf.normalized_24h | |
| 55 | + cf.parser_confidence, cf.normalized_24h, | |
| 56 | + sp.fc_credits_24h, sp.sf_credits_24h, | |
| 57 | + bf.backfill_id, bf.backfill_status, bf.backfill_percent, bf.backfill_pages, bf.backfill_items, bf.backfill_updated_at, bf.backfill_reached_date | |
| 52 | 58 | from connectors c |
| 53 | 59 | left join connector_health h on h.connector_id = c.id |
| 54 | 60 | left join last_run lr on lr.connector_id = c.id |
| 55 | 61 | left join day d on d.connector_id = c.id |
| 56 | 62 | left join conf cf on cf.connector_id = c.id |
| 63 | + left join spend sp on sp.connector_id = c.id | |
| 64 | + left join bf on bf.connector_id = c.id | |
| 57 | 65 | order by c.priority = 'high' desc, c.id`); |
| 58 | 66 | } |
| 59 | 67 | |
@@ -67,9 +75,59 @@ export async function connectorDetail(id: string) { | ||
| 67 | 75 | const normalized = await run(sql`select status, count(*)::int as n from normalized_records where connector_id = ${id} group by status`); |
| 68 | 76 | const normalizedSample = await run(sql`select id, kind, status, match_method, match_confidence, asset_id, reject_reason, created_at, payload->>'rawTitle' as raw_title, payload->>'price' as price, payload->>'currency' as currency from normalized_records where connector_id = ${id} order by created_at desc limit 10`); |
| 69 | 77 | const costs = await run(sql`select date_trunc('day', occurred_at)::date as day, kind, sum(credits)::float as credits, sum(usd_est)::float as usd from costs where connector_id = ${id} and occurred_at >= now() - interval '30 days' group by 1, 2 order by 1 desc`); |
| 70 | − const outputs = await one(sql`select (select count(*)::int from sales where connector_id = ${id}) as sales, (select count(*)::int from listings where connector_id = ${id}) as listings, (select count(*)::int from price_observations where connector_id = ${id}) as observations`); | |
| 78 | + const outputs = await one(sql`select (select count(*)::int from sales where connector_id = ${id}) as sales, (select count(*)::int from listings where connector_id = ${id}) as listings, (select count(*)::int from price_observations where connector_id = ${id}) as observations, (select count(*)::int from certificates where ${id} = any(source_ids) or last_source_url like '%' || ${connector.source_id as string} || '%') as certificates`); | |
| 79 | + const backfills = await run(sql`select * from connector_backfills where connector_id = ${id} order by started_at desc limit 10`); | |
| 80 | + const fieldStats = await run(sql`select field, day::text as day, total, nulls from connector_field_stats where connector_id = ${id} and day >= (current_date - interval '7 days')::date order by day desc, field`); | |
| 71 | 81 | const anomalies = runs.flatMap((r) => (Array.isArray(r.anomalies) ? (r.anomalies as string[]) : [])).slice(0, 30); |
| 72 | − return { connector, source, health, runs, rawSample, normalized, normalizedSample, costs, outputs, anomalies }; | |
| 82 | + return { connector, source, health, runs, rawSample, normalized, normalizedSample, costs, outputs, anomalies, backfills, fieldStats }; | |
| 83 | +} | |
| 84 | + | |
| 85 | +export async function rawRecordDetail(id: string) { | |
| 86 | + return one(sql`select id, connector_id, source_id, run_id, engine, url, external_id, kind, fetched_at, content_hash, http_status, payload, snapshot_ref, parser_version, connector_version, processed_at, process_error from raw_records where id = ${id}`); | |
| 87 | +} | |
| 88 | + | |
| 89 | +export async function normalizedRecordDetail(id: string) { | |
| 90 | + return one(sql`select n.*, r.url as raw_url, r.engine as raw_engine from normalized_records n left join raw_records r on r.id = n.raw_record_id where n.id = ${id}`); | |
| 91 | +} | |
| 92 | + | |
| 93 | +/** Global coverage dashboard (SPEC §30). */ | |
| 94 | +export async function coverage() { | |
| 95 | + const totals = await one(sql`select | |
| 96 | + (select count(*)::int from connectors) as connectors, | |
| 97 | + (select count(*)::int from connectors where status = 'active') as connectors_active, | |
| 98 | + (select count(*)::int from listings where availability = 'available') as live_listings, | |
| 99 | + (select count(*)::int from sales) as sales, | |
| 100 | + (select count(*)::int from sales where sale_date < now() - interval '1 year') as sales_older_1y, | |
| 101 | + (select min(sale_date) from sales) as oldest_sale, | |
| 102 | + (select count(*)::int from auction_lots) as auction_lots, | |
| 103 | + (select count(distinct auction_house) from auctions) as auction_houses, | |
| 104 | + (select count(distinct grader) from certificates) as graders_with_certs, | |
| 105 | + (select count(*)::int from certificates) as certificates, | |
| 106 | + (select count(*)::int from population_reports) as population_reports, | |
| 107 | + (select count(*)::int from assets) as assets, | |
| 108 | + (select count(*)::int from asset_variants) as variants, | |
| 109 | + (select count(*)::int from price_observations) as observations, | |
| 110 | + (select count(*)::int from raw_records where fetched_at >= now() - interval '24 hours') as raw_24h, | |
| 111 | + (select count(*)::int from normalized_records where created_at >= now() - interval '24 hours') as normalized_24h, | |
| 112 | + (select count(*)::int from sales where created_at >= now() - interval '24 hours') as sales_24h, | |
| 113 | + (select count(*)::int from listings where first_seen_at >= now() - interval '24 hours') as listings_24h, | |
| 114 | + (select count(*)::int from connector_backfills where status = 'running') as backfills_running, | |
| 115 | + (select count(*)::int from connector_backfills where status = 'completed') as backfills_completed`); | |
| 116 | + const byStatus = await run(sql`select coalesce(h.status, 'unknown') as status, count(*)::int as n from connectors c left join connector_health h on h.connector_id = c.id group by 1 order by n desc`); | |
| 117 | + const byCategory = await run(sql`select cat as category, count(distinct c.id)::int as connectors, count(distinct c.id) filter (where h.status = 'healthy')::int as healthy from connectors c cross join unnest(c.categories) as cat left join connector_health h on h.connector_id = c.id group by 1 order by connectors desc, 1`); | |
| 118 | + const byCountry = await run(sql`select coalesce(c.meta->>'country', c.regions[1], 'global') as country, count(*)::int as connectors from connectors c group by 1 order by connectors desc`); | |
| 119 | + const bySourceType = await run(sql`select s.source_type, count(*)::int as connectors from connectors c join sources s on s.id = c.source_id group by 1 order by 2 desc`); | |
| 120 | + const byEngine = await run(sql`select engine_priority[1] as engine, count(*)::int as connectors from connectors group by 1 order by 2 desc`); | |
| 121 | + const byCapability = await run(sql`select cap as capability, count(*)::int as connectors from connectors c cross join jsonb_array_elements_text(coalesce(c.meta->'capabilities', '[]'::jsonb)) as cap group by 1 order by 2 desc`); | |
| 122 | + const salesByFamily = await run(sql`select a.family_slug, count(*)::int as sales, count(*) filter (where s.sale_date >= now() - interval '30 days')::int as sales_30d, min(s.sale_date)::date as oldest from sales s join assets a on a.id = s.asset_id group by 1 order by sales desc`); | |
| 123 | + const listingsByFamily = await run(sql`select a.family_slug, count(*)::int as live from listings l join assets a on a.id = l.asset_id where l.availability = 'available' group by 1 order by live desc`); | |
| 124 | + const daily = await run(sql`select d::date as day, | |
| 125 | + (select count(*)::int from raw_records r where r.fetched_at >= d and r.fetched_at < d + interval '1 day') as raw, | |
| 126 | + (select count(*)::int from sales s where s.created_at >= d and s.created_at < d + interval '1 day') as sales, | |
| 127 | + (select count(*)::int from listings l where l.first_seen_at >= d and l.first_seen_at < d + interval '1 day') as listings | |
| 128 | + from generate_series(current_date - interval '13 days', current_date, interval '1 day') d order by 1`); | |
| 129 | + const salesByYear = await run(sql`select extract(year from sale_date)::int as year, count(*)::int as sales from sales group by 1 order by 1`); | |
| 130 | + return { totals, byStatus, byCategory, byCountry, bySourceType, byEngine, byCapability, salesByFamily, listingsByFamily, daily, salesByYear }; | |
| 73 | 131 | } |
| 74 | 132 | |
| 75 | 133 | export async function costsOverview(days = 30) { |
modified
connectors/README.md
+4 −0
@@ -16,3 +16,7 @@ Rules: never invent data, keep source quirks inside the connector, store the sou | ||
| 16 | 16 | keep the native currency, respect robots/terms/rate limits, and never bypass access controls (§179). |
| 17 | 17 | |
| 18 | 18 | Module contract: `export default (meta: ConnectorMeta) => RareIndexConnector`. |
| 19 | + | |
| 20 | +Full guide: `docs/connectors/ADDING_A_CONNECTOR.md` (scaffold with `pnpm connector:new`, adapters for Shopify/ | |
| 21 | +WooCommerce/sitemaps/RSS/PDF/JSON-LD, per-host policies in `domains.json` + `domains.d/`, source catalogue in | |
| 22 | +`data/sources/entries/`, resumable backfills, health and drift monitoring). | |
added
connectors/api/1stdibs/README.md
+13 −0
@@ -0,0 +1,13 @@ | ||
| 1 | +# 1stdibs — 1stDibs marketplace (dealer listings) | |
| 2 | + | |
| 3 | +Browse pages (`/furniture/seating/`, `/jewelry/`, `/jewelry/watches/wrist-watches/`, | |
| 4 | +`/fashion/handbags-purses-bags/`, `/art/prints-works-on-paper/` … `?page=N`, 60 items, ≤ 50 pages) embed | |
| 5 | +the server-rendered Relay store in `<script id="serverVars_data">`. `parseBrowseHtml` dereferences the | |
| 6 | +`Item` records: title, `serviceId`, display prices in 10 currencies (USD kept), seller company/id, | |
| 7 | +designers, location, period/style line, materials, measurements, photos, sold/hold flags. | |
| 8 | + | |
| 9 | +- One raw record per page (trimmed items, not the 3 MB store), one `listing` per item. | |
| 10 | +- Category from seed vertical + category code + attribute line/title (`design_furniture` / `antiques` / | |
| 11 | + decorative-object slugs / jewelry / watch brands / `luxury_handbags` / art). | |
| 12 | +- Cursor `{seedIndex, page}` with seed rotation; 5 s politeness. | |
| 13 | +- Identifiers: `firstdibs_item_id` (`f_51602042`), `firstdibs_seller_id`. | |
added
connectors/api/1stdibs/index.test.ts
+54 −0
@@ -0,0 +1,54 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 3 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 4 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 5 | +import createConnector, { makeDeref, pageUrl, parseBrowseHtml } from './index.js'; | |
| 6 | + | |
| 7 | +const connector = createConnector(localMeta(metaJson)); | |
| 8 | + | |
| 9 | +describe('1stdibs', () => { | |
| 10 | + runFixtureSuite(connector, it, expect); | |
| 11 | + | |
| 12 | + it('parses Item records out of the serverVars_data Relay store', () => { | |
| 13 | + const html = String((loadFixture('1stdibs', 'seating-chairs-p1').raw.payload as { snapshot?: string }).snapshot ?? ''); | |
| 14 | + const p = parseBrowseHtml(html); | |
| 15 | + expect(p).not.toBeNull(); | |
| 16 | + expect(p!.items.length).toBeGreaterThanOrEqual(2); | |
| 17 | + expect(p!.totalResults).toBeGreaterThan(1000); | |
| 18 | + expect(p!.maxPages).toBe(50); | |
| 19 | + const it0 = p!.items[0]!; | |
| 20 | + expect(it0.serviceId).toMatch(/^f_\d+$/); | |
| 21 | + expect(it0.url).toMatch(/^https:\/\/www\.1stdibs\.com\/furniture\/.+\/id-f_\d+\/$/); | |
| 22 | + expect(it0.prices.USD).toBeGreaterThan(0); | |
| 23 | + expect(it0.prices.EUR).toBeGreaterThan(0); | |
| 24 | + expect(it0.seller.serviceId).toMatch(/^f_\d+$/); | |
| 25 | + expect(it0.images[0]).toMatch(/1stdibscdn\.com/); | |
| 26 | + expect(it0.categoryCode).toMatch(/^F_/); | |
| 27 | + expect(pageUrl('/furniture/seating/', 2)).toBe('https://www.1stdibs.com/furniture/seating/?page=2'); | |
| 28 | + }); | |
| 29 | + | |
| 30 | + it('dereferences __ref / __refs chains', () => { | |
| 31 | + const store = { a: { __typename: 'A', b: { __ref: 'b' }, list: { __refs: ['c', 'c'] } }, b: { __typename: 'B', v: 1 }, c: { __typename: 'C', v: 2 } }; | |
| 32 | + const deref = makeDeref(store as never); | |
| 33 | + expect(deref(store.a)).toEqual({ __typename: 'A', b: { __typename: 'B', v: 1 }, list: [{ __typename: 'C', v: 2 }, { __typename: 'C', v: 2 }] }); | |
| 34 | + expect(parseBrowseHtml('<html><body>no store</body></html>')).toBeNull(); | |
| 35 | + }); | |
| 36 | + | |
| 37 | + it('normalises items into USD dealer listings with designer, seller and identifiers', async () => { | |
| 38 | + const fx = loadFixture('1stdibs', 'seating-chairs-p1'); | |
| 39 | + const out = await connector.normalize(fx.raw); | |
| 40 | + expect(out.length).toBeGreaterThan(0); | |
| 41 | + for (const r of out) { | |
| 42 | + if (r.kind !== 'listing') throw new Error('expected listing'); | |
| 43 | + expect(r.currency).toBe('USD'); | |
| 44 | + expect(r.price).toBeGreaterThan(0); | |
| 45 | + expect(r.attributes.identifiers.firstdibs_item_id).toMatch(/^f_\d+$/); | |
| 46 | + expect(['design_furniture', 'antiques']).toContain(r.attributes.categorySlug); | |
| 47 | + expect(r.seller).toBeTruthy(); | |
| 48 | + } | |
| 49 | + const jw = loadFixture('1stdibs', 'wrist-watches-p1'); | |
| 50 | + const jwOut = await connector.normalize(jw.raw); | |
| 51 | + expect(jwOut.length).toBeGreaterThan(0); | |
| 52 | + for (const r of jwOut) expect(['rolex', 'omega', 'patek_philippe', 'audemars_piguet', 'other_watches']).toContain((r as { attributes: { categorySlug: string } }).attributes.categorySlug); | |
| 53 | + }); | |
| 54 | +}); | |
added
connectors/api/1stdibs/index.ts
+324 −0
@@ -0,0 +1,324 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { designSlug, plainText, readSeedCursor, yearOrDecade, type DesignVertical } from '../_g10-lib/index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * 1stDibs — the largest online marketplace for vintage/antique furniture, design, jewelry, watches, | |
| 8 | + * fashion and art (USD by default). Browse pages (/furniture/seating/, /jewelry/, … ?page=N, 60 items) | |
| 9 | + * embed the server-rendered Relay store in <script id="serverVars_data" type="application/json">; | |
| 10 | + * we read the `Item` records from it (title, serviceId, prices in ten currencies, seller company, | |
| 11 | + * creators/designers, location, period/style attributes, materials, measurements, photos, sold/hold | |
| 12 | + * flags) and never call 1stDibs' GraphQL endpoint ourselves. Asking prices → listings. | |
| 13 | + */ | |
| 14 | +const BASE = 'https://www.1stdibs.com'; | |
| 15 | +const PARSER_VERSION = '1.0.0'; | |
| 16 | +const PAGE_SIZE = 60; | |
| 17 | + | |
| 18 | +export const SeedSchema = z.object({ path: z.string(), slug: z.string().nullable().optional(), vertical: z.enum(['furniture', 'lighting', 'decor', 'art', 'jewelry', 'watches', 'fashion', 'tableware', 'rugs', 'pens', 'unknown']).default('unknown') }); | |
| 19 | +export type Seed = z.infer<typeof SeedSchema>; | |
| 20 | + | |
| 21 | +export const ItemSchema = z.object({ | |
| 22 | + serviceId: z.string(), | |
| 23 | + title: z.string(), | |
| 24 | + url: z.string(), | |
| 25 | + prices: z.record(z.string(), z.number()), | |
| 26 | + amountType: z.string().nullable(), | |
| 27 | + isSold: z.boolean(), | |
| 28 | + isOnHold: z.boolean(), | |
| 29 | + isUnavailable: z.boolean(), | |
| 30 | + isNewListing: z.boolean().nullable(), | |
| 31 | + seller: z.object({ serviceId: z.string().nullable(), company: z.string().nullable() }), | |
| 32 | + creators: z.array(z.string()), | |
| 33 | + country: z.string().nullable(), | |
| 34 | + location: z.string().nullable(), | |
| 35 | + attributesText: z.string().nullable(), | |
| 36 | + materials: z.string().nullable(), | |
| 37 | + description: z.string().nullable(), | |
| 38 | + measurement: z.string().nullable(), | |
| 39 | + categoryCode: z.string().nullable(), | |
| 40 | + browseUrl: z.string().nullable(), | |
| 41 | + vertical: z.string().nullable(), | |
| 42 | + images: z.array(z.string()), | |
| 43 | +}); | |
| 44 | +export type Item = z.infer<typeof ItemSchema>; | |
| 45 | + | |
| 46 | +export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), totalResults: z.number().int().nullable(), maxPages: z.number().int().nullable(), items: z.array(ItemSchema), snapshot: z.string().optional() }); | |
| 47 | +export type PagePayload = z.infer<typeof PagePayloadSchema>; | |
| 48 | + | |
| 49 | +const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pagesPerSeed: z.number().int().min(1).default(1), seedsPerRun: z.number().int().min(1).default(3), maxPagesPerSeed: z.number().int().min(1).default(50) }); | |
| 50 | + | |
| 51 | +type Rec = Record<string, unknown>; | |
| 52 | + | |
| 53 | +/** Relay normalized store → dereferenced view (bounded depth). */ | |
| 54 | +export function makeDeref(store: Record<string, Rec>) { | |
| 55 | + const deref = (v: unknown, depth = 0): unknown => { | |
| 56 | + if (depth > 4 || v === null || typeof v !== 'object') return v; | |
| 57 | + if (Array.isArray(v)) return v.map((x) => deref(x, depth + 1)); | |
| 58 | + const o = v as Rec; | |
| 59 | + if (typeof o.__ref === 'string') return deref(store[o.__ref] ?? null, depth + 1); | |
| 60 | + if (Array.isArray(o.__refs)) return o.__refs.map((r) => deref(store[String(r)] ?? null, depth + 1)); | |
| 61 | + const out: Rec = {}; | |
| 62 | + for (const [k, x] of Object.entries(o)) if (k !== '__id') out[k] = deref(x, depth + 1); | |
| 63 | + return out; | |
| 64 | + }; | |
| 65 | + return deref; | |
| 66 | +} | |
| 67 | + | |
| 68 | +function firstKey(o: Rec, prefix: string): unknown { | |
| 69 | + const k = Object.keys(o).find((x) => x === prefix || x.startsWith(`${prefix}(`)); | |
| 70 | + return k ? o[k] : undefined; | |
| 71 | +} | |
| 72 | + | |
| 73 | +const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null); | |
| 74 | + | |
| 75 | +/** Parse the serverVars_data bootstrap of a browse page. Exported for tests. */ | |
| 76 | +export function parseBrowseHtml(htmlText: string): { items: Item[]; totalResults: number | null; maxPages: number | null } | null { | |
| 77 | + const m = htmlText.match(/<script id="serverVars_data" type="application\/json">([\s\S]*?)<\/script>/); | |
| 78 | + if (!m) return null; | |
| 79 | + let root: Rec; | |
| 80 | + try { | |
| 81 | + root = JSON.parse(m[1]!) as Rec; | |
| 82 | + } catch { | |
| 83 | + return null; | |
| 84 | + } | |
| 85 | + const store = ((root.dbl as Rec | undefined)?.relayData ?? (root.relay as Rec | undefined)?.store ?? null) as Record<string, Rec> | null; | |
| 86 | + if (!store || typeof store !== 'object') return null; | |
| 87 | + const deref = makeDeref(store); | |
| 88 | + const items: Item[] = []; | |
| 89 | + let totalResults: number | null = null; | |
| 90 | + let maxPages: number | null = null; | |
| 91 | + for (const rec of Object.values(store)) { | |
| 92 | + if (!rec || typeof rec !== 'object') continue; | |
| 93 | + if (rec.__typename === 'ItemSearchQueryConnection') { | |
| 94 | + if (typeof rec.totalResults === 'number') totalResults = rec.totalResults; | |
| 95 | + if (typeof rec.displayMaxNumberOfPages === 'number') maxPages = rec.displayMaxNumberOfPages; | |
| 96 | + continue; | |
| 97 | + } | |
| 98 | + if (rec.__typename !== 'Item' || typeof rec.serviceId !== 'string' || typeof rec.title !== 'string') continue; | |
| 99 | + const dp = deref(firstKey(rec, 'displayPrice')) as Array<{ convertedAmountList?: Array<{ amount?: number; currency?: string }>; amountType?: string }> | undefined; | |
| 100 | + const prices: Record<string, number> = {}; | |
| 101 | + let amountType: string | null = null; | |
| 102 | + for (const d of dp ?? []) { | |
| 103 | + amountType ??= d.amountType ?? null; | |
| 104 | + for (const c of d.convertedAmountList ?? []) if (c.currency && typeof c.amount === 'number' && c.amount > 0) prices[c.currency] ??= c.amount; | |
| 105 | + } | |
| 106 | + const track = rec.ecommerceTrackingParams as { price?: number; convertedAmounts?: Record<string, number> } | undefined; | |
| 107 | + if (!Object.keys(prices).length && track?.convertedAmounts) for (const [k, v] of Object.entries(track.convertedAmounts)) if (typeof v === 'number' && v > 0) prices[k] = v; | |
| 108 | + const seller = deref(rec.seller) as { serviceId?: string; sellerProfile?: { company?: string } } | null; | |
| 109 | + const creators = (deref(rec.creators) as Array<{ creator?: { displayName?: string } }> | null) ?? []; | |
| 110 | + const address = deref(rec.address) as { englishCountryName?: string } | null; | |
| 111 | + const qv = deref(rec.quickViewDisplay) as { paragraphs?: Array<{ key?: string; text?: string }> } | null; | |
| 112 | + const para = (key: string) => str(qv?.paragraphs?.find((p) => p.key === key)?.text); | |
| 113 | + const meas = deref(rec.measurement) as Rec | null; | |
| 114 | + const measList = meas ? (firstKey(meas, 'display') as Array<{ unit?: string; value?: string }> | undefined) : undefined; | |
| 115 | + const photos = (deref(firstKey(rec, 'photos')) as Array<{ masterOrZoomPath?: string; versions?: Array<{ webPath?: string }> }> | null) ?? []; | |
| 116 | + const images = photos.map((p) => p.versions?.find((v) => v.webPath?.includes('width=768'))?.webPath ?? p.masterOrZoomPath ?? null).filter((x): x is string => Boolean(x)); | |
| 117 | + const link = deref(rec.linkData) as { path?: string } | null; | |
| 118 | + const path = str(rec.localizedPdpUrl) ?? str(link?.path); | |
| 119 | + if (!path) continue; | |
| 120 | + items.push({ | |
| 121 | + serviceId: rec.serviceId, | |
| 122 | + title: rec.title, | |
| 123 | + url: path.startsWith('http') ? path : `${BASE}${path}`, | |
| 124 | + prices, | |
| 125 | + amountType, | |
| 126 | + isSold: rec.isSold === true, | |
| 127 | + isOnHold: rec.isOnHold === true, | |
| 128 | + isUnavailable: rec.isUnavailable === true, | |
| 129 | + isNewListing: typeof rec.isNewListing === 'boolean' ? rec.isNewListing : null, | |
| 130 | + seller: { serviceId: str(seller?.serviceId), company: str(seller?.sellerProfile?.company) }, | |
| 131 | + creators: creators.map((c) => str(c?.creator?.displayName)).filter((x): x is string => Boolean(x)), | |
| 132 | + country: str(address?.englishCountryName), | |
| 133 | + location: para('location'), | |
| 134 | + attributesText: para('attributes'), | |
| 135 | + materials: para('materials'), | |
| 136 | + description: plainText(para('description'), 1200), | |
| 137 | + measurement: str(measList?.find((x) => x.unit === 'IN')?.value ?? measList?.[0]?.value), | |
| 138 | + categoryCode: str(rec.categoryCode), | |
| 139 | + browseUrl: str(rec.browseUrl), | |
| 140 | + vertical: str(rec.vertical), | |
| 141 | + images: [...new Set(images)].slice(0, 4), | |
| 142 | + }); | |
| 143 | + } | |
| 144 | + return { items, totalResults, maxPages }; | |
| 145 | +} | |
| 146 | + | |
| 147 | +export function pageUrl(seedPath: string, page: number): string { | |
| 148 | + const p = seedPath.endsWith('/') ? seedPath : `${seedPath}/`; | |
| 149 | + return `${BASE}${p}${page > 1 ? `?page=${page}` : ''}`; | |
| 150 | +} | |
| 151 | + | |
| 152 | +/** | |
| 153 | + * Fixture helper: rebuild a minimal page whose serverVars_data store only holds the search connection | |
| 154 | + * record plus the first `n` Item records and everything reachable from them through __ref/__refs. | |
| 155 | + */ | |
| 156 | +export function trimStoreHtml(htmlText: string, n = 3): string { | |
| 157 | + const m = htmlText.match(/<script id="serverVars_data" type="application\/json">([\s\S]*?)<\/script>/); | |
| 158 | + if (!m) return ''; | |
| 159 | + const root = JSON.parse(m[1]!) as Rec; | |
| 160 | + const store = ((root.dbl as Rec | undefined)?.relayData ?? {}) as Record<string, Rec>; | |
| 161 | + const keep = new Set<string>(); | |
| 162 | + const queue: string[] = []; | |
| 163 | + for (const [k, v] of Object.entries(store)) { | |
| 164 | + if (v?.__typename === 'ItemSearchQueryConnection') keep.add(k); | |
| 165 | + } | |
| 166 | + let picked = 0; | |
| 167 | + for (const [k, v] of Object.entries(store)) { | |
| 168 | + if (v?.__typename === 'Item' && picked < n) { | |
| 169 | + queue.push(k); | |
| 170 | + picked++; | |
| 171 | + } | |
| 172 | + } | |
| 173 | + const collect = (v: unknown) => { | |
| 174 | + if (!v || typeof v !== 'object') return; | |
| 175 | + if (Array.isArray(v)) return v.forEach(collect); | |
| 176 | + const o = v as Rec; | |
| 177 | + if (typeof o.__ref === 'string') queue.push(o.__ref); | |
| 178 | + if (Array.isArray(o.__refs)) for (const r of o.__refs) queue.push(String(r)); | |
| 179 | + for (const [k, x] of Object.entries(o)) if (k !== '__ref' && k !== '__refs') collect(x); | |
| 180 | + }; | |
| 181 | + while (queue.length) { | |
| 182 | + const k = queue.shift()!; | |
| 183 | + if (keep.has(k) || !store[k]) continue; | |
| 184 | + keep.add(k); | |
| 185 | + collect(store[k]); | |
| 186 | + } | |
| 187 | + const small: Record<string, Rec> = {}; | |
| 188 | + for (const k of keep) { | |
| 189 | + const rec = store[k]!; | |
| 190 | + // the connection record references every item on the page; keep only its scalar fields | |
| 191 | + small[k] = rec.__typename === 'ItemSearchQueryConnection' ? Object.fromEntries(Object.entries(rec).filter(([, v]) => v === null || typeof v !== 'object')) : rec; | |
| 192 | + } | |
| 193 | + return `<!doctype html><html><head><script id="serverVars_data" type="application/json">${JSON.stringify({ dbl: { relayData: small } })}</script></head><body></body></html>`; | |
| 194 | +} | |
| 195 | + | |
| 196 | +function verticalFor(seed: Seed, item: Item): DesignVertical { | |
| 197 | + if (seed.vertical !== 'unknown') return seed.vertical; | |
| 198 | + const v = item.vertical ?? ''; | |
| 199 | + const code = item.categoryCode ?? ''; | |
| 200 | + if (v === 'jewelry') return code.startsWith('J_WAT') ? 'watches' : 'jewelry'; | |
| 201 | + if (v === 'fashion') return 'fashion'; | |
| 202 | + if (v === 'art') return 'art'; | |
| 203 | + if (code.startsWith('F_LIG')) return 'lighting'; | |
| 204 | + if (code.startsWith('F_DEC') || code.startsWith('F_SER')) return 'decor'; | |
| 205 | + if (code.startsWith('F_RUG') || code.startsWith('F_TEX')) return 'rugs'; | |
| 206 | + return v === 'furniture' ? 'furniture' : 'unknown'; | |
| 207 | +} | |
| 208 | + | |
| 209 | +export class FirstDibsConnector extends BaseConnector { | |
| 210 | + readonly version = '1.0.0'; | |
| 211 | + readonly parserVersion = PARSER_VERSION; | |
| 212 | + protected override minIntervalMs = 5000; | |
| 213 | + override readonly urlPatterns = [/^https?:\/\/(www\.)?1stdibs\.com\/(?:furniture|jewelry|fashion|art)\/.+\/id-([a-z]_\d+)\/?/i]; | |
| 214 | + private readonly cfg: z.infer<typeof ConfigSchema>; | |
| 215 | + | |
| 216 | + constructor(meta: ConnectorMeta) { | |
| 217 | + super(meta); | |
| 218 | + this.cfg = ConfigSchema.parse(meta.config); | |
| 219 | + } | |
| 220 | + | |
| 221 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 222 | + const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds; | |
| 223 | + const backfill = ctx.options.mode === 'backfill'; | |
| 224 | + const maxPages = backfill ? Math.min(this.policy.backfillMaxPages, this.cfg.maxPagesPerSeed) : this.cfg.pagesPerSeed; | |
| 225 | + const start = readSeedCursor(ctx.options.cursor, seeds.length); | |
| 226 | + const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun); | |
| 227 | + let count = 0; | |
| 228 | + let items = 0; | |
| 229 | + for (let k = 0; k < seedsThisRun; k++) { | |
| 230 | + const seedIndex = (start.seedIndex + k) % seeds.length; | |
| 231 | + const seed = seeds[seedIndex]!; | |
| 232 | + let page = k === 0 ? start.page : 1; | |
| 233 | + for (; page <= maxPages; page++) { | |
| 234 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 235 | + const url = pageUrl(seed.path, page); | |
| 236 | + await this.throttle(url); | |
| 237 | + const res = await ctx.fetch(url, { | |
| 238 | + engines: ['api'], | |
| 239 | + responseType: 'text', | |
| 240 | + timeoutMs: 90_000, | |
| 241 | + expect: ['title', 'price', 'currency'], | |
| 242 | + parse: (r) => { | |
| 243 | + const p = r.html ? parseBrowseHtml(r.html) : null; | |
| 244 | + const priced = p?.items.find((i) => i.prices.USD); | |
| 245 | + return p?.items.length ? { title: p.items[0]!.title, price: priced?.prices.USD ?? null, currency: priced ? 'USD' : null } : null; | |
| 246 | + }, | |
| 247 | + minQuality: 0.3, | |
| 248 | + }); | |
| 249 | + const parsed = res.success && res.html ? parseBrowseHtml(res.html) : null; | |
| 250 | + if (!parsed) { | |
| 251 | + ctx.anomaly(res.success ? 'schema_drift' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'serverVars_data missing'}`); | |
| 252 | + break; | |
| 253 | + } | |
| 254 | + if (!parsed.items.length) { | |
| 255 | + if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no Item records`); | |
| 256 | + break; | |
| 257 | + } | |
| 258 | + count++; | |
| 259 | + items += parsed.items.length; | |
| 260 | + const payload: PagePayload = { kind: 'listing_page', url, seed, page, totalResults: parsed.totalResults, maxPages: parsed.maxPages, items: parsed.items }; | |
| 261 | + yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 262 | + await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() }); | |
| 263 | + await ctx.progress({ page, totalPages: parsed.maxPages, itemsProcessed: items }); | |
| 264 | + if (parsed.items.length < PAGE_SIZE || (parsed.maxPages !== null && page >= parsed.maxPages)) break; | |
| 265 | + } | |
| 266 | + const nextSeed = (seedIndex + 1) % seeds.length; | |
| 267 | + await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) }); | |
| 268 | + } | |
| 269 | + } | |
| 270 | + | |
| 271 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 272 | + const p = PagePayloadSchema.parse(raw.payload); | |
| 273 | + const out: NormalizedRecord[] = []; | |
| 274 | + for (const it of p.items) { | |
| 275 | + const vertical = verticalFor(p.seed, it); | |
| 276 | + const categorySlug = p.seed.slug ?? designSlug(it.attributesText ?? it.browseUrl, `${it.title} ${it.attributesText ?? ''}`, vertical); | |
| 277 | + if (!categorySlug) continue; | |
| 278 | + const currency = it.prices.USD ? 'USD' : Object.keys(it.prices)[0] ?? null; | |
| 279 | + const price = currency ? it.prices[currency]! : null; | |
| 280 | + const { year, decade } = yearOrDecade(`${it.title} ${it.attributesText ?? ''}`); | |
| 281 | + const period = it.attributesText?.match(/\b(1[5-9]th Century|20th Century|21st Century|Antique|Vintage|Mid-Century Modern|Art Deco|Art Nouveau|Victorian|Georgian|Regency|Louis X[VI]+|Bauhaus|Scandinavian Modern|Hollywood Regency|Brutalist|Space Age|Postmodern|Memphis)\b/i)?.[0] ?? null; | |
| 282 | + const attributes = AssetAttributesSchema.parse({ | |
| 283 | + categorySlug, | |
| 284 | + brand: it.creators[0] ?? null, | |
| 285 | + name: it.title, | |
| 286 | + year, | |
| 287 | + material: it.materials, | |
| 288 | + size: it.measurement, | |
| 289 | + country: null, | |
| 290 | + identifiers: { firstdibs_item_id: it.serviceId, ...(it.seller.serviceId ? { firstdibs_seller_id: it.seller.serviceId } : {}) }, | |
| 291 | + metadata: { decade, period, creators: it.creators, attributes_text: it.attributesText, category_code: it.categoryCode, browse_url: it.browseUrl, vertical: it.vertical, converted_prices: it.prices, amount_type: it.amountType, item_country: it.country, is_new_listing: it.isNewListing }, | |
| 292 | + }); | |
| 293 | + out.push( | |
| 294 | + NormalizedListingSchema.parse({ | |
| 295 | + kind: 'listing', | |
| 296 | + connectorId: this.meta.id, | |
| 297 | + sourceId: this.meta.sourceId, | |
| 298 | + sourceUrl: it.url, | |
| 299 | + externalId: it.serviceId, | |
| 300 | + rawTitle: it.title, | |
| 301 | + description: it.description, | |
| 302 | + imageUrls: it.images, | |
| 303 | + attributes, | |
| 304 | + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, | |
| 305 | + condition: { condition: null, conditionRaw: null, completeness: null }, | |
| 306 | + observedAt: raw.fetchedAt, | |
| 307 | + confidence: price ? 0.82 : 0.6, | |
| 308 | + parserVersion: PARSER_VERSION, | |
| 309 | + listingType: 'fixed_price', | |
| 310 | + price, | |
| 311 | + currency, | |
| 312 | + seller: it.seller.company, | |
| 313 | + location: it.location ?? it.country, | |
| 314 | + availability: it.isSold ? 'sold' : it.isOnHold || it.isUnavailable ? 'ended' : 'available', | |
| 315 | + }), | |
| 316 | + ); | |
| 317 | + } | |
| 318 | + return out; | |
| 319 | + } | |
| 320 | +} | |
| 321 | + | |
| 322 | +export default function createConnector(meta: ConnectorMeta): FirstDibsConnector { | |
| 323 | + return new FirstDibsConnector(meta); | |
| 324 | +} | |
added
connectors/api/1stdibs/meta.json
+47 −0
@@ -0,0 +1,47 @@ | ||
| 1 | +{ | |
| 2 | + "id": "1stdibs", | |
| 3 | + "displayName": "1stDibs (vintage design, antiques, jewelry, watches, fashion & art listings)", | |
| 4 | + "sourceId": "1stdibs", | |
| 5 | + "sourceName": "1stDibs", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://www.1stdibs.com", | |
| 8 | + "module": "api/1stdibs", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["design_furniture", "antiques", "porcelain", "glass_crystal", "silver", "clocks", "jewelry", "gemstones", "rolex", "patek_philippe", "audemars_piguet", "omega", "other_watches", "luxury_handbags", "art", "contemporary_art", "photography"], | |
| 11 | + "regions": ["US", "GB", "FR", "IT", "DE"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["USD"], | |
| 14 | + "supportsListings": true, | |
| 15 | + "supportsSold": false, | |
| 16 | + "supportsAuctions": false, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "medium", | |
| 23 | + "trustScore": 0.8, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.1stdibs.com/about/user-agreement/", | |
| 26 | + "acquisitionMethod": "public HTML — server-rendered Relay store (serverVars_data JSON) on browse pages", | |
| 27 | + "historicalDepth": "none", | |
| 28 | + "accessNotes": "1stDibs browse pages (/furniture/seating/, /furniture/lighting/, /furniture/decorative-objects/, /jewelry/, /jewelry/watches/wrist-watches/, /fashion/handbags-purses-bags/, /art/prints-works-on-paper/ … with ?page=N, 60 items per page, at most 50 pages per browse path as capped by the site) are served to the honest RareIndexBot UA as ~3 MB server-rendered documents whose <script id=\"serverVars_data\"> JSON contains the Relay store used to hydrate the grid. We read the `Item` records from that store: title, serviceId (f_########), the retail display price converted by 1stDibs into USD/EUR/GBP/CAD/AUD/CHF/MXN/NOK/SEK/DKK (USD kept as the listing price, the others in metadata), seller company + dealer id, designer/creator names, dealer city/country, period-style attribute line, materials, measurements, photos, sold/on-hold flags. We never call 1stDibs' GraphQL (/soa/graphql — reserved to the site, robots.txt allows only that exact path) nor the /search/ paths robots.txt disallows; product detail pages are not fetched. The page carries an F5/Shape client-detection script, but no challenge was served to plain HTTPS requests; if 1stDibs starts blocking, the connector fails loudly (schema_drift/page_fetch_failed) — no evasion engine is configured. User Agreement (checked 2026-09-08) has no clause on automated access to public pages. Dealer asking prices only → listings (never sales); 5 s between requests, 1 in flight, 3 browse paths × 1 page per daily run with seed rotation (≈180 listings/run, ~9 MB), backfill capped at the site's 50-page window. Category mapping: seed vertical + category code + attribute line/title → design_furniture (default), antiques (period evidence), porcelain/glass/silver/clocks (decorative objects), jewelry/gemstones, watch brand slugs, luxury_handbags, art/photography/contemporary_art; rugs & textiles skipped.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "pagesPerSeed": 1, | |
| 33 | + "seedsPerRun": 3, | |
| 34 | + "maxPagesPerSeed": 50, | |
| 35 | + "seeds": [ | |
| 36 | + { "path": "/furniture/seating/", "vertical": "furniture" }, | |
| 37 | + { "path": "/furniture/tables/", "vertical": "furniture" }, | |
| 38 | + { "path": "/furniture/storage-case-pieces/", "vertical": "furniture" }, | |
| 39 | + { "path": "/furniture/lighting/", "vertical": "lighting" }, | |
| 40 | + { "path": "/furniture/decorative-objects/", "vertical": "decor" }, | |
| 41 | + { "path": "/jewelry/", "vertical": "jewelry" }, | |
| 42 | + { "path": "/jewelry/watches/wrist-watches/", "vertical": "watches" }, | |
| 43 | + { "path": "/fashion/handbags-purses-bags/", "slug": "luxury_handbags", "vertical": "fashion" }, | |
| 44 | + { "path": "/art/prints-works-on-paper/", "vertical": "art" } | |
| 45 | + ] | |
| 46 | + } | |
| 47 | +} | |
added
connectors/api/401-games/README.md
+44 −0
@@ -0,0 +1,44 @@ | ||
| 1 | +# 401 Games connector (`401-games`) | |
| 2 | + | |
| 3 | +- Source: https://store.401games.ca · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Canada's largest TCG/board-game retailer (Toronto). Public Shopify storefront with one umbrella collection per game plus sports and non-sport cards. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `all-magic-the-gathering` | `magic_the_gathering` | Magic: The Gathering | | |
| 14 | +| `all-pokemon` | `pokemon` | Pokémon | | |
| 15 | +| `all-yugioh` | `yugioh` | Yu-Gi-Oh! | | |
| 16 | +| `all-one-piece-card-game` | `one_piece_card_game` | One Piece | | |
| 17 | +| `all-disney-lorcana` | `disney_lorcana` | Disney Lorcana | | |
| 18 | +| `all-weiss-schwarz` | `weiss_schwarz` | Weiß Schwarz | | |
| 19 | +| `all-digimon-card-game` | `digimon_tcg` | Digimon | | |
| 20 | +| `all-dragon-ball-super` | `dragon_ball_tcg` | Dragon Ball Super | | |
| 21 | +| `all-star-wars-unlimited` | `star_wars_tcg` | Star Wars Unlimited | | |
| 22 | +| `all-cardfight-vanguard` | `other_tcg` | Cardfight!! Vanguard | | |
| 23 | +| `all-riftbound-league-of-legends-tcg` | `other_tcg` | Riftbound | | |
| 24 | +| `all-gundam-card-game` | `other_tcg` | Gundam Card Game | | |
| 25 | +| `all-sports-cards` | `sports_cards` | — | | |
| 26 | +| `all-non-sports-trading-cards` | `non_sport_cards` | — | | |
| 27 | +| `all-board-games` | `board_games` | — | | |
| 28 | + | |
| 29 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 30 | +- `^(?=[\s\S]*\bhockey\b)[\s\S]*\| all-sports-cards$` → `hockey_cards` | |
| 31 | +- `^(?=[\s\S]*\bbaseball\b)[\s\S]*\| all-sports-cards$` → `baseball_cards` | |
| 32 | +- `^(?=[\s\S]*\bbasketball\b)[\s\S]*\| all-sports-cards$` → `basketball_cards` | |
| 33 | +- `^(?=[\s\S]*\bfootball\b)[\s\S]*\| all-sports-cards$` → `football_cards` | |
| 34 | +- `^(?=[\s\S]*\b(soccer|premier league|uefa|fifa)\b)[\s\S]*\| all-sports-cards$` → `soccer_cards` | |
| 35 | +- `^(?=[\s\S]*\b(formula 1|formula one|\bf1\b)\b)[\s\S]*\| all-sports-cards$` → `f1_cards` | |
| 36 | + | |
| 37 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live` | |
| 38 | + | |
| 39 | +## Access & compliance | |
| 40 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: supplies/dice/modelling collections, event tickets, staff-loyalty and "acorn" migration collections, the 0.00-priced Easy-Buy buylist placeholders (dropped: no price). | |
| 41 | + | |
| 42 | +## Fixtures & tests | |
| 43 | +`data/fixtures/401-games/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts 401-games`), trimmed single-product payloads incl. a sold-out variant and a graded item. | |
| 44 | +`pnpm vitest run connectors/api/401-games` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts 401-games`. | |
added
connectors/api/401-games/index.test.ts
+46 −0
@@ -0,0 +1,46 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * 401 Games — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/401-games/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "2023-24 Upper Deck Series 2 Hockey Hobby Box", | |
| 15 | + "productType": "Sports Cards", | |
| 16 | + "collection": "all-sports-cards", | |
| 17 | + "categorySlug": "hockey_cards" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "2024 Topps Chrome Baseball Hobby Box", | |
| 21 | + "collection": "all-sports-cards", | |
| 22 | + "categorySlug": "baseball_cards" | |
| 23 | + }, | |
| 24 | + { | |
| 25 | + "title": "Easy-Buy Buylist Submission (Pokemon)", | |
| 26 | + "collection": "all-pokemon", | |
| 27 | + "categorySlug": null | |
| 28 | + }, | |
| 29 | + { | |
| 30 | + "title": "Pokemon - SWSH Darkness Ablaze Online Pack (Unused Digital Code)", | |
| 31 | + "collection": "all-pokemon", | |
| 32 | + "categorySlug": null | |
| 33 | + }, | |
| 34 | + { | |
| 35 | + "title": "Ultra Pro Deck Box - Black", | |
| 36 | + "collection": "all-magic-the-gathering", | |
| 37 | + "categorySlug": null | |
| 38 | + }, | |
| 39 | + { | |
| 40 | + "title": "Mareep (Japanese) - 036/172 - No Rarity", | |
| 41 | + "collection": "all-pokemon", | |
| 42 | + "categorySlug": "pokemon", | |
| 43 | + "franchise": "Pokémon" | |
| 44 | + } | |
| 45 | + ] | |
| 46 | +}); | |
added
connectors/api/401-games/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * 401 Games — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/401-games/meta.json
+172 −0
@@ -0,0 +1,172 @@ | ||
| 1 | +{ | |
| 2 | + "id": "401-games", | |
| 3 | + "displayName": "401 Games (Canadian TCG & sports-card & board-game store, CAD)", | |
| 4 | + "sourceId": "401-games", | |
| 5 | + "sourceName": "401 Games", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://store.401games.ca", | |
| 8 | + "module": "api/401-games", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "pokemon", | |
| 15 | + "yugioh", | |
| 16 | + "one_piece_card_game", | |
| 17 | + "disney_lorcana", | |
| 18 | + "weiss_schwarz", | |
| 19 | + "digimon_tcg", | |
| 20 | + "dragon_ball_tcg", | |
| 21 | + "star_wars_tcg", | |
| 22 | + "other_tcg", | |
| 23 | + "sports_cards", | |
| 24 | + "non_sport_cards", | |
| 25 | + "board_games", | |
| 26 | + "hockey_cards", | |
| 27 | + "baseball_cards", | |
| 28 | + "basketball_cards", | |
| 29 | + "football_cards", | |
| 30 | + "soccer_cards", | |
| 31 | + "f1_cards" | |
| 32 | + ], | |
| 33 | + "regions": [ | |
| 34 | + "CA" | |
| 35 | + ], | |
| 36 | + "languages": [ | |
| 37 | + "en" | |
| 38 | + ], | |
| 39 | + "currency": [ | |
| 40 | + "CAD" | |
| 41 | + ], | |
| 42 | + "supportsListings": true, | |
| 43 | + "supportsSold": false, | |
| 44 | + "supportsAuctions": false, | |
| 45 | + "supportsImages": true, | |
| 46 | + "supportsCatalog": false, | |
| 47 | + "supportsPopulation": false, | |
| 48 | + "supportsLookup": true, | |
| 49 | + "refreshFrequencyMinutes": 720, | |
| 50 | + "priority": "medium", | |
| 51 | + "trustScore": 0.75, | |
| 52 | + "attributionRequired": true, | |
| 53 | + "termsUrl": "https://store.401games.ca/policies/terms-of-service", | |
| 54 | + "accessNotes": "401 Games (store.401games.ca) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 15 configured collections (all-magic-the-gathering, all-pokemon, all-yugioh, all-one-piece-card-game, all-disney-lorcana, all-weiss-schwarz, all-digimon-card-game, all-dragon-ball-super … (+7 more, see config.collections)) and /products/<handle>.json for URL lookups (~430k products; MTG 158k, Yu-Gi-Oh! 60k, Pokémon 54k). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Third-party grades in titles (PSA/BGS/CGC/ICCS/PMG…) are parsed by parseGradeFromTitle; cert numbers are not extracted. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — supplies/dice/modelling collections, event tickets, staff-loyalty and \"acorn\" migration collections, the 0.00-priced Easy-Buy buylist placeholders (dropped: no price). No personal data is collected; seller = the store itself.", | |
| 55 | + "enabled": true, | |
| 56 | + "schemaVersion": "1.0", | |
| 57 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 58 | + "historicalDepth": "none", | |
| 59 | + "requires": [], | |
| 60 | + "config": { | |
| 61 | + "currency": "CAD", | |
| 62 | + "market": "CA", | |
| 63 | + "seller": "401 Games", | |
| 64 | + "location": "Toronto, ON, Canada", | |
| 65 | + "collections": [ | |
| 66 | + { | |
| 67 | + "handle": "all-magic-the-gathering", | |
| 68 | + "categorySlug": "magic_the_gathering", | |
| 69 | + "franchise": "Magic: The Gathering" | |
| 70 | + }, | |
| 71 | + { | |
| 72 | + "handle": "all-pokemon", | |
| 73 | + "categorySlug": "pokemon", | |
| 74 | + "franchise": "Pokémon" | |
| 75 | + }, | |
| 76 | + { | |
| 77 | + "handle": "all-yugioh", | |
| 78 | + "categorySlug": "yugioh", | |
| 79 | + "franchise": "Yu-Gi-Oh!" | |
| 80 | + }, | |
| 81 | + { | |
| 82 | + "handle": "all-one-piece-card-game", | |
| 83 | + "categorySlug": "one_piece_card_game", | |
| 84 | + "franchise": "One Piece" | |
| 85 | + }, | |
| 86 | + { | |
| 87 | + "handle": "all-disney-lorcana", | |
| 88 | + "categorySlug": "disney_lorcana", | |
| 89 | + "franchise": "Disney Lorcana" | |
| 90 | + }, | |
| 91 | + { | |
| 92 | + "handle": "all-weiss-schwarz", | |
| 93 | + "categorySlug": "weiss_schwarz", | |
| 94 | + "franchise": "Weiß Schwarz" | |
| 95 | + }, | |
| 96 | + { | |
| 97 | + "handle": "all-digimon-card-game", | |
| 98 | + "categorySlug": "digimon_tcg", | |
| 99 | + "franchise": "Digimon" | |
| 100 | + }, | |
| 101 | + { | |
| 102 | + "handle": "all-dragon-ball-super", | |
| 103 | + "categorySlug": "dragon_ball_tcg", | |
| 104 | + "franchise": "Dragon Ball Super" | |
| 105 | + }, | |
| 106 | + { | |
| 107 | + "handle": "all-star-wars-unlimited", | |
| 108 | + "categorySlug": "star_wars_tcg", | |
| 109 | + "franchise": "Star Wars Unlimited" | |
| 110 | + }, | |
| 111 | + { | |
| 112 | + "handle": "all-cardfight-vanguard", | |
| 113 | + "categorySlug": "other_tcg", | |
| 114 | + "franchise": "Cardfight!! Vanguard" | |
| 115 | + }, | |
| 116 | + { | |
| 117 | + "handle": "all-riftbound-league-of-legends-tcg", | |
| 118 | + "categorySlug": "other_tcg", | |
| 119 | + "franchise": "Riftbound" | |
| 120 | + }, | |
| 121 | + { | |
| 122 | + "handle": "all-gundam-card-game", | |
| 123 | + "categorySlug": "other_tcg", | |
| 124 | + "franchise": "Gundam Card Game" | |
| 125 | + }, | |
| 126 | + { | |
| 127 | + "handle": "all-sports-cards", | |
| 128 | + "categorySlug": "sports_cards" | |
| 129 | + }, | |
| 130 | + { | |
| 131 | + "handle": "all-non-sports-trading-cards", | |
| 132 | + "categorySlug": "non_sport_cards" | |
| 133 | + }, | |
| 134 | + { | |
| 135 | + "handle": "all-board-games", | |
| 136 | + "categorySlug": "board_games" | |
| 137 | + } | |
| 138 | + ], | |
| 139 | + "rules": [ | |
| 140 | + { | |
| 141 | + "match": "^(?=[\\s\\S]*\\bhockey\\b)[\\s\\S]*\\| all-sports-cards$", | |
| 142 | + "categorySlug": "hockey_cards" | |
| 143 | + }, | |
| 144 | + { | |
| 145 | + "match": "^(?=[\\s\\S]*\\bbaseball\\b)[\\s\\S]*\\| all-sports-cards$", | |
| 146 | + "categorySlug": "baseball_cards" | |
| 147 | + }, | |
| 148 | + { | |
| 149 | + "match": "^(?=[\\s\\S]*\\bbasketball\\b)[\\s\\S]*\\| all-sports-cards$", | |
| 150 | + "categorySlug": "basketball_cards" | |
| 151 | + }, | |
| 152 | + { | |
| 153 | + "match": "^(?=[\\s\\S]*\\bfootball\\b)[\\s\\S]*\\| all-sports-cards$", | |
| 154 | + "categorySlug": "football_cards" | |
| 155 | + }, | |
| 156 | + { | |
| 157 | + "match": "^(?=[\\s\\S]*\\b(soccer|premier league|uefa|fifa)\\b)[\\s\\S]*\\| all-sports-cards$", | |
| 158 | + "categorySlug": "soccer_cards" | |
| 159 | + }, | |
| 160 | + { | |
| 161 | + "match": "^(?=[\\s\\S]*\\b(formula 1|formula one|\\bf1\\b)\\b)[\\s\\S]*\\| all-sports-cards$", | |
| 162 | + "categorySlug": "f1_cards" | |
| 163 | + } | |
| 164 | + ], | |
| 165 | + "defaultCategory": null, | |
| 166 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live", | |
| 167 | + "keepOutOfStock": true, | |
| 168 | + "fetchBarcodes": false, | |
| 169 | + "wholeShop": false, | |
| 170 | + "pageSize": 250 | |
| 171 | + } | |
| 172 | +} | |
added
connectors/api/_g1-cards-eu-jp-lib/index.ts
+261 −0
@@ -0,0 +1,261 @@ | ||
| 1 | +/** | |
| 2 | + * Helpers shared by the g1-cards-eu-jp connectors (cardmarket-priceguide, limitless-tcg, yuyu-tei, | |
| 3 | + * hareruya, magi, cardrush, cardtrader). Kept inside connectors/api (not the framework). | |
| 4 | + */ | |
| 5 | +import { normalizeCondition, parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 6 | + | |
| 7 | +export const BOT_UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)'; | |
| 8 | +export const HTML_HEADERS = { 'user-agent': BOT_UA, accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'accept-language': 'ja,en;q=0.8' }; | |
| 9 | +export const JSON_HEADERS = { 'user-agent': BOT_UA, accept: 'application/json, text/plain, */*;q=0.8' }; | |
| 10 | + | |
| 11 | +/** UTC midnight of a Date (observation dates are day-precise). */ | |
| 12 | +export function dayOf(d: Date): Date { | |
| 13 | + return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); | |
| 14 | +} | |
| 15 | + | |
| 16 | +/** "2026-09-07T02:48:04+0200" | "2026-09-07" → UTC midnight of that calendar day (source's own date), else null. */ | |
| 17 | +export function isoDay(s: string | null | undefined): Date | null { | |
| 18 | + if (!s) return null; | |
| 19 | + const m = s.match(/^(\d{4})-(\d{2})-(\d{2})/); | |
| 20 | + if (!m) return null; | |
| 21 | + const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]))); | |
| 22 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 23 | +} | |
| 24 | + | |
| 25 | +/** "118,000 円" | "¥ 187,614" | "780円" | "¥1,590" | "4200000" → integer yen; null when absent or ≤ 0. */ | |
| 26 | +export function yen(s: string | number | null | undefined): number | null { | |
| 27 | + if (s === null || s === undefined) return null; | |
| 28 | + if (typeof s === 'number') return Number.isFinite(s) && s > 0 ? Math.round(s) : null; | |
| 29 | + const m = s.replace(/[，,\s]/g, '').match(/(\d+)/); | |
| 30 | + if (!m) return null; | |
| 31 | + const n = Number.parseInt(m[1]!, 10); | |
| 32 | + return Number.isFinite(n) && n > 0 ? n : null; | |
| 33 | +} | |
| 34 | + | |
| 35 | +/** "$0.25" | "0.02€" | "1,585.73€" | "€1,179.09" → { amount, currency } (EN number format), else null. */ | |
| 36 | +export function usdEur(s: string | null | undefined): { amount: number; currency: 'USD' | 'EUR' } | null { | |
| 37 | + if (!s) return null; | |
| 38 | + const t = s.trim(); | |
| 39 | + const currency = /€|\bEUR\b/.test(t) ? 'EUR' : /\$|\bUSD\b/.test(t) ? 'USD' : null; | |
| 40 | + if (!currency) return null; | |
| 41 | + const m = t.replace(/,/g, '').match(/(\d+(?:\.\d+)?)/); | |
| 42 | + if (!m) return null; | |
| 43 | + const amount = Number.parseFloat(m[1]!); | |
| 44 | + return Number.isFinite(amount) && amount > 0 ? { amount, currency } : null; | |
| 45 | +} | |
| 46 | + | |
| 47 | +export function cleanText(s: string | null | undefined): string | null { | |
| 48 | + if (!s) return null; | |
| 49 | + const t = s.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"').replace(/</g, '<').replace(/>/g, '>').replace(/\s+/g, ' ').trim(); | |
| 50 | + return t || null; | |
| 51 | +} | |
| 52 | + | |
| 53 | +/** "113/076" → { number: "113", total: "076" }; "OP15-118" → { number: "OP15-118", total: null }; "-" → nulls. */ | |
| 54 | +export function splitCardNumber(s: string | null | undefined): { number: string | null; total: string | null } { | |
| 55 | + if (!s) return { number: null, total: null }; | |
| 56 | + const t = s.trim(); | |
| 57 | + if (!t || t === '-' || t === '—') return { number: null, total: null }; | |
| 58 | + const m = t.match(/^([A-Za-z0-9]+)\s*\/\s*([A-Za-z0-9-]+)$/); | |
| 59 | + if (m) return { number: m[1]!, total: m[2]! }; | |
| 60 | + return { number: t, total: null }; | |
| 61 | +} | |
| 62 | + | |
| 63 | +/** Grade parser tolerant of Japanese shop notation: "PSA10鑑定済", "【PSA10】", "BGS9.5" → grader/grade. */ | |
| 64 | +export function jpGrade(title: string): { grader: string | null; grade: string | null; qualifier: string | null } { | |
| 65 | + const spaced = title.replace(/(PSA|BGS|CGC|SGC|ACE|TAG)(\d)/gi, '$1 $2').replace(/鑑定済/g, ' graded '); | |
| 66 | + const g = parseGradeFromTitle(spaced); | |
| 67 | + return g; | |
| 68 | +} | |
| 69 | + | |
| 70 | +/** Japanese shop condition labels → taxonomy condition slug; null when the scale is shop-specific (状態A/B…) — never guessed. */ | |
| 71 | +export function jpCondition(raw: string | null | undefined): string | null { | |
| 72 | + if (!raw) return null; | |
| 73 | + const t = raw.trim(); | |
| 74 | + const direct = normalizeCondition('trading_cards', t); | |
| 75 | + if (direct) return direct; | |
| 76 | + if (/^(NM|near mint)$/i.test(t)) return 'near_mint'; | |
| 77 | + if (/^EX[+-]?$/i.test(t)) return 'excellent'; | |
| 78 | + if (/^(HP|heavily played)$/i.test(t)) return 'good'; | |
| 79 | + if (/^(DMG|damaged)$/i.test(t)) return 'poor'; | |
| 80 | + return null; | |
| 81 | +} | |
| 82 | + | |
| 83 | +export interface JpCardTitle { | |
| 84 | + /** card name with all tags stripped (EN half preferred for bilingual "JP/EN" names) */ | |
| 85 | + name: string; | |
| 86 | + /** contents of the leading 〔…〕/[…] tags: condition or grading ("状態A-", "PSA10鑑定済", "PLD") */ | |
| 87 | + conditionRaw: string | null; | |
| 88 | + /** 【…】 tag: rarity (Pokémon/One Piece shops) or set code (MTG shops) — caller decides via `bracket` */ | |
| 89 | + rarity: string | null; | |
| 90 | + setCode: string | null; | |
| 91 | + /** {…} tag: "013/068" → number 013, total 068; "OP01-001"; "-" → null */ | |
| 92 | + number: string | null; | |
| 93 | + total: string | null; | |
| 94 | + /** 《日本語》/《英語》 → "Japanese"/"English"; null when absent */ | |
| 95 | + language: string | null; | |
| 96 | + /** parenthesised hints: "(RR仕様)", "(パラレル)", "(MA仕様/英語版)" */ | |
| 97 | + notes: string[]; | |
| 98 | + /** trailing "N枚" (count of cards in the listing) */ | |
| 99 | + quantity: number | null; | |
| 100 | + sealed: boolean; | |
| 101 | + grader: string | null; | |
| 102 | + grade: string | null; | |
| 103 | + qualifier: string | null; | |
| 104 | +} | |
| 105 | + | |
| 106 | +const LANG_MAP: Record<string, string> = { 日本語: 'Japanese', 英語: 'English', 韓国語: 'Korean', 中国語: 'Chinese', 簡体字: 'Chinese', 繁体字: 'Chinese', ドイツ語: 'German', フランス語: 'French', イタリア語: 'Italian', スペイン語: 'Spanish', ポルトガル語: 'Portuguese', ロシア語: 'Russian' }; | |
| 107 | + | |
| 108 | +/** | |
| 109 | + * Parse the title conventions shared by Japanese single-card shops (Cardrush, Magi sellers, Yuyu-tei): | |
| 110 | + * "〔PSA10鑑定済〕ミロカロスδ-デルタ種【★】{013/068}" | |
| 111 | + * "〔状態A-〕ミュウツーV(RR仕様)【P】{273/S-P} 1枚" | |
| 112 | + * "[PLD](黒枠)ラノワールのエルフ/Llanowar Elves《日本語》【4ED】" | |
| 113 | + * "ブースターパック 受け継がれる意志【未開封BOX】{-}" | |
| 114 | + * `bracket` tells whether the 【…】 tag is a rarity (default) or a set code (MTG shops). | |
| 115 | + */ | |
| 116 | +export function parseJpCardTitle(title: string, opts: { bracket?: 'rarity' | 'set' } = {}): JpCardTitle { | |
| 117 | + let t = title.replace(/\s+/g, ' ').trim(); | |
| 118 | + const conds: string[] = []; | |
| 119 | + // leading condition/grading tags | |
| 120 | + for (;;) { | |
| 121 | + const m = t.match(/^(?:〔([^〕]*)〕|\[([^\]]*)\])\s*/); | |
| 122 | + if (!m) break; | |
| 123 | + conds.push((m[1] ?? m[2] ?? '').trim()); | |
| 124 | + t = t.slice(m[0].length); | |
| 125 | + } | |
| 126 | + let sealed = false; | |
| 127 | + let rarity: string | null = null; | |
| 128 | + let setCode: string | null = null; | |
| 129 | + t = t.replace(/【([^】]*)】/g, (_, inner: string) => { | |
| 130 | + const v = inner.trim(); | |
| 131 | + if (/^(PSA|BGS|CGC|SGC|ACE|TAG|ARS)\s*\d/i.test(v)) conds.push(v); // "【PSA10】" = grading tag, not a rarity | |
| 132 | + else if (/未開封|BOX|パック|カートン/i.test(v)) sealed = true; | |
| 133 | + else if (opts.bracket === 'set') setCode = setCode ?? (v || null); | |
| 134 | + else rarity = rarity ?? (v && v !== '-' ? v : null); | |
| 135 | + return ' '; | |
| 136 | + }); | |
| 137 | + const g = jpGrade(conds.join(' ')); | |
| 138 | + let number: string | null = null; | |
| 139 | + let total: string | null = null; | |
| 140 | + t = t.replace(/\{([^}]*)\}/g, (_, inner: string) => { | |
| 141 | + // "{ST15-005[OP16]}" → number ST15-005, set code OP16 (reprint origin) | |
| 142 | + const withSet = inner.match(/^(.*?)\[([A-Za-z0-9-]+)\]\s*$/); | |
| 143 | + const s = splitCardNumber(withSet ? withSet[1]! : inner); | |
| 144 | + if (withSet && opts.bracket !== 'set') setCode = setCode ?? withSet[2]!; | |
| 145 | + if (s.number && number === null) { | |
| 146 | + number = s.number; | |
| 147 | + total = s.total; | |
| 148 | + } | |
| 149 | + return ' '; | |
| 150 | + }); | |
| 151 | + let language: string | null = null; | |
| 152 | + t = t.replace(/《([^》]*)》/g, (_, inner: string) => { | |
| 153 | + language = language ?? LANG_MAP[inner.trim()] ?? inner.trim() ?? null; | |
| 154 | + return ' '; | |
| 155 | + }); | |
| 156 | + let quantity: number | null = null; | |
| 157 | + const q = t.match(/(?:^|\s)(\d{1,3})枚(?:セット|組)?\s*$/); | |
| 158 | + if (q) { | |
| 159 | + quantity = Number(q[1]); | |
| 160 | + t = t.slice(0, q.index).trim(); | |
| 161 | + } | |
| 162 | + const notes: string[] = []; | |
| 163 | + t = t.replace(/[（(]([^()（）]*)[)）]/g, (_, inner: string) => { | |
| 164 | + const v = inner.trim(); | |
| 165 | + if (v) notes.push(v); | |
| 166 | + if (/未開封/.test(v)) sealed = true; | |
| 167 | + return ' '; | |
| 168 | + }); | |
| 169 | + // bare "120/114" card number outside braces (private sellers often omit the braces) | |
| 170 | + if (number === null) { | |
| 171 | + const bare = t.match(/(?:^|\s)(\d{1,3})\s*\/\s*(\d{1,3})(?=\s|$)/); | |
| 172 | + if (bare) { | |
| 173 | + number = bare[1]!; | |
| 174 | + total = bare[2]!; | |
| 175 | + t = t.replace(bare[0], ' '); | |
| 176 | + } | |
| 177 | + } | |
| 178 | + let name = t.replace(/\s+/g, ' ').trim(); | |
| 179 | + // bilingual MTG names "ラノワールのエルフ/Llanowar Elves" → prefer the Latin half | |
| 180 | + const bi = name.match(/^(.+?)\/(.+)$/); | |
| 181 | + if (bi && /[A-Za-z]/.test(bi[2]!) && !/[A-Za-z]/.test(bi[1]!)) name = bi[2]!.trim(); | |
| 182 | + const conditionRaw = conds.filter((c) => c && !/PSA|BGS|CGC|SGC|ACE|TAG|鑑定/i.test(c)).join(' / ') || (conds.length ? conds.join(' / ') : null); | |
| 183 | + return { name: name || title.trim(), conditionRaw, rarity, setCode, number, total, language, notes, quantity, sealed, grader: g.grader, grade: g.grade, qualifier: g.qualifier }; | |
| 184 | +} | |
| 185 | + | |
| 186 | +/** | |
| 187 | + * JSON.parse for JSON-LD blocks that embed raw line breaks inside string values (magi item pages do): | |
| 188 | + * escapes control characters found inside strings, then parses. Returns null on failure. | |
| 189 | + */ | |
| 190 | +export function lenientJsonParse(text: string): unknown | null { | |
| 191 | + try { | |
| 192 | + return JSON.parse(text); | |
| 193 | + } catch { | |
| 194 | + let out = ''; | |
| 195 | + let inStr = false; | |
| 196 | + for (let i = 0; i < text.length; i++) { | |
| 197 | + const ch = text[i]!; | |
| 198 | + if (inStr) { | |
| 199 | + if (ch === '\\') { | |
| 200 | + out += ch + (text[i + 1] ?? ''); | |
| 201 | + i++; | |
| 202 | + continue; | |
| 203 | + } | |
| 204 | + if (ch === '"') inStr = false; | |
| 205 | + else if (ch === '\n') { | |
| 206 | + out += '\\n'; | |
| 207 | + continue; | |
| 208 | + } else if (ch === '\r') continue; | |
| 209 | + else if (ch === '\t') { | |
| 210 | + out += '\\t'; | |
| 211 | + continue; | |
| 212 | + } | |
| 213 | + } else if (ch === '"') inStr = true; | |
| 214 | + out += ch; | |
| 215 | + } | |
| 216 | + try { | |
| 217 | + return JSON.parse(out); | |
| 218 | + } catch { | |
| 219 | + return null; | |
| 220 | + } | |
| 221 | + } | |
| 222 | +} | |
| 223 | + | |
| 224 | +/** All JSON-LD objects of a given @type in a document (lenient about raw newlines inside strings). */ | |
| 225 | +export function jsonLdObjects(doc: string, type: string): Record<string, unknown>[] { | |
| 226 | + const out: Record<string, unknown>[] = []; | |
| 227 | + const re = /<script[^>]+type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi; | |
| 228 | + let m: RegExpExecArray | null; | |
| 229 | + while ((m = re.exec(doc))) { | |
| 230 | + const parsed = lenientJsonParse(m[1]!.trim()); | |
| 231 | + const items = Array.isArray(parsed) ? parsed : parsed && typeof parsed === 'object' && '@graph' in (parsed as object) ? ((parsed as { '@graph': unknown[] })['@graph'] ?? []) : [parsed]; | |
| 232 | + for (const it of items) { | |
| 233 | + if (!it || typeof it !== 'object') continue; | |
| 234 | + const t = (it as { '@type'?: string | string[] })['@type']; | |
| 235 | + const types = Array.isArray(t) ? t : t ? [t] : []; | |
| 236 | + if (types.includes(type)) out.push(it as Record<string, unknown>); | |
| 237 | + } | |
| 238 | + } | |
| 239 | + return out; | |
| 240 | +} | |
| 241 | + | |
| 242 | +/** Variant label from Japanese printing hints: パラレル → Parallel, ミラー → Mirror, SR仕様 → "SR". */ | |
| 243 | +export function jpVariant(notes: string[], name?: string): string | null { | |
| 244 | + const hay = [...notes, name ?? ''].join(' '); | |
| 245 | + if (/パラレル/.test(hay)) return 'Parallel'; | |
| 246 | + if (/マスターボールミラー/.test(hay)) return 'Master Ball Mirror'; | |
| 247 | + if (/モンスターボールミラー/.test(hay)) return 'Poké Ball Mirror'; | |
| 248 | + if (/ミラー/.test(hay)) return 'Mirror'; | |
| 249 | + if (/コミックパラレル|コミパラ/.test(hay)) return 'Comic Parallel'; | |
| 250 | + if (/箔押し|foil/i.test(hay)) return 'Foil'; | |
| 251 | + return null; | |
| 252 | +} | |
| 253 | + | |
| 254 | +/** Mystery packs / grab bags / lots sold by Japanese shops — never priced as cards. */ | |
| 255 | +export const JP_EXCLUDE_RE = /オリパ|福袋|まとめ売り|詰め合わせ|セット販売|スリーブ|プレイマット|デッキケース|サプライ|ストレージ|ローダー|募集用/; | |
| 256 | + | |
| 257 | +/** Bundle detection: "3枚", "4コン", "セット", "まとめ" (single card listings say "1枚"). */ | |
| 258 | +export function isJpBundle(title: string, quantity: number | null): boolean { | |
| 259 | + if (quantity !== null && quantity > 1) return true; | |
| 260 | + return /\d+コン|セット(?!ブースター)|まとめ|複数枚/.test(title) && !/スターターセット|ex?スタート|構築済み/.test(title); | |
| 261 | +} | |
added
connectors/api/_g1-cards-eu-jp-lib/smoke.ts
+61 −0
@@ -0,0 +1,61 @@ | ||
| 1 | +/** | |
| 2 | + * Live smoke for the g1-cards-eu-jp connectors: real router + crawl context, probe mode, small limits, | |
| 3 | + * seeds chosen to keep traffic (and Scrapfly credits) minimal. Postgres is not needed. | |
| 4 | + * Usage: pnpm tsx connectors/api/_g1-cards-eu-jp-lib/smoke.ts [ids…] | |
| 5 | + */ | |
| 6 | +import { createCrawlContext, createRouter, loadConnector } from '@rareindex/connectors'; | |
| 7 | +import { childLogger } from '@rareindex/shared'; | |
| 8 | + | |
| 9 | +try { | |
| 10 | + process.loadEnvFile('.env'); | |
| 11 | +} catch { | |
| 12 | + /* no .env */ | |
| 13 | +} | |
| 14 | + | |
| 15 | +const PLAN: Record<string, { limit: number; seeds?: string[] }> = { | |
| 16 | + 'cardmarket-priceguide': { limit: 5, seeds: ['18'] }, // One Piece: smallest files (~4 MB) | |
| 17 | + 'limitless-tcg': { limit: 1, seeds: ['MEG', 'M6'] }, | |
| 18 | + 'yuyu-tei': { limit: 1, seeds: ['poc'] }, | |
| 19 | + hareruya: { limit: 1 }, | |
| 20 | + magi: { limit: 1, seeds: ['31'] }, | |
| 21 | + cardrush: { limit: 1, seeds: ['www.cardrush-mtg.jp'] }, // plain-HTTP store only (no Scrapfly credits) | |
| 22 | + cardtrader: { limit: 1 }, | |
| 23 | +}; | |
| 24 | + | |
| 25 | +const ids = process.argv.slice(2).length ? process.argv.slice(2) : Object.keys(PLAN); | |
| 26 | +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY }); | |
| 27 | +for (const id of ids) { | |
| 28 | + const started = Date.now(); | |
| 29 | + const connector = await loadConnector(id); | |
| 30 | + const plan = PLAN[id] ?? { limit: 1 }; | |
| 31 | + const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: plan.limit, seeds: plan.seeds }, log: childLogger({ connector: id, level: 'warn' }) }); | |
| 32 | + let raws = 0; | |
| 33 | + let normalized = 0; | |
| 34 | + const kinds: Record<string, number> = {}; | |
| 35 | + let printed = 0; | |
| 36 | + try { | |
| 37 | + for await (const raw of connector.crawl(ctx)) { | |
| 38 | + raws++; | |
| 39 | + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); | |
| 40 | + normalized += out.length; | |
| 41 | + for (const r of out) { | |
| 42 | + kinds[r.kind] = (kinds[r.kind] ?? 0) + 1; | |
| 43 | + if (printed < 3) { | |
| 44 | + printed++; | |
| 45 | + const summary = | |
| 46 | + r.kind === 'price_observation' | |
| 47 | + ? `${r.priceKind} ${r.price} ${r.currency} @ ${r.observationDate.toISOString().slice(0, 10)}` | |
| 48 | + : r.kind === 'listing' | |
| 49 | + ? `${r.price} ${r.currency} ${r.availability} ids=${JSON.stringify(r.attributes.identifiers)}` | |
| 50 | + : r.kind === 'catalog_item' | |
| 51 | + ? `set=${r.attributes.set ?? '-'} #${r.attributes.number ?? '-'} variant=${r.attributes.variant ?? '-'} ids=${JSON.stringify(r.attributes.identifiers)}` | |
| 52 | + : ''; | |
| 53 | + console.log(` [${id}] ${r.kind}: ${'rawTitle' in r ? r.rawTitle : ''} → ${summary}`); | |
| 54 | + } | |
| 55 | + } | |
| 56 | + } | |
| 57 | + } catch (err) { | |
| 58 | + console.log(` [${id}] crawl error: ${err instanceof Error ? err.message : String(err)}`); | |
| 59 | + } | |
| 60 | + console.log(`[${id}] raw=${raws} normalized=${normalized} kinds=${JSON.stringify(kinds)} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)} ms=${Date.now() - started}`); | |
| 61 | +} | |
added
connectors/api/_g10-lib/capture.ts
+139 −0
@@ -0,0 +1,139 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture for the g10 connectors (real router + crawl context, trimmed payloads, trimmed HTML | |
| 3 | + * snapshots for parser tests). Usage: pnpm tsx connectors/api/_g10-lib/capture.ts [id…] | |
| 4 | + * Gated connectors (ebay-browse, etsy, trademe) are captured only when their env vars are present. | |
| 5 | + */ | |
| 6 | +import { readFileSync } from 'node:fs'; | |
| 7 | +import path from 'node:path'; | |
| 8 | +import { fileURLToPath } from 'node:url'; | |
| 9 | +import { createCrawlContext, createRouter, type ConnectorMeta, type RareIndexConnector, type RawRecordInput } from '@rareindex/connectors'; | |
| 10 | +import { saveFixture, type Fixture } from '@rareindex/connectors/testing'; | |
| 11 | +import { childLogger } from '@rareindex/shared'; | |
| 12 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 13 | + | |
| 14 | +const here = path.dirname(fileURLToPath(import.meta.url)); | |
| 15 | +const apiDir = path.resolve(here, '..'); | |
| 16 | +const router = createRouter({}); | |
| 17 | +const ids = process.argv.slice(2).length ? process.argv.slice(2) : ['whisky-auction', 'chairish', 'pamono', 'peyton-street-pens', '1stdibs', 'chatterley-luxuries']; | |
| 18 | + | |
| 19 | +async function load(id: string): Promise<{ connector: RareIndexConnector; meta: ConnectorMeta; mod: Record<string, unknown> }> { | |
| 20 | + const meta = localMeta(JSON.parse(readFileSync(path.join(apiDir, id, 'meta.json'), 'utf8'))); | |
| 21 | + const mod = (await import(path.join(apiDir, id, 'index.ts'))) as Record<string, unknown> & { default: (m: ConnectorMeta) => RareIndexConnector }; | |
| 22 | + return { connector: mod.default(meta), meta, mod }; | |
| 23 | +} | |
| 24 | + | |
| 25 | +function ctxFor(meta: ConnectorMeta, seeds?: string[], limit = 2) { | |
| 26 | + return createCrawlContext({ router, meta, options: { mode: 'probe', limit, ...(seeds ? { seeds } : {}) }, log: childLogger({ connector: meta.id, level: 'warn' }) }); | |
| 27 | +} | |
| 28 | + | |
| 29 | +async function firstRaw(connector: RareIndexConnector, meta: ConnectorMeta, seeds?: string[]): Promise<RawRecordInput | null> { | |
| 30 | + for await (const raw of connector.crawl(ctxFor(meta, seeds, 1))) return raw; | |
| 31 | + return null; | |
| 32 | +} | |
| 33 | + | |
| 34 | +function fixture(raw: RawRecordInput, expect: Fixture['expect'], note: string): Fixture { | |
| 35 | + return { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload: raw.payload, fetchedAt: raw.fetchedAt ?? new Date() }, expect, note }; | |
| 36 | +} | |
| 37 | + | |
| 38 | +const today = new Date().toISOString().slice(0, 10); | |
| 39 | + | |
| 40 | +for (const id of ids) { | |
| 41 | + const { connector, meta, mod } = await load(id); | |
| 42 | + console.log(`[capture] ${id}`); | |
| 43 | + try { | |
| 44 | + if (id === 'whisky-auction') { | |
| 45 | + const parseAuctionList = mod.parseAuctionList as (h: string) => Array<{ id: string }>; | |
| 46 | + const parseLotPage = mod.parseLotPage as (h: string) => { lots: unknown[]; total: number | null; auctionTitle: string | null }; | |
| 47 | + const trimLotHtml = mod.trimLotHtml as (h: string, n?: number) => string; | |
| 48 | + const pageUrl = mod.pageUrl as (a: string, s: { category: string; slug: 'whisky' | 'rum' | 'cognac' | 'wine' }, i: number, n: number) => string; | |
| 49 | + const ctx = ctxFor(meta); | |
| 50 | + const list = await ctx.fetch('https://whisky.auction/auctions', { engines: ['api'], responseType: 'text', minQuality: 0 }); | |
| 51 | + const entries = parseAuctionList(list.html!); | |
| 52 | + // keep the first 4 auction entries of the real list page as the HTML snapshot | |
| 53 | + const m = list.html!.match(/<div class="wa-grid-item">[\s\S]*?<\/div>\s*<\/div>\s*<\/div>\s*<\/div>/g) ?? []; | |
| 54 | + const listHtml = `<!doctype html><html><body><div id="pastauctions" class="wa-grid top">${m.slice(0, 4).join('\n')}</div></body></html>`; | |
| 55 | + saveFixture(id, 'auctions-list', { raw: { url: 'https://whisky.auction/auctions', externalId: 'auctions-list', kind: 'sale', engine: 'api', fetchedAt: list.fetchedAt, payload: { kind: 'lot_page', url: 'https://whisky.auction/auctions', auctionId: entries[0]!.id, auctionTitle: null, seed: { category: '', slug: 'whisky' }, pageIndex: 0, pageSize: 90, total: 0, lots: [], snapshot: listHtml } }, expect: { count: 0 }, note: `Live capture ${today}: /auctions past-auction list trimmed to 4 entries (parser test only; no lots).` }); | |
| 56 | + const seeds = meta.config.seeds as Array<{ category: string; slug: 'whisky' | 'rum' | 'cognac' | 'wine' }>; | |
| 57 | + const auctionId = '201'; | |
| 58 | + for (const [name, seed, pick] of [ | |
| 59 | + ['auction-201-whisky-p0', seeds[0]!, (lots: unknown[]) => lots.slice(0, 6)], | |
| 60 | + ['auction-201-rum-p0', seeds[1]!, (lots: unknown[]) => [...lots.slice(0, 4), ...lots.slice(-3)]], | |
| 61 | + ] as const) { | |
| 62 | + const url = pageUrl(auctionId, seed, 0, 90); | |
| 63 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 }); | |
| 64 | + const parsed = parseLotPage(res.html!); | |
| 65 | + const payload = { kind: 'lot_page', url, auctionId, auctionTitle: parsed.auctionTitle, seed, pageIndex: 0, pageSize: 90, total: parsed.total, lots: pick(parsed.lots), snapshot: trimLotHtml(res.html!, 3) }; | |
| 66 | + saveFixture(id, name, { raw: { url, externalId: `auction:${auctionId}:${seed.slug}:p0`, kind: 'sale', engine: 'api', fetchedAt: res.fetchedAt, payload }, expect: { minCount: 1, kinds: ['sale'], requiredFields: ['price', 'currency', 'saleDate', 'attributes.identifiers.whisky_auction_lot'] }, note: `Live capture ${today} of whisky.auction auction 201 (May 2026), ${seed.slug} filter, first grid page sorted by price desc; lots trimmed (${name.includes('rum') ? 'first 4 + last 3, the tail includes unsold NotMet lots' : 'first 6'}), HTML snapshot trimmed to 3 lot cards.` }); | |
| 67 | + } | |
| 68 | + continue; | |
| 69 | + } | |
| 70 | + if (id === 'chairish' || id === 'pamono' || id === 'peyton-street-pens' || id === '1stdibs') { | |
| 71 | + const seeds = meta.config.seeds as Array<{ path: string }>; | |
| 72 | + const picks: Array<[string, string, number]> = id === 'chairish' ? [['vintage-furniture-p2', '/collection/vintage-furniture', 2], ['decor-p1', '/collection/decor', 1]] : id === 'pamono' ? [['furniture-p1', '/furniture', 1], ['jewelry-watches-p1', '/jewelry-watches', 1]] : id === 'peyton-street-pens' ? [['parker-p1', '/pens-by-brand/parker/', 1], ['montblanc-p1', '/pen-makers-europe/montblanc/', 1]] : [['seating-chairs-p1', '/furniture/seating/chairs/', 1], ['wrist-watches-p1', '/jewelry/watches/wrist-watches/', 1]]; | |
| 73 | + void seeds; | |
| 74 | + const pageUrl = mod.pageUrl as (p: string, n: number) => string; | |
| 75 | + const ctx = ctxFor(meta); | |
| 76 | + for (const [name, seedPath, page] of picks) { | |
| 77 | + const url = pageUrl(seedPath, page); | |
| 78 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 90_000 }); | |
| 79 | + if (!res.success || !res.html) throw new Error(`${url}: ${res.error ?? res.httpStatus}`); | |
| 80 | + const seedCfg = (meta.config.seeds as Array<Record<string, unknown>>).find((s) => s.path === seedPath) ?? { path: seedPath }; | |
| 81 | + let payload: Record<string, unknown>; | |
| 82 | + let snapshot: string | undefined; | |
| 83 | + if (id === 'chairish') { | |
| 84 | + const items = (mod.parseBrowseHtml as (h: string) => unknown[])(res.html).slice(0, 6); | |
| 85 | + snapshot = (mod.trimBrowseHtml as (h: string, n: number) => string)(res.html, 3); | |
| 86 | + payload = { kind: 'listing_page', url, seed: seedCfg, page, items, snapshot }; | |
| 87 | + } else if (id === 'pamono') { | |
| 88 | + const cards = (mod.parseCategoryHtml as (h: string) => unknown[])(res.html).slice(0, 6); | |
| 89 | + snapshot = (mod.trimCategoryHtml as (h: string, n: number) => string)(res.html, 3); | |
| 90 | + payload = { kind: 'listing_page', url, seed: seedCfg, page, currency: (mod.parseStoreCurrency as (h: string) => string | null)(res.html), cards, snapshot }; | |
| 91 | + } else if (id === 'peyton-street-pens') { | |
| 92 | + const parsed = (mod.parseCategoryHtml as (h: string) => { cards: unknown[]; totalPages: number | null })(res.html); | |
| 93 | + snapshot = (mod.trimCategoryHtml as (h: string, n: number) => string)(res.html, 3); | |
| 94 | + payload = { kind: 'listing_page', url, seed: seedCfg, page, totalPages: parsed.totalPages, cards: parsed.cards.slice(0, 6), snapshot }; | |
| 95 | + } else { | |
| 96 | + const parsed = (mod.parseBrowseHtml as (h: string) => { items: unknown[]; totalResults: number | null; maxPages: number | null })(res.html)!; | |
| 97 | + snapshot = (mod.trimStoreHtml as (h: string, n: number) => string)(res.html, 3); | |
| 98 | + payload = { kind: 'listing_page', url, seed: seedCfg, page, totalResults: parsed.totalResults, maxPages: parsed.maxPages, items: parsed.items.slice(0, 6) }; | |
| 99 | + // the Relay snapshot is stored as a separate .html next to the fixture | |
| 100 | + const fx = fixture({ url, externalId: `${seedPath}:p${page}`, kind: 'listing', engine: 'api', payload: { ...payload, snapshot }, fetchedAt: res.fetchedAt }, { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', 'attributes.identifiers.firstdibs_item_id'] }, `Live capture ${today} of ${url}; items trimmed to 6, Relay store snapshot reduced to the records reachable from the first 3 items.`); | |
| 101 | + saveFixture(id, name, fx); | |
| 102 | + continue; | |
| 103 | + } | |
| 104 | + const req = id === 'chairish' ? 'attributes.identifiers.chairish_product_id' : id === 'pamono' ? 'attributes.identifiers.pamono_sku' : 'seller'; | |
| 105 | + saveFixture(id, name, fixture({ url, externalId: `${seedPath}:p${page}`, kind: 'listing', engine: 'api', payload, fetchedAt: res.fetchedAt }, { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', req] }, `Live capture ${today} of ${url}; items trimmed to 6, HTML snapshot trimmed to 3 cards.`)); | |
| 106 | + } | |
| 107 | + continue; | |
| 108 | + } | |
| 109 | + if (id === 'chatterley-luxuries') { | |
| 110 | + for (const [name, handle, wantSku] of [['pens-p1', 'pens', true], ['lighters-p1', 'lighters', false]] as const) { | |
| 111 | + const m2 = { ...meta, config: { ...meta.config, collections: (meta.config.collections as Array<{ handle: string }>).filter((c) => c.handle === handle) } }; | |
| 112 | + const c2 = (mod.default as (m: ConnectorMeta) => RareIndexConnector)(m2); | |
| 113 | + let chosen: RawRecordInput | null = null; | |
| 114 | + for await (const raw of c2.crawl(ctxFor(m2, undefined, 12))) { | |
| 115 | + const sku = ((raw.payload as { product: { sku?: string | null; prices?: { price?: string } } }).product.sku ?? '').trim(); | |
| 116 | + const priced = Number((raw.payload as { product: { prices?: { price?: string } } }).product.prices?.price ?? 0) > 0; | |
| 117 | + if (priced && (!wantSku || sku)) { | |
| 118 | + chosen = raw; | |
| 119 | + break; | |
| 120 | + } | |
| 121 | + } | |
| 122 | + if (!chosen) throw new Error(`no product captured for ${handle}`); | |
| 123 | + const p = chosen.payload as { product: Record<string, unknown> }; | |
| 124 | + p.product = { ...p.product, description: String(p.product.description ?? '').slice(0, 1500), images: (p.product.images as unknown[]).slice(0, 2) }; | |
| 125 | + saveFixture(id, name, fixture(chosen, { count: 1, kinds: ['listing'], requiredFields: ['price', 'currency'] }, `Live capture ${today}: first priced product of the '${handle}' category via the WooCommerce Store API (description/images trimmed).`)); | |
| 126 | + } | |
| 127 | + continue; | |
| 128 | + } | |
| 129 | + // gated official APIs: capture only when credentials exist | |
| 130 | + const raw = await firstRaw(connector, meta); | |
| 131 | + if (!raw) { | |
| 132 | + console.log(` [${id}] nothing captured (gated / no credentials)`); | |
| 133 | + continue; | |
| 134 | + } | |
| 135 | + saveFixture(id, `live-${today}`, fixture(raw, { minCount: 1 }, `Live capture ${today}.`)); | |
| 136 | + } catch (err) { | |
| 137 | + console.error(` [${id}] capture failed:`, err instanceof Error ? err.message : err); | |
| 138 | + } | |
| 139 | +} | |
added
connectors/api/_g10-lib/index.ts
+211 −0
@@ -0,0 +1,211 @@ | ||
| 1 | +/** | |
| 2 | + * Helpers shared by the g10 connectors (global marketplaces + wine/whisky/design/pens sources). | |
| 3 | + * Kept inside connectors/api (not the framework). Nothing here guesses data: every mapper returns | |
| 4 | + * null when the source gives no evidence (SPEC §192). | |
| 5 | + */ | |
| 6 | +import { watchBrand } from '../_auction-lib/categories.js'; | |
| 7 | + | |
| 8 | +export { isBundleTitle, safeYear, watchBrand, slugFromTitle, hintFromLabel } from '../_auction-lib/categories.js'; | |
| 9 | + | |
| 10 | +/** Strip HTML tags/entities into a compact single-line text (descriptions). */ | |
| 11 | +export function plainText(html: string | null | undefined, max = 2000): string | null { | |
| 12 | + if (!html) return null; | |
| 13 | + const t = html | |
| 14 | + .replace(/<br\s*\/?>/gi, ' ') | |
| 15 | + .replace(/<[^>]+>/g, ' ') | |
| 16 | + .replace(/ /g, ' ') | |
| 17 | + .replace(/&/g, '&') | |
| 18 | + .replace(/"/g, '"') | |
| 19 | + .replace(/'|'/g, "'") | |
| 20 | + .replace(/</g, '<') | |
| 21 | + .replace(/>/g, '>') | |
| 22 | + .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))) | |
| 23 | + .replace(/\s+/g, ' ') | |
| 24 | + .trim(); | |
| 25 | + return t ? t.slice(0, max) : null; | |
| 26 | +} | |
| 27 | + | |
| 28 | +/** "$1,448" | "1448" | 1448 → number (> 0) or null. */ | |
| 29 | +export function moneyNum(v: unknown): number | null { | |
| 30 | + if (v === null || v === undefined || v === '') return null; | |
| 31 | + const n = typeof v === 'number' ? v : Number.parseFloat(String(v).replace(/[^0-9.]/g, '')); | |
| 32 | + return Number.isFinite(n) && n > 0 ? n : null; | |
| 33 | +} | |
| 34 | + | |
| 35 | +/** | |
| 36 | + * Year evidence in a design/antiques title. "1930s" is a decade, not a year → year stays null and the | |
| 37 | + * decade is kept separately; "circa 1965" / "(1965)" / ", 1965" → year 1965. | |
| 38 | + */ | |
| 39 | +export function yearOrDecade(title: string): { year: number | null; decade: string | null } { | |
| 40 | + const dec = title.match(/\b(1[6-9]\d0|20[0-2]0)['’]?s\b/); | |
| 41 | + const yr = [...title.matchAll(/\b(1[6-9]\d{2}|20[0-2]\d)\b(?!['’]?s)/g)].map((m) => Number(m[1])).filter((y) => y <= new Date().getUTCFullYear()); | |
| 42 | + return { year: yr.length ? yr[0]! : null, decade: dec ? `${dec[1]}s` : null }; | |
| 43 | +} | |
| 44 | + | |
| 45 | +/** Century / period words that make a furniture or decorative lot an antique rather than "design". */ | |
| 46 | +const ANTIQUE_RE = /\b(antique|1[5-8]th[- ]century|19th[- ]century|georgian|regency|victorian|edwardian|louis\s?x(?:iv|v|vi)|empire|biedermeier|baroque|rococo|renaissance|gothic|queen anne|chippendale|hepplewhite|sheraton|federal period|napoleon iii|gustavian|directoire|william iv|jacobean|elizabethan|tudor|17[0-9]0s|18[0-9]0s|16[0-9]0s|ming dynasty|qing dynasty|kangxi|qianlong|meiji|edo period)\b/i; | |
| 47 | +const PORCELAIN_RE = /\b(porcelain|ceramic|earthenware|stoneware|faience|majolica|delft|meissen|sèvres|sevres|wedgwood|royal copenhagen|limoges|imari|satsuma|pottery|terracotta|bisque)\b/i; | |
| 48 | +const GLASS_RE = /\b(glass|murano|lalique|baccarat|daum|gallé|galle|orrefors|kosta|iittala|holmegaard|crystal|paperweight|venini|steuben|waterford)\b/i; | |
| 49 | +const SILVER_RE = /\b(sterling|silver[- ]plate|silverplate|\bsilver\b|vermeil|silver-gilt|pewter|tea (?:set|service)|flatware|salver|tankard|christofle|georg jensen|tiffany & co\.? sterling)\b/i; | |
| 50 | +const CLOCK_RE = /\b(clock|barometer|chronometer|regulator|longcase|carriage clock|mantel clock|cuckoo)\b/i; | |
| 51 | +const WATCH_RE = /\b(wristwatch|wrist watch|watch|chronograph|pocket watch)\b/i; | |
| 52 | +const JEWEL_RE = /\b(ring|necklace|bracelet|brooch|earrings?|pendant|diamond|sapphire|ruby|emerald|carat|tiara|cufflinks|bangle|choker|cameo)\b/i; | |
| 53 | +const HANDBAG_RE = /\b(handbag|birkin|kelly|tote|clutch|shoulder bag|crossbody|satchel|purse|pochette|top handle)\b/i; | |
| 54 | +const PHOTO_RE = /\b(gelatin silver|photograph|silver print|c-print|chromogenic|platinum print|daguerreotype|albumen|photo(?:graphy)? print)\b/i; | |
| 55 | +const ART_RE = /\b(oil on canvas|oil on board|oil on panel|acrylic|watercolou?r|gouache|lithograph|screenprint|serigraph|etching|engraving|woodcut|linocut|giclée|giclee|sculpture|bronze|drawing|pastel|ink on paper|mixed media|painting|print\b|edition of|signed and numbered)\b/i; | |
| 56 | +const CONTEMPORARY_RE = /\b(banksy|kaws|murakami|hirst|koons|kusama|basquiat|haring|warhol|richter|hockney|kapoor|kiefer|condo|nara|stik|invader|shepard fairey|obey)\b/i; | |
| 57 | +const RUG_RE = /\b(rug|carpet|kilim|runner|tapestry|textile|pillow|cushion|throw|blanket|quilt|wallpaper|curtain|fabric|linen)\b/i; | |
| 58 | +const LIGHTING_RE = /\b(lamp|chandelier|sconce|pendant light|light fixture|lantern|flush mount|floor lamp|table lamp|wall light)\b/i; | |
| 59 | +const PEN_RE = /\b(fountain pen|ballpoint|rollerball|mechanical pencil|pen set|pen and pencil|writing instrument|dip pen|desk set|nib\b)/i; | |
| 60 | +const LIGHTER_RE = /\b(lighter|table lighter|pocket lighter|zippo|dunhill rollagas)\b/i; | |
| 61 | + | |
| 62 | +export type DesignVertical = 'furniture' | 'lighting' | 'decor' | 'art' | 'jewelry' | 'watches' | 'fashion' | 'tableware' | 'rugs' | 'pens' | 'unknown'; | |
| 63 | + | |
| 64 | +/** | |
| 65 | + * Map a design/antiques marketplace item to a taxonomy slug from its category label + title. | |
| 66 | + * Returns null when the item is not a collectible asset class we track (rugs, pillows, wallpaper, new | |
| 67 | + * production accessories…). Furniture and lighting default to `design_furniture`; period/antique | |
| 68 | + * evidence moves them to `antiques`. | |
| 69 | + */ | |
| 70 | +export function designSlug(category: string | null | undefined, title: string, vertical: DesignVertical = 'unknown'): string | null { | |
| 71 | + const c = (category ?? '').toLowerCase(); | |
| 72 | + const t = title; | |
| 73 | + const v: DesignVertical = vertical !== 'unknown' ? vertical : /jewel/.test(c) ? 'jewelry' : /watch/.test(c) ? 'watches' : /handbag|bag|fashion|wallet|accessor/.test(c) ? 'fashion' : /\bart\b|paint|print|photograph|sculpture|drawing/.test(c) ? 'art' : /rug|textile|pillow|wallpaper|curtain|bedding|throw/.test(c) ? 'rugs' : /light|lamp|chandelier|sconce/.test(c) ? 'lighting' : /tableware|barware|serveware|dinnerware|glassware|silver|flatware|vase|ceramic|porcelain|decor|mirror|object|sculpture|clock|accent|accessor/.test(c) ? 'decor' : /furniture|seating|chair|sofa|table|desk|storage|cabinet|bed|bench|dresser|case/.test(c) ? 'furniture' : /pen|writing/.test(c) ? 'pens' : 'unknown'; | |
| 74 | + | |
| 75 | + if (v === 'rugs') return null; | |
| 76 | + if (v === 'pens') return LIGHTER_RE.test(t) ? 'lighters' : 'pens'; | |
| 77 | + if (v === 'watches' || (WATCH_RE.test(t) && !/\bwatch (?:box|stand|winder|case|holder)/i.test(t) && v !== 'furniture' && v !== 'lighting')) return watchBrand(t).slug; | |
| 78 | + if (v === 'jewelry') return /\b(loose|unmounted|gia certified|rough)\b/i.test(t) ? 'gemstones' : 'jewelry'; | |
| 79 | + if (v === 'fashion') return HANDBAG_RE.test(t) || /bag/.test(c) ? 'luxury_handbags' : 'fashion_streetwear'; | |
| 80 | + if (v === 'art') return CONTEMPORARY_RE.test(t) ? 'contemporary_art' : PHOTO_RE.test(t) ? 'photography' : 'art'; | |
| 81 | + if (v === 'decor' || v === 'tableware') { | |
| 82 | + if (CLOCK_RE.test(t)) return 'clocks'; | |
| 83 | + if (SILVER_RE.test(t) && !GLASS_RE.test(t) && !PORCELAIN_RE.test(t)) return 'silver'; | |
| 84 | + if (PORCELAIN_RE.test(t)) return 'porcelain'; | |
| 85 | + if (GLASS_RE.test(t)) return 'glass_crystal'; | |
| 86 | + if (PHOTO_RE.test(t)) return 'photography'; | |
| 87 | + if (/sculpture|statue|bust\b/.test(c) || /\b(sculpture|statue|bust)\b/i.test(t)) return CONTEMPORARY_RE.test(t) ? 'contemporary_art' : 'art'; | |
| 88 | + if (JEWEL_RE.test(t) && /jewel/.test(c)) return 'jewelry'; | |
| 89 | + if (RUG_RE.test(t)) return null; | |
| 90 | + if (LIGHTING_RE.test(t)) return ANTIQUE_RE.test(t) ? 'antiques' : 'design_furniture'; | |
| 91 | + if (/tableware|barware|serveware|dinnerware|glassware/.test(c)) return ANTIQUE_RE.test(t) ? 'antiques' : null; | |
| 92 | + return ANTIQUE_RE.test(t) ? 'antiques' : 'design_furniture'; | |
| 93 | + } | |
| 94 | + // furniture / lighting / unknown | |
| 95 | + if (CLOCK_RE.test(t) && !/\bclock (?:table|cabinet)/i.test(t)) return 'clocks'; | |
| 96 | + if (RUG_RE.test(t) && !/\b(chair|sofa|table|cabinet|bench|stool|lamp)\b/i.test(t)) return null; | |
| 97 | + if (v === 'unknown') { | |
| 98 | + if (ART_RE.test(t) && !/\b(chair|sofa|table|cabinet|lamp|desk)\b/i.test(t)) return CONTEMPORARY_RE.test(t) ? 'contemporary_art' : 'art'; | |
| 99 | + if (PORCELAIN_RE.test(t) && /\b(vase|bowl|plate|figurine|figure|jar|charger|dish|service|tureen)\b/i.test(t)) return 'porcelain'; | |
| 100 | + if (GLASS_RE.test(t) && /\b(vase|bowl|decanter|paperweight|goblet|glasses|sculpture)\b/i.test(t)) return 'glass_crystal'; | |
| 101 | + if (SILVER_RE.test(t) && /\b(tray|salver|tea|coffee|flatware|bowl|candlestick|tankard)\b/i.test(t)) return 'silver'; | |
| 102 | + if (PEN_RE.test(t)) return 'pens'; | |
| 103 | + if (LIGHTER_RE.test(t)) return 'lighters'; | |
| 104 | + } | |
| 105 | + return ANTIQUE_RE.test(t) ? 'antiques' : 'design_furniture'; | |
| 106 | +} | |
| 107 | + | |
| 108 | +/** Designer / maker from titles like "Coffee Table by Maison Jansen, 1970s" or "Sideboard from Greaves & Thomas". */ | |
| 109 | +export function makerFromTitle(title: string): string | null { | |
| 110 | + const m = title.match(/\b(?:by|from|for|attributed to|attr\.?(?: to)?)\s+([A-Z][\w&'.\- ]{1,40}?)(?:,|\s+for\s+|\s+(?:circa|ca\.|c\.)\s|\s+\d{4}|\s*\(|$)/); | |
| 111 | + if (!m) return null; | |
| 112 | + const name = m[1]!.trim().replace(/\s+/g, ' '); | |
| 113 | + if (/^(the|a|an|his|her)\b/i.test(name) || name.length < 3) return null; | |
| 114 | + return name; | |
| 115 | +} | |
| 116 | + | |
| 117 | +/** "(Near Mint, Restored)" | "(Excellent, Works Well)" → the parenthetical condition text. */ | |
| 118 | +export function conditionFromParens(title: string): string | null { | |
| 119 | + const parts = [...title.matchAll(/\(([^()]{3,60})\)/g)].map((m) => m[1]!.trim()); | |
| 120 | + const hit = parts.find((p) => /\b(mint|excellent|very good|good|fair|poor|restored|works well|new old stock|nos\b|unused|used|worn|damaged|repaired|as is)\b/i.test(p)); | |
| 121 | + return hit ?? null; | |
| 122 | +} | |
| 123 | + | |
| 124 | +const COND_MAP: Array<[RegExp, string]> = [ | |
| 125 | + [/\bnear mint\b/i, 'near_mint'], | |
| 126 | + [/\b(new old stock|nos|new in box|unused|brand new|mint)\b/i, 'mint'], | |
| 127 | + [/\bexcellent\b/i, 'excellent'], | |
| 128 | + [/\bvery good\b/i, 'very_good'], | |
| 129 | + [/\bgood\b/i, 'good'], | |
| 130 | + [/\bfair\b/i, 'fair'], | |
| 131 | + [/\b(poor|damaged|for parts|as is)\b/i, 'poor'], | |
| 132 | +]; | |
| 133 | +export function normalizeConditionWord(raw: string | null): string | null { | |
| 134 | + if (!raw) return null; | |
| 135 | + for (const [re, slug] of COND_MAP) if (re.test(raw)) return slug; | |
| 136 | + return null; | |
| 137 | +} | |
| 138 | + | |
| 139 | +/** .NET JSON date "/Date(1725000000000)/" (Trade Me) → Date or null. */ | |
| 140 | +export function dotNetDate(s: string | null | undefined): Date | null { | |
| 141 | + if (!s) return null; | |
| 142 | + const m = String(s).match(/\/Date\((-?\d+)(?:[+-]\d{4})?\)\//); | |
| 143 | + if (m) { | |
| 144 | + const d = new Date(Number(m[1])); | |
| 145 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 146 | + } | |
| 147 | + const d = new Date(s); | |
| 148 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 149 | +} | |
| 150 | + | |
| 151 | +/** Epoch seconds (Etsy) → Date or null. */ | |
| 152 | +export function epochSeconds(n: number | null | undefined): Date | null { | |
| 153 | + if (!n || !Number.isFinite(n)) return null; | |
| 154 | + const d = new Date(n * 1000); | |
| 155 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 156 | +} | |
| 157 | + | |
| 158 | +/** ISO-8601 string → Date or null (never throws). */ | |
| 159 | +export function isoDate(s: string | null | undefined): Date | null { | |
| 160 | + if (!s) return null; | |
| 161 | + const d = new Date(s); | |
| 162 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 163 | +} | |
| 164 | + | |
| 165 | +/** "15 May 2026" | "01 Sep 2026" → UTC midnight. */ | |
| 166 | +export function dateDMonY(s: string | null | undefined): Date | null { | |
| 167 | + const m = s?.match(/(\d{1,2})\s+([A-Za-z]{3,9})\.?\s+(\d{4})/); | |
| 168 | + if (!m) return null; | |
| 169 | + const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; | |
| 170 | + const mo = months.indexOf(m[2]!.slice(0, 3).toLowerCase()); | |
| 171 | + if (mo < 0) return null; | |
| 172 | + return new Date(Date.UTC(Number(m[3]), mo, Number(m[1]))); | |
| 173 | +} | |
| 174 | + | |
| 175 | +/** | |
| 176 | + * Process-wide OAuth token cache for gated official APIs (eBay client-credentials, Artsy xapp…). | |
| 177 | + * Keyed by client id + scope; refreshes 60 s before expiry. Lives in the connector layer on purpose | |
| 178 | + * (no framework edits): the framework only knows that the connector `requires` the env vars. | |
| 179 | + */ | |
| 180 | +interface CachedToken { | |
| 181 | + token: string; | |
| 182 | + expiresAt: number; | |
| 183 | +} | |
| 184 | +const tokenCache = new Map<string, CachedToken>(); | |
| 185 | + | |
| 186 | +export async function cachedToken(key: string, fetcher: () => Promise<{ token: string; expiresInSeconds: number }>): Promise<string> { | |
| 187 | + const hit = tokenCache.get(key); | |
| 188 | + if (hit && hit.expiresAt > Date.now()) return hit.token; | |
| 189 | + const fresh = await fetcher(); | |
| 190 | + tokenCache.set(key, { token: fresh.token, expiresAt: Date.now() + Math.max(30, fresh.expiresInSeconds - 60) * 1000 }); | |
| 191 | + return fresh.token; | |
| 192 | +} | |
| 193 | + | |
| 194 | +/** Test hook. */ | |
| 195 | +export function clearTokenCache(): void { | |
| 196 | + tokenCache.clear(); | |
| 197 | +} | |
| 198 | + | |
| 199 | +/** Cursor helper shared by the seed × page crawlers: resume at (seedIndex, page) and rotate seeds between runs. */ | |
| 200 | +export interface SeedPageCursor { | |
| 201 | + seedIndex?: number; | |
| 202 | + page?: number; | |
| 203 | + done?: boolean; | |
| 204 | + at?: string; | |
| 205 | +} | |
| 206 | +export function readSeedCursor(cursor: Record<string, unknown> | undefined, seedCount: number): { seedIndex: number; page: number } { | |
| 207 | + const c = (cursor ?? {}) as SeedPageCursor; | |
| 208 | + const seedIndex = typeof c.seedIndex === 'number' && c.seedIndex >= 0 && c.seedIndex < seedCount && !c.done ? c.seedIndex : 0; | |
| 209 | + const page = typeof c.page === 'number' && c.page >= 1 && !c.done ? c.page : 1; | |
| 210 | + return { seedIndex, page }; | |
| 211 | +} | |
added
connectors/api/_g3-shops-na-lib/capture.ts
+144 −0
@@ -0,0 +1,144 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture for the g3-shops-na Shopify store connectors (SPEC §14: fixtures are real captures, trimmed). | |
| 3 | + * | |
| 4 | + * pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts <connector-id> [<connector-id>…] [--max-collections 8] | |
| 5 | + * | |
| 6 | + * For each connector it reads meta.json, walks the configured collections' public | |
| 7 | + * /collections/<handle>/products.json?limit=100&page=1 (honest UA, 1.5 s apart), and saves up to three | |
| 8 | + * trimmed single-product payloads into data/fixtures/<id>/: a graded item when the store lists any, a | |
| 9 | + * product with an out-of-stock variant, and an in-stock product — spread over distinct collections. | |
| 10 | + * Payload shape = ShopifyPayload ({ collection, product }); trimming keeps everything `normalize` reads. | |
| 11 | + */ | |
| 12 | +import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs'; | |
| 13 | +import path from 'node:path'; | |
| 14 | +import { fileURLToPath } from 'node:url'; | |
| 15 | +import { adapters, type ConnectorMeta } from '@rareindex/connectors'; | |
| 16 | +import { fixtureDir, saveFixture } from '@rareindex/connectors/testing'; | |
| 17 | +import { parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 18 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 19 | + | |
| 20 | +const UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)'; | |
| 21 | +const here = path.dirname(fileURLToPath(import.meta.url)); | |
| 22 | +const root = path.resolve(here, '../../..'); | |
| 23 | +const args = process.argv.slice(2); | |
| 24 | +const maxIdx = args.indexOf('--max-collections'); | |
| 25 | +const maxCollections = maxIdx >= 0 ? Number(args[maxIdx + 1]) : 8; | |
| 26 | +const ids = args.filter((a, i) => !a.startsWith('--') && (maxIdx < 0 || i !== maxIdx + 1)); | |
| 27 | +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 28 | + | |
| 29 | +interface Candidate { | |
| 30 | + handle: string; | |
| 31 | + product: adapters.ShopifyProduct; | |
| 32 | + categorySlug: string; | |
| 33 | + graded: boolean; | |
| 34 | + hasOos: boolean; | |
| 35 | + hasAvailable: boolean; | |
| 36 | +} | |
| 37 | + | |
| 38 | +function trim(p: adapters.ShopifyProduct, keepVariantIds: Set<number>): adapters.ShopifyProduct { | |
| 39 | + const variants = p.variants.filter((v) => keepVariantIds.has(v.id)).slice(0, 4); | |
| 40 | + return { | |
| 41 | + id: p.id, | |
| 42 | + title: p.title, | |
| 43 | + handle: p.handle, | |
| 44 | + body_html: p.body_html ? p.body_html.slice(0, 400) : p.body_html, | |
| 45 | + published_at: p.published_at, | |
| 46 | + updated_at: p.updated_at, | |
| 47 | + vendor: p.vendor, | |
| 48 | + product_type: p.product_type, | |
| 49 | + tags: Array.isArray(p.tags) ? p.tags.slice(0, 12) : p.tags, | |
| 50 | + variants: variants.map((v) => ({ id: v.id, title: v.title, sku: v.sku, barcode: v.barcode, price: v.price, compare_at_price: v.compare_at_price, available: v.available, featured_image: v.featured_image ? { src: v.featured_image.src } : v.featured_image })), | |
| 51 | + images: p.images.slice(0, 2).map((i) => ({ src: i.src })), | |
| 52 | + }; | |
| 53 | +} | |
| 54 | + | |
| 55 | +async function captureStore(id: string): Promise<void> { | |
| 56 | + const dir = path.join(root, 'connectors/api', id); | |
| 57 | + const meta: ConnectorMeta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 58 | + const cfg = adapters.ShopifyConfigSchema.parse(meta.config); | |
| 59 | + const site = meta.sourceUrl.replace(/\/+$/, ''); | |
| 60 | + const candidates: Candidate[] = []; | |
| 61 | + const handles = cfg.collections.map((c) => c.handle).slice(0, maxCollections); | |
| 62 | + for (const handle of handles) { | |
| 63 | + const url = `${site}/collections/${handle}/products.json?limit=100&page=1`; | |
| 64 | + await sleep(1500); | |
| 65 | + const res = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' } }); | |
| 66 | + if (!res.ok) { | |
| 67 | + console.warn(` [${id}] ${handle}: HTTP ${res.status}`); | |
| 68 | + continue; | |
| 69 | + } | |
| 70 | + const json = (await res.json()) as { products?: unknown[] }; | |
| 71 | + const products = (json.products ?? []).map((x) => adapters.ShopifyProductSchema.safeParse(x)).filter((r) => r.success).map((r) => r.data); | |
| 72 | + let mapped = 0; | |
| 73 | + for (const product of products) { | |
| 74 | + const sp = adapters.shopifyToStorefrontProduct(site, { collection: handle, product }); | |
| 75 | + const m = adapters.mapProduct(cfg, sp); | |
| 76 | + if (!m) continue; | |
| 77 | + const priced = product.variants.filter((v) => Number(v.price) > 0); | |
| 78 | + if (!priced.length) continue; | |
| 79 | + mapped++; | |
| 80 | + const g = parseGradeFromTitle(product.title); | |
| 81 | + // "graded" = a real third-party grader with a grade (or an ICCS/PMG/PCGS/NGC coin/note grade the parser does not read yet); | |
| 82 | + // parseGradeFromTitle also answers grader "raw" for colourways like "Raw Indigo" — not a grade. | |
| 83 | + const graded = Boolean(g.grader && g.grader !== 'raw' && (g.grade !== null || /\b(iccs|pmg|pcgs|ngc|anacs)\b/i.test(product.title))); | |
| 84 | + candidates.push({ handle, product, categorySlug: m.categorySlug, graded, hasOos: priced.some((v) => v.available === false), hasAvailable: priced.some((v) => v.available === true) }); | |
| 85 | + } | |
| 86 | + console.log(` [${id}] ${handle}: ${products.length} products, ${mapped} mapped`); | |
| 87 | + } | |
| 88 | + if (!candidates.length) { | |
| 89 | + console.error(` [${id}] no candidates — nothing saved`); | |
| 90 | + return; | |
| 91 | + } | |
| 92 | + const picks: Array<{ kind: string; c: Candidate; why: string }> = []; | |
| 93 | + const used = new Set<string>(); | |
| 94 | + const fresh = (x: Candidate) => !picks.some((p) => p.c.product.id === x.product.id); | |
| 95 | + const take = (kind: string, pred: (c: Candidate) => boolean, why: string) => { | |
| 96 | + const c = candidates.find((x) => pred(x) && fresh(x) && !used.has(x.handle)) ?? candidates.find((x) => pred(x) && fresh(x)); | |
| 97 | + if (!c) return; | |
| 98 | + picks.push({ kind, c, why }); | |
| 99 | + used.add(c.handle); | |
| 100 | + }; | |
| 101 | + take('graded', (c) => c.graded, 'title carries a third-party grade (grader parsed by parseGradeFromTitle)'); | |
| 102 | + take('out-of-stock', (c) => c.hasOos && c.hasAvailable, 'mixed availability: at least one variant sold out → ended listing'); | |
| 103 | + take('out-of-stock', (c) => c.hasOos && !picks.some((p) => p.kind === 'out-of-stock'), 'sold-out product → ended listing'); | |
| 104 | + take('available', (c) => c.hasAvailable, 'in-stock product with asking price'); | |
| 105 | + while (picks.length < 3) { | |
| 106 | + const next = candidates.find((c) => !used.has(c.handle) && !picks.some((p) => p.c.product.id === c.product.id)); | |
| 107 | + if (!next) break; | |
| 108 | + picks.push({ kind: 'available', c: next, why: 'additional collection coverage' }); | |
| 109 | + used.add(next.handle); | |
| 110 | + } | |
| 111 | + // fresh capture replaces the previous one | |
| 112 | + const fdir = fixtureDir(id); | |
| 113 | + if (existsSync(fdir)) for (const f of readdirSync(fdir)) if (f.endsWith('.json')) rmSync(path.join(fdir, f)); | |
| 114 | + const seenNames = new Set<string>(); | |
| 115 | + for (const { kind, c, why } of picks) { | |
| 116 | + const priced = c.product.variants.filter((v) => Number(v.price) > 0); | |
| 117 | + const keep = new Set<number>(); | |
| 118 | + const oos = priced.find((v) => v.available === false); | |
| 119 | + const avail = priced.find((v) => v.available === true); | |
| 120 | + if (avail) keep.add(avail.id); | |
| 121 | + if (oos) keep.add(oos.id); | |
| 122 | + for (const v of priced) if (keep.size < 4) keep.add(v.id); | |
| 123 | + const product = trim(c.product, keep); | |
| 124 | + let name = `${c.handle}-${kind}`.slice(0, 80); | |
| 125 | + while (seenNames.has(name)) name = `${name}-2`; | |
| 126 | + seenNames.add(name); | |
| 127 | + const now = new Date(); | |
| 128 | + saveFixture(id, name, { | |
| 129 | + raw: { url: `${site}/products/${product.handle}`, externalId: String(product.id), kind: 'listing', engine: 'api', fetchedAt: now, payload: { collection: c.handle, product } }, | |
| 130 | + expect: { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', 'sourceUrl', 'attributes.categorySlug'], first: { currency: cfg.currency, 'attributes.categorySlug': c.categorySlug } }, | |
| 131 | + note: `Live capture ${now.toISOString().slice(0, 10)} of ${site}/collections/${c.handle}/products.json?limit=100&page=1 (product ${product.id}); trimmed to what normalize() reads: body_html ≤400 chars, ≤2 images, ≤4 variants (${keep.size} kept${oos ? ', incl. one sold-out' : ''}). Chosen because: ${why}.`, | |
| 132 | + }); | |
| 133 | + console.log(` [${id}] saved ${name} (${c.categorySlug}) ← ${product.title.slice(0, 70)}`); | |
| 134 | + } | |
| 135 | +} | |
| 136 | + | |
| 137 | +for (const id of ids) { | |
| 138 | + console.log(`== ${id}`); | |
| 139 | + try { | |
| 140 | + await captureStore(id); | |
| 141 | + } catch (err) { | |
| 142 | + console.error(` [${id}] failed: ${(err as Error).message}`); | |
| 143 | + } | |
| 144 | +} | |
added
connectors/api/_g3-shops-na-lib/probe.ts
+51 −0
@@ -0,0 +1,51 @@ | ||
| 1 | +/** | |
| 2 | + * Live smoke for the g3-shops-na Shopify store connectors — real router + crawl context, probe mode, | |
| 3 | + * no database needed (pattern of connectors/api/_lib/smoke.ts). | |
| 4 | + * | |
| 5 | + * pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts <connector-id> [<connector-id>…] [--limit 3] | |
| 6 | + */ | |
| 7 | +import { readFileSync } from 'node:fs'; | |
| 8 | +import path from 'node:path'; | |
| 9 | +import { fileURLToPath } from 'node:url'; | |
| 10 | +import { createCrawlContext, createRouter, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors'; | |
| 11 | +import { childLogger } from '@rareindex/shared'; | |
| 12 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 13 | + | |
| 14 | +const here = path.dirname(fileURLToPath(import.meta.url)); | |
| 15 | +const root = path.resolve(here, '../../..'); | |
| 16 | +const args = process.argv.slice(2); | |
| 17 | +const limIdx = args.indexOf('--limit'); | |
| 18 | +const limit = limIdx >= 0 ? Number(args[limIdx + 1]) : 3; | |
| 19 | +const ids = args.filter((a, i) => !a.startsWith('--') && (limIdx < 0 || i !== limIdx + 1)); | |
| 20 | +const router = createRouter({}); | |
| 21 | + | |
| 22 | +for (const id of ids) { | |
| 23 | + const started = Date.now(); | |
| 24 | + const dir = path.join(root, 'connectors/api', id); | |
| 25 | + const meta: ConnectorMeta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 26 | + const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector }; | |
| 27 | + const connector = mod.default(meta); | |
| 28 | + const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) }); | |
| 29 | + let raws = 0; | |
| 30 | + let normalized = 0; | |
| 31 | + const cats = new Map<string, number>(); | |
| 32 | + let printed = 0; | |
| 33 | + try { | |
| 34 | + for await (const raw of connector.crawl(ctx)) { | |
| 35 | + raws++; | |
| 36 | + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); | |
| 37 | + normalized += out.length; | |
| 38 | + for (const r of out) { | |
| 39 | + if (r.kind !== 'listing') continue; | |
| 40 | + cats.set(r.attributes.categorySlug, (cats.get(r.attributes.categorySlug) ?? 0) + 1); | |
| 41 | + if (printed < 2) { | |
| 42 | + printed++; | |
| 43 | + console.log(` [${id}] ${r.rawTitle.slice(0, 80)} → ${r.price} ${r.currency} ${r.availability} cat=${r.attributes.categorySlug} grade=${r.grade.grader ?? '-'}${r.grade.grade ?? ''} sku=${r.attributes.identifiers.sku ?? '-'}`); | |
| 44 | + } | |
| 45 | + } | |
| 46 | + } | |
| 47 | + } catch (err) { | |
| 48 | + console.error(` [${id}] crawl error: ${(err as Error).message}`); | |
| 49 | + } | |
| 50 | + console.log(`[${id}] raw=${raws} normalized=${normalized} cats=${JSON.stringify(Object.fromEntries(cats))} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)} ms=${Date.now() - started}`); | |
| 51 | +} | |
added
connectors/api/_g3-shops-na-lib/shopify-market.ts
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * ShopifyStoreConnector pinned to the shop's home market. | |
| 5 | + * | |
| 6 | + * Shopify Markets localises `products.json` prices by the requester's IP as soon as the request carries an | |
| 7 | + * Accept-Language header: a Canadian crawler reading a US shop gets CAD-converted prices while the JSON has no | |
| 8 | + * currency field — the listing would be silently mislabelled as USD (§192). The market pin (public | |
| 9 | + * `localization=<CC>` cookie) now lives in the SDK adapter itself (`config.market`, default `meta.regions[0]`); | |
| 10 | + * this subclass only enforces that a market is known for the g3 fleet. | |
| 11 | + */ | |
| 12 | +export class MarketPinnedShopifyConnector extends ShopifyStoreConnector { | |
| 13 | + constructor(meta: ConnectorMeta) { | |
| 14 | + super(meta); | |
| 15 | + if (!this.market) throw new Error(`${meta.id}: config.market (or regions[0]) must be an ISO-3166 alpha-2 country code`); | |
| 16 | + } | |
| 17 | +} | |
added
connectors/api/_g3-shops-na-lib/store-suite.ts
+209 −0
@@ -0,0 +1,209 @@ | ||
| 1 | +import { readFileSync } from 'node:fs'; | |
| 2 | +import path from 'node:path'; | |
| 3 | +import { fileURLToPath } from 'node:url'; | |
| 4 | +import { adapters, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors'; | |
| 5 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 7 | + | |
| 8 | +/** | |
| 9 | + * Shared vitest suite for the g3-shops-na Shopify storefront connectors. | |
| 10 | + * A store connector has no code of its own, so the tests exercise the metadata: every configured | |
| 11 | + * collection/rule points at an existing taxonomy slug, the collection → category mapping resolves through | |
| 12 | + * the SDK's `mapProduct`, rules/exclusions compile and behave on sample titles, and the live-captured | |
| 13 | + * fixtures normalise into listings in the store's native currency (runFixtureSuite invariants). | |
| 14 | + */ | |
| 15 | +type Vitest = { | |
| 16 | + describe: (name: string, fn: () => void) => void; | |
| 17 | + it: (name: string, fn: () => Promise<void> | void) => void; | |
| 18 | + expect: (v: unknown) => any; | |
| 19 | +}; | |
| 20 | + | |
| 21 | +export interface StoreCase { | |
| 22 | + /** product title as the store would publish it */ | |
| 23 | + title: string; | |
| 24 | + productType?: string | null; | |
| 25 | + tags?: string[]; | |
| 26 | + collection: string | null; | |
| 27 | + /** expected taxonomy slug, or null when the product must be skipped (supplies, apparel…) */ | |
| 28 | + categorySlug: string | null; | |
| 29 | + brand?: string | null; | |
| 30 | + franchise?: string | null; | |
| 31 | +} | |
| 32 | + | |
| 33 | +export interface StoreSuiteOptions { | |
| 34 | + /** product_type used for the neutral probe product (stores whose exclude regex keys on the type) */ | |
| 35 | + probeType?: string; | |
| 36 | + /** extra rule/exclusion expectations */ | |
| 37 | + cases?: StoreCase[]; | |
| 38 | +} | |
| 39 | + | |
| 40 | +const here = path.dirname(fileURLToPath(import.meta.url)); | |
| 41 | +const TAXONOMY = path.resolve(here, '../../../data/taxonomy/categories.json'); | |
| 42 | + | |
| 43 | +export function taxonomySlugs(): Set<string> { | |
| 44 | + const doc = JSON.parse(readFileSync(TAXONOMY, 'utf8')) as { nodes: Array<{ slug: string }> }; | |
| 45 | + return new Set(doc.nodes.map((n) => n.slug)); | |
| 46 | +} | |
| 47 | + | |
| 48 | +function neutralProduct(handle: string | null, title: string, productType: string | null, tags: string[] = []): adapters.StorefrontProduct { | |
| 49 | + return { | |
| 50 | + id: 'probe', | |
| 51 | + title, | |
| 52 | + url: 'https://example.com/products/probe', | |
| 53 | + description: null, | |
| 54 | + vendor: null, | |
| 55 | + productType, | |
| 56 | + tags, | |
| 57 | + collection: handle, | |
| 58 | + images: [], | |
| 59 | + publishedAt: null, | |
| 60 | + updatedAt: null, | |
| 61 | + variants: [{ id: 'v', title: null, sku: null, barcode: null, price: 10, compareAtPrice: null, available: true, quantity: null, image: null }], | |
| 62 | + }; | |
| 63 | +} | |
| 64 | + | |
| 65 | +export function describeShopifyStore(rawMeta: unknown, create: (meta: ConnectorMeta) => RareIndexConnector, v: Vitest, opts: StoreSuiteOptions = {}): void { | |
| 66 | + const meta = localMeta(rawMeta); | |
| 67 | + const connector = create(meta); | |
| 68 | + const cfg = adapters.ShopifyConfigSchema.parse(meta.config); | |
| 69 | + const site = meta.sourceUrl.replace(/\/+$/, ''); | |
| 70 | + | |
| 71 | + v.describe(meta.id, () => { | |
| 72 | + runFixtureSuite(connector, v.it, v.expect); | |
| 73 | + | |
| 74 | + v.it('declares a coherent Shopify store connector (metadata, refresh class, currency)', async () => { | |
| 75 | + v.expect(meta.acquisitionMethod).toMatch(/shopify/i); | |
| 76 | + v.expect(meta.enginePriority).toEqual(['api']); | |
| 77 | + v.expect(meta.supportsListings).toBe(true); | |
| 78 | + v.expect(meta.supportsSold).toBe(false); | |
| 79 | + v.expect(meta.currency).toEqual([cfg.currency]); | |
| 80 | + v.expect([720, 1440]).toContain(meta.refreshFrequencyMinutes); | |
| 81 | + v.expect(meta.accessNotes).toMatch(/robots/i); | |
| 82 | + v.expect(meta.accessNotes).toMatch(/products\.json/); | |
| 83 | + v.expect(cfg.collections.length).toBeGreaterThan(0); | |
| 84 | + v.expect(cfg.seller).toBeTruthy(); | |
| 85 | + // market pinning (Shopify Markets geo-pricing guard): the shop's home market must match its region/currency | |
| 86 | + const market = (connector as { market?: string }).market; | |
| 87 | + v.expect(market).toBe(meta.regions[0]); | |
| 88 | + v.expect(meta.config.market).toBe(market); | |
| 89 | + v.expect({ CA: 'CAD', US: 'USD' }[market as 'CA' | 'US']).toBe(cfg.currency); | |
| 90 | + v.expect(meta.accessNotes).toMatch(/localization=/); | |
| 91 | + }); | |
| 92 | + | |
| 93 | + v.it('pins the shop market on every request (localization cookie + matching Accept-Language)', async () => { | |
| 94 | + const seen: Array<Record<string, string> | undefined> = []; | |
| 95 | + const ctrl = new AbortController(); // one request is enough: abort so the crawl does not throttle through every collection | |
| 96 | + const fake = { | |
| 97 | + meta, | |
| 98 | + options: { mode: 'probe', limit: 1 }, | |
| 99 | + signal: ctrl.signal, | |
| 100 | + engineStats: {}, | |
| 101 | + anomalies: [] as string[], | |
| 102 | + log: { info() {}, warn() {}, error() {}, debug() {} }, | |
| 103 | + anomaly() {}, | |
| 104 | + async setCursor() {}, | |
| 105 | + async progress() {}, | |
| 106 | + async fetch(_url: string, opts?: { headers?: Record<string, string> }) { | |
| 107 | + seen.push(opts?.headers); | |
| 108 | + ctrl.abort(); | |
| 109 | + return { success: false, engine: 'api', url: _url, finalUrl: _url, httpStatus: 503, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error: 'test', costCredits: 0, durationMs: 0, fetchedAt: new Date() }; | |
| 110 | + }, | |
| 111 | + } as unknown as Parameters<typeof connector.crawl>[0]; | |
| 112 | + for await (const _ of connector.crawl(fake)) void _; | |
| 113 | + v.expect(seen.length).toBeGreaterThan(0); | |
| 114 | + for (const h of seen) { | |
| 115 | + v.expect(h?.cookie).toBe(`localization=${meta.regions[0]}`); | |
| 116 | + v.expect(h?.['accept-language']).toMatch(new RegExp(`^en-${meta.regions[0]}`)); | |
| 117 | + } | |
| 118 | + }); | |
| 119 | + | |
| 120 | + v.it('maps every configured collection and rule onto an existing taxonomy slug', async () => { | |
| 121 | + const slugs = taxonomySlugs(); | |
| 122 | + const handles = new Set<string>(); | |
| 123 | + for (const c of cfg.collections) { | |
| 124 | + v.expect(handles.has(c.handle)).toBe(false); | |
| 125 | + handles.add(c.handle); | |
| 126 | + if (c.categorySlug) v.expect(slugs.has(c.categorySlug)).toBe(true); | |
| 127 | + } | |
| 128 | + for (const r of cfg.rules) { | |
| 129 | + v.expect(slugs.has(r.categorySlug)).toBe(true); | |
| 130 | + v.expect(() => new RegExp(r.match, 'i')).not.toThrow(); | |
| 131 | + } | |
| 132 | + if (cfg.defaultCategory) v.expect(slugs.has(cfg.defaultCategory)).toBe(true); | |
| 133 | + v.expect(() => new RegExp(cfg.exclude, 'i')).not.toThrow(); | |
| 134 | + if (cfg.titlePattern) v.expect(() => new RegExp(cfg.titlePattern!, 'i')).not.toThrow(); | |
| 135 | + // meta.categories must list what the mapping can emit | |
| 136 | + const emitted = new Set([...cfg.collections.map((c) => c.categorySlug).filter(Boolean), ...cfg.rules.map((r) => r.categorySlug)]); | |
| 137 | + for (const s of emitted) v.expect(meta.categories).toContain(s); | |
| 138 | + }); | |
| 139 | + | |
| 140 | + v.it('resolves collection → category through the SDK mapProduct', async () => { | |
| 141 | + for (const c of cfg.collections) { | |
| 142 | + if (!c.categorySlug) continue; | |
| 143 | + const m = adapters.mapProduct(cfg, neutralProduct(c.handle, 'Probe item', opts.probeType ?? null)); | |
| 144 | + v.expect(m).not.toBeNull(); | |
| 145 | + v.expect(m!.categorySlug).toBe(c.categorySlug); | |
| 146 | + if (c.brand) v.expect(m!.brand).toBe(c.brand); | |
| 147 | + if (c.franchise) v.expect(m!.franchise).toBe(c.franchise); | |
| 148 | + } | |
| 149 | + }); | |
| 150 | + | |
| 151 | + v.it('applies title/type rules and exclusions on sample products', async () => { | |
| 152 | + for (const cs of opts.cases ?? []) { | |
| 153 | + const m = adapters.mapProduct(cfg, neutralProduct(cs.collection, cs.title, cs.productType ?? opts.probeType ?? null, cs.tags ?? [])); | |
| 154 | + if (cs.categorySlug === null) { | |
| 155 | + v.expect(m).toBeNull(); | |
| 156 | + } else { | |
| 157 | + v.expect(m).not.toBeNull(); | |
| 158 | + v.expect(m!.categorySlug).toBe(cs.categorySlug); | |
| 159 | + if (cs.brand !== undefined) v.expect(m!.brand).toBe(cs.brand); | |
| 160 | + if (cs.franchise !== undefined) v.expect(m!.franchise).toBe(cs.franchise); | |
| 161 | + } | |
| 162 | + } | |
| 163 | + }); | |
| 164 | + | |
| 165 | + v.it('normalises fixtures into listings in the store currency with seller, source URL and variant SKUs', async () => { | |
| 166 | + const names = listFixtures(meta.id); | |
| 167 | + v.expect(names.length).toBeGreaterThanOrEqual(2); | |
| 168 | + const collections = new Set<string>(); | |
| 169 | + for (const name of names) { | |
| 170 | + const fx = loadFixture(meta.id, name); | |
| 171 | + const payload = fx.raw.payload as { collection: string | null; product: { variants: Array<{ sku?: string | null }> } }; | |
| 172 | + if (payload.collection) collections.add(payload.collection); | |
| 173 | + const out = await connector.normalize(fx.raw); | |
| 174 | + v.expect(out.length).toBeGreaterThan(0); | |
| 175 | + for (const r of out) { | |
| 176 | + v.expect(r.kind).toBe('listing'); | |
| 177 | + if (r.kind !== 'listing') continue; | |
| 178 | + v.expect(r.currency).toBe(cfg.currency); | |
| 179 | + v.expect(r.seller).toBe(cfg.seller); | |
| 180 | + v.expect(r.listingType).toBe('fixed_price'); | |
| 181 | + v.expect(r.sourceUrl.startsWith(`${site}/products/`)).toBe(true); | |
| 182 | + v.expect(['available', 'ended', 'unknown']).toContain(r.availability); | |
| 183 | + v.expect(r.price).toBeGreaterThan(0); | |
| 184 | + v.expect(r.confidence).toBeLessThan(1); | |
| 185 | + v.expect(r.attributes.metadata?.collection).toBe(payload.collection); | |
| 186 | + } | |
| 187 | + const skus = payload.product.variants.filter((x) => x.sku).length; | |
| 188 | + if (skus) v.expect(out.some((r) => r.kind === 'listing' && r.attributes.identifiers.sku)).toBe(true); | |
| 189 | + } | |
| 190 | + // fixtures should span more than one collection when the store has several | |
| 191 | + if (cfg.collections.length > 1) v.expect(collections.size).toBeGreaterThanOrEqual(Math.min(2, names.length)); | |
| 192 | + }); | |
| 193 | + | |
| 194 | + v.it('keeps out-of-stock variants as ended listings (or drops them when configured)', async () => { | |
| 195 | + let sawEnded = false; | |
| 196 | + for (const name of listFixtures(meta.id)) { | |
| 197 | + const fx = loadFixture(meta.id, name); | |
| 198 | + const payload = fx.raw.payload as { product: { variants: Array<{ available?: boolean | null; price?: string | number | null }> } }; | |
| 199 | + const oos = payload.product.variants.filter((x) => x.available === false && Number(x.price) > 0).length; | |
| 200 | + const out = await connector.normalize(fx.raw); | |
| 201 | + const ended = out.filter((r) => r.kind === 'listing' && r.availability === 'ended').length; | |
| 202 | + if (cfg.keepOutOfStock) v.expect(ended).toBe(oos); | |
| 203 | + else v.expect(ended).toBe(0); | |
| 204 | + if (ended) sawEnded = true; | |
| 205 | + } | |
| 206 | + if (cfg.keepOutOfStock) v.expect(typeof sawEnded).toBe('boolean'); | |
| 207 | + }); | |
| 208 | + }); | |
| 209 | +} | |
added
connectors/api/_g4-shops-intl-lib/capture.ts
+108 −0
@@ -0,0 +1,108 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture for the g4-shops-intl storefront connectors. | |
| 3 | + * | |
| 4 | + * pnpm tsx connectors/api/_g4-shops-intl-lib/capture.ts <connectorId> <collectionHandle> <fixtureName> [--pick <regex>] [--index <n>] [--note "..."] | |
| 5 | + * | |
| 6 | + * Fetches one public storefront page (Shopify `/collections/<handle>/products.json?limit=50` or the | |
| 7 | + * WooCommerce Store API `/wp-json/wc/store/v1/products?category=<id>`), picks one product (first, by | |
| 8 | + * index, or first whose title matches --pick), trims it to what `normalize` needs (short body, ≤ 3 | |
| 9 | + * images, ≤ 6 variants, ≤ 12 tags) and saves data/fixtures/<id>/<name>.json with an `expect` block | |
| 10 | + * derived from the connector's own normalisation of that live record. Never hand-written (SPEC §14). | |
| 11 | + */ | |
| 12 | +import { readFileSync } from 'node:fs'; | |
| 13 | +import path from 'node:path'; | |
| 14 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 15 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 16 | + | |
| 17 | +const UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)'; | |
| 18 | +const [id, handle, name, ...rest] = process.argv.slice(2); | |
| 19 | +if (!id || !handle || !name) { | |
| 20 | + console.error('usage: capture.ts <connectorId> <collectionHandle> <fixtureName> [--pick <regex>] [--index <n>] [--note "..."]'); | |
| 21 | + process.exit(1); | |
| 22 | +} | |
| 23 | +const flags: Record<string, string> = {}; | |
| 24 | +for (let i = 0; i < rest.length; i++) if (rest[i]!.startsWith('--')) flags[rest[i]!.slice(2)] = rest[i + 1] ?? 'true', i++; | |
| 25 | + | |
| 26 | +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..'); | |
| 27 | +const metaPath = path.join(root, 'connectors/api', id, 'meta.json'); | |
| 28 | +const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(metaPath, 'utf8'))); | |
| 29 | +const mod = (await import(path.join(root, 'connectors/api', id, 'index.ts'))) as { default: (m: typeof meta) => { normalize(raw: any): Promise<unknown[]> } }; | |
| 30 | +const connector = mod.default(meta); | |
| 31 | +const site = meta.sourceUrl.replace(/\/+$/, ''); | |
| 32 | +const isWoo = /WooCommerce/i.test(meta.acquisitionMethod ?? ''); | |
| 33 | + | |
| 34 | +/** Shopify occasionally emits raw control characters inside description strings; blank them before parsing. */ | |
| 35 | +function tolerantJson(s: string): unknown { | |
| 36 | + return JSON.parse(s.replace(/[\x00-\x1f]/g, " ")); | |
| 37 | +} | |
| 38 | + | |
| 39 | +async function get(url: string): Promise<{ status: number; body: unknown }> { | |
| 40 | + const res = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' } }); | |
| 41 | + const text = await res.text(); | |
| 42 | + return { status: res.status, body: tolerantJson(text) }; | |
| 43 | +} | |
| 44 | + | |
| 45 | +const fetchedAt = new Date(); | |
| 46 | +let url: string; | |
| 47 | +let payload: unknown; | |
| 48 | +let recordUrl: string; | |
| 49 | +let externalId: string; | |
| 50 | +if (isWoo) { | |
| 51 | + const cats = (await get(`${site}/wp-json/wc/store/v1/products/categories?per_page=100`)).body as Array<{ id: number; slug: string }>; | |
| 52 | + const cat = cats.find((c) => c.slug === handle); | |
| 53 | + if (!cat) throw new Error(`category ${handle} not found`); | |
| 54 | + url = `${site}/wp-json/wc/store/v1/products?per_page=50&category=${cat.id}`; | |
| 55 | + const list = (await get(url)).body as Array<Record<string, any>>; | |
| 56 | + const pick = choose(list, (p) => String(p.name)); | |
| 57 | + const p = pick as Record<string, any>; | |
| 58 | + const trimmed = { | |
| 59 | + id: p.id, name: p.name, slug: p.slug, permalink: p.permalink, sku: p.sku ?? null, | |
| 60 | + short_description: String(p.short_description ?? '').slice(0, 400), prices: p.prices, | |
| 61 | + images: (p.images ?? []).slice(0, 3).map((i: { src: string }) => ({ src: i.src })), | |
| 62 | + categories: (p.categories ?? []).map((c: { id: number; name: string; slug: string }) => ({ id: c.id, name: c.name, slug: c.slug })), | |
| 63 | + tags: (p.tags ?? []).slice(0, 12), is_in_stock: p.is_in_stock ?? null, stock_availability: p.stock_availability, type: p.type, brands: p.brands, | |
| 64 | + }; | |
| 65 | + payload = { category: handle, product: trimmed }; | |
| 66 | + recordUrl = p.permalink; | |
| 67 | + externalId = String(p.id); | |
| 68 | +} else { | |
| 69 | + url = `${site}/collections/${handle}/products.json?limit=50`; | |
| 70 | + const body = (await get(url)).body as { products: Array<Record<string, any>> }; | |
| 71 | + const p = choose(body.products, (x) => String(x.title)) as Record<string, any>; | |
| 72 | + const trimmed = { | |
| 73 | + id: p.id, title: p.title, handle: p.handle, | |
| 74 | + body_html: p.body_html ? String(p.body_html).replace(/\s+/g, ' ').slice(0, 400) : null, | |
| 75 | + published_at: p.published_at ?? null, updated_at: p.updated_at ?? null, vendor: p.vendor ?? null, product_type: p.product_type ?? null, | |
| 76 | + tags: (Array.isArray(p.tags) ? p.tags : String(p.tags ?? '').split(',').map((t: string) => t.trim()).filter(Boolean)).slice(0, 12), | |
| 77 | + variants: (p.variants ?? []).slice(0, 6).map((v: Record<string, any>) => ({ id: v.id, title: v.title ?? null, sku: v.sku ?? null, barcode: v.barcode ?? null, price: v.price ?? null, compare_at_price: v.compare_at_price ?? null, available: v.available ?? null, featured_image: v.featured_image ? { src: v.featured_image.src } : null })), | |
| 78 | + images: (p.images ?? []).slice(0, 3).map((i: { src: string }) => ({ src: i.src })), | |
| 79 | + }; | |
| 80 | + payload = { collection: handle, product: trimmed }; | |
| 81 | + recordUrl = `${site}/products/${p.handle}`; | |
| 82 | + externalId = String(p.id); | |
| 83 | +} | |
| 84 | + | |
| 85 | +function choose<T>(list: T[], title: (x: T) => string): T { | |
| 86 | + if (!list.length) throw new Error(`no products returned by ${url}`); | |
| 87 | + if (flags.pick) { | |
| 88 | + const re = new RegExp(flags.pick, 'i'); | |
| 89 | + const hit = list.find((x) => re.test(title(x))); | |
| 90 | + if (!hit) throw new Error(`no product matching /${flags.pick}/ among: ${list.slice(0, 15).map(title).join(' · ')}`); | |
| 91 | + return hit; | |
| 92 | + } | |
| 93 | + const i = Number(flags.index ?? 0); | |
| 94 | + return list[Math.min(i, list.length - 1)]!; | |
| 95 | +} | |
| 96 | + | |
| 97 | +const raw = { url: recordUrl, externalId, kind: 'listing' as const, engine: 'api' as const, fetchedAt, payload }; | |
| 98 | +const out = (await connector.normalize(raw)) as Array<Record<string, any>>; | |
| 99 | +if (!out.length) throw new Error(`normalize() produced no listing for "${(payload as any).product.title ?? (payload as any).product.name}" — mapping/exclude rejects it; pick another product`); | |
| 100 | +const first = out[0]!; | |
| 101 | +const expect = { | |
| 102 | + minCount: 1, | |
| 103 | + kinds: ['listing'], | |
| 104 | + requiredFields: ['price', 'currency', 'attributes.categorySlug', 'sourceUrl', 'externalId'], | |
| 105 | + first: { currency: first.currency, 'attributes.categorySlug': first.attributes.categorySlug, listingType: 'fixed_price', ...(first.grade?.grader ? { 'grade.grader': first.grade.grader, 'grade.grade': first.grade.grade } : {}) }, | |
| 106 | +}; | |
| 107 | +saveFixture(id, name, { raw, expect, note: `${flags.note ?? 'Live capture'} — ${url} (trimmed: body ≤ 400 chars, ≤ 3 images, ≤ 6 variants, ≤ 12 tags). Captured ${fetchedAt.toISOString().slice(0, 10)} with the RareIndexBot UA.` }); | |
| 108 | +console.log(`✔ data/fixtures/${id}/${name}.json ← "${(payload as any).product.title ?? (payload as any).product.name}" → ${out.length} listing(s): ${first.attributes.categorySlug} ${first.price} ${first.currency}${first.grade?.grader ? ` grade=${first.grade.grader} ${first.grade.grade}` : ''} avail=${first.availability}`); | |
added
connectors/api/_g4-shops-intl-lib/smoke.ts
+47 −0
@@ -0,0 +1,47 @@ | ||
| 1 | +/** | |
| 2 | + * Live smoke probe for the g4-shops-intl storefront connectors (no Postgres needed): | |
| 3 | + * real router + crawl context, probe mode, limit N (default 5), one shop after another. | |
| 4 | + * | |
| 5 | + * pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts totalcards ninoma [--limit 5] | |
| 6 | + */ | |
| 7 | +import { readFileSync } from 'node:fs'; | |
| 8 | +import path from 'node:path'; | |
| 9 | +import { ConnectorMetaSchema, createCrawlContext, createRouter, type RareIndexConnector } from '@rareindex/connectors'; | |
| 10 | +import { childLogger } from '@rareindex/shared'; | |
| 11 | + | |
| 12 | +const args = process.argv.slice(2); | |
| 13 | +const limitIdx = args.indexOf('--limit'); | |
| 14 | +const limit = limitIdx >= 0 ? Number(args[limitIdx + 1]) : 5; | |
| 15 | +const ids = args.filter((a, i) => !a.startsWith('--') && (limitIdx < 0 || i !== limitIdx + 1)); | |
| 16 | +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..'); | |
| 17 | +const router = createRouter({}); | |
| 18 | + | |
| 19 | +for (const id of ids) { | |
| 20 | + const started = Date.now(); | |
| 21 | + const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(root, 'connectors/api', id, 'meta.json'), 'utf8'))); | |
| 22 | + const mod = (await import(path.join(root, 'connectors/api', id, 'index.ts'))) as { default: (m: typeof meta) => RareIndexConnector }; | |
| 23 | + const connector = mod.default(meta); | |
| 24 | + const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) }); | |
| 25 | + let raws = 0; | |
| 26 | + let normalized = 0; | |
| 27 | + let skipped = 0; | |
| 28 | + const cats = new Map<string, number>(); | |
| 29 | + let sample = ''; | |
| 30 | + try { | |
| 31 | + for await (const raw of connector.crawl(ctx)) { | |
| 32 | + raws++; | |
| 33 | + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); | |
| 34 | + if (!out.length) skipped++; | |
| 35 | + normalized += out.length; | |
| 36 | + for (const r of out) { | |
| 37 | + if (r.kind !== 'listing') continue; | |
| 38 | + cats.set(r.attributes.categorySlug, (cats.get(r.attributes.categorySlug) ?? 0) + 1); | |
| 39 | + if (!sample) sample = `"${r.rawTitle}" → ${r.attributes.categorySlug} ${r.price} ${r.currency} ${r.availability}${r.grade.grader ? ` ${r.grade.grader} ${r.grade.grade}` : ''}`; | |
| 40 | + } | |
| 41 | + } | |
| 42 | + } catch (e) { | |
| 43 | + console.log(`[${id}] CRASH ${(e as Error).message}`); | |
| 44 | + } | |
| 45 | + console.log(`[${id}] raw=${raws} listings=${normalized} unmapped=${skipped} cats=${JSON.stringify(Object.fromEntries(cats))} anomalies=${JSON.stringify(ctx.anomalies)} engines=${JSON.stringify(ctx.engineStats)} ms=${Date.now() - started}`); | |
| 46 | + if (sample) console.log(` e.g. ${sample}`); | |
| 47 | +} | |
added
connectors/api/_g4-shops-intl-lib/suite.ts
+124 −0
@@ -0,0 +1,124 @@ | ||
| 1 | +import { adapters, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors'; | |
| 2 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 3 | +import { SUPPORTED_CURRENCIES, type NormalizedListing } from '@rareindex/shared'; | |
| 4 | + | |
| 5 | +type It = (name: string, fn: () => Promise<void> | void) => void; | |
| 6 | +// vitest's `expect` is typed loosely here so the helper does not depend on vitest at type level. | |
| 7 | +type Expect = (v: unknown) => any; | |
| 8 | + | |
| 9 | +/** | |
| 10 | + * Shared vitest suite for the g4-shops-intl storefront connectors (metadata-only Shopify/WooCommerce | |
| 11 | + * stores). Runs the framework fixture suite plus the invariants every store connector must honour: | |
| 12 | + * native currency = config.currency, every category is one the connector declares, listings only | |
| 13 | + * (asking prices are never sales, SPEC §111), seller label present, stable external ids, no zero prices. | |
| 14 | + */ | |
| 15 | +export function storeSuite(meta: ConnectorMeta, connector: RareIndexConnector, it: It, expect: Expect): void { | |
| 16 | + const cfg = adapters.StorefrontConfigSchema.parse(meta.config); | |
| 17 | + runFixtureSuite(connector, it, expect); | |
| 18 | + | |
| 19 | + it('declares a supported native currency and complete compliance metadata', () => { | |
| 20 | + expect(SUPPORTED_CURRENCIES as readonly string[]).toContain(cfg.currency); | |
| 21 | + expect(meta.currency).toContain(cfg.currency); | |
| 22 | + expect(Boolean(meta.accessNotes && meta.accessNotes.length > 120 && !/TODO/.test(meta.accessNotes))).toBe(true); | |
| 23 | + expect(meta.supportsListings).toBe(true); | |
| 24 | + expect(meta.supportsSold).toBe(false); | |
| 25 | + expect(meta.refreshFrequencyMinutes).toBeGreaterThanOrEqual(360); | |
| 26 | + expect(cfg.collections.length + (meta.config.wholeShop ? 1 : 0)).toBeGreaterThan(0); | |
| 27 | + for (const c of cfg.collections) expect(/^[a-z0-9-]+$/.test(c.handle)).toBe(true); | |
| 28 | + expect(new Set(cfg.collections.map((c) => c.handle)).size).toBe(cfg.collections.length); | |
| 29 | + }); | |
| 30 | + | |
| 31 | + it('every collection and rule maps onto a category the connector declares', () => { | |
| 32 | + const declared = new Set(meta.categories); | |
| 33 | + for (const c of cfg.collections) if (c.categorySlug) expect(declared.has(c.categorySlug)).toBe(true); | |
| 34 | + for (const r of cfg.rules) { | |
| 35 | + expect(declared.has(r.categorySlug)).toBe(true); | |
| 36 | + expect(() => new RegExp(r.match, 'i')).not.toThrow(); | |
| 37 | + } | |
| 38 | + expect(() => new RegExp(cfg.exclude, 'i')).not.toThrow(); | |
| 39 | + if (cfg.titlePattern) expect(() => new RegExp(cfg.titlePattern!, 'i')).not.toThrow(); | |
| 40 | + }); | |
| 41 | + | |
| 42 | + it('normalises every fixture into listings in the shop currency with stable ids', async () => { | |
| 43 | + for (const name of listFixtures(meta.id)) { | |
| 44 | + const fx = loadFixture(meta.id, name); | |
| 45 | + const out = await connector.normalize(fx.raw); | |
| 46 | + expect(out.length).toBeGreaterThan(0); | |
| 47 | + const ids = new Set<string>(); | |
| 48 | + for (const r of out) { | |
| 49 | + expect(r.kind).toBe('listing'); | |
| 50 | + if (r.kind !== 'listing') continue; | |
| 51 | + expect(r.currency).toBe(cfg.currency); | |
| 52 | + expect(r.price).toBeGreaterThan(0); | |
| 53 | + expect(r.listingType).toBe('fixed_price'); | |
| 54 | + expect(r.seller).toBe(cfg.seller ?? null); | |
| 55 | + expect(meta.categories).toContain(r.attributes.categorySlug); | |
| 56 | + expect(r.sourceUrl.startsWith(meta.sourceUrl)).toBe(true); | |
| 57 | + expect(Boolean(r.externalId && r.externalId.length > 0)).toBe(true); | |
| 58 | + expect(ids.has(r.externalId!)).toBe(false); | |
| 59 | + ids.add(r.externalId!); | |
| 60 | + expect(r.confidence).toBeLessThan(1); | |
| 61 | + expect(['available', 'ended', 'unknown']).toContain(r.availability); | |
| 62 | + } | |
| 63 | + } | |
| 64 | + }); | |
| 65 | +} | |
| 66 | + | |
| 67 | +export interface MappingCase { | |
| 68 | + title: string; | |
| 69 | + /** collection handle (Shopify) / category slug (WooCommerce) the product was listed under */ | |
| 70 | + collection: string | null; | |
| 71 | + type?: string | null; | |
| 72 | + tags?: string[]; | |
| 73 | + vendor?: string | null; | |
| 74 | + /** optional single variant title (e.g. "Near Mint Foil", "UK 9") */ | |
| 75 | + variant?: string | null; | |
| 76 | + /** expected taxonomy slug, or null when the product must be skipped (accessory, unmapped…) */ | |
| 77 | + expect: string | null; | |
| 78 | + set?: string | null; | |
| 79 | + number?: string | null; | |
| 80 | + name?: string; | |
| 81 | + grader?: string | null; | |
| 82 | + grade?: string | null; | |
| 83 | + brand?: string | null; | |
| 84 | + series?: string | null; | |
| 85 | + franchise?: string | null; | |
| 86 | + conditionRaw?: string | null; | |
| 87 | + year?: number | null; | |
| 88 | +} | |
| 89 | + | |
| 90 | +/** | |
| 91 | + * Parser unit test on synthetic storefront payloads: checks the collection → rule → exclude chain | |
| 92 | + * and titlePattern extraction for representative titles seen live. Complements the real fixtures. | |
| 93 | + */ | |
| 94 | +export async function expectMapping(meta: ConnectorMeta, connector: RareIndexConnector, expect: Expect, cases: MappingCase[]): Promise<void> { | |
| 95 | + const woo = /woocommerce/i.test(meta.acquisitionMethod ?? ''); | |
| 96 | + const cfg = adapters.StorefrontConfigSchema.parse(meta.config); | |
| 97 | + const site = meta.sourceUrl.replace(/\/+$/, ''); | |
| 98 | + let n = 1; | |
| 99 | + for (const c of cases) { | |
| 100 | + const id = 900000 + n++; | |
| 101 | + const payload = woo | |
| 102 | + ? { category: c.collection, product: { id, name: c.title, slug: `p-${id}`, permalink: `${site}/product/p-${id}/`, sku: null, prices: { price: '1999', regular_price: '1999', currency_code: cfg.currency, currency_minor_unit: 2 }, images: [], categories: c.type ? c.type.split(' / ').map((name) => ({ name, slug: name.toLowerCase().replace(/\s+/g, '-') })) : [], tags: (c.tags ?? []).map((name) => ({ name })), is_in_stock: true, type: 'simple', brands: c.vendor ? [{ name: c.vendor }] : undefined } } | |
| 103 | + : { collection: c.collection, product: { id, title: c.title, handle: `p-${id}`, vendor: c.vendor ?? null, product_type: c.type ?? null, tags: c.tags ?? [], variants: [{ id: id * 10, title: c.variant ?? 'Default Title', sku: null, price: '19.99', available: true }], images: [] } }; | |
| 104 | + const out = (await connector.normalize({ url: `${site}/products/p-${id}`, externalId: String(id), kind: 'listing', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload })) as NormalizedListing[]; | |
| 105 | + const label = `"${c.title}" [${c.collection ?? '-'}]`; | |
| 106 | + if (c.expect === null) { | |
| 107 | + expect({ label, count: out.length }).toEqual({ label, count: 0 }); | |
| 108 | + continue; | |
| 109 | + } | |
| 110 | + expect({ label, count: out.length }).toEqual({ label, count: 1 }); | |
| 111 | + const r = out[0]!; | |
| 112 | + expect({ label, slug: r.attributes.categorySlug }).toEqual({ label, slug: c.expect }); | |
| 113 | + if (c.set !== undefined) expect({ label, set: r.attributes.set }).toEqual({ label, set: c.set }); | |
| 114 | + if (c.number !== undefined) expect({ label, number: r.attributes.number }).toEqual({ label, number: c.number }); | |
| 115 | + if (c.name !== undefined) expect({ label, name: r.attributes.name }).toEqual({ label, name: c.name }); | |
| 116 | + if (c.grader !== undefined) expect({ label, grader: r.grade.grader }).toEqual({ label, grader: c.grader }); | |
| 117 | + if (c.grade !== undefined) expect({ label, grade: r.grade.grade }).toEqual({ label, grade: c.grade }); | |
| 118 | + if (c.brand !== undefined) expect({ label, brand: r.attributes.brand }).toEqual({ label, brand: c.brand }); | |
| 119 | + if (c.series !== undefined) expect({ label, series: r.attributes.series }).toEqual({ label, series: c.series }); | |
| 120 | + if (c.franchise !== undefined) expect({ label, franchise: r.attributes.franchise }).toEqual({ label, franchise: c.franchise }); | |
| 121 | + if (c.conditionRaw !== undefined) expect({ label, conditionRaw: r.condition.conditionRaw }).toEqual({ label, conditionRaw: c.conditionRaw }); | |
| 122 | + if (c.year !== undefined) expect({ label, year: r.attributes.year }).toEqual({ label, year: c.year }); | |
| 123 | + } | |
| 124 | +} | |
added
connectors/api/_g6-comics-toys-games-lib/_inspect.ts
+21 −0
@@ -0,0 +1,21 @@ | ||
| 1 | +import { readFileSync } from 'node:fs'; | |
| 2 | +import path from 'node:path'; | |
| 3 | +import { listFixtures, loadFixture } from '@rareindex/connectors/testing'; | |
| 4 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 5 | +const root = '/Users/simon-pierreboucher/Desktop/Projets/apps-web/rareindex/connectors'; | |
| 6 | +const MODULES: Record<string, string> = { mycomicshop: 'scrapfly/mycomicshop', 'entertainment-earth': 'scrapfly/entertainment-earth', 'miniature-market': 'api/miniature-market', videogametrader: 'api/videogametrader', 'mattel-creations': 'api/mattel-creations', comiclink: 'scrapfly/comiclink', gcd: 'api/gcd' }; | |
| 7 | +for (const id of process.argv.slice(2)) { | |
| 8 | + const meta = localMeta(JSON.parse(readFileSync(path.join(root, MODULES[id]!, 'meta.json'), 'utf8'))); | |
| 9 | + const mod = await import(path.join(root, MODULES[id]!, 'index.ts')); | |
| 10 | + const c = mod.default(meta); | |
| 11 | + for (const name of listFixtures(id)) { | |
| 12 | + const out = await c.normalize(loadFixture(id, name).raw); | |
| 13 | + console.log(`\n=== ${id}/${name}: ${out.length} records`); | |
| 14 | + for (const r of out.slice(0, Number(process.env.N ?? 3))) { | |
| 15 | + const { attributes, grade, condition, ...rest } = r as any; | |
| 16 | + const { metadata, identifiers, ...a } = attributes ?? {}; | |
| 17 | + const compact = Object.fromEntries(Object.entries({ ...rest, ...Object.fromEntries(Object.entries(a).filter(([, v]) => v !== null)), identifiers, grade, condition, metadata }).filter(([k]) => !['connectorId', 'sourceId', 'parserVersion', 'observedAt', 'imageUrls', 'description'].includes(k))); | |
| 18 | + console.log(JSON.stringify(compact)); | |
| 19 | + } | |
| 20 | + } | |
| 21 | +} | |
added
connectors/api/_g6-comics-toys-games-lib/capture-env.ts
+28 −0
@@ -0,0 +1,28 @@ | ||
| 1 | +/** | |
| 2 | + * Capture/smoke helper for the g6 connectors: load connectors/domains.json plus ONLY this group's | |
| 3 | + * domains.d fragment through `setDomains()`. Parallel contributors may leave a temporarily invalid | |
| 4 | + * fragment in domains.d; this keeps our live captures runnable without touching their files. | |
| 5 | + * (Production uses the normal loader, which validates every fragment.) | |
| 6 | + */ | |
| 7 | +import { readFileSync } from 'node:fs'; | |
| 8 | +import path from 'node:path'; | |
| 9 | +import { fileURLToPath } from 'node:url'; | |
| 10 | +import { setDomains } from '@rareindex/connectors'; | |
| 11 | + | |
| 12 | +const here = path.dirname(fileURLToPath(import.meta.url)); | |
| 13 | +const root = path.resolve(here, '../../..'); | |
| 14 | + | |
| 15 | +export function useGroupDomains(): void { | |
| 16 | + const base = JSON.parse(readFileSync(path.join(root, 'connectors/domains.json'), 'utf8')) as { version?: string; defaults?: Record<string, unknown>; domains?: Record<string, unknown> }; | |
| 17 | + const frag = JSON.parse(readFileSync(path.join(root, 'connectors/domains.d/g6-comics-toys-games.json'), 'utf8')) as { domains?: Record<string, unknown> }; | |
| 18 | + setDomains({ version: base.version ?? '1.0', defaults: (base.defaults ?? {}) as never, domains: { ...(base.domains ?? {}), ...(frag.domains ?? {}) } as never }); | |
| 19 | +} | |
| 20 | + | |
| 21 | +/** Loads repo-root .env (FIRECRAWL_API_KEY / SCRAPFLY_API_KEY) when present. */ | |
| 22 | +export function loadEnv(): void { | |
| 23 | + try { | |
| 24 | + process.loadEnvFile?.(path.join(root, '.env')); | |
| 25 | + } catch { | |
| 26 | + /* no .env — fine for api-only connectors */ | |
| 27 | + } | |
| 28 | +} | |
added
connectors/api/_g6-comics-toys-games-lib/capture.ts
+73 −0
@@ -0,0 +1,73 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture for the g6 connectors (real pages/APIs, payloads trimmed, snapshots cut down). | |
| 3 | + * Usage: pnpm tsx connectors/api/_g6-comics-toys-games-lib/capture.ts <connectorId> [limit] [seed...] | |
| 4 | + * e.g. … capture.ts videogametrader 2 | |
| 5 | + * … capture.ts mycomicshop 1 78991 | |
| 6 | + * … capture.ts entertainment-earth 1 | |
| 7 | + */ | |
| 8 | +import { readFileSync } from 'node:fs'; | |
| 9 | +import path from 'node:path'; | |
| 10 | +import { fileURLToPath } from 'node:url'; | |
| 11 | +import { createCrawlContext, createRouter, type RawRecordInput } from '@rareindex/connectors'; | |
| 12 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 13 | +import { childLogger } from '@rareindex/shared'; | |
| 14 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 15 | +import { loadEnv, useGroupDomains } from './capture-env.js'; | |
| 16 | + | |
| 17 | +loadEnv(); | |
| 18 | +useGroupDomains(); | |
| 19 | +const here = path.dirname(fileURLToPath(import.meta.url)); | |
| 20 | +const connectorsDir = path.resolve(here, '../..'); | |
| 21 | +const [id, limitArg, ...seeds] = process.argv.slice(2); | |
| 22 | +if (!id) throw new Error('usage: capture.ts <connectorId> [limit] [seed...]'); | |
| 23 | +const MODULES: Record<string, string> = { mycomicshop: 'scrapfly/mycomicshop', 'entertainment-earth': 'scrapfly/entertainment-earth', 'miniature-market': 'api/miniature-market', videogametrader: 'api/videogametrader', 'mattel-creations': 'api/mattel-creations', comiclink: 'scrapfly/comiclink', gcd: 'api/gcd', estarland: 'scrapfly/estarland' }; | |
| 24 | +const modPath = MODULES[id]; | |
| 25 | +if (!modPath) throw new Error(`unknown connector ${id}`); | |
| 26 | +const meta = localMeta(JSON.parse(readFileSync(path.join(connectorsDir, modPath, 'meta.json'), 'utf8'))); | |
| 27 | +const mod = (await import(path.join(connectorsDir, modPath, 'index.ts'))) as { default: (m: typeof meta) => { crawl: (ctx: never) => AsyncIterable<RawRecordInput>; normalize: (r: never) => Promise<unknown[]> } }; | |
| 28 | +const connector = mod.default(meta); | |
| 29 | +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY }); | |
| 30 | +const limit = Number(limitArg ?? 1) || 1; | |
| 31 | +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit, ...(seeds.length ? { seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) }); | |
| 32 | + | |
| 33 | +/** Keep only `n` items of the listy part of a payload, and cut a snapshot to the first `keep` element blocks. */ | |
| 34 | +function trim(payload: Record<string, unknown>, snapshot: string | null | undefined): Record<string, unknown> { | |
| 35 | + const p = { ...payload }; | |
| 36 | + for (const key of ['issues', 'tiles', 'boxes', 'items', 'cards']) { | |
| 37 | + if (Array.isArray(p[key])) p[key] = (p[key] as unknown[]).slice(0, id === 'mycomicshop' ? 6 : 30); | |
| 38 | + } | |
| 39 | + if (snapshot) { | |
| 40 | + const marker = id === 'mycomicshop' ? '<li class="issue">' : id === 'entertainment-earth' ? '<div class="grid-view item' : id === 'miniature-market' ? '<div class="card product-box' : id === 'estarland' ? '<a href="/product-description/' : null; | |
| 41 | + if (marker) { | |
| 42 | + const start = snapshot.indexOf(marker); | |
| 43 | + const blocks = snapshot.slice(start).split(marker).slice(1).map((b) => marker + b).filter((b) => (id === 'estarland' ? b.includes('productConditionHolder') : true)).slice(0, id === 'mycomicshop' ? 2 : 4); | |
| 44 | + const head = id === 'miniature-market' ? snapshot.slice(snapshot.indexOf('<nav aria-label="Pagination"'), snapshot.indexOf('<nav aria-label="Pagination"') + 600) : id === 'entertainment-earth' ? '<a href="?page=2">2</a><a href="?page=73">73</a>' : id === 'estarland' ? snapshot.slice(snapshot.indexOf('<div class="commingsoon_pagig">'), snapshot.indexOf('<div class="commingsoon_pagig">') + 1500) : ''; | |
| 45 | + p.snapshot = `<html><head><title>${(snapshot.match(/<title>([^<]*)/)?.[1] ?? '').trim()}</title></head><body>${head}${blocks.join('\n')}</body></html>`; | |
| 46 | + } | |
| 47 | + } | |
| 48 | + return p; | |
| 49 | +} | |
| 50 | + | |
| 51 | +let n = 0; | |
| 52 | +const urlSeeds = seeds.filter((s) => /^https?:\/\//.test(s)); | |
| 53 | +const lookup = (connector as unknown as { lookup?: (url: string, ctx: unknown) => Promise<RawRecordInput[]> }).lookup; | |
| 54 | +async function* source(): AsyncIterable<RawRecordInput> { | |
| 55 | + if (urlSeeds.length && lookup) { | |
| 56 | + for (const u of urlSeeds) yield* await lookup.call(connector, u, ctx); | |
| 57 | + return; | |
| 58 | + } | |
| 59 | + yield* connector.crawl(ctx as never); | |
| 60 | +} | |
| 61 | +for await (const raw of source()) { | |
| 62 | + const payload = trim((raw.payload ?? {}) as Record<string, unknown>, raw.snapshot); | |
| 63 | + const normalized = await connector.normalize({ url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload, fetchedAt: raw.fetchedAt ?? new Date() } as never); | |
| 64 | + const name = `${id === 'videogametrader' || id === 'mattel-creations' ? String((raw.payload as { collection?: string }).collection ?? 'shop') + '-' : ''}${String(raw.externalId ?? n).replace(/[^A-Za-z0-9]+/g, '-').replace(/^-|-$/g, '')}`; | |
| 65 | + console.log(`[${id}] ${raw.url} → externalId=${raw.externalId} normalized=${normalized.length}`); | |
| 66 | + saveFixture(id, name, { | |
| 67 | + raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload, fetchedAt: raw.fetchedAt ?? new Date() }, | |
| 68 | + expect: { minCount: normalized.length ? 1 : 0, kinds: id === 'mycomicshop' ? ['listing', 'auction_lot'] : [raw.kind] }, | |
| 69 | + note: `Live capture of ${raw.url} on ${new Date().toISOString().slice(0, 10)} (payload lists trimmed; snapshot cut to a few blocks).`, | |
| 70 | + }); | |
| 71 | + n++; | |
| 72 | +} | |
| 73 | +console.log(`[${id}] fixtures=${n} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)}`); | |
added
connectors/api/_g6-comics-toys-games-lib/comics.ts
+141 −0
@@ -0,0 +1,141 @@ | ||
| 1 | +/** | |
| 2 | + * Shared parsing helpers for the comics / toys / games connectors (gcd, comiclink, mycomicshop, | |
| 3 | + * entertainment-earth, miniature-market, videogametrader, mattel-creations). Kept inside | |
| 4 | + * connectors/ (not the framework). Nothing here guesses: unknown → null (SPEC §192). | |
| 5 | + */ | |
| 6 | +import { getGrader } from '@rareindex/taxonomy'; | |
| 7 | + | |
| 8 | +export const BOT_UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)'; | |
| 9 | + | |
| 10 | +/** Comic condition labels (Overstreet scale) as printed by dealers/auction houses. */ | |
| 11 | +export const COMIC_GRADE_LABEL = '(?:GEM\\s*MT|GEM|MT|NM/MT|NM/M|NM\\+|NM-|NM|VF/NM|VFNM|VF\\+|VF-|VF|FN/VF|FNVF|FN\\+|FN-|FN|VG/FN|VGFN|VGF|VG\\+|VG-|VG|GD/VG|GDVG|GVG|GD\\+|GD-|GD|FR/GD|FRGD|FR|PR|P|M|Mint|Near Mint|Very Fine|Fine|Very Good|Good|Fair|Poor)'; | |
| 12 | +const GRADERS = '(CGC|CBCS|PGX|EGS|CGG)'; | |
| 13 | +const GRADED_RE = new RegExp(`\\b${GRADERS}\\b[\\s:-]*(?:${COMIC_GRADE_LABEL}(?![A-Za-z]))?[\\s:-]*(\\d{1,2}(?:\\.\\d)?)`, 'i'); | |
| 14 | +const RAW_RE = new RegExp(`(?<![A-Za-z])(${COMIC_GRADE_LABEL})(?![A-Za-z])[\\s:-]*(\\d{1,2}(?:\\.\\d)?)?`, 'i'); | |
| 15 | + | |
| 16 | +export interface ComicGrade { | |
| 17 | + /** taxonomy grader slug (cgc, cbcs, pgx) or 'raw' when a dealer condition is given, null when nothing found */ | |
| 18 | + grader: string | null; | |
| 19 | + /** numeric grade as printed ("9.4"); null for descriptive-only raw conditions */ | |
| 20 | + grade: string | null; | |
| 21 | + /** condition label as printed ("NM", "VF/NM", "Fine") */ | |
| 22 | + label: string | null; | |
| 23 | + qualifier: string | null; | |
| 24 | +} | |
| 25 | + | |
| 26 | +/** "CGC 9.4 NM" · "SOLD in CBCS 9.6" · "VF- 7.5" · "Fine" → grader/grade/label. */ | |
| 27 | +export function parseComicGrade(text: string | null | undefined): ComicGrade { | |
| 28 | + if (!text) return { grader: null, grade: null, label: null, qualifier: null }; | |
| 29 | + const qualifier = /signature\s*series/i.test(text) ? 'Signature Series' : /restored|\bRESTORED\b|\(R\)/.test(text) && /\bCGC\b|\bCBCS\b/i.test(text) ? 'Restored' : /qualified/i.test(text) ? 'Qualified' : null; | |
| 30 | + const g = text.match(GRADED_RE); | |
| 31 | + if (g) { | |
| 32 | + const slug = getGrader(g[1]!)?.slug ?? g[1]!.toLowerCase(); | |
| 33 | + const label = text.match(new RegExp(`\\b${GRADERS}\\b[\\s:-]*(${COMIC_GRADE_LABEL})(?![A-Za-z])`, 'i'))?.[2] ?? text.match(new RegExp(`\\d(?:\\.\\d)?\\s+(${COMIC_GRADE_LABEL})(?![A-Za-z])`, 'i'))?.[1] ?? null; | |
| 34 | + return { grader: slug, grade: g[2] ?? null, label: label ?? null, qualifier }; | |
| 35 | + } | |
| 36 | + const r = text.match(RAW_RE); | |
| 37 | + if (r) return { grader: 'raw', grade: r[2] ?? null, label: r[1]!, qualifier }; | |
| 38 | + return { grader: null, grade: null, label: null, qualifier }; | |
| 39 | +} | |
| 40 | + | |
| 41 | +export interface ComicTitleParts { | |
| 42 | + series: string; | |
| 43 | + issue: string | null; | |
| 44 | + /** "(1963 1st Series)" / "(1963-2011)" style qualifier as printed */ | |
| 45 | + seriesYears: string | null; | |
| 46 | + /** single publication year when the title carries exactly one year */ | |
| 47 | + year: number | null; | |
| 48 | + variant: string | null; | |
| 49 | + isLot: boolean; | |
| 50 | +} | |
| 51 | + | |
| 52 | +/** | |
| 53 | + * "AMAZING SPIDER-MAN #129" · "Amazing Spider-Man (1963 1st Series) 300" · "X-MEN (1963-2011) #282" | |
| 54 | + * → { series: 'Amazing Spider-Man', issue: '129', seriesYears, year }. | |
| 55 | + */ | |
| 56 | +export function parseComicTitle(raw: string): ComicTitleParts { | |
| 57 | + let t = raw.replace(/\s+/g, ' ').trim(); | |
| 58 | + const isLot = /\b(group lot|lot of \d+|\d+\s+(?:issue|comic)s?\b.*\blot\b|collection of)\b/i.test(t); | |
| 59 | + const paren = t.match(/\(((?:19|20)\d{2})(?:\s*-\s*(\d{2,4}))?(?:\s+([^)]{1,30}))?\)/); | |
| 60 | + const seriesYears = paren ? paren[0].slice(1, -1).trim() : null; | |
| 61 | + const year = paren && !paren[2] ? Number(paren[1]) : null; | |
| 62 | + if (paren) t = t.replace(paren[0], ' '); | |
| 63 | + const hashIssue = t.match(/#\s*(\d+[A-Za-z]?(?:\.\d+)?(?:\/\d+)?)/); | |
| 64 | + let issue = hashIssue?.[1] ?? null; | |
| 65 | + if (hashIssue) t = t.replace(hashIssue[0], ' '); | |
| 66 | + else { | |
| 67 | + // MyComicShop style: "Amazing Spider-Man (1963 1st Series) 300" → trailing bare number | |
| 68 | + const tail = t.match(/\s(\d{1,4}[A-Za-z]?)(?:\s+(?:CGC|CBCS|PGX)\b.*)?$/); | |
| 69 | + if (tail) { | |
| 70 | + issue = tail[1]!; | |
| 71 | + t = t.slice(0, tail.index).trim(); | |
| 72 | + } | |
| 73 | + } | |
| 74 | + // strip grade tails ("CGC 9.4 NM", "VF 8.0") and sale words | |
| 75 | + t = t.replace(new RegExp(`\\b(?:CGC|CBCS|PGX)\\b.*$`, 'i'), ' ').replace(/\bSOLD\b.*$/i, ' '); | |
| 76 | + const variantM = t.match(/\b(variant|newsstand|direct edition|2nd print(?:ing)?|3rd print(?:ing)?|facsimile|sketch cover|virgin)\b/i); | |
| 77 | + const variant = variantM ? variantM[1]!.replace(/\b\w/g, (c) => c.toUpperCase()) : null; | |
| 78 | + if (variantM) t = t.replace(new RegExp(`\\b${variantM[1]!}(?:\\s+variant)?\\b`, 'i'), ' '); | |
| 79 | + const series = t | |
| 80 | + .replace(/\s+/g, ' ') | |
| 81 | + .trim() | |
| 82 | + .replace(/\s+comic books?$/i, '') | |
| 83 | + .toLowerCase() | |
| 84 | + .replace(/(^|[\s(/-])([a-z])/g, (m, pre: string, c: string) => pre + c.toUpperCase()); | |
| 85 | + return { series: series || raw.trim(), issue, seriesYears, year, variant, isLot }; | |
| 86 | +} | |
| 87 | + | |
| 88 | +/** Publisher (and optional language) → taxonomy slug by publisher family. Never returns null: family 'comics' is the honest fallback. */ | |
| 89 | +export function publisherCategory(publisher: string | null | undefined, language?: string | null): 'marvel_comics' | 'dc_comics' | 'manga' | 'independent_comics' | 'comics' { | |
| 90 | + const p = (publisher ?? '').toLowerCase(); | |
| 91 | + if (language && /^(ja|jp|japanese)$/i.test(language)) return 'manga'; | |
| 92 | + if (/\b(marvel|timely|atlas comics|atlas \[|marvel comics|epic comics|icon comics|max comics)\b/.test(p)) return 'marvel_comics'; | |
| 93 | + if (/^dc\b|\bdc comics\b|\bdetective comics\b|\bnational (?:periodical|comics|allied)|\bvertigo\b|\bwildstorm\b|\ball-american\b/.test(p)) return 'dc_comics'; | |
| 94 | + if (/\b(shueisha|kodansha|shogakukan|viz media|tokyopop|square enix|kadokawa|hakusensha|akita shoten)\b/.test(p)) return 'manga'; | |
| 95 | + if (p) return 'independent_comics'; | |
| 96 | + return 'comics'; | |
| 97 | +} | |
| 98 | + | |
| 99 | +/** "Label #4013069001" · "Label #16-19B5702-003" → certification number (CGC/CBCS). */ | |
| 100 | +export function parseLabelNumber(text: string | null | undefined): string | null { | |
| 101 | + if (!text) return null; | |
| 102 | + const m = text.match(/\bLabel\s*#\s*([0-9][0-9A-Za-z-]{5,})/i) ?? text.match(/\b(?:cert(?:ification)?|serial)\s*#?\s*:?\s*([0-9][0-9A-Za-z-]{6,})/i); | |
| 103 | + return m ? m[1]! : null; | |
| 104 | +} | |
| 105 | + | |
| 106 | +/** "$1,365" · "$1,030.00" → number (> 0) or null. */ | |
| 107 | +export function usd(s: string | null | undefined): number | null { | |
| 108 | + if (!s) return null; | |
| 109 | + const m = s.replace(/,/g, '').match(/\$?\s*(\d+(?:\.\d+)?)/); | |
| 110 | + if (!m) return null; | |
| 111 | + const n = Number(m[1]); | |
| 112 | + return Number.isFinite(n) && n > 0 ? n : null; | |
| 113 | +} | |
| 114 | + | |
| 115 | +/** Month names → 0-based month, accepts "Sept". */ | |
| 116 | +export function monthIndex(s: string): number | null { | |
| 117 | + const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; | |
| 118 | + const i = months.indexOf(s.slice(0, 3).toLowerCase()); | |
| 119 | + return i < 0 ? null : i; | |
| 120 | +} | |
| 121 | + | |
| 122 | +/** | |
| 123 | + * ComicLink session labels → the month the session ended (UTC first-of-month) at MONTH precision. | |
| 124 | + * "Spring Featured: Comics (5-6/26)" → 2026-06 · "Fall Featured (Oct-Nov)" + year 2022 → 2022-11 · | |
| 125 | + * "11-12/2019" → 2019-12 · "Jan/Feb 2024 Premium …" → 2024-02 · "March Premium…" + year → year-03. | |
| 126 | + */ | |
| 127 | +export function sessionEndMonth(label: string, fallbackYear: number | null): { year: number; month: number } | null { | |
| 128 | + const l = label.replace(/\s+/g, ' ').trim(); | |
| 129 | + let m = l.match(/(\d{1,2})\s*-\s*(\d{1,2})\s*\/\s*(\d{2,4})/) ?? l.match(/\b(\d{1,2})\s*\/\s*(\d{2,4})\b/); | |
| 130 | + if (m) { | |
| 131 | + const end = m.length === 4 ? Number(m[2]) : Number(m[1]); | |
| 132 | + const yr = Number(m[m.length - 1]); | |
| 133 | + const year = yr < 100 ? 2000 + yr : yr; | |
| 134 | + if (end >= 1 && end <= 12) return { year, month: end }; | |
| 135 | + } | |
| 136 | + const names = [...l.matchAll(/\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\b/gi)].map((x) => monthIndex(x[1]!)).filter((x): x is number => x !== null); | |
| 137 | + const yearM = l.match(/\b(20\d{2})\b/); | |
| 138 | + const year = yearM ? Number(yearM[1]) : fallbackYear; | |
| 139 | + if (names.length && year) return { year, month: names[names.length - 1]! + 1 }; | |
| 140 | + return null; | |
| 141 | +} | |
added
connectors/api/_g7-auctions-na-lib/index.ts
+263 −0
@@ -0,0 +1,263 @@ | ||
| 1 | +/** | |
| 2 | + * Helpers shared by the North-American auction-house connectors of group g7 (Miller & Miller, Wright, | |
| 3 | + * LAMA, Doyle, Fanatics Collect, Clean Sweep, HiBid). Source vocabulary and page-shape parsing live | |
| 4 | + * here; canonical models stay clean. Everything is pure and unit-testable on saved HTML. | |
| 5 | + */ | |
| 6 | +import { parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 7 | +import type { CurrencyCode } from '@rareindex/shared'; | |
| 8 | +import { hintFromLabel, isBundleTitle, safeYear, slugFromTitle, type DeptHint } from '../_auction-lib/categories.js'; | |
| 9 | +import { gradeOf, lotAttributes, makeLot, popCultureCategory, sportsCategory } from '../_memorabilia-lib/index.js'; | |
| 10 | +import { makeSale } from '../../firecrawl/_carlib/index.js'; | |
| 11 | + | |
| 12 | +export { gradeOf, isBundleTitle, lotAttributes, makeLot, makeSale, safeYear, sportsCategory }; | |
| 13 | + | |
| 14 | +// --------------------------------------------------------------------------------------------- | |
| 15 | +// JSON scanning inside HTML / JS text | |
| 16 | +// --------------------------------------------------------------------------------------------- | |
| 17 | + | |
| 18 | +/** Index just past the balanced JSON value (object/array) that starts at `start`; -1 when incomplete. */ | |
| 19 | +export function scanJsonEnd(text: string, start: number): number { | |
| 20 | + const open = text[start]; | |
| 21 | + if (open !== '{' && open !== '[') return -1; | |
| 22 | + let depth = 0; | |
| 23 | + let inStr = false; | |
| 24 | + for (let i = start; i < text.length; i++) { | |
| 25 | + const ch = text[i]!; | |
| 26 | + if (inStr) { | |
| 27 | + if (ch === '\\') i++; | |
| 28 | + else if (ch === '"') inStr = false; | |
| 29 | + continue; | |
| 30 | + } | |
| 31 | + if (ch === '"') inStr = true; | |
| 32 | + else if (ch === '{' || ch === '[') depth++; | |
| 33 | + else if (ch === '}' || ch === ']') { | |
| 34 | + depth--; | |
| 35 | + if (depth === 0) return i + 1; | |
| 36 | + } | |
| 37 | + } | |
| 38 | + return -1; | |
| 39 | +} | |
| 40 | + | |
| 41 | +/** Parse the balanced JSON value starting at `start` (null when malformed). */ | |
| 42 | +export function parseJsonAt<T = unknown>(text: string, start: number): T | null { | |
| 43 | + const end = scanJsonEnd(text, start); | |
| 44 | + if (end < 0) return null; | |
| 45 | + try { | |
| 46 | + return JSON.parse(text.slice(start, end)) as T; | |
| 47 | + } catch { | |
| 48 | + return null; | |
| 49 | + } | |
| 50 | +} | |
| 51 | + | |
| 52 | +/** | |
| 53 | + * Every JSON object that contains `marker` and starts with `objectStart` (e.g. Auction Mobility rows | |
| 54 | + * start with `{"row_id":"` and are typed by `"type":"auction-lot-summary"`). Nested objects that also | |
| 55 | + * match are skipped by jumping past each parsed object. | |
| 56 | + */ | |
| 57 | +export function jsonObjectsWithMarker<T = Record<string, unknown>>(text: string, marker: string, objectStart = '{"row_id":"'): T[] { | |
| 58 | + const out: T[] = []; | |
| 59 | + let from = 0; | |
| 60 | + for (;;) { | |
| 61 | + const i = text.indexOf(marker, from); | |
| 62 | + if (i < 0) break; | |
| 63 | + const start = text.lastIndexOf(objectStart, i); | |
| 64 | + if (start < 0) { | |
| 65 | + from = i + marker.length; | |
| 66 | + continue; | |
| 67 | + } | |
| 68 | + const end = scanJsonEnd(text, start); | |
| 69 | + if (end < 0 || end < i) { | |
| 70 | + from = i + marker.length; | |
| 71 | + continue; | |
| 72 | + } | |
| 73 | + try { | |
| 74 | + out.push(JSON.parse(text.slice(start, end)) as T); | |
| 75 | + } catch { | |
| 76 | + /* malformed slice: skip */ | |
| 77 | + } | |
| 78 | + from = Math.max(end, i + marker.length); | |
| 79 | + } | |
| 80 | + return out; | |
| 81 | +} | |
| 82 | + | |
| 83 | +/** Minimal HTML entity decoding for attribute payloads (Inertia `data-page`). */ | |
| 84 | +export function decodeEntities(s: string): string { | |
| 85 | + return s | |
| 86 | + .replace(/"/g, '"') | |
| 87 | + .replace(/�?39;|'/g, "'") | |
| 88 | + .replace(/</g, '<') | |
| 89 | + .replace(/>/g, '>') | |
| 90 | + .replace(/ /g, ' ') | |
| 91 | + .replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n))) | |
| 92 | + .replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCodePoint(Number.parseInt(n, 16))) | |
| 93 | + .replace(/&/g, '&'); | |
| 94 | +} | |
| 95 | + | |
| 96 | +/** Inertia.js page payload: `<div id="app" data-page="{…}">` (Wright / LAMA). */ | |
| 97 | +export function inertiaPage<T = Record<string, unknown>>(html: string): T | null { | |
| 98 | + const m = html.match(/<div[^>]+id="app"[^>]+data-page="([^"]+)"/); | |
| 99 | + if (!m) return null; | |
| 100 | + try { | |
| 101 | + return JSON.parse(decodeEntities(m[1]!)) as T; | |
| 102 | + } catch { | |
| 103 | + return null; | |
| 104 | + } | |
| 105 | +} | |
| 106 | + | |
| 107 | +/** Next.js App Router flight payload: concatenated `self.__next_f.push([1,"…"])` chunks, JS-string decoded. */ | |
| 108 | +export function nextFlightText(html: string): string { | |
| 109 | + const re = /self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)/g; | |
| 110 | + const parts: string[] = []; | |
| 111 | + let m: RegExpExecArray | null; | |
| 112 | + while ((m = re.exec(html))) { | |
| 113 | + try { | |
| 114 | + parts.push(JSON.parse(`"${m[1]!}"`) as string); | |
| 115 | + } catch { | |
| 116 | + /* skip undecodable chunk */ | |
| 117 | + } | |
| 118 | + } | |
| 119 | + return parts.join(''); | |
| 120 | +} | |
| 121 | + | |
| 122 | +/** First JSON object following `key` (e.g. `"prefetchedItemData":`) in a flight/JS text. */ | |
| 123 | +export function jsonAfterKey<T = Record<string, unknown>>(text: string, key: string, from = 0): T | null { | |
| 124 | + const i = text.indexOf(key, from); | |
| 125 | + if (i < 0) return null; | |
| 126 | + const start = text.indexOf('{', i + key.length); | |
| 127 | + if (start < 0 || start - (i + key.length) > 4) return null; | |
| 128 | + return parseJsonAt<T>(text, start); | |
| 129 | +} | |
| 130 | + | |
| 131 | +/** HiBid: `<script id="hibid-state" type="application/json">` → Apollo normalised cache. */ | |
| 132 | +export function hibidApolloState(html: string): Record<string, Record<string, unknown>> | null { | |
| 133 | + const m = html.match(/<script id="hibid-state" type="application\/json">([\s\S]*?)<\/script>/); | |
| 134 | + if (!m) return null; | |
| 135 | + try { | |
| 136 | + const st = JSON.parse(decodeEntities(m[1]!)) as Record<string, unknown>; | |
| 137 | + const cache = (st['apollo.state'] ?? st) as Record<string, Record<string, unknown>>; | |
| 138 | + return cache && typeof cache === 'object' ? cache : null; | |
| 139 | + } catch { | |
| 140 | + return null; | |
| 141 | + } | |
| 142 | +} | |
| 143 | + | |
| 144 | +/** Resolve an Apollo `{ __ref }` (or inline object) against the cache. */ | |
| 145 | +export function apolloRef<T = Record<string, unknown>>(cache: Record<string, Record<string, unknown>>, v: unknown): T | null { | |
| 146 | + if (!v || typeof v !== 'object') return null; | |
| 147 | + const ref = (v as { __ref?: string }).__ref; | |
| 148 | + if (ref) return (cache[ref] as T | undefined) ?? null; | |
| 149 | + return v as T; | |
| 150 | +} | |
| 151 | + | |
| 152 | +// --------------------------------------------------------------------------------------------- | |
| 153 | +// Dates & money | |
| 154 | +// --------------------------------------------------------------------------------------------- | |
| 155 | + | |
| 156 | +const MONTHS: Record<string, number> = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, sept: 8, oct: 9, nov: 10, dec: 11 }; | |
| 157 | + | |
| 158 | +/** "Sep 2, 2026 10:00 EST" | "Apr 16, 2026" → UTC date (US Eastern wall clock converted when a time zone is given). */ | |
| 159 | +export function parseUsDate(s: string | null | undefined): Date | null { | |
| 160 | + if (!s) return null; | |
| 161 | + const m = s.match(/([A-Za-z]{3,9})\.?\s+(\d{1,2}),?\s+(\d{4})(?:\s+(\d{1,2}):(\d{2})\s*([AP]M)?\s*(E[SD]T|C[SD]T|M[SD]T|P[SD]T|UTC)?)?/); | |
| 162 | + if (!m) return null; | |
| 163 | + const mo = MONTHS[m[1]!.slice(0, 4).toLowerCase()] ?? MONTHS[m[1]!.slice(0, 3).toLowerCase()]; | |
| 164 | + if (mo === undefined) return null; | |
| 165 | + const y = Number(m[3]); | |
| 166 | + const d = Number(m[2]); | |
| 167 | + if (!m[4]) return new Date(Date.UTC(y, mo, d)); | |
| 168 | + let hour = Number(m[4]); | |
| 169 | + if (m[6] === 'PM' && hour < 12) hour += 12; | |
| 170 | + if (m[6] === 'AM' && hour === 12) hour = 0; | |
| 171 | + const offsets: Record<string, number> = { EST: 5, EDT: 4, CST: 6, CDT: 5, MST: 7, MDT: 6, PST: 8, PDT: 7, UTC: 0 }; | |
| 172 | + const off = offsets[m[7] ?? 'EST'] ?? 5; | |
| 173 | + return new Date(Date.UTC(y, mo, d, hour + off, Number(m[5]))); | |
| 174 | +} | |
| 175 | + | |
| 176 | +/** "april-2025" | "April 2025" → first day of that month (UTC). Precision is the month — callers must say so. */ | |
| 177 | +export function monthYearDate(s: string | null | undefined): Date | null { | |
| 178 | + if (!s) return null; | |
| 179 | + const m = s.match(/([A-Za-z]{3,9})[\s-]+(\d{4})/); | |
| 180 | + if (!m) return null; | |
| 181 | + const mo = MONTHS[m[1]!.slice(0, 3).toLowerCase()]; | |
| 182 | + if (mo === undefined) return null; | |
| 183 | + return new Date(Date.UTC(Number(m[2]), mo, 1)); | |
| 184 | +} | |
| 185 | + | |
| 186 | +/** ISO 8601 (with or without zone; zone-less values are treated as UTC) → Date or null. */ | |
| 187 | +export function isoDate(s: string | null | undefined): Date | null { | |
| 188 | + if (!s) return null; | |
| 189 | + const iso = /[zZ]$|[+-]\d{2}:?\d{2}$/.test(s) ? s : `${s}Z`; | |
| 190 | + const d = new Date(iso); | |
| 191 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 192 | +} | |
| 193 | + | |
| 194 | +/** "$2,048" | "2048.00" | 2048 → positive number or null. */ | |
| 195 | +export function amount(v: unknown): number | null { | |
| 196 | + if (v === null || v === undefined || v === '') return null; | |
| 197 | + const n = typeof v === 'number' ? v : Number.parseFloat(String(v).replace(/[^0-9.-]/g, '')); | |
| 198 | + return Number.isFinite(n) && n > 0 ? n : null; | |
| 199 | +} | |
| 200 | + | |
| 201 | +export function isCurrency(s: string | null | undefined): s is CurrencyCode { | |
| 202 | + return s === 'USD' || s === 'CAD' || s === 'EUR' || s === 'GBP' || s === 'CHF' || s === 'HKD' || s === 'AUD' || s === 'JPY'; | |
| 203 | +} | |
| 204 | + | |
| 205 | +// --------------------------------------------------------------------------------------------- | |
| 206 | +// Category mapping for general-purpose houses | |
| 207 | +// --------------------------------------------------------------------------------------------- | |
| 208 | + | |
| 209 | +const SPORTS_LABEL = /sports?\s*(cards?|memorabilia)|hockey|baseball|basketball|football|trading cards?|game[- ]used/i; | |
| 210 | +const POP_LABEL = /pop culture|toys?|comics?|advertising|petroliana|coin[- ]op|movie|music|entertainment|disney|star wars/i; | |
| 211 | + | |
| 212 | +/** | |
| 213 | + * Department/sale label + lot title → taxonomy slug. Sports-card houses route through `sportsCategory`; | |
| 214 | + * pop-culture sales through `popCultureCategory`; everything else through the shared auction mapper. | |
| 215 | + * Returns null when nothing confident matched (the connector counts an anomaly and skips the lot). | |
| 216 | + */ | |
| 217 | +export function houseCategory(label: string | null | undefined, title: string, fallback: string | null = null): string | null { | |
| 218 | + const l = label ?? ''; | |
| 219 | + if (SPORTS_LABEL.test(l)) return sportsCategory(title); | |
| 220 | + if (/petroliana|advertising|soda|gas|oil|signs?/i.test(l)) return /\b(sign|clock|thermometer|display|calendar|poster|globe|tin|can|bottle|crate)\b/i.test(title) ? 'advertising' : slugFromTitle(title, 'unknown') ?? 'advertising'; | |
| 221 | + if (POP_LABEL.test(l)) return popCultureCategory(title); | |
| 222 | + const hint: DeptHint = hintFromLabel(l); | |
| 223 | + const slug = slugFromTitle(title, hint); | |
| 224 | + if (slug) return slug; | |
| 225 | + if (hint === 'design') return 'design_furniture'; | |
| 226 | + if (hint === 'art' || hint === 'contemporary' || hint === 'prints') return 'art'; | |
| 227 | + if (hint === 'furniture' || hint === 'decorative' || hint === 'asian' || hint === 'tribal' || hint === 'antiquities') return 'antiques'; | |
| 228 | + if (hint === 'books') return 'books'; | |
| 229 | + if (hint === 'jewelry') return 'jewelry'; | |
| 230 | + if (hint === 'watches') return 'other_watches'; | |
| 231 | + if (hint === 'coins') return 'coins'; | |
| 232 | + return fallback; | |
| 233 | +} | |
| 234 | + | |
| 235 | +/** Card/TCG-aware mapper for sports-card marketplaces (Fanatics Collect). */ | |
| 236 | +export function cardHouseCategory(title: string): string { | |
| 237 | + if (/\bpok[eé]mon\b|\bcharizard\b|\bpikachu\b/i.test(title)) return 'pokemon'; | |
| 238 | + if (/magic:? the gathering|\bmtg\b|black lotus|\bmox\b/i.test(title)) return 'magic_the_gathering'; | |
| 239 | + if (/yu-?gi-?oh/i.test(title)) return 'yugioh'; | |
| 240 | + if (/\bone piece\b/i.test(title) && /\b(card|tcg|op0\d|leader|alt art|manga)\b/i.test(title)) return 'one_piece_card_game'; | |
| 241 | + if (/\blorcana\b/i.test(title)) return 'disney_lorcana'; | |
| 242 | + if (/\bdragon ?ball\b/i.test(title) && /\bcard|tcg|fusion world\b/i.test(title)) return 'dragon_ball_tcg'; | |
| 243 | + if (/\bdigimon\b/i.test(title)) return 'digimon_tcg'; | |
| 244 | + if (/\bflesh and blood\b/i.test(title)) return 'flesh_and_blood'; | |
| 245 | + if (/\bweiss schwarz\b/i.test(title)) return 'weiss_schwarz'; | |
| 246 | + if (/\b(marvel|dc|star wars|garbage pail|wacky pack|non-?sport)\b/i.test(title) && /\bcard|topps|panini|upper deck|fleer|skybox\b/i.test(title)) return 'non_sport_cards'; | |
| 247 | + if (/\b(sealed|booster box|hobby box|wax box|blaster|case)\b/i.test(title) && !/\bcard\b/i.test(title)) return sportsCategory(title); | |
| 248 | + if (/\b(cgc|cbcs)\b.*\b#\s?\d+/i.test(title) && !/\bcard\b/i.test(title)) return popCultureCategory(title); | |
| 249 | + return sportsCategory(title); | |
| 250 | +} | |
| 251 | + | |
| 252 | +/** Grade for sale records: title grading via taxonomy, plus PSA/DNA-style authentication left as null grader. */ | |
| 253 | +export function saleGrade(title: string): { grader: string | null; grade: string | null; qualifier: string | null } { | |
| 254 | + const g = gradeOf(title); | |
| 255 | + const q = parseGradeFromTitle(title).qualifier; | |
| 256 | + return { grader: g.grader, grade: g.grade, qualifier: q }; | |
| 257 | +} | |
| 258 | + | |
| 259 | +/** Certification number written in a title, e.g. "PSA 10 (cert 12345678)" / "Cert #: 12345678". */ | |
| 260 | +export function certFromTitle(title: string): string | null { | |
| 261 | + const m = title.match(/\bcert(?:ification)?\.?\s*(?:no\.?|#|number)?[:\s#]*(\d{7,12})\b/i) ?? title.match(/\b(?:PSA|BGS|SGC|CGC|CBCS)\b[^()]{0,40}\((\d{7,12})\)/i); | |
| 262 | + return m ? m[1]! : null; | |
| 263 | +} | |
added
connectors/api/_g7-auctions-na-lib/smoke.ts
+76 −0
@@ -0,0 +1,76 @@ | ||
| 1 | +/** | |
| 2 | + * Live smoke + fixture capture for the g7 North-American auction connectors (works before the registry is | |
| 3 | + * rebuilt: the connector's own meta.json is used). | |
| 4 | + * Usage: pnpm tsx connectors/api/_g7-auctions-na-lib/smoke.ts <engine>/<id> [--limit N] [--seed URL]... [--capture name] [--trim N] [--mode probe|incremental|backfill] | |
| 5 | + */ | |
| 6 | +import { readFileSync } from 'node:fs'; | |
| 7 | +import path from 'node:path'; | |
| 8 | +import { pathToFileURL } from 'node:url'; | |
| 9 | +import { ConnectorMetaSchema, createCrawlContext, createRouter, DomainsFileSchema, setDomains, type ConnectorFactory, type ConnectorMeta } from '@rareindex/connectors'; | |
| 10 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 11 | +import { childLogger } from '@rareindex/shared'; | |
| 12 | + | |
| 13 | +// Parallel connector agents may leave an invalid fragment in connectors/domains.d/ which would make | |
| 14 | +// loadDomains() throw for everybody; pre-seed the policy cache from domains.json + this group's fragment. | |
| 15 | +{ | |
| 16 | + const base = JSON.parse(readFileSync(path.resolve(process.cwd(), 'connectors/domains.json'), 'utf8')) as { version?: string; defaults?: unknown; domains?: Record<string, unknown> }; | |
| 17 | + const frag = JSON.parse(readFileSync(path.resolve(process.cwd(), 'connectors/domains.d/g7-auctions-na.json'), 'utf8')) as { domains?: Record<string, unknown> }; | |
| 18 | + setDomains(DomainsFileSchema.parse({ ...base, domains: { ...(base.domains ?? {}), ...(frag.domains ?? {}) } })); | |
| 19 | +} | |
| 20 | + | |
| 21 | +const [modulePath, ...rest] = process.argv.slice(2); | |
| 22 | +if (!modulePath) throw new Error('usage: smoke.ts <engine>/<id> [--limit N] [--seed URL] [--capture name] [--trim N] [--mode m]'); | |
| 23 | +const opt = { limit: 3, seeds: [] as string[], capture: null as string | null, trim: 12, mode: 'probe' as 'probe' | 'incremental' | 'backfill', maxRaw: 4 }; | |
| 24 | +for (let i = 0; i < rest.length; i++) { | |
| 25 | + const a = rest[i]!; | |
| 26 | + if (a === '--limit') opt.limit = Number(rest[++i]); | |
| 27 | + else if (a === '--seed') opt.seeds.push(rest[++i]!); | |
| 28 | + else if (a === '--capture') opt.capture = rest[++i]!; | |
| 29 | + else if (a === '--trim') opt.trim = Number(rest[++i]); | |
| 30 | + else if (a === '--mode') opt.mode = rest[++i] as typeof opt.mode; | |
| 31 | + else if (a === '--max-raw') opt.maxRaw = Number(rest[++i]); | |
| 32 | +} | |
| 33 | + | |
| 34 | +const dir = path.resolve(process.cwd(), 'connectors', modulePath); | |
| 35 | +const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 36 | +const factory = (await import(pathToFileURL(path.join(dir, 'index.ts')).href)).default as (meta: ConnectorMeta) => ReturnType<ConnectorFactory>; | |
| 37 | +const connector = await factory(meta); | |
| 38 | +const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY }); | |
| 39 | +const ctx = createCrawlContext({ router, meta, options: { mode: opt.mode, limit: opt.limit, ...(opt.seeds.length ? { seeds: opt.seeds } : {}) }, log: childLogger({ connector: meta.id, smoke: true }), onCursor: async (c) => console.log('cursor →', JSON.stringify(c).slice(0, 300)) }); | |
| 40 | + | |
| 41 | +/** Trim list-shaped payloads so fixtures stay small (keeps every scalar field, first N items). */ | |
| 42 | +function trim(payload: unknown): unknown { | |
| 43 | + if (!payload || typeof payload !== 'object') return payload; | |
| 44 | + const p = { ...(payload as Record<string, unknown>) }; | |
| 45 | + for (const k of Object.keys(p)) if (Array.isArray(p[k]) && (p[k] as unknown[]).length > opt.trim) p[k] = (p[k] as unknown[]).slice(0, opt.trim); | |
| 46 | + return p; | |
| 47 | +} | |
| 48 | + | |
| 49 | +let rawCount = 0; | |
| 50 | +let total = 0; | |
| 51 | +let captured = 0; | |
| 52 | +const kinds: Record<string, number> = {}; | |
| 53 | +for await (const raw of connector.crawl(ctx)) { | |
| 54 | + rawCount++; | |
| 55 | + const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }; | |
| 56 | + const records = await connector.normalize(rawLike); | |
| 57 | + total += records.length; | |
| 58 | + for (const r of records) kinds[r.kind] = (kinds[r.kind] ?? 0) + 1; | |
| 59 | + console.log(`raw#${rawCount} ${raw.kind} ${raw.url} → ${records.length} records`); | |
| 60 | + for (const r of records.slice(0, 2)) console.log(JSON.stringify(r).slice(0, 900)); | |
| 61 | + if (opt.capture && captured < 3 && records.length) { | |
| 62 | + const trimmed = trim(JSON.parse(JSON.stringify(rawLike.payload))); | |
| 63 | + const recs = await connector.normalize({ ...rawLike, payload: trimmed }); | |
| 64 | + const first = recs[0]!; | |
| 65 | + const name = `${opt.capture}-${captured + 1}`; | |
| 66 | + saveFixture(meta.id, name, { | |
| 67 | + raw: { ...rawLike, payload: trimmed }, | |
| 68 | + expect: { minCount: 1, kinds: [...new Set(recs.map((r) => r.kind))], first: { kind: first.kind, ...('auctionHouse' in first && first.auctionHouse ? { auctionHouse: first.auctionHouse } : {}), ...('currency' in first && first.currency ? { currency: first.currency } : {}) } }, | |
| 69 | + note: `Captured live by connectors/api/_g7-auctions-na-lib/smoke.ts on ${new Date().toISOString().slice(0, 10)} from ${raw.url} (payload lists trimmed to ${opt.trim} items; ${recs.length} records).`, | |
| 70 | + }); | |
| 71 | + captured++; | |
| 72 | + console.log(` saved fixture data/fixtures/${meta.id}/${name}.json`); | |
| 73 | + } | |
| 74 | + if (rawCount >= opt.maxRaw) break; | |
| 75 | +} | |
| 76 | +console.log(JSON.stringify({ rawCount, totalNormalized: total, kinds, engineStats: ctx.engineStats, anomalies: ctx.anomalies }, null, 1)); | |
added
connectors/api/_g8-auctions-eu-apac-lib/capture.ts
+52 −0
@@ -0,0 +1,52 @@ | ||
| 1 | +/** | |
| 2 | + * Live smoke + fixture capture for the g8 auction connectors (plain HTTP, no paid engines unless the | |
| 3 | + * connector's meta says so). Runs the real router/context in probe mode and saves the first raw records. | |
| 4 | + * Usage: pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts <connectorId> [--save] [--limit N] [--name prefix] | |
| 5 | + */ | |
| 6 | +import { readFileSync } from 'node:fs'; | |
| 7 | +import path from 'node:path'; | |
| 8 | +import { ConnectorMetaSchema, DomainsFileSchema, DOMAINS_PATH, createCrawlContext, createRouter, setDomains, type RareIndexConnector } from '@rareindex/connectors'; | |
| 9 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 10 | +import { childLogger } from '@rareindex/shared'; | |
| 11 | + | |
| 12 | +// Load domains.json + only this group's fragment (other groups' work-in-progress fragments may be invalid while agents run in parallel). | |
| 13 | +{ | |
| 14 | + const base = DomainsFileSchema.parse(JSON.parse(readFileSync(DOMAINS_PATH, 'utf8'))); | |
| 15 | + const frag = JSON.parse(readFileSync(path.resolve(process.cwd(), 'connectors/domains.d/g8-auctions-eu-apac.json'), 'utf8')) as { domains: Record<string, unknown> }; | |
| 16 | + setDomains({ ...base, domains: { ...base.domains, ...(frag.domains as typeof base.domains) } }); | |
| 17 | +} | |
| 18 | + | |
| 19 | +const [id, ...rest] = process.argv.slice(2); | |
| 20 | +if (!id) throw new Error('usage: capture.ts <connectorId> [--save] [--limit N] [--name prefix]'); | |
| 21 | +const save = rest.includes('--save'); | |
| 22 | +const limit = rest.includes('--limit') ? Number(rest[rest.indexOf('--limit') + 1]) : 2; | |
| 23 | +const prefix = rest.includes('--name') ? rest[rest.indexOf('--name') + 1]! : 'sale'; | |
| 24 | + | |
| 25 | +const dir = path.resolve(process.cwd(), 'connectors/api', id); | |
| 26 | +const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 27 | +const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: typeof meta) => RareIndexConnector }; | |
| 28 | +const connector = mod.default(meta); | |
| 29 | +const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY }); | |
| 30 | +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) }); | |
| 31 | + | |
| 32 | +let i = 0; | |
| 33 | +const started = Date.now(); | |
| 34 | +for await (const raw of connector.crawl(ctx)) { | |
| 35 | + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); | |
| 36 | + const sales = out.filter((r) => r.kind === 'sale'); | |
| 37 | + const lots = out.filter((r) => r.kind === 'auction_lot'); | |
| 38 | + const currencies = [...new Set(sales.map((r) => (r.kind === 'sale' ? r.currency : '')))]; | |
| 39 | + console.log(`raw ${raw.externalId} (${raw.engine}, ${raw.httpStatus}) → ${out.length} records: ${sales.length} sales ${JSON.stringify(currencies)}, ${lots.length} auction_lot`); | |
| 40 | + for (const r of out.slice(0, 2)) console.log(' ', JSON.stringify(r).slice(0, 360)); | |
| 41 | + if (save) { | |
| 42 | + const name = `${prefix}-${String(raw.externalId ?? i).replace(/[^a-z0-9]+/gi, '-').toLowerCase()}`; | |
| 43 | + saveFixture(meta.id, name, { | |
| 44 | + raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, | |
| 45 | + expect: { minCount: Math.max(1, Math.min(5, out.length)), kinds: ['sale', 'auction_lot'], requiredFields: ['rawTitle', 'attributes.categorySlug'] }, | |
| 46 | + note: `Live capture via the real router (engine ${raw.engine}) from ${raw.url} on ${new Date().toISOString().slice(0, 10)}; payload trimmed by the connector's own parser.`, | |
| 47 | + }); | |
| 48 | + console.log(` saved fixture ${name}`); | |
| 49 | + } | |
| 50 | + i++; | |
| 51 | +} | |
| 52 | +console.log(`done raw=${i} ms=${Date.now() - started} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)}`); | |
added
connectors/api/_g8-auctions-eu-apac-lib/drouot-platform.ts
+108 −0
@@ -0,0 +1,108 @@ | ||
| 1 | +/** | |
| 2 | + * Parser for the Drouot "site générique" platform used by many French houses (Ader, Osenat, …): | |
| 3 | + * - past-sales index: /ventes-passees?year=YYYY (Ader) or /resultats-ventes-passees?year=YYYY (Osenat) with | |
| 4 | + * `.calendrier.entry` blocks (title, "jeudi 09 juillet 2026 à 14h00", venue, /catalogue/<id>-slug link); | |
| 5 | + * - catalogue: /catalogue/<id>?offset=N&max=50 with `.product` cards (lot number, title, description, image, | |
| 6 | + * "Estimation : 150 - 200 EUR", "Résultat : 1 081 EUR") and a page-level `.explicationResultats` note that | |
| 7 | + * says whether results are "avec frais" (premium-inclusive) or "sans frais" (hammer) — read per page, never assumed. | |
| 8 | + */ | |
| 9 | +import type { CurrencyCode } from '@rareindex/shared'; | |
| 10 | +import { parseEuDate, parseEuMoney } from './index.js'; | |
| 11 | +import { absolute, chunksBetween, pick, textOf, type ParsedLot, type ParsedSalePage, type SaleRef } from './sale-results.js'; | |
| 12 | + | |
| 13 | +export interface DrouotHouse { | |
| 14 | + base: string; | |
| 15 | + currency: CurrencyCode; | |
| 16 | +} | |
| 17 | + | |
| 18 | +/** Index page → sales (only entries that link to a /catalogue/ — sales without an online catalogue are skipped). */ | |
| 19 | +export function parseDrouotIndex(htmlText: string, house: DrouotHouse): SaleRef[] { | |
| 20 | + const out: SaleRef[] = []; | |
| 21 | + const seen = new Set<string>(); | |
| 22 | + for (const chunk of chunksBetween(htmlText, /<div class="calendrier entry[^"]*"/)) { | |
| 23 | + const link = chunk.match(/href="(\/catalogue\/(\d+)[^"]*)"/); | |
| 24 | + if (!link) continue; | |
| 25 | + const id = link[2]!; | |
| 26 | + if (seen.has(id)) continue; | |
| 27 | + seen.add(id); | |
| 28 | + const title = pick(chunk, /<h2>\s*<a[^>]*>([\s\S]*?)<\/a>/) ?? pick(chunk, /alt="([^"]*)"/) ?? id; | |
| 29 | + const dateText = pick(chunk, /bloc_vente_date">[\s\S]*?<\/i>([\s\S]*?)<\/div>/); | |
| 30 | + const venue = pick(chunk, /bloc_vente_lieu">[\s\S]*?<\/i>([\s\S]*?)<\/div>/); | |
| 31 | + const date = dateText ? parseEuDate(dateText) : null; | |
| 32 | + out.push({ id, title, url: `${house.base}${link[1]!.split('?')[0]}`, date: date ? date.toISOString() : null, location: venue ? cleanVenue(venue) : null, extra: { date_text: dateText } }); | |
| 33 | + } | |
| 34 | + return out; | |
| 35 | +} | |
| 36 | + | |
| 37 | +function cleanVenue(v: string): string { | |
| 38 | + return v.replace(/\s*,\s*/g, ', ').replace(/(www\.[^\s,]+)(?:, \1)+/g, '$1').trim(); | |
| 39 | +} | |
| 40 | + | |
| 41 | +/** Catalogue page → lots + sale header (title/date/venue) + premium basis. */ | |
| 42 | +export function parseDrouotCatalogue(htmlText: string, sale: SaleRef, house: DrouotHouse, pageSize = 50): ParsedSalePage | null { | |
| 43 | + if (!/class="[^"]*\bproduct\b[^"]*"/.test(htmlText) && !/nbre_lot_haut/.test(htmlText)) return null; | |
| 44 | + const header = { | |
| 45 | + title: pick(htmlText, /<h1 class="nom_vente">([\s\S]*?)<\/h1>/), | |
| 46 | + dateText: pick(htmlText, /<div class="date_vente">([\s\S]*?)<\/div>/), | |
| 47 | + venue: pick(htmlText, /<div class="lieu_vente">([\s\S]*?)<\/div>/), | |
| 48 | + }; | |
| 49 | + const countText = pick(htmlText, /nbre_lot_haut">([\s\S]*?)<\/div>/); | |
| 50 | + const m = countText?.match(/Lots?\s+(\d+)\s+à\s+(\d+)\s+sur\s+(\d+)/i); | |
| 51 | + const total = m ? Number(m[3]) : null; | |
| 52 | + const to = m ? Number(m[2]) : null; | |
| 53 | + const pageNote = pick(htmlText, /explicationResultats">([\s\S]*?)<\/div>/); | |
| 54 | + const pagePremium = premiumFromNote(pageNote); | |
| 55 | + const lots: ParsedLot[] = []; | |
| 56 | + for (const chunk of chunksBetween(htmlText, /<div class="ordre_[a-z]+ product[^"]*"/, /<div class="pagination_catalogue">|<footer|id="footer"/)) { | |
| 57 | + const lotId = chunk.match(/lotId(\d+)/)?.[1] ?? null; | |
| 58 | + const href = chunk.match(/href="(\/lot\/\d+\/[^"]+)"/)?.[1] ?? null; | |
| 59 | + const lotNo = pick(chunk, /<span class='lotnum'>([^<]*)</) ?? pick(chunk, /class="lotnum">([^<]*)</); | |
| 60 | + const title = pick(chunk, /<div class="product-title">[\s\S]*?<a[^>]*>([\s\S]*?)<\/a>/); | |
| 61 | + const desc = pick(chunk, /<h2 id="lotDesc-\d+">([\s\S]*?)<\/h2>/); | |
| 62 | + if (!lotNo || !(title || desc)) continue; | |
| 63 | + const image = chunk.match(/<img src="([^"]+)" class="lot_visu"/)?.[1] ?? chunk.match(/<img src="(https:\/\/cdn\.drouot\.com[^"]+)"/)?.[1] ?? null; | |
| 64 | + const est = pick(chunk, /estimAff4">([\s\S]*?)<\/div>/); | |
| 65 | + const estM = est?.match(/([\d\s.,]+)\s*-\s*([\d\s.,]+)\s*([A-Z]{3})?/); | |
| 66 | + const resText = pick(chunk, /sale-flash2?">\s*Résultat\s*:?\s*<nobr>([\s\S]*?)<\/nobr>/) ?? pick(chunk, /Résultat\s*:?\s*<nobr>([^<]*)</); | |
| 67 | + const price = resText ? parseEuMoney(resText, house.currency) : null; | |
| 68 | + const note = pick(chunk, /explicationResultats">([\s\S]*?)<\/div>/); | |
| 69 | + const premiumIncluded = premiumFromNote(note) ?? pagePremium; | |
| 70 | + // the card title is truncated ("Georges HUGNET (1906-1974)..."); the lotDesc heading carries the full first line | |
| 71 | + // also when the short card title is just the artist/maker and the lotDesc heading expands on it | |
| 72 | + const truncated = !title || /\.{3}$|…$/.test(title) || (!!desc && desc.length > title.length && desc.toLowerCase().startsWith(title.toLowerCase())); | |
| 73 | + const fullTitle = truncated && desc ? desc : (title ?? desc)!.replace(/\.{3}$|…$/, '').trim(); | |
| 74 | + lots.push({ | |
| 75 | + lotNo: lotNo.replace(/\s+/g, ''), | |
| 76 | + title: fullTitle, | |
| 77 | + subtitle: null, | |
| 78 | + description: desc && desc !== fullTitle ? desc : null, | |
| 79 | + url: absolute(house.base, href) ?? `${sale.url}#lot${lotNo}`, | |
| 80 | + image: image ? image.replace(/&/g, '&') : null, | |
| 81 | + price: price?.amount ?? null, | |
| 82 | + currency: price?.currency ?? house.currency, | |
| 83 | + premiumIncluded, | |
| 84 | + estimateLow: estM ? parseEuMoney(`${estM[1]} ${estM[3] ?? house.currency}`, house.currency)?.amount ?? null : null, | |
| 85 | + estimateHigh: estM ? parseEuMoney(`${estM[2]} ${estM[3] ?? house.currency}`, house.currency)?.amount ?? null : null, | |
| 86 | + date: null, | |
| 87 | + sold: price !== null, | |
| 88 | + extra: { drouot_lot_id: lotId, premium_note: note ?? pageNote ?? null }, | |
| 89 | + }); | |
| 90 | + } | |
| 91 | + const date = header.dateText ? parseEuDate(header.dateText) : null; | |
| 92 | + return { | |
| 93 | + lots, | |
| 94 | + hasMore: total !== null && to !== null ? to < total : lots.length >= pageSize, | |
| 95 | + totalLots: total, | |
| 96 | + sale: { title: header.title ?? undefined, date: date ? date.toISOString() : undefined, location: header.venue ? cleanVenue(header.venue) : undefined, extra: { premium_note: pageNote ?? null, date_text: header.dateText ?? null } }, | |
| 97 | + }; | |
| 98 | +} | |
| 99 | + | |
| 100 | +/** "Résultats avec frais" → true; "Résultats sans frais" → false; anything else → null (unknown). */ | |
| 101 | +export function premiumFromNote(note: string | null | undefined): boolean | null { | |
| 102 | + if (!note) return null; | |
| 103 | + if (/avec\s+frais|frais\s+inclus|frais\s+compris|incl\w*\s+(?:buyer|premium|fees)/i.test(note)) return true; | |
| 104 | + if (/sans\s+frais|hors\s+frais|prix\s+marteau|hammer/i.test(note)) return false; | |
| 105 | + return null; | |
| 106 | +} | |
| 107 | + | |
| 108 | +export { textOf }; | |
added
connectors/api/_g8-auctions-eu-apac-lib/goauction-platform.ts
+114 −0
@@ -0,0 +1,114 @@ | ||
| 1 | +/** | |
| 2 | + * Parser for the "goauction" UK auction-house platform (Sworders, Chiswick Auctions, Dominic Winter …): | |
| 3 | + * - results calendar: `.auction-calendar-item` blocks with title, "Tuesday 21 April 2026" (or a range), sale number | |
| 4 | + * and a link carrying `au=<auctionId>`; | |
| 5 | + * - lot grid: `.auction-grid-lot` / `.auction-lot` cards with "Lot 12 - Title", optional `.sub-title`, image, | |
| 6 | + * "Sold for £1,800" (only sold lots carry a price; unsold lots show nothing), 48–96 per page, `<link rel="next">`. | |
| 7 | + * Prices are what the house publishes as "Sold for" — the pages do not say hammer vs premium, so the basis is | |
| 8 | + * left to each connector (null = unknown unless the house's terms say otherwise). | |
| 9 | + */ | |
| 10 | +import type { CurrencyCode } from '@rareindex/shared'; | |
| 11 | +import { parseEuMoney } from './index.js'; | |
| 12 | +import { absolute, chunksBetween, pick, textOf, type ParsedLot, type ParsedSalePage, type SaleRef } from './sale-results.js'; | |
| 13 | + | |
| 14 | +export interface GoauctionHouse { | |
| 15 | + base: string; | |
| 16 | + currency: CurrencyCode; | |
| 17 | + /** path of the lot-list page: 'details' → /auction/details/<slug>/?au=ID ; 'search' → /Auction/Search?au=ID&sd=2 */ | |
| 18 | + listStyle: 'details' | 'search'; | |
| 19 | +} | |
| 20 | + | |
| 21 | +/** | |
| 22 | + * "Tuesday 21 April 2026" | "Tuesday 1 September - Monday 7 September 2026" | "20th August 2026" | "4th Sep, 2026 12:00" | |
| 23 | + * → first day ISO (UTC midnight). | |
| 24 | + */ | |
| 25 | +export function parseUkDate(text: string | null | undefined): string | null { | |
| 26 | + if (!text) return null; | |
| 27 | + const t = text.replace(/\s+/g, ' ').trim(); | |
| 28 | + const range = t.match(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})?,?\s*-\s*[A-Za-z]*,?\s*(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9}),?\s+(\d{4})/); | |
| 29 | + const m = range ? { d: range[1]!, mon: range[2] ?? range[4]!, y: range[5]! } : (() => { | |
| 30 | + const s = t.match(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(\d{4})/); | |
| 31 | + return s ? { d: s[1]!, mon: s[2]!, y: s[3]! } : null; | |
| 32 | + })(); | |
| 33 | + if (!m) return null; | |
| 34 | + const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; | |
| 35 | + const mo = months.indexOf(m.mon.slice(0, 3).toLowerCase()); | |
| 36 | + if (mo < 0) return null; | |
| 37 | + const d = new Date(Date.UTC(Number(m.y), mo, Number(m.d))); | |
| 38 | + return Number.isNaN(d.getTime()) ? null : d.toISOString(); | |
| 39 | +} | |
| 40 | + | |
| 41 | +export function parseGoauctionCalendar(htmlText: string, house: GoauctionHouse): SaleRef[] { | |
| 42 | + const out: SaleRef[] = []; | |
| 43 | + const seen = new Set<string>(); | |
| 44 | + for (const chunk of chunksBetween(htmlText, /<div class="auction-calendar-item[^"]*"/)) { | |
| 45 | + const au = chunk.match(/[?&](?:amp;)?au=(\d+)/)?.[1]; | |
| 46 | + if (!au || seen.has(au)) continue; | |
| 47 | + const href = chunk.match(/href=['"]([^'"]*[?&](?:amp;)?au=\d+[^'"]*)['"]/)?.[1] ?? null; | |
| 48 | + if (!href) continue; | |
| 49 | + seen.add(au); | |
| 50 | + const title = pick(chunk, /<H[3-6]>([\s\S]*?)<\/H[3-6]>/i) ?? pick(chunk, /alt="([^"]*)"/) ?? `Auction ${au}`; | |
| 51 | + // Sworders: <h5>Tuesday 21 April 2026</h5>; Chiswick (timed): "Starts: 24th Aug, 2026 17:00 … Ends: 4th Sep, 2026 12:00"; | |
| 52 | + // Dominic Winter (sessions): "Lot: 1 to 410 - 2nd Sep, 2026 10:00" → first session day. | |
| 53 | + const text = textOf(chunk.replace(/<a[^>]*>|<\/a>/g, ' ')); | |
| 54 | + const startsText = text.match(/Starts:\s*(\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]{3,9},?\s+\d{4})/i)?.[1] ?? null; | |
| 55 | + const endsText = text.match(/Ends:\s*(\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]{3,9},?\s+\d{4})/i)?.[1] ?? null; | |
| 56 | + const dateText = pick(chunk, /<h5>([\s\S]*?)<\/h5>/) ?? pick(chunk, /<p class="auction-calendar-date">([\s\S]*?)<\/p>/) ?? startsText ?? text.match(/((?:\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]+,?\s*-\s*)?\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]{3,9},?\s+\d{4})/)?.[1] ?? null; | |
| 57 | + const saleNo = pick(chunk, /Sale number:\s*([A-Z0-9]+)/i); | |
| 58 | + const timed = /timed online|Starts:/i.test(chunk); | |
| 59 | + const resultsPublished = /View Results|Download PDF results/i.test(chunk); | |
| 60 | + out.push({ id: au, title, url: absolute(house.base, href.replace(/&/g, '&')) ?? `${house.base}/auction/details/?au=${au}`, date: parseUkDate(dateText), location: null, extra: { sale_number: saleNo, timed, date_text: dateText, end_date: parseUkDate(endsText), results_published: resultsPublished } }); | |
| 61 | + } | |
| 62 | + return out; | |
| 63 | +} | |
| 64 | + | |
| 65 | +export function parseGoauctionLots(htmlText: string, sale: SaleRef, house: GoauctionHouse): ParsedSalePage | null { | |
| 66 | + if (!/class="auction-lot"|class='auction-grid-lot|class="auction-lot-title"/.test(htmlText)) return null; | |
| 67 | + const lots: ParsedLot[] = []; | |
| 68 | + for (const chunk of chunksBetween(htmlText, /<div class="auction-lot">/, /<nav class="pagination">|<footer|id="footer"/)) { | |
| 69 | + const href = chunk.match(/href="([^"]*\/auction\/lot\/[^"]+)"/i)?.[1] ?? null; | |
| 70 | + const titleBlock = chunk.match(/<span class='lot-title[^']*'>([\s\S]*?)<\/span>\s*<\/a>/)?.[1] ?? chunk.match(/<p class="auction-lot-title">([\s\S]*?)<\/p>/)?.[1] ?? null; | |
| 71 | + if (!titleBlock) continue; | |
| 72 | + const sub = pick(titleBlock, /<span class='sub-title[^']*'>([\s\S]*?)<\/span>/); | |
| 73 | + const main = textOf(titleBlock.replace(/<span class='sub-title[^']*'>[\s\S]*?<\/span>/, '')); | |
| 74 | + const lm = main.match(/^Lot\s+(\S+?)\s*(?:-|–|\s)\s*([\s\S]*)$/i); | |
| 75 | + const lotNo = lm?.[1]?.replace(/[,:]$/, '') ?? chunk.match(/alt="(?:Lot\s+)?(\d+[A-Za-z]?)\s*-/)?.[1] ?? null; | |
| 76 | + const title = (lm?.[2] ?? main).trim(); | |
| 77 | + if (!lotNo || !title) continue; | |
| 78 | + const soldText = pick(chunk, /<strong[^>]*>\s*(Sold for[^<]*)<\/strong>/i); | |
| 79 | + const price = soldText ? parseEuMoney(soldText.replace(/^Sold for\s*/i, ''), house.currency, 'en') : null; | |
| 80 | + const image = chunk.match(/<img (?:src|data-lazy)="([^"]+)"/)?.[1] ?? null; | |
| 81 | + const lotId = href?.match(/[?&](?:amp;)?lot=(\d+)/)?.[1] ?? null; | |
| 82 | + lots.push({ | |
| 83 | + lotNo, | |
| 84 | + title, | |
| 85 | + subtitle: sub || null, | |
| 86 | + description: null, | |
| 87 | + url: absolute(house.base, href?.replace(/&/g, '&')) ?? sale.url, | |
| 88 | + image: image ? image.replace(/&/g, '&') : null, | |
| 89 | + price: price?.amount ?? null, | |
| 90 | + currency: price?.currency ?? house.currency, | |
| 91 | + premiumIncluded: null, | |
| 92 | + estimateLow: null, | |
| 93 | + estimateHigh: null, | |
| 94 | + date: null, | |
| 95 | + sold: price !== null, | |
| 96 | + extra: { platform_lot_id: lotId, sold_text: soldText }, | |
| 97 | + }); | |
| 98 | + } | |
| 99 | + const next = /<link rel="next" href="[^"]+"/.test(htmlText) || /class="next"[^>]*href=|rel="next"/.test(htmlText); | |
| 100 | + const header = pick(htmlText, /<h1[^>]*>([\s\S]*?)<\/h1>/); | |
| 101 | + const dateText = pick(htmlText, /((?:\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]+\s*-\s*)?\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]{3,9}\s+\d{4})\s*\|/); | |
| 102 | + return { lots, hasMore: next, totalLots: null, sale: { title: header && header.length < 160 ? header : undefined, date: parseUkDate(dateText) ?? undefined } }; | |
| 103 | +} | |
| 104 | + | |
| 105 | +export function goauctionPageUrl(sale: SaleRef, page: number, house: GoauctionHouse): string { | |
| 106 | + const u = new URL(sale.url); | |
| 107 | + if (house.listStyle === 'search') { | |
| 108 | + const au = u.searchParams.get('au') ?? sale.id; | |
| 109 | + return `${house.base}/Auction/Search?au=${au}&sd=2${page > 1 ? `&pn=${page}` : ''}&g=1`; | |
| 110 | + } | |
| 111 | + u.searchParams.set('g', '1'); | |
| 112 | + if (page > 1) u.searchParams.set('pn', String(page)); | |
| 113 | + return u.toString(); | |
| 114 | +} | |
added
connectors/api/_g8-auctions-eu-apac-lib/index.ts
+317 −0
@@ -0,0 +1,317 @@ | ||
| 1 | +/** | |
| 2 | + * Shared helpers for the European / Nordic / APAC auction-house connectors (group g8-auctions-eu-apac). | |
| 3 | + * Multilingual date + money parsing (fr/de/it/es/nl/sv/da/fi/en), bundle detection in those languages, | |
| 4 | + * cautious year extraction for continental title conventions ("1900-tal", "20. Jh.", "1950er"). | |
| 5 | + * Kept inside connectors/api (not the framework). Pure functions, fixture-testable. | |
| 6 | + */ | |
| 7 | +import { parsePrice, SUPPORTED_CURRENCIES, type CurrencyCode } from '@rareindex/shared'; | |
| 8 | + | |
| 9 | +/** Month names / abbreviations → 0-based month. Covers en, fr, de, it, es, nl, sv, da, fi (+ common abbreviations). */ | |
| 10 | +const MONTHS: Record<string, number> = { | |
| 11 | + // en | |
| 12 | + jan: 0, january: 0, feb: 1, february: 1, mar: 2, march: 2, apr: 3, april: 3, may: 4, jun: 5, june: 5, jul: 6, july: 6, aug: 7, august: 7, sep: 8, sept: 8, september: 8, oct: 9, october: 9, nov: 10, november: 10, dec: 11, december: 11, | |
| 13 | + // fr | |
| 14 | + janvier: 0, janv: 0, février: 1, fevrier: 1, févr: 1, fevr: 1, mars: 2, avril: 3, avr: 3, mai: 4, juin: 5, juillet: 6, juil: 6, août: 7, aout: 7, septembre: 8, octobre: 9, novembre: 10, décembre: 11, decembre: 11, déc: 11, | |
| 15 | + // de | |
| 16 | + januar: 0, jänner: 0, februar: 1, märz: 2, maerz: 2, mrz: 2, juni: 5, juli: 6, oktober: 9, okt: 9, dezember: 11, dez: 11, | |
| 17 | + // it | |
| 18 | + gennaio: 0, gen: 0, febbraio: 1, marzo: 2, aprile: 3, maggio: 4, mag: 4, giugno: 5, giu: 5, luglio: 6, lug: 6, agosto: 7, ago: 7, settembre: 8, set: 8, ottobre: 9, ott: 9, dicembre: 11, dic: 11, | |
| 19 | + // es | |
| 20 | + enero: 0, ene: 0, febrero: 1, abril: 3, abr: 3, mayo: 4, junio: 5, julio: 6, septiembre: 8, setiembre: 8, octubre: 9, noviembre: 10, diciembre: 11, | |
| 21 | + // nl | |
| 22 | + januari: 0, februari: 1, maart: 2, mrt: 2, mei: 4, augustus: 7, // (juni/juli/september/oktober/november/december shared) | |
| 23 | + // sv / da / no | |
| 24 | + mars_sv: 2, // placeholder never matched; sv "mars" already mapped via fr | |
| 25 | + maj: 4, // sv/da | |
| 26 | + // fi | |
| 27 | + tammikuu: 0, tammikuuta: 0, helmikuu: 1, helmikuuta: 1, maaliskuu: 2, maaliskuuta: 2, huhtikuu: 3, huhtikuuta: 3, toukokuu: 4, toukokuuta: 4, kesäkuu: 5, kesäkuuta: 5, heinäkuu: 6, heinäkuuta: 6, elokuu: 7, elokuuta: 7, syyskuu: 8, syyskuuta: 8, lokakuu: 9, lokakuuta: 9, marraskuu: 10, marraskuuta: 10, joulukuu: 11, joulukuuta: 11, | |
| 28 | +}; | |
| 29 | + | |
| 30 | +function monthIndex(word: string): number | null { | |
| 31 | + const w = word.toLowerCase().replace(/\.$/, ''); | |
| 32 | + if (w in MONTHS && w !== 'mars_sv') return MONTHS[w]!; | |
| 33 | + // truncated forms ("sept", "déc", "okt") | |
| 34 | + const hit = Object.keys(MONTHS).find((k) => k.length >= 3 && w.length >= 3 && k.startsWith(w) && k !== 'mars_sv'); | |
| 35 | + return hit !== undefined ? MONTHS[hit]! : null; | |
| 36 | +} | |
| 37 | + | |
| 38 | +/** | |
| 39 | + * Parse a date written in any of the supported languages. Handles: | |
| 40 | + * "12 juin 2026" · "12. Juni 2026" · "12 giugno 2026" · "12 de junio de 2026" · "12 juni 2026" · "den 12 juni 2026" | |
| 41 | + * "June 12, 2026" · "12/06/2026" · "12.06.2026" · "2026-06-12" · "12 juin 2026 14:00" (time ignored → UTC midnight) | |
| 42 | + * "22–23 juin 2026" / "22-23. Juni 2026" (range → first day). | |
| 43 | + * Returns null instead of guessing; day-first for numeric forms. | |
| 44 | + */ | |
| 45 | +export function parseEuDate(raw: string | null | undefined): Date | null { | |
| 46 | + if (!raw) return null; | |
| 47 | + const s = raw.replace(/[ ]/g, ' ').replace(/\s+/g, ' ').trim(); | |
| 48 | + if (!s) return null; | |
| 49 | + const iso = s.match(/\b(\d{4})-(\d{2})-(\d{2})(?!\d)/); | |
| 50 | + if (iso) return utc(Number(iso[1]), Number(iso[2]) - 1, Number(iso[3])); | |
| 51 | + // day [–day] [de] month [de] year (fr/de/it/es/nl/sv/da/fi/en) | |
| 52 | + const dmy = s.match(/\b(\d{1,2})(?:\s?[-–/]\s?\d{1,2})?\.?(?:er|e|º|°|th|st|nd|rd)?\s+(?:de\s+|den\s+)?([A-Za-zÀ-ÿ]{3,12})\.?\s+(?:de\s+)?(\d{4})\b/); | |
| 53 | + if (dmy) { | |
| 54 | + const mo = monthIndex(dmy[2]!); | |
| 55 | + if (mo !== null) return utc(Number(dmy[3]), mo, Number(dmy[1])); | |
| 56 | + } | |
| 57 | + // month day, year (en) | |
| 58 | + const mdy = s.match(/\b([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})\b/); | |
| 59 | + if (mdy) { | |
| 60 | + const mo = monthIndex(mdy[1]!); | |
| 61 | + if (mo !== null) return utc(Number(mdy[3]), mo, Number(mdy[2])); | |
| 62 | + } | |
| 63 | + // numeric day-first: 12/06/2026, 12.06.2026, 12-06-2026 | |
| 64 | + const num = s.match(/\b(\d{1,2})[./-](\d{1,2})[./-](\d{4})\b/); | |
| 65 | + if (num) { | |
| 66 | + const d = Number(num[1]); | |
| 67 | + const m = Number(num[2]); | |
| 68 | + if (m >= 1 && m <= 12 && d >= 1 && d <= 31) return utc(Number(num[3]), m - 1, d); | |
| 69 | + } | |
| 70 | + // month year only ("juin 2026") → first of month (month precision) | |
| 71 | + const my = s.match(/\b([A-Za-zÀ-ÿ]{3,12})\.?\s+(\d{4})\b/); | |
| 72 | + if (my) { | |
| 73 | + const mo = monthIndex(my[1]!); | |
| 74 | + if (mo !== null) return utc(Number(my[2]), mo, 1); | |
| 75 | + } | |
| 76 | + return null; | |
| 77 | +} | |
| 78 | + | |
| 79 | +function utc(y: number, m: number, d: number): Date | null { | |
| 80 | + const dt = new Date(Date.UTC(y, m, d)); | |
| 81 | + return Number.isNaN(dt.getTime()) || dt.getUTCMonth() !== m ? null : dt; | |
| 82 | +} | |
| 83 | + | |
| 84 | +/** Unix seconds (Auctionet `ends_at`) → Date. */ | |
| 85 | +export function fromUnix(sec: number | null | undefined): Date | null { | |
| 86 | + if (!sec || !Number.isFinite(sec)) return null; | |
| 87 | + const d = new Date(sec * 1000); | |
| 88 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 89 | +} | |
| 90 | + | |
| 91 | +export type NumberLocale = 'eu' | 'en' | 'ch'; | |
| 92 | + | |
| 93 | +/** | |
| 94 | + * Parse a money string written with continental conventions. `locale` disambiguates the trailing-3-digit | |
| 95 | + * case: "1.250" → 1250 under 'eu' (dot = thousands) but 1.25 under 'en'; "1'250" (Swiss) → 1250. | |
| 96 | + * Currency: explicit symbol/code wins ("€", "CHF", "SEK", "kr", "£", "HK$", "¥"), else `fallback`. | |
| 97 | + * Returns null for no number / non-positive amounts. Never invents a currency. | |
| 98 | + */ | |
| 99 | +export function parseEuMoney(text: string | null | undefined, fallback: CurrencyCode | null, locale: NumberLocale = 'eu'): { amount: number; currency: CurrencyCode; confidence: number } | null { | |
| 100 | + if (!text) return null; | |
| 101 | + let s = text.replace(/[ ]/g, ' ').replace(/\s+/g, ' ').trim(); | |
| 102 | + if (!s) return null; | |
| 103 | + // Swiss apostrophe thousands and "Fr." prefix | |
| 104 | + if (locale === 'ch') s = s.replace(/(\d)'(\d{3})/g, '$1$2').replace(/\bFr\.\s?/, 'CHF '); | |
| 105 | + // Danish/Swedish/Norwegian "kr" ambiguity: keep as fallback unless explicit code present | |
| 106 | + const explicit = detectCurrency(s); | |
| 107 | + const currency = explicit ?? fallback; | |
| 108 | + if (!currency) return null; | |
| 109 | + // Normalise EU thousands groups written with spaces or dots when a comma decimal follows or when 3 trailing digits | |
| 110 | + let m = s.match(/-?\d[\d .,']*\d|\d/); | |
| 111 | + if (!m) return null; | |
| 112 | + let n = m[0].replace(/[ ']/g, ''); | |
| 113 | + let confidence = 0.95; | |
| 114 | + if (locale !== 'en') { | |
| 115 | + if (/,\d{1,2}$/.test(n)) n = n.replace(/\./g, '').replace(',', '.'); | |
| 116 | + else if (/^\d{1,3}(\.\d{3})+$/.test(n)) n = n.replace(/\./g, ''); | |
| 117 | + else if (/^\d{1,3}(,\d{3})+$/.test(n)) n = n.replace(/,/g, ''); // some EU sites still print en groups | |
| 118 | + else if (/\.\d{3}$/.test(n) && !/,/.test(n)) { | |
| 119 | + n = n.replace(/\./g, ''); | |
| 120 | + confidence = 0.85; | |
| 121 | + } else if (/,\d{3}$/.test(n)) { | |
| 122 | + n = n.replace(/,/g, ''); | |
| 123 | + confidence = 0.85; | |
| 124 | + } else n = n.replace(',', '.'); | |
| 125 | + } else { | |
| 126 | + const p = parsePrice(m[0], currency); | |
| 127 | + if (!p || p.amount <= 0) return null; | |
| 128 | + return { amount: p.amount, currency, confidence: p.confidence }; | |
| 129 | + } | |
| 130 | + const amount = Number.parseFloat(n); | |
| 131 | + if (!Number.isFinite(amount) || amount <= 0) return null; | |
| 132 | + return { amount, currency, confidence }; | |
| 133 | +} | |
| 134 | + | |
| 135 | +const CURRENCY_TOKENS: Array<[RegExp, CurrencyCode]> = [ | |
| 136 | + [/\bHK\s?\$|\bHKD\b/i, 'HKD'], | |
| 137 | + [/\bA\s?\$|\bAU\s?\$|\bAUD\b/i, 'AUD'], | |
| 138 | + [/\bNZ\s?\$|\bNZD\b/i, 'NZD'], | |
| 139 | + [/\bUS\s?\$|\bUSD\b/i, 'USD'], | |
| 140 | + [/\bCA\s?\$|\bCAD\b/i, 'CAD'], | |
| 141 | + [/\bS\s?\$|\bSGD\b/i, 'SGD'], | |
| 142 | + [/€|\bEUR\b|\beuros?\b/i, 'EUR'], | |
| 143 | + [/£|\bGBP\b/i, 'GBP'], | |
| 144 | + [/¥|￥|\bJPY\b|円/i, 'JPY'], | |
| 145 | + [/\bCHF\b|\bSFr\.?/i, 'CHF'], | |
| 146 | + [/\bSEK\b|\bskr\b/i, 'SEK'], | |
| 147 | + [/\bDKK\b|\bdkr\b/i, 'DKK'], | |
| 148 | + [/\bNOK\b|\bnkr\b/i, 'NOK'], | |
| 149 | + [/\bPLN\b|zł/i, 'PLN'], | |
| 150 | + [/\bCZK\b|Kč/i, 'CZK'], | |
| 151 | + [/\bCNY\b|\bRMB\b/i, 'CNY'], | |
| 152 | + [/\bTWD\b|\bNT\$/i, 'TWD'], | |
| 153 | +]; | |
| 154 | + | |
| 155 | +/** Explicit currency in a string (code or unambiguous symbol); null when absent or ambiguous ("kr", "$"). */ | |
| 156 | +export function detectCurrency(text: string): CurrencyCode | null { | |
| 157 | + for (const [re, code] of CURRENCY_TOKENS) if (re.test(text)) return code; | |
| 158 | + return null; | |
| 159 | +} | |
| 160 | + | |
| 161 | +export function isSupportedCurrency(code: string | null | undefined): code is CurrencyCode { | |
| 162 | + return !!code && (SUPPORTED_CURRENCIES as readonly string[]).includes(code); | |
| 163 | +} | |
| 164 | + | |
| 165 | +/** | |
| 166 | + * Bundle / multi-item detection across the group's languages. Pairs ("ett par", "Paar", "paire", "coppia", | |
| 167 | + * "a pair") are NOT bundles (a pair of candlesticks is one collectible unit); three or more distinct pieces, | |
| 168 | + * "collection/lot of", "Konvolut", "samling", "parti", "lotto di", "lote de", "partij" are. | |
| 169 | + */ | |
| 170 | +export function isBundleMultilingual(title: string): boolean { | |
| 171 | + const t = title; | |
| 172 | + if (/\b(collection of|group of|lot of|assorted|quantity of|mixed lot|various|job lot)\b/i.test(t)) return true; | |
| 173 | + if (/\b(konvolut|sammlung|posten|nachlass|\d+\s*-?\s*teilig|\d+\s*st(?:ück|k)\.?)\b/i.test(t)) return true; // de | |
| 174 | + if (/\b(samling|parti|diverse|blandat|\d+\s*st\b|\d+\s*delar|\d+\s*stk)\b/i.test(t)) return true; // sv/da/no | |
| 175 | + if (/\b(lot de|ensemble de|réunion de|reunion de|collection de|suite de|lot comprenant)\b/i.test(t)) return true; // fr | |
| 176 | + if (/\b(lotto di|gruppo di|insieme di|collezione di)\b/i.test(t)) return true; // it | |
| 177 | + if (/\b(lote de|conjunto de|colecci[oó]n de)\b/i.test(t)) return true; // es | |
| 178 | + if (/\b(partij|collectie van|set van \d+|diverse)\b/i.test(t)) return true; // nl | |
| 179 | + if (/\b(erä|kokoelma)\b/i.test(t)) return true; // fi | |
| 180 | + const count = parenCount(t); | |
| 181 | + return count !== null && count >= 3; | |
| 182 | +} | |
| 183 | + | |
| 184 | +/** Trailing "(4)" / "(12)" piece count used by Nordic/German houses; null when absent. */ | |
| 185 | +export function parenCount(title: string): number | null { | |
| 186 | + const m = title.match(/\((\d{1,3})\)\.?\s*$/) ?? title.match(/\b(\d{1,3})\s*(?:st|stk|pcs|pieces|pièces|pezzi|piezas|stuks|Stück|Teile)\b\.?/i); | |
| 187 | + if (!m) return null; | |
| 188 | + const n = Number(m[1]); | |
| 189 | + return n >= 2 && n <= 999 ? n : null; | |
| 190 | +} | |
| 191 | + | |
| 192 | +/** | |
| 193 | + * Year from a continental title, ignoring century/decade conventions: "1900-tal", "1900/2000-tal", "1950er", | |
| 194 | + * "1950s", "20. Jh.", "XIXe siècle", "circa 1900" (kept), "1800-talets". Returns null when nothing safe. | |
| 195 | + */ | |
| 196 | +export function yearFromTitle(title: string): number | null { | |
| 197 | + const now = new Date().getUTCFullYear(); | |
| 198 | + const re = /\b(1[5-9]\d{2}|20\d{2})\b(?![-–/]\s?\d{2,4}|-?\s?tal|er\b|s\b|-?\s?talet|\/)/g; | |
| 199 | + let m: RegExpExecArray | null; | |
| 200 | + while ((m = re.exec(title))) { | |
| 201 | + const y = Number(m[1]); | |
| 202 | + // skip when the token is preceded by "/" (second half of "1900/2000-tal") or followed by "-tal" | |
| 203 | + const before = title[m.index - 1] ?? ''; | |
| 204 | + if (before === '/' || before === '-' || before === '–') continue; | |
| 205 | + if (y >= 1500 && y <= now) return y; | |
| 206 | + } | |
| 207 | + return null; | |
| 208 | +} | |
| 209 | + | |
| 210 | +/** Strip tags, decode common entities, collapse whitespace; truncate to `max` chars. */ | |
| 211 | +export function stripHtml(s: string | null | undefined, max = 600): string | null { | |
| 212 | + if (!s) return null; | |
| 213 | + const t = s | |
| 214 | + .replace(/<br\s*\/?>/gi, ' ') | |
| 215 | + .replace(/<\/p>/gi, ' ') | |
| 216 | + .replace(/<[^>]+>/g, ' ') | |
| 217 | + .replace(/ /g, ' ') | |
| 218 | + .replace(/&/g, '&') | |
| 219 | + .replace(/</g, '<') | |
| 220 | + .replace(/>/g, '>') | |
| 221 | + .replace(/"/g, '"') | |
| 222 | + .replace(/'|'/g, "'") | |
| 223 | + .replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n))) | |
| 224 | + .replace(/\s+/g, ' ') | |
| 225 | + .trim(); | |
| 226 | + if (!t) return null; | |
| 227 | + return t.length > max ? `${t.slice(0, max - 1)}…` : t; | |
| 228 | +} | |
| 229 | + | |
| 230 | +/** Location string → ISO country when obvious (used for `location` display, never for currency). */ | |
| 231 | +export function countryFromLocation(loc: string | null | undefined): string | null { | |
| 232 | + if (!loc) return null; | |
| 233 | + const l = loc.toLowerCase(); | |
| 234 | + if (/stockholm|göteborg|goteborg|malmö|malmo|uppsala|helsingborg|sweden|sverige/.test(l)) return 'SE'; | |
| 235 | + if (/köln|cologne|hamburg|berlin|münchen|munich|frankfurt|düsseldorf|stuttgart|germany|deutschland/.test(l)) return 'DE'; | |
| 236 | + if (/københavn|copenhagen|aarhus|denmark|danmark/.test(l)) return 'DK'; | |
| 237 | + if (/helsinki|helsingfors|turku|finland/.test(l)) return 'FI'; | |
| 238 | + if (/madrid|barcelona|valencia|sevilla|spain|españa/.test(l)) return 'ES'; | |
| 239 | + if (/london|manchester|edinburgh|glasgow|united kingdom|england|scotland/.test(l)) return 'GB'; | |
| 240 | + if (/paris|lyon|marseille|bordeaux|france/.test(l)) return 'FR'; | |
| 241 | + if (/wien|vienna|austria|österreich/.test(l)) return 'AT'; | |
| 242 | + if (/zürich|zurich|genève|geneva|basel|switzerland|schweiz/.test(l)) return 'CH'; | |
| 243 | + if (/oslo|bergen|norway|norge/.test(l)) return 'NO'; | |
| 244 | + if (/amsterdam|rotterdam|den haag|netherlands|nederland/.test(l)) return 'NL'; | |
| 245 | + if (/bruxelles|brussel|antwerpen|belgium|belgique/.test(l)) return 'BE'; | |
| 246 | + if (/milano|roma|torino|genova|italy|italia/.test(l)) return 'IT'; | |
| 247 | + return null; | |
| 248 | +} | |
| 249 | + | |
| 250 | +/** | |
| 251 | + * Translate department / sale-title vocabulary from fr/de/it/es/nl/sv into the English keywords that | |
| 252 | + * `_auction-lib`'s `hintFromLabel` understands, so continental sale titles map to the same department hints. | |
| 253 | + */ | |
| 254 | +const LABEL_VOCAB: Array<[RegExp, string]> = [ | |
| 255 | + [/\b(montres?|horlogerie|uhren|armbanduhren|orologi|relojes|horloges|klockor|armbandsur)\b/i, 'watches'], | |
| 256 | + [/\b(bijoux|joaillerie|schmuck|juwelen|gioielli|joyas|sieraden|smycken|juweelen)\b/i, 'jewellery'], | |
| 257 | + [/\b(vins?|spiritueux|wein|weine|spirituosen|vini|vinos|wijn|whisky|champagne|viner)\b/i, 'wine'], | |
| 258 | + [/\b(monnaies|numismatique|münzen|numismatik|monete|monedas|munten|mynt|medailles|médailles)\b/i, 'coins'], | |
| 259 | + [/\b(timbres|philatélie|briefmarken|philatelie|francobolli|filatelia|sellos|postzegels|frimärken)\b/i, 'stamps'], | |
| 260 | + [/\b(automobiles?|voitures|motorcars|automobilia|autos?|automobili|coches|auto's|bilar)\b/i, 'motor cars'], | |
| 261 | + [/\b(motos?|motocyclettes|motorräder|motociclette|motorfietsen|motorcyklar)\b/i, 'motorcycles'], | |
| 262 | + [/\b(bandes? dessinées?|bd\b|comics|fumetti|cómics|strips|serier)\b/i, 'comics'], | |
| 263 | + [/\b(livres?|manuscrits|bücher|bibliothek|libri|libros|boeken|böcker|autographes|autographen|cartes anciennes|landkarten)\b/i, 'books'], | |
| 264 | + [/\b(photographies?|photographie|fotografie|fotografia|fotografía|foto's|fotografi)\b/i, 'photographs'], | |
| 265 | + [/\b(jouets|spielzeug|giocattoli|juguetes|speelgoed|leksaker|poupées|puppen|trains|modellbahn)\b/i, 'toys'], | |
| 266 | + [/\b(militaria|décorations|orden|ehrenzeichen|armes|waffen|armi|armas|wapens|vapen)\b/i, 'militaria'], | |
| 267 | + [/\b(design|arts? décoratifs? du xxe|kunstgewerbe|arredi di design|diseño|vormgeving)\b/i, 'design'], | |
| 268 | + [/\b(mobilier|meubles|möbel|arredi|mobili|muebles|meubelen|möbler|objets d'art|kunstgewerbe|antiquités|antiquitäten|antichità|antigüedades|antiek|antikviteter|tapis|teppiche|tappeti|alfombras)\b/i, 'furniture'], | |
| 269 | + [/\b(argenterie|orfèvrerie|silber|argenti|plata|zilver|silver)\b/i, 'silver'], | |
| 270 | + [/\b(céramiques?|porcelaine|porzellan|keramik|ceramiche|porcellane|cerámica|porcelana|porselein|keramiek|porslin|keramik)\b/i, 'ceramics'], | |
| 271 | + [/\b(verre|verrerie|glas|vetri|vidrio|glaswerk)\b/i, 'glass'], | |
| 272 | + [/\b(art contemporain|art moderne|après-guerre|post-war|moderne kunst|zeitgenössische kunst|arte moderna|arte contemporanea|arte contemporáneo|moderne en hedendaagse kunst|modern konst|samtida konst|street art|urban art|estampes|graphik|druckgraphik|editions)\b/i, 'contemporary'], | |
| 273 | + [/\b(tableaux|peintures|dessins|gemälde|zeichnungen|alte meister|maîtres anciens|dipinti|disegni|pinturas|dibujos|schilderijen|tekeningen|målningar|sculptures|skulpturen|sculture|esculturas|beelden|konst|kunst|arte|art)\b/i, 'fine art'], | |
| 274 | + [/\b(art d'asie|arts d'asie|asiatique|asiatica|asiatische kunst|chine|chinois|japon|japonais|arte asiatica|arte asiático|aziatische kunst|asiatisk|china|japan|japanese|chinese)\b/i, 'asian'], | |
| 275 | + [/\b(arts premiers|art tribal|art africain|art océanien|arts d'afrique|stammeskunst|arte tribale|arte africano|tribale kunst|afrikansk)\b/i, 'tribal'], | |
| 276 | + [/\b(archéologie|antiquités classiques|antike|archeologia|arqueología|archeologie|antikviteter klassiska)\b/i, 'antiquities'], | |
| 277 | + [/\b(mode|haute couture|maroquinerie|sacs|hermès|vuitton|chanel|taschen|borse|bolsos|tassen|väskor|vintage fashion)\b/i, 'handbags'], | |
| 278 | + [/\b(instruments? scientifiques?|wissenschaftliche instrumente|strumenti scientifici|marine|nautica|technica)\b/i, 'scientific instruments'], | |
| 279 | + [/\b(cinéma|affiches|plakate|manifesti|carteles|affiches|filmplakat)\b/i, 'movie posters'], | |
| 280 | + [/\b(musique|musik|musica|música|muziek|instruments de musique|musikinstrumente|strumenti musicali|vinyles|schallplatten)\b/i, 'music'], | |
| 281 | + [/\b(sport|sports|sportmemorabilia)\b/i, 'sports'], | |
| 282 | + [/\b(stylos|schreibgeräte|penne|plumas|pennen|pennor)\b/i, 'pens'], | |
| 283 | + [/\b(parfums?|parfüm|profumi|perfumes)\b/i, 'perfume'], | |
| 284 | + [/\b(appareils photo|kameras|fotocamere|cámaras|camera's|kameror)\b/i, 'cameras'], | |
| 285 | + [/\b(jeux vidéo|videospiele|videogiochi|videojuegos|videogames|tv-spel)\b/i, 'video games'], | |
| 286 | + [/\b(pendules|horloges|uhren und pendulen|orologi da tavolo|relojes de pared)\b/i, 'clocks'], | |
| 287 | +]; | |
| 288 | + | |
| 289 | +/** Sale/department label (any supported language) → English keyword string for `hintFromLabel`. */ | |
| 290 | +export function englishLabel(label: string | null | undefined): string { | |
| 291 | + if (!label) return ''; | |
| 292 | + const hits: string[] = []; | |
| 293 | + for (const [re, en] of LABEL_VOCAB) if (re.test(label)) hits.push(en); | |
| 294 | + return hits.length ? hits.join(' ') : label; | |
| 295 | +} | |
| 296 | + | |
| 297 | +/** | |
| 298 | + * Strong per-lot department cues in the group's languages (a Rolex filed under "Moderne Kunst" is still a | |
| 299 | + * watch). Returns a department hint understood by `_auction-lib` or null when the lot text has no strong cue. | |
| 300 | + */ | |
| 301 | +export function strongLotHint(text: string): 'watches' | 'jewelry' | 'wine' | 'coins' | 'stamps' | 'handbags' | 'cars' | 'motorcycles' | 'cameras' | 'comics' | 'books' | 'asian' | 'antiquities' | null { | |
| 302 | + const t = text; | |
| 303 | + if (/\b(armbanduhr|armbanduhren|taschenuhr|montre|montres|orologio|orologi|reloj|relojes|horloge|polshorloge|armbandsur|fickur|wristwatch|pocket watch|chronograph|chronographe|cronografo|rolex|patek philippe|audemars piguet|omega|breitling|iwc|jaeger[- ]lecoultre|vacheron|cartier tank|tudor|panerai|hublot|zenith|longines|breguet|blancpain)\b/i.test(t) && !/\b(poster|affiche|plakat|book|livre|buch|catalogue|katalog)\b/i.test(t)) return 'watches'; | |
| 304 | + if (/\b(bague|collier|bracelet|broche|boucles d'oreilles|ring|halskette|armband|brosche|ohrringe|anello|collana|bracciale|spilla|orecchini|anillo|collar|pulsera|pendientes|diamant|diamanten|diamante|brillant|saphir|rubis|émeraude|smaragd|rubin|zaffiro|rubino|smeraldo|diamond|sapphire|emerald|ruby|carats?|\d+(?:[.,]\d+)?\s*ct\b|earrings?|necklace|brooch|pendant|tiara|bangle|cufflinks|halsband|örhängen|armbandsur)\b/i.test(t) && !/\barmbandsur\b/i.test(t)) return 'jewelry'; | |
| 305 | + if (/\b(whisky|whiskey|bourbon|cognac|armagnac|champagne|bordeaux|bourgogne|burgundy|château|chateau|domaine|magnum|bouteilles?|flaschen?|bottiglie?|botellas?|flaskor|wein|vino|vin\b|riesling|barolo|brunello|romanée|pétrus|petrus|lafite|latour|margaux|mouton|yquem|krug|dom pérignon|macallan|yamazaki|hibiki|springbank|bowmore|ardbeg)\b/i.test(t)) return 'wine'; | |
| 306 | + if (/\b(münze|münzen|mynt|monnaie|monnaies|moneta|monete|moneda|monedas|munt|munten|coin|coins|dukat|ducat|taler|thaler|riksdaler|sovereign|20 francs or|louis d'or|napoléon|napoleon 20|goldmünze|silbermünze|banknote|geldschein|billet de banque|sedel|sedlar|banconota|billete)\b/i.test(t)) return 'coins'; | |
| 307 | + if (/\b(briefmarke|briefmarken|timbre|timbres|francobollo|francobolli|sello|sellos|postzegel|postzegels|frimärke|frimärken|stamp|stamps|philatel\w*|postal history)\b/i.test(t)) return 'stamps'; | |
| 308 | + if (/\b(hermès|hermes birkin|birkin|kelly bag|sac kelly|chanel (?:timeless|classic|flap|2\.55)|louis vuitton|vuitton|goyard|handtasche|sac à main|borsa|bolso|handbag|tote bag)\b/i.test(t)) return 'handbags'; | |
| 309 | + if (/\b(motorrad|motocyclette|motocicletta|motocicleta|motorfiets|motorcykel|motorcycle|ducati|harley[- ]davidson|vespa|moto guzzi|bsa\b|norton|triumph bonneville)\b/i.test(t)) return 'motorcycles'; | |
| 310 | + if (/\b(chassis|châssis|fahrgestell|telaio|bastidor|numéro de série|vin\b|coupé|cabriolet|roadster|berline|limousine|spider|spyder|berlinetta)\b/i.test(t) && /\b(ferrari|porsche|mercedes|bmw|jaguar|aston martin|bentley|rolls[- ]royce|alfa romeo|lancia|maserati|bugatti|citroën|citroen|peugeot|renault|volvo|saab|ford|chevrolet|cadillac|lamborghini|mclaren|austin|mg\b|triumph|lotus|fiat|volkswagen|vw\b)\b/i.test(t)) return 'cars'; | |
| 311 | + if (/\b(leica|hasselblad|rolleiflex|nikon f\d?|contax|kamera|appareil photo|fotocamera|cámara|objektiv|objectif)\b/i.test(t)) return 'cameras'; | |
| 312 | + if (/\b(bande dessinée|bandes dessinées|planche originale|comic|comics|fumetto|fumetti|serietidning|tintin|astérix|asterix|hergé|uderzo|franquin|manga)\b/i.test(t)) return 'comics'; | |
| 313 | + if (/\b(dynasty|dynastie|kangxi|qianlong|yongzheng|wanli|ming|qing|song|tang|han\b|edo|meiji|taisho|satsuma|imari|kutani|celadon|cloisonné|netsuke|okimono|inro|tsuba|thangka|khmer|gandhara|famille rose|famille verte|blanc de chine|kakiemon|arita|chinesisch|chinoise|chinese|japanisch|japonais|japanese|tibetan|tibétain)\b/i.test(t)) return 'asian'; | |
| 314 | + if (/\b(roman|romaine|römisch|romano|greek|grec|griechisch|greco|etruscan|étrusque|egyptian|égyptien|ägyptisch|egizio|mesopotamian|sumerian|bactrian|hellenistic|hellénistique|\d+(?:st|nd|rd|th)? century (?:bc|b\.c\.)|av\. ?j\.-c\.|v\. ?chr\.|a\.c\.|\bbc\b)\b/i.test(t)) return 'antiquities'; | |
| 315 | + if (/\b(édition originale|first edition|erstausgabe|prima edizione|primera edición|incunable|incunabula|manuscrit|manuscript|handschrift|folio|in-4|in-8|in-12|reliure|einband|legatura|exemplaire numéroté)\b/i.test(t)) return 'books'; | |
| 316 | + return null; | |
| 317 | +} | |
added
connectors/api/_g8-auctions-eu-apac-lib/sale-results.ts
+383 −0
@@ -0,0 +1,383 @@ | ||
| 1 | +/** | |
| 2 | + * Generic "past sales → lot results" connector skeleton shared by the g8 auction-house connectors. | |
| 3 | + * A house exposes (1) an index of past sales and (2) per-sale lot pages (HTML or embedded JSON) that list | |
| 4 | + * lot number, title, realised price and estimate. Subclasses implement the three parsers; crawl | |
| 5 | + * (resumable, backfill-aware) and normalize (sale / auction_lot, native currency, premium basis labelled) | |
| 6 | + * are shared so every house behaves identically. | |
| 7 | + */ | |
| 8 | +import { z } from 'zod'; | |
| 9 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 10 | +import { parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 11 | +import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, type CurrencyCode, type ExtractionResult, type NormalizedRecord } from '@rareindex/shared'; | |
| 12 | +import { brandFromSlug, hintFromLabel, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js'; | |
| 13 | +import { englishLabel, isBundleMultilingual, strongLotHint, yearFromTitle } from './index.js'; | |
| 14 | + | |
| 15 | +export const SaleRefSchema = z.object({ | |
| 16 | + id: z.string(), | |
| 17 | + title: z.string(), | |
| 18 | + url: z.string(), | |
| 19 | + /** ISO date (source's own sale date), null when the index does not show it */ | |
| 20 | + date: z.string().nullable(), | |
| 21 | + location: z.string().nullable(), | |
| 22 | + extra: z.record(z.string(), z.unknown()).default({}), | |
| 23 | +}); | |
| 24 | +export type SaleRef = z.infer<typeof SaleRefSchema>; | |
| 25 | + | |
| 26 | +export const ParsedLotSchema = z.object({ | |
| 27 | + lotNo: z.string(), | |
| 28 | + title: z.string(), | |
| 29 | + subtitle: z.string().nullable().default(null), | |
| 30 | + description: z.string().nullable().default(null), | |
| 31 | + url: z.string(), | |
| 32 | + image: z.string().nullable().default(null), | |
| 33 | + /** realised price as published (hammer or premium-inclusive per `premiumIncluded`) */ | |
| 34 | + price: z.number().nullable().default(null), | |
| 35 | + currency: z.string().nullable().default(null), | |
| 36 | + premiumIncluded: z.boolean().nullable().default(null), | |
| 37 | + estimateLow: z.number().nullable().default(null), | |
| 38 | + estimateHigh: z.number().nullable().default(null), | |
| 39 | + /** sale date override (timed sales where each lot closes on its own day) */ | |
| 40 | + date: z.string().nullable().default(null), | |
| 41 | + sold: z.boolean(), | |
| 42 | + extra: z.record(z.string(), z.unknown()).default({}), | |
| 43 | +}); | |
| 44 | +export type ParsedLot = z.infer<typeof ParsedLotSchema>; | |
| 45 | + | |
| 46 | +export const SaleResultsPayloadSchema = z.object({ | |
| 47 | + kind: z.literal('sale_results'), | |
| 48 | + url: z.string(), | |
| 49 | + sale: SaleRefSchema, | |
| 50 | + page: z.number(), | |
| 51 | + totalLots: z.number().nullable(), | |
| 52 | + lots: z.array(ParsedLotSchema), | |
| 53 | +}); | |
| 54 | +export type SaleResultsPayload = z.infer<typeof SaleResultsPayloadSchema>; | |
| 55 | + | |
| 56 | +export interface ParsedSalePage { | |
| 57 | + lots: ParsedLot[]; | |
| 58 | + hasMore: boolean; | |
| 59 | + totalLots: number | null; | |
| 60 | + /** sale-level facts discovered on the lot page (date, location, premium basis) */ | |
| 61 | + sale?: Partial<Pick<SaleRef, 'date' | 'location' | 'title'>> & { extra?: Record<string, unknown> }; | |
| 62 | +} | |
| 63 | + | |
| 64 | +export interface HouseConfig { | |
| 65 | + houseName: string; | |
| 66 | + defaultCurrency: CurrencyCode; | |
| 67 | + /** default `location` for sale records when the source does not give one */ | |
| 68 | + location: string | null; | |
| 69 | + /** identifiers key, e.g. "aguttes_lot" → "<saleId>/<lotNo>" */ | |
| 70 | + idKey: string; | |
| 71 | + /** default buyer-premium basis when the page does not label it (null = unknown) */ | |
| 72 | + premiumIncluded: boolean | null; | |
| 73 | + /** engines for page fetches (default ['api']) */ | |
| 74 | + engines?: Array<'api' | 'firecrawl' | 'scrapfly'>; | |
| 75 | + responseType?: 'text' | 'json'; | |
| 76 | + /** taxonomy slug when nothing matches (null = drop the lot) */ | |
| 77 | + fallbackSlug: string | null; | |
| 78 | + /** politeness between requests (ms) */ | |
| 79 | + minIntervalMs?: number; | |
| 80 | + /** cap of sale pages fetched per incremental run */ | |
| 81 | + maxPagesPerSale?: number; | |
| 82 | + /** drop lots whose currency could not be read from the source (multi-currency houses) instead of defaulting */ | |
| 83 | + requireCurrency?: boolean; | |
| 84 | +} | |
| 85 | + | |
| 86 | +type Cursor = { done?: string[]; pending?: Record<string, string>; backfill?: { index: number; page: number; itemsProcessed: number }; finished?: boolean }; | |
| 87 | + | |
| 88 | +/** Keep the pending map bounded (most recent 200 entries). */ | |
| 89 | +function trimPending(p: Record<string, string>): Record<string, string> { | |
| 90 | + const entries = Object.entries(p).sort((a, b) => b[1].localeCompare(a[1])).slice(0, 200); | |
| 91 | + return Object.fromEntries(entries); | |
| 92 | +} | |
| 93 | + | |
| 94 | +export abstract class SaleResultsConnector extends BaseConnector { | |
| 95 | + readonly parserVersion = '1.0.0'; | |
| 96 | + abstract readonly house: HouseConfig; | |
| 97 | + | |
| 98 | + /** Fetch + parse the index of past sales (most recent first is not required; we sort by date). */ | |
| 99 | + abstract listSales(ctx: CrawlContext): Promise<SaleRef[]>; | |
| 100 | + /** URL of page `page` (1-based) of a sale's lot list. */ | |
| 101 | + abstract salePageUrl(sale: SaleRef, page: number): string; | |
| 102 | + /** Parse one lot page (HTML text or JSON). Return null when the document is not a lot page. */ | |
| 103 | + abstract parseSalePage(res: ExtractionResult, sale: SaleRef, page: number): ParsedSalePage | null; | |
| 104 | + /** | |
| 105 | + * Taxonomy slug for a lot: strong multilingual lot cues (a Rolex in a "Moderne Kunst" sale is a watch) win, | |
| 106 | + * then the department hint derived from the sale title / department (any supported language), then a keyword | |
| 107 | + * sweep over title + description, then the house fallback (null = drop). | |
| 108 | + */ | |
| 109 | + categoryFor(sale: SaleRef, lot: ParsedLot): string | null { | |
| 110 | + return resolveCategory(`${sale.title} ${String(sale.extra.department ?? '')} ${String(sale.extra.title_fr ?? '')}`, lot, this.house.fallbackSlug); | |
| 111 | + } | |
| 112 | + | |
| 113 | + protected get salesPerRun(): number { | |
| 114 | + return Number(this.meta.config.salesPerRun ?? 3); | |
| 115 | + } | |
| 116 | + | |
| 117 | + protected async fetchSalePage(ctx: CrawlContext, sale: SaleRef, page: number): Promise<{ url: string; res: ExtractionResult; parsed: ParsedSalePage | null }> { | |
| 118 | + const url = this.salePageUrl(sale, page); | |
| 119 | + await this.throttle(url); | |
| 120 | + const res = await ctx.fetch(url, { | |
| 121 | + engines: this.house.engines ?? ['api'], | |
| 122 | + responseType: this.house.responseType ?? 'text', | |
| 123 | + timeoutMs: 60_000, | |
| 124 | + expect: ['title', 'price'], | |
| 125 | + minQuality: 0.3, | |
| 126 | + parse: (r) => { | |
| 127 | + const p = this.parseSalePage(r, sale, page); | |
| 128 | + const sold = p?.lots.find((l) => l.price); | |
| 129 | + return p?.lots.length ? { title: p.lots[0]!.title, price: sold?.price ?? null, currency: sold?.currency ?? null } : null; | |
| 130 | + }, | |
| 131 | + }); | |
| 132 | + const parsed = res.success ? this.parseSalePage(res, sale, page) : null; | |
| 133 | + return { url, res, parsed }; | |
| 134 | + } | |
| 135 | + | |
| 136 | + /** Yield every page of one sale (stops at hasMore=false, empty page or the per-sale cap). */ | |
| 137 | + protected async *crawlSale(ctx: CrawlContext, sale: SaleRef, startPage = 1, onPage?: (page: number, lots: number) => Promise<void>): AsyncIterable<RawRecordInput> { | |
| 138 | + const cap = this.house.maxPagesPerSale ?? 40; | |
| 139 | + let page = startPage; | |
| 140 | + for (; page < startPage + cap; page++) { | |
| 141 | + if (ctx.signal?.aborted) return; | |
| 142 | + const { url, res, parsed } = await this.fetchSalePage(ctx, sale, page); | |
| 143 | + if (!parsed) { | |
| 144 | + ctx.anomaly(page === 1 ? 'sale_parse_failed' : 'pagination_failure', `${sale.id} p${page}: ${res.error ?? res.httpStatus}`); | |
| 145 | + return; | |
| 146 | + } | |
| 147 | + if (parsed.sale) Object.assign(sale, { date: parsed.sale.date ?? sale.date, location: parsed.sale.location ?? sale.location, title: parsed.sale.title ?? sale.title, extra: { ...sale.extra, ...(parsed.sale.extra ?? {}) } }); | |
| 148 | + if (parsed.lots.length === 0) { | |
| 149 | + if (page === 1) ctx.anomaly('selector_missing', `${sale.id}: no lots parsed`); | |
| 150 | + return; | |
| 151 | + } | |
| 152 | + const payload: SaleResultsPayload = { kind: 'sale_results', url, sale: { ...sale }, page, totalLots: parsed.totalLots, lots: parsed.lots }; | |
| 153 | + yield { url, externalId: `${sale.id}:p${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 154 | + if (onPage) await onPage(page, parsed.lots.length); | |
| 155 | + if (!parsed.hasMore) return; | |
| 156 | + } | |
| 157 | + ctx.anomaly('pagination_failure', `${sale.id}: page cap ${cap} reached`); | |
| 158 | + } | |
| 159 | + | |
| 160 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 161 | + const cursor = (ctx.options.cursor ?? {}) as Cursor; | |
| 162 | + // keep sales that have started (timed sales expose their end date in extra.end_date → wait for it) | |
| 163 | + const sales = (await this.listSales(ctx)).filter((s) => { | |
| 164 | + const when = typeof s.extra.end_date === 'string' ? s.extra.end_date : s.date; | |
| 165 | + return !when || new Date(when).getTime() <= Date.now() + 86_400_000; | |
| 166 | + }); | |
| 167 | + if (sales.length === 0) { | |
| 168 | + ctx.anomaly('past_list_failed', 'no past sales found on the index'); | |
| 169 | + return; | |
| 170 | + } | |
| 171 | + if (ctx.options.mode === 'backfill') { | |
| 172 | + yield* this.backfill(ctx, sales, cursor); | |
| 173 | + return; | |
| 174 | + } | |
| 175 | + sales.sort((a, b) => (b.date ?? '').localeCompare(a.date ?? '')); | |
| 176 | + const done = new Set<string>(cursor.done ?? []); | |
| 177 | + // sales seen without any realised price yet (still running / results not published) → re-check after a few days | |
| 178 | + const pending: Record<string, string> = { ...(cursor.pending ?? {}) }; | |
| 179 | + const recheckMs = Number(this.meta.config.pendingRecheckDays ?? 3) * 86_400_000; | |
| 180 | + const maxUnfinished = Number(this.meta.config.maxUnfinishedPerRun ?? Math.max(this.salesPerRun * 3, 12)); | |
| 181 | + let processed = 0; | |
| 182 | + let unfinished = 0; | |
| 183 | + let count = 0; | |
| 184 | + for (const sale of sales) { | |
| 185 | + if (ctx.signal?.aborted || this.reached(ctx, count) || processed >= this.salesPerRun || unfinished >= maxUnfinished) break; | |
| 186 | + if (done.has(sale.id)) continue; | |
| 187 | + const seenAt = pending[sale.id] ? new Date(pending[sale.id]!).getTime() : 0; | |
| 188 | + if (seenAt && Date.now() - seenAt < recheckMs && ctx.options.mode !== 'probe') continue; | |
| 189 | + let pages = 0; | |
| 190 | + let complete = true; | |
| 191 | + for await (const raw of this.crawlSale(ctx, sale)) { | |
| 192 | + pages++; | |
| 193 | + count++; | |
| 194 | + yield raw; | |
| 195 | + // A first page without a single realised price = the sale is still running or results are not published yet: | |
| 196 | + // keep its lots as auction_lot records but do not paginate further and do not mark the sale done. | |
| 197 | + if (pages === 1 && !(raw.payload as SaleResultsPayload).lots.some((l) => l.sold)) { | |
| 198 | + complete = false; | |
| 199 | + unfinished++; | |
| 200 | + break; | |
| 201 | + } | |
| 202 | + if (this.reached(ctx, count)) break; | |
| 203 | + } | |
| 204 | + if (!complete) { | |
| 205 | + pending[sale.id] = new Date().toISOString(); | |
| 206 | + await ctx.setCursor({ done: [...done].slice(-500), pending: trimPending(pending) }); | |
| 207 | + continue; | |
| 208 | + } | |
| 209 | + processed++; | |
| 210 | + if (pages > 0 && !this.reached(ctx, count)) { | |
| 211 | + done.add(sale.id); | |
| 212 | + delete pending[sale.id]; | |
| 213 | + await ctx.setCursor({ done: [...done].slice(-500), pending: trimPending(pending) }); | |
| 214 | + } | |
| 215 | + } | |
| 216 | + } | |
| 217 | + | |
| 218 | + /** Backfill: every past sale, oldest first, resumable at (sale index, page); ends with {finished:true}. */ | |
| 219 | + protected async *backfill(ctx: CrawlContext, sales: SaleRef[], cursor: Cursor): AsyncIterable<RawRecordInput> { | |
| 220 | + if (cursor.finished) return; | |
| 221 | + sales.sort((a, b) => (a.date ?? '').localeCompare(b.date ?? '') || a.id.localeCompare(b.id)); | |
| 222 | + let index = cursor.backfill?.index ?? 0; | |
| 223 | + let itemsProcessed = cursor.backfill?.itemsProcessed ?? 0; | |
| 224 | + let startPage = cursor.backfill?.page ?? 1; | |
| 225 | + let fetched = 0; | |
| 226 | + const maxPages = this.policy.backfillMaxPages; | |
| 227 | + for (; index < sales.length; index++, startPage = 1) { | |
| 228 | + const sale = sales[index]!; | |
| 229 | + let stop = false; | |
| 230 | + for await (const raw of this.crawlSale(ctx, sale, startPage, async (page, lots) => { | |
| 231 | + itemsProcessed += lots; | |
| 232 | + fetched++; | |
| 233 | + await ctx.setCursor({ backfill: { index, page: page + 1, itemsProcessed } }); | |
| 234 | + await ctx.progress({ page: index + 1, totalPages: sales.length, itemsProcessed, reachedDate: sale.date ? new Date(sale.date) : null, cursor: { index, page: page + 1 } }); | |
| 235 | + if (fetched >= maxPages || ctx.signal?.aborted) stop = true; | |
| 236 | + })) { | |
| 237 | + yield raw; | |
| 238 | + if (stop) return; | |
| 239 | + } | |
| 240 | + await ctx.setCursor({ backfill: { index: index + 1, page: 1, itemsProcessed } }); | |
| 241 | + } | |
| 242 | + await ctx.setCursor({ finished: true, backfill: { index: sales.length, page: 1, itemsProcessed } }); | |
| 243 | + await ctx.progress({ page: sales.length, totalPages: sales.length, itemsProcessed, cursor: { finished: true } }); | |
| 244 | + } | |
| 245 | + | |
| 246 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 247 | + const p = SaleResultsPayloadSchema.parse(raw.payload); | |
| 248 | + const out: NormalizedRecord[] = []; | |
| 249 | + const saleDate = p.sale.date ? new Date(p.sale.date) : null; | |
| 250 | + for (const lot of p.lots) { | |
| 251 | + if (this.house.requireCurrency && !lot.currency) continue; | |
| 252 | + const slug = this.categoryFor(p.sale, lot); | |
| 253 | + if (!slug) continue; | |
| 254 | + const text = `${lot.title} ${lot.subtitle ?? ''}`.trim(); | |
| 255 | + const g = parseGradeFromTitle(text); | |
| 256 | + const isWatch = ['rolex', 'omega', 'patek_philippe', 'audemars_piguet', 'other_watches'].includes(slug); | |
| 257 | + const attributes = AssetAttributesSchema.parse({ | |
| 258 | + categorySlug: slug, | |
| 259 | + name: lot.title, | |
| 260 | + model: lot.subtitle, | |
| 261 | + brand: brandFromSlug(slug, text), | |
| 262 | + // "réf. 5513" / "Ref 16233" — normalise the French/German abbreviation before the English reference parser | |
| 263 | + reference: isWatch ? watchReference(text) ?? watchReference(text.replace(/\br[ée]f(?:[ée]rence|erenz)?\.?\s*/gi, 'Ref. ')) : null, | |
| 264 | + year: yearFromTitle(text), | |
| 265 | + identifiers: { [this.house.idKey]: `${p.sale.id}/${lot.lotNo}` }, | |
| 266 | + metadata: { sale_id: p.sale.id, sale_title: p.sale.title, sale_url: p.sale.url, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, ...p.sale.extra, ...lot.extra }, | |
| 267 | + }); | |
| 268 | + const base = { | |
| 269 | + connectorId: this.meta.id, | |
| 270 | + sourceId: this.meta.sourceId, | |
| 271 | + sourceUrl: lot.url, | |
| 272 | + externalId: `${p.sale.id}:${lot.lotNo}`, | |
| 273 | + rawTitle: lot.subtitle ? `${lot.title} — ${lot.subtitle}` : lot.title, | |
| 274 | + description: lot.description, | |
| 275 | + imageUrls: lot.image ? [lot.image] : [], | |
| 276 | + attributes, | |
| 277 | + grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, qualifier: null, certificationNumber: null }, | |
| 278 | + condition: { condition: null, conditionRaw: null, completeness: null }, | |
| 279 | + observedAt: raw.fetchedAt, | |
| 280 | + confidence: this.confidenceFor(slug, lot), | |
| 281 | + parserVersion: this.parserVersion, | |
| 282 | + }; | |
| 283 | + const currency = (lot.currency ?? this.house.defaultCurrency) as CurrencyCode; | |
| 284 | + const lotDate = lot.date ? new Date(lot.date) : saleDate; | |
| 285 | + const location = p.sale.location ?? this.house.location; | |
| 286 | + if (lot.sold && lot.price && lotDate && !Number.isNaN(lotDate.getTime())) { | |
| 287 | + out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate: lotDate, price: lot.price, currency, buyerPremiumIncluded: lot.premiumIncluded ?? this.house.premiumIncluded, quantity: 1, isBundle: isBundleMultilingual(text), location, auctionHouse: this.house.houseName, lotNumber: lot.lotNo })); | |
| 288 | + } else { | |
| 289 | + const status = lotDate && lotDate.getTime() < raw.fetchedAt.getTime() ? 'ended' : lotDate ? 'upcoming' : 'unknown'; | |
| 290 | + out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: this.house.houseName, auctionName: p.sale.title, lotNumber: lot.lotNo, startsAt: lotDate, endsAt: lotDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currentBid: lot.sold ? lot.price : null, currency, status, location })); | |
| 291 | + } | |
| 292 | + } | |
| 293 | + return out; | |
| 294 | + } | |
| 295 | + | |
| 296 | + protected confidenceFor(_slug: string, _lot: ParsedLot): number { | |
| 297 | + return 0.85; | |
| 298 | + } | |
| 299 | +} | |
| 300 | + | |
| 301 | +/** Shared category resolution (see `SaleResultsConnector.categoryFor`). Exported for connectors with extra rules. */ | |
| 302 | +export function resolveCategory(saleLabel: string, lot: ParsedLot, fallbackSlug: string | null): string | null { | |
| 303 | + const text = `${lot.title} ${lot.subtitle ?? ''}`.trim(); | |
| 304 | + const full = `${text} ${lot.description ?? ''}`.slice(0, 500); | |
| 305 | + const strong = strongLotHint(full); | |
| 306 | + const saleHint: DeptHint = hintFromLabel(englishLabel(saleLabel)); | |
| 307 | + const hint: DeptHint = strong ?? saleHint; | |
| 308 | + if (hint === 'wine') { | |
| 309 | + if (/whisky|whiskey|威士忌|ウイスキー|bourbon|macallan|yamazaki|山崎|hibiki|響|yoichi|余市|karuizawa|輕井澤|軽井沢|springbank|bowmore|ardbeg|glenfiddich|dalmore|laphroaig|brora|port ellen/i.test(full)) return 'whisky'; | |
| 310 | + if (/cognac|armagnac|calvados|干邑|白蘭地|ブランデー/i.test(full)) return 'cognac'; | |
| 311 | + if (/\brum\b|\brhum\b|朗姆|ラム酒/i.test(full)) return 'rum'; | |
| 312 | + return 'wine'; | |
| 313 | + } | |
| 314 | + if (hint === 'asian' || hint === 'antiquities') return slugFromTitle(text, hint) ?? 'antiques'; | |
| 315 | + if (hint === 'cars') { | |
| 316 | + if (/\b(helmet|casque|helm|poster|affiche|plakat|trophy|trophée|suit|combinaison|gloves|gants|photograph|photographie|model|maquette|modell|miniature|book|livre|buch|sign|plaque|enamel|programme|program|watch|montre|mascot|mascotte|badge)\b/i.test(full) && !/\b(chassis|châssis|fahrgestell|telaio|\bvin\b|immatricul|registration|kilom|mileage)\b/i.test(full)) return 'automotive_memorabilia'; | |
| 317 | + return slugFromTitle(full, 'cars') ?? 'automobiles'; | |
| 318 | + } | |
| 319 | + return slugFromTitle(text, hint) ?? slugFromTitle(full, hint) ?? (hint !== 'unknown' ? slugFromTitle('', hint) : null) ?? (saleHint !== 'unknown' ? slugFromTitle('', saleHint) : null) ?? fallbackSlug; | |
| 320 | +} | |
| 321 | + | |
| 322 | +/** Unescape HTML entities commonly found in server-rendered auction pages. */ | |
| 323 | +export function decodeEntities(s: string): string { | |
| 324 | + return s | |
| 325 | + .replace(/ | /g, ' ') | |
| 326 | + .replace(/&/g, '&') | |
| 327 | + .replace(/</g, '<') | |
| 328 | + .replace(/>/g, '>') | |
| 329 | + .replace(/"/g, '"') | |
| 330 | + .replace(/'|'|’|’/g, "'") | |
| 331 | + .replace(/€/g, '€') | |
| 332 | + .replace(/£/g, '£') | |
| 333 | + .replace(/é/g, 'é') | |
| 334 | + .replace(/è/g, 'è') | |
| 335 | + .replace(/à/g, 'à') | |
| 336 | + .replace(/ç/g, 'ç') | |
| 337 | + .replace(/ô/g, 'ô') | |
| 338 | + .replace(/ê/g, 'ê') | |
| 339 | + .replace(/ü/g, 'ü') | |
| 340 | + .replace(/ö/g, 'ö') | |
| 341 | + .replace(/ä/g, 'ä') | |
| 342 | + .replace(/ß/g, 'ß') | |
| 343 | + .replace(/º/g, 'º') | |
| 344 | + .replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n))) | |
| 345 | + .replace(/&#x([0-9a-f]+);/gi, (_, h: string) => String.fromCodePoint(Number.parseInt(h, 16))); | |
| 346 | +} | |
| 347 | + | |
| 348 | +/** Text content of an HTML fragment (tags stripped, entities decoded, whitespace collapsed). */ | |
| 349 | +export function textOf(fragment: string | null | undefined): string { | |
| 350 | + if (!fragment) return ''; | |
| 351 | + return decodeEntities(fragment.replace(/<br\s*\/?>/gi, ' ').replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim(); | |
| 352 | +} | |
| 353 | + | |
| 354 | +/** Split an HTML document into item chunks starting at each match of `startRe` (the chunk runs to the next match). */ | |
| 355 | +export function chunksBetween(htmlText: string, startRe: RegExp, endBoundary?: RegExp): string[] { | |
| 356 | + const re = new RegExp(startRe.source, startRe.flags.includes('g') ? startRe.flags : `${startRe.flags}g`); | |
| 357 | + const idx: number[] = []; | |
| 358 | + let m: RegExpExecArray | null; | |
| 359 | + while ((m = re.exec(htmlText))) idx.push(m.index); | |
| 360 | + if (idx.length === 0) return []; | |
| 361 | + let end = htmlText.length; | |
| 362 | + if (endBoundary) { | |
| 363 | + const tail = htmlText.slice(idx[idx.length - 1]!); | |
| 364 | + const e = tail.search(endBoundary); | |
| 365 | + if (e > 0) end = idx[idx.length - 1]! + e; | |
| 366 | + } | |
| 367 | + return idx.map((s, k) => htmlText.slice(s, idx[k + 1] ?? end)).filter((c) => c.length > 0); | |
| 368 | +} | |
| 369 | + | |
| 370 | +/** First capture group of `re` in `s`, entity-decoded and trimmed; null when absent. */ | |
| 371 | +export function pick(s: string, re: RegExp): string | null { | |
| 372 | + const m = s.match(re); | |
| 373 | + return m ? textOf(m[1] ?? m[0]) || null : null; | |
| 374 | +} | |
| 375 | + | |
| 376 | +export function absolute(base: string, href: string | null | undefined): string | null { | |
| 377 | + if (!href) return null; | |
| 378 | + try { | |
| 379 | + return new URL(decodeEntities(href), base).toString(); | |
| 380 | + } catch { | |
| 381 | + return null; | |
| 382 | + } | |
| 383 | +} | |
added
connectors/api/_g9-asia-watch-sneaker-lib/capture.ts
+103 −0
@@ -0,0 +1,103 @@ | ||
| 1 | +/** | |
| 2 | + * Live smoke + fixture capture for the g9 connectors (real router + crawl context, no database). | |
| 3 | + * | |
| 4 | + * pnpm tsx connectors/api/_g9-asia-watch-sneaker-lib/capture.ts <connectorId> [--limit 1] [--seeds a,b] [--mode probe|backfill] | |
| 5 | + * [--name fixtureName] [--trim 12] [--no-save] [--cursor '{"json":1}'] [--lookup <url>] | |
| 6 | + * | |
| 7 | + * Payload arrays (rows/items/tiles/data/list/offers) are trimmed to --trim entries before saving so fixtures stay small | |
| 8 | + * but remain genuine live captures. | |
| 9 | + */ | |
| 10 | +import { existsSync, readFileSync } from 'node:fs'; | |
| 11 | +import path from 'node:path'; | |
| 12 | +import { ConnectorMetaSchema, DOMAINS_DIR, DOMAINS_PATH, createCrawlContext, createRouter, loadDomains, setDomains, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors'; | |
| 13 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 14 | + | |
| 15 | +/** Another group's domains.d fragment may be temporarily invalid; fall back to domains.json + our own fragment so smoke runs stay possible. */ | |
| 16 | +function ensureDomains() { | |
| 17 | + try { | |
| 18 | + loadDomains(); | |
| 19 | + } catch (err) { | |
| 20 | + console.warn(`[capture] domains.d contains an invalid fragment (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); using domains.json + g9 fragment only`); | |
| 21 | + const base = JSON.parse(readFileSync(DOMAINS_PATH, 'utf8')) as { version?: string; defaults?: unknown; domains?: Record<string, unknown> }; | |
| 22 | + const frag = JSON.parse(readFileSync(path.join(DOMAINS_DIR, 'g9-asia-watch-sneaker.json'), 'utf8')) as { domains?: Record<string, unknown> }; | |
| 23 | + setDomains({ version: base.version ?? '1.0', defaults: (base.defaults ?? {}) as never, domains: { ...(base.domains ?? {}), ...(frag.domains ?? {}) } as never }); | |
| 24 | + } | |
| 25 | +} | |
| 26 | + | |
| 27 | +function loadEnv() { | |
| 28 | + const p = path.resolve(process.cwd(), '.env'); | |
| 29 | + if (!existsSync(p)) return; | |
| 30 | + for (const line of readFileSync(p, 'utf8').split('\n')) { | |
| 31 | + const m = line.match(/^([A-Z0-9_]+)=(.*)$/); | |
| 32 | + if (m && !process.env[m[1]!]) process.env[m[1]!] = m[2]!.replace(/^"(.*)"$/, '$1'); | |
| 33 | + } | |
| 34 | +} | |
| 35 | + | |
| 36 | +function flag(name: string): string | undefined { | |
| 37 | + const i = process.argv.indexOf(`--${name}`); | |
| 38 | + return i >= 0 ? process.argv[i + 1] : undefined; | |
| 39 | +} | |
| 40 | + | |
| 41 | +function trimPayload(payload: unknown, n: number): unknown { | |
| 42 | + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return payload; | |
| 43 | + const out: Record<string, unknown> = { ...(payload as Record<string, unknown>) }; | |
| 44 | + for (const k of ['rows', 'items', 'tiles', 'data', 'list', 'offers', 'cards', 'hits', 'Items', 'products']) { | |
| 45 | + if (Array.isArray(out[k]) && (out[k] as unknown[]).length > n) out[k] = (out[k] as unknown[]).slice(0, n); | |
| 46 | + } | |
| 47 | + return out; | |
| 48 | +} | |
| 49 | + | |
| 50 | +async function main() { | |
| 51 | + loadEnv(); | |
| 52 | + ensureDomains(); | |
| 53 | + const id = process.argv[2]; | |
| 54 | + if (!id || id.startsWith('--')) throw new Error('usage: capture.ts <connectorId> [--limit N] [--seeds a,b] [--name x] [--trim N] [--no-save] [--lookup url]'); | |
| 55 | + const limit = Number(flag('limit') ?? 1); | |
| 56 | + const trim = Number(flag('trim') ?? 12); | |
| 57 | + const save = !process.argv.includes('--no-save'); | |
| 58 | + const seeds = flag('seeds')?.split(',').map((s) => s.trim()).filter(Boolean); | |
| 59 | + const mode = (flag('mode') ?? 'probe') as 'probe' | 'incremental' | 'backfill'; | |
| 60 | + const cursor = flag('cursor') ? (JSON.parse(flag('cursor')!) as Record<string, unknown>) : undefined; | |
| 61 | + const dir = ['api', 'firecrawl', 'scrapfly'].map((d) => path.resolve('connectors', d, id)).find((d) => existsSync(path.join(d, 'meta.json'))); | |
| 62 | + if (!dir) throw new Error(`no meta.json for ${id}`); | |
| 63 | + const metaRaw = JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')) as Record<string, unknown>; | |
| 64 | + const meta: ConnectorMeta = ConnectorMetaSchema.parse({ ...metaRaw, module: metaRaw.module ?? `${path.basename(path.dirname(dir))}/${id}` }); | |
| 65 | + const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector }; | |
| 66 | + const connector = mod.default(meta); | |
| 67 | + const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY }); | |
| 68 | + const ctx = createCrawlContext({ router, meta, options: { mode, limit, seeds, cursor } }); | |
| 69 | + const lookupUrl = flag('lookup'); | |
| 70 | + const raws = lookupUrl ? await connector.lookup!(lookupUrl, ctx) : null; | |
| 71 | + let n = 0; | |
| 72 | + const iter = raws ? (async function* () { for (const r of raws) yield r; })() : connector.crawl(ctx); | |
| 73 | + for await (const raw of iter) { | |
| 74 | + n++; | |
| 75 | + const records = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); | |
| 76 | + const kinds = records.reduce<Record<string, number>>((acc, r) => ((acc[r.kind] = (acc[r.kind] ?? 0) + 1), acc), {}); | |
| 77 | + console.log(`\n[${id}] raw #${n} ${raw.url} engine=${raw.engine} http=${raw.httpStatus ?? '-'} → ${records.length} records ${JSON.stringify(kinds)}`); | |
| 78 | + for (const r of records.slice(0, 4)) { | |
| 79 | + const a = r as Record<string, unknown>; | |
| 80 | + const price = 'price' in r ? `${(r as { price: unknown }).price} ${(r as { currency?: string }).currency ?? ''}` : 'currentBid' in r ? `bid ${(r as { currentBid: unknown }).currentBid} ${(r as { currency?: string }).currency ?? ''}` : ''; | |
| 81 | + const date = 'saleDate' in r ? (r as { saleDate: Date }).saleDate.toISOString().slice(0, 10) : 'endsAt' in r && (r as { endsAt: Date | null }).endsAt ? `ends ${(r as { endsAt: Date }).endsAt.toISOString().slice(0, 10)}` : ''; | |
| 82 | + const attrs = 'attributes' in r ? JSON.stringify({ cat: r.attributes.categorySlug, brand: r.attributes.brand, ref: r.attributes.reference, ids: r.attributes.identifiers }).slice(0, 200) : ''; | |
| 83 | + console.log(' -', r.kind, '|', String((a.rawTitle as string | undefined) ?? (a.title as string | undefined) ?? '').slice(0, 80), '|', price, '|', date, '|', attrs, '|', 'grade' in r && r.grade.grade ? `${r.grade.grader} ${r.grade.grade}` : ''); | |
| 84 | + } | |
| 85 | + if (save) { | |
| 86 | + const name = flag('name') ? (n === 1 ? flag('name')! : `${flag('name')}-${n}`) : (raw.externalId ?? `raw-${n}`).replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '').toLowerCase().slice(0, 80); | |
| 87 | + const payload = trimPayload(raw.payload, trim); | |
| 88 | + const trimmedRecords = await connector.normalize({ ...raw, payload, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); | |
| 89 | + saveFixture(id, name, { | |
| 90 | + raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload }, | |
| 91 | + expect: { minCount: Math.min(trimmedRecords.length, 1), kinds: [...new Set(trimmedRecords.map((r) => r.kind))] }, | |
| 92 | + note: `Captured live from ${raw.url} by connectors/api/_g9-asia-watch-sneaker-lib/capture.ts (payload arrays trimmed to ${trim} entries).`, | |
| 93 | + }); | |
| 94 | + console.log(` saved fixture data/fixtures/${id}/${name}.json (${trimmedRecords.length} records after trim)`); | |
| 95 | + } | |
| 96 | + } | |
| 97 | + console.log(`\n[${id}] done: ${n} raw records; engine stats ${JSON.stringify(ctx.engineStats)}; anomalies ${JSON.stringify(ctx.anomalies)}`); | |
| 98 | +} | |
| 99 | + | |
| 100 | +main().catch((e) => { | |
| 101 | + console.error(e); | |
| 102 | + process.exit(1); | |
| 103 | +}); | |
added
connectors/api/_g9-asia-watch-sneaker-lib/index.ts
+220 −0
@@ -0,0 +1,220 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 3 | +import { extractYear } from '@rareindex/shared'; | |
| 4 | +import { WATCH_BRANDS, caseSize, watchCategory, watchCompleteness, watchConditionRaw, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Helpers shared by the g9 group (Japanese / Taiwanese / Korean marketplaces, watch dealers and auction | |
| 8 | + * platforms, sneaker shops). Multilingual (ja / zh / ko) title parsing, JST/KST dates, CJK bundle and | |
| 9 | + * condition hints, watch title decomposition. Kept inside connectors/api, not the framework. | |
| 10 | + */ | |
| 11 | + | |
| 12 | +// ---------- seeds ---------- | |
| 13 | +export const KeywordSeedSchema = z.object({ | |
| 14 | + /** search keyword in the source's language */ | |
| 15 | + q: z.string(), | |
| 16 | + /** taxonomy slug applied to every row of the seed (refined by title when it is a family slug) */ | |
| 17 | + category: z.string(), | |
| 18 | + /** card / product language when the seed implies one ("Japanese", "Chinese", "Korean", "English") */ | |
| 19 | + language: z.string().nullable().default(null), | |
| 20 | +}); | |
| 21 | +export type KeywordSeed = z.infer<typeof KeywordSeedSchema>; | |
| 22 | + | |
| 23 | +// ---------- dates ---------- | |
| 24 | +/** "2026-09-09 18:32:00" (JST wall clock) | "2026-09-08T17:07:22+09:00" | ISO with Z → UTC Date; null when unparseable. */ | |
| 25 | +export function parseJstDateTime(s: string | null | undefined): Date | null { | |
| 26 | + if (!s) return null; | |
| 27 | + const t = s.trim(); | |
| 28 | + if (/[+-]\d{2}:\d{2}$|Z$/.test(t)) { | |
| 29 | + const d = new Date(t); | |
| 30 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 31 | + } | |
| 32 | + const m = t.match(/^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2})(?::(\d{2}))?)?$/); | |
| 33 | + if (!m) return null; | |
| 34 | + const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4] ?? 0) - 9, Number(m[5] ?? 0), Number(m[6] ?? 0))); | |
| 35 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 36 | +} | |
| 37 | + | |
| 38 | +/** Unix seconds (number or numeric string) → Date; null when missing/invalid. */ | |
| 39 | +export function unixToDate(v: number | string | null | undefined): Date | null { | |
| 40 | + if (v === null || v === undefined || v === '') return null; | |
| 41 | + const n = typeof v === 'number' ? v : Number(v); | |
| 42 | + if (!Number.isFinite(n) || n <= 0) return null; | |
| 43 | + const d = new Date(n * 1000); | |
| 44 | + return Number.isNaN(d.getTime()) ? null : d; | |
| 45 | +} | |
| 46 | + | |
| 47 | +/** "2026-07-16" → UTC midnight. */ | |
| 48 | +export function isoDateOnly(s: string | null | undefined): Date | null { | |
| 49 | + const m = (s ?? '').match(/^(\d{4})-(\d{2})-(\d{2})/); | |
| 50 | + return m ? new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]))) : null; | |
| 51 | +} | |
| 52 | + | |
| 53 | +// ---------- CJK text ---------- | |
| 54 | +/** Full-width → half-width, "PSA10" / "ＰＳＡ１０" / "BGS9.5" → "PSA 10" so the shared grade parser reads them; brackets → spaces. */ | |
| 55 | +export function normaliseCjkGradeText(title: string): string { | |
| 56 | + return title | |
| 57 | + .normalize('NFKC') | |
| 58 | + .replace(/\b(PSA|BGS|CGC|SGC|ARS|ACE|TAG)\s*(\d{1,2}(?:\.\d)?)/gi, '$1 $2') | |
| 59 | + .replace(/【|】|\[|\]|「|」|『|』|（|）/g, ' ') | |
| 60 | + .replace(/\s+/g, ' ') | |
| 61 | + .trim(); | |
| 62 | +} | |
| 63 | + | |
| 64 | +export function cjkGrade(title: string): { grader: string | null; grade: string | null; qualifier: string | null } { | |
| 65 | + const g = parseGradeFromTitle(normaliseCjkGradeText(title)); | |
| 66 | + return { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier }; | |
| 67 | +} | |
| 68 | + | |
| 69 | +/** Lots of many items (ja / zh / ko). Deliberately narrow: "セット" alone also names sealed box sets. */ | |
| 70 | +export const BUNDLE_RE_CJK = /まとめ|セット売り|大量|\d+\s*枚セット|\d+\s*点セット|おまとめ|引退品|バラ売り不可|合售|整套出售|一起賣|\d+\s*張(?:一組|合售)|\d+\s*件組|묶음|일괄|\d+\s*장\s*세트|\bbundle\b|\blot of\b/i; | |
| 71 | +export function isCjkBundle(title: string): boolean { | |
| 72 | + return BUNDLE_RE_CJK.test(title.normalize('NFKC')); | |
| 73 | +} | |
| 74 | + | |
| 75 | +/** Coarse condition hint from a CJK title: "New" | "Used" | null. */ | |
| 76 | +export function cjkConditionRaw(title: string): string | null { | |
| 77 | + const t = title.normalize('NFKC'); | |
| 78 | + if (/未開封|新品未使用|新品|未使用|全新|未拆|새상품|미개봉|新品同様|\bBNIB\b|\bDS\b|\bdeadstock\b|\bunworn\b/i.test(t)) return 'New'; | |
| 79 | + if (/中古|二手|중고|used|pre-?owned/i.test(t)) return 'Used'; | |
| 80 | + return null; | |
| 81 | +} | |
| 82 | + | |
| 83 | +/** Card / product language implied by a CJK title. */ | |
| 84 | +export function cjkLanguage(title: string): string | null { | |
| 85 | + const t = title.normalize('NFKC'); | |
| 86 | + if (/英語版|英文版|영문|북미판|\bEN\b|\bENG\b|english/i.test(t)) return 'English'; | |
| 87 | + if (/日本語版|日版|日文版|일판|일본판|\bJP\b|\bJPN\b|japanese/i.test(t)) return 'Japanese'; | |
| 88 | + if (/中文版|简中|繁中|簡中|中文/.test(t)) return 'Chinese'; | |
| 89 | + if (/한글판|한판|\bKR\b|korean/i.test(t)) return 'Korean'; | |
| 90 | + return null; | |
| 91 | +} | |
| 92 | + | |
| 93 | +/** Family slugs whose rows are refined by title keywords. */ | |
| 94 | +const FAMILY_SLUGS = new Set(['trading_cards', 'watches', 'sneakers', 'video_games', 'designer_toys', 'action_figures', 'lego']); | |
| 95 | + | |
| 96 | +const CATEGORY_KEYWORDS: Array<[RegExp, string]> = [ | |
| 97 | + [/ポケモンカード|ポケカ|寶可夢|宝可梦|포켓몬\s*카드|pok[eé]mon/i, 'pokemon'], | |
| 98 | + [/遊戯王|遊戲王|游戏王|유희왕|yu-?gi-?oh/i, 'yugioh'], | |
| 99 | + [/ワンピースカード|ワンピカ|ONE PIECE\s*カード|海賊王卡|航海王卡|원피스\s*카드|one piece card/i, 'one_piece_card_game'], | |
| 100 | + [/マジック[:：]?ザ[・･]?ギャザリング|\bMTG\b|魔法風雲會|magic:? the gathering/i, 'magic_the_gathering'], | |
| 101 | + [/デジモンカード|digimon card/i, 'digimon_tcg'], | |
| 102 | + [/ヴァイスシュヴァルツ|weiss schwarz/i, 'weiss_schwarz'], | |
| 103 | + [/ロレックス|勞力士|劳力士|롤렉스|\brolex\b/i, 'rolex'], | |
| 104 | + [/パテック|百達翡麗|百达翡丽|파텍|patek/i, 'patek_philippe'], | |
| 105 | + [/オーデマ|愛彼|爱彼|오데마|audemars/i, 'audemars_piguet'], | |
| 106 | + [/オメガ|歐米茄|欧米茄|오메가|\bomega\b/i, 'omega'], | |
| 107 | + [/腕時計|手錶|手表|시계|セイコー|\bseiko\b|g-?shock|カルティエ|cartier|tudor|チューダー|breitling|ブライトリング|IWC|panerai|hublot|zenith|longines|tissot|casio/i, 'other_watches'], | |
| 108 | + [/ジョーダン|\bjordan\b|ナイキ|\bnike\b|나이키|조던|耐吉|\bdunk\b|air max|air force/i, 'nike_jordan'], | |
| 109 | + [/アディダス|adidas|yeezy|イージー|이지|아디다스|愛迪達/i, 'adidas_yeezy'], | |
| 110 | + [/new balance|ニューバランス|뉴발란스|asics|アシックス|salomon|puma|converse|vans|hoka/i, 'new_balance_asics_other'], | |
| 111 | + [/ガンプラ|ガンダム|鋼彈|钢弹|건담|gundam/i, 'gundam'], | |
| 112 | + [/BE@RBRICK|ベアブリック|\bKAWS\b|be@rbrick|bearbrick|designer toy|sofubi|ソフビ/i, 'designer_toys'], | |
| 113 | + [/\bLEGO\b|レゴ|樂高|乐高|레고/i, 'lego_sets'], | |
| 114 | + [/funko|ファンコ/i, 'funko'], | |
| 115 | + [/ファミコン|スーファミ|スーパーファミコン|ゲームボーイ|nintendo|任天堂|닌텐도|game ?boy|\bswitch\b|\bN64\b|gamecube|\bwii\b/i, 'nintendo_games'], | |
| 116 | + [/playstation|プレイステーション|プレステ|\bPS[1-5]\b|플레이스테이션/i, 'playstation_games'], | |
| 117 | + [/セガ|\bsega\b|メガドライブ|dreamcast|saturn/i, 'sega_games'], | |
| 118 | + [/figma|フィギュア|figure|피규어|公仔|ねんどろいど|nendoroid|hot toys|S\.H\.Figuarts/i, 'action_figures'], | |
| 119 | +]; | |
| 120 | + | |
| 121 | +/** Seed category, refined by title keywords when the seed is a family slug (trading_cards, watches, sneakers…). */ | |
| 122 | +export function refineCategory(seedCategory: string, title: string): string { | |
| 123 | + if (!FAMILY_SLUGS.has(seedCategory)) return seedCategory; | |
| 124 | + const t = title.normalize('NFKC'); | |
| 125 | + for (const [re, slug] of CATEGORY_KEYWORDS) if (re.test(t)) return slug; | |
| 126 | + return seedCategory; | |
| 127 | +} | |
| 128 | + | |
| 129 | +/** Best-effort category for a title with no seed hint (Bunjang / Ruten category ids are source-specific). */ | |
| 130 | +export function categoryFromTitle(title: string): string | null { | |
| 131 | + const t = title.normalize('NFKC'); | |
| 132 | + for (const [re, slug] of CATEGORY_KEYWORDS) if (re.test(t)) return slug; | |
| 133 | + return null; | |
| 134 | +} | |
| 135 | + | |
| 136 | +// ---------- watches ---------- | |
| 137 | +export interface WatchTitleParts { | |
| 138 | + brand: string | null; | |
| 139 | + categorySlug: string; | |
| 140 | + reference: string | null; | |
| 141 | + material: string | null; | |
| 142 | + size: string | null; | |
| 143 | + completeness: string | null; | |
| 144 | + conditionRaw: string | null; | |
| 145 | + year: number | null; | |
| 146 | +} | |
| 147 | + | |
| 148 | +const BRAND_ALIASES: Array<[RegExp, string]> = [ | |
| 149 | + [/^a\.? ?lange ?(?:&|and|und) ?s[oö]hne/i, 'A. Lange & Söhne'], | |
| 150 | + [/^patek ?philippe/i, 'Patek Philippe'], | |
| 151 | + [/^audemars ?piguet/i, 'Audemars Piguet'], | |
| 152 | + [/^vacheron ?constantin/i, 'Vacheron Constantin'], | |
| 153 | + [/^jaeger[- ]?lecoultre/i, 'Jaeger-LeCoultre'], | |
| 154 | + [/^tag ?heuer/i, 'TAG Heuer'], | |
| 155 | + [/^grand ?seiko/i, 'Grand Seiko'], | |
| 156 | + [/^f\.? ?p\.? ?journe/i, 'F.P. Journe'], | |
| 157 | + [/^h\.? ?moser/i, 'H. Moser & Cie'], | |
| 158 | + [/^girard[- ]?perregaux/i, 'Girard-Perregaux'], | |
| 159 | + [/^glash[uü]tte ?original/i, 'Glashütte Original'], | |
| 160 | + [/^richard ?mille/i, 'Richard Mille'], | |
| 161 | + [/^bell ?& ?ross/i, 'Bell & Ross'], | |
| 162 | + [/^christopher ?ward/i, 'Christopher Ward'], | |
| 163 | + [/^ming\b/i, 'Ming'], | |
| 164 | +]; | |
| 165 | + | |
| 166 | +/** Brand at the start of a watch title (WATCH_BRANDS list + aliases); null when unknown. */ | |
| 167 | +export function watchBrandFromTitle(title: string): string | null { | |
| 168 | + const t = title.trim(); | |
| 169 | + for (const [re, name] of BRAND_ALIASES) if (re.test(t)) return name; | |
| 170 | + const sorted = [...WATCH_BRANDS].sort((a, b) => b.length - a.length); | |
| 171 | + for (const b of sorted) if (t.toLowerCase().startsWith(b.toLowerCase())) return b; | |
| 172 | + return null; | |
| 173 | +} | |
| 174 | + | |
| 175 | +/** Letter-prefixed references the shared regex misses: SBGE201G / SBGA211 (Seiko), IW371446 (IWC), PAM01312 (Panerai), M79030N-0001 (Tudor), CAR2A1Z (TAG Heuer). */ | |
| 176 | +const ALNUM_REF_RE = /\b([A-Z]{1,4}\d{3,6}[A-Z]{0,2}(?:-\d{4})?|[A-Z]{3}\d[A-Z]\d[A-Z])\b/; | |
| 177 | +const NOT_A_REF = new Set(['18K', '14K', '9K', '24K', 'GMT', 'UTC', 'COSC', 'PVD', 'DLC', 'SS', 'YG', 'WG', 'RG', 'PT', 'TI']); | |
| 178 | +export function alnumReferenceFromText(text: string): string | null { | |
| 179 | + for (const m of text.matchAll(new RegExp(ALNUM_REF_RE.source, 'g'))) { | |
| 180 | + const ref = m[1]!; | |
| 181 | + if (NOT_A_REF.has(ref) || /^(18|19|20)\d{2}$/.test(ref) || !/\d{3}/.test(ref)) continue; | |
| 182 | + return ref; | |
| 183 | + } | |
| 184 | + return null; | |
| 185 | +} | |
| 186 | + | |
| 187 | +export function watchFromTitle(title: string, brandHint?: string | null): WatchTitleParts { | |
| 188 | + const brand = brandHint?.trim() || watchBrandFromTitle(title); | |
| 189 | + const rest = brand ? title.replace(new RegExp(`^${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*`, 'i'), '') : title; | |
| 190 | + return { | |
| 191 | + brand, | |
| 192 | + categorySlug: watchCategory(brand), | |
| 193 | + reference: watchReferenceFromText(rest) ?? alnumReferenceFromText(rest), | |
| 194 | + material: watchMaterial(title), | |
| 195 | + size: caseSize(title), | |
| 196 | + completeness: watchCompleteness(title), | |
| 197 | + conditionRaw: watchConditionRaw(title), | |
| 198 | + year: extractYear(title), | |
| 199 | + }; | |
| 200 | +} | |
| 201 | + | |
| 202 | +/** "USD 4,380" | "$79,900" | "€ 8.850" → { amount, currency } using the shared parser conventions; null when absent. */ | |
| 203 | +export { parsePrice } from '@rareindex/shared'; | |
| 204 | + | |
| 205 | +// ---------- misc ---------- | |
| 206 | +export function intOrNull(v: unknown): number | null { | |
| 207 | + if (v === null || v === undefined || v === '') return null; | |
| 208 | + const n = typeof v === 'number' ? v : Number.parseInt(String(v).replace(/[^0-9-]/g, ''), 10); | |
| 209 | + return Number.isFinite(n) ? n : null; | |
| 210 | +} | |
| 211 | + | |
| 212 | +export function posNumber(v: unknown): number | null { | |
| 213 | + if (v === null || v === undefined || v === '') return null; | |
| 214 | + const n = typeof v === 'number' ? v : Number(String(v).replace(/[^0-9.]/g, '')); | |
| 215 | + return Number.isFinite(n) && n > 0 ? n : null; | |
| 216 | +} | |
| 217 | + | |
| 218 | +export function cleanTitle(s: string): string { | |
| 219 | + return s.normalize('NFKC').replace(/\s+/g, ' ').trim(); | |
| 220 | +} | |
added
connectors/api/ader/README.md
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +# ader — Ader (Paris, Hôtel Drouot) sale results | |
| 2 | + | |
| 3 | +Drouot "site générique" platform (shared parser `_g8-auctions-eu-apac-lib/drouot-platform.ts`, also used by | |
| 4 | +`osenat`). Index `/ventes-passees?year=YYYY` (2004→today) → catalogue `/catalogue/<id>?offset=N&max=50` | |
| 5 | +(50 lot cards: number, title, description, cdn.drouot.com image, estimate, `Résultat : 1 081 EUR`). | |
| 6 | + | |
| 7 | +- The page note `Résultats avec frais` / `sans frais` is read per page and per lot → `buyerPremiumIncluded` | |
| 8 | + (Ader publishes premium-inclusive results; null when the note is missing). | |
| 9 | +- `sale` for lots with a result, `auction_lot` (ended) for unsold lots. Identifiers `ader_lot = <catalogue>/<lot>`. | |
| 10 | +- Backfill: every year from `backfillFromYear`, oldest sale first, resumable. | |
| 11 | +- Fixtures: `pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts ader --save`. | |
added
connectors/api/ader/index.test.ts
+77 −0
@@ -0,0 +1,77 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 3 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 6 | +import createConnector, { backfillYears } from './index.js'; | |
| 7 | +import { parseDrouotCatalogue, parseDrouotIndex, premiumFromNote } from '../_g8-auctions-eu-apac-lib/drouot-platform.js'; | |
| 8 | + | |
| 9 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 10 | +const connector = createConnector(meta); | |
| 11 | +const attrsOf = (r: NormalizedRecord) => { | |
| 12 | + if (!('attributes' in r)) throw new Error(`record kind ${r.kind} has no attributes`); | |
| 13 | + return r.attributes; | |
| 14 | +}; | |
| 15 | +const HOUSE = { base: 'https://www.ader-paris.fr', currency: 'EUR' as const }; | |
| 16 | + | |
| 17 | +const INDEX = `<div class="calendrier entry clearfix Vente183542" id="183542"><div class="entry-title"><h2><a href="/vente/183542-art-militaire-et-decorations" class="cataPasDispo">Art militaire et décorations</a></h2></div><div class="bloc_vente_date"><i class="icon-calendar3"></i> jeudi 23 juillet 2026 à 20h00 </div></div> | |
| 18 | +<div class="calendrier entry clearfix Vente182770 Etude197 Etude2" id="182770"> <div class="col-md-2 entry-image couverture_catalogue"> <a href="/catalogue/182770-art-dapres-guerre-et-contemporain"> <img src="https://cdn.drouot.com/d/image/vente?size=phare&path=97/182770/fb56" alt="Art d’après-guerre et contemporain"></a></div><div class="entry-title"><h2><a href="/catalogue/182770-art-dapres-guerre-et-contemporain">Art d’après-guerre et contemporain</a></h2></div><div class="entry-content"><div class="bloc_vente_date"><i class="icon-calendar3"></i> jeudi 09 juillet 2026 à 14h00 </div><div class="bloc_vente_lieu"><i class="icon-globe"></i> Salle 6 - Hôtel Drouot , 9, rue Drouot 75009 Paris </div></div></div>`; | |
| 19 | + | |
| 20 | +const CATALOGUE = `<h1 class="nom_vente"> Art d’après-guerre et contemporain </h1> <div class="date_vente"> jeudi 09 juillet 2026 14:00 </div> <div class="lieu_vente"> Salle 6 - Hôtel Drouot , 9, rue Drouot 75009 Paris </div> | |
| 21 | +<div class="nbre_lot_haut"> Lots 1 à 50 sur 230 </div> | |
| 22 | +<div class="ordre_false product clearfix vente182770" id="lot1 lotId34329776"> <div class="product-image"> <a href="/lot/182770/34329776-georges-hugnet"> <img src="https://cdn.drouot.com/d/image/lot?size=phare&path=97/182770/7672" class="lot_visu" alt="Georges HUGNET..."></a> <div class="sale-flash"> Résultat : <nobr>593 EUR</nobr> </div></div> <div class="product-desc"> <div class="num_lot"> <span class='lotlabel'>Lot</span><span class='lotlabelnum'> nº</span><span class='lotnum'>1 </span> </div> <div class="product-title"> <h2> <a href="/lot/182770/34329776-georges-hugnet"> Georges HUGNET (1906-1974)... </a> </h2> </div> <div class="product-description"> <a href="/lot/182770/34329776-georges-hugnet"> <h2 id="lotDesc-34329776">Georges HUGNET (1906-1974) et André BEAUDIN (1904-1974) Composition, collage, 1955</h2> </a> </div> <div class="product-price"> <div class="estimLabelAff4">Estimation :</div> <div class="estimAff4"> 150 - 200 EUR </div> <div class="sale-flash2"> Résultat <nobr>593 EUR</nobr> </div> <div class="explicationResultats"> Résultats avec frais </div> </div> </div> </div> | |
| 23 | +<div class="ordre_false product clearfix vente182770" id="lot2 lotId34329777"> <div class="product-desc"> <div class="num_lot"> <span class='lotnum'>2 </span> </div> <div class="product-title"> <h2> <a href="/lot/182770/34329777-x"> ROLEX Submariner réf. 5513 </a> </h2> </div> <div class="product-price"> <div class="estimAff4"> 8 000 - 12 000 EUR </div> </div> </div> </div> | |
| 24 | +<div class="pagination_catalogue"><a href="/catalogue/182770?lang=fr&search=&offset=50&max=50">2</a></div>`; | |
| 25 | + | |
| 26 | +describe('ader', () => { | |
| 27 | + runFixtureSuite(connector, it, expect); | |
| 28 | + | |
| 29 | + it('fixtures: EUR results labelled from the page note (Ader = avec frais)', async () => { | |
| 30 | + let sales = 0; | |
| 31 | + for (const name of listFixtures('ader')) { | |
| 32 | + for (const r of await connector.normalize(loadFixture('ader', name).raw)) { | |
| 33 | + if (!('attributes' in r)) continue; | |
| 34 | + expect(r.attributes.identifiers.ader_lot).toMatch(/^\d+\/\S+$/); | |
| 35 | + expect(r.sourceUrl).toMatch(/^https:\/\/www\.ader-paris\.fr\//); | |
| 36 | + if (r.kind === 'sale') { | |
| 37 | + sales++; | |
| 38 | + expect(r.currency).toBe('EUR'); | |
| 39 | + expect(r.buyerPremiumIncluded).toBe(true); | |
| 40 | + expect(r.auctionHouse).toBe('Ader'); | |
| 41 | + } | |
| 42 | + } | |
| 43 | + } | |
| 44 | + expect(sales).toBeGreaterThan(10); | |
| 45 | + }); | |
| 46 | + | |
| 47 | + it('parses the Drouot-platform index (skips sales without catalogue) and a catalogue page', async () => { | |
| 48 | + const sales = parseDrouotIndex(INDEX, HOUSE); | |
| 49 | + expect(sales).toHaveLength(1); | |
| 50 | + expect(sales[0]).toMatchObject({ id: '182770', title: 'Art d’après-guerre et contemporain', url: 'https://www.ader-paris.fr/catalogue/182770-art-dapres-guerre-et-contemporain', date: '2026-07-09T00:00:00.000Z', location: 'Salle 6 - Hôtel Drouot, 9, rue Drouot 75009 Paris' }); | |
| 51 | + const p = parseDrouotCatalogue(CATALOGUE, sales[0]!, HOUSE)!; | |
| 52 | + expect(p.totalLots).toBe(230); | |
| 53 | + expect(p.hasMore).toBe(true); | |
| 54 | + expect(p.sale?.date).toBe('2026-07-09T00:00:00.000Z'); | |
| 55 | + expect(p.lots).toHaveLength(2); | |
| 56 | + expect(p.lots[0]).toMatchObject({ lotNo: '1', title: 'Georges HUGNET (1906-1974) et André BEAUDIN (1904-1974) Composition, collage, 1955', price: 593, currency: 'EUR', premiumIncluded: true, estimateLow: 150, estimateHigh: 200, sold: true, url: 'https://www.ader-paris.fr/lot/182770/34329776-georges-hugnet', image: 'https://cdn.drouot.com/d/image/lot?size=phare&path=97/182770/7672' }); | |
| 57 | + expect(p.lots[0]!.description).toBeNull(); | |
| 58 | + expect(p.lots[1]).toMatchObject({ lotNo: '2', price: null, sold: false, estimateLow: 8000, estimateHigh: 12000, premiumIncluded: true }); | |
| 59 | + Object.assign(sales[0]!, p.sale, { extra: { ...sales[0]!.extra, ...p.sale!.extra } }); | |
| 60 | + const out = await connector.normalize({ url: sales[0]!.url, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'sale_results', url: sales[0]!.url, sale: sales[0]!, page: 1, totalLots: 230, lots: p.lots } }); | |
| 61 | + expect(out.map((r) => r.kind)).toEqual(['sale', 'auction_lot']); | |
| 62 | + expect(attrsOf(out[0]!).categorySlug).toBe('contemporary_art'); | |
| 63 | + expect(attrsOf(out[1]!).categorySlug).toBe('rolex'); | |
| 64 | + expect(attrsOf(out[1]!).reference).toBe('5513'); | |
| 65 | + expect(connector.salePageUrl(sales[0]!, 2)).toBe('https://www.ader-paris.fr/catalogue/182770?lang=fr&search=&offset=50&max=50'); | |
| 66 | + expect(parseDrouotCatalogue('<html><body>rien</body></html>', sales[0]!, HOUSE)).toBeNull(); | |
| 67 | + }); | |
| 68 | + | |
| 69 | + it('premium notes and backfill years', () => { | |
| 70 | + expect(premiumFromNote('Résultats avec frais')).toBe(true); | |
| 71 | + expect(premiumFromNote('Résultats sans frais')).toBe(false); | |
| 72 | + expect(premiumFromNote(null)).toBeNull(); | |
| 73 | + expect(premiumFromNote('Résultats')).toBeNull(); | |
| 74 | + expect(backfillYears(2024)[0]).toBe(2024); | |
| 75 | + expect(backfillYears(2024).at(-1)).toBe(new Date().getUTCFullYear()); | |
| 76 | + }); | |
| 77 | +}); | |
added
connectors/api/ader/index.ts
+55 −0
@@ -0,0 +1,55 @@ | ||
| 1 | +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors'; | |
| 2 | +import type { ExtractionResult } from '@rareindex/shared'; | |
| 3 | +import { parseDrouotCatalogue, parseDrouotIndex } from '../_g8-auctions-eu-apac-lib/drouot-platform.js'; | |
| 4 | +import { SaleResultsConnector, type HouseConfig, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Ader (Paris, Hôtel Drouot) — Drouot "site générique" platform. Past sales per year at /ventes-passees?year=YYYY, | |
| 8 | + * lots at /catalogue/<id>?offset=N&max=50 with "Résultat : 1 081 EUR" and the page note "Résultats avec frais" | |
| 9 | + * (premium-inclusive) read per page. | |
| 10 | + */ | |
| 11 | +const BASE = 'https://www.ader-paris.fr'; | |
| 12 | +const HOUSE = { base: BASE, currency: 'EUR' as const }; | |
| 13 | +const PAGE = 50; | |
| 14 | + | |
| 15 | +export class AderConnector extends SaleResultsConnector { | |
| 16 | + readonly version = '1.0.0'; | |
| 17 | + readonly house: HouseConfig = { houseName: 'Ader', defaultCurrency: 'EUR', location: 'Paris, France', idKey: 'ader_lot', premiumIncluded: null, fallbackSlug: 'antiques', minIntervalMs: 2500, maxPagesPerSale: 30 }; | |
| 18 | + protected override minIntervalMs = 2500; | |
| 19 | + | |
| 20 | + async listSales(ctx: CrawlContext): Promise<SaleRef[]> { | |
| 21 | + const years = (this.meta.config.years as number[] | undefined) ?? [new Date().getUTCFullYear()]; | |
| 22 | + const all: SaleRef[] = []; | |
| 23 | + for (const y of ctx.options.mode === 'backfill' ? backfillYears(this.meta.config.backfillFromYear) : years) { | |
| 24 | + const url = `${BASE}/ventes-passees?year=${y}`; | |
| 25 | + await this.throttle(url); | |
| 26 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 27 | + if (!res.success || !res.html) { | |
| 28 | + ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 29 | + continue; | |
| 30 | + } | |
| 31 | + all.push(...parseDrouotIndex(res.html, HOUSE)); | |
| 32 | + } | |
| 33 | + return all; | |
| 34 | + } | |
| 35 | + | |
| 36 | + salePageUrl(sale: SaleRef, page: number): string { | |
| 37 | + return page > 1 ? `${sale.url.replace(/\/catalogue\/(\d+)[^?]*/, '/catalogue/$1')}?lang=fr&search=&offset=${(page - 1) * PAGE}&max=${PAGE}` : sale.url; | |
| 38 | + } | |
| 39 | + | |
| 40 | + parseSalePage(res: ExtractionResult, sale: SaleRef): ParsedSalePage | null { | |
| 41 | + return res.html ? parseDrouotCatalogue(res.html, sale, HOUSE, PAGE) : null; | |
| 42 | + } | |
| 43 | +} | |
| 44 | + | |
| 45 | +export function backfillYears(from: unknown): number[] { | |
| 46 | + const start = Number(from ?? 2004); | |
| 47 | + const now = new Date().getUTCFullYear(); | |
| 48 | + const out: number[] = []; | |
| 49 | + for (let y = start; y <= now; y++) out.push(y); | |
| 50 | + return out; | |
| 51 | +} | |
| 52 | + | |
| 53 | +export default function createConnector(meta: ConnectorMeta) { | |
| 54 | + return new AderConnector(meta); | |
| 55 | +} | |
added
connectors/api/ader/meta.json
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +{ | |
| 2 | + "id": "ader", | |
| 3 | + "displayName": "Ader (Paris, Hôtel Drouot) — sale results", | |
| 4 | + "sourceId": "ader", | |
| 5 | + "sourceName": "Ader", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.ader-paris.fr", | |
| 8 | + "module": "api/ader", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["art", "contemporary_art", "photography", "design_furniture", "antiques", "books", "maps", "historical_documents", "comics", "jewelry", "other_watches", "wine", "coins", "stamps", "militaria", "medals", "porcelain", "silver", "glass_crystal", "vintage_toys", "movie_posters", "cameras"], | |
| 11 | + "regions": ["FR"], | |
| 12 | + "languages": ["fr"], | |
| 13 | + "currency": ["EUR"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "medium", | |
| 23 | + "trustScore": 0.9, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.ader-paris.fr/conditions-generales-vente", | |
| 26 | + "acquisitionMethod": "server-rendered HTML (Drouot site-générique platform)", | |
| 27 | + "historicalDepth": "decades", | |
| 28 | + "accessNotes": "Ader (Paris, sales at Hôtel Drouot) runs on the Drouot 'site générique' platform. Public pages read: /ventes-passees?year=YYYY (past sales of a year: title, 'jeudi 09 juillet 2026 à 14h00', venue, /catalogue/<id> link — sales without an online catalogue are skipped) and /catalogue/<id>?offset=N&max=50 (lot cards: number, title, description, cdn.drouot.com image, 'Estimation : 150 - 200 EUR', 'Résultat : 1 081 EUR'; unsold lots have no result). Each page states 'Résultats avec frais' — we read that note per page/lot and set buyer_premium_included accordingly (true for Ader; never assumed, stays null when the note is missing). robots.txt disallows only /recherche/*; CGV contain no clause on automated access (they define 'prix marteau' vs fees). Years 2004→today available for backfill (one index request per year). 2.5 s politeness; salesPerRun caps incremental runs. No login, no bidder data.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "years": [2026], | |
| 33 | + "backfillFromYear": 2004, | |
| 34 | + "salesPerRun": 2 | |
| 35 | + } | |
| 36 | +} | |
added
connectors/api/aguttes/README.md
+13 −0
@@ -0,0 +1,13 @@ | ||
| 1 | +# aguttes — Aguttes (Neuilly-sur-Seine, FR) sale results | |
| 2 | + | |
| 3 | +Artisio-powered Next.js site: each public catalogue page `/catalogue/<id>?page=N` embeds lots + auction in | |
| 4 | +`__NEXT_DATA__` (`hammer_price` = hammer, EUR; estimates; status sold/unsold; images; premium tiers). | |
| 5 | +Past catalogues come from `sitemap-fr-ventes-passees.xml` (≈1 170 sales). | |
| 6 | + | |
| 7 | +- Built on `_g8-auctions-eu-apac-lib/sale-results.ts` (shared crawl/normalize): `sale` for sold lots | |
| 8 | + (`buyerPremiumIncluded=false`), `auction_lot` (ended) for unsold; identifiers `aguttes_lot = <catalogue>/<lot_no>`. | |
| 9 | +- Incremental: newest catalogues first, `salesPerRun` per run, cursor `{done:[…]}`; backfill oldest→newest with | |
| 10 | + `{backfill:{index,page,itemsProcessed}}` + `progress()`, ends `{finished:true}`. | |
| 11 | +- Categories: sale title (fr/en) → department hint via `englishLabel` + title keywords; car sales fall back to | |
| 12 | + `automobiles` / `automotive_memorabilia`. | |
| 13 | +- Fixtures: `pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts aguttes --save`. | |
added
connectors/api/aguttes/index.test.ts
+94 −0
@@ -0,0 +1,94 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 3 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 6 | +import createConnector, { humanizeSlug, parseCataloguePage, parseSalesSitemap } from './index.js'; | |
| 7 | + | |
| 8 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 9 | +const connector = createConnector(meta); | |
| 10 | +const attrsOf = (r: NormalizedRecord) => { | |
| 11 | + if (!('attributes' in r)) throw new Error(`record kind ${r.kind} has no attributes`); | |
| 12 | + return r.attributes; | |
| 13 | +}; | |
| 14 | + | |
| 15 | +const NEXT = { | |
| 16 | + props: { | |
| 17 | + pageProps: { | |
| 18 | + auction: { uuid: 'a-1', sale_no: 'G000002', title: { en: 'Tour Auto 2025 • The Official Sale', fr: 'Tour Auto 2025 • La Vente Officielle' }, start_date: '2025-04-07T14:00:00.000Z', status: 'completed', type: 'live', currency: { code: 'EUR' }, premiums: [{ percent: 25, amount_over: 0 }], branch: { name: 'Aguttes', city: 'Neuilly-sur-seine', country_code: 'FR' } }, | |
| 19 | + auctionLots: { | |
| 20 | + count: 39, | |
| 21 | + limit: 24, | |
| 22 | + page: 1, | |
| 23 | + results: [ | |
| 24 | + { uuid: 'l-1', lot_no: '1', status: 'sold', low: '15000.00', high: '25000.00', hammer_price: '20000.00', title: { en: 'Michael Schumacher', fr: 'Michael Schumacher' }, quantity: 1, num_of_bids: 0, primary_image: { data: { lg: { url: 'https://media.app.artisio.co/x_lg.jpeg' } } }, dynamic_fields: { en: { title: 'Michael Schumacher', artist: '', car_brand: 'Ferrari', description: 'Official BELL DOMINATOR helmet, worn during the 1999 pre-season tests<br>Numbered 9th.' } } }, | |
| 25 | + { uuid: 'l-2', lot_no: '2', status: 'unsold', low: '1200.00', high: '1800.00', hammer_price: null, title: { en: 'Ferrari F2008', fr: 'Ferrari F2008' }, quantity: 1, dynamic_fields: { en: { description: 'Nose cone.' } } }, | |
| 26 | + { uuid: 'l-3', lot_no: '35', status: 'sold', low: '100000.00', high: '150000.00', hammer_price: '235000.00', title: { en: 'CARTIER', fr: 'CARTIER' }, dynamic_fields: { en: { description: 'Tank wristwatch, yellow gold, circa 1975.' } } }, | |
| 27 | + ], | |
| 28 | + }, | |
| 29 | + }, | |
| 30 | + }, | |
| 31 | +}; | |
| 32 | +const HTML = `<html><body><a href="/lot/tour-auto-2025-la-vente-officielle-a-1/michael-schumacher-l-1">x</a><a href="/lot/tour-auto-2025-la-vente-officielle-a-1/cartier-l-3">y</a><script id="__NEXT_DATA__" type="application/json">${JSON.stringify(NEXT)}</script></body></html>`; | |
| 33 | + | |
| 34 | +describe('aguttes', () => { | |
| 35 | + runFixtureSuite(connector, it, expect); | |
| 36 | + | |
| 37 | + it('fixtures: EUR hammer-price sales with the Aguttes lot identifier', async () => { | |
| 38 | + let sales = 0; | |
| 39 | + for (const name of listFixtures('aguttes')) { | |
| 40 | + const out = await connector.normalize(loadFixture('aguttes', name).raw); | |
| 41 | + for (const r of out) { | |
| 42 | + if (!('attributes' in r)) continue; | |
| 43 | + expect(r.attributes.identifiers.aguttes_lot).toMatch(/^[^/]+\/\S+$/); | |
| 44 | + expect(r.sourceUrl.startsWith('https://www.aguttes.com/')).toBe(true); | |
| 45 | + if (r.kind === 'sale') { | |
| 46 | + sales++; | |
| 47 | + expect(r.currency).toBe('EUR'); | |
| 48 | + expect(r.buyerPremiumIncluded).toBe(false); | |
| 49 | + expect(r.auctionHouse).toBe('Aguttes'); | |
| 50 | + expect(r.lotNumber).toBeTruthy(); | |
| 51 | + } | |
| 52 | + } | |
| 53 | + } | |
| 54 | + expect(sales).toBeGreaterThan(5); | |
| 55 | + }); | |
| 56 | + | |
| 57 | + it('parses the sitemap (newest catalogues first) and slugs', () => { | |
| 58 | + const xml = `<urlset><url><loc>https://www.aguttes.com/catalogue/11442</loc></url><url><loc>https://www.aguttes.com/catalogue/160425</loc></url><url><loc>https://www.aguttes.com/catalogue/livres-manuscrits-6339ab29-3d37-453b-aeeb-d33046b92b2f</loc></url></urlset>`; | |
| 59 | + const sales = parseSalesSitemap(xml); | |
| 60 | + expect(sales.map((s) => s.id)).toEqual(['livres-manuscrits-6339ab29-3d37-453b-aeeb-d33046b92b2f', '160425', '11442']); | |
| 61 | + expect(sales[0]!.title).toBe('livres manuscrits'); | |
| 62 | + expect(humanizeSlug('livres-manuscrits-6339ab29-3d37-453b-aeeb-d33046b92b2f')).toBe('livres manuscrits'); | |
| 63 | + }); | |
| 64 | + | |
| 65 | + it('parses a catalogue page from __NEXT_DATA__ (hammer, estimates, sold/unsold, pagination, lot URLs)', async () => { | |
| 66 | + const sale = { id: '160425', title: '160425', url: 'https://www.aguttes.com/catalogue/160425', date: null, location: null, extra: {} }; | |
| 67 | + const p = parseCataloguePage(HTML, sale, 1)!; | |
| 68 | + expect(p.totalLots).toBe(39); | |
| 69 | + expect(p.hasMore).toBe(true); | |
| 70 | + expect(p.sale?.date).toBe('2025-04-07T14:00:00.000Z'); | |
| 71 | + expect(p.sale?.title).toBe('Tour Auto 2025 • The Official Sale'); | |
| 72 | + expect(p.lots).toHaveLength(3); | |
| 73 | + expect(p.lots[0]).toMatchObject({ lotNo: '1', price: 20000, currency: 'EUR', premiumIncluded: false, estimateLow: 15000, estimateHigh: 25000, sold: true, url: 'https://www.aguttes.com/lot/tour-auto-2025-la-vente-officielle-a-1/michael-schumacher-l-1', image: 'https://media.app.artisio.co/x_lg.jpeg' }); | |
| 74 | + expect(p.lots[0]!.description).toBe('Official BELL DOMINATOR helmet, worn during the 1999 pre-season tests Numbered 9th.'); | |
| 75 | + expect(p.lots[1]).toMatchObject({ lotNo: '2', price: null, sold: false }); | |
| 76 | + expect(p.lots[1]!.url).toContain('#lot-2'); | |
| 77 | + Object.assign(sale, p.sale, { extra: p.sale!.extra }); | |
| 78 | + const out = await connector.normalize({ url: sale.url, kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload: { kind: 'sale_results', url: sale.url, sale, page: 1, totalLots: 39, lots: p.lots } }); | |
| 79 | + expect(out.map((r) => r.kind)).toEqual(['sale', 'auction_lot', 'sale']); | |
| 80 | + const helmet = out[0]!; | |
| 81 | + if (helmet.kind === 'sale') { | |
| 82 | + expect(helmet.saleDate.toISOString()).toBe('2025-04-07T14:00:00.000Z'); | |
| 83 | + expect(helmet.attributes.categorySlug).toBe('automotive_memorabilia'); | |
| 84 | + expect(helmet.location).toBe('Neuilly-sur-seine, France'); | |
| 85 | + expect(helmet.attributes.metadata.sale_no).toBe('G000002'); | |
| 86 | + } | |
| 87 | + const cartier = out[2]!; | |
| 88 | + expect(attrsOf(cartier).categorySlug).toBe('other_watches'); | |
| 89 | + expect(attrsOf(cartier).brand).toBe('Cartier'); | |
| 90 | + const unsold = out[1]!; | |
| 91 | + if (unsold.kind === 'auction_lot') expect(unsold.status).toBe('ended'); | |
| 92 | + expect(parseCataloguePage('<html></html>', sale, 1)).toBeNull(); | |
| 93 | + }); | |
| 94 | +}); | |
added
connectors/api/aguttes/index.ts
+132 −0
@@ -0,0 +1,132 @@ | ||
| 1 | +import { html as H, type ConnectorMeta, type CrawlContext } from '@rareindex/connectors'; | |
| 2 | +import type { ExtractionResult } from '@rareindex/shared'; | |
| 3 | +import { stripHtml } from '../_g8-auctions-eu-apac-lib/index.js'; | |
| 4 | +import { SaleResultsConnector, resolveCategory, type HouseConfig, type ParsedLot, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Aguttes (Neuilly-sur-Seine / Paris) — Next.js site on the Artisio auction platform. Every catalogue page | |
| 8 | + * (/catalogue/<id-or-slug>?page=N) embeds the lot list and the auction object in __NEXT_DATA__ with | |
| 9 | + * numeric `hammer_price` (hammer, EUR), estimates, status (sold/unsold), images and bilingual titles. | |
| 10 | + * Past sales are enumerated from the public sitemap (sitemap-fr-ventes-passees.xml). | |
| 11 | + */ | |
| 12 | +const BASE = 'https://www.aguttes.com'; | |
| 13 | + | |
| 14 | +type NextLot = { uuid?: string; lot_no?: string; status?: string; low?: string | number | null; high?: string | number | null; hammer_price?: string | number | null; title?: Record<string, string> | string; quantity?: number; end_date?: string | null; num_of_bids?: number; primary_image?: { data?: Record<string, { url?: string }> } | null; dynamic_fields?: Record<string, Record<string, unknown>> }; | |
| 15 | +type NextAuction = { uuid?: string; sale_no?: string; title?: Record<string, string>; start_date?: string | null; end_date?: string | null; status?: string; type?: string; currency?: { code?: string }; premiums?: Array<{ percent?: number; amount_over?: number }>; branch?: { name?: string; city?: string; country_code?: string }; department_uuid?: string | null }; | |
| 16 | +type PageProps = { auction?: NextAuction; auctionLots?: { count?: number; limit?: number; page?: number; results?: NextLot[] }; drouotVente?: unknown }; | |
| 17 | + | |
| 18 | +function num(v: unknown): number | null { | |
| 19 | + if (v === null || v === undefined || v === '') return null; | |
| 20 | + const n = Number(v); | |
| 21 | + return Number.isFinite(n) && n > 0 ? n : null; | |
| 22 | +} | |
| 23 | + | |
| 24 | +export function humanizeSlug(slug: string): string { | |
| 25 | + return slug.replace(/-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, '').replace(/-/g, ' ').replace(/\s+/g, ' ').trim(); | |
| 26 | +} | |
| 27 | + | |
| 28 | +/** Sitemap of past sales → SaleRef list (newest last in the sitemap → we reverse). */ | |
| 29 | +export function parseSalesSitemap(xml: string): SaleRef[] { | |
| 30 | + const locs = [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((m) => m[1]!.replace(/&/g, '&')); | |
| 31 | + const out: SaleRef[] = []; | |
| 32 | + const seen = new Set<string>(); | |
| 33 | + for (const url of locs) { | |
| 34 | + const m = url.match(/\/catalogue\/([^/?#]+)$/); | |
| 35 | + if (!m || seen.has(m[1]!)) continue; | |
| 36 | + seen.add(m[1]!); | |
| 37 | + out.push({ id: m[1]!, title: humanizeSlug(m[1]!), url, date: null, location: null, extra: {} }); | |
| 38 | + } | |
| 39 | + return out.reverse(); | |
| 40 | +} | |
| 41 | + | |
| 42 | +/** Parse a catalogue page: lots + auction facts from __NEXT_DATA__; lot URLs from the rendered anchors. */ | |
| 43 | +export function parseCataloguePage(htmlText: string, sale: SaleRef, page: number): ParsedSalePage | null { | |
| 44 | + const data = H.nextData(htmlText) as { props?: { pageProps?: PageProps } } | null; | |
| 45 | + const pp = data?.props?.pageProps; | |
| 46 | + if (!pp?.auctionLots || !Array.isArray(pp.auctionLots.results)) return null; | |
| 47 | + const auction = pp.auction ?? {}; | |
| 48 | + const currency = auction.currency?.code ?? 'EUR'; | |
| 49 | + const hrefs = [...htmlText.matchAll(/href="(\/lot\/[^"]+)"/g)].map((m) => m[1]!.replace(/&/g, '&')); | |
| 50 | + const lots: ParsedLot[] = []; | |
| 51 | + for (const l of pp.auctionLots.results) { | |
| 52 | + const lotNo = String(l.lot_no ?? '').trim(); | |
| 53 | + const en = (l.dynamic_fields?.en ?? {}) as Record<string, unknown>; | |
| 54 | + const fr = (l.dynamic_fields?.fr ?? {}) as Record<string, unknown>; | |
| 55 | + const rawTitle = (typeof l.title === 'object' && l.title ? (l.title.en || l.title.fr) : typeof l.title === 'string' ? l.title : (en.title as string | undefined) || (fr.title as string | undefined)) ?? ''; | |
| 56 | + const title = stripHtml(rawTitle, 300) ?? ''; | |
| 57 | + if (!lotNo || !title) continue; | |
| 58 | + const uuid = String(l.uuid ?? ''); | |
| 59 | + const href = hrefs.find((h) => uuid && h.endsWith(uuid)) ?? (typeof en.url_legacy === 'string' ? en.url_legacy : null); | |
| 60 | + const url = href ? `${BASE}${href}` : `${sale.url}?page=${page}#lot-${lotNo}`; | |
| 61 | + const img = l.primary_image?.data; | |
| 62 | + const image = img?.lg?.url ?? img?.xlg?.url ?? img?.sm?.url ?? null; | |
| 63 | + const description = stripHtml((en.description as string | undefined) ?? (fr.description as string | undefined) ?? null, 500); | |
| 64 | + const hammer = num(l.hammer_price); | |
| 65 | + const sold = l.status === 'sold' && hammer !== null; | |
| 66 | + lots.push({ | |
| 67 | + lotNo, | |
| 68 | + title: String(title).trim(), | |
| 69 | + subtitle: typeof en.artist === 'string' && en.artist.trim() && en.artist.trim() !== String(title).trim() ? en.artist.trim() : null, | |
| 70 | + description, | |
| 71 | + url, | |
| 72 | + image, | |
| 73 | + price: hammer, | |
| 74 | + currency, | |
| 75 | + premiumIncluded: false, | |
| 76 | + estimateLow: num(l.low), | |
| 77 | + estimateHigh: num(l.high), | |
| 78 | + date: l.end_date ?? null, | |
| 79 | + sold, | |
| 80 | + extra: { status: l.status ?? null, lot_uuid: uuid || null, num_of_bids: l.num_of_bids ?? null, car_brand: (en.car_brand as string | undefined) || null, quantity: l.quantity ?? null }, | |
| 81 | + }); | |
| 82 | + } | |
| 83 | + const count = Number(pp.auctionLots.count ?? lots.length); | |
| 84 | + const limit = Number(pp.auctionLots.limit ?? 24); | |
| 85 | + const cur = Number(pp.auctionLots.page ?? page); | |
| 86 | + const title = auction.title?.en || auction.title?.fr || sale.title; | |
| 87 | + return { | |
| 88 | + lots, | |
| 89 | + hasMore: cur * limit < count, | |
| 90 | + totalLots: Number.isFinite(count) ? count : null, | |
| 91 | + sale: { | |
| 92 | + title, | |
| 93 | + date: auction.start_date ?? null, | |
| 94 | + location: auction.branch?.city ? `${auction.branch.city}, France` : null, | |
| 95 | + extra: { sale_no: auction.sale_no ?? null, auction_uuid: auction.uuid ?? null, auction_status: auction.status ?? null, auction_type: auction.type ?? null, premiums: auction.premiums ?? null, title_fr: auction.title?.fr ?? null }, | |
| 96 | + }, | |
| 97 | + }; | |
| 98 | +} | |
| 99 | + | |
| 100 | +export class AguttesConnector extends SaleResultsConnector { | |
| 101 | + readonly version = '1.0.0'; | |
| 102 | + readonly house: HouseConfig = { houseName: 'Aguttes', defaultCurrency: 'EUR', location: 'Neuilly-sur-Seine, France', idKey: 'aguttes_lot', premiumIncluded: false, fallbackSlug: 'antiques', minIntervalMs: 2000 }; | |
| 103 | + protected override minIntervalMs = 2000; | |
| 104 | + | |
| 105 | + async listSales(ctx: CrawlContext): Promise<SaleRef[]> { | |
| 106 | + const url = String(this.meta.config.pastSalesSitemap ?? `${BASE}/sitemap-fr-ventes-passees.xml`); | |
| 107 | + await this.throttle(url); | |
| 108 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 109 | + if (!res.success || !res.html) { | |
| 110 | + ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 111 | + return []; | |
| 112 | + } | |
| 113 | + return parseSalesSitemap(res.html); | |
| 114 | + } | |
| 115 | + | |
| 116 | + salePageUrl(sale: SaleRef, page: number): string { | |
| 117 | + return page > 1 ? `${sale.url}?page=${page}` : sale.url; | |
| 118 | + } | |
| 119 | + | |
| 120 | + parseSalePage(res: ExtractionResult, sale: SaleRef, page: number): ParsedSalePage | null { | |
| 121 | + return res.html ? parseCataloguePage(res.html, sale, page) : null; | |
| 122 | + } | |
| 123 | + | |
| 124 | + override categoryFor(sale: SaleRef, lot: ParsedLot): string | null { | |
| 125 | + const brand = String(lot.extra.car_brand ?? ''); | |
| 126 | + return resolveCategory(`${sale.title} ${String(sale.extra.title_fr ?? '')}`, brand ? { ...lot, subtitle: `${lot.subtitle ?? ''} ${brand}`.trim() } : lot, this.house.fallbackSlug); | |
| 127 | + } | |
| 128 | +} | |
| 129 | + | |
| 130 | +export default function createConnector(meta: ConnectorMeta) { | |
| 131 | + return new AguttesConnector(meta); | |
| 132 | +} | |
added
connectors/api/aguttes/meta.json
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +{ | |
| 2 | + "id": "aguttes", | |
| 3 | + "displayName": "Aguttes (sale results)", | |
| 4 | + "sourceId": "aguttes", | |
| 5 | + "sourceName": "Aguttes", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.aguttes.com", | |
| 8 | + "module": "api/aguttes", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["art", "contemporary_art", "photography", "design_furniture", "antiques", "jewelry", "other_watches", "rolex", "omega", "patek_philippe", "luxury_handbags", "wine", "whisky", "comics", "books", "maps", "automobiles", "motorcycles", "automotive_memorabilia", "coins", "stamps", "porcelain", "silver", "glass_crystal", "militaria", "medals", "sports_memorabilia"], | |
| 11 | + "regions": ["FR"], | |
| 12 | + "languages": ["fr", "en"], | |
| 13 | + "currency": ["EUR"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "medium", | |
| 23 | + "trustScore": 0.9, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.aguttes.com/legals/conditions-generales-d-utilisation", | |
| 26 | + "acquisitionMethod": "embedded JSON (__NEXT_DATA__, Artisio platform)", | |
| 27 | + "historicalDepth": "years", | |
| 28 | + "accessNotes": "Aguttes (Neuilly-sur-Seine, Lyon, Bruxelles) runs on the Artisio platform behind a Next.js site. We read the public sitemap https://www.aguttes.com/sitemap-fr-ventes-passees.xml (≈1 170 past catalogues back to the 2010s) and each public catalogue page /catalogue/<id>?page=N whose server-rendered __NEXT_DATA__ embeds the lot list (lot_no, status sold/unsold, hammer_price, low/high estimate, bilingual title, description, images) and the auction object (sale_no, start_date, currency EUR, premium tiers, branch). hammer_price is explicitly the HAMMER (buyer_premium_included=false; premium tiers kept in metadata.premiums). robots.txt: Allow / for *, Content-Signal search=yes, ai-train=no, use=reference (we use it as reference data, no model training); CGU contain no clause on automated access. 24 lots per page, 2 s politeness, salesPerRun caps each incremental run; backfill walks every catalogue oldest→newest with a resumable (index, page) cursor. No login, no bidder data (paddle numbers discarded).", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "pastSalesSitemap": "https://www.aguttes.com/sitemap-fr-ventes-passees.xml", | |
| 33 | + "salesPerRun": 3 | |
| 34 | + } | |
| 35 | +} | |
added
connectors/api/aste-bolaffi/README.md
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +# aste-bolaffi — Aste Bolaffi (Turin) risultati | |
| 2 | + | |
| 3 | +Laravel SSR: `/it/results` (113 auctions: DD.MM.YYYY date, city, department heading) → `/it/auction/<id>?page=N` | |
| 4 | +(10 lots/page: `Lotto N`, title, description, image, `Base asta: € 5 | Aggiudicato a : € 12.500`; sale date from | |
| 5 | +`getLocationDate('YYYY-MM-DD …')`). | |
| 6 | + | |
| 7 | +- `Aggiudicato a` is not labelled hammer/with-fees → `buyerPremiumIncluded=null`, EUR. `sale` when a result exists, | |
| 8 | + `auction_lot` (ended) otherwise. Identifiers `bolaffi_lot = <auction>/<lotto>`. 3 s politeness (many small pages). | |
| 9 | +- Fixtures: `pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts aste-bolaffi --save`. | |
added
connectors/api/aste-bolaffi/index.test.ts
+65 −0
@@ -0,0 +1,65 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 3 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 6 | +import createConnector, { parseAuctionPage, parseResultsIndex } from './index.js'; | |
| 7 | + | |
| 8 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 9 | +const connector = createConnector(meta); | |
| 10 | +const attrsOf = (r: NormalizedRecord) => { | |
| 11 | + if (!('attributes' in r)) throw new Error(`record kind ${r.kind} has no attributes`); | |
| 12 | + return r.attributes; | |
| 13 | +}; | |
| 14 | + | |
| 15 | +const INDEX = `<h2 class="title-border-risultati">Special Sales</h2> <div class="row "> <div class="col-left-table col-md-2 col-xs-2 tab-risultati-1"> <h3>07.05.2023</h3> </div> <div class="col-md-6 col-xs-4 tab-risultati"> <h3>Torino</h3> </div> <div class="col-md-2 col-xs-2 tab-risultati text-center"> <h3 class="cms-underline"><a href="https://www.astebolaffi.it/it/auction/1204">Catalogo</a> </h3> </div> <div class="col-right-table col-md-2 col-xs-2 tab-risultati-2"> <h3 class="cms-underline"><a href="https://www.astebolaffi.it/it/results/1204">Risultati</a> </h3> </div> </div> | |
| 16 | +<h2 class="title-border-risultati">Orologi</h2> <div class="row "> <div class="col-left-table col-md-2 col-xs-2 tab-risultati-1"> <h3>12.06.2026</h3> </div> <div class="col-md-6 col-xs-4 tab-risultati"> <h3>Milano</h3> </div> <div class="col-md-2"> <h3 class="cms-underline"><a href="https://www.astebolaffi.it/it/auction/3011">Catalogo</a> </h3> </div> </div>`; | |
| 17 | + | |
| 18 | +const AUCTION = `<title> Orologi da polso | Aste Bolaffi </title><script>$("#start_date").text(getLocationDate('2026-06-12 15:00:00'));</script> | |
| 19 | +<div class="row sezione-news lot-row"> <div class="col col-md-8 col-xs-12 "> <div class="news-left lot-description"> <div class="bg-dettaglio-asta img-wrapper"> <p class="thumb-lot"> <a href="https://www.astebolaffi.it/it/lot/3011/1/detail"> <img src="https://www.astebolaffi.it/storage/lots/3011/1.jpg" style="display: inline;"> </a> </p> </div> <div class="testo-right"> <h5 class="asta-title"> <a href="https://www.astebolaffi.it/it/lot/3011/1/detail">Lotto 1</a> </h5> <p class="lot-dida"> <strong>ROLEX SUBMARINER REF. 5513</strong> <br /> <p><strong>ROLEX SUBMARINER REF. 5513</strong> <br></p><p><em class="em_wide"><strong></strong> <br></em>Acciaio, anni '70, quadrante nero</p> </p> <p> Base asta: <span class="asta-euro">€ 5.000</span> <span class="cms-gold">|</span> Aggiudicato a : <span class="asta-euro">€ 12.500</span> </p> </div> </div> </div> </div> | |
| 20 | +<div class="row sezione-news lot-row"> <div class="testo-right"> <h5 class="asta-title"> <a href="https://www.astebolaffi.it/it/lot/3011/2/detail">Lotto 2</a> </h5> <p class="lot-dida"> <strong>OMEGA SPEEDMASTER</strong> <br /> </p> <p> Base asta: <span class="asta-euro">€ 2.000</span> </p> </div> </div> | |
| 21 | +<ul class="pagination"><li><a href="https://www.astebolaffi.it/it/auction/3011?page=2">2</a></li><li><a href="https://www.astebolaffi.it/it/auction/3011?page=9">9</a></li></ul>`; | |
| 22 | + | |
| 23 | +describe('aste-bolaffi', () => { | |
| 24 | + runFixtureSuite(connector, it, expect); | |
| 25 | + | |
| 26 | + it('fixtures: EUR results ("Aggiudicato a", premium basis unknown)', async () => { | |
| 27 | + let sales = 0; | |
| 28 | + for (const name of listFixtures('aste-bolaffi')) { | |
| 29 | + for (const r of await connector.normalize(loadFixture('aste-bolaffi', name).raw)) { | |
| 30 | + if (!('attributes' in r)) continue; | |
| 31 | + expect(r.attributes.identifiers.bolaffi_lot).toMatch(/^\d+\/\S+$/); | |
| 32 | + expect(r.sourceUrl).toMatch(/^https:\/\/www\.astebolaffi\.it\/it\/lot\/\d+\/\S+\/detail$/); | |
| 33 | + if (r.kind === 'sale') { | |
| 34 | + sales++; | |
| 35 | + expect(r.currency).toBe('EUR'); | |
| 36 | + expect(r.buyerPremiumIncluded).toBeNull(); | |
| 37 | + expect(r.auctionHouse).toBe('Aste Bolaffi'); | |
| 38 | + } | |
| 39 | + } | |
| 40 | + } | |
| 41 | + expect(sales).toBeGreaterThan(5); | |
| 42 | + }); | |
| 43 | + | |
| 44 | + it('parses the results index (Italian dates, departments) and an auction page (Italian prices, pagination)', async () => { | |
| 45 | + const sales = parseResultsIndex(INDEX); | |
| 46 | + expect(sales.map((s) => s.id)).toEqual(['1204', '3011']); | |
| 47 | + expect(sales[0]).toMatchObject({ date: '2023-05-07T00:00:00.000Z', location: 'Torino, Italy', url: 'https://www.astebolaffi.it/it/auction/1204' }); | |
| 48 | + expect(sales[0]!.extra.department).toBe('Special Sales'); | |
| 49 | + expect(sales[1]!.title).toBe('Orologi — Asta 3011'); | |
| 50 | + const p = parseAuctionPage(AUCTION, sales[1]!, 1)!; | |
| 51 | + expect(p.hasMore).toBe(true); | |
| 52 | + expect(parseAuctionPage(AUCTION, sales[1]!, 9)!.hasMore).toBe(false); | |
| 53 | + expect(p.sale?.date).toBe('2026-06-12T00:00:00.000Z'); | |
| 54 | + expect(p.lots).toHaveLength(2); | |
| 55 | + expect(p.lots[0]).toMatchObject({ lotNo: '1', title: 'ROLEX SUBMARINER REF. 5513', price: 12500, currency: 'EUR', premiumIncluded: null, estimateLow: 5000, sold: true, url: 'https://www.astebolaffi.it/it/lot/3011/1/detail', image: 'https://www.astebolaffi.it/storage/lots/3011/1.jpg' }); | |
| 56 | + expect(p.lots[0]!.description).toContain("Acciaio, anni '70"); | |
| 57 | + expect(p.lots[1]).toMatchObject({ lotNo: '2', price: null, sold: false, estimateLow: 2000 }); | |
| 58 | + const out = await connector.normalize({ url: sales[1]!.url, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'sale_results', url: sales[1]!.url, sale: { ...sales[1]!, ...p.sale, extra: sales[1]!.extra }, page: 1, totalLots: null, lots: p.lots } }); | |
| 59 | + expect(out.map((r) => r.kind)).toEqual(['sale', 'auction_lot']); | |
| 60 | + expect(attrsOf(out[0]!).categorySlug).toBe('rolex'); | |
| 61 | + expect(attrsOf(out[0]!).reference).toBe('5513'); | |
| 62 | + expect(attrsOf(out[1]!).categorySlug).toBe('omega'); | |
| 63 | + expect(parseAuctionPage('<html>x</html>', sales[1]!, 1)).toBeNull(); | |
| 64 | + }); | |
| 65 | +}); | |
added
connectors/api/aste-bolaffi/index.ts
+107 −0
@@ -0,0 +1,107 @@ | ||
| 1 | +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors'; | |
| 2 | +import type { ExtractionResult } from '@rareindex/shared'; | |
| 3 | +import { parseEuDate, parseEuMoney } from '../_g8-auctions-eu-apac-lib/index.js'; | |
| 4 | +import { SaleResultsConnector, chunksBetween, pick, textOf, type HouseConfig, type ParsedLot, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Aste Bolaffi (Turin — stamps, coins, banknotes, watches, wine, cars, memorabilia). Laravel site, server-rendered: | |
| 8 | + * /it/results lists every past auction (date, city, /it/auction/<id>); /it/auction/<id>?page=N shows 10 lots per | |
| 9 | + * page with "Lotto N", title, "Base asta: € 5 | Aggiudicato a : € 5"; the sale date is embedded as | |
| 10 | + * getLocationDate('2023-05-07 10:00:00'). "Aggiudicato a" is not labelled hammer/with-fees → premium basis null. | |
| 11 | + */ | |
| 12 | +const BASE = 'https://www.astebolaffi.it'; | |
| 13 | + | |
| 14 | +export function parseResultsIndex(htmlText: string): SaleRef[] { | |
| 15 | + const out: SaleRef[] = []; | |
| 16 | + const seen = new Set<string>(); | |
| 17 | + // rows: "<h3>07.05.2023</h3> … <h3>Torino</h3> … /it/auction/1204 … /it/results/1204" | |
| 18 | + for (const chunk of chunksBetween(htmlText, /<div class="row\s*">\s*<div class="col-left-table/)) { | |
| 19 | + const id = chunk.match(/\/it\/(?:auction|results)\/(\d+)"/)?.[1]; | |
| 20 | + if (!id || seen.has(id)) continue; | |
| 21 | + seen.add(id); | |
| 22 | + const h3s = [...chunk.matchAll(/<h3[^>]*>([\s\S]*?)<\/h3>/g)].map((m) => textOf(m[1])); | |
| 23 | + const dateText = h3s.find((t) => /^\d{2}\.\d{2}\.\d{4}$/.test(t)) ?? null; | |
| 24 | + const city = h3s.find((t) => t && !/^\d{2}\.\d{2}\.\d{4}$/.test(t) && !/Catalogo|Risultati/i.test(t)) ?? null; | |
| 25 | + const date = dateText ? parseEuDate(dateText) : null; | |
| 26 | + out.push({ id, title: `Asta ${id}`, url: `${BASE}/it/auction/${id}`, date: date ? date.toISOString() : null, location: city ? `${city}, Italy` : null, extra: { date_text: dateText } }); | |
| 27 | + } | |
| 28 | + // section headings ("Special Sales", "Francobolli", …) precede their rows → attach as department | |
| 29 | + let dept: string | null = null; | |
| 30 | + const seq = [...htmlText.matchAll(/<h2 class="title-border-risultati">([\s\S]*?)<\/h2>|\/it\/auction\/(\d+)"/g)]; | |
| 31 | + const deptById = new Map<string, string>(); | |
| 32 | + for (const m of seq) { | |
| 33 | + if (m[1]) dept = textOf(m[1]); | |
| 34 | + else if (m[2] && dept) deptById.set(m[2], dept); | |
| 35 | + } | |
| 36 | + return out.map((s) => ({ ...s, title: deptById.get(s.id) ? `${deptById.get(s.id)} — Asta ${s.id}` : s.title, extra: { ...s.extra, department: deptById.get(s.id) ?? null } })); | |
| 37 | +} | |
| 38 | + | |
| 39 | +export function parseAuctionPage(htmlText: string, sale: SaleRef, page: number): ParsedSalePage | null { | |
| 40 | + if (!/lot-row|asta-title/.test(htmlText)) return null; | |
| 41 | + const lots: ParsedLot[] = []; | |
| 42 | + for (const chunk of chunksBetween(htmlText, /<div class="row sezione-news lot-row">/)) { | |
| 43 | + const href = chunk.match(/href="(https:\/\/www\.astebolaffi\.it\/it\/lot\/(\d+)\/([^/"]+)\/detail)"/); | |
| 44 | + const lotNo = pick(chunk, /<h5 class="asta-title">[\s\S]*?<a[^>]*>\s*Lotto\s*([^<]*)<\/a>/) ?? href?.[3] ?? null; | |
| 45 | + if (!href || !lotNo) continue; | |
| 46 | + const titleBlock = chunk.match(/<p class="lot-dida">([\s\S]*?)<\/p>\s*<p>\s*Base asta/)?.[1] ?? chunk.match(/<p class="lot-dida">([\s\S]*?)<\/div>/)?.[1] ?? ''; | |
| 47 | + const strong = pick(titleBlock, /<strong>([\s\S]*?)<\/strong>/); | |
| 48 | + const full = textOf(titleBlock); | |
| 49 | + const title = strong || full.split(/\s{2,}/)[0] || ''; | |
| 50 | + if (!title) continue; | |
| 51 | + const desc = full.replace(title, '').replace(new RegExp(title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), '').trim(); | |
| 52 | + const start = pick(chunk, /Base asta:\s*<span class="asta-euro">([\s\S]*?)<\/span>/); | |
| 53 | + const sold = pick(chunk, /Aggiudicato a\s*:\s*<span class="asta-euro">([\s\S]*?)<\/span>/); | |
| 54 | + const price = sold ? parseEuMoney(sold, 'EUR') : null; | |
| 55 | + const image = chunk.match(/<img src="([^"]+)"/)?.[1] ?? null; | |
| 56 | + lots.push({ | |
| 57 | + lotNo: lotNo.trim(), | |
| 58 | + title, | |
| 59 | + subtitle: null, | |
| 60 | + description: desc || null, | |
| 61 | + url: href[1]!, | |
| 62 | + image: image && !/no_image/.test(image) ? image : null, | |
| 63 | + price: price?.amount ?? null, | |
| 64 | + currency: 'EUR', | |
| 65 | + premiumIncluded: null, | |
| 66 | + estimateLow: start ? parseEuMoney(start, 'EUR')?.amount ?? null : null, | |
| 67 | + estimateHigh: null, | |
| 68 | + date: null, | |
| 69 | + sold: price !== null, | |
| 70 | + extra: { starting_price_text: start, aggiudicato_text: sold }, | |
| 71 | + }); | |
| 72 | + } | |
| 73 | + const dateM = htmlText.match(/getLocationDate\('(\d{4}-\d{2}-\d{2})[^']*'\)/); | |
| 74 | + const last = [...htmlText.matchAll(/\?page=(\d+)"/g)].map((m) => Number(m[1])); | |
| 75 | + const maxPage = last.length ? Math.max(...last) : page; | |
| 76 | + const title = pick(htmlText, /<title>([\s\S]*?)<\/title>/); | |
| 77 | + return { lots, hasMore: page < maxPage, totalLots: null, sale: { date: dateM ? `${dateM[1]}T00:00:00.000Z` : undefined, title: title && !/^Lotto/i.test(title) && title.length < 120 ? title.replace(/\s*\|\s*Aste Bolaffi\s*$/, '') : undefined } }; | |
| 78 | +} | |
| 79 | + | |
| 80 | +export class AsteBolaffiConnector extends SaleResultsConnector { | |
| 81 | + readonly version = '1.0.0'; | |
| 82 | + readonly house: HouseConfig = { houseName: 'Aste Bolaffi', defaultCurrency: 'EUR', location: 'Torino, Italy', idKey: 'bolaffi_lot', premiumIncluded: null, fallbackSlug: 'antiques', minIntervalMs: 3000, maxPagesPerSale: 40 }; | |
| 83 | + protected override minIntervalMs = 3000; | |
| 84 | + | |
| 85 | + async listSales(ctx: CrawlContext): Promise<SaleRef[]> { | |
| 86 | + const url = String(this.meta.config.resultsUrl ?? `${BASE}/it/results`); | |
| 87 | + await this.throttle(url); | |
| 88 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 89 | + if (!res.success || !res.html) { | |
| 90 | + ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 91 | + return []; | |
| 92 | + } | |
| 93 | + return parseResultsIndex(res.html); | |
| 94 | + } | |
| 95 | + | |
| 96 | + salePageUrl(sale: SaleRef, page: number): string { | |
| 97 | + return page > 1 ? `${sale.url}?page=${page}` : sale.url; | |
| 98 | + } | |
| 99 | + | |
| 100 | + parseSalePage(res: ExtractionResult, sale: SaleRef, page: number): ParsedSalePage | null { | |
| 101 | + return res.html ? parseAuctionPage(res.html, sale, page) : null; | |
| 102 | + } | |
| 103 | +} | |
| 104 | + | |
| 105 | +export default function createConnector(meta: ConnectorMeta) { | |
| 106 | + return new AsteBolaffiConnector(meta); | |
| 107 | +} | |
added
connectors/api/aste-bolaffi/meta.json
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +{ | |
| 2 | + "id": "aste-bolaffi", | |
| 3 | + "displayName": "Aste Bolaffi (Turin) — risultati", | |
| 4 | + "sourceId": "aste-bolaffi", | |
| 5 | + "sourceName": "Aste Bolaffi", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.astebolaffi.it", | |
| 8 | + "module": "api/aste-bolaffi", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["stamps", "coins", "banknotes", "medals", "other_watches", "rolex", "omega", "patek_philippe", "wine", "whisky", "automobiles", "motorcycles", "automotive_memorabilia", "books", "maps", "historical_documents", "autographs", "movie_posters", "comics", "vintage_toys", "sports_memorabilia", "jewelry", "art", "antiques"], | |
| 11 | + "regions": ["IT"], | |
| 12 | + "languages": ["it", "en"], | |
| 13 | + "currency": ["EUR"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "medium", | |
| 23 | + "trustScore": 0.88, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.astebolaffi.it/cms/termini-e-condizioni", | |
| 26 | + "acquisitionMethod": "server-rendered HTML (Laravel)", | |
| 27 | + "historicalDepth": "years", | |
| 28 | + "accessNotes": "Aste Bolaffi (Turin; Gruppo Bolaffi — stamps, coins, banknotes, watches, wine & spirits, classic cars, posters, comics, memorabilia). Public pages read: /it/results (113 past auctions: date DD.MM.YYYY, city, /it/auction/<id>) and /it/auction/<id>?page=N (10 lots per page: 'Lotto N', title, description, image, 'Base asta: € 5 | Aggiudicato a : € 5'; the sale date is in getLocationDate('YYYY-MM-DD …')). 'Aggiudicato a' (price knocked down) is not labelled hammer vs fee-inclusive on the public page → buyer_premium_included null. robots.txt: Allow all; termini e condizioni contain no clause on automated access. 3 s politeness (10 lots/page → many pages); salesPerRun caps incremental runs; resumable backfill over the results list. No login, no bidder data.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "resultsUrl": "https://www.astebolaffi.it/it/results", | |
| 33 | + "salesPerRun": 1 | |
| 34 | + } | |
| 35 | +} | |
added
connectors/api/auctionet/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# auctionet — Auctionet (Nordic/European online auction platform) | |
| 2 | + | |
| 3 | +Public, unauthenticated JSON endpoint `https://auctionet.com/api/v2/items.json` (`is=ended`, `per_page`, `page`, | |
| 4 | +`category_id`, `country_code`), plus `/api/v2/categories.json` for the taxonomy tree. ~100 houses | |
| 5 | +(Stockholms Auktionsverk, Crafoord, Uppsala Auktionskammare, Bruun Rasmussen online, Helsingborgs…), 3.9 M closed | |
| 6 | +lots since 2011, native SEK / EUR / DKK / GBP. | |
| 7 | + | |
| 8 | +- **Records**: `sale` for `state=sold` (price = winning bid = hammer; `buyerPremiumIncluded=false`, each house adds | |
| 9 | + its own buyer's fee — `metadata.price_basis`), `auction_lot` (status `ended`) for `unsold`. | |
| 10 | +- **Incremental**: pages through the most recently ended lots until the previous checkpoint (`sinceEndsAt`) is met; | |
| 11 | + `incrementalMaxPages` caps a run. **Backfill**: category × country slices (the API caps every query at | |
| 12 | + 10 000 items → `apiItemCap / perPage` pages per slice), cursor `{sliceIndex, page, itemsProcessed}`, `progress()` | |
| 13 | + per page, `{done:true}` at the end. | |
| 14 | +- **Taxonomy**: `CATEGORY_MAP` (Auctionet id → slug + department hint) refined by `_auction-lib` title rules and | |
| 15 | + multilingual cues (sv/de/da/fi/es). Licence weapons, firearms, vehicle parts, boats, mopeds, bicycles, garden, | |
| 16 | + tools and consumer electronics are skipped. | |
| 17 | +- **Identifiers**: `auctionet_item_id` (deterministic, unique per lot). | |
| 18 | +- Tests: `pnpm vitest run connectors/api/auctionet` · fixtures: `pnpm tsx connectors/api/auctionet/_capture.ts --save`. | |
added
connectors/api/auctionet/_capture.ts
+49 −0
@@ -0,0 +1,49 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture + smoke for the Auctionet connector (public JSON API, no key). | |
| 3 | + * Usage: pnpm tsx connectors/api/auctionet/_capture.ts [--save] [--smoke] | |
| 4 | + */ | |
| 5 | +import { ConnectorMetaSchema, createCrawlContext, createRouter } from '@rareindex/connectors'; | |
| 6 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 7 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 8 | +import createConnector, { parseItemsPage } from './index.js'; | |
| 9 | + | |
| 10 | +const UA = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'application/json' }; | |
| 11 | +const save = process.argv.includes('--save'); | |
| 12 | +const smoke = process.argv.includes('--smoke'); | |
| 13 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 14 | +const connector = createConnector(meta); | |
| 15 | + | |
| 16 | +const captures: Array<{ name: string; url: string; slice: { categoryId: number | null; countryCode: string | null }; page: number; note: string; expect: Record<string, unknown> }> = [ | |
| 17 | + { name: 'ended-page-1', url: 'https://auctionet.com/api/v2/items.json?is=ended&per_page=25&page=1', slice: { categoryId: null, countryCode: null }, page: 1, note: 'Live capture of the public items.json endpoint (is=ended, 25 most recently closed lots, mixed houses/currencies). Trimmed with trimItem().', expect: { minCount: 10, kinds: ['sale', 'auction_lot'], requiredFields: ['attributes.categorySlug', 'attributes.identifiers.auctionet_item_id'] } }, | |
| 18 | + { name: 'wristwatches-de-eur', url: 'https://auctionet.com/api/v2/items.json?is=ended&per_page=25&page=1&category_id=15&country_code=DE', slice: { categoryId: 15, countryCode: 'DE' }, page: 1, note: 'Live capture: backfill slice Wristwatches (category 15) × Germany (EUR, German titles) — exercises watch brand/reference parsing and Konvolut bundle detection.', expect: { minCount: 5, kinds: ['sale', 'auction_lot'] } }, | |
| 19 | + { name: 'search-with-unsold', url: 'https://auctionet.com/api/v2/items.json?is=ended&per_page=25&q=1955+SEK', slice: { categoryId: null, countryCode: null }, page: 1, note: 'Live capture of an ended search page containing unsold lots (state=unsold, no bids) → auction_lot records with status ended; edge case for the sold/unsold split.', expect: { minCount: 3, kinds: ['sale', 'auction_lot'] } }, | |
| 20 | +]; | |
| 21 | + | |
| 22 | +for (const c of captures) { | |
| 23 | + const res = await fetch(c.url, { headers: UA }); | |
| 24 | + if (!res.ok) throw new Error(`${c.url}: HTTP ${res.status}`); | |
| 25 | + const json = await res.json(); | |
| 26 | + const payload = parseItemsPage(json, c.url, c.page, c.slice); | |
| 27 | + if (!payload) throw new Error(`parse failed ${c.url}`); | |
| 28 | + const raw = { url: c.url, externalId: `capture:${c.name}`, kind: 'sale' as const, engine: 'api' as const, fetchedAt: new Date(), payload }; | |
| 29 | + const out = await connector.normalize(raw); | |
| 30 | + const sales = out.filter((r) => r.kind === 'sale'); | |
| 31 | + const lots = out.filter((r) => r.kind === 'auction_lot'); | |
| 32 | + console.log(`${c.name}: ${payload.items.length} items → ${out.length} records (${sales.length} sales, ${lots.length} auction_lot); states=${JSON.stringify([...new Set(payload.items.map((i) => i.state))])} currencies=${JSON.stringify([...new Set(payload.items.map((i) => i.currency))])}`); | |
| 33 | + for (const r of out.slice(0, 2)) console.log(' ', JSON.stringify(r).slice(0, 300)); | |
| 34 | + if (save) saveFixture(meta.id, c.name, { raw, expect: c.expect as never, note: c.note }); | |
| 35 | + await new Promise((r) => setTimeout(r, 1500)); | |
| 36 | +} | |
| 37 | + | |
| 38 | +if (smoke) { | |
| 39 | + const router = createRouter({}); | |
| 40 | + const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2 } }); | |
| 41 | + let raws = 0; | |
| 42 | + let recs = 0; | |
| 43 | + for await (const raw of connector.crawl(ctx)) { | |
| 44 | + raws++; | |
| 45 | + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); | |
| 46 | + recs += out.length; | |
| 47 | + } | |
| 48 | + console.log(`smoke: raw=${raws} normalized=${recs} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)}`); | |
| 49 | +} | |
added
connectors/api/auctionet/index.test.ts
+193 −0
@@ -0,0 +1,193 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 3 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 4 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 5 | +import createConnector, { categoryFor, leafCategoryIds, normalizeItem, parseItemsPage, trimItem } from './index.js'; | |
| 6 | +import { isBundleMultilingual, parenCount, parseEuDate, parseEuMoney, yearFromTitle } from '../_g8-auctions-eu-apac-lib/index.js'; | |
| 7 | + | |
| 8 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 9 | +const connector = createConnector(meta); | |
| 10 | + | |
| 11 | +const API_ITEM = { | |
| 12 | + id: 5224154, | |
| 13 | + catalog_number: null, | |
| 14 | + auction_id: 9524392, | |
| 15 | + currency: 'EUR', | |
| 16 | + reserve_met: true, | |
| 17 | + reserve_amount: 200, | |
| 18 | + estimate: 450, | |
| 19 | + upper_estimate: null, | |
| 20 | + starting_bid_amount: 25, | |
| 21 | + hammered: false, | |
| 22 | + state: 'sold', | |
| 23 | + title: 'CORTÉBERT, WALTHAM, IMMERFORT ETC., Armbanduhren Konvolut, 20. Jahrhundert. (4).', | |
| 24 | + description: '<p>Konvolut bestehend aus vier Armbanduhren: </p>\n\n<p>Cortébert, Handaufzug, 1950er Jahre.</p>', | |
| 25 | + condition: '<p>Altersgemäße Gebrauchsspuren.</p>', | |
| 26 | + company_id: 237, | |
| 27 | + category_id: 15, | |
| 28 | + ends_at: 1788854835, | |
| 29 | + published_at: 1788164174, | |
| 30 | + type: 'online', | |
| 31 | + location: 'Cologne', | |
| 32 | + house: 'Stockholms Auktionsverk Köln', | |
| 33 | + placement: 'auf Anfrage', | |
| 34 | + url: 'https://auctionet.com/en/5224154-cortebert-waltham-immerfort-etc-wristwatch-collection-20th-century-4', | |
| 35 | + images: [{ thumb: 't.jpg', mini: 'm.jpg', w640: 'https://images.auctionet.com/thumbs/w640_item_5224154_x.jpg', hd: 'hd.jpg' }], | |
| 36 | + bids: [ | |
| 37 | + { id: 42504089, bidder: 7, amount: 220, your_bid: false, reserve_met: true, auto: false, timestamp: 1788854655 }, | |
| 38 | + { id: 42502618, bidder: 4, amount: 200, your_bid: false, reserve_met: true, auto: false, timestamp: 1788846567 }, | |
| 39 | + ], | |
| 40 | +}; | |
| 41 | + | |
| 42 | +describe('auctionet', () => { | |
| 43 | + runFixtureSuite(connector, it, expect); | |
| 44 | + | |
| 45 | + it('fixtures: sold lots become hammer-basis sales in native currency, unsold lots become ended auction_lots', async () => { | |
| 46 | + let sales = 0; | |
| 47 | + let lots = 0; | |
| 48 | + for (const name of listFixtures('auctionet')) { | |
| 49 | + const fx = loadFixture('auctionet', name); | |
| 50 | + const out = await connector.normalize(fx.raw); | |
| 51 | + for (const r of out) { | |
| 52 | + if (!('attributes' in r)) continue; | |
| 53 | + expect(r.attributes.identifiers.auctionet_item_id).toMatch(/^\d+$/); | |
| 54 | + expect(r.sourceUrl).toMatch(/^https:\/\/auctionet\.com\/en\/(events\/[^/]+\/)?\d+-/); | |
| 55 | + if (r.kind === 'sale') { | |
| 56 | + sales++; | |
| 57 | + expect(['SEK', 'EUR', 'DKK', 'GBP']).toContain(r.currency); | |
| 58 | + expect(r.buyerPremiumIncluded).toBe(false); | |
| 59 | + expect(r.attributes.metadata.price_basis).toBe('winning_bid_excl_buyer_fee'); | |
| 60 | + expect(r.auctionHouse).toBeTruthy(); | |
| 61 | + expect(r.saleDate.getUTCFullYear()).toBeGreaterThanOrEqual(2011); | |
| 62 | + } else if (r.kind === 'auction_lot') { | |
| 63 | + lots++; | |
| 64 | + expect(r.status).toBe('ended'); | |
| 65 | + } | |
| 66 | + } | |
| 67 | + } | |
| 68 | + expect(sales).toBeGreaterThan(20); | |
| 69 | + expect(lots).toBeGreaterThan(0); | |
| 70 | + }); | |
| 71 | + | |
| 72 | + it('trims the API item and normalises it (winning bid = hammer, EUR, bundle of 4)', () => { | |
| 73 | + const item = trimItem(API_ITEM); | |
| 74 | + expect(item).toMatchObject({ id: 5224154, currency: 'EUR', state: 'sold', winningBid: 220, winningBidAt: 1788854655, bidCount: 2, house: 'Stockholms Auktionsverk Köln', location: 'Cologne', categoryId: 15 }); | |
| 75 | + expect(item.images).toEqual(['https://images.auctionet.com/thumbs/w640_item_5224154_x.jpg']); | |
| 76 | + expect(item.description).toBe('Konvolut bestehend aus vier Armbanduhren: Cortébert, Handaufzug, 1950er Jahre.'); | |
| 77 | + const rec = normalizeItem(meta, item, new Date('2026-09-08T10:00:00Z')); | |
| 78 | + expect(rec?.kind).toBe('sale'); | |
| 79 | + if (rec?.kind !== 'sale') return; | |
| 80 | + expect(rec.price).toBe(220); | |
| 81 | + expect(rec.currency).toBe('EUR'); | |
| 82 | + expect(rec.saleDate.toISOString()).toBe('2026-09-08T08:07:15.000Z'); | |
| 83 | + expect(rec.isBundle).toBe(true); | |
| 84 | + expect(rec.attributes.categorySlug).toBe('other_watches'); | |
| 85 | + expect(rec.attributes.year).toBeNull(); // "20. Jahrhundert", "(4)" — no year | |
| 86 | + expect(rec.buyerPremiumIncluded).toBe(false); | |
| 87 | + expect(rec.lotNumber).toBeNull(); | |
| 88 | + expect(rec.condition.conditionRaw).toBe('Altersgemäße Gebrauchsspuren.'); | |
| 89 | + }); | |
| 90 | + | |
| 91 | + it('unsold and out-of-taxonomy items', () => { | |
| 92 | + const unsold = trimItem({ ...API_ITEM, id: 1, state: 'unsold', bids: [], category_id: 128, title: 'MYNT, 5 kronor, Sverige 1955.' }); | |
| 93 | + const rec = normalizeItem(meta, unsold, new Date()); | |
| 94 | + expect(rec?.kind).toBe('auction_lot'); | |
| 95 | + if (rec?.kind === 'auction_lot') { | |
| 96 | + expect(rec.status).toBe('ended'); | |
| 97 | + expect(rec.currentBid).toBeNull(); | |
| 98 | + expect(rec.estimateLow).toBe(450); | |
| 99 | + expect(rec.attributes.categorySlug).toBe('coins'); | |
| 100 | + expect(rec.attributes.year).toBe(1955); | |
| 101 | + } | |
| 102 | + const weapon = trimItem({ ...API_ITEM, id: 2, category_id: 61, title: 'GEVÄR, Husqvarna.' }); | |
| 103 | + expect(normalizeItem(meta, weapon, new Date())).toBeNull(); | |
| 104 | + const unknownCurrency = trimItem({ ...API_ITEM, id: 3, currency: 'XXX' }); | |
| 105 | + expect(normalizeItem(meta, unknownCurrency, new Date())).toBeNull(); | |
| 106 | + }); | |
| 107 | + | |
| 108 | + it('parses a page and pagination metadata', () => { | |
| 109 | + const p = parseItemsPage({ items: [API_ITEM], pagination: { current_page: 1, total_pages: 50, total_entries: 3917226 } }, 'https://auctionet.com/api/v2/items.json?is=ended', 1, { categoryId: null, countryCode: null }); | |
| 110 | + expect(p?.items).toHaveLength(1); | |
| 111 | + expect(p?.totalEntries).toBe(3917226); | |
| 112 | + expect(parseItemsPage({ error: 'x' }, 'u', 1, { categoryId: null, countryCode: null })).toBeNull(); | |
| 113 | + }); | |
| 114 | + | |
| 115 | + it('maps Auctionet categories + multilingual titles to taxonomy slugs', () => { | |
| 116 | + expect(categoryFor(15, 'ARMBANDSUR, Rolex, Datejust, ref 16233, stål och guld.').slug).toBe('rolex'); | |
| 117 | + expect(categoryFor(15, 'ARMBANDSUR, Omega, Seamaster, stål, dam, manuell.').slug).toBe('omega'); | |
| 118 | + expect(categoryFor(110, 'FICKUR, silver, 1800-talets slut.').slug).toBe('other_watches'); | |
| 119 | + expect(categoryFor(128, 'ÄLDRE SVENSKA SEDLAR, 49 st, 1960-80-tal.').slug).toBe('banknotes'); | |
| 120 | + expect(categoryFor(128, 'MYNT, 1 riksdaler 1776.').slug).toBe('coins'); | |
| 121 | + expect(categoryFor(135, 'ORDEN, Vasaorden, riddartecken, guld.').slug).toBe('medals'); | |
| 122 | + expect(categoryFor(136, 'FRIMÄRKEN, samling i album.').slug).toBe('stamps'); | |
| 123 | + expect(categoryFor(19, 'SOFFBORD, Bruno Mathsson, Karl Mathsson, 1960-tal.').slug).toBe('antiques'); | |
| 124 | + expect(categoryFor(18, 'HANS J. WEGNER, armchair "The Chair", Johannes Hansen, Denmark.').slug).toBe('design_furniture'); | |
| 125 | + expect(categoryFor(28, 'OLJEMÅLNING, signerad, landskap.').slug).toBe('art'); | |
| 126 | + expect(categoryFor(26, 'FOTOGRAFI, gelatin silver print.').slug).toBe('photography'); | |
| 127 | + expect(categoryFor(170, 'MACALLAN 18 Years Old Sherry Oak, 70 cl.').slug).toBe('whisky'); | |
| 128 | + expect(categoryFor(170, 'CHÂTEAU MARGAUX 1990, 1 flaska.').slug).toBe('wine'); | |
| 129 | + expect(categoryFor(49, 'HERMÈS, handväska "Birkin 35", Togo.').slug).toBe('luxury_handbags'); | |
| 130 | + expect(categoryFor(268, 'POKÉMON, Charizard, Base Set, holo.').slug).toBe('pokemon'); | |
| 131 | + expect(categoryFor(211, 'SERIETIDNINGAR, Kalle Anka & Co, 1950-tal, 12 st.').slug).toBe('independent_comics'); | |
| 132 | + expect(categoryFor(275, 'MODELLBILAR, Dinky Toys, 5 st.').slug).toBe('model_cars'); | |
| 133 | + expect(categoryFor(215, 'VOLVO P1800 S, 1966.').slug).toBe('automobiles'); | |
| 134 | + expect(categoryFor(61, 'GEVÄR').slug).toBeNull(); | |
| 135 | + expect(categoryFor(253, 'MOTOR').slug).toBeNull(); | |
| 136 | + expect(categoryFor(999, 'Random unknown thing').slug).toBeNull(); | |
| 137 | + expect(categoryFor(999, 'LEICA M3 camera body').slug).toBe('cameras'); | |
| 138 | + expect(leafCategoryIds()).not.toContain(16); // parent "Furniture" | |
| 139 | + expect(leafCategoryIds()).toContain(19); | |
| 140 | + expect(leafCategoryIds()).toContain(42); // Mirrors (childless root) | |
| 141 | + expect(leafCategoryIds()).not.toContain(61); // licence weapons | |
| 142 | + }); | |
| 143 | + | |
| 144 | + it('shared multilingual helpers', () => { | |
| 145 | + expect(parseEuDate('12 juin 2026')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 146 | + expect(parseEuDate('jeudi 09 juillet 2026 14:00')?.toISOString()).toBe('2026-07-09T00:00:00.000Z'); | |
| 147 | + expect(parseEuDate('12. Juni 2026')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 148 | + expect(parseEuDate('1. März 2025')?.toISOString()).toBe('2025-03-01T00:00:00.000Z'); | |
| 149 | + expect(parseEuDate('12 giugno 2026')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 150 | + expect(parseEuDate('12 de junio de 2026')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 151 | + expect(parseEuDate('den 12 juni 2026')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 152 | + expect(parseEuDate('22–23 juin 2026')?.toISOString()).toBe('2026-06-22T00:00:00.000Z'); | |
| 153 | + expect(parseEuDate('June 12, 2026')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 154 | + expect(parseEuDate('12.06.2026')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 155 | + expect(parseEuDate('2026-06-12T14:00:00Z')?.toISOString()).toBe('2026-06-12T00:00:00.000Z'); | |
| 156 | + expect(parseEuDate('12 joulukuuta 2025')?.toISOString()).toBe('2025-12-12T00:00:00.000Z'); | |
| 157 | + expect(parseEuDate('nonsense')).toBeNull(); | |
| 158 | + | |
| 159 | + expect(parseEuMoney('1.250,00 €', 'EUR')).toMatchObject({ amount: 1250, currency: 'EUR' }); | |
| 160 | + expect(parseEuMoney('1 250 €', 'EUR')).toMatchObject({ amount: 1250, currency: 'EUR' }); | |
| 161 | + expect(parseEuMoney('1 081 EUR', 'EUR')).toMatchObject({ amount: 1081, currency: 'EUR' }); | |
| 162 | + expect(parseEuMoney('593EUR', 'EUR')).toMatchObject({ amount: 593, currency: 'EUR' }); | |
| 163 | + expect(parseEuMoney('€ 12.500', 'EUR')).toMatchObject({ amount: 12500, currency: 'EUR' }); | |
| 164 | + expect(parseEuMoney("CHF 1'250", 'CHF', 'ch')).toMatchObject({ amount: 1250, currency: 'CHF' }); | |
| 165 | + expect(parseEuMoney('12 000 SEK', 'SEK')).toMatchObject({ amount: 12000, currency: 'SEK' }); | |
| 166 | + expect(parseEuMoney('4 000 kr', 'SEK')).toMatchObject({ amount: 4000, currency: 'SEK' }); | |
| 167 | + expect(parseEuMoney('£1,250', 'GBP', 'en')).toMatchObject({ amount: 1250, currency: 'GBP' }); | |
| 168 | + expect(parseEuMoney('HK$1,250,000', 'HKD', 'en')).toMatchObject({ amount: 1_250_000, currency: 'HKD' }); | |
| 169 | + expect(parseEuMoney('¥1,250,000', 'JPY', 'en')).toMatchObject({ amount: 1_250_000, currency: 'JPY' }); | |
| 170 | + expect(parseEuMoney('AU$2,400', 'AUD', 'en')).toMatchObject({ amount: 2400, currency: 'AUD' }); | |
| 171 | + expect(parseEuMoney('1.250', 'EUR')).toMatchObject({ amount: 1250 }); | |
| 172 | + expect(parseEuMoney('1.25', 'EUR')).toMatchObject({ amount: 1.25 }); | |
| 173 | + expect(parseEuMoney('n/a', 'EUR')).toBeNull(); | |
| 174 | + expect(parseEuMoney('1 250', null)).toBeNull(); | |
| 175 | + | |
| 176 | + expect(isBundleMultilingual('Armbanduhren Konvolut (4)')).toBe(true); | |
| 177 | + expect(isBundleMultilingual('SEDLAR, 49 st')).toBe(true); | |
| 178 | + expect(isBundleMultilingual('Lot de trois montres')).toBe(true); | |
| 179 | + expect(isBundleMultilingual('Lotto di sei tazze')).toBe(true); | |
| 180 | + expect(isBundleMultilingual('SÄNGBORD, ett par')).toBe(false); | |
| 181 | + expect(isBundleMultilingual('ARMBANDSUR, Rolex, Datejust')).toBe(false); | |
| 182 | + expect(parenCount('LJUSSTAKAR, ett par, silver. (2).')).toBe(2); | |
| 183 | + expect(parenCount('12 st glas')).toBe(12); | |
| 184 | + | |
| 185 | + expect(yearFromTitle('SÄNGBORD, ett par, 1900/2000-tal')).toBeNull(); | |
| 186 | + expect(yearFromTitle('SOFFBORD, 1960-tal')).toBeNull(); | |
| 187 | + expect(yearFromTitle('Cortébert, 1950er Jahre')).toBeNull(); | |
| 188 | + expect(yearFromTitle('Rolex Datejust, 1980s')).toBeNull(); | |
| 189 | + expect(yearFromTitle('VOLVO P1800 S, 1966.')).toBe(1966); | |
| 190 | + expect(yearFromTitle('MYNT, 1 riksdaler 1776.')).toBe(1776); | |
| 191 | + expect(yearFromTitle('CHÂTEAU MARGAUX 1990')).toBe(1990); | |
| 192 | + }); | |
| 193 | +}); | |
added
connectors/api/auctionet/index.ts
+390 −0
@@ -0,0 +1,390 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, adapters, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 4 | +import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 5 | +import { brandFromSlug, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js'; | |
| 6 | +import { fromUnix, isBundleMultilingual, isSupportedCurrency, parenCount, stripHtml, yearFromTitle } from '../_g8-auctions-eu-apac-lib/index.js'; | |
| 7 | + | |
| 8 | +/** | |
| 9 | + * Auctionet — public JSON API (https://auctionet.com/api/v2/items.json). One raw record per API page | |
| 10 | + * (trimmed items); normalise → `sale` (winning bid = hammer, native SEK/EUR/DKK/GBP) for sold lots and | |
| 11 | + * `auction_lot` (status ended) for unsold ones. Titles are in the consigning house's language | |
| 12 | + * (sv/de/da/fi/es/en); taxonomy comes from Auctionet's category tree + title keywords. | |
| 13 | + */ | |
| 14 | + | |
| 15 | +const PARSER_VERSION = '1.0.0'; | |
| 16 | + | |
| 17 | +export const ItemSchema = z.object({ | |
| 18 | + id: z.number(), | |
| 19 | + catalogNumber: z.string().nullable(), | |
| 20 | + auctionId: z.number().nullable(), | |
| 21 | + currency: z.string(), | |
| 22 | + estimate: z.number().nullable(), | |
| 23 | + upperEstimate: z.number().nullable(), | |
| 24 | + reserveMet: z.boolean().nullable(), | |
| 25 | + state: z.string(), | |
| 26 | + hammered: z.boolean().nullable(), | |
| 27 | + title: z.string(), | |
| 28 | + description: z.string().nullable(), | |
| 29 | + condition: z.string().nullable(), | |
| 30 | + companyId: z.number().nullable(), | |
| 31 | + categoryId: z.number().nullable(), | |
| 32 | + endsAt: z.number().nullable(), | |
| 33 | + publishedAt: z.number().nullable(), | |
| 34 | + type: z.string().nullable(), | |
| 35 | + location: z.string().nullable(), | |
| 36 | + house: z.string().nullable(), | |
| 37 | + url: z.string(), | |
| 38 | + images: z.array(z.string()), | |
| 39 | + winningBid: z.number().nullable(), | |
| 40 | + winningBidAt: z.number().nullable(), | |
| 41 | + bidCount: z.number(), | |
| 42 | +}); | |
| 43 | +export type Item = z.infer<typeof ItemSchema>; | |
| 44 | + | |
| 45 | +export const PagePayloadSchema = z.object({ | |
| 46 | + kind: z.literal('items_page'), | |
| 47 | + url: z.string(), | |
| 48 | + page: z.number(), | |
| 49 | + totalEntries: z.number().nullable(), | |
| 50 | + slice: z.object({ categoryId: z.number().nullable(), countryCode: z.string().nullable() }), | |
| 51 | + items: z.array(ItemSchema), | |
| 52 | +}); | |
| 53 | +export type PagePayload = z.infer<typeof PagePayloadSchema>; | |
| 54 | + | |
| 55 | +type ApiItem = Record<string, unknown>; | |
| 56 | + | |
| 57 | +function n(v: unknown): number | null { | |
| 58 | + if (v === null || v === undefined || v === '') return null; | |
| 59 | + const x = typeof v === 'number' ? v : Number(v); | |
| 60 | + return Number.isFinite(x) ? x : null; | |
| 61 | +} | |
| 62 | + | |
| 63 | +/** Keep only what normalize() needs; drops bidder numbers, suggested bids, placement, thumbnails. */ | |
| 64 | +export function trimItem(i: ApiItem): Item { | |
| 65 | + const bids = Array.isArray(i.bids) ? (i.bids as Array<{ amount?: unknown; timestamp?: unknown }>) : []; | |
| 66 | + const amounts = bids.map((b) => n(b.amount) ?? 0); | |
| 67 | + const top = amounts.length ? Math.max(...amounts) : null; | |
| 68 | + const topBid = bids.find((b) => n(b.amount) === top); | |
| 69 | + const images = Array.isArray(i.images) ? (i.images as Array<{ w640?: string; hd?: string }>).map((im) => im.w640 ?? im.hd ?? '').filter(Boolean).slice(0, 3) : []; | |
| 70 | + return ItemSchema.parse({ | |
| 71 | + id: Number(i.id), | |
| 72 | + catalogNumber: i.catalog_number !== null && i.catalog_number !== undefined ? String(i.catalog_number) : null, | |
| 73 | + auctionId: n(i.auction_id), | |
| 74 | + currency: String(i.currency ?? ''), | |
| 75 | + estimate: n(i.estimate), | |
| 76 | + upperEstimate: n(i.upper_estimate), | |
| 77 | + reserveMet: typeof i.reserve_met === 'boolean' ? i.reserve_met : null, | |
| 78 | + state: String(i.state ?? ''), | |
| 79 | + hammered: typeof i.hammered === 'boolean' ? i.hammered : null, | |
| 80 | + title: String(i.title ?? '').trim(), | |
| 81 | + description: stripHtml(typeof i.description === 'string' ? i.description : null, 600), | |
| 82 | + condition: stripHtml(typeof i.condition === 'string' ? i.condition : null, 300), | |
| 83 | + companyId: n(i.company_id), | |
| 84 | + categoryId: n(i.category_id), | |
| 85 | + endsAt: n(i.ends_at), | |
| 86 | + publishedAt: n(i.published_at), | |
| 87 | + type: typeof i.type === 'string' ? i.type : null, | |
| 88 | + location: typeof i.location === 'string' && i.location ? i.location : null, | |
| 89 | + house: typeof i.house === 'string' && i.house ? i.house : null, | |
| 90 | + url: String(i.url ?? ''), | |
| 91 | + images, | |
| 92 | + winningBid: top && top > 0 ? top : null, | |
| 93 | + winningBidAt: topBid ? n(topBid.timestamp) : null, | |
| 94 | + bidCount: bids.length, | |
| 95 | + }); | |
| 96 | +} | |
| 97 | + | |
| 98 | +export function parseItemsPage(json: unknown, url: string, page: number, slice: { categoryId: number | null; countryCode: string | null }): PagePayload | null { | |
| 99 | + const j = json as { items?: ApiItem[]; pagination?: { total_entries?: number; total_pages?: number } } | null; | |
| 100 | + if (!j || !Array.isArray(j.items)) return null; | |
| 101 | + return { kind: 'items_page', url, page, totalEntries: n(j.pagination?.total_entries), slice, items: j.items.filter((i) => i && typeof i === 'object' && i.id !== undefined).map(trimItem) }; | |
| 102 | +} | |
| 103 | + | |
| 104 | +/** | |
| 105 | + * Auctionet category id → default taxonomy slug + department hint for title refinement. | |
| 106 | + * `null` slug = out of taxonomy / regulated (licence weapons, firearms, vehicle parts…) → item skipped. | |
| 107 | + * Ids captured live from /api/v2/categories.json (2026-09-08); unknown ids fall back to keyword sweep. | |
| 108 | + */ | |
| 109 | +export const CATEGORY_MAP: Record<number, { slug: string | null; hint: DeptHint; path: string }> = { | |
| 110 | + 25: { slug: 'art', hint: 'art', path: 'Art' }, 119: { slug: 'art', hint: 'art', path: 'Art > Drawings' }, 27: { slug: 'art', hint: 'prints', path: 'Art > Engravings & Prints' }, 30: { slug: 'art', hint: 'art', path: 'Art > Other' }, 28: { slug: 'art', hint: 'art', path: 'Art > Paintings' }, 26: { slug: 'photography', hint: 'photographs', path: 'Art > Photography' }, 29: { slug: 'art', hint: 'art', path: 'Art > Sculptures & Bronzes' }, | |
| 111 | + 117: { slug: 'antiques', hint: 'asian', path: 'Asiatica' }, 319: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Bronzes' }, 325: { slug: 'porcelain', hint: 'ceramics', path: 'Asiatica > Ceramics & Porcelain' }, 323: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Cloisonné' }, 320: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Jade' }, 355: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Other' }, 321: { slug: 'art', hint: 'asian', path: 'Asiatica > Scrolls' }, 322: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Textiles' }, 324: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Wood' }, | |
| 112 | + 50: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts' }, 206: { slug: 'historical_documents', hint: 'books', path: 'Books, Maps & Manuscripts > Autographs & Manuscripts' }, 204: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts > Books' }, 205: { slug: 'maps', hint: 'books', path: 'Books, Maps & Manuscripts > Maps' }, 207: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts > Other' }, | |
| 113 | + 35: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles' }, 36: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Carpets' }, 285: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > European' }, 287: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Oriental' }, 286: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Persian' }, 37: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Textiles' }, | |
| 114 | + 9: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain' }, 10: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > European' }, 11: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Oriental' }, 12: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Rest of the world' }, 210: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Tableware' }, | |
| 115 | + 31: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches' }, 258: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Carriage & Miniature Clocks' }, 32: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Longcase clocks' }, 33: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Mantel clocks' }, 34: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Other clocks' }, 110: { slug: 'other_watches', hint: 'watches', path: 'Clocks & Watches > Pocket & Stop Watches' }, 127: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Wall Clocks' }, 15: { slug: 'other_watches', hint: 'watches', path: 'Clocks & Watches > Wristwatches' }, | |
| 116 | + 46: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps' }, 128: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps > Coins & Banknotes' }, 135: { slug: 'medals', hint: 'militaria', path: 'Coins, Medals & Stamps > Orders & Medals' }, 131: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps > Other' }, 136: { slug: 'stamps', hint: 'stamps', path: 'Coins, Medals & Stamps > Stamps' }, | |
| 117 | + 261: { slug: 'antiques', hint: 'popular_culture', path: 'Collectables' }, 262: { slug: 'advertising', hint: 'popular_culture', path: 'Collectables > Advertising & Signs' }, 269: { slug: 'music', hint: 'music', path: 'Collectables > Audio, Vinyl & Hi-Fi' }, 268: { slug: 'trading_cards', hint: 'cards', path: 'Collectables > Collectible trading cards' }, 54: { slug: 'sports_memorabilia', hint: 'sports', path: 'Collectables > Fishing equipment' }, 265: { slug: 'movie_memorabilia', hint: 'movies', path: 'Collectables > Movie memorabilia' }, 266: { slug: 'music_memorabilia', hint: 'music', path: 'Collectables > Music memorabilia' }, 51: { slug: 'musical_instruments', hint: 'music', path: 'Collectables > Musical instruments' }, 267: { slug: 'antiques', hint: 'popular_culture', path: 'Collectables > Other collectables' }, 263: { slug: 'pens', hint: 'pens', path: 'Collectables > Pens' }, 264: { slug: 'sports_memorabilia', hint: 'sports', path: 'Collectables > Sports memorabilia' }, 45: { slug: 'scientific_instruments', hint: 'science', path: 'Collectables > Technica & Nautica' }, | |
| 118 | + 134: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica' }, 283: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > African tribal art' }, 284: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > Other' }, 282: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > Sami Arts & Crafts' }, | |
| 119 | + 16: { slug: 'antiques', hint: 'furniture', path: 'Furniture' }, 18: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Armchairs & Chairs' }, 24: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Chests of drawers' }, 280: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Coffee Tables' }, 23: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Cupboards, Cabinets & Shelves' }, 279: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Desks' }, 22: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Dining room furniture' }, 281: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Dining tables' }, 17: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Other' }, 20: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Sofas & Seatings' }, 19: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Tables' }, | |
| 120 | + 270: { slug: null, hint: 'unknown', path: 'Garden & Architectural' }, 272: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Architectural details' }, 21: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Garden' }, 271: { slug: 'antiques', hint: 'furniture', path: 'Garden & Architectural > Garden Sculptures & Urns' }, 273: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Other' }, | |
| 121 | + 6: { slug: 'glass_crystal', hint: 'glass', path: 'Glass' }, 208: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Art glass' }, 8: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Other' }, 7: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Tableware' }, 209: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Utility glass' }, | |
| 122 | + 13: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones' }, 106: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Bracelets' }, 107: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Brooches & Pendants' }, 259: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Costume Jewellery' }, 111: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Cufflinks & Tie Pins' }, 115: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Earrings' }, 48: { slug: 'gemstones', hint: 'jewelry', path: 'Jewellery & Gemstones > Gemstones' }, 14: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Jewellery' }, 109: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Jewellery Suites' }, 104: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Necklace' }, 118: { slug: 'antiques', hint: 'silver', path: 'Jewellery & Gemstones > Objet de vertu & Miscellaneous' }, 112: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Rings' }, 108: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Tiara' }, | |
| 123 | + 59: { slug: null, hint: 'unknown', path: 'Licence weapons' }, 64: { slug: null, hint: 'unknown', path: 'Licence weapons > Airguns' }, 63: { slug: null, hint: 'unknown', path: 'Licence weapons > Combi/Combo' }, 60: { slug: null, hint: 'unknown', path: 'Licence weapons > Double express rifles' }, 70: { slug: null, hint: 'unknown', path: 'Licence weapons > Drilling' }, 65: { slug: null, hint: 'unknown', path: 'Licence weapons > Military weapons' }, 69: { slug: null, hint: 'unknown', path: 'Licence weapons > Other weapons' }, 67: { slug: null, hint: 'unknown', path: 'Licence weapons > Pistols' }, 68: { slug: null, hint: 'unknown', path: 'Licence weapons > Revolvers' }, 61: { slug: null, hint: 'unknown', path: 'Licence weapons > Rifles' }, 62: { slug: null, hint: 'unknown', path: 'Licence weapons > Shotguns' }, | |
| 124 | + 1: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps' }, 4: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Candlesticks' }, 3: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Ceiling lights' }, 203: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Chandeliers' }, 2: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Floor lights' }, 5: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Other lighting' }, 125: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Table Lamps' }, 124: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Wall Lights' }, | |
| 125 | + 42: { slug: 'antiques', hint: 'decorative', path: 'Mirrors' }, | |
| 126 | + 43: { slug: 'antiques', hint: 'unknown', path: 'Miscellaneous' }, 47: { slug: 'antiques', hint: 'unknown', path: 'Miscellaneous > Miscellaneous' }, 133: { slug: null, hint: 'unknown', path: 'Miscellaneous > Modern Tools' }, 52: { slug: null, hint: 'unknown', path: 'Miscellaneous > Modern consumer electronics' }, | |
| 127 | + 57: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses' }, 71: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses > Cameras & accessories' }, 66: { slug: 'scientific_instruments', hint: 'science', path: 'Photo, Cameras & Lenses > Optics' }, 72: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses > Other' }, | |
| 128 | + 38: { slug: 'silver', hint: 'silver', path: 'Silver & Metals' }, 40: { slug: 'antiques', hint: 'decorative', path: 'Silver & Metals > Other metals' }, 41: { slug: 'antiques', hint: 'decorative', path: 'Silver & Metals > Pewter, Brass & Copper' }, 39: { slug: 'silver', hint: 'silver', path: 'Silver & Metals > Silver' }, 213: { slug: 'silver', hint: 'silver', path: 'Silver & Metals > Silver plated' }, | |
| 129 | + 58: { slug: 'antiques', hint: 'furniture', path: 'Swedish Folk Art' }, 121: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Bowls & Boxes' }, 122: { slug: 'antiques', hint: 'furniture', path: 'Swedish Folk Art > Furniture' }, 123: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Other' }, 120: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Tools & Gears' }, | |
| 130 | + 44: { slug: 'vintage_toys', hint: 'toys', path: 'Toys' }, 276: { slug: 'action_figures', hint: 'toys', path: 'Toys > Action figures & Sci-Fi' }, 211: { slug: 'independent_comics', hint: 'comics', path: 'Toys > Comics' }, 274: { slug: 'dolls', hint: 'toys', path: 'Toys > Dolls & Teddybears' }, 275: { slug: 'model_cars', hint: 'toys', path: 'Toys > Model cars' }, 278: { slug: 'model_trains', hint: 'toys', path: 'Toys > Model railways' }, 277: { slug: 'vintage_toys', hint: 'toys', path: 'Toys > Other toys' }, 212: { slug: 'vintage_toys', hint: 'toys', path: 'Toys > Toys' }, | |
| 131 | + 249: { slug: null, hint: 'cars', path: 'Vehicles, Boats & Parts' }, 255: { slug: 'automotive_memorabilia', hint: 'automobilia', path: 'Vehicles, Boats & Parts > Automobilia & Transport' }, 132: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Bicycles' }, 250: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Boats & Accessories' }, 253: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Car parts' }, 215: { slug: 'automobiles', hint: 'cars', path: 'Vehicles, Boats & Parts > Cars' }, 254: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Moped parts' }, 216: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Mopeds' }, 252: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Motorcycle parts' }, 251: { slug: 'motorcycles', hint: 'motorcycles', path: 'Vehicles, Boats & Parts > Motorcycles' }, 256: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Other' }, | |
| 132 | + 49: { slug: 'fashion_streetwear', hint: 'fashion', path: 'Vintage & Designer Fashion' }, | |
| 133 | + 137: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria' }, 257: { slug: null, hint: 'unknown', path: 'Weapons & Militaria > Airguns' }, 138: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Armour & Uniform' }, 130: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Edged weapons' }, 129: { slug: null, hint: 'unknown', path: 'Weapons & Militaria > Guns & Rifles' }, 214: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Other' }, | |
| 134 | + 170: { slug: 'wine', hint: 'wine', path: 'Wine, Port & Spirits' }, | |
| 135 | +}; | |
| 136 | + | |
| 137 | +/** Multilingual title cues that are more reliable than the (English-only) auction-lib sweep. */ | |
| 138 | +const LOCAL_CUES: Array<[RegExp, string]> = [ | |
| 139 | + [/\b(armbandsur|armbanduhr|wristwatch|herrur|damur|reloj de pulsera|rannekello)\b/i, 'other_watches'], | |
| 140 | + [/\b(fickur|taschenuhr|pocket watch|reloj de bolsillo|taskukello)\b/i, 'other_watches'], | |
| 141 | + [/\b(sedlar|sedel|banknote|geldschein|billete)\b/i, 'banknotes'], | |
| 142 | + [/\b(mynt|münze|münzen|coin|moneda|kolikko|ducat|dukat|riksdaler|daler|thaler|taler)\b/i, 'coins'], | |
| 143 | + [/\b(frimärken|frimärke|briefmarke|stamp|sello|postimerkki)\b/i, 'stamps'], | |
| 144 | + [/\b(orden|medalj|medaille|medal|orden och medaljer)\b/i, 'medals'], | |
| 145 | + [/\b(serietidning|serietidningar|comic|comics|tegneserie|sarjakuva)\b/i, 'independent_comics'], | |
| 146 | + [/\b(lp|vinyl|skivor|schallplatte)\b/i, 'music'], | |
| 147 | + [/\b(kamera|camera|leica|hasselblad|rolleiflex)\b/i, 'cameras'], | |
| 148 | + [/\b(whisky|whiskey|bourbon|cognac|armagnac|rom|rhum|rum)\b/i, 'whisky'], | |
| 149 | + [/\b(handväska|handtasche|handbag|bolso|hermès|hermes|chanel|louis vuitton)\b/i, 'luxury_handbags'], | |
| 150 | + [/\b(sneakers|air jordan|nike|yeezy)\b/i, 'sneakers'], | |
| 151 | + [/\blego\b/i, 'lego_sets'], | |
| 152 | + [/\b(pokémon|pokemon|magic: the gathering|yu-gi-oh)\b/i, 'trading_cards'], | |
| 153 | +]; | |
| 154 | + | |
| 155 | +/** Resolve the taxonomy slug for an item; null → skip (regulated/out-of-taxonomy category). */ | |
| 156 | +export function categoryFor(categoryId: number | null, title: string): { slug: string | null; hint: DeptHint; path: string | null } { | |
| 157 | + const c = categoryId !== null ? CATEGORY_MAP[categoryId] : undefined; | |
| 158 | + if (c && c.slug === null) return { slug: null, hint: c.hint, path: c.path }; | |
| 159 | + const hint: DeptHint = c?.hint ?? 'unknown'; | |
| 160 | + let slug: string | null = null; | |
| 161 | + if (hint === 'watches') slug = slugFromTitle(title, 'watches'); | |
| 162 | + else if (hint === 'wine') slug = slugFromTitle(title, 'wine'); | |
| 163 | + else if (hint === 'coins') slug = LOCAL_CUES.find(([re, s]) => re.test(title) && ['banknotes', 'coins', 'medals'].includes(s))?.[1] ?? c?.slug ?? 'coins'; | |
| 164 | + else if (hint === 'toys') slug = /\blego\b/i.test(title) ? 'lego_sets' : c && c.slug !== 'vintage_toys' ? c.slug : slugFromTitle(title, 'toys') ?? 'vintage_toys'; | |
| 165 | + else if (hint === 'cards' || hint === 'comics' || hint === 'popular_culture' || hint === 'fashion' || hint === 'cars') slug = slugFromTitle(title, hint) ?? c?.slug ?? null; | |
| 166 | + else if (hint === 'furniture' || hint === 'decorative') slug = slugFromTitle(title, 'furniture') ?? c?.slug ?? null; | |
| 167 | + else if (hint === 'unknown') slug = LOCAL_CUES.find(([re]) => re.test(title))?.[1] ?? slugFromTitle(title) ?? c?.slug ?? null; | |
| 168 | + else slug = c?.slug ?? null; | |
| 169 | + // multilingual refinements for lots filed under broad categories | |
| 170 | + if (slug && ['antiques', 'art', 'vintage_toys', 'fashion_streetwear'].includes(slug)) { | |
| 171 | + const cue = LOCAL_CUES.find(([re]) => re.test(title))?.[1]; | |
| 172 | + if (cue && !(slug === 'art' && cue === 'music')) slug = cue; | |
| 173 | + } | |
| 174 | + if (slug === 'trading_cards') slug = slugFromTitle(title, 'cards') ?? 'other_tcg'; | |
| 175 | + return { slug, hint, path: c?.path ?? null }; | |
| 176 | +} | |
| 177 | + | |
| 178 | +function withParams(base: string, params: Record<string, string | number | null | undefined>): string { | |
| 179 | + return adapters.withParams(base, params); | |
| 180 | +} | |
| 181 | + | |
| 182 | +type Cursor = { sinceEndsAt?: number; inProgress?: { newest: number; page: number } | null; sliceIndex?: number; page?: number; itemsProcessed?: number; done?: boolean }; | |
| 183 | + | |
| 184 | +export class AuctionetConnector extends BaseConnector { | |
| 185 | + readonly version = '1.0.0'; | |
| 186 | + readonly parserVersion = PARSER_VERSION; | |
| 187 | + protected override minIntervalMs = 1500; | |
| 188 | + | |
| 189 | + private get apiBase(): string { | |
| 190 | + return String(this.meta.config.apiBase ?? 'https://auctionet.com/api/v2'); | |
| 191 | + } | |
| 192 | + private get perPage(): number { | |
| 193 | + return Math.min(500, Math.max(10, Number(this.meta.config.perPage ?? 100))); | |
| 194 | + } | |
| 195 | + private get maxPage(): number { | |
| 196 | + return Math.floor(Number(this.meta.config.apiItemCap ?? 10_000) / this.perPage); | |
| 197 | + } | |
| 198 | + | |
| 199 | + private async fetchPage(ctx: CrawlContext, page: number, slice: { categoryId: number | null; countryCode: string | null }): Promise<{ payload: PagePayload | null; url: string; res: Awaited<ReturnType<CrawlContext['fetch']>> }> { | |
| 200 | + const url = withParams(`${this.apiBase}/items.json`, { is: 'ended', per_page: this.perPage, page, category_id: slice.categoryId, country_code: slice.countryCode }); | |
| 201 | + await this.throttle(url); | |
| 202 | + const res = await ctx.fetch(url, { | |
| 203 | + engines: ['api'], | |
| 204 | + responseType: 'json', | |
| 205 | + timeoutMs: 60_000, | |
| 206 | + expect: ['title', 'price', 'currency', 'date'], | |
| 207 | + parse: (r) => { | |
| 208 | + const p = parseItemsPage(r.json, url, page, slice); | |
| 209 | + const sold = p?.items.find((i) => i.winningBid); | |
| 210 | + return p ? { title: p.items[0]?.title, price: sold?.winningBid ?? null, currency: sold?.currency ?? null, date: sold?.endsAt ?? null } : null; | |
| 211 | + }, | |
| 212 | + }); | |
| 213 | + const payload = res.success ? parseItemsPage(res.json, url, page, slice) : null; | |
| 214 | + return { payload, url, res }; | |
| 215 | + } | |
| 216 | + | |
| 217 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 218 | + if (ctx.options.mode === 'backfill') { | |
| 219 | + yield* this.backfill(ctx); | |
| 220 | + return; | |
| 221 | + } | |
| 222 | + const cursor = (ctx.options.cursor ?? {}) as Cursor; | |
| 223 | + const since = cursor.sinceEndsAt ?? 0; | |
| 224 | + const maxPages = Math.min(Number(this.meta.config.incrementalMaxPages ?? 60), this.maxPage); | |
| 225 | + let newest = cursor.inProgress?.newest ?? 0; | |
| 226 | + let page = cursor.inProgress?.page ?? 1; | |
| 227 | + let count = 0; | |
| 228 | + const slice = { categoryId: null, countryCode: null }; | |
| 229 | + for (; page <= maxPages; page++) { | |
| 230 | + if (ctx.signal?.aborted || this.reached(ctx, count)) break; | |
| 231 | + const { payload, url, res } = await this.fetchPage(ctx, page, slice); | |
| 232 | + if (!payload) { | |
| 233 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 234 | + break; | |
| 235 | + } | |
| 236 | + if (payload.items.length === 0) break; | |
| 237 | + const pageNewest = Math.max(...payload.items.map((i) => i.endsAt ?? 0)); | |
| 238 | + newest = Math.max(newest, pageNewest); | |
| 239 | + const fresh = payload.items.filter((i) => (i.endsAt ?? 0) > since); | |
| 240 | + const reachedCheckpoint = fresh.length < payload.items.length; | |
| 241 | + if (fresh.length) { | |
| 242 | + count++; | |
| 243 | + yield { url, externalId: `ended:p${page}:${fresh[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { ...payload, items: fresh }, fetchedAt: res.fetchedAt }; | |
| 244 | + } | |
| 245 | + if (reachedCheckpoint) break; | |
| 246 | + await ctx.setCursor({ sinceEndsAt: since, inProgress: { newest, page: page + 1 } }); | |
| 247 | + } | |
| 248 | + if (ctx.options.mode !== 'probe') await ctx.setCursor({ sinceEndsAt: Math.max(since, newest), inProgress: null }); | |
| 249 | + } | |
| 250 | + | |
| 251 | + /** Backfill: category × country slices (each capped at apiItemCap items by the API), resumable. */ | |
| 252 | + private async *backfill(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 253 | + const cursor = (ctx.options.cursor ?? {}) as Cursor; | |
| 254 | + if (cursor.done) return; | |
| 255 | + const slices = this.slices(ctx.options.categories); | |
| 256 | + let sliceIndex = cursor.sliceIndex ?? 0; | |
| 257 | + let page = cursor.page ?? 1; | |
| 258 | + let itemsProcessed = cursor.itemsProcessed ?? 0; | |
| 259 | + let count = 0; | |
| 260 | + let reachedDate: Date | null = null; | |
| 261 | + const maxPagesRun = this.policy.backfillMaxPages; | |
| 262 | + let fetched = 0; | |
| 263 | + for (; sliceIndex < slices.length; sliceIndex++, page = 1) { | |
| 264 | + const slice = slices[sliceIndex]!; | |
| 265 | + let totalPages: number | null = null; | |
| 266 | + for (; page <= this.maxPage; page++) { | |
| 267 | + if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= maxPagesRun) { | |
| 268 | + await ctx.setCursor({ sliceIndex, page, itemsProcessed }); | |
| 269 | + return; | |
| 270 | + } | |
| 271 | + const { payload, url, res } = await this.fetchPage(ctx, page, slice); | |
| 272 | + fetched++; | |
| 273 | + if (!payload) { | |
| 274 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 275 | + break; | |
| 276 | + } | |
| 277 | + if (payload.totalEntries !== null) totalPages = Math.min(this.maxPage, Math.ceil(payload.totalEntries / this.perPage)); | |
| 278 | + if (payload.items.length === 0) break; | |
| 279 | + count++; | |
| 280 | + itemsProcessed += payload.items.length; | |
| 281 | + const oldest = Math.min(...payload.items.map((i) => i.endsAt ?? Number.MAX_SAFE_INTEGER)); | |
| 282 | + if (Number.isFinite(oldest) && oldest < Number.MAX_SAFE_INTEGER) reachedDate = fromUnix(oldest); | |
| 283 | + yield { url, externalId: `backfill:c${slice.categoryId ?? 'all'}:${slice.countryCode ?? 'all'}:p${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 284 | + await ctx.setCursor({ sliceIndex, page: page + 1, itemsProcessed }); | |
| 285 | + await ctx.progress({ page: sliceIndex * this.maxPage + page, totalPages: slices.length * this.maxPage, itemsProcessed, reachedDate, cursor: { sliceIndex, page: page + 1 } }); | |
| 286 | + if (totalPages !== null && page >= totalPages) break; | |
| 287 | + } | |
| 288 | + } | |
| 289 | + await ctx.setCursor({ done: true, itemsProcessed }); | |
| 290 | + await ctx.progress({ page: slices.length * this.maxPage, totalPages: slices.length * this.maxPage, itemsProcessed, reachedDate, cursor: { done: true } }); | |
| 291 | + } | |
| 292 | + | |
| 293 | + /** Leaf categories kept in taxonomy (optionally restricted to requested slugs) × configured countries. */ | |
| 294 | + slices(categorySlugs?: string[]): Array<{ categoryId: number | null; countryCode: string | null }> { | |
| 295 | + const configured = (this.meta.config.backfillCategoryIds as number[] | undefined) ?? []; | |
| 296 | + let ids = configured.length ? configured : leafCategoryIds(); | |
| 297 | + if (categorySlugs?.length) { | |
| 298 | + const want = new Set(categorySlugs); | |
| 299 | + ids = ids.filter((id) => want.has(CATEGORY_MAP[id]!.slug!) || want.has(familyOf(CATEGORY_MAP[id]!.slug!))); | |
| 300 | + } | |
| 301 | + const countries = (this.meta.config.countryCodes as string[] | undefined) ?? [null]; | |
| 302 | + const out: Array<{ categoryId: number | null; countryCode: string | null }> = []; | |
| 303 | + for (const id of ids) for (const cc of countries.length ? countries : [null]) out.push({ categoryId: id, countryCode: cc }); | |
| 304 | + return out; | |
| 305 | + } | |
| 306 | + | |
| 307 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 308 | + const p = PagePayloadSchema.parse(raw.payload); | |
| 309 | + const out: NormalizedRecord[] = []; | |
| 310 | + for (const item of p.items) { | |
| 311 | + const rec = normalizeItem(this.meta, item, raw.fetchedAt, p.slice); | |
| 312 | + if (rec) out.push(rec); | |
| 313 | + } | |
| 314 | + return out; | |
| 315 | + } | |
| 316 | +} | |
| 317 | + | |
| 318 | +/** Leaf categories kept in taxonomy: a node is a leaf when no other node's path extends it (roots without children count). */ | |
| 319 | +export function leafCategoryIds(): number[] { | |
| 320 | + const paths = Object.values(CATEGORY_MAP).map((c) => c.path); | |
| 321 | + return Object.entries(CATEGORY_MAP) | |
| 322 | + .filter(([, c]) => c.slug !== null && !paths.some((p) => p.startsWith(`${c.path} > `))) | |
| 323 | + .map(([id]) => Number(id)); | |
| 324 | +} | |
| 325 | + | |
| 326 | +function familyOf(slug: string): string { | |
| 327 | + const fam: Record<string, string> = { rolex: 'watches', omega: 'watches', patek_philippe: 'watches', audemars_piguet: 'watches', other_watches: 'watches', independent_comics: 'comics', marvel_comics: 'comics', dc_comics: 'comics', manga: 'comics', lego_sets: 'lego', banknotes: 'coins', medals: 'militaria', whisky: 'wine', cognac: 'wine', rum: 'wine' }; | |
| 328 | + return fam[slug] ?? slug; | |
| 329 | +} | |
| 330 | + | |
| 331 | +export function normalizeItem(meta: ConnectorMeta, item: Item, observedAt: Date, slice?: { categoryId: number | null; countryCode: string | null }): NormalizedRecord | null { | |
| 332 | + const cat = categoryFor(item.categoryId, item.title); | |
| 333 | + if (!cat.slug) return null; | |
| 334 | + if (!isSupportedCurrency(item.currency)) return null; | |
| 335 | + const endsAt = fromUnix(item.endsAt); | |
| 336 | + if (!endsAt) return null; | |
| 337 | + const g = parseGradeFromTitle(item.title); | |
| 338 | + const count = parenCount(item.title); | |
| 339 | + const isBundle = isBundleMultilingual(item.title); | |
| 340 | + const brand = brandFromSlug(cat.slug, item.title); | |
| 341 | + const reference = ['rolex', 'omega', 'patek_philippe', 'audemars_piguet', 'other_watches'].includes(cat.slug) ? watchReference(item.title) : null; | |
| 342 | + const attributes = AssetAttributesSchema.parse({ | |
| 343 | + categorySlug: cat.slug, | |
| 344 | + name: item.title, | |
| 345 | + brand, | |
| 346 | + reference, | |
| 347 | + year: yearFromTitle(item.title), | |
| 348 | + identifiers: { auctionet_item_id: String(item.id) }, | |
| 349 | + metadata: { | |
| 350 | + house: item.house, | |
| 351 | + company_id: item.companyId, | |
| 352 | + auction_id: item.auctionId, | |
| 353 | + auctionet_category_id: item.categoryId, | |
| 354 | + auctionet_category: cat.path, | |
| 355 | + estimate: item.estimate, | |
| 356 | + upper_estimate: item.upperEstimate, | |
| 357 | + reserve_met: item.reserveMet, | |
| 358 | + bid_count: item.bidCount, | |
| 359 | + auction_type: item.type, | |
| 360 | + state: item.state, | |
| 361 | + price_basis: 'winning_bid_excl_buyer_fee', | |
| 362 | + slice_country: slice?.countryCode ?? null, | |
| 363 | + }, | |
| 364 | + }); | |
| 365 | + const base = { | |
| 366 | + connectorId: meta.id, | |
| 367 | + sourceId: meta.sourceId, | |
| 368 | + sourceUrl: item.url, | |
| 369 | + externalId: String(item.id), | |
| 370 | + rawTitle: item.title, | |
| 371 | + description: item.description, | |
| 372 | + imageUrls: item.images, | |
| 373 | + attributes, | |
| 374 | + grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, qualifier: null, certificationNumber: null }, | |
| 375 | + condition: { condition: null, conditionRaw: item.condition, completeness: null }, | |
| 376 | + observedAt, | |
| 377 | + confidence: cat.hint === 'unknown' ? 0.75 : 0.85, | |
| 378 | + parserVersion: PARSER_VERSION, | |
| 379 | + }; | |
| 380 | + const currency = item.currency; | |
| 381 | + if (item.state === 'sold' && item.winningBid) { | |
| 382 | + return NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate: endsAt, price: item.winningBid, currency, buyerPremiumIncluded: false, quantity: count && !isBundle ? count : 1, isBundle, location: item.location, auctionHouse: item.house ?? 'Auctionet', lotNumber: item.catalogNumber }); | |
| 383 | + } | |
| 384 | + const status = item.state === 'published' ? (endsAt.getTime() > observedAt.getTime() ? 'live' : 'ended') : 'ended'; | |
| 385 | + return NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: item.house ?? 'Auctionet', auctionName: null, lotNumber: item.catalogNumber, startsAt: fromUnix(item.publishedAt), endsAt, estimateLow: item.estimate, estimateHigh: item.upperEstimate, currentBid: item.winningBid, currency, status, location: item.location }); | |
| 386 | +} | |
| 387 | + | |
| 388 | +export default function createConnector(meta: ConnectorMeta) { | |
| 389 | + return new AuctionetConnector(meta); | |
| 390 | +} | |
added
connectors/api/auctionet/meta.json
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +{ | |
| 2 | + "id": "auctionet", | |
| 3 | + "displayName": "Auctionet (Nordic & European online auctions — results)", | |
| 4 | + "sourceId": "auctionet", | |
| 5 | + "sourceName": "Auctionet", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://auctionet.com", | |
| 8 | + "module": "api/auctionet", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["art", "contemporary_art", "photography", "antiques", "design_furniture", "porcelain", "glass_crystal", "silver", "clocks", "watches", "rolex", "omega", "patek_philippe", "audemars_piguet", "other_watches", "jewelry", "gemstones", "coins", "banknotes", "medals", "stamps", "books", "maps", "historical_documents", "autographs", "cameras", "vintage_toys", "action_figures", "dolls", "model_cars", "model_trains", "comics", "trading_cards", "music", "music_memorabilia", "movie_memorabilia", "musical_instruments", "pens", "sports_memorabilia", "advertising", "scientific_instruments", "automobiles", "motorcycles", "automotive_memorabilia", "fashion_streetwear", "luxury_handbags", "wine", "whisky", "militaria"], | |
| 11 | + "regions": ["SE", "DE", "DK", "FI", "ES", "GB"], | |
| 12 | + "languages": ["sv", "de", "da", "fi", "es", "en"], | |
| 13 | + "currency": ["SEK", "EUR", "DKK", "GBP"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 360, | |
| 22 | + "priority": "high", | |
| 23 | + "trustScore": 0.85, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://auctionet.com/en/terms_of_use", | |
| 26 | + "acquisitionMethod": "public JSON API (undocumented, unauthenticated)", | |
| 27 | + "historicalDepth": "years", | |
| 28 | + "accessNotes": "Auctionet is the Nordic/European online auction platform used by ~100 houses (Stockholms Auktionsverk, Crafoord, Uppsala Auktionskammare, Bruun Rasmussen online, Stockholms Auktionsverk Köln, Helsingborgs Auktionskammare…). We read ONLY the public, unauthenticated JSON endpoint https://auctionet.com/api/v2/items.json (the same endpoint the site's own search uses; verified HTTP 200 with the RareIndexBot UA, no key, no cookies) plus /api/v2/categories.json once per run. Parameters used: is=ended (closed lots), per_page (≤ 500; we use 100), page (the API caps every query at 10 000 items, page > cap returns an empty list), category_id and country_code for backfill slices. robots.txt disallows only /admin/ and /*/my paths; /api/ is not disallowed; terms_of_use contain no clause on automated access. Each item carries state (sold/unsold/published), currency (SEK/EUR/DKK/GBP — native, never converted), estimate, ends_at (unix, = sale date), house, location, catalog_number, bids[] whose top amount is the winning bid. Auctionet publishes the WINNING BID = hammer price; each house adds its own buyer's fee on top, so buyer_premium_included=false and metadata.price_basis='winning_bid_excl_buyer_fee'. We store no bidder data (bidder numbers are anonymous and discarded), no user data. Items in licence-weapon, firearm, vehicle-part, boat, moped, bicycle, garden, tool and consumer-electronics categories are skipped (out of taxonomy / regulated). Politeness 1.5 s between requests; incremental runs page through the most recently ended lots until the previous checkpoint (ends_at) is reached; backfill iterates category × country slices (≤ 10 000 items each) with a resumable cursor.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "apiBase": "https://auctionet.com/api/v2", | |
| 33 | + "perPage": 100, | |
| 34 | + "incrementalMaxPages": 60, | |
| 35 | + "apiItemCap": 10000, | |
| 36 | + "countryCodes": ["SE", "DE", "DK", "FI", "ES", "GB"], | |
| 37 | + "backfillCategoryIds": [] | |
| 38 | + } | |
| 39 | +} | |
added
connectors/api/bernaerts/README.md
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# bernaerts — Bernaerts Auctioneers (Antwerp) results | |
| 2 | + | |
| 3 | +AuctionMobility front-end `live.bernaerts.eu`: `/auctions/past` and `/auctions/<row_id>/<slug>?page=N` embed the | |
| 4 | +platform JSON in `viewVars` (auctions: title/dates/lot_count/currency; lots: `lot_number`, `title`, `sold_price`, | |
| 5 | +estimates, `status`, cover image, lot URL, 36 per page with `query_info.total_num_results`). | |
| 6 | + | |
| 7 | +- `sale` for `status=sold` (`sold_price` = hammer → `buyerPremiumIncluded=false`, EUR), `auction_lot` for unsold. | |
| 8 | +- The token-protected `production4-server.auctionmobility.com` API is never called. | |
| 9 | +- Identifiers: `bernaerts_lot = <auction row_id>/<lot number>`. | |
| 10 | +- Fixtures: `pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts bernaerts --save`. | |
added
connectors/api/bernaerts/index.test.ts
+62 −0
@@ -0,0 +1,62 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 3 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 6 | +import createConnector, { parseLotsPage, parsePastAuctions, parseViewVars } from './index.js'; | |
| 7 | + | |
| 8 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 9 | +const connector = createConnector(meta); | |
| 10 | +const attrsOf = (r: NormalizedRecord) => { | |
| 11 | + if (!('attributes' in r)) throw new Error(`record kind ${r.kind} has no attributes`); | |
| 12 | + return r.attributes; | |
| 13 | +}; | |
| 14 | + | |
| 15 | +const PAST = `<script type="text/javascript" nonce="x"> viewVars = ${JSON.stringify({ auctions: { result_page: [{ row_id: '4-KD099X', title: 'Modern vs. Classic (Lot 100-402)', auction_type: 'timed_then_live', time_start: '2026-03-13T13:30:00Z', time_start_live_auction: '2026-03-31T12:00:00Z', effective_end_time: '2026-03-31T17:00:00Z', location_name: 'Antwerp, Belgium', lot_count: 298, sold_lot_count: 242, currency_code: 'EUR', _detail_url: '/auctions/4-KD099X/modern-vs-classic-lot-100-402', publication_status: 'full', total_hammer_price: '411000.00', total_sold_value: '407800.00' }], query_info: { page_size: 20, page_start_offset: 0, total_num_results: 319 } } })};</script>`; | |
| 16 | +const LOTS = `<script> viewVars = ${JSON.stringify({ auction: { row_id: '4-KD099X', title: 'Modern vs. Classic (Lot 100-402)', time_start_live_auction: '2026-03-31T12:00:00Z', effective_end_time: '2026-03-31T17:00:00Z', location_name: 'Antwerp, Belgium', total_hammer_price: '411000.00' }, lots: { result_page: [ | |
| 17 | + { row_id: '4-KDZAWS', lot_number: 100, title: 'A Shang Dynasty (1600-1100 BC) wine jug, Gu. China.', sold_price: '7000.00', estimate_low: '3000.00', estimate_high: '4000.00', currency_code: 'EUR', status: 'sold', cover_thumbnail: 'https://images4-cdn.auctionmobility.com/x/0100.jpg', _detail_url: '/lots/view/4-KDZAWS/a-shang-dynasty-wine-jug', truncated_description: 'Bronze, H 28 cm.' }, | |
| 18 | + { row_id: '4-KDZAWT', lot_number: 101, lot_number_extension: 'A', title: 'Rolex Submariner ref. 5513, steel, 1970s', sold_price: null, estimate_low: '8000.00', estimate_high: '12000.00', currency_code: 'EUR', status: 'unsold', _detail_url: '/lots/view/4-KDZAWT/rolex' }, | |
| 19 | +], query_info: { page_size: 36, page_start_offset: 0, total_num_results: 298, next_page: 'https://production4-server.auctionmobility.com/v1/auction/4-KD099X/lots?o=36' } } })};</script>`; | |
| 20 | + | |
| 21 | +describe('bernaerts', () => { | |
| 22 | + runFixtureSuite(connector, it, expect); | |
| 23 | + | |
| 24 | + it('fixtures: EUR hammer sales from the embedded AuctionMobility JSON', async () => { | |
| 25 | + let sales = 0; | |
| 26 | + for (const name of listFixtures('bernaerts')) { | |
| 27 | + for (const r of await connector.normalize(loadFixture('bernaerts', name).raw)) { | |
| 28 | + if (!('attributes' in r)) continue; | |
| 29 | + expect(r.attributes.identifiers.bernaerts_lot).toMatch(/^4-[A-Z0-9]+\/\S+$/); | |
| 30 | + expect(r.sourceUrl).toMatch(/^https:\/\/live\.bernaerts\.eu\//); | |
| 31 | + if (r.kind === 'sale') { | |
| 32 | + sales++; | |
| 33 | + expect(r.currency).toBe('EUR'); | |
| 34 | + expect(r.buyerPremiumIncluded).toBe(false); | |
| 35 | + expect(r.auctionHouse).toBe('Bernaerts'); | |
| 36 | + } | |
| 37 | + } | |
| 38 | + } | |
| 39 | + expect(sales).toBeGreaterThan(10); | |
| 40 | + }); | |
| 41 | + | |
| 42 | + it('parses viewVars, the past-auction list and a lots page (pagination from query_info)', async () => { | |
| 43 | + expect(parseViewVars('<p>no</p>')).toBeNull(); | |
| 44 | + const sales = parsePastAuctions(PAST); | |
| 45 | + expect(sales).toHaveLength(1); | |
| 46 | + expect(sales[0]).toMatchObject({ id: '4-KD099X', title: 'Modern vs. Classic (Lot 100-402)', url: 'https://live.bernaerts.eu/auctions/4-KD099X/modern-vs-classic-lot-100-402', date: '2026-03-31T12:00:00Z', location: 'Antwerp, Belgium' }); | |
| 47 | + expect(sales[0]!.extra.total_hammer_price).toBe(411000); | |
| 48 | + const p = parseLotsPage(LOTS, sales[0]!)!; | |
| 49 | + expect(p.totalLots).toBe(298); | |
| 50 | + expect(p.hasMore).toBe(true); | |
| 51 | + expect(p.lots).toHaveLength(2); | |
| 52 | + expect(p.lots[0]).toMatchObject({ lotNo: '100', price: 7000, currency: 'EUR', premiumIncluded: false, estimateLow: 3000, estimateHigh: 4000, sold: true, url: 'https://live.bernaerts.eu/lots/view/4-KDZAWS/a-shang-dynasty-wine-jug', image: 'https://images4-cdn.auctionmobility.com/x/0100.jpg', description: 'Bronze, H 28 cm.' }); | |
| 53 | + expect(p.lots[1]).toMatchObject({ lotNo: '101A', price: null, sold: false }); | |
| 54 | + const out = await connector.normalize({ url: sales[0]!.url, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'sale_results', url: sales[0]!.url, sale: sales[0]!, page: 1, totalLots: 298, lots: p.lots } }); | |
| 55 | + expect(out.map((r) => r.kind)).toEqual(['sale', 'auction_lot']); | |
| 56 | + expect(attrsOf(out[0]!).categorySlug).toBe('antiques'); | |
| 57 | + expect(attrsOf(out[1]!).categorySlug).toBe('rolex'); | |
| 58 | + expect(attrsOf(out[1]!).reference).toBe('5513'); | |
| 59 | + if (out[0]!.kind === 'sale') expect(out[0]!.saleDate.toISOString()).toBe('2026-03-31T12:00:00.000Z'); | |
| 60 | + expect(parseLotsPage('<html></html>', sales[0]!)).toBeNull(); | |
| 61 | + }); | |
| 62 | +}); | |
added
connectors/api/bernaerts/index.ts
+125 −0
@@ -0,0 +1,125 @@ | ||
| 1 | +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors'; | |
| 2 | +import type { ExtractionResult } from '@rareindex/shared'; | |
| 3 | +import { isSupportedCurrency, stripHtml } from '../_g8-auctions-eu-apac-lib/index.js'; | |
| 4 | +import { SaleResultsConnector, type HouseConfig, type ParsedLot, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Bernaerts Auctioneers (Antwerp) — AuctionMobility front-end (live.bernaerts.eu). Every page embeds the | |
| 8 | + * platform's JSON in `viewVars = {...}`: /auctions/past lists finished auctions (row_id, title, dates, | |
| 9 | + * lot_count, currency_code, _detail_url); /auctions/<row_id>/<slug>?page=N embeds 36 lots per page | |
| 10 | + * (lot_number, title, sold_price, estimate_low/high, status, currency_code, cover_thumbnail, _detail_url, | |
| 11 | + * schema.org Product jsonld). `sold_price` is the hammer (the auction object separately totals | |
| 12 | + * total_hammer_price and total_sold_value; buyer's premium 30 % is stated in the description). | |
| 13 | + */ | |
| 14 | +const BASE = 'https://live.bernaerts.eu'; | |
| 15 | + | |
| 16 | +type AmAuction = { row_id?: string; title?: string; auction_type?: string; time_start?: string | null; time_start_live_auction?: string | null; effective_end_time?: string | null; location_name?: string | null; lot_count?: number; sold_lot_count?: number; currency_code?: string; _detail_url?: string; publication_status?: string; total_hammer_price?: string | null; total_sold_value?: string | null; default_buyers_premium?: string | number | null }; | |
| 17 | +type AmLot = { row_id?: string; lot_number?: number | string; lot_number_extension?: string | null; title?: string; truncated_description?: string | null; artist?: string | null; sold_price?: string | number | null; estimate_low?: string | number | null; estimate_high?: string | number | null; currency_code?: string | null; status?: string | null; cover_thumbnail?: string | null; _detail_url?: string; extended_end_time?: string | null; auction?: { row_id?: string; effective_end_time?: string | null; currency_code?: string } | null; is_mixed_lot?: boolean | null; quantity?: number | null; when_produced?: string | null; condition?: string | null; dimensions?: string | null }; | |
| 18 | +type QueryInfo = { page_size?: number; page_start_offset?: number; total_num_results?: number; next_page?: string | null }; | |
| 19 | + | |
| 20 | +/** Extract the `viewVars = {...};` JSON literal from an AuctionMobility page. */ | |
| 21 | +export function parseViewVars(htmlText: string): Record<string, unknown> | null { | |
| 22 | + const i = htmlText.indexOf('viewVars = '); | |
| 23 | + if (i < 0) return null; | |
| 24 | + const start = htmlText.indexOf('{', i); | |
| 25 | + const end = htmlText.indexOf('</script>', start); | |
| 26 | + if (start < 0 || end < 0) return null; | |
| 27 | + const literal = htmlText.slice(start, end).trim().replace(/;\s*$/, ''); | |
| 28 | + try { | |
| 29 | + return JSON.parse(literal) as Record<string, unknown>; | |
| 30 | + } catch { | |
| 31 | + return null; | |
| 32 | + } | |
| 33 | +} | |
| 34 | + | |
| 35 | +function money(v: unknown): number | null { | |
| 36 | + if (v === null || v === undefined || v === '') return null; | |
| 37 | + const n = Number(v); | |
| 38 | + return Number.isFinite(n) && n > 0 ? n : null; | |
| 39 | +} | |
| 40 | + | |
| 41 | +export function parsePastAuctions(htmlText: string): SaleRef[] { | |
| 42 | + const vv = parseViewVars(htmlText); | |
| 43 | + const page = (vv?.auctions as { result_page?: AmAuction[] } | undefined)?.result_page; | |
| 44 | + if (!Array.isArray(page)) return []; | |
| 45 | + return page | |
| 46 | + .filter((a) => a.row_id && a._detail_url && a.publication_status !== 'hidden') | |
| 47 | + .map((a) => ({ | |
| 48 | + id: a.row_id!, | |
| 49 | + title: String(a.title ?? a.row_id).trim(), | |
| 50 | + url: `${BASE}${a._detail_url}`, | |
| 51 | + date: a.time_start_live_auction ?? a.effective_end_time ?? a.time_start ?? null, | |
| 52 | + location: a.location_name ?? null, | |
| 53 | + extra: { auction_type: a.auction_type ?? null, lot_count: a.lot_count ?? null, sold_lot_count: a.sold_lot_count ?? null, currency_code: a.currency_code ?? null, ends_at: a.effective_end_time ?? null, total_hammer_price: money(a.total_hammer_price), total_sold_value: money(a.total_sold_value) }, | |
| 54 | + })); | |
| 55 | +} | |
| 56 | + | |
| 57 | +export function parseLotsPage(htmlText: string, sale: SaleRef): ParsedSalePage | null { | |
| 58 | + const vv = parseViewVars(htmlText); | |
| 59 | + const lotsBlock = vv?.lots as { result_page?: AmLot[]; query_info?: QueryInfo } | undefined; | |
| 60 | + if (!lotsBlock || !Array.isArray(lotsBlock.result_page)) return null; | |
| 61 | + const qi = lotsBlock.query_info ?? {}; | |
| 62 | + const lots: ParsedLot[] = []; | |
| 63 | + for (const l of lotsBlock.result_page) { | |
| 64 | + const lotNo = `${l.lot_number ?? ''}${l.lot_number_extension ?? ''}`.trim(); | |
| 65 | + const title = stripHtml(l.title ?? '', 300); | |
| 66 | + if (!lotNo || !title) continue; | |
| 67 | + const price = money(l.sold_price); | |
| 68 | + const cur = (l.currency_code ?? l.auction?.currency_code ?? String(sale.extra.currency_code ?? '') ?? '').toUpperCase(); | |
| 69 | + lots.push({ | |
| 70 | + lotNo, | |
| 71 | + title, | |
| 72 | + subtitle: l.artist ? stripHtml(l.artist, 120) : null, | |
| 73 | + description: stripHtml(l.truncated_description ?? null, 500), | |
| 74 | + url: l._detail_url ? `${BASE}${l._detail_url}` : sale.url, | |
| 75 | + image: l.cover_thumbnail ?? null, | |
| 76 | + price, | |
| 77 | + currency: isSupportedCurrency(cur) ? cur : null, | |
| 78 | + premiumIncluded: false, | |
| 79 | + estimateLow: money(l.estimate_low), | |
| 80 | + estimateHigh: money(l.estimate_high), | |
| 81 | + date: l.extended_end_time ?? l.auction?.effective_end_time ?? null, | |
| 82 | + sold: l.status === 'sold' && price !== null, | |
| 83 | + extra: { lot_row_id: l.row_id ?? null, status: l.status ?? null, is_mixed_lot: l.is_mixed_lot ?? null, quantity: l.quantity ?? null, when_produced: l.when_produced ?? null, condition: l.condition ?? null, dimensions: l.dimensions ?? null }, | |
| 84 | + }); | |
| 85 | + } | |
| 86 | + const total = typeof qi.total_num_results === 'number' ? qi.total_num_results : null; | |
| 87 | + const offset = typeof qi.page_start_offset === 'number' ? qi.page_start_offset : 0; | |
| 88 | + const size = typeof qi.page_size === 'number' ? qi.page_size : lots.length; | |
| 89 | + const auction = vv?.auction as AmAuction | undefined; | |
| 90 | + return { | |
| 91 | + lots, | |
| 92 | + hasMore: total !== null ? offset + size < total : Boolean(qi.next_page), | |
| 93 | + totalLots: total, | |
| 94 | + sale: auction ? { title: auction.title ?? undefined, date: auction.time_start_live_auction ?? auction.effective_end_time ?? undefined, location: auction.location_name ?? undefined, extra: { total_hammer_price: money(auction.total_hammer_price), total_sold_value: money(auction.total_sold_value), auction_type: auction.auction_type ?? null } } : undefined, | |
| 95 | + }; | |
| 96 | +} | |
| 97 | + | |
| 98 | +export class BernaertsConnector extends SaleResultsConnector { | |
| 99 | + readonly version = '1.0.0'; | |
| 100 | + readonly house: HouseConfig = { houseName: 'Bernaerts', defaultCurrency: 'EUR', location: 'Antwerp, Belgium', idKey: 'bernaerts_lot', premiumIncluded: false, fallbackSlug: 'antiques', minIntervalMs: 2000, maxPagesPerSale: 30 }; | |
| 101 | + protected override minIntervalMs = 2000; | |
| 102 | + | |
| 103 | + async listSales(ctx: CrawlContext): Promise<SaleRef[]> { | |
| 104 | + const url = String(this.meta.config.pastUrl ?? `${BASE}/auctions/past`); | |
| 105 | + await this.throttle(url); | |
| 106 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 107 | + if (!res.success || !res.html) { | |
| 108 | + ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 109 | + return []; | |
| 110 | + } | |
| 111 | + return parsePastAuctions(res.html); | |
| 112 | + } | |
| 113 | + | |
| 114 | + salePageUrl(sale: SaleRef, page: number): string { | |
| 115 | + return page > 1 ? `${sale.url}?page=${page}` : sale.url; | |
| 116 | + } | |
| 117 | + | |
| 118 | + parseSalePage(res: ExtractionResult, sale: SaleRef): ParsedSalePage | null { | |
| 119 | + return res.html ? parseLotsPage(res.html, sale) : null; | |
| 120 | + } | |
| 121 | +} | |
| 122 | + | |
| 123 | +export default function createConnector(meta: ConnectorMeta) { | |
| 124 | + return new BernaertsConnector(meta); | |
| 125 | +} | |
added
connectors/api/bernaerts/meta.json
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +{ | |
| 2 | + "id": "bernaerts", | |
| 3 | + "displayName": "Bernaerts Auctioneers (Antwerp) — results", | |
| 4 | + "sourceId": "bernaerts", | |
| 5 | + "sourceName": "Bernaerts", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://live.bernaerts.eu", | |
| 8 | + "module": "api/bernaerts", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["art", "contemporary_art", "photography", "design_furniture", "antiques", "porcelain", "silver", "glass_crystal", "jewelry", "other_watches", "books", "comics", "wine", "vintage_toys"], | |
| 11 | + "regions": ["BE"], | |
| 12 | + "languages": ["nl", "fr", "en"], | |
| 13 | + "currency": ["EUR"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "low", | |
| 23 | + "trustScore": 0.85, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.bernaerts.eu/nl/algemene-voorwaarden", | |
| 26 | + "acquisitionMethod": "embedded JSON (AuctionMobility viewVars) on public pages", | |
| 27 | + "historicalDepth": "years", | |
| 28 | + "accessNotes": "Bernaerts (Antwerp) publishes its catalogues and results on live.bernaerts.eu (AuctionMobility). We read two public page types only: /auctions/past (finished auctions, JSON embedded in the page's viewVars: title, dates, lot_count, currency, detail URL) and /auctions/<row_id>/<slug>?page=N (36 lots per page embedded as JSON: lot_number, title, sold_price, estimate_low/high, status sold/unsold, cover image, lot URL, schema.org Product). The platform's direct API (production4-server.auctionmobility.com, requires an authorization token) is deliberately NOT called. sold_price is the HAMMER price (the auction JSON totals total_hammer_price separately and the sale description states a 30 % + €2 buyer's premium) → buyer_premium_included=false, EUR. robots.txt: Allow / for *; algemene voorwaarden contain no clause on automated access. Bidder registrations/paddles present in the JSON are discarded. 2 s politeness; salesPerRun caps incremental runs; backfill walks the past-auction list with a resumable cursor.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "pastUrl": "https://live.bernaerts.eu/auctions/past", | |
| 33 | + "salesPerRun": 2 | |
| 34 | + } | |
| 35 | +} | |
added
connectors/api/big-b-comics/README.md
+45 −0
@@ -0,0 +1,45 @@ | ||
| 1 | +# Big B Comics connector (`big-b-comics`) | |
| 2 | + | |
| 3 | +- Source: https://www.bigbcomics.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Ontario comic-shop chain. Shopify storefront; CGC slabs titled 'Series Year #issue - CGC 9.8 - $price'; vintage comics by character; toys and trading cards. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `cgc-graded-comics` | `comics` | — | | |
| 14 | +| `vintage-comics` | `comics` | — | | |
| 15 | +| `comics` | `comics` | — | | |
| 16 | +| `ratio-variants` | `comics` | — | | |
| 17 | +| `marvel-comics` | `marvel_comics` | Marvel | | |
| 18 | +| `vintage-spider-man` | `marvel_comics` | Marvel | | |
| 19 | +| `vintage-x-men` | `marvel_comics` | Marvel | | |
| 20 | +| `dc-comics` | `dc_comics` | DC | | |
| 21 | +| `vintage-batman` | `dc_comics` | DC | | |
| 22 | +| `indie-comics` | `independent_comics` | — | | |
| 23 | +| `vintage-star-wars` | `comics` | Star Wars | | |
| 24 | +| `trading-cards` | `non_sport_cards` | — | | |
| 25 | +| `pokemon-cards` | `pokemon` | Pokémon | | |
| 26 | +| `funko-pops` | `funko` | Funko | | |
| 27 | +| `mcfarlane-toys` | `action_figures` | McFarlane Toys | | |
| 28 | +| `super7-reaction-figures` | `action_figures` | Super7 / ReAction | | |
| 29 | +| `hasbro-toy-group` | `action_figures` | Hasbro | | |
| 30 | +| `banpresto-toys` | `action_figures` | Banpresto | | |
| 31 | + | |
| 32 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 33 | +- `^(?=[\s\S]*\b(marvel|spider-?man|x-men|avengers|wolverine|hulk|deadpool|iron man|captain america|fantastic four|daredevil|venom|punisher)\b)[\s\S]*\| (comics|vintage-comics|cgc-graded-comics|ratio-variants)$` → `marvel_comics` (franchise Marvel) | |
| 34 | +- `^(?=[\s\S]*\b(dc comics|batman|superman|wonder woman|justice league|the flash|green lantern|aquaman|detective comics|action comics|harley quinn|joker|nightwing|swamp thing)\b)[\s\S]*\| (comics|vintage-comics|cgc-graded-comics|ratio-variants)$` → `dc_comics` (franchise DC) | |
| 35 | + | |
| 36 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|bags? (and|&) boards|\bmylar|comic bags|backing boards|supplies|graphic novel|\btpb\b|trade paperback|hardcover|omnibus|\bmanga\b` | |
| 37 | + | |
| 38 | +Title pattern: `^(?<name>[^#]+?) (?:\d{4} )?#(?<number>\d+[A-Za-z]?)` (name / set / number) | |
| 39 | + | |
| 40 | +## Access & compliance | |
| 41 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: graphic novels/manga/TPB collections (books, not collectibles), pre-orders, supplies. | |
| 42 | + | |
| 43 | +## Fixtures & tests | |
| 44 | +`data/fixtures/big-b-comics/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts big-b-comics`), trimmed single-product payloads incl. a sold-out variant and a graded item. | |
| 45 | +`pnpm vitest run connectors/api/big-b-comics` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts big-b-comics`. | |
added
connectors/api/big-b-comics/index.test.ts
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Big B Comics — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/big-b-comics/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "Spawn 1992 #4 Newsstand ed. - CGC 9.4 - $55.00", | |
| 15 | + "productType": "Key Issues", | |
| 16 | + "collection": "cgc-graded-comics", | |
| 17 | + "categorySlug": "comics" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "Deadpool 1997 #54 Direct Edition - CGC 9.8 - $300.00", | |
| 21 | + "collection": "cgc-graded-comics", | |
| 22 | + "categorySlug": "marvel_comics", | |
| 23 | + "franchise": "Marvel" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "title": "Batman 1940 #232", | |
| 27 | + "collection": "vintage-comics", | |
| 28 | + "categorySlug": "dc_comics" | |
| 29 | + }, | |
| 30 | + { | |
| 31 | + "title": "BCW Comic Bags and Boards Current 100ct", | |
| 32 | + "collection": "comics", | |
| 33 | + "categorySlug": null | |
| 34 | + } | |
| 35 | + ] | |
| 36 | +}); | |
added
connectors/api/big-b-comics/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Big B Comics — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/big-b-comics/meta.json
+161 −0
@@ -0,0 +1,161 @@ | ||
| 1 | +{ | |
| 2 | + "id": "big-b-comics", | |
| 3 | + "displayName": "Big B Comics (Canadian comic & sports-card & TCG store, CAD)", | |
| 4 | + "sourceId": "big-b-comics", | |
| 5 | + "sourceName": "Big B Comics", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.bigbcomics.com", | |
| 8 | + "module": "api/big-b-comics", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "comics", | |
| 14 | + "marvel_comics", | |
| 15 | + "dc_comics", | |
| 16 | + "independent_comics", | |
| 17 | + "non_sport_cards", | |
| 18 | + "pokemon", | |
| 19 | + "funko", | |
| 20 | + "action_figures" | |
| 21 | + ], | |
| 22 | + "regions": [ | |
| 23 | + "CA" | |
| 24 | + ], | |
| 25 | + "languages": [ | |
| 26 | + "en" | |
| 27 | + ], | |
| 28 | + "currency": [ | |
| 29 | + "CAD" | |
| 30 | + ], | |
| 31 | + "supportsListings": true, | |
| 32 | + "supportsSold": false, | |
| 33 | + "supportsAuctions": false, | |
| 34 | + "supportsImages": true, | |
| 35 | + "supportsCatalog": false, | |
| 36 | + "supportsPopulation": false, | |
| 37 | + "supportsLookup": true, | |
| 38 | + "refreshFrequencyMinutes": 720, | |
| 39 | + "priority": "medium", | |
| 40 | + "trustScore": 0.75, | |
| 41 | + "attributionRequired": true, | |
| 42 | + "termsUrl": "https://www.bigbcomics.com/policies/terms-of-service", | |
| 43 | + "accessNotes": "Big B Comics (bigbcomics.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 18 configured collections (cgc-graded-comics, vintage-comics, comics, ratio-variants, marvel-comics, vintage-spider-man, vintage-x-men, dc-comics … (+10 more, see config.collections)) and /products/<handle>.json for URL lookups (~12.7k products; 3.1k vintage comics, 300 CGC slabs, 400 trading cards). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Third-party grades in titles (PSA/BGS/CGC/ICCS/PMG…) are parsed by parseGradeFromTitle; cert numbers are not extracted. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — graphic novels/manga/TPB collections (books, not collectibles), pre-orders, supplies. No personal data is collected; seller = the store itself.", | |
| 44 | + "enabled": true, | |
| 45 | + "schemaVersion": "1.0", | |
| 46 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 47 | + "historicalDepth": "none", | |
| 48 | + "requires": [], | |
| 49 | + "config": { | |
| 50 | + "currency": "CAD", | |
| 51 | + "market": "CA", | |
| 52 | + "seller": "Big B Comics", | |
| 53 | + "location": "Hamilton / Barrie / Niagara Falls, ON, Canada", | |
| 54 | + "collections": [ | |
| 55 | + { | |
| 56 | + "handle": "cgc-graded-comics", | |
| 57 | + "categorySlug": "comics" | |
| 58 | + }, | |
| 59 | + { | |
| 60 | + "handle": "vintage-comics", | |
| 61 | + "categorySlug": "comics" | |
| 62 | + }, | |
| 63 | + { | |
| 64 | + "handle": "comics", | |
| 65 | + "categorySlug": "comics" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "handle": "ratio-variants", | |
| 69 | + "categorySlug": "comics" | |
| 70 | + }, | |
| 71 | + { | |
| 72 | + "handle": "marvel-comics", | |
| 73 | + "categorySlug": "marvel_comics", | |
| 74 | + "franchise": "Marvel" | |
| 75 | + }, | |
| 76 | + { | |
| 77 | + "handle": "vintage-spider-man", | |
| 78 | + "categorySlug": "marvel_comics", | |
| 79 | + "franchise": "Marvel" | |
| 80 | + }, | |
| 81 | + { | |
| 82 | + "handle": "vintage-x-men", | |
| 83 | + "categorySlug": "marvel_comics", | |
| 84 | + "franchise": "Marvel" | |
| 85 | + }, | |
| 86 | + { | |
| 87 | + "handle": "dc-comics", | |
| 88 | + "categorySlug": "dc_comics", | |
| 89 | + "franchise": "DC" | |
| 90 | + }, | |
| 91 | + { | |
| 92 | + "handle": "vintage-batman", | |
| 93 | + "categorySlug": "dc_comics", | |
| 94 | + "franchise": "DC" | |
| 95 | + }, | |
| 96 | + { | |
| 97 | + "handle": "indie-comics", | |
| 98 | + "categorySlug": "independent_comics" | |
| 99 | + }, | |
| 100 | + { | |
| 101 | + "handle": "vintage-star-wars", | |
| 102 | + "categorySlug": "comics", | |
| 103 | + "franchise": "Star Wars" | |
| 104 | + }, | |
| 105 | + { | |
| 106 | + "handle": "trading-cards", | |
| 107 | + "categorySlug": "non_sport_cards" | |
| 108 | + }, | |
| 109 | + { | |
| 110 | + "handle": "pokemon-cards", | |
| 111 | + "categorySlug": "pokemon", | |
| 112 | + "franchise": "Pokémon" | |
| 113 | + }, | |
| 114 | + { | |
| 115 | + "handle": "funko-pops", | |
| 116 | + "categorySlug": "funko", | |
| 117 | + "brand": "Funko" | |
| 118 | + }, | |
| 119 | + { | |
| 120 | + "handle": "mcfarlane-toys", | |
| 121 | + "categorySlug": "action_figures", | |
| 122 | + "brand": "McFarlane Toys" | |
| 123 | + }, | |
| 124 | + { | |
| 125 | + "handle": "super7-reaction-figures", | |
| 126 | + "categorySlug": "action_figures", | |
| 127 | + "brand": "Super7", | |
| 128 | + "series": "ReAction" | |
| 129 | + }, | |
| 130 | + { | |
| 131 | + "handle": "hasbro-toy-group", | |
| 132 | + "categorySlug": "action_figures", | |
| 133 | + "brand": "Hasbro" | |
| 134 | + }, | |
| 135 | + { | |
| 136 | + "handle": "banpresto-toys", | |
| 137 | + "categorySlug": "action_figures", | |
| 138 | + "brand": "Banpresto" | |
| 139 | + } | |
| 140 | + ], | |
| 141 | + "rules": [ | |
| 142 | + { | |
| 143 | + "match": "^(?=[\\s\\S]*\\b(marvel|spider-?man|x-men|avengers|wolverine|hulk|deadpool|iron man|captain america|fantastic four|daredevil|venom|punisher)\\b)[\\s\\S]*\\| (comics|vintage-comics|cgc-graded-comics|ratio-variants)$", | |
| 144 | + "categorySlug": "marvel_comics", | |
| 145 | + "franchise": "Marvel" | |
| 146 | + }, | |
| 147 | + { | |
| 148 | + "match": "^(?=[\\s\\S]*\\b(dc comics|batman|superman|wonder woman|justice league|the flash|green lantern|aquaman|detective comics|action comics|harley quinn|joker|nightwing|swamp thing)\\b)[\\s\\S]*\\| (comics|vintage-comics|cgc-graded-comics|ratio-variants)$", | |
| 149 | + "categorySlug": "dc_comics", | |
| 150 | + "franchise": "DC" | |
| 151 | + } | |
| 152 | + ], | |
| 153 | + "defaultCategory": null, | |
| 154 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|bags? (and|&) boards|\\bmylar|comic bags|backing boards|supplies|graphic novel|\\btpb\\b|trade paperback|hardcover|omnibus|\\bmanga\\b", | |
| 155 | + "keepOutOfStock": true, | |
| 156 | + "fetchBarcodes": false, | |
| 157 | + "wholeShop": false, | |
| 158 | + "pageSize": 250, | |
| 159 | + "titlePattern": "^(?<name>[^#]+?) (?:\\d{4} )?#(?<number>\\d+[A-Za-z]?)" | |
| 160 | + } | |
| 161 | +} | |
added
connectors/api/bunjang/README.md
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# bunjang — Bunjang 번개장터 (Korea) | |
| 2 | + | |
| 3 | +KRW asking prices from Korea's largest C2C app/site, via the public search JSON of the mobile web. | |
| 4 | + | |
| 5 | +- `api.bunjang.co.kr/api/1/find_v2.json?q=<kw>&order=date&page=<0..>&n=100&…` → `num_found` + rows (pid, name, price, status, category id, tags, region label, favourites, image template, last update). | |
| 6 | +- One raw record per page → `listing` (fixed price, `availability` from `status`: 0 selling, 3 sold); sponsored rows dropped; `identifiers.bunjang_pid`; Korean titles parsed for PSA grades, language (일판 / 북미판 / 한글판), condition (`used` enum + 미개봉/중고 hints), bundles (묶음, 일괄). | |
| 7 | +- `update_time` is a bump/edit time, kept in metadata (not `listedAt`). Seller ids/nicknames are never stored; only the coarse region label. | |
| 8 | +- Seeds `{ q, category, language }`; category refined from the title first, seller hashtags second. | |
| 9 | + | |
| 10 | +Smoke: `pnpm tsx connectors/api/_g9-asia-watch-sneaker-lib/capture.ts bunjang --limit 1 --seeds '롤렉스 시계'`. | |
added
connectors/api/bunjang/index.test.ts
+53 −0
@@ -0,0 +1,53 @@ | ||
| 1 | +import { readFileSync } from 'node:fs'; | |
| 2 | +import path from 'node:path'; | |
| 3 | +import { describe, expect, it } from 'vitest'; | |
| 4 | +import { getConnectorMeta } from '@rareindex/connectors'; | |
| 5 | +import { fixtureDir, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import createConnector, { searchUrl, toItem } from './index.js'; | |
| 7 | + | |
| 8 | +const connector = createConnector(getConnectorMeta('bunjang')); | |
| 9 | +const sample = JSON.parse(readFileSync(path.join(fixtureDir('bunjang'), 'samples', 'find-v2.json'), 'utf8')) as { num_found: number; list: Record<string, unknown>[] }; | |
| 10 | + | |
| 11 | +describe('bunjang', () => { | |
| 12 | + runFixtureSuite(connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps a raw find_v2 row (KRW price string, image template, status, category)', () => { | |
| 15 | + const it0 = toItem(sample.list[0]!); | |
| 16 | + expect(it0).not.toBeNull(); | |
| 17 | + expect(it0!.pid).toBe('407596186'); | |
| 18 | + expect(it0!.price).toBe(100000); | |
| 19 | + expect(it0!.image).toBe('https://media.bunjang.co.kr/product/407596186_1_1778563857_w600.jpg'); | |
| 20 | + expect(it0!.status).toBe('0'); | |
| 21 | + expect(it0!.categoryId).toBe('940100001'); | |
| 22 | + expect(it0!.used).toBe(2); | |
| 23 | + expect(sample.num_found).toBeGreaterThan(100); | |
| 24 | + expect(toItem({ ...sample.list[0]!, type: 'BANNER' })).toBeNull(); | |
| 25 | + }); | |
| 26 | + | |
| 27 | + it('builds the 0-based search URL', () => { | |
| 28 | + expect(searchUrl('포켓몬카드', 2)).toBe('https://api.bunjang.co.kr/api/1/find_v2.json?q=%ED%8F%AC%EC%BC%93%EB%AA%AC%EC%B9%B4%EB%93%9C&order=date&page=2&n=100&req_ref=search&stat_device=w&version=5'); | |
| 29 | + }); | |
| 30 | + | |
| 31 | + it('normalises the fixture into KRW listings with grade, condition, language and metadata', async () => { | |
| 32 | + const fx = loadFixture('bunjang', 'pokemon-psa10-search-p0'); | |
| 33 | + const out = await connector.normalize(fx.raw); | |
| 34 | + expect(out.length).toBeGreaterThan(5); | |
| 35 | + const l = out[0]; | |
| 36 | + if (l?.kind !== 'listing') throw new Error('expected listing'); | |
| 37 | + expect(l.currency).toBe('KRW'); | |
| 38 | + expect(l.price).toBe(100000); | |
| 39 | + expect(l.attributes.categorySlug).toBe('pokemon'); | |
| 40 | + expect(l.attributes.identifiers.bunjang_pid).toBe('407596186'); | |
| 41 | + expect(l.attributes.language).toBe('English'); // 북미판 = North American print | |
| 42 | + expect(l.grade.grader).toBe('psa'); | |
| 43 | + expect(l.grade.grade).toBe('10'); | |
| 44 | + expect(l.condition.conditionRaw).toBe('Used'); // used = 2 | |
| 45 | + expect(l.availability).toBe('available'); | |
| 46 | + expect(l.listedAt).toBeNull(); | |
| 47 | + expect(typeof l.attributes.metadata.updated_at).toBe('string'); | |
| 48 | + expect(l.sourceUrl).toBe('https://m.bunjang.co.kr/products/407596186'); | |
| 49 | + expect(l.seller).toBeNull(); | |
| 50 | + const jp = out.find((r) => r.kind === 'listing' && /일판/.test(r.rawTitle)); | |
| 51 | + expect(jp && jp.kind === 'listing' ? jp.attributes.language : null).toBe('Japanese'); | |
| 52 | + }); | |
| 53 | +}); | |
added
connectors/api/bunjang/index.ts
+174 −0
@@ -0,0 +1,174 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { normalizeCondition } from '@rareindex/taxonomy'; | |
| 4 | +import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 5 | +import { KeywordSeedSchema, cjkConditionRaw, cjkGrade, cjkLanguage, cleanTitle, intOrNull, isCjkBundle, posNumber, refineCategory, unixToDate, type KeywordSeed } from '../_g9-asia-watch-sneaker-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Bunjang (번개장터, Korea) — public search JSON used by m.bunjang.co.kr. KRW asking prices → `listing`. | |
| 9 | + * Titles are Korean; PSA/BGS grades and JP/EN/KR language hints are parsed from them. | |
| 10 | + */ | |
| 11 | + | |
| 12 | +const API = 'https://api.bunjang.co.kr/api/1/find_v2.json'; | |
| 13 | +const SITE = 'https://m.bunjang.co.kr'; | |
| 14 | +const PARSER_VERSION = '1.0.0'; | |
| 15 | +const PAGE_SIZE = 100; | |
| 16 | + | |
| 17 | +export const ItemSchema = z.object({ | |
| 18 | + pid: z.string(), | |
| 19 | + name: z.string(), | |
| 20 | + price: z.number(), | |
| 21 | + image: z.string().nullable(), | |
| 22 | + /** "0" = on sale, "1" = reserved, "3" = sold (as used by the mobile site) */ | |
| 23 | + status: z.string().nullable(), | |
| 24 | + /** unix seconds of the last update/bump */ | |
| 25 | + updateTime: z.number().nullable(), | |
| 26 | + /** 1 = new, 2 = used (source enum) */ | |
| 27 | + used: z.number().nullable(), | |
| 28 | + categoryId: z.string().nullable(), | |
| 29 | + tag: z.string().nullable(), | |
| 30 | + location: z.string().nullable(), | |
| 31 | + freeShipping: z.boolean().nullable(), | |
| 32 | + bizseller: z.boolean().nullable(), | |
| 33 | + numFaved: z.number().nullable(), | |
| 34 | + ad: z.boolean().nullable(), | |
| 35 | +}); | |
| 36 | +export type Item = z.infer<typeof ItemSchema>; | |
| 37 | +export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), seed: KeywordSeedSchema, page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) }); | |
| 38 | +export type PagePayload = z.infer<typeof PagePayloadSchema>; | |
| 39 | + | |
| 40 | +const Response = z.object({ result: z.string().optional(), num_found: z.number().nullable().optional(), list: z.array(z.record(z.string(), z.unknown())).default([]) }); | |
| 41 | + | |
| 42 | +export function toItem(raw: Record<string, unknown>): Item | null { | |
| 43 | + const pid = raw.pid !== undefined && raw.pid !== null ? String(raw.pid) : null; | |
| 44 | + const name = typeof raw.name === 'string' ? raw.name : null; | |
| 45 | + const price = posNumber(raw.price); | |
| 46 | + if (!pid || !name || price === null) return null; | |
| 47 | + if (raw.type !== undefined && raw.type !== 'PRODUCT') return null; | |
| 48 | + const img = typeof raw.product_image === 'string' ? raw.product_image.replace('{res}', '600').replace('{cnt}', '1') : null; | |
| 49 | + return { | |
| 50 | + pid, | |
| 51 | + name, | |
| 52 | + price, | |
| 53 | + image: img, | |
| 54 | + status: raw.status === undefined || raw.status === null ? null : String(raw.status), | |
| 55 | + updateTime: intOrNull(raw.update_time), | |
| 56 | + used: intOrNull(raw.used), | |
| 57 | + categoryId: raw.category_id === undefined || raw.category_id === null ? null : String(raw.category_id), | |
| 58 | + tag: typeof raw.tag === 'string' ? raw.tag : null, | |
| 59 | + location: typeof raw.location === 'string' && raw.location ? raw.location : null, | |
| 60 | + freeShipping: typeof raw.free_shipping === 'boolean' ? raw.free_shipping : null, | |
| 61 | + bizseller: typeof raw.bizseller === 'boolean' ? raw.bizseller : null, | |
| 62 | + numFaved: intOrNull(raw.num_faved), | |
| 63 | + ad: typeof raw.ad === 'boolean' ? raw.ad : null, | |
| 64 | + }; | |
| 65 | +} | |
| 66 | + | |
| 67 | +export function searchUrl(q: string, page: number, n = PAGE_SIZE): string { | |
| 68 | + return `${API}?q=${encodeURIComponent(q)}&order=date&page=${page}&n=${n}&req_ref=search&stat_device=w&version=5`; | |
| 69 | +} | |
| 70 | + | |
| 71 | +export class BunjangConnector extends BaseConnector { | |
| 72 | + readonly version = '1.0.0'; | |
| 73 | + readonly parserVersion = PARSER_VERSION; | |
| 74 | + protected override minIntervalMs = 2500; | |
| 75 | + | |
| 76 | + private seeds(ctx: CrawlContext): KeywordSeed[] { | |
| 77 | + if (ctx.options.seeds?.length) return ctx.options.seeds.map((q) => KeywordSeedSchema.parse({ q, category: String(this.meta.config.defaultCategory ?? 'trading_cards') })); | |
| 78 | + const seeds = z.array(KeywordSeedSchema).parse(this.meta.config.seeds ?? []); | |
| 79 | + const filter = ctx.options.categories; | |
| 80 | + return filter?.length ? seeds.filter((s) => filter.includes(s.category)) : seeds; | |
| 81 | + } | |
| 82 | + | |
| 83 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 84 | + const seeds = this.seeds(ctx); | |
| 85 | + const pages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 1); | |
| 86 | + const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number; page?: number }; | |
| 87 | + let count = 0; | |
| 88 | + for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) { | |
| 89 | + const seed = seeds[si]!; | |
| 90 | + let page = si === (cur.seedIndex ?? 0) && cur.page !== undefined ? cur.page : 0; // Bunjang pages are 0-based | |
| 91 | + for (; page < pages; page++) { | |
| 92 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 93 | + const url = searchUrl(seed.q, page); | |
| 94 | + await this.throttle(url); | |
| 95 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => { | |
| 96 | + const parsed = Response.safeParse(r.json); | |
| 97 | + const first = parsed.success ? parsed.data.list.map(toItem).find(Boolean) : null; | |
| 98 | + return first ? { title: first.name, price: first.price } : parsed.success && parsed.data.list.length === 0 ? { title: 'empty', price: 1 } : null; | |
| 99 | + } }); | |
| 100 | + const parsed = Response.safeParse(res.json); | |
| 101 | + if (!res.success || !parsed.success) { | |
| 102 | + ctx.anomaly(res.success ? 'schema_drift' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 103 | + break; | |
| 104 | + } | |
| 105 | + const items = parsed.data.list.map(toItem).filter((x): x is Item => Boolean(x) && !(x as Item).ad); | |
| 106 | + if (!items.length) break; | |
| 107 | + const payload: PagePayload = { kind: 'search_page', seed, page, total: parsed.data.num_found ?? null, items }; | |
| 108 | + count++; | |
| 109 | + yield { url, externalId: `search:${seed.q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 110 | + await ctx.setCursor({ seedIndex: si, page: page + 1, at: new Date().toISOString() }); | |
| 111 | + await ctx.progress({ page: page + 1, totalPages: payload.total ? Math.min(pages, Math.ceil(payload.total / PAGE_SIZE)) : null, itemsProcessed: count }); | |
| 112 | + if (parsed.data.list.length < PAGE_SIZE) break; | |
| 113 | + } | |
| 114 | + await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() }); | |
| 115 | + } | |
| 116 | + await ctx.setCursor({ done: true, at: new Date().toISOString() }); | |
| 117 | + } | |
| 118 | + | |
| 119 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 120 | + const p = PagePayloadSchema.parse(raw.payload); | |
| 121 | + const out: NormalizedRecord[] = []; | |
| 122 | + const seen = new Set<string>(); | |
| 123 | + for (const it of p.items) { | |
| 124 | + if (seen.has(it.pid)) continue; | |
| 125 | + seen.add(it.pid); | |
| 126 | + const title = cleanTitle(it.name); | |
| 127 | + // Title first; seller hashtags (often listing several brands) only when the title alone stays at the family level. | |
| 128 | + let categorySlug = refineCategory(p.seed.category, title); | |
| 129 | + if (categorySlug === p.seed.category && it.tag) categorySlug = refineCategory(p.seed.category, it.tag); | |
| 130 | + const conditionRaw = it.used === 1 ? 'New' : it.used === 2 ? 'Used' : cjkConditionRaw(title); | |
| 131 | + const availability = it.status === '0' ? 'available' : it.status === '3' ? 'sold' : it.status === '1' ? 'available' : 'unknown'; | |
| 132 | + out.push( | |
| 133 | + NormalizedListingSchema.parse({ | |
| 134 | + kind: 'listing', | |
| 135 | + connectorId: this.meta.id, | |
| 136 | + sourceId: this.meta.sourceId, | |
| 137 | + sourceUrl: `${SITE}/products/${it.pid}`, | |
| 138 | + externalId: it.pid, | |
| 139 | + rawTitle: it.name, | |
| 140 | + imageUrls: it.image ? [it.image] : [], | |
| 141 | + attributes: AssetAttributesSchema.parse({ | |
| 142 | + categorySlug, | |
| 143 | + name: title, | |
| 144 | + language: p.seed.language ?? cjkLanguage(title), | |
| 145 | + country: 'KR', | |
| 146 | + identifiers: { bunjang_pid: it.pid }, | |
| 147 | + // update_time is the last bump/edit, not the original listing time → metadata only | |
| 148 | + metadata: { bunjang_category_id: it.categoryId, tags: it.tag, favourites: it.numFaved, business_seller: it.bizseller, free_shipping: it.freeShipping, region: it.location, status_code: it.status, seed_query: p.seed.q, updated_at: unixToDate(it.updateTime)?.toISOString() ?? null, is_bundle: isCjkBundle(title) }, | |
| 149 | + }), | |
| 150 | + grade: { ...cjkGrade(title), certificationNumber: null }, | |
| 151 | + condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null }, | |
| 152 | + observedAt: raw.fetchedAt, | |
| 153 | + confidence: 0.65, | |
| 154 | + parserVersion: PARSER_VERSION, | |
| 155 | + listingType: 'fixed_price', | |
| 156 | + price: it.price, | |
| 157 | + currency: 'KRW', | |
| 158 | + seller: null, | |
| 159 | + location: it.location ? `${it.location}, South Korea` : 'South Korea', | |
| 160 | + quantity: null, | |
| 161 | + listedAt: null, | |
| 162 | + endsAt: null, | |
| 163 | + availability, | |
| 164 | + bidCount: null, | |
| 165 | + }), | |
| 166 | + ); | |
| 167 | + } | |
| 168 | + return out; | |
| 169 | + } | |
| 170 | +} | |
| 171 | + | |
| 172 | +export default function createConnector(meta: ConnectorMeta) { | |
| 173 | + return new BunjangConnector(meta); | |
| 174 | +} | |
added
connectors/api/bunjang/meta.json
+50 −0
@@ -0,0 +1,50 @@ | ||
| 1 | +{ | |
| 2 | + "id": "bunjang", | |
| 3 | + "displayName": "Bunjang 번개장터 (Korea C2C marketplace — listings)", | |
| 4 | + "sourceId": "bunjang", | |
| 5 | + "sourceName": "Bunjang (번개장터)", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://m.bunjang.co.kr", | |
| 8 | + "module": "api/bunjang", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["pokemon", "yugioh", "one_piece_card_game", "trading_cards", "rolex", "other_watches", "nike_jordan", "adidas_yeezy", "sneakers", "action_figures", "gundam", "lego_sets", "designer_toys"], | |
| 11 | + "regions": ["KR"], | |
| 12 | + "country": "KR", | |
| 13 | + "languages": ["ko"], | |
| 14 | + "currency": ["KRW"], | |
| 15 | + "supportsListings": true, | |
| 16 | + "supportsSold": false, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 360, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.6, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://m.bunjang.co.kr/terms", | |
| 27 | + "acquisitionMethod": "public search JSON used by the mobile site, direct HTTP", | |
| 28 | + "historicalDepth": "none", | |
| 29 | + "accessNotes": "Reads the public, unauthenticated search endpoint the mobile site itself loads: `api.bunjang.co.kr/api/1/find_v2.json?q=<kw>&order=date&page=<0..>&n=100` (pid, title, KRW price, status, category id, tags, region label, favourites, image template, last-update time). api.bunjang.co.kr answers /robots.txt with a 403 JSON (no rules); m.bunjang.co.kr robots.txt allows `*` except /login, /apps, /talk2 (AI-training crawlers are named and disallowed separately; RareIndexBot is a market-data bot under `*`). Only asking prices are stored as listings — Bunjang shows no dated sale prices publicly. Sponsored rows (`ad: true`) are dropped. Seller ids/nicknames, chat and personal data are never read; the coarse region label (e.g. a district) is kept as listing location only. Item detail pages/API are not fetched. 2.5 s between requests, concurrency 1.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "requires": [], | |
| 33 | + "config": { | |
| 34 | + "seeds": [ | |
| 35 | + { "q": "포켓몬카드 PSA10", "category": "pokemon", "language": null }, | |
| 36 | + { "q": "포켓몬카드 일판", "category": "pokemon", "language": "Japanese" }, | |
| 37 | + { "q": "유희왕 PSA", "category": "yugioh", "language": null }, | |
| 38 | + { "q": "원피스 카드 PSA", "category": "one_piece_card_game", "language": null }, | |
| 39 | + { "q": "롤렉스 시계", "category": "rolex", "language": null }, | |
| 40 | + { "q": "그랜드세이코", "category": "other_watches", "language": null }, | |
| 41 | + { "q": "조던1 새상품", "category": "nike_jordan", "language": null }, | |
| 42 | + { "q": "이지 부스트 새상품", "category": "adidas_yeezy", "language": null }, | |
| 43 | + { "q": "건담 MG 미개봉", "category": "gundam", "language": null }, | |
| 44 | + { "q": "레고 미개봉", "category": "lego_sets", "language": null }, | |
| 45 | + { "q": "베어브릭 1000%", "category": "designer_toys", "language": null } | |
| 46 | + ], | |
| 47 | + "pagesPerSeed": 1, | |
| 48 | + "defaultCategory": "trading_cards" | |
| 49 | + } | |
| 50 | +} | |
added
connectors/api/capsule-toronto/README.md
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +# Capsule Toronto connector (`capsule-toronto`) | |
| 2 | + | |
| 3 | +- Source: https://www.capsuletoronto.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Toronto sneaker/streetwear boutique (Nike, Jordan, New Balance, adidas, ASICS, Converse). Shopify storefront; titles end with the manufacturer style code (e.g. IX3529-010). | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `footwear` | `sneakers` | — | | |
| 14 | +| `womens-footwear` | `sneakers` | — | | |
| 15 | +| `jordan` | `nike_jordan` | Jordan | | |
| 16 | +| `nike` | `nike_jordan` | Nike | | |
| 17 | +| `new-balance` | `new_balance_asics_other` | New Balance | | |
| 18 | +| `asics` | `new_balance_asics_other` | ASICS | | |
| 19 | +| `converse` | `new_balance_asics_other` | Converse | | |
| 20 | + | |
| 21 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 22 | +- `\b(air )?jordan\b|\bnike\b` → `nike_jordan` | |
| 23 | +- `\badidas\b|\byeezy\b` → `adidas_yeezy` | |
| 24 | +- `new balance|\basics\b|\bsalomon\b|\bhoka\b|\bpuma\b|\breebok\b|\bsaucony\b|\bconverse\b|\bvans\b|\bon running\b|\bmizuno\b` → `new_balance_asics_other` | |
| 25 | + | |
| 26 | +Excluded (regex): `gift card|^[^|]*\|(?![^|]*Footwear \|)[^|]*\||birkenstock|\bsandals?\b|\bslides?\b|\bclogs?\b|\bmules?\b|\bloafers?\b|\bboots?\b|\bugg\b|\bcrocs\b|\bsocks?\b|\blaces\b|insoles?|cleaner|shoe trees?` | |
| 27 | + | |
| 28 | +Title pattern: `^(?<name>.+?) (?<number>[A-Z]{1,3}\d{3,6}-\d{3}|\d{6}-\d{3}|[A-Z]{1,2}\d{3,4}[A-Z]{0,3}\d{0,3})$` (name / set / number) | |
| 29 | + | |
| 30 | +## Access & compliance | |
| 31 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: apparel (t-shirts, sweaters, outerwear, bottoms), headwear, accessories, bags — the exclude regex also drops anything whose product_type is not Footwear. | |
| 32 | + | |
| 33 | +## Fixtures & tests | |
| 34 | +`data/fixtures/capsule-toronto/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts capsule-toronto`), trimmed single-product payloads incl. a sold-out variant. | |
| 35 | +`pnpm vitest run connectors/api/capsule-toronto` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts capsule-toronto`. | |
added
connectors/api/capsule-toronto/index.test.ts
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Capsule Toronto — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/capsule-toronto/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "probeType": "Footwear", | |
| 13 | + "cases": [ | |
| 14 | + { | |
| 15 | + "title": "Air Jordan 1 Low SE BLACK/TOUR YELLOW IX3529-010", | |
| 16 | + "productType": "Footwear", | |
| 17 | + "collection": "footwear", | |
| 18 | + "categorySlug": "nike_jordan" | |
| 19 | + }, | |
| 20 | + { | |
| 21 | + "title": "New Balance 990v6 GREY M990GL6", | |
| 22 | + "productType": "Footwear", | |
| 23 | + "collection": "footwear", | |
| 24 | + "categorySlug": "new_balance_asics_other" | |
| 25 | + }, | |
| 26 | + { | |
| 27 | + "title": "Nike Women's Air Max 1 '87 SAIL/BLACK DD9702-100", | |
| 28 | + "productType": "Women's Footwear", | |
| 29 | + "collection": "womens-footwear", | |
| 30 | + "categorySlug": "nike_jordan" | |
| 31 | + }, | |
| 32 | + { | |
| 33 | + "title": "Nike ACG Wolf Tree Polartec Fleece", | |
| 34 | + "productType": "Sweaters", | |
| 35 | + "collection": "nike", | |
| 36 | + "categorySlug": null | |
| 37 | + } | |
| 38 | + ] | |
| 39 | +}); | |
added
connectors/api/capsule-toronto/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Capsule Toronto — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/capsule-toronto/meta.json
+107 −0
@@ -0,0 +1,107 @@ | ||
| 1 | +{ | |
| 2 | + "id": "capsule-toronto", | |
| 3 | + "displayName": "Capsule Toronto (Canadian sneaker store, CAD)", | |
| 4 | + "sourceId": "capsule-toronto", | |
| 5 | + "sourceName": "Capsule Toronto", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.capsuletoronto.com", | |
| 8 | + "module": "api/capsule-toronto", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "sneakers", | |
| 14 | + "nike_jordan", | |
| 15 | + "new_balance_asics_other", | |
| 16 | + "adidas_yeezy" | |
| 17 | + ], | |
| 18 | + "regions": [ | |
| 19 | + "CA" | |
| 20 | + ], | |
| 21 | + "languages": [ | |
| 22 | + "en" | |
| 23 | + ], | |
| 24 | + "currency": [ | |
| 25 | + "CAD" | |
| 26 | + ], | |
| 27 | + "supportsListings": true, | |
| 28 | + "supportsSold": false, | |
| 29 | + "supportsAuctions": false, | |
| 30 | + "supportsImages": true, | |
| 31 | + "supportsCatalog": false, | |
| 32 | + "supportsPopulation": false, | |
| 33 | + "supportsLookup": true, | |
| 34 | + "refreshFrequencyMinutes": 720, | |
| 35 | + "priority": "medium", | |
| 36 | + "trustScore": 0.75, | |
| 37 | + "attributionRequired": true, | |
| 38 | + "termsUrl": "https://www.capsuletoronto.com/policies/terms-of-service", | |
| 39 | + "accessNotes": "Capsule Toronto (capsuletoronto.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 7 configured collections (footwear, womens-footwear, jordan, nike, new-balance, asics, converse) and /products/<handle>.json for URL lookups (~9k footwear SKUs; Nike 2.4k, Jordan 1k, New Balance 900). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — apparel (t-shirts, sweaters, outerwear, bottoms), headwear, accessories, bags — the exclude regex also drops anything whose product_type is not Footwear. No personal data is collected; seller = the store itself.", | |
| 40 | + "enabled": true, | |
| 41 | + "schemaVersion": "1.0", | |
| 42 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 43 | + "historicalDepth": "none", | |
| 44 | + "requires": [], | |
| 45 | + "config": { | |
| 46 | + "currency": "CAD", | |
| 47 | + "market": "CA", | |
| 48 | + "seller": "Capsule Toronto", | |
| 49 | + "location": "Toronto, ON, Canada", | |
| 50 | + "collections": [ | |
| 51 | + { | |
| 52 | + "handle": "footwear", | |
| 53 | + "categorySlug": "sneakers" | |
| 54 | + }, | |
| 55 | + { | |
| 56 | + "handle": "womens-footwear", | |
| 57 | + "categorySlug": "sneakers" | |
| 58 | + }, | |
| 59 | + { | |
| 60 | + "handle": "jordan", | |
| 61 | + "categorySlug": "nike_jordan", | |
| 62 | + "brand": "Jordan" | |
| 63 | + }, | |
| 64 | + { | |
| 65 | + "handle": "nike", | |
| 66 | + "categorySlug": "nike_jordan", | |
| 67 | + "brand": "Nike" | |
| 68 | + }, | |
| 69 | + { | |
| 70 | + "handle": "new-balance", | |
| 71 | + "categorySlug": "new_balance_asics_other", | |
| 72 | + "brand": "New Balance" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "handle": "asics", | |
| 76 | + "categorySlug": "new_balance_asics_other", | |
| 77 | + "brand": "ASICS" | |
| 78 | + }, | |
| 79 | + { | |
| 80 | + "handle": "converse", | |
| 81 | + "categorySlug": "new_balance_asics_other", | |
| 82 | + "brand": "Converse" | |
| 83 | + } | |
| 84 | + ], | |
| 85 | + "rules": [ | |
| 86 | + { | |
| 87 | + "match": "\\b(air )?jordan\\b|\\bnike\\b", | |
| 88 | + "categorySlug": "nike_jordan" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "match": "\\badidas\\b|\\byeezy\\b", | |
| 92 | + "categorySlug": "adidas_yeezy" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "match": "new balance|\\basics\\b|\\bsalomon\\b|\\bhoka\\b|\\bpuma\\b|\\breebok\\b|\\bsaucony\\b|\\bconverse\\b|\\bvans\\b|\\bon running\\b|\\bmizuno\\b", | |
| 96 | + "categorySlug": "new_balance_asics_other" | |
| 97 | + } | |
| 98 | + ], | |
| 99 | + "defaultCategory": null, | |
| 100 | + "exclude": "gift card|^[^|]*\\|(?![^|]*Footwear \\|)[^|]*\\||birkenstock|\\bsandals?\\b|\\bslides?\\b|\\bclogs?\\b|\\bmules?\\b|\\bloafers?\\b|\\bboots?\\b|\\bugg\\b|\\bcrocs\\b|\\bsocks?\\b|\\blaces\\b|insoles?|cleaner|shoe trees?", | |
| 101 | + "keepOutOfStock": true, | |
| 102 | + "fetchBarcodes": false, | |
| 103 | + "wholeShop": false, | |
| 104 | + "pageSize": 250, | |
| 105 | + "titlePattern": "^(?<name>.+?) (?<number>[A-Z]{1,3}\\d{3,6}-\\d{3}|\\d{6}-\\d{3}|[A-Z]{1,2}\\d{3,4}[A-Z]{0,3}\\d{0,3})$" | |
| 106 | + } | |
| 107 | +} | |
added
connectors/api/cardboard-memories/README.md
+45 −0
@@ -0,0 +1,45 @@ | ||
| 1 | +# Cardboard Memories connector (`cardboard-memories`) | |
| 2 | + | |
| 3 | +- Source: https://www.cardboardmemories.ca · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +New Brunswick comic & sports-card shop. Shopify storefront; hockey/football/baseball boxes with UPC-like SKUs, Yu-Gi-Oh! singles, comics by publisher, Funko. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `hockey-cards` | `hockey_cards` | — | | |
| 14 | +| `football-cards-1` | `football_cards` | — | | |
| 15 | +| `baseball-cards` | `baseball_cards` | — | | |
| 16 | +| `basketball-cards` | `basketball_cards` | — | | |
| 17 | +| `sports-cards` | `sports_cards` | — | | |
| 18 | +| `upper-deck-authenticated` | `sports_memorabilia` | Upper Deck Authenticated | | |
| 19 | +| `yu-gi-oh-singles` | `yugioh` | Yu-Gi-Oh! | | |
| 20 | +| `pokemon` | `pokemon` | Pokémon | | |
| 21 | +| `pokemon-booster-boxes` | `pokemon` | Pokémon | | |
| 22 | +| `magic-the-gathering` | `magic_the_gathering` | Magic: The Gathering | | |
| 23 | +| `magic-the-gathering-boxes` | `magic_the_gathering` | Magic: The Gathering | | |
| 24 | +| `funko-pop` | `funko` | Funko | | |
| 25 | +| `comic-books` | `comics` | — | | |
| 26 | +| `dc-comics` | `dc_comics` | DC | | |
| 27 | + | |
| 28 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 29 | +- `^(?=[\s\S]*\bhockey\b)[\s\S]*\| sports-cards$` → `hockey_cards` | |
| 30 | +- `^(?=[\s\S]*\bbaseball\b)[\s\S]*\| sports-cards$` → `baseball_cards` | |
| 31 | +- `^(?=[\s\S]*\bbasketball\b)[\s\S]*\| sports-cards$` → `basketball_cards` | |
| 32 | +- `^(?=[\s\S]*\bfootball\b)[\s\S]*\| sports-cards$` → `football_cards` | |
| 33 | +- `^(?=[\s\S]*\b(soccer|premier league|uefa|fifa)\b)[\s\S]*\| sports-cards$` → `soccer_cards` | |
| 34 | +- `^(?=[\s\S]*\b(formula 1|formula one|\bf1\b)\b)[\s\S]*\| sports-cards$` → `f1_cards` | |
| 35 | +- `^(?=[\s\S]*\b(marvel|spider-?man|x-men|avengers|wolverine|hulk|deadpool|iron man|captain america|fantastic four|daredevil|venom|punisher)\b)[\s\S]*\| (comic-books)$` → `marvel_comics` (franchise Marvel) | |
| 36 | +- `^(?=[\s\S]*\b(dc comics|batman|superman|wonder woman|justice league|the flash|green lantern|aquaman|detective comics|action comics|harley quinn|joker|nightwing|swamp thing)\b)[\s\S]*\| (comic-books)$` → `dc_comics` (franchise DC) | |
| 37 | + | |
| 38 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live|bags? (and|&) boards|\bmylar|comic bags|backing boards|magnetic holder|one-touch|\bpaint` | |
| 39 | + | |
| 40 | +## Access & compliance | |
| 41 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: RPG books, miniatures, paints, supplies (binders, sleeves, deck boxes), trade paperbacks. | |
| 42 | + | |
| 43 | +## Fixtures & tests | |
| 44 | +`data/fixtures/cardboard-memories/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts cardboard-memories`), trimmed single-product payloads incl. a sold-out variant. | |
| 45 | +`pnpm vitest run connectors/api/cardboard-memories` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts cardboard-memories`. | |
added
connectors/api/cardboard-memories/index.test.ts
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Cardboard Memories — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/cardboard-memories/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "Upper Deck - 2021-22 - Hockey - Series 1 - Trading Card Hobby Box", | |
| 15 | + "productType": "Sports Cards", | |
| 16 | + "collection": "hockey-cards", | |
| 17 | + "categorySlug": "hockey_cards" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "2023 Panini Prizm Football Blaster Box", | |
| 21 | + "productType": "Sports Cards", | |
| 22 | + "collection": "sports-cards", | |
| 23 | + "categorySlug": "football_cards" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "title": "Ultra Pro One-Touch Magnetic Holder 35pt", | |
| 27 | + "collection": "hockey-cards", | |
| 28 | + "categorySlug": null | |
| 29 | + }, | |
| 30 | + { | |
| 31 | + "title": "Amazing Spider-Man #300 Facsimile", | |
| 32 | + "collection": "comic-books", | |
| 33 | + "categorySlug": "marvel_comics" | |
| 34 | + } | |
| 35 | + ] | |
| 36 | +}); | |
added
connectors/api/cardboard-memories/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Cardboard Memories — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/cardboard-memories/meta.json
+170 −0
@@ -0,0 +1,170 @@ | ||
| 1 | +{ | |
| 2 | + "id": "cardboard-memories", | |
| 3 | + "displayName": "Cardboard Memories (Canadian sports-card & TCG & collector-toy store, CAD)", | |
| 4 | + "sourceId": "cardboard-memories", | |
| 5 | + "sourceName": "Cardboard Memories", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.cardboardmemories.ca", | |
| 8 | + "module": "api/cardboard-memories", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "hockey_cards", | |
| 14 | + "football_cards", | |
| 15 | + "baseball_cards", | |
| 16 | + "basketball_cards", | |
| 17 | + "sports_cards", | |
| 18 | + "sports_memorabilia", | |
| 19 | + "yugioh", | |
| 20 | + "pokemon", | |
| 21 | + "magic_the_gathering", | |
| 22 | + "funko", | |
| 23 | + "comics", | |
| 24 | + "dc_comics", | |
| 25 | + "soccer_cards", | |
| 26 | + "f1_cards", | |
| 27 | + "marvel_comics" | |
| 28 | + ], | |
| 29 | + "regions": [ | |
| 30 | + "CA" | |
| 31 | + ], | |
| 32 | + "languages": [ | |
| 33 | + "en" | |
| 34 | + ], | |
| 35 | + "currency": [ | |
| 36 | + "CAD" | |
| 37 | + ], | |
| 38 | + "supportsListings": true, | |
| 39 | + "supportsSold": false, | |
| 40 | + "supportsAuctions": false, | |
| 41 | + "supportsImages": true, | |
| 42 | + "supportsCatalog": false, | |
| 43 | + "supportsPopulation": false, | |
| 44 | + "supportsLookup": true, | |
| 45 | + "refreshFrequencyMinutes": 720, | |
| 46 | + "priority": "medium", | |
| 47 | + "trustScore": 0.75, | |
| 48 | + "attributionRequired": true, | |
| 49 | + "termsUrl": "https://www.cardboardmemories.ca/policies/terms-of-service", | |
| 50 | + "accessNotes": "Cardboard Memories (cardboardmemories.ca) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 14 configured collections (hockey-cards, football-cards-1, baseball-cards, basketball-cards, sports-cards, upper-deck-authenticated, yu-gi-oh-singles, pokemon … (+6 more, see config.collections)) and /products/<handle>.json for URL lookups (~26k comics, 1.1k hockey card products, Yu-Gi-Oh!/Pokémon singles, Funko). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — RPG books, miniatures, paints, supplies (binders, sleeves, deck boxes), trade paperbacks. No personal data is collected; seller = the store itself.", | |
| 51 | + "enabled": true, | |
| 52 | + "schemaVersion": "1.0", | |
| 53 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 54 | + "historicalDepth": "none", | |
| 55 | + "requires": [], | |
| 56 | + "config": { | |
| 57 | + "currency": "CAD", | |
| 58 | + "market": "CA", | |
| 59 | + "seller": "Cardboard Memories", | |
| 60 | + "location": "Moncton, NB, Canada", | |
| 61 | + "collections": [ | |
| 62 | + { | |
| 63 | + "handle": "hockey-cards", | |
| 64 | + "categorySlug": "hockey_cards" | |
| 65 | + }, | |
| 66 | + { | |
| 67 | + "handle": "football-cards-1", | |
| 68 | + "categorySlug": "football_cards" | |
| 69 | + }, | |
| 70 | + { | |
| 71 | + "handle": "baseball-cards", | |
| 72 | + "categorySlug": "baseball_cards" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "handle": "basketball-cards", | |
| 76 | + "categorySlug": "basketball_cards" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "sports-cards", | |
| 80 | + "categorySlug": "sports_cards" | |
| 81 | + }, | |
| 82 | + { | |
| 83 | + "handle": "upper-deck-authenticated", | |
| 84 | + "categorySlug": "sports_memorabilia", | |
| 85 | + "brand": "Upper Deck Authenticated" | |
| 86 | + }, | |
| 87 | + { | |
| 88 | + "handle": "yu-gi-oh-singles", | |
| 89 | + "categorySlug": "yugioh", | |
| 90 | + "franchise": "Yu-Gi-Oh!" | |
| 91 | + }, | |
| 92 | + { | |
| 93 | + "handle": "pokemon", | |
| 94 | + "categorySlug": "pokemon", | |
| 95 | + "franchise": "Pokémon" | |
| 96 | + }, | |
| 97 | + { | |
| 98 | + "handle": "pokemon-booster-boxes", | |
| 99 | + "categorySlug": "pokemon", | |
| 100 | + "franchise": "Pokémon" | |
| 101 | + }, | |
| 102 | + { | |
| 103 | + "handle": "magic-the-gathering", | |
| 104 | + "categorySlug": "magic_the_gathering", | |
| 105 | + "franchise": "Magic: The Gathering" | |
| 106 | + }, | |
| 107 | + { | |
| 108 | + "handle": "magic-the-gathering-boxes", | |
| 109 | + "categorySlug": "magic_the_gathering", | |
| 110 | + "franchise": "Magic: The Gathering" | |
| 111 | + }, | |
| 112 | + { | |
| 113 | + "handle": "funko-pop", | |
| 114 | + "categorySlug": "funko", | |
| 115 | + "brand": "Funko" | |
| 116 | + }, | |
| 117 | + { | |
| 118 | + "handle": "comic-books", | |
| 119 | + "categorySlug": "comics" | |
| 120 | + }, | |
| 121 | + { | |
| 122 | + "handle": "dc-comics", | |
| 123 | + "categorySlug": "dc_comics", | |
| 124 | + "franchise": "DC" | |
| 125 | + } | |
| 126 | + ], | |
| 127 | + "rules": [ | |
| 128 | + { | |
| 129 | + "match": "^(?=[\\s\\S]*\\bhockey\\b)[\\s\\S]*\\| sports-cards$", | |
| 130 | + "categorySlug": "hockey_cards" | |
| 131 | + }, | |
| 132 | + { | |
| 133 | + "match": "^(?=[\\s\\S]*\\bbaseball\\b)[\\s\\S]*\\| sports-cards$", | |
| 134 | + "categorySlug": "baseball_cards" | |
| 135 | + }, | |
| 136 | + { | |
| 137 | + "match": "^(?=[\\s\\S]*\\bbasketball\\b)[\\s\\S]*\\| sports-cards$", | |
| 138 | + "categorySlug": "basketball_cards" | |
| 139 | + }, | |
| 140 | + { | |
| 141 | + "match": "^(?=[\\s\\S]*\\bfootball\\b)[\\s\\S]*\\| sports-cards$", | |
| 142 | + "categorySlug": "football_cards" | |
| 143 | + }, | |
| 144 | + { | |
| 145 | + "match": "^(?=[\\s\\S]*\\b(soccer|premier league|uefa|fifa)\\b)[\\s\\S]*\\| sports-cards$", | |
| 146 | + "categorySlug": "soccer_cards" | |
| 147 | + }, | |
| 148 | + { | |
| 149 | + "match": "^(?=[\\s\\S]*\\b(formula 1|formula one|\\bf1\\b)\\b)[\\s\\S]*\\| sports-cards$", | |
| 150 | + "categorySlug": "f1_cards" | |
| 151 | + }, | |
| 152 | + { | |
| 153 | + "match": "^(?=[\\s\\S]*\\b(marvel|spider-?man|x-men|avengers|wolverine|hulk|deadpool|iron man|captain america|fantastic four|daredevil|venom|punisher)\\b)[\\s\\S]*\\| (comic-books)$", | |
| 154 | + "categorySlug": "marvel_comics", | |
| 155 | + "franchise": "Marvel" | |
| 156 | + }, | |
| 157 | + { | |
| 158 | + "match": "^(?=[\\s\\S]*\\b(dc comics|batman|superman|wonder woman|justice league|the flash|green lantern|aquaman|detective comics|action comics|harley quinn|joker|nightwing|swamp thing)\\b)[\\s\\S]*\\| (comic-books)$", | |
| 159 | + "categorySlug": "dc_comics", | |
| 160 | + "franchise": "DC" | |
| 161 | + } | |
| 162 | + ], | |
| 163 | + "defaultCategory": null, | |
| 164 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live|bags? (and|&) boards|\\bmylar|comic bags|backing boards|magnetic holder|one-touch|\\bpaint", | |
| 165 | + "keepOutOfStock": true, | |
| 166 | + "fetchBarcodes": false, | |
| 167 | + "wholeShop": false, | |
| 168 | + "pageSize": 250 | |
| 169 | + } | |
| 170 | +} | |
added
connectors/api/cardmarket-priceguide/README.md
+12 −0
@@ -0,0 +1,12 @@ | ||
| 1 | +# cardmarket-priceguide | |
| 2 | + | |
| 3 | +Cardmarket's public bulk files (no key, no storefront crawling): | |
| 4 | + | |
| 5 | +- `productCatalog/priceGuide/price_guide_<idGame>.json` — daily EUR guide per product: `avg`, `low`, `trend`, `avg1`, `avg7`, `avg30` and the `*-foil` twins. `createdAt` of the file is the observation date. | |
| 6 | +- `productCatalog/productList/products_singles_<idGame>.json` / `products_nonsingles_<idGame>.json` — `idProduct`, `name`, `idCategory`/`categoryName`, `idExpansion`, `idMetacard`, `dateAdded`. | |
| 7 | + | |
| 8 | +Game ids (probed live; 403 = absent): 1 Magic · 2 WoW · 3 Yu-Gi-Oh! · 5 Spoils · 6 Pokémon · 7 Force of Will · 8 Vanguard · 9 Final Fantasy · 10 Weiss Schwarz · 11 Dragoborne · 12 My Little Pony · 13 Dragon Ball Super · 15 SW Destiny · 16 Flesh and Blood · 17 Digimon · 18 One Piece · 19 Lorcana · 20 Battle Spirits Saga · 21 SW Unlimited · 22 Riftbound. `config.games` maps ids → taxonomy slug. | |
| 9 | + | |
| 10 | +Output: one `catalog_item` per product (`identifiers.cardmarket_id` = idProduct, the key MTGJSON also emits), a second one with variant `Foil` (`Reverse Holo` for Pokémon) when foil prices exist, and `price_observation`s per field (`avg`→mid, `low`→low, `trend`→trend, `avg1`→market, `avg7`→average_7d, `avg30`→average_30d). The files carry no expansion names, numbers or images. | |
| 11 | + | |
| 12 | +Cursor `{ gameIdx, offset }`; ≤ 3 requests per game per run. Fixtures: `pnpm tsx connectors/api/cardmarket-priceguide/_capture.ts`. | |
added
connectors/api/cardmarket-priceguide/_capture.ts
+48 −0
@@ -0,0 +1,48 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture: joins one game's price guide with its product list and saves a few products | |
| 3 | + * (Magic Black Lotus-like foil card, a Pokémon card with a disambiguator, a One Piece numbered card, | |
| 4 | + * a sealed non-single). Usage: pnpm tsx connectors/api/cardmarket-priceguide/_capture.ts | |
| 5 | + */ | |
| 6 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 7 | +import meta from './meta.json' with { type: 'json' }; | |
| 8 | +import { PriceRowSchema, ProductSchema, hasAnyPrice, type CardmarketPayload, type PriceRow } from './index.js'; | |
| 9 | +import { JSON_HEADERS } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 10 | + | |
| 11 | +const BASE = 'https://downloads.s3.cardmarket.com/productCatalog'; | |
| 12 | +const get = async (u: string) => (await (await fetch(u, { headers: JSON_HEADERS })).json()) as { createdAt?: string; priceGuides?: unknown[]; products?: unknown[] }; | |
| 13 | + | |
| 14 | +async function game(gameId: number) { | |
| 15 | + const guide = await get(`${BASE}/priceGuide/price_guide_${gameId}.json`); | |
| 16 | + const prices = new Map<number, PriceRow>(); | |
| 17 | + for (const r of guide.priceGuides ?? []) { | |
| 18 | + const p = PriceRowSchema.safeParse(r); | |
| 19 | + if (p.success) prices.set(p.data.idProduct, p.data); | |
| 20 | + } | |
| 21 | + const singles = await get(`${BASE}/productList/products_singles_${gameId}.json`); | |
| 22 | + const nonsingles = await get(`${BASE}/productList/products_nonsingles_${gameId}.json`); | |
| 23 | + return { guide, prices, singles, nonsingles }; | |
| 24 | +} | |
| 25 | + | |
| 26 | +const games = (meta.config.games as Record<string, { slug: string; name: string; franchise?: string | null; brand?: string | null; foilVariant?: string }>); | |
| 27 | + | |
| 28 | +async function capture(gameId: number, name: string, pick: (products: unknown[], prices: Map<number, PriceRow>) => unknown, single: boolean, note: string) { | |
| 29 | + const g = await game(gameId); | |
| 30 | + const rawProduct = pick((single ? g.singles.products : g.nonsingles.products) ?? [], g.prices); | |
| 31 | + if (!rawProduct) throw new Error(`${name}: product not found`); | |
| 32 | + const product = ProductSchema.parse(rawProduct); | |
| 33 | + const cfg = games[String(gameId)]!; | |
| 34 | + const payload: CardmarketPayload = { gameId, game: { slug: cfg.slug, name: cfg.name, franchise: cfg.franchise ?? null, brand: cfg.brand ?? null, foilVariant: cfg.foilVariant ?? 'Foil' }, product, prices: g.prices.get(product.idProduct) ?? null, single, priceGuideCreatedAt: g.guide.createdAt ?? null, productListCreatedAt: (single ? g.singles : g.nonsingles).createdAt ?? null }; | |
| 35 | + saveFixture('cardmarket-priceguide', name, { | |
| 36 | + raw: { url: `${BASE}/priceGuide/price_guide_${gameId}.json#idProduct=${product.idProduct}`, externalId: String(product.idProduct), kind: 'catalog_item', engine: 'api', fetchedAt: new Date(), payload }, | |
| 37 | + expect: { minCount: 2, kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.identifiers.cardmarket_id', 'attributes.name'] }, | |
| 38 | + note: `Live capture ${new Date().toISOString().slice(0, 10)} — ${note} (price guide createdAt ${g.guide.createdAt})`, | |
| 39 | + }); | |
| 40 | + console.log(name, product.idProduct, product.name, JSON.stringify(g.prices.get(product.idProduct))); | |
| 41 | +} | |
| 42 | + | |
| 43 | +const byName = (re: RegExp, needFoil = false) => (products: unknown[], prices: Map<number, PriceRow>) => products.find((p) => re.test(String((p as { name: string }).name)) && (!needFoil || hasAnyPrice(prices.get((p as { idProduct: number }).idProduct), true))); | |
| 44 | + | |
| 45 | +await capture(1, 'magic-foil-single', byName(/^Lightning Bolt$/, true), true, 'Magic single with foil and non-foil price rows'); | |
| 46 | +await capture(6, 'pokemon-disambiguated', byName(/^Charizard \[/), true, 'Pokémon single whose name carries a [move | set] disambiguator; foil = Reverse Holo'); | |
| 47 | +await capture(18, 'one-piece-numbered', byName(/^Roronoa Zoro \(OP01-001\)$/), true, 'One Piece single with the card number in parentheses'); | |
| 48 | +await capture(1, 'magic-nonsingle-booster', byName(/^Alpha Booster$/), false, 'Magic sealed product from products_nonsingles (no foil, completeness sealed)'); | |
added
connectors/api/cardmarket-priceguide/index.test.ts
+63 −0
@@ -0,0 +1,63 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import createConnector, { PRICE_FIELDS, hasAnyPrice, parseProductName } from './index.js'; | |
| 6 | +import { isoDay } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 7 | + | |
| 8 | +const connector = createConnector(localMeta(meta)); | |
| 9 | + | |
| 10 | +describe('cardmarket-priceguide', () => { | |
| 11 | + runFixtureSuite(connector, it, expect); | |
| 12 | + | |
| 13 | + it('parses Cardmarket product names', () => { | |
| 14 | + expect(parseProductName('Kakuna [Bug Bite | Primal Clash]')).toEqual({ name: 'Kakuna', number: null, version: null, disambiguation: 'Bug Bite | Primal Clash' }); | |
| 15 | + expect(parseProductName('Roronoa Zoro (OP01-001)')).toEqual({ name: 'Roronoa Zoro', number: 'OP01-001', version: null, disambiguation: null }); | |
| 16 | + expect(parseProductName('Yokomon (BT1-001)').number).toBe('BT1-001'); | |
| 17 | + expect(parseProductName('Auron (1-001)').number).toBe('1-001'); | |
| 18 | + expect(parseProductName('Forest (V.1)')).toEqual({ name: 'Forest', number: null, version: 'V.1', disambiguation: null }); | |
| 19 | + expect(parseProductName('Whis's Coercion').name).toBe("Whis's Coercion"); | |
| 20 | + // unknown parentheticals stay part of the name (never guessed as numbers) | |
| 21 | + expect(parseProductName('Sol Ring (Extended Art)').name).toBe('Sol Ring (Extended Art)'); | |
| 22 | + }); | |
| 23 | + | |
| 24 | + it('dates observations with the file createdAt (source date, not fetch time)', () => { | |
| 25 | + expect(isoDay('2026-09-07T02:48:04+0200')?.toISOString()).toBe('2026-09-07T00:00:00.000Z'); | |
| 26 | + }); | |
| 27 | + | |
| 28 | + it('detects foil rows and maps every price field to a priceKind', () => { | |
| 29 | + expect(hasAnyPrice({ idProduct: 1, avg: null, low: null, 'trend-foil': 0.36 }, true)).toBe(true); | |
| 30 | + expect(hasAnyPrice({ idProduct: 1, avg: null, low: null, 'trend-foil': 0.36 }, false)).toBe(false); | |
| 31 | + expect(hasAnyPrice(null)).toBe(false); | |
| 32 | + expect(PRICE_FIELDS.map(([, k]) => k)).toEqual(['mid', 'low', 'trend', 'market', 'average_7d', 'average_30d']); | |
| 33 | + }); | |
| 34 | + | |
| 35 | + it('emits cardmarket_id identifiers, EUR observations and a foil variant when foil prices exist', async () => { | |
| 36 | + for (const name of listFixtures('cardmarket-priceguide')) { | |
| 37 | + const fx = loadFixture('cardmarket-priceguide', name); | |
| 38 | + const out = await connector.normalize(fx.raw); | |
| 39 | + const cats = out.filter((r) => r.kind === 'catalog_item'); | |
| 40 | + const obs = out.filter((r) => r.kind === 'price_observation'); | |
| 41 | + expect(cats.length).toBeGreaterThanOrEqual(1); | |
| 42 | + expect(obs.length).toBeGreaterThanOrEqual(1); | |
| 43 | + for (const c of cats) { | |
| 44 | + if (c.kind !== 'catalog_item') continue; | |
| 45 | + expect(c.attributes.identifiers.cardmarket_id).toMatch(/^\d+$/); | |
| 46 | + expect(c.sourceUrl).toContain('downloads.s3.cardmarket.com'); | |
| 47 | + } | |
| 48 | + for (const o of obs) { | |
| 49 | + if (o.kind !== 'price_observation') continue; | |
| 50 | + expect(o.currency).toBe('EUR'); | |
| 51 | + expect(o.price).toBeGreaterThan(0); | |
| 52 | + expect(o.observationDate.getUTCHours()).toBe(0); | |
| 53 | + expect(o.observationDate.getTime()).toBeLessThanOrEqual(Date.now()); | |
| 54 | + expect((o.attributes.metadata as { cardmarket_field: string }).cardmarket_field).toBeTruthy(); | |
| 55 | + } | |
| 56 | + const payload = fx.raw.payload as { prices: Record<string, number | null> | null; single: boolean; game: { foilVariant: string } }; | |
| 57 | + const foilPresent = Boolean(payload.single && payload.prices && Object.entries(payload.prices).some(([k, v]) => k.endsWith('-foil') && typeof v === 'number' && v > 0)); | |
| 58 | + const foilCat = cats.find((c) => c.kind === 'catalog_item' && c.attributes.variant === payload.game.foilVariant); | |
| 59 | + expect(Boolean(foilCat)).toBe(foilPresent); | |
| 60 | + if (!payload.single) for (const c of cats) if (c.kind === 'catalog_item') expect(c.condition.completeness).toBe('sealed'); | |
| 61 | + } | |
| 62 | + }); | |
| 63 | +}); | |
added
connectors/api/cardmarket-priceguide/index.ts
+289 −0
@@ -0,0 +1,289 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { attrs, catalogItem, makeTitle, priceObservation } from '../_lib/shared.js'; | |
| 5 | +import { JSON_HEADERS, dayOf, isoDay } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Cardmarket public bulk files: daily price guide (EUR) joined with the product catalog per game. | |
| 9 | + * One raw record per product; normalize emits a catalog item (plus a foil variant when foil prices | |
| 10 | + * exist) and one price observation per Cardmarket price field, dated by the file's own createdAt. | |
| 11 | + */ | |
| 12 | +const BASE = 'https://downloads.s3.cardmarket.com/productCatalog'; | |
| 13 | +const PARSER_VERSION = '1.0.0'; | |
| 14 | + | |
| 15 | +const GameCfgSchema = z.object({ slug: z.string(), name: z.string(), franchise: z.string().nullable().optional(), brand: z.string().nullable().optional(), foilVariant: z.string().default('Foil') }); | |
| 16 | +export type GameCfg = z.infer<typeof GameCfgSchema>; | |
| 17 | + | |
| 18 | +export const ProductSchema = z.object({ | |
| 19 | + idProduct: z.number().int(), | |
| 20 | + name: z.string(), | |
| 21 | + idCategory: z.number().int(), | |
| 22 | + categoryName: z.string(), | |
| 23 | + idExpansion: z.number().int().nullable().optional(), | |
| 24 | + idMetacard: z.number().int().nullable().optional(), | |
| 25 | + dateAdded: z.string().nullable().optional(), | |
| 26 | +}); | |
| 27 | +export type CardmarketProduct = z.infer<typeof ProductSchema>; | |
| 28 | + | |
| 29 | +const price = z.number().nullable().optional(); | |
| 30 | +export const PriceRowSchema = z.object({ | |
| 31 | + idProduct: z.number().int(), | |
| 32 | + idCategory: z.number().int().optional(), | |
| 33 | + avg: price, | |
| 34 | + low: price, | |
| 35 | + trend: price, | |
| 36 | + avg1: price, | |
| 37 | + avg7: price, | |
| 38 | + avg30: price, | |
| 39 | + 'avg-foil': price, | |
| 40 | + 'low-foil': price, | |
| 41 | + 'trend-foil': price, | |
| 42 | + 'avg1-foil': price, | |
| 43 | + 'avg7-foil': price, | |
| 44 | + 'avg30-foil': price, | |
| 45 | +}); | |
| 46 | +export type PriceRow = z.infer<typeof PriceRowSchema>; | |
| 47 | + | |
| 48 | +const RawPayloadSchema = z.object({ | |
| 49 | + gameId: z.number().int(), | |
| 50 | + game: GameCfgSchema, | |
| 51 | + product: ProductSchema, | |
| 52 | + prices: PriceRowSchema.nullable(), | |
| 53 | + single: z.boolean(), | |
| 54 | + priceGuideCreatedAt: z.string().nullable(), | |
| 55 | + productListCreatedAt: z.string().nullable(), | |
| 56 | +}); | |
| 57 | +export type CardmarketPayload = z.infer<typeof RawPayloadSchema>; | |
| 58 | + | |
| 59 | +const PriceGuideFileSchema = z.object({ version: z.number().optional(), createdAt: z.string().nullable().optional(), priceGuides: z.array(z.unknown()) }); | |
| 60 | +const ProductFileSchema = z.object({ version: z.number().optional(), createdAt: z.string().nullable().optional(), products: z.array(z.unknown()) }); | |
| 61 | + | |
| 62 | +/** Cardmarket price field → RareIndex priceKind. */ | |
| 63 | +export const PRICE_FIELDS: Array<[keyof PriceRow, 'market' | 'low' | 'mid' | 'trend' | 'average_7d' | 'average_30d']> = [ | |
| 64 | + ['avg', 'mid'], | |
| 65 | + ['low', 'low'], | |
| 66 | + ['trend', 'trend'], | |
| 67 | + ['avg1', 'market'], | |
| 68 | + ['avg7', 'average_7d'], | |
| 69 | + ['avg30', 'average_30d'], | |
| 70 | +]; | |
| 71 | + | |
| 72 | +export function hasAnyPrice(row: PriceRow | null | undefined, foil = false): boolean { | |
| 73 | + if (!row) return false; | |
| 74 | + return PRICE_FIELDS.some(([f]) => { | |
| 75 | + const v = row[foil ? (`${f}-foil` as keyof PriceRow) : f]; | |
| 76 | + return typeof v === 'number' && v > 0; | |
| 77 | + }); | |
| 78 | +} | |
| 79 | + | |
| 80 | +/** | |
| 81 | + * Cardmarket product names embed disambiguators: "Kakuna [Bug Bite | Primal Clash]", "Roronoa Zoro (OP01-001)", | |
| 82 | + * "Auron (1-001)", "Forest (V.1)". Returns the clean name plus the parts we can use. | |
| 83 | + */ | |
| 84 | +export function parseProductName(raw: string): { name: string; number: string | null; version: string | null; disambiguation: string | null } { | |
| 85 | + let name = raw.replace(/'/g, "'").replace(/&/g, '&').replace(/\s+/g, ' ').trim(); | |
| 86 | + let disambiguation: string | null = null; | |
| 87 | + let number: string | null = null; | |
| 88 | + let version: string | null = null; | |
| 89 | + const br = name.match(/\s*\[([^\]]+)\]\s*$/); | |
| 90 | + if (br) { | |
| 91 | + disambiguation = br[1]!.trim(); | |
| 92 | + name = name.slice(0, br.index).trim(); | |
| 93 | + } | |
| 94 | + for (;;) { | |
| 95 | + const m = name.match(/\s*\(([^()]+)\)\s*$/); | |
| 96 | + if (!m) break; | |
| 97 | + const inner = m[1]!.trim(); | |
| 98 | + if (/^V\.\s?\d+$/i.test(inner)) version = version ?? inner.replace(/\s/g, ''); | |
| 99 | + else if (/^(?:[A-Z]{1,6}\d{0,3}[A-Z]?-\d{1,4}[A-Za-z]{0,3}|\d{1,2}-\d{3}[A-Z]?|[A-Z]{1,3}\d{1,3}-\d{1,3})$/.test(inner)) number = number ?? inner; | |
| 100 | + else break; | |
| 101 | + name = name.slice(0, m.index).trim(); | |
| 102 | + } | |
| 103 | + return { name: name || raw.trim(), number, version, disambiguation }; | |
| 104 | +} | |
| 105 | + | |
| 106 | +const SEALED_RE = /booster|box|display|bundle|deck|case|collection|tin|pack|kit|set$|starter|bundle|toolkit|fat pack|gift/i; | |
| 107 | + | |
| 108 | +export class CardmarketPriceGuideConnector extends BaseConnector { | |
| 109 | + readonly version = '1.0.0'; | |
| 110 | + readonly parserVersion = PARSER_VERSION; | |
| 111 | + protected override minIntervalMs = 2000; | |
| 112 | + | |
| 113 | + private games(): Array<[number, GameCfg]> { | |
| 114 | + const cfg = (this.meta.config.games ?? {}) as Record<string, unknown>; | |
| 115 | + return Object.entries(cfg) | |
| 116 | + .map(([id, c]) => [Number(id), GameCfgSchema.parse(c)] as [number, GameCfg]) | |
| 117 | + .sort((a, b) => a[0] - b[0]); | |
| 118 | + } | |
| 119 | + | |
| 120 | + private async file<T>(ctx: CrawlContext, url: string, schema: z.ZodType<T>): Promise<{ data: T | null; status: number | null; fetchedAt: Date; error: string | null }> { | |
| 121 | + await this.throttle(url); | |
| 122 | + const res = await ctx.fetch(url, { engines: ['api'], headers: JSON_HEADERS, timeoutMs: 240_000, minQuality: 0 }); | |
| 123 | + if (!res.success || res.json === null || res.json === undefined) return { data: null, status: res.httpStatus ?? null, fetchedAt: res.fetchedAt, error: res.error ?? `HTTP ${res.httpStatus}` }; | |
| 124 | + const parsed = schema.safeParse(res.json); | |
| 125 | + if (!parsed.success) return { data: null, status: res.httpStatus ?? null, fetchedAt: res.fetchedAt, error: `schema: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}` }; | |
| 126 | + return { data: parsed.data, status: res.httpStatus ?? null, fetchedAt: res.fetchedAt, error: null }; | |
| 127 | + } | |
| 128 | + | |
| 129 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 130 | + let games = this.games(); | |
| 131 | + if (ctx.options.seeds?.length) games = games.filter(([id]) => ctx.options.seeds!.includes(String(id))); | |
| 132 | + const includeNonSingles = this.meta.config.includeNonSingles !== false; | |
| 133 | + const onlyPriced = this.meta.config.onlyPriced !== false; | |
| 134 | + let gameIdx = Number(ctx.options.cursor?.gameIdx ?? 0); | |
| 135 | + let offset = Number(ctx.options.cursor?.offset ?? 0); | |
| 136 | + let count = 0; | |
| 137 | + let processed = 0; | |
| 138 | + for (; gameIdx < games.length; gameIdx++, offset = 0) { | |
| 139 | + if (ctx.signal?.aborted) return; | |
| 140 | + const [gameId, game] = games[gameIdx]!; | |
| 141 | + const guideUrl = `${BASE}/priceGuide/price_guide_${gameId}.json`; | |
| 142 | + const guide = await this.file(ctx, guideUrl, PriceGuideFileSchema); | |
| 143 | + if (!guide.data) { | |
| 144 | + ctx.anomaly('page_fetch_failed', `${guideUrl}: ${guide.error}`); | |
| 145 | + continue; | |
| 146 | + } | |
| 147 | + const prices = new Map<number, PriceRow>(); | |
| 148 | + let badRows = 0; | |
| 149 | + for (const row of guide.data.priceGuides) { | |
| 150 | + const p = PriceRowSchema.safeParse(row); | |
| 151 | + if (p.success) prices.set(p.data.idProduct, p.data); | |
| 152 | + else badRows++; | |
| 153 | + } | |
| 154 | + if (badRows) ctx.anomaly('schema_drift', `price_guide_${gameId}: ${badRows} unparseable rows`); | |
| 155 | + const priceGuideCreatedAt = guide.data.createdAt ?? null; | |
| 156 | + | |
| 157 | + const lists: Array<{ product: CardmarketProduct; single: boolean }> = []; | |
| 158 | + let productListCreatedAt: string | null = null; | |
| 159 | + const kinds: Array<['singles' | 'nonsingles', boolean]> = includeNonSingles ? [['singles', true], ['nonsingles', false]] : [['singles', true]]; | |
| 160 | + for (const [kind, single] of kinds) { | |
| 161 | + const url = `${BASE}/productList/products_${kind}_${gameId}.json`; | |
| 162 | + const f = await this.file(ctx, url, ProductFileSchema); | |
| 163 | + if (!f.data) { | |
| 164 | + ctx.anomaly('page_fetch_failed', `${url}: ${f.error}`); | |
| 165 | + continue; | |
| 166 | + } | |
| 167 | + productListCreatedAt = productListCreatedAt ?? f.data.createdAt ?? null; | |
| 168 | + let bad = 0; | |
| 169 | + for (const raw of f.data.products) { | |
| 170 | + const p = ProductSchema.safeParse(raw); | |
| 171 | + if (!p.success) { | |
| 172 | + bad++; | |
| 173 | + continue; | |
| 174 | + } | |
| 175 | + lists.push({ product: p.data, single }); | |
| 176 | + } | |
| 177 | + if (bad) ctx.anomaly('schema_drift', `products_${kind}_${gameId}: ${bad} unparseable products`); | |
| 178 | + } | |
| 179 | + const items = onlyPriced ? lists.filter((x) => hasAnyPrice(prices.get(x.product.idProduct)) || hasAnyPrice(prices.get(x.product.idProduct), true)) : lists; | |
| 180 | + ctx.log.info({ gameId, products: lists.length, priced: items.length, createdAt: priceGuideCreatedAt }, 'cardmarket bulk files loaded'); | |
| 181 | + for (; offset < items.length; offset++) { | |
| 182 | + if (ctx.signal?.aborted) return; | |
| 183 | + if (this.reached(ctx, count)) { | |
| 184 | + await ctx.setCursor({ gameIdx, offset }); | |
| 185 | + return; | |
| 186 | + } | |
| 187 | + const { product, single } = items[offset]!; | |
| 188 | + count++; | |
| 189 | + processed++; | |
| 190 | + const payload: CardmarketPayload = { gameId, game, product, prices: prices.get(product.idProduct) ?? null, single, priceGuideCreatedAt, productListCreatedAt }; | |
| 191 | + yield { url: `${guideUrl}#idProduct=${product.idProduct}`, externalId: String(product.idProduct), kind: 'catalog_item', engine: 'api', httpStatus: guide.status, payload, fetchedAt: guide.fetchedAt }; | |
| 192 | + if (offset > 0 && offset % 2000 === 0) await ctx.setCursor({ gameIdx, offset: offset + 1 }); | |
| 193 | + } | |
| 194 | + await ctx.setCursor({ gameIdx: gameIdx + 1, offset: 0, priceGuideCreatedAt }); | |
| 195 | + await ctx.progress({ page: gameIdx + 1, totalPages: games.length, itemsProcessed: processed }); | |
| 196 | + } | |
| 197 | + await ctx.setCursor({ gameIdx: 0, offset: 0, completedAt: new Date().toISOString(), done: true }); | |
| 198 | + } | |
| 199 | + | |
| 200 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 201 | + const { gameId, game, product, prices, single, priceGuideCreatedAt } = RawPayloadSchema.parse(raw.payload); | |
| 202 | + const parsed = parseProductName(product.name); | |
| 203 | + const observedAt = raw.fetchedAt; | |
| 204 | + const obsDate = isoDay(priceGuideCreatedAt) ?? dayOf(observedAt); | |
| 205 | + const sealed = !single && SEALED_RE.test(product.categoryName); | |
| 206 | + const priceKinds = new Set((this.meta.config.priceKinds as string[] | undefined) ?? PRICE_FIELDS.map(([f]) => f)); | |
| 207 | + const ids: Record<string, string> = { cardmarket_id: String(product.idProduct) }; | |
| 208 | + if (single && product.idMetacard) ids.cardmarket_metacard_id = String(product.idMetacard); | |
| 209 | + if (product.idExpansion) ids.cardmarket_expansion_id = String(product.idExpansion); | |
| 210 | + const build = (variant: string | null) => | |
| 211 | + attrs({ | |
| 212 | + categorySlug: game.slug, | |
| 213 | + franchise: game.franchise ?? null, | |
| 214 | + brand: game.brand ?? null, | |
| 215 | + name: parsed.name, | |
| 216 | + number: parsed.number, | |
| 217 | + variant, | |
| 218 | + language: null, | |
| 219 | + identifiers: ids, | |
| 220 | + metadata: { | |
| 221 | + cardmarket_game_id: gameId, | |
| 222 | + cardmarket_game: game.name, | |
| 223 | + cardmarket_category: product.categoryName, | |
| 224 | + cardmarket_category_id: product.idCategory, | |
| 225 | + cardmarket_expansion_id: product.idExpansion ?? null, | |
| 226 | + single, | |
| 227 | + sealed, | |
| 228 | + version: parsed.version, | |
| 229 | + disambiguation: parsed.disambiguation, | |
| 230 | + date_added: product.dateAdded && !product.dateAdded.startsWith('0000') ? product.dateAdded : null, | |
| 231 | + }, | |
| 232 | + }); | |
| 233 | + const out: NormalizedRecord[] = []; | |
| 234 | + const variants: Array<{ variant: string | null; foil: boolean }> = [{ variant: null, foil: false }]; | |
| 235 | + if (single && hasAnyPrice(prices, true)) variants.push({ variant: game.foilVariant, foil: true }); | |
| 236 | + for (const { variant, foil } of variants) { | |
| 237 | + const a = build(variant); | |
| 238 | + const rawTitle = makeTitle({ name: parsed.name, number: parsed.number, variant }) + (parsed.disambiguation ? ` [${parsed.disambiguation}]` : '') + (single ? '' : ` — ${product.categoryName}`); | |
| 239 | + out.push( | |
| 240 | + catalogItem({ | |
| 241 | + kind: 'catalog_item', | |
| 242 | + connectorId: this.meta.id, | |
| 243 | + sourceId: this.meta.sourceId, | |
| 244 | + sourceUrl: raw.url, | |
| 245 | + externalId: `${product.idProduct}${foil ? ':foil' : ''}`, | |
| 246 | + rawTitle, | |
| 247 | + imageUrls: [], | |
| 248 | + attributes: a, | |
| 249 | + condition: { completeness: sealed ? 'sealed' : null }, | |
| 250 | + observedAt, | |
| 251 | + confidence: 0.8, | |
| 252 | + parserVersion: PARSER_VERSION, | |
| 253 | + releaseDate: null, | |
| 254 | + }), | |
| 255 | + ); | |
| 256 | + if (!prices) continue; | |
| 257 | + for (const [field, priceKind] of PRICE_FIELDS) { | |
| 258 | + if (!priceKinds.has(field)) continue; | |
| 259 | + const key = foil ? (`${field}-foil` as keyof PriceRow) : field; | |
| 260 | + const v = prices[key]; | |
| 261 | + if (typeof v !== 'number' || !(v > 0)) continue; | |
| 262 | + out.push( | |
| 263 | + priceObservation({ | |
| 264 | + kind: 'price_observation', | |
| 265 | + connectorId: this.meta.id, | |
| 266 | + sourceId: this.meta.sourceId, | |
| 267 | + sourceUrl: raw.url, | |
| 268 | + externalId: `${product.idProduct}:${foil ? 'foil' : 'base'}:${field}`, | |
| 269 | + rawTitle, | |
| 270 | + imageUrls: [], | |
| 271 | + attributes: { ...a, metadata: { ...a.metadata, cardmarket_field: key, foil } }, | |
| 272 | + condition: { completeness: sealed ? 'sealed' : null }, | |
| 273 | + observedAt, | |
| 274 | + confidence: 0.75, | |
| 275 | + parserVersion: PARSER_VERSION, | |
| 276 | + priceKind, | |
| 277 | + price: v, | |
| 278 | + currency: 'EUR', | |
| 279 | + observationDate: obsDate, | |
| 280 | + sampleSize: null, | |
| 281 | + }), | |
| 282 | + ); | |
| 283 | + } | |
| 284 | + } | |
| 285 | + return out; | |
| 286 | + } | |
| 287 | +} | |
| 288 | + | |
| 289 | +export default (meta: ConnectorMeta) => new CardmarketPriceGuideConnector(meta); | |
added
connectors/api/cardmarket-priceguide/meta.json
+56 −0
@@ -0,0 +1,56 @@ | ||
| 1 | +{ | |
| 2 | + "id": "cardmarket-priceguide", | |
| 3 | + "displayName": "Cardmarket price guide (bulk files)", | |
| 4 | + "sourceId": "cardmarket-priceguide", | |
| 5 | + "sourceName": "Cardmarket", | |
| 6 | + "sourceType": "pricing_guide", | |
| 7 | + "sourceUrl": "https://downloads.s3.cardmarket.com/productCatalog/priceGuide/price_guide_1.json", | |
| 8 | + "module": "api/cardmarket-priceguide", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["magic_the_gathering", "yugioh", "pokemon", "final_fantasy_tcg", "weiss_schwarz", "dragon_ball_tcg", "flesh_and_blood", "digimon_tcg", "one_piece_card_game", "disney_lorcana", "star_wars_tcg", "other_tcg"], | |
| 11 | + "regions": ["EU", "DE"], | |
| 12 | + "country": "DE", | |
| 13 | + "languages": ["en"], | |
| 14 | + "currency": ["EUR"], | |
| 15 | + "supportsListings": false, | |
| 16 | + "supportsSold": false, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": false, | |
| 19 | + "supportsCatalog": true, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 1440, | |
| 23 | + "priority": "high", | |
| 24 | + "trustScore": 0.85, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.cardmarket.com/en/Magic/Help/TermsOfService", | |
| 27 | + "acquisitionMethod": "public bulk JSON files (daily price guide + product catalog)", | |
| 28 | + "historicalDepth": "none", | |
| 29 | + "capabilities": ["price_guide", "catalog"], | |
| 30 | + "accessNotes": "Reads only Cardmarket's public, unauthenticated bulk files on downloads.s3.cardmarket.com/productCatalog/: priceGuide/price_guide_<idGame>.json (one row per product: avg, low, trend, avg1, avg7, avg30 and the *-foil equivalents, EUR, regenerated daily — the file's own createdAt is used as observationDate) and productList/products_singles_<idGame>.json + products_nonsingles_<idGame>.json (idProduct, name, idCategory/categoryName, idExpansion, idMetacard, dateAdded). Game ids were discovered live by probing 1–30 (HTTP 403 = no such game): 1 Magic, 2 WoW TCG, 3 Yu-Gi-Oh!, 5 The Spoils, 6 Pokémon, 7 Force of Will, 8 Cardfight!! Vanguard, 9 Final Fantasy TCG, 10 Weiss Schwarz, 11 Dragoborne, 12 My Little Pony, 13 Dragon Ball Super, 15 Star Wars Destiny, 16 Flesh and Blood, 17 Digimon, 18 One Piece, 19 Lorcana, 20 Battle Spirits Saga, 21 Star Wars Unlimited, 22 Riftbound. Files are 1–26 MB each and fetched at most once per game per run (≤ 3 requests per game, 2 s apart, 1 concurrent). The bucket has no robots.txt (S3 AccessDenied on /robots.txt); the storefront cardmarket.com is Cloudflare-protected and is deliberately NEVER crawled — no product pages, no search, no seller data. The bulk files carry no expansion names, card numbers or images, so catalog items rely on identifiers.cardmarket_id (= idProduct, the same key MTGJSON emits for mcmId) for entity resolution; Cardmarket 'foil' rows become variant 'Foil' (Magic & most games) or 'Reverse Holo' (Pokémon, Cardmarket's foil flag for Pokémon). Guide values are price observations, never sales.", | |
| 31 | + "enabled": true, | |
| 32 | + "schemaVersion": "1.0", | |
| 33 | + "config": { | |
| 34 | + "games": { | |
| 35 | + "1": { "slug": "magic_the_gathering", "name": "Magic: The Gathering", "franchise": "Magic: The Gathering", "brand": "Wizards of the Coast" }, | |
| 36 | + "3": { "slug": "yugioh", "name": "Yu-Gi-Oh!", "franchise": "Yu-Gi-Oh!", "brand": "Konami" }, | |
| 37 | + "6": { "slug": "pokemon", "name": "Pokémon", "franchise": "Pokémon", "brand": "The Pokémon Company", "foilVariant": "Reverse Holo" }, | |
| 38 | + "9": { "slug": "final_fantasy_tcg", "name": "Final Fantasy TCG", "franchise": "Final Fantasy", "brand": "Square Enix" }, | |
| 39 | + "10": { "slug": "weiss_schwarz", "name": "Weiss Schwarz", "franchise": "Weiß Schwarz", "brand": "Bushiroad" }, | |
| 40 | + "13": { "slug": "dragon_ball_tcg", "name": "Dragon Ball Super Card Game", "franchise": "Dragon Ball", "brand": "Bandai" }, | |
| 41 | + "16": { "slug": "flesh_and_blood", "name": "Flesh and Blood", "franchise": "Flesh and Blood", "brand": "Legend Story Studios" }, | |
| 42 | + "17": { "slug": "digimon_tcg", "name": "Digimon Card Game", "franchise": "Digimon", "brand": "Bandai" }, | |
| 43 | + "18": { "slug": "one_piece_card_game", "name": "One Piece Card Game", "franchise": "One Piece", "brand": "Bandai" }, | |
| 44 | + "19": { "slug": "disney_lorcana", "name": "Disney Lorcana", "franchise": "Disney Lorcana", "brand": "Ravensburger" }, | |
| 45 | + "21": { "slug": "star_wars_tcg", "name": "Star Wars: Unlimited", "franchise": "Star Wars", "brand": "Fantasy Flight Games" }, | |
| 46 | + "15": { "slug": "star_wars_tcg", "name": "Star Wars: Destiny", "franchise": "Star Wars", "brand": "Fantasy Flight Games" }, | |
| 47 | + "20": { "slug": "other_tcg", "name": "Battle Spirits Saga", "franchise": "Battle Spirits Saga", "brand": "Bandai" }, | |
| 48 | + "22": { "slug": "other_tcg", "name": "Riftbound", "franchise": "Riftbound: League of Legends TCG", "brand": "Riot Games" }, | |
| 49 | + "7": { "slug": "other_tcg", "name": "Force of Will", "franchise": "Force of Will", "brand": "Force of Will Co." }, | |
| 50 | + "8": { "slug": "other_tcg", "name": "Cardfight!! Vanguard", "franchise": "Cardfight!! Vanguard", "brand": "Bushiroad" } | |
| 51 | + }, | |
| 52 | + "includeNonSingles": true, | |
| 53 | + "onlyPriced": true, | |
| 54 | + "priceKinds": ["avg", "low", "trend", "avg1", "avg7", "avg30"] | |
| 55 | + } | |
| 56 | +} | |
added
connectors/api/cardmerchant-nz/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Card Merchant NZ connector (`cardmerchant-nz`) | |
| 2 | + | |
| 3 | +- Source: https://cardmerchant.co.nz · Auckland TCG store on BinderPOS: ~112k MTG singles (5k in stock), Yu-Gi-Oh!, Flesh and Blood, Riftbound and Grand Archive singles with "Name [Set]" titles and condition variants; sealed MTG, Yu-Gi-Oh!, One Piece, Lorcana, Digimon, Dragon Ball and Star Wars Unlimited. | |
| 4 | +- Country/currency: NZ / NZD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, a `titlePattern` that extracts name/set/number, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (14): `mtg-singles-instock` → magic_the_gathering, `mtg-sealed` → magic_the_gathering, `yu-gi-oh-singles` → yugioh, `yugioh-sealed` → yugioh, `fab-singles` → flesh_and_blood, `flesh-and-blood-sealed` → flesh_and_blood, `riftbound-singles` → other_tcg, `grand-archive-singles` → other_tcg, `one-piece-tcg` → one_piece_card_game, `pokemon` → pokemon, `disney-lorcana-tcg` → disney_lorcana, `star-wars-unlimited` → star_wars_tcg, `digimon` → digimon_tcg, `dragonball-super` → dragon_ball_tcg. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/cardmerchant-nz/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts cardmerchant-nz`. | |
added
connectors/api/cardmerchant-nz/index.test.ts
+62 −0
@@ -0,0 +1,62 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('cardmerchant-nz', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "CookyYummy [JUSH-EN017] Collector's Rare", | |
| 18 | + "collection": "yu-gi-oh-singles", | |
| 19 | + "type": "Yugioh Single", | |
| 20 | + "tags": [ | |
| 21 | + "1st Edition", | |
| 22 | + "Justice Hunters" | |
| 23 | + ], | |
| 24 | + "variant": "Near Mint / Lightly Played 1st Edition", | |
| 25 | + "expect": "yugioh", | |
| 26 | + "name": "CookyYummy", | |
| 27 | + "set": "JUSH-EN017", | |
| 28 | + "conditionRaw": "Near Mint" | |
| 29 | + }, | |
| 30 | + { | |
| 31 | + "title": "Heart of Fyendal [ANQ001] (Compendium of Rathe - Antiquity Pack)", | |
| 32 | + "collection": "fab-singles", | |
| 33 | + "type": "Flesh And Blood Single", | |
| 34 | + "variant": "Moderately Played", | |
| 35 | + "expect": "flesh_and_blood", | |
| 36 | + "name": "Heart of Fyendal", | |
| 37 | + "set": "ANQ001", | |
| 38 | + "conditionRaw": "Moderately Played" | |
| 39 | + }, | |
| 40 | + { | |
| 41 | + "title": "Lightning Bolt [Magic 2011]", | |
| 42 | + "collection": "mtg-singles-instock", | |
| 43 | + "type": "MTG Single", | |
| 44 | + "variant": "Near Mint", | |
| 45 | + "expect": "magic_the_gathering", | |
| 46 | + "set": "Magic 2011" | |
| 47 | + }, | |
| 48 | + { | |
| 49 | + "title": "One Piece TCG Double Pack Set Vol 9 [DP-09]", | |
| 50 | + "collection": "one-piece-tcg", | |
| 51 | + "type": "One Piece Sealed", | |
| 52 | + "expect": "one_piece_card_game" | |
| 53 | + }, | |
| 54 | + { | |
| 55 | + "title": "Dragon Shield Matte Sleeves - Black (100)", | |
| 56 | + "collection": "mtg-sealed", | |
| 57 | + "type": "Accessories", | |
| 58 | + "expect": null | |
| 59 | + } | |
| 60 | + ]); | |
| 61 | + }); | |
| 62 | +}); | |
added
connectors/api/cardmerchant-nz/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Card Merchant NZ — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/cardmerchant-nz/meta.json
+136 −0
@@ -0,0 +1,136 @@ | ||
| 1 | +{ | |
| 2 | + "id": "cardmerchant-nz", | |
| 3 | + "displayName": "Card Merchant NZ", | |
| 4 | + "sourceId": "cardmerchant-nz", | |
| 5 | + "sourceName": "Card Merchant NZ", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://cardmerchant.co.nz", | |
| 8 | + "module": "api/cardmerchant-nz", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "yugioh", | |
| 15 | + "flesh_and_blood", | |
| 16 | + "other_tcg", | |
| 17 | + "one_piece_card_game", | |
| 18 | + "pokemon", | |
| 19 | + "disney_lorcana", | |
| 20 | + "star_wars_tcg", | |
| 21 | + "digimon_tcg", | |
| 22 | + "dragon_ball_tcg" | |
| 23 | + ], | |
| 24 | + "regions": [ | |
| 25 | + "NZ" | |
| 26 | + ], | |
| 27 | + "country": "NZ", | |
| 28 | + "languages": [ | |
| 29 | + "en" | |
| 30 | + ], | |
| 31 | + "currency": [ | |
| 32 | + "NZD" | |
| 33 | + ], | |
| 34 | + "supportsListings": true, | |
| 35 | + "supportsSold": false, | |
| 36 | + "supportsAuctions": false, | |
| 37 | + "supportsImages": true, | |
| 38 | + "supportsCatalog": false, | |
| 39 | + "supportsPopulation": false, | |
| 40 | + "supportsLookup": true, | |
| 41 | + "refreshFrequencyMinutes": 720, | |
| 42 | + "priority": "medium", | |
| 43 | + "trustScore": 0.65, | |
| 44 | + "attributionRequired": true, | |
| 45 | + "termsUrl": "https://cardmerchant.co.nz/policies/terms-of-service", | |
| 46 | + "accessNotes": "Card Merchant NZ (cardmerchant.co.nz) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (mtg-singles-instock, mtg-sealed, yu-gi-oh-singles, yugioh-sealed, fab-singles, flesh-and-blood-sealed, riftbound-singles, grand-archive-singles, one-piece-tcg, pokemon, disney-lorcana-tcg, star-wars-unlimited, digimon, dragonball-super). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency NZD (/meta.json currency NZD, Shopify.currency rate 1.0), GST included. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 47 | + "enabled": true, | |
| 48 | + "schemaVersion": "1.0", | |
| 49 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 50 | + "historicalDepth": "none", | |
| 51 | + "requires": [], | |
| 52 | + "config": { | |
| 53 | + "currency": "NZD", | |
| 54 | + "seller": "Card Merchant NZ", | |
| 55 | + "location": null, | |
| 56 | + "collections": [ | |
| 57 | + { | |
| 58 | + "handle": "mtg-singles-instock", | |
| 59 | + "categorySlug": "magic_the_gathering", | |
| 60 | + "franchise": "Magic: The Gathering" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "mtg-sealed", | |
| 64 | + "categorySlug": "magic_the_gathering", | |
| 65 | + "franchise": "Magic: The Gathering" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "handle": "yu-gi-oh-singles", | |
| 69 | + "categorySlug": "yugioh", | |
| 70 | + "franchise": "Yu-Gi-Oh!" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "yugioh-sealed", | |
| 74 | + "categorySlug": "yugioh", | |
| 75 | + "franchise": "Yu-Gi-Oh!" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "handle": "fab-singles", | |
| 79 | + "categorySlug": "flesh_and_blood", | |
| 80 | + "franchise": "Flesh and Blood" | |
| 81 | + }, | |
| 82 | + { | |
| 83 | + "handle": "flesh-and-blood-sealed", | |
| 84 | + "categorySlug": "flesh_and_blood", | |
| 85 | + "franchise": "Flesh and Blood" | |
| 86 | + }, | |
| 87 | + { | |
| 88 | + "handle": "riftbound-singles", | |
| 89 | + "categorySlug": "other_tcg", | |
| 90 | + "franchise": "League of Legends" | |
| 91 | + }, | |
| 92 | + { | |
| 93 | + "handle": "grand-archive-singles", | |
| 94 | + "categorySlug": "other_tcg" | |
| 95 | + }, | |
| 96 | + { | |
| 97 | + "handle": "one-piece-tcg", | |
| 98 | + "categorySlug": "one_piece_card_game", | |
| 99 | + "franchise": "One Piece" | |
| 100 | + }, | |
| 101 | + { | |
| 102 | + "handle": "pokemon", | |
| 103 | + "categorySlug": "pokemon", | |
| 104 | + "franchise": "Pokémon" | |
| 105 | + }, | |
| 106 | + { | |
| 107 | + "handle": "disney-lorcana-tcg", | |
| 108 | + "categorySlug": "disney_lorcana", | |
| 109 | + "franchise": "Disney Lorcana" | |
| 110 | + }, | |
| 111 | + { | |
| 112 | + "handle": "star-wars-unlimited", | |
| 113 | + "categorySlug": "star_wars_tcg", | |
| 114 | + "franchise": "Star Wars" | |
| 115 | + }, | |
| 116 | + { | |
| 117 | + "handle": "digimon", | |
| 118 | + "categorySlug": "digimon_tcg", | |
| 119 | + "franchise": "Digimon" | |
| 120 | + }, | |
| 121 | + { | |
| 122 | + "handle": "dragonball-super", | |
| 123 | + "categorySlug": "dragon_ball_tcg", | |
| 124 | + "franchise": "Dragon Ball" | |
| 125 | + } | |
| 126 | + ], | |
| 127 | + "rules": [], | |
| 128 | + "defaultCategory": null, | |
| 129 | + "exclude": "gift card|\\bsleeves?\\b|deck box|deck case|deck holder|binder|playmat|play mat|toploader|top loader|storage box|card case|portfolio|\\balbum\\b|\\bdice\\b|spindown|life counter|tokens? only|bundle of|mystery (box|pack|bundle|bag)|repack|\\bticket|event entry|entry fee|weekly league|display case|acrylic (stand|display|case)|card holder|card protector|magnetic holder|semi-rigid|perfect fit|penny sleeve|team bag|card saver|\\baccessor|dice set|d20|d6\\b|\\bpaint|brush|primer|glue|hobby tool|cutter|tweezers|sticker|keyring|key ring|lanyard|badge|coaster|mug\\b|t-shirt|hoodie|\\bcap\\b|socks|towel|blanket|poster|wall scroll|calendar|advent|\\| accessories \\||folder|custom|board game|dragon shield|ultimate guard|gamegenic|\\bsleeve", | |
| 130 | + "keepOutOfStock": true, | |
| 131 | + "titlePattern": "^(?<name>.+?)\\s*\\[(?<set>[^\\]]+)\\]", | |
| 132 | + "pageSize": 250, | |
| 133 | + "fetchBarcodes": false, | |
| 134 | + "wholeShop": false | |
| 135 | + } | |
| 136 | +} | |
added
connectors/api/cardrush/README.md
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +# cardrush | |
| 2 | + | |
| 3 | +Cardrush (カードラッシュ) — Japanese single-card retailer, one storefront per game on the same platform. | |
| 4 | + | |
| 5 | +| store | game | access | | |
| 6 | +|---|---|---| | |
| 7 | +| www.cardrush-mtg.jp | Magic | plain HTTP | | |
| 8 | +| www.cardrush-op.jp | One Piece | plain HTTP | | |
| 9 | +| www.cardrush.jp | Yu-Gi-Oh! | plain HTTP | | |
| 10 | +| www.cardrush-dm.jp | Duel Masters | plain HTTP | | |
| 11 | +| www.cardrush-digimon.jp | Digimon | plain HTTP | | |
| 12 | +| www.cardrush-pokemon.jp | Pokémon | Cloudflare challenge → Scrapfly ASP (jp, no JS, ~25 credits/page, 1 page per list per run) | | |
| 13 | + | |
| 14 | +Pages: `https://<store>/product-list/<id>?page=<n>` (100 items, `.item_data` blocks, `a.to_next_page` pager). Titles: `[COND]JP/EN《language》【SET】` (MTG, `bracket: set`) or `〔状態/PSA10鑑定済〕Name(仕様)【Rarity】{No/Total}` (others), parsed by `_g1-cards-eu-jp-lib/parseJpCardTitle`. | |
| 15 | + | |
| 16 | +Output: `listing` (JPY, seller Cardrush, `ended` when 在庫なし, `sealed` for 【未開封BOX】, PSA grades from titles; オリパ/supplies skipped). Cursor `{ storeIdx, listIdx, page }`. | |
| 17 | +Fixtures: `pnpm tsx connectors/api/cardrush/_capture.ts`. | |
added
connectors/api/cardrush/_capture.ts
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture for cardrush: one MTG list page (plain HTTP, set-code brackets), one One Piece | |
| 3 | + * page (rarity brackets, sealed boxes) and — if a Scrapfly capture of the Pokémon store exists at | |
| 4 | + * /tmp/cr_pokemon_list2.html (taken once during research, 25 credits) — the Pokémon page. | |
| 5 | + * Usage: pnpm tsx connectors/api/cardrush/_capture.ts | |
| 6 | + */ | |
| 7 | +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; | |
| 8 | +import path from 'node:path'; | |
| 9 | +import { fixtureDir, saveFixture } from '@rareindex/connectors/testing'; | |
| 10 | +import meta from './meta.json' with { type: 'json' }; | |
| 11 | +import { parseListPage, type CardrushPayload, type CardrushStore } from './index.js'; | |
| 12 | +import { HTML_HEADERS } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 13 | + | |
| 14 | +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 15 | +mkdirSync(fixtureDir('cardrush'), { recursive: true }); | |
| 16 | +const stores = meta.config.stores as unknown as CardrushStore[]; | |
| 17 | + | |
| 18 | +async function capture(host: string, listId: string, name: string, doc: string | null, engine: 'api' | 'scrapfly', keep: number, note: string) { | |
| 19 | + const store = stores.find((s) => s.host === host)!; | |
| 20 | + const url = `https://${host}/product-list/${listId}`; | |
| 21 | + const page = doc ?? (await (await fetch(url, { headers: HTML_HEADERS })).text()); | |
| 22 | + if (name === 'mtg-list-page1') writeFileSync(path.join(fixtureDir('cardrush'), 'list-page.html'), page); | |
| 23 | + const { title, items, hasNext } = parseListPage(page, host); | |
| 24 | + const soldOut = items.find((i) => i.soldOut); | |
| 25 | + const kept = items.slice(0, keep); | |
| 26 | + if (soldOut && !kept.includes(soldOut)) kept[kept.length - 1] = soldOut; | |
| 27 | + const payload: CardrushPayload = { store: { host, categorySlug: store.categorySlug, franchise: store.franchise ?? null, brand: store.brand ?? null, bracket: store.bracket ?? 'rarity' }, listId, listName: title ?? store.lists[listId]!, page: 1, items: kept }; | |
| 28 | + saveFixture('cardrush', name, { | |
| 29 | + raw: { url, externalId: `${host}:${listId}:p1`, kind: 'listing', engine, fetchedAt: new Date(), payload }, | |
| 30 | + expect: { minCount: 5, kinds: ['listing'], requiredFields: ['price', 'attributes.name'] }, | |
| 31 | + note: `Live capture ${new Date().toISOString().slice(0, 10)} — ${url} via ${engine} (${note}; ${items.length} items, hasNext=${hasNext}, first ${kept.length} kept incl. a sold-out row when present)`, | |
| 32 | + }); | |
| 33 | + console.log(name, title, items.length, hasNext, kept[0]); | |
| 34 | +} | |
| 35 | + | |
| 36 | +await capture('www.cardrush-mtg.jp', '100', 'mtg-list-page1', null, 'api', 16, 'Magic 4th Edition list: [COND]JP/EN name《language》【SET】'); | |
| 37 | +await wait(4000); | |
| 38 | +await capture('www.cardrush-op.jp', '4', 'one-piece-sealed-boxes', null, 'api', 8, 'One Piece sealed boxes (【未開封BOX】 → completeness sealed)'); | |
| 39 | +await wait(4000); | |
| 40 | +await capture('www.cardrush-op.jp', '17', 'one-piece-red-singles', null, 'api', 12, 'One Piece red singles with 【rarity】{number}'); | |
| 41 | +const pk = '/tmp/cr_pokemon_list2.html'; | |
| 42 | +if (existsSync(pk)) await capture('www.cardrush-pokemon.jp', '2', 'pokemon-scrapfly-page1', readFileSync(pk, 'utf8'), 'scrapfly', 12, 'Pokémon store behind Cloudflare, page captured through Scrapfly ASP (country jp, no JS); PSA-graded titles'); | |
added
connectors/api/cardrush/index.test.ts
+68 −0
@@ -0,0 +1,68 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import meta from './meta.json' with { type: 'json' }; | |
| 5 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 6 | +import { fixtureDir, listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 7 | +import createConnector, { parseListPage } from './index.js'; | |
| 8 | +import { jpCondition, parseJpCardTitle } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 9 | + | |
| 10 | +const connector = createConnector(localMeta(meta)); | |
| 11 | + | |
| 12 | +describe('cardrush', () => { | |
| 13 | + runFixtureSuite(connector, it, expect); | |
| 14 | + | |
| 15 | + it('parses MTG and Pokémon store titles', () => { | |
| 16 | + const m = parseJpCardTitle('[PLD](黒枠)ラノワールのエルフ/Llanowar Elves《日本語》【4ED】', { bracket: 'set' }); | |
| 17 | + expect(m).toMatchObject({ name: 'Llanowar Elves', conditionRaw: 'PLD', setCode: '4ED', language: 'Japanese', notes: ['黒枠'], rarity: null }); | |
| 18 | + expect(jpCondition('PLD')).toBeNull(); | |
| 19 | + const e = parseJpCardTitle('[EX]暗黒の儀式/Dark Ritual《英語》【4ED】', { bracket: 'set' }); | |
| 20 | + expect(e).toMatchObject({ name: 'Dark Ritual', language: 'English', setCode: '4ED' }); | |
| 21 | + expect(jpCondition(e.conditionRaw)).toBe('excellent'); | |
| 22 | + const p = parseJpCardTitle('〔PSA10鑑定済〕ミロカロスδ-デルタ種【★】{013/068}'); | |
| 23 | + expect(p).toMatchObject({ name: 'ミロカロスδ-デルタ種', grader: 'psa', grade: '10', rarity: '★', number: '013', total: '068' }); | |
| 24 | + const box = parseJpCardTitle('ブースターパック 受け継がれる意志【未開封BOX】{-}'); | |
| 25 | + expect(box.sealed).toBe(true); | |
| 26 | + expect(box.number).toBeNull(); | |
| 27 | + const mixed = parseJpCardTitle('〔※状態難/PSA8鑑定済〕リザードン LV.76(かえん/マークあり)【★】{旧裏}'); | |
| 28 | + expect(mixed.grade).toBe('8'); | |
| 29 | + expect(mixed.number).toBe('旧裏'); | |
| 30 | + }); | |
| 31 | + | |
| 32 | + it('parses the saved list page: items, yen prices, stock, sold-out flag, pager', () => { | |
| 33 | + const doc = readFileSync(path.join(fixtureDir('cardrush'), 'list-page.html'), 'utf8'); | |
| 34 | + const { title, items, hasNext } = parseListPage(doc, 'www.cardrush-mtg.jp'); | |
| 35 | + expect(title).toBeTruthy(); | |
| 36 | + expect(items.length).toBe(100); | |
| 37 | + expect(hasNext).toBe(true); | |
| 38 | + const first = items[0]!; | |
| 39 | + expect(first.url).toMatch(/^https:\/\/www\.cardrush-mtg\.jp\/product\/\d+$/); | |
| 40 | + expect(first.priceJpy).toBeGreaterThan(0); | |
| 41 | + expect(first.image).toMatch(/^https:\/\//); | |
| 42 | + expect(items.some((i) => i.soldOut && i.stock === 0)).toBe(true); | |
| 43 | + expect(items.some((i) => !i.soldOut && (i.stock ?? 0) > 0)).toBe(true); | |
| 44 | + }); | |
| 45 | + | |
| 46 | + it('emits JPY listings per store with the right category, language and availability', async () => { | |
| 47 | + for (const name of listFixtures('cardrush')) { | |
| 48 | + const fx = loadFixture('cardrush', name); | |
| 49 | + const out = await connector.normalize(fx.raw); | |
| 50 | + expect(out.length).toBeGreaterThan(0); | |
| 51 | + const payload = fx.raw.payload as { store: { categorySlug: string }; items: Array<{ soldOut: boolean; title: string }> }; | |
| 52 | + for (const r of out) { | |
| 53 | + if (r.kind !== 'listing') continue; | |
| 54 | + expect(r.currency).toBe('JPY'); | |
| 55 | + expect(r.price).toBeGreaterThan(0); | |
| 56 | + expect(r.seller).toBe('Cardrush'); | |
| 57 | + expect(r.attributes.categorySlug).toBe(payload.store.categorySlug); | |
| 58 | + expect(['Japanese', 'English', 'Korean', 'Chinese']).toContain(r.attributes.language); | |
| 59 | + expect(r.attributes.identifiers.cardrush_product_id).toMatch(/^www\.cardrush[a-z-]*\.jp\/\d+$/); | |
| 60 | + } | |
| 61 | + const ended = out.filter((r) => r.kind === 'listing' && r.availability === 'ended').length; | |
| 62 | + const soldOutIn = payload.items.filter((i) => i.soldOut && !/オリパ|サプライ/.test(i.title)).length; | |
| 63 | + expect(ended).toBe(soldOutIn); | |
| 64 | + if (name.includes('sealed')) for (const r of out) if (r.kind === 'listing') expect(r.condition.completeness).toBe('sealed'); | |
| 65 | + if (name.includes('pokemon')) expect(out.some((r) => r.kind === 'listing' && r.grade.grader === 'psa' && r.grade.grade)).toBe(true); | |
| 66 | + } | |
| 67 | + }); | |
| 68 | +}); | |
added
connectors/api/cardrush/index.ts
+176 −0
@@ -0,0 +1,176 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { NormalizedListingSchema, type Engine, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { attrs } from '../_lib/shared.js'; | |
| 5 | +import { HTML_HEADERS, JP_EXCLUDE_RE, cleanText, isJpBundle, jpCondition, jpVariant, parseJpCardTitle, yen } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Cardrush (カードラッシュ) — one storefront per game on the same shop platform. Category listing pages | |
| 9 | + * (product-list/<id>?page=n) → one raw record per page; normalize emits one JPY listing per product. | |
| 10 | + */ | |
| 11 | +const PARSER_VERSION = '1.0.0'; | |
| 12 | + | |
| 13 | +const StoreSchema = z.object({ | |
| 14 | + host: z.string(), | |
| 15 | + categorySlug: z.string(), | |
| 16 | + franchise: z.string().nullable().optional(), | |
| 17 | + brand: z.string().nullable().optional(), | |
| 18 | + bracket: z.enum(['rarity', 'set']).default('rarity'), | |
| 19 | + engines: z.array(z.enum(['api', 'feed', 'firecrawl', 'scrapfly', 'browser', 'manual'])).optional(), | |
| 20 | + pagesPerList: z.number().int().positive().optional(), | |
| 21 | + lists: z.record(z.string(), z.string()), | |
| 22 | +}); | |
| 23 | +export type CardrushStore = z.infer<typeof StoreSchema>; | |
| 24 | + | |
| 25 | +export const ItemSchema = z.object({ productId: z.string(), url: z.string(), title: z.string(), modelNumber: z.string().nullable(), priceJpy: z.number().nullable(), stock: z.number().int().nullable(), soldOut: z.boolean(), image: z.string().nullable() }); | |
| 26 | +export type CardrushItem = z.infer<typeof ItemSchema>; | |
| 27 | +const RawPayloadSchema = z.object({ store: StoreSchema.omit({ lists: true, engines: true, pagesPerList: true }), listId: z.string(), listName: z.string(), page: z.number().int(), items: z.array(ItemSchema) }); | |
| 28 | +export type CardrushPayload = z.infer<typeof RawPayloadSchema>; | |
| 29 | + | |
| 30 | +/** Parse a product-list page: .item_data blocks + pager. */ | |
| 31 | +export function parseListPage(doc: string, host: string): { title: string | null; items: CardrushItem[]; hasNext: boolean } { | |
| 32 | + const $ = html.load(doc); | |
| 33 | + const items: CardrushItem[] = []; | |
| 34 | + $('.item_data').each((_, el) => { | |
| 35 | + const $el = $(el); | |
| 36 | + const productId = $el.attr('data-product-id')?.trim() || $el.find('a.item_data_link').attr('href')?.match(/\/product\/(\d+)/)?.[1] || ''; | |
| 37 | + const href = $el.find('a.item_data_link').attr('href') ?? (productId ? `https://${host}/product/${productId}` : null); | |
| 38 | + const title = cleanText($el.find('.goods_name').first().text()); | |
| 39 | + if (!productId || !href || !title) return; | |
| 40 | + const modelNumber = cleanText($el.find('.model_number_value').first().text()); | |
| 41 | + const priceJpy = yen(cleanText($el.find('.selling_price .figure').first().text())); | |
| 42 | + const stockText = cleanText($el.find('.stock').first().text()) ?? ''; | |
| 43 | + const soldOut = $el.find('.stock.soldout').length > 0 || /在庫なし|売り切れ|SOLD/i.test(stockText); | |
| 44 | + const stockM = stockText.replace(/[，,]/g, '').match(/(\d+)\s*(?:枚|点|個)/); | |
| 45 | + const stock = soldOut ? 0 : stockM ? Number(stockM[1]) : null; | |
| 46 | + const img = $el.find('.global_photo img').first(); | |
| 47 | + const image = img.attr('data-x2') ?? img.attr('src') ?? null; | |
| 48 | + items.push({ productId, url: href.startsWith('http') ? href : `https://${host}${href}`, title, modelNumber, priceJpy, stock, soldOut, image }); | |
| 49 | + }); | |
| 50 | + const hasNext = $('.pager a.to_next_page').length > 0; | |
| 51 | + const title = cleanText($('title').first().text())?.split(' - ')[0]?.trim() ?? null; | |
| 52 | + return { title, items, hasNext }; | |
| 53 | +} | |
| 54 | + | |
| 55 | +export class CardrushConnector extends BaseConnector { | |
| 56 | + readonly version = '1.0.0'; | |
| 57 | + readonly parserVersion = PARSER_VERSION; | |
| 58 | + protected override minIntervalMs = 4000; | |
| 59 | + | |
| 60 | + private stores(): CardrushStore[] { | |
| 61 | + return z.array(StoreSchema).parse(this.meta.config.stores ?? []); | |
| 62 | + } | |
| 63 | + | |
| 64 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 65 | + let stores = this.stores(); | |
| 66 | + if (ctx.options.seeds?.length) stores = stores.filter((s) => ctx.options.seeds!.some((seed) => seed === s.host || seed === s.categorySlug || seed.startsWith(`${s.host}/`))); | |
| 67 | + const backfill = ctx.options.mode === 'backfill'; | |
| 68 | + let storeIdx = Number(ctx.options.cursor?.storeIdx ?? 0); | |
| 69 | + let listIdx = Number(ctx.options.cursor?.listIdx ?? 0); | |
| 70 | + let page = Number(ctx.options.cursor?.page ?? 1); | |
| 71 | + let count = 0; | |
| 72 | + for (; storeIdx < stores.length; storeIdx++, listIdx = 0) { | |
| 73 | + const store = stores[storeIdx]!; | |
| 74 | + const lists = Object.entries(store.lists); | |
| 75 | + const maxPages = backfill ? this.policy.backfillMaxPages : Number(store.pagesPerList ?? this.meta.config.pagesPerList ?? 3); | |
| 76 | + const engines = (store.engines ?? ['api']) as Engine[]; | |
| 77 | + for (; listIdx < lists.length; listIdx++, page = 1) { | |
| 78 | + const [listId, listName] = lists[listIdx]!; | |
| 79 | + for (; page <= maxPages; page++) { | |
| 80 | + if (ctx.signal?.aborted) return; | |
| 81 | + if (this.reached(ctx, count)) { | |
| 82 | + await ctx.setCursor({ storeIdx, listIdx, page }); | |
| 83 | + return; | |
| 84 | + } | |
| 85 | + const url = `https://${store.host}/product-list/${listId}${page > 1 ? `?page=${page}` : ''}`; | |
| 86 | + await this.throttle(url); | |
| 87 | + const res = await ctx.fetch(url, { | |
| 88 | + engines, | |
| 89 | + headers: HTML_HEADERS, | |
| 90 | + responseType: 'text', | |
| 91 | + renderJs: false, | |
| 92 | + country: 'jp', | |
| 93 | + expect: ['title', 'price'], | |
| 94 | + parse: (r) => { | |
| 95 | + const parsed = parseListPage(r.html ?? '', store.host); | |
| 96 | + return { title: parsed.items.length ? 'ok' : null, price: parsed.items.some((i) => i.priceJpy) ? 1 : null }; | |
| 97 | + }, | |
| 98 | + }); | |
| 99 | + if (!res.success || !res.html) { | |
| 100 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 101 | + break; | |
| 102 | + } | |
| 103 | + const { title, items, hasNext } = parseListPage(res.html, store.host); | |
| 104 | + if (!items.length) { | |
| 105 | + if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no .item_data blocks`); | |
| 106 | + break; | |
| 107 | + } | |
| 108 | + count++; | |
| 109 | + const payload: CardrushPayload = { store: { host: store.host, categorySlug: store.categorySlug, franchise: store.franchise ?? null, brand: store.brand ?? null, bracket: store.bracket }, listId, listName: title ?? listName, page, items }; | |
| 110 | + yield { url, externalId: `${store.host}:${listId}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 111 | + await ctx.setCursor({ storeIdx, listIdx, page: page + 1 }); | |
| 112 | + await ctx.progress({ page, totalPages: null, itemsProcessed: count }); | |
| 113 | + if (!hasNext) break; | |
| 114 | + } | |
| 115 | + } | |
| 116 | + } | |
| 117 | + await ctx.setCursor({ storeIdx: 0, listIdx: 0, page: 1, completedAt: new Date().toISOString(), done: true }); | |
| 118 | + } | |
| 119 | + | |
| 120 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 121 | + const p = RawPayloadSchema.parse(raw.payload); | |
| 122 | + const out: NormalizedRecord[] = []; | |
| 123 | + const observedAt = raw.fetchedAt; | |
| 124 | + const seen = new Set<string>(); | |
| 125 | + for (const it of p.items) { | |
| 126 | + if (it.priceJpy === null || seen.has(it.productId)) continue; | |
| 127 | + seen.add(it.productId); | |
| 128 | + if (JP_EXCLUDE_RE.test(it.title)) continue; | |
| 129 | + const t = parseJpCardTitle(it.title, { bracket: p.store.bracket }); | |
| 130 | + const language = t.language ?? (t.notes.some((n) => /英語/.test(n)) ? 'English' : 'Japanese'); | |
| 131 | + const isBundle = isJpBundle(it.title, t.quantity); | |
| 132 | + const variant = jpVariant(t.notes, t.name); | |
| 133 | + const a = attrs({ | |
| 134 | + categorySlug: p.store.categorySlug, | |
| 135 | + franchise: p.store.franchise ?? null, | |
| 136 | + brand: p.store.brand ?? null, | |
| 137 | + setCode: t.setCode, | |
| 138 | + name: t.name, | |
| 139 | + number: t.number, | |
| 140 | + variant, | |
| 141 | + language, | |
| 142 | + rarity: t.rarity, | |
| 143 | + // model_number is the shop's katakana/romaji reading of the name (a search key), NOT a SKU/GTIN → never emitted as `sku` | |
| 144 | + identifiers: { cardrush_product_id: `${p.store.host}/${it.productId}`, ...(it.modelNumber ? { cardrush_model_number: it.modelNumber } : {}) }, | |
| 145 | + metadata: { store: p.store.host, list_id: p.listId, list_name: p.listName, total: t.total, notes: t.notes, quantity: t.quantity, sealed: t.sealed }, | |
| 146 | + }); | |
| 147 | + out.push( | |
| 148 | + NormalizedListingSchema.parse({ | |
| 149 | + kind: 'listing', | |
| 150 | + connectorId: this.meta.id, | |
| 151 | + sourceId: this.meta.sourceId, | |
| 152 | + sourceUrl: it.url, | |
| 153 | + externalId: `${p.store.host}:${it.productId}`, | |
| 154 | + rawTitle: it.title, | |
| 155 | + imageUrls: it.image ? [it.image] : [], | |
| 156 | + attributes: a, | |
| 157 | + grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier, certificationNumber: null }, | |
| 158 | + condition: { condition: jpCondition(t.conditionRaw), conditionRaw: t.conditionRaw, completeness: t.sealed ? 'sealed' : null }, | |
| 159 | + observedAt, | |
| 160 | + confidence: t.grade ? 0.8 : 0.75, | |
| 161 | + parserVersion: PARSER_VERSION, | |
| 162 | + listingType: 'fixed_price', | |
| 163 | + price: it.priceJpy, | |
| 164 | + currency: 'JPY', | |
| 165 | + seller: 'Cardrush', | |
| 166 | + location: 'JP', | |
| 167 | + quantity: isBundle ? t.quantity : it.stock, | |
| 168 | + availability: it.soldOut ? 'ended' : 'available', | |
| 169 | + }), | |
| 170 | + ); | |
| 171 | + } | |
| 172 | + return out; | |
| 173 | + } | |
| 174 | +} | |
| 175 | + | |
| 176 | +export default (meta: ConnectorMeta) => new CardrushConnector(meta); | |
added
connectors/api/cardrush/meta.json
+44 −0
@@ -0,0 +1,44 @@ | ||
| 1 | +{ | |
| 2 | + "id": "cardrush", | |
| 3 | + "displayName": "Cardrush (カードラッシュ) singles", | |
| 4 | + "sourceId": "cardrush", | |
| 5 | + "sourceName": "Cardrush", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.cardrush-mtg.jp", | |
| 8 | + "module": "api/cardrush", | |
| 9 | + "enginePriority": ["api", "scrapfly"], | |
| 10 | + "categories": ["magic_the_gathering", "one_piece_card_game", "yugioh", "digimon_tcg", "pokemon", "other_tcg"], | |
| 11 | + "regions": ["JP"], | |
| 12 | + "country": "JP", | |
| 13 | + "languages": ["ja"], | |
| 14 | + "currency": ["JPY"], | |
| 15 | + "supportsListings": true, | |
| 16 | + "supportsSold": false, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 1440, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.8, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.cardrush-mtg.jp/law", | |
| 27 | + "acquisitionMethod": "public product-list HTML pages (plain HTTP; Scrapfly ASP only for the Cloudflare-fronted Pokémon store)", | |
| 28 | + "historicalDepth": "none", | |
| 29 | + "capabilities": ["live_listings", "images"], | |
| 30 | + "accessNotes": "Cardrush is a Japanese single-card retailer running one storefront per game on the same shop platform: www.cardrush-mtg.jp (Magic), www.cardrush-op.jp (One Piece), www.cardrush.jp (Yu-Gi-Oh!), www.cardrush-dm.jp (Duel Masters), www.cardrush-digimon.jp (Digimon) and www.cardrush-pokemon.jp (Pokémon). Category listing pages https://<store>/product-list/<id>?page=<n> are public, server-rendered (100 items per page, pager with a 'next' link) and give product id, title ([condition] name 《language》【set】 or 〔condition/grade〕name【rarity】{number}), JPY price incl. tax, stock/在庫なし and image. robots.txt on every store disallows only GPTBot/Bytespider/TikTokSpider/meta-externalagent (the Pokémon store adds a Cloudflare 'Content-Signal: search=yes, ai-train=no' block and more AI crawlers) — RareIndexBot is allowed everywhere. Five stores answer plain HTTP with the honest UA; the Pokémon store sits behind a Cloudflare managed challenge (403 for any non-browser client), so its public listing pages are fetched through Scrapfly ASP (country jp, no JS rendering, ~25 credits per page) and limited to one page per category per run — no login, no private data. Requests ≥ 4 s apart (8 s on the Pokémon store), 1 concurrent. Not fetched: product detail pages (only linked), buy-list (買取) prices, mystery packs (オリパ) and supplies categories, carts. Shop asking prices → listings, never sales.", | |
| 31 | + "enabled": true, | |
| 32 | + "schemaVersion": "1.0", | |
| 33 | + "config": { | |
| 34 | + "stores": [ | |
| 35 | + { "host": "www.cardrush-mtg.jp", "categorySlug": "magic_the_gathering", "franchise": "Magic: The Gathering", "brand": "Wizards of the Coast", "bracket": "set", "lists": { "2": "スタンダード(パック別)", "3": "モダン(パック別)", "4": "レガシー(パック別)", "5": "特殊セット(パック別)", "7": "統率者" } }, | |
| 36 | + { "host": "www.cardrush-op.jp", "categorySlug": "one_piece_card_game", "franchise": "One Piece", "brand": "Bandai", "bracket": "rarity", "lists": { "17": "赤", "18": "緑", "19": "青", "20": "紫", "21": "黒", "22": "黄", "23": "多色", "4": "未開封BOX" } }, | |
| 37 | + { "host": "www.cardrush.jp", "categorySlug": "yugioh", "franchise": "Yu-Gi-Oh!", "brand": "Konami", "bracket": "rarity", "lists": { "291": "コレクター向け", "384": "未開封BOX" } }, | |
| 38 | + { "host": "www.cardrush-dm.jp", "categorySlug": "other_tcg", "franchise": "Duel Masters", "brand": "Takara Tomy", "bracket": "rarity", "lists": { "2": "光", "4": "火", "3": "闇", "5": "自然", "6": "水", "7": "無", "8": "多" } }, | |
| 39 | + { "host": "www.cardrush-digimon.jp", "categorySlug": "digimon_tcg", "franchise": "Digimon", "brand": "Bandai", "bracket": "rarity", "lists": { "2": "デジモン", "3": "テイマー", "4": "デジタマ", "5": "オプション", "7": "未開封商品" } }, | |
| 40 | + { "host": "www.cardrush-pokemon.jp", "categorySlug": "pokemon", "franchise": "Pokémon", "brand": "The Pokémon Company", "bracket": "rarity", "engines": ["scrapfly"], "pagesPerList": 1, "lists": { "3": "草", "2": "炎", "5": "水", "4": "雷", "9": "超", "7": "闘", "16": "悪", "10": "鋼", "8": "妖", "6": "竜", "11": "無", "23": "未開封BOX" } } | |
| 41 | + ], | |
| 42 | + "pagesPerList": 3 | |
| 43 | + } | |
| 44 | +} | |
added
connectors/api/cardtrader/README.md
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +# cardtrader (gated) | |
| 2 | + | |
| 3 | +CardTrader API v2 — `https://api.cardtrader.com/api/v2`, `Authorization: Bearer $CARDTRADER_API_TOKEN` (token from any CardTrader account's settings page; unauthenticated calls return 401). Docs: https://www.cardtrader.com/docs/api/full. | |
| 4 | + | |
| 5 | +Flow: `GET /games` → `GET /expansions` (filtered to tracked games via `config.gameSlugs`, newest expansion ids first) → `GET /blueprints/export?expansion_id=` (catalog: `scryfall_id`, `card_market_ids` → `cardmarket_id`, `tcg_player_id`, image, `fixed_properties.collector_number`/rarity) → `GET /marketplace/products?expansion_id=` (25 cheapest public offers per blueprint: cents + currency, condition/language/foil properties, seller type/country, graded, bundle_size). | |
| 6 | + | |
| 7 | +Output: `catalog_item` per blueprint + `listing` per offer (EUR/USD; private sellers anonymised; `bundle_size` → quantity; graded flagged). Rate limits: 200 req/10 s global, 10 req/s marketplace — connector uses 1 req/s. Cursor `{ expIdx }`. | |
| 8 | + | |
| 9 | +Fixtures are built from the documentation's example responses (no token in this environment) and are labelled as such in their `note`. | |
added
connectors/api/cardtrader/index.test.ts
+60 −0
@@ -0,0 +1,60 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import { createCrawlContext, createRouter, missingRequirements } from '@rareindex/connectors'; | |
| 6 | +import createConnector, { offerProperties, slugForGame } from './index.js'; | |
| 7 | + | |
| 8 | +const connector = createConnector(localMeta(meta)); | |
| 9 | + | |
| 10 | +describe('cardtrader (gated)', () => { | |
| 11 | + runFixtureSuite(connector, it, expect); | |
| 12 | + | |
| 13 | + it('is reported as gated until CARDTRADER_API_TOKEN exists and never crashes without it', async () => { | |
| 14 | + expect(meta.requires).toEqual(['CARDTRADER_API_TOKEN']); | |
| 15 | + const saved = process.env.CARDTRADER_API_TOKEN; | |
| 16 | + delete process.env.CARDTRADER_API_TOKEN; | |
| 17 | + try { | |
| 18 | + expect(missingRequirements(connector.meta)).toEqual(['CARDTRADER_API_TOKEN']); | |
| 19 | + const ctx = createCrawlContext({ router: createRouter({}), meta: connector.meta, options: { mode: 'probe', limit: 1 } }); | |
| 20 | + const seen: unknown[] = []; | |
| 21 | + for await (const r of connector.crawl(ctx)) seen.push(r); | |
| 22 | + expect(seen).toEqual([]); | |
| 23 | + expect(ctx.engineStats).toEqual({}); | |
| 24 | + } finally { | |
| 25 | + if (saved !== undefined) process.env.CARDTRADER_API_TOKEN = saved; | |
| 26 | + } | |
| 27 | + }); | |
| 28 | + | |
| 29 | + it('maps games and offer properties', () => { | |
| 30 | + const map = meta.config.gameSlugs as Record<string, string>; | |
| 31 | + expect(slugForGame('Magic Magic: the Gathering', map)).toBe('magic_the_gathering'); | |
| 32 | + expect(slugForGame('Pokemon', map)).toBe('pokemon'); | |
| 33 | + expect(slugForGame('Warhammer 40k', map)).toBeNull(); | |
| 34 | + const p = offerProperties({ condition: 'Moderately Played', signed: false, mtg_foil: true, mtg_language: 'en', altered: false }); | |
| 35 | + expect(p).toMatchObject({ conditionRaw: 'Moderately Played', condition: 'very_good', language: 'English', foil: true, signed: false }); | |
| 36 | + expect(offerProperties({ condition: 'Near Mint', pokemon_language: 'jp', pokemon_reverse: true }).reverse).toBe(true); | |
| 37 | + }); | |
| 38 | + | |
| 39 | + it('emits a catalog item with cross-marketplace ids and one listing per public offer', async () => { | |
| 40 | + for (const name of listFixtures('cardtrader')) { | |
| 41 | + const fx = loadFixture('cardtrader', name); | |
| 42 | + const out = await connector.normalize(fx.raw); | |
| 43 | + const cats = out.filter((r) => r.kind === 'catalog_item'); | |
| 44 | + expect(cats.length).toBe(1); | |
| 45 | + const c = cats[0]!; | |
| 46 | + if (c.kind === 'catalog_item') { | |
| 47 | + expect(c.attributes.identifiers.cardtrader_blueprint_id).toMatch(/^\d+$/); | |
| 48 | + expect(c.attributes.set).toBeTruthy(); | |
| 49 | + } | |
| 50 | + for (const r of out) { | |
| 51 | + if (r.kind !== 'listing') continue; | |
| 52 | + expect(['EUR', 'USD']).toContain(r.currency); | |
| 53 | + expect(r.price).toBeGreaterThan(0); | |
| 54 | + expect(r.listingType).toBe('fixed_price'); | |
| 55 | + // private sellers' usernames are never stored | |
| 56 | + if ((r.attributes.metadata as { seller_type: string }).seller_type !== 'professional') expect(r.seller).toBeNull(); | |
| 57 | + } | |
| 58 | + } | |
| 59 | + }); | |
| 60 | +}); | |
added
connectors/api/cardtrader/index.ts
+217 −0
@@ -0,0 +1,217 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, missingRequirements, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { normalizeCondition } from '@rareindex/taxonomy'; | |
| 5 | +import { attrs, catalogItem, makeTitle } from '../_lib/shared.js'; | |
| 6 | +import { JSON_HEADERS } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 7 | + | |
| 8 | +/** | |
| 9 | + * CardTrader API v2 (gated: CARDTRADER_API_TOKEN). games → expansions → blueprints/export (catalog with | |
| 10 | + * Scryfall/Cardmarket/TCGplayer ids) → marketplace/products (cheapest public offers, EUR/USD). | |
| 11 | + * One raw record per blueprint (with its offers); normalize emits a catalog item and one listing per offer. | |
| 12 | + */ | |
| 13 | +const API = 'https://api.cardtrader.com/api/v2'; | |
| 14 | +const PARSER_VERSION = '1.0.0'; | |
| 15 | + | |
| 16 | +export const GameSchema = z.object({ id: z.number().int(), name: z.string(), display_name: z.string().nullable().optional() }); | |
| 17 | +export const ExpansionSchema = z.object({ id: z.number().int(), game_id: z.number().int(), code: z.string().nullable().optional(), name: z.string() }); | |
| 18 | +export const BlueprintSchema = z.object({ | |
| 19 | + id: z.number().int(), | |
| 20 | + name: z.string(), | |
| 21 | + version: z.string().nullable().optional(), | |
| 22 | + game_id: z.number().int(), | |
| 23 | + category_id: z.number().int().nullable().optional(), | |
| 24 | + expansion_id: z.number().int().nullable().optional(), | |
| 25 | + image_url: z.string().nullable().optional(), | |
| 26 | + scryfall_id: z.string().nullable().optional(), | |
| 27 | + card_market_ids: z.array(z.number()).nullable().optional(), | |
| 28 | + tcg_player_id: z.union([z.string(), z.number()]).nullable().optional(), | |
| 29 | + fixed_properties: z.record(z.string(), z.unknown()).nullable().optional(), | |
| 30 | +}); | |
| 31 | +export type Blueprint = z.infer<typeof BlueprintSchema>; | |
| 32 | +export const ProductSchema = z.object({ | |
| 33 | + id: z.number().int(), | |
| 34 | + blueprint_id: z.number().int(), | |
| 35 | + name_en: z.string().nullable().optional(), | |
| 36 | + quantity: z.number().int().nullable().optional(), | |
| 37 | + price: z.object({ cents: z.number(), currency: z.string() }), | |
| 38 | + description: z.string().nullable().optional(), | |
| 39 | + properties_hash: z.record(z.string(), z.unknown()).default({}), | |
| 40 | + expansion: z.object({ id: z.number().int().optional(), code: z.string().nullable().optional(), name_en: z.string().nullable().optional() }).nullable().optional(), | |
| 41 | + user: z.object({ id: z.number().int().optional(), username: z.string().nullable().optional(), country_code: z.string().nullable().optional(), user_type: z.string().nullable().optional(), can_sell_via_hub: z.boolean().optional() }).nullable().optional(), | |
| 42 | + graded: z.boolean().nullable().optional(), | |
| 43 | + on_vacation: z.boolean().nullable().optional(), | |
| 44 | + bundle_size: z.number().int().nullable().optional(), | |
| 45 | +}); | |
| 46 | +export type Product = z.infer<typeof ProductSchema>; | |
| 47 | +const RawPayloadSchema = z.object({ game: GameSchema, categorySlug: z.string(), expansion: ExpansionSchema, blueprint: BlueprintSchema, products: z.array(ProductSchema) }); | |
| 48 | +export type CardtraderPayload = z.infer<typeof RawPayloadSchema>; | |
| 49 | + | |
| 50 | +const CURRENCIES = new Set(['EUR', 'USD', 'GBP', 'CHF', 'CAD', 'AUD', 'JPY', 'SEK', 'NOK', 'DKK', 'PLN', 'CZK']); | |
| 51 | + | |
| 52 | +/** Game name → taxonomy slug via the configurable substring map (null = not tracked). */ | |
| 53 | +export function slugForGame(name: string, map: Record<string, string>): string | null { | |
| 54 | + const s = name.toLowerCase(); | |
| 55 | + for (const [needle, slug] of Object.entries(map)) if (s.includes(needle)) return slug; | |
| 56 | + return null; | |
| 57 | +} | |
| 58 | + | |
| 59 | +/** properties_hash → normalised bits (condition slug, language, foil, first edition, signed/altered). */ | |
| 60 | +export function offerProperties(h: Record<string, unknown>): { conditionRaw: string | null; condition: string | null; language: string | null; foil: boolean; firstEdition: boolean; signed: boolean; altered: boolean; reverse: boolean } { | |
| 61 | + const conditionRaw = typeof h.condition === 'string' ? h.condition : null; | |
| 62 | + const langKey = Object.keys(h).find((k) => /_language$|^language$/.test(k)); | |
| 63 | + const lang = langKey && typeof h[langKey] === 'string' ? (h[langKey] as string) : null; | |
| 64 | + const LANG: Record<string, string> = { en: 'English', it: 'Italian', fr: 'French', de: 'German', es: 'Spanish', pt: 'Portuguese', jp: 'Japanese', ja: 'Japanese', ko: 'Korean', ru: 'Russian', zh: 'Chinese', 'zh-cn': 'Chinese', 'zh-tw': 'Chinese' }; | |
| 65 | + const foil = Object.entries(h).some(([k, v]) => /_foil$|^foil$/.test(k) && (v === true || v === 'true')); | |
| 66 | + const reverse = Object.entries(h).some(([k, v]) => /reverse/.test(k) && (v === true || v === 'true')); | |
| 67 | + const firstEdition = Object.entries(h).some(([k, v]) => /first_edition/.test(k) && (v === true || v === 'true')); | |
| 68 | + return { conditionRaw, condition: normalizeCondition('trading_cards', conditionRaw), language: lang ? (LANG[lang.toLowerCase()] ?? lang) : null, foil, firstEdition, signed: h.signed === true, altered: h.altered === true, reverse }; | |
| 69 | +} | |
| 70 | + | |
| 71 | +export class CardtraderConnector extends BaseConnector { | |
| 72 | + readonly version = '1.0.0'; | |
| 73 | + readonly parserVersion = PARSER_VERSION; | |
| 74 | + protected override minIntervalMs = 1000; | |
| 75 | + | |
| 76 | + private headers(): Record<string, string> { | |
| 77 | + return { ...JSON_HEADERS, authorization: `Bearer ${process.env.CARDTRADER_API_TOKEN ?? ''}` }; | |
| 78 | + } | |
| 79 | + | |
| 80 | + private async get<T>(ctx: CrawlContext, path: string, schema: z.ZodType<T>): Promise<T | null> { | |
| 81 | + const url = `${API}${path}`; | |
| 82 | + await this.throttle(url); | |
| 83 | + const res = await ctx.fetch(url, { engines: ['api'], headers: this.headers(), minQuality: 0 }); | |
| 84 | + if (!res.success || res.json === null) { | |
| 85 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 86 | + return null; | |
| 87 | + } | |
| 88 | + const parsed = schema.safeParse(res.json); | |
| 89 | + if (!parsed.success) { | |
| 90 | + ctx.anomaly('schema_drift', `${url}: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`); | |
| 91 | + return null; | |
| 92 | + } | |
| 93 | + return parsed.data; | |
| 94 | + } | |
| 95 | + | |
| 96 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 97 | + const missing = missingRequirements(this.meta); | |
| 98 | + if (missing.length) { | |
| 99 | + ctx.log.warn({ missing }, 'cardtrader disabled: missing API token'); | |
| 100 | + return; | |
| 101 | + } | |
| 102 | + const slugMap = (this.meta.config.gameSlugs ?? {}) as Record<string, string>; | |
| 103 | + const includeMarketplace = this.meta.config.includeMarketplace !== false; | |
| 104 | + const backfill = ctx.options.mode === 'backfill'; | |
| 105 | + const perRun = backfill ? Infinity : Number(this.meta.config.expansionsPerRun ?? 30); | |
| 106 | + const games = await this.get(ctx, '/games', z.array(GameSchema.loose())); | |
| 107 | + if (!games) throw new Error('cardtrader: /games failed'); | |
| 108 | + const tracked = new Map<number, { game: z.infer<typeof GameSchema>; slug: string }>(); | |
| 109 | + for (const g of games) { | |
| 110 | + const slug = slugForGame(`${g.name} ${g.display_name ?? ''}`, slugMap); | |
| 111 | + if (slug) tracked.set(g.id, { game: GameSchema.parse(g), slug }); | |
| 112 | + } | |
| 113 | + const allExp = await this.get(ctx, '/expansions', z.array(ExpansionSchema.loose())); | |
| 114 | + if (!allExp) throw new Error('cardtrader: /expansions failed'); | |
| 115 | + let expansions = allExp.filter((e) => tracked.has(e.game_id)).map((e) => ExpansionSchema.parse(e)).sort((a, b) => b.id - a.id); | |
| 116 | + if (ctx.options.seeds?.length) expansions = expansions.filter((e) => ctx.options.seeds!.includes(String(e.id)) || (e.code && ctx.options.seeds!.includes(e.code))); | |
| 117 | + expansions = expansions.slice(0, Number.isFinite(perRun) ? perRun : expansions.length); | |
| 118 | + let expIdx = Number(ctx.options.cursor?.expIdx ?? 0); | |
| 119 | + let count = 0; | |
| 120 | + for (; expIdx < expansions.length; expIdx++) { | |
| 121 | + if (ctx.signal?.aborted) return; | |
| 122 | + const expansion = expansions[expIdx]!; | |
| 123 | + const { game, slug } = tracked.get(expansion.game_id)!; | |
| 124 | + const blueprints = await this.get(ctx, `/blueprints/export?expansion_id=${expansion.id}`, z.array(z.unknown())); | |
| 125 | + if (!blueprints) continue; | |
| 126 | + const offers = new Map<number, Product[]>(); | |
| 127 | + if (includeMarketplace) { | |
| 128 | + const mp = await this.get(ctx, `/marketplace/products?expansion_id=${expansion.id}`, z.record(z.string(), z.array(z.unknown()))); | |
| 129 | + for (const [bp, list] of Object.entries(mp ?? {})) { | |
| 130 | + const parsed = list.map((p) => ProductSchema.safeParse(p)).filter((p) => p.success).map((p) => (p as { data: Product }).data); | |
| 131 | + offers.set(Number(bp), parsed); | |
| 132 | + } | |
| 133 | + } | |
| 134 | + for (const raw of blueprints) { | |
| 135 | + const bp = BlueprintSchema.safeParse(raw); | |
| 136 | + if (!bp.success) { | |
| 137 | + ctx.anomaly('parse_failure_blueprint', `${expansion.id}: ${bp.error.issues[0]?.message}`); | |
| 138 | + continue; | |
| 139 | + } | |
| 140 | + if (this.reached(ctx, count)) { | |
| 141 | + await ctx.setCursor({ expIdx }); | |
| 142 | + return; | |
| 143 | + } | |
| 144 | + count++; | |
| 145 | + const payload: CardtraderPayload = { game, categorySlug: slug, expansion, blueprint: bp.data, products: offers.get(bp.data.id) ?? [] }; | |
| 146 | + yield { url: `https://www.cardtrader.com/cards/${bp.data.id}`, externalId: String(bp.data.id), kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() }; | |
| 147 | + } | |
| 148 | + await ctx.setCursor({ expIdx: expIdx + 1 }); | |
| 149 | + await ctx.progress({ page: expIdx + 1, totalPages: expansions.length, itemsProcessed: count }); | |
| 150 | + } | |
| 151 | + await ctx.setCursor({ expIdx: 0, completedAt: new Date().toISOString(), done: true }); | |
| 152 | + } | |
| 153 | + | |
| 154 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 155 | + const { game, categorySlug, expansion, blueprint, products } = RawPayloadSchema.parse(raw.payload); | |
| 156 | + const fp = blueprint.fixed_properties ?? {}; | |
| 157 | + const str = (v: unknown) => (typeof v === 'string' && v.trim() ? v.trim() : typeof v === 'number' ? String(v) : null); | |
| 158 | + const number = str(fp.collector_number) ?? str(fp.number); | |
| 159 | + const rarity = str(Object.entries(fp).find(([k]) => /rarity/.test(k))?.[1]); | |
| 160 | + const ids: Record<string, string> = { cardtrader_blueprint_id: String(blueprint.id) }; | |
| 161 | + if (blueprint.scryfall_id) ids.scryfall_id = blueprint.scryfall_id; | |
| 162 | + if (blueprint.card_market_ids?.length) ids.cardmarket_id = String(blueprint.card_market_ids[0]); | |
| 163 | + if (blueprint.tcg_player_id) ids.tcgplayer_id = String(blueprint.tcg_player_id); | |
| 164 | + const base = attrs({ | |
| 165 | + categorySlug, | |
| 166 | + franchise: game.display_name ?? game.name, | |
| 167 | + set: expansion.name, | |
| 168 | + setCode: expansion.code?.toUpperCase() ?? null, | |
| 169 | + name: blueprint.name, | |
| 170 | + number, | |
| 171 | + variant: blueprint.version ?? null, | |
| 172 | + language: null, | |
| 173 | + rarity, | |
| 174 | + identifiers: ids, | |
| 175 | + metadata: { cardtrader_game_id: game.id, cardtrader_expansion_id: expansion.id, category_id: blueprint.category_id ?? null, fixed_properties: fp }, | |
| 176 | + }); | |
| 177 | + const images = blueprint.image_url ? [blueprint.image_url.startsWith('http') ? blueprint.image_url : `https://www.cardtrader.com${blueprint.image_url}`] : []; | |
| 178 | + const rawTitle = makeTitle({ name: blueprint.name, set: expansion.name, number, variant: blueprint.version ?? null }); | |
| 179 | + const out: NormalizedRecord[] = [catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: String(blueprint.id), rawTitle, imageUrls: images, attributes: base, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: null })]; | |
| 180 | + for (const p of products) { | |
| 181 | + const currency = p.price.currency.toUpperCase(); | |
| 182 | + if (!CURRENCIES.has(currency) || !(p.price.cents > 0)) continue; | |
| 183 | + const props = offerProperties(p.properties_hash); | |
| 184 | + const variant = [blueprint.version, props.firstEdition ? '1st Edition' : null, props.reverse ? 'Reverse Holo' : props.foil ? 'Foil' : null].filter(Boolean).join(' ') || null; | |
| 185 | + const bundle = (p.bundle_size ?? 1) > 1; | |
| 186 | + const professional = p.user?.user_type === 'professional'; | |
| 187 | + out.push( | |
| 188 | + NormalizedListingSchema.parse({ | |
| 189 | + kind: 'listing', | |
| 190 | + connectorId: this.meta.id, | |
| 191 | + sourceId: this.meta.sourceId, | |
| 192 | + sourceUrl: raw.url, | |
| 193 | + externalId: `${blueprint.id}:${p.id}`, | |
| 194 | + rawTitle: `${p.name_en ?? blueprint.name} · ${expansion.name}${variant ? ` ${variant}` : ''}${props.conditionRaw ? ` · ${props.conditionRaw}` : ''}${props.language ? ` (${props.language})` : ''}`, | |
| 195 | + description: p.description?.slice(0, 500) ?? null, | |
| 196 | + imageUrls: images, | |
| 197 | + attributes: { ...base, variant, language: props.language, metadata: { ...base.metadata, properties: p.properties_hash, signed: props.signed, altered: props.altered, graded: p.graded ?? false, bundle_size: p.bundle_size ?? 1, seller_type: p.user?.user_type ?? null, seller_country: p.user?.country_code ?? null, cardtrader_zero: p.user?.can_sell_via_hub ?? null } }, | |
| 198 | + grade: { grader: null, grade: null, qualifier: p.graded ? 'graded (grader not exposed by API)' : null, certificationNumber: null }, | |
| 199 | + condition: { condition: props.condition, conditionRaw: props.conditionRaw, completeness: null }, | |
| 200 | + observedAt: raw.fetchedAt, | |
| 201 | + confidence: 0.8, | |
| 202 | + parserVersion: PARSER_VERSION, | |
| 203 | + listingType: 'fixed_price', | |
| 204 | + price: p.price.cents / 100, | |
| 205 | + currency, | |
| 206 | + seller: professional ? (p.user?.username ?? null) : null, | |
| 207 | + location: p.user?.country_code ?? null, | |
| 208 | + quantity: bundle ? p.bundle_size : (p.quantity ?? null), | |
| 209 | + availability: p.on_vacation ? 'unknown' : 'available', | |
| 210 | + }), | |
| 211 | + ); | |
| 212 | + } | |
| 213 | + return out; | |
| 214 | + } | |
| 215 | +} | |
| 216 | + | |
| 217 | +export default (meta: ConnectorMeta) => new CardtraderConnector(meta); | |
added
connectors/api/cardtrader/meta.json
+54 −0
@@ -0,0 +1,54 @@ | ||
| 1 | +{ | |
| 2 | + "id": "cardtrader", | |
| 3 | + "displayName": "CardTrader (API v2 — blueprints & marketplace)", | |
| 4 | + "sourceId": "cardtrader", | |
| 5 | + "sourceName": "CardTrader", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://www.cardtrader.com", | |
| 8 | + "module": "api/cardtrader", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["magic_the_gathering", "pokemon", "yugioh", "one_piece_card_game", "disney_lorcana", "flesh_and_blood", "digimon_tcg", "dragon_ball_tcg", "star_wars_tcg", "other_tcg"], | |
| 11 | + "regions": ["EU", "IT"], | |
| 12 | + "country": "IT", | |
| 13 | + "languages": ["en", "it", "es"], | |
| 14 | + "currency": ["EUR", "USD"], | |
| 15 | + "supportsListings": true, | |
| 16 | + "supportsSold": false, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": true, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 1440, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.8, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.cardtrader.com/en/terms", | |
| 27 | + "acquisitionMethod": "official REST API v2 (Bearer token from a CardTrader account)", | |
| 28 | + "historicalDepth": "none", | |
| 29 | + "requires": ["CARDTRADER_API_TOKEN"], | |
| 30 | + "capabilities": ["catalog", "live_listings", "images"], | |
| 31 | + "accessNotes": "CardTrader (Italy) is Europe's second card marketplace and publishes a documented REST API at https://api.cardtrader.com/api/v2 (docs: https://www.cardtrader.com/docs/api/full). Every call needs `Authorization: Bearer <token>`; the token is issued to any registered CardTrader account from its settings page — unauthenticated calls return 401 (verified live), so the connector is gated behind CARDTRADER_API_TOKEN. Endpoints used, all read-only: GET /games, GET /expansions (id, game_id, code, name), GET /blueprints/export?expansion_id=<id> (one blueprint per printing with scryfall_id, card_market_ids, tcg_player_id, image_url, fixed_properties such as collector_number/rarity) and GET /marketplace/products?expansion_id=<id> (the 25 cheapest public offers per blueprint: price cents + currency, quantity, properties_hash condition/language/foil, seller username/country/type, graded, bundle_size). Rate limits per docs: 200 requests / 10 s overall, 10 req/s on /marketplace/products (we stay at ≤ 1 req/s). The public website (Cloudflare) is never crawled. Seller usernames are personal data for 'normal' (private) sellers and are never stored; only user_type/country_code are kept. Offers are asking prices → listings, never sales; graded and bundle offers are flagged. Fixtures were built from the official documentation's example responses (no token available) and say so in their note.", | |
| 32 | + "enabled": true, | |
| 33 | + "schemaVersion": "1.0", | |
| 34 | + "config": { | |
| 35 | + "gameSlugs": { | |
| 36 | + "magic": "magic_the_gathering", | |
| 37 | + "pokemon": "pokemon", | |
| 38 | + "yugioh": "yugioh", | |
| 39 | + "one piece": "one_piece_card_game", | |
| 40 | + "lorcana": "disney_lorcana", | |
| 41 | + "flesh and blood": "flesh_and_blood", | |
| 42 | + "digimon": "digimon_tcg", | |
| 43 | + "dragon ball": "dragon_ball_tcg", | |
| 44 | + "star wars": "star_wars_tcg", | |
| 45 | + "vanguard": "other_tcg", | |
| 46 | + "riftbound": "other_tcg", | |
| 47 | + "union arena": "other_tcg", | |
| 48 | + "battle spirits": "other_tcg", | |
| 49 | + "gundam": "other_tcg" | |
| 50 | + }, | |
| 51 | + "expansionsPerRun": 30, | |
| 52 | + "includeMarketplace": true | |
| 53 | + } | |
| 54 | +} | |
added
connectors/api/cdncoin/README.md
+42 −0
@@ -0,0 +1,42 @@ | ||
| 1 | +# Canadian Coin & Currency connector (`cdncoin`) | |
| 2 | + | |
| 3 | +- Source: https://www.cdncoin.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Richmond Hill numismatic dealer: Canadian decimal, certified coins (ICCS/PCGS), PMG notes, world/US/ancient coins, medals & tokens. Shopify storefront with SKU per item. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `certified` | `coins` | — | | |
| 14 | +| `pcgs` | `coins` | — | | |
| 15 | +| `canadian-collections` | `coins` | — | | |
| 16 | +| `world-coins` | `coins` | — | | |
| 17 | +| `us-coins` | `coins` | — | | |
| 18 | +| `ancient-coins` | `coins` | — | | |
| 19 | +| `pre-confederation-tokens` | `coins` | — | | |
| 20 | +| `royal-canadian-mint-products-2` | `coins` | — | | |
| 21 | +| `canadian-paper-money` | `banknotes` | — | | |
| 22 | +| `pmg` | `banknotes` | — | | |
| 23 | +| `world-paper-money` | `banknotes` | — | | |
| 24 | +| `us-paper-money` | `banknotes` | — | | |
| 25 | +| `replacement-notes` | `banknotes` | — | | |
| 26 | +| `bank-of-canada-1937` | `banknotes` | — | | |
| 27 | +| `dominion-province-of-canada` | `banknotes` | — | | |
| 28 | +| `chartered` | `banknotes` | — | | |
| 29 | +| `medals-exonumia` | `medals` | — | | |
| 30 | + | |
| 31 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 32 | +- `\bPMG\b|\bbanknotes?\b|\bpaper money\b|\| (Canadian Paper Money|World Paper Money|US Paper Money|Chartered Bank Notes?|Dominion of Canada|Bank of Canada)[^|]* \||\bBC-\d|\bDC-\d` → `banknotes` | |
| 33 | +- `\bmedal(lion)?s?\b` → `medals` | |
| 34 | + | |
| 35 | +Excluded (regex): `gift card|supplies|\balbums?\b|\bholders?\b|\bcapsules?\b|catalogue|catalog|magnifier|cleaning|\bflips?\b|\bpages?\b|\brolls?\b|\bbooks?\b|\btubes?\b|\bcases?\b|storage|\bbinder|\bloupe\b|\bgloves?\b|\bscale\b|\bmaps?\b|\bframe` | |
| 36 | + | |
| 37 | +## Access & compliance | |
| 38 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: bullion (silver/gold), gift/theme collections ("for nature lovers"…), supplies, new-arrivals umbrella. | |
| 39 | + | |
| 40 | +## Fixtures & tests | |
| 41 | +`data/fixtures/cdncoin/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts cdncoin`), trimmed single-product payloads incl. a sold-out variant and a graded item. | |
| 42 | +`pnpm vitest run connectors/api/cdncoin` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts cdncoin`. | |
added
connectors/api/cdncoin/index.test.ts
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Canadian Coin & Currency — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/cdncoin/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "$1 1935 Dbl XXV; Dbl Voyageur ICCS MS-65", | |
| 15 | + "productType": "Canadian Decimal Coins", | |
| 16 | + "collection": "certified", | |
| 17 | + "categorySlug": "coins" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "$20 1954 Bank of Canada Beattie-Coyne PMG 65 EPQ", | |
| 21 | + "productType": "Canadian Paper Money", | |
| 22 | + "collection": "certified", | |
| 23 | + "categorySlug": "banknotes" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "title": "Lighthouse Coin Album Canada Cents", | |
| 27 | + "collection": "canadian-collections", | |
| 28 | + "categorySlug": null | |
| 29 | + } | |
| 30 | + ] | |
| 31 | +}); | |
added
connectors/api/cdncoin/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Canadian Coin & Currency — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/cdncoin/meta.json
+136 −0
@@ -0,0 +1,136 @@ | ||
| 1 | +{ | |
| 2 | + "id": "cdncoin", | |
| 3 | + "displayName": "Canadian Coin & Currency (Canadian coin & banknote store, CAD)", | |
| 4 | + "sourceId": "cdncoin", | |
| 5 | + "sourceName": "Canadian Coin & Currency", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.cdncoin.com", | |
| 8 | + "module": "api/cdncoin", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "coins", | |
| 14 | + "banknotes", | |
| 15 | + "medals" | |
| 16 | + ], | |
| 17 | + "regions": [ | |
| 18 | + "CA" | |
| 19 | + ], | |
| 20 | + "languages": [ | |
| 21 | + "en" | |
| 22 | + ], | |
| 23 | + "currency": [ | |
| 24 | + "CAD" | |
| 25 | + ], | |
| 26 | + "supportsListings": true, | |
| 27 | + "supportsSold": false, | |
| 28 | + "supportsAuctions": false, | |
| 29 | + "supportsImages": true, | |
| 30 | + "supportsCatalog": false, | |
| 31 | + "supportsPopulation": false, | |
| 32 | + "supportsLookup": true, | |
| 33 | + "refreshFrequencyMinutes": 720, | |
| 34 | + "priority": "medium", | |
| 35 | + "trustScore": 0.75, | |
| 36 | + "attributionRequired": true, | |
| 37 | + "termsUrl": "https://www.cdncoin.com/policies/terms-of-service", | |
| 38 | + "accessNotes": "Canadian Coin & Currency (cdncoin.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 17 configured collections (certified, pcgs, canadian-collections, world-coins, us-coins, ancient-coins, pre-confederation-tokens, royal-canadian-mint-products-2 … (+9 more, see config.collections)) and /products/<handle>.json for URL lookups (~23k products; 4.7k certified (ICCS/PCGS/PMG), 4k Canadian paper money). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Third-party grades in titles (PSA/BGS/CGC/ICCS/PMG…) are parsed by parseGradeFromTitle; cert numbers are not extracted. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — bullion (silver/gold), gift/theme collections (\"for nature lovers\"…), supplies, new-arrivals umbrella. No personal data is collected; seller = the store itself.", | |
| 39 | + "enabled": true, | |
| 40 | + "schemaVersion": "1.0", | |
| 41 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 42 | + "historicalDepth": "none", | |
| 43 | + "requires": [], | |
| 44 | + "config": { | |
| 45 | + "currency": "CAD", | |
| 46 | + "market": "CA", | |
| 47 | + "seller": "Canadian Coin & Currency", | |
| 48 | + "location": "Richmond Hill, ON, Canada", | |
| 49 | + "collections": [ | |
| 50 | + { | |
| 51 | + "handle": "certified", | |
| 52 | + "categorySlug": "coins" | |
| 53 | + }, | |
| 54 | + { | |
| 55 | + "handle": "pcgs", | |
| 56 | + "categorySlug": "coins" | |
| 57 | + }, | |
| 58 | + { | |
| 59 | + "handle": "canadian-collections", | |
| 60 | + "categorySlug": "coins" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "world-coins", | |
| 64 | + "categorySlug": "coins" | |
| 65 | + }, | |
| 66 | + { | |
| 67 | + "handle": "us-coins", | |
| 68 | + "categorySlug": "coins" | |
| 69 | + }, | |
| 70 | + { | |
| 71 | + "handle": "ancient-coins", | |
| 72 | + "categorySlug": "coins" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "handle": "pre-confederation-tokens", | |
| 76 | + "categorySlug": "coins" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "royal-canadian-mint-products-2", | |
| 80 | + "categorySlug": "coins" | |
| 81 | + }, | |
| 82 | + { | |
| 83 | + "handle": "canadian-paper-money", | |
| 84 | + "categorySlug": "banknotes" | |
| 85 | + }, | |
| 86 | + { | |
| 87 | + "handle": "pmg", | |
| 88 | + "categorySlug": "banknotes" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "handle": "world-paper-money", | |
| 92 | + "categorySlug": "banknotes" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "handle": "us-paper-money", | |
| 96 | + "categorySlug": "banknotes" | |
| 97 | + }, | |
| 98 | + { | |
| 99 | + "handle": "replacement-notes", | |
| 100 | + "categorySlug": "banknotes" | |
| 101 | + }, | |
| 102 | + { | |
| 103 | + "handle": "bank-of-canada-1937", | |
| 104 | + "categorySlug": "banknotes" | |
| 105 | + }, | |
| 106 | + { | |
| 107 | + "handle": "dominion-province-of-canada", | |
| 108 | + "categorySlug": "banknotes" | |
| 109 | + }, | |
| 110 | + { | |
| 111 | + "handle": "chartered", | |
| 112 | + "categorySlug": "banknotes" | |
| 113 | + }, | |
| 114 | + { | |
| 115 | + "handle": "medals-exonumia", | |
| 116 | + "categorySlug": "medals" | |
| 117 | + } | |
| 118 | + ], | |
| 119 | + "rules": [ | |
| 120 | + { | |
| 121 | + "match": "\\bPMG\\b|\\bbanknotes?\\b|\\bpaper money\\b|\\| (Canadian Paper Money|World Paper Money|US Paper Money|Chartered Bank Notes?|Dominion of Canada|Bank of Canada)[^|]* \\||\\bBC-\\d|\\bDC-\\d", | |
| 122 | + "categorySlug": "banknotes" | |
| 123 | + }, | |
| 124 | + { | |
| 125 | + "match": "\\bmedal(lion)?s?\\b", | |
| 126 | + "categorySlug": "medals" | |
| 127 | + } | |
| 128 | + ], | |
| 129 | + "defaultCategory": null, | |
| 130 | + "exclude": "gift card|supplies|\\balbums?\\b|\\bholders?\\b|\\bcapsules?\\b|catalogue|catalog|magnifier|cleaning|\\bflips?\\b|\\bpages?\\b|\\brolls?\\b|\\bbooks?\\b|\\btubes?\\b|\\bcases?\\b|storage|\\bbinder|\\bloupe\\b|\\bgloves?\\b|\\bscale\\b|\\bmaps?\\b|\\bframe", | |
| 131 | + "keepOutOfStock": true, | |
| 132 | + "fetchBarcodes": false, | |
| 133 | + "wholeShop": false, | |
| 134 | + "pageSize": 250 | |
| 135 | + } | |
| 136 | +} | |
added
connectors/api/chairish/README.md
+12 −0
@@ -0,0 +1,12 @@ | ||
| 1 | +# chairish — Chairish vintage design marketplace (listings) | |
| 2 | + | |
| 3 | +Reads the schema.org `Product` JSON-LD array embedded in Chairish browse pages | |
| 4 | +(`/collection/<slug>`, `/style/<slug>`, `?page=N`, 48 per page): name, images, brand, color, material, | |
| 5 | +dimensions, Offer (USD price, availability, condition, category path, seller city/country). | |
| 6 | + | |
| 7 | +- One raw record per page, one `listing` per product (asking prices, never sales). | |
| 8 | +- Category: seed vertical + offer category + title → `design_furniture` / `antiques` / `porcelain` / | |
| 9 | + `glass_crystal` / `silver` / `clocks` / `art` / `jewelry` / watch brand / `luxury_handbags`; rugs and | |
| 10 | + textiles skipped. | |
| 11 | +- Cursor `{seedIndex, page}` rotates seeds between runs; `lookup()` handles `/product/<id>/…` URLs. | |
| 12 | +- Identifier: `chairish_product_id`. | |
added
connectors/api/chairish/index.test.ts
+53 −0
@@ -0,0 +1,53 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 3 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 4 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 5 | +import { designSlug } from '../_g10-lib/index.js'; | |
| 6 | +import createConnector, { hasNextPage, pageUrl, parseBrowseHtml } from './index.js'; | |
| 7 | + | |
| 8 | +const connector = createConnector(localMeta(metaJson)); | |
| 9 | + | |
| 10 | +describe('chairish', () => { | |
| 11 | + runFixtureSuite(connector, it, expect); | |
| 12 | + | |
| 13 | + it('parses JSON-LD products from a browse page snapshot', () => { | |
| 14 | + const html = String((loadFixture('chairish', 'vintage-furniture-p2').raw.payload as { snapshot?: string }).snapshot ?? ''); | |
| 15 | + const items = parseBrowseHtml(html); | |
| 16 | + expect(items.length).toBeGreaterThanOrEqual(2); | |
| 17 | + const first = items[0]!; | |
| 18 | + expect(first.id).toMatch(/^\d+$/); | |
| 19 | + expect(first.url).toMatch(/^https:\/\/www\.chairish\.com\/product\/\d+\//); | |
| 20 | + expect(first.price).toBeGreaterThan(0); | |
| 21 | + expect(first.currency).toBe('USD'); | |
| 22 | + expect(first.category).toMatch(/^Furniture/); | |
| 23 | + expect(first.availability).toBe('available'); | |
| 24 | + expect(first.images[0]).toMatch(/chairish-prod/); | |
| 25 | + expect(hasNextPage(html)).toBe(true); | |
| 26 | + expect(hasNextPage('<html><head></head><body></body></html>')).toBe(false); | |
| 27 | + }); | |
| 28 | + | |
| 29 | + it('builds page urls and maps categories', () => { | |
| 30 | + expect(pageUrl('/collection/lighting', 1)).toBe('https://www.chairish.com/collection/lighting'); | |
| 31 | + expect(pageUrl('/collection/lighting', 3)).toBe('https://www.chairish.com/collection/lighting?page=3'); | |
| 32 | + expect(designSlug('Furniture > Chairs > Chaises', 'Adrian Pearsall for Craft Associates Chaise Lounge', 'furniture')).toBe('design_furniture'); | |
| 33 | + expect(designSlug('Furniture > Chairs', 'Set of Five Louis XVI Style Carved Beech Dining Chairs, 19th Century', 'furniture')).toBe('antiques'); | |
| 34 | + expect(designSlug('Decor > Vases', 'Murano Glass Vase by Venini, 1960s', 'decor')).toBe('glass_crystal'); | |
| 35 | + expect(designSlug('Decor > Clocks', 'Howard Miller Mantel Clock', 'decor')).toBe('clocks'); | |
| 36 | + expect(designSlug('Rugs > Area Rugs', 'Vintage Persian Rug 8x10', 'rugs')).toBeNull(); | |
| 37 | + expect(designSlug('Jewelry > Watches', 'Rolex Datejust 1601', 'watches')).toBe('rolex'); | |
| 38 | + }); | |
| 39 | + | |
| 40 | + it('normalises listings in USD with the seller location and product id', async () => { | |
| 41 | + const fx = loadFixture('chairish', 'vintage-furniture-p2'); | |
| 42 | + const out = await connector.normalize(fx.raw); | |
| 43 | + expect(out.length).toBeGreaterThan(0); | |
| 44 | + for (const r of out) { | |
| 45 | + if (r.kind !== 'listing') throw new Error('expected listing'); | |
| 46 | + expect(r.currency).toBe('USD'); | |
| 47 | + expect(r.price).toBeGreaterThan(0); | |
| 48 | + expect(r.attributes.identifiers.chairish_product_id).toMatch(/^\d+$/); | |
| 49 | + expect(['design_furniture', 'antiques', 'clocks', 'porcelain', 'glass_crystal', 'silver', 'art']).toContain(r.attributes.categorySlug); | |
| 50 | + expect(r.listingType).toBe('fixed_price'); | |
| 51 | + } | |
| 52 | + }); | |
| 53 | +}); | |
added
connectors/api/chairish/index.ts
+255 −0
@@ -0,0 +1,255 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, adapters, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { designSlug, makerFromTitle, plainText, readSeedCursor, yearOrDecade, type DesignVertical } from '../_g10-lib/index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Chairish — curated vintage/antique furniture, decor, art and jewelry marketplace (US, USD). | |
| 8 | + * Public browse pages (/collection/<slug>, /style/<slug>, ?page=N, 48 items) embed one schema.org Product | |
| 9 | + * per listing in JSON-LD (name, description, images, brand, color, material, dimensions, Offer with price, | |
| 10 | + * currency, availability, condition, category path and the seller's city/country). We read only that | |
| 11 | + * structured block. Asking prices → listings (never sales). | |
| 12 | + */ | |
| 13 | +const BASE = 'https://www.chairish.com'; | |
| 14 | +const PARSER_VERSION = '1.0.0'; | |
| 15 | +const PAGE_SIZE = 48; | |
| 16 | + | |
| 17 | +export const SeedSchema = z.object({ path: z.string(), slug: z.string().nullable().optional(), vertical: z.enum(['furniture', 'lighting', 'decor', 'art', 'jewelry', 'watches', 'fashion', 'tableware', 'rugs', 'pens', 'unknown']).optional() }); | |
| 18 | +export type Seed = z.infer<typeof SeedSchema>; | |
| 19 | + | |
| 20 | +export const ItemSchema = z.object({ | |
| 21 | + id: z.string(), | |
| 22 | + url: z.string(), | |
| 23 | + name: z.string(), | |
| 24 | + description: z.string().nullable(), | |
| 25 | + images: z.array(z.string()), | |
| 26 | + brand: z.string().nullable(), | |
| 27 | + color: z.string().nullable(), | |
| 28 | + material: z.string().nullable(), | |
| 29 | + category: z.string().nullable(), | |
| 30 | + price: z.number().nullable(), | |
| 31 | + currency: z.string().nullable(), | |
| 32 | + availability: z.enum(['available', 'sold', 'ended', 'unknown']), | |
| 33 | + condition: z.string().nullable(), | |
| 34 | + sellerCity: z.string().nullable(), | |
| 35 | + sellerCountry: z.string().nullable(), | |
| 36 | + dimensions: z.string().nullable(), | |
| 37 | +}); | |
| 38 | +export type Item = z.infer<typeof ItemSchema>; | |
| 39 | + | |
| 40 | +export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), items: z.array(ItemSchema), snapshot: z.string().optional() }); | |
| 41 | +export type PagePayload = z.infer<typeof PagePayloadSchema>; | |
| 42 | + | |
| 43 | +const ConfigSchema = z.object({ seeds: z.array(SeedSchema).min(1), pagesPerSeed: z.number().int().min(1).default(2), seedsPerRun: z.number().int().min(1).default(4) }); | |
| 44 | + | |
| 45 | +const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null); | |
| 46 | +const dim = (v: unknown): string | null => { | |
| 47 | + if (!v || typeof v !== 'object') return null; | |
| 48 | + const r = v as { value?: unknown; unitText?: unknown; unitCode?: unknown }; | |
| 49 | + const val = r.value !== undefined ? String(r.value) : null; | |
| 50 | + return val ? `${val}${str(r.unitText) ?? (r.unitCode === 'INH' ? ' in' : '')}` : null; | |
| 51 | +}; | |
| 52 | + | |
| 53 | +/** JSON-LD Product[] → trimmed items. Exported for tests. */ | |
| 54 | +export function parseBrowseHtml(htmlText: string): Item[] { | |
| 55 | + const products = adapters.productsFromHtml(htmlText); | |
| 56 | + const out: Item[] = []; | |
| 57 | + for (const p of products) { | |
| 58 | + const url = p.url ?? ''; | |
| 59 | + const id = url.match(/\/product\/(\d+)\//)?.[1]; | |
| 60 | + if (!id || !p.name) continue; | |
| 61 | + const offer = p.offers[0]; | |
| 62 | + const rawOffer = (Array.isArray(p.raw.offers) ? p.raw.offers[0] : p.raw.offers) as Record<string, unknown> | undefined; | |
| 63 | + const seller = rawOffer?.seller as { address?: { addressLocality?: unknown; addressCountry?: { name?: unknown } | string } } | undefined; | |
| 64 | + const country = seller?.address?.addressCountry; | |
| 65 | + const dims = [dim(p.raw.width), dim(p.raw.depth), dim(p.raw.height)].filter(Boolean); | |
| 66 | + out.push({ | |
| 67 | + id, | |
| 68 | + url: url.startsWith('http') ? url : `${BASE}${url}`, | |
| 69 | + name: p.name, | |
| 70 | + description: plainText(p.description, 1500), | |
| 71 | + images: p.images.slice(0, 6), | |
| 72 | + brand: p.brand, | |
| 73 | + color: str(p.raw.color), | |
| 74 | + material: str(p.raw.material), | |
| 75 | + category: str(rawOffer?.category), | |
| 76 | + price: offer?.price ?? null, | |
| 77 | + currency: offer?.currency ?? null, | |
| 78 | + availability: offer?.availability ?? 'unknown', | |
| 79 | + condition: offer?.condition ?? null, | |
| 80 | + sellerCity: str(seller?.address?.addressLocality), | |
| 81 | + sellerCountry: typeof country === 'string' ? country : str(country?.name), | |
| 82 | + dimensions: dims.length === 3 ? `W ${dims[0]} × D ${dims[1]} × H ${dims[2]}` : null, | |
| 83 | + }); | |
| 84 | + } | |
| 85 | + return out; | |
| 86 | +} | |
| 87 | + | |
| 88 | +/** Keep only the JSON-LD scripts (first `n` products) for a compact fixture snapshot. */ | |
| 89 | +export function trimBrowseHtml(htmlText: string, n = 3): string { | |
| 90 | + const $ = H.load(htmlText); | |
| 91 | + const scripts = $('script[type="application/ld+json"]').toArray(); | |
| 92 | + const kept: string[] = []; | |
| 93 | + for (const s of scripts) { | |
| 94 | + const txt = $(s).contents().text(); | |
| 95 | + try { | |
| 96 | + const j = JSON.parse(txt) as unknown; | |
| 97 | + if (Array.isArray(j)) kept.push(JSON.stringify(j.slice(0, n))); | |
| 98 | + else kept.push(txt); | |
| 99 | + } catch { | |
| 100 | + /* skip */ | |
| 101 | + } | |
| 102 | + } | |
| 103 | + const next = $('link[rel="next"], a[rel="next"]').first().attr('href'); | |
| 104 | + return `<!doctype html><html><head><title>${$('title').text()}</title>${next ? `<link rel="next" href="${next}">` : ''}${kept.map((k) => `<script type="application/ld+json">${k}</script>`).join('')}</head><body></body></html>`; | |
| 105 | +} | |
| 106 | + | |
| 107 | +/** Chairish marks the following page with <link rel="next"> (and a[rel=next]); absent on the last page. */ | |
| 108 | +export function hasNextPage(htmlText: string): boolean { | |
| 109 | + const $ = H.load(htmlText); | |
| 110 | + return $('link[rel="next"], a[rel="next"]').length > 0; | |
| 111 | +} | |
| 112 | + | |
| 113 | +export function pageUrl(seedPath: string, page: number): string { | |
| 114 | + return `${BASE}${seedPath}${page > 1 ? `?page=${page}` : ''}`; | |
| 115 | +} | |
| 116 | + | |
| 117 | +function verticalFor(seed: Seed): DesignVertical { | |
| 118 | + if (seed.vertical) return seed.vertical; | |
| 119 | + const p = seed.path; | |
| 120 | + if (/lighting|lamps/.test(p)) return 'lighting'; | |
| 121 | + if (/\/art\b|paintings|prints|photograph/.test(p)) return 'art'; | |
| 122 | + if (/jewelry/.test(p)) return 'jewelry'; | |
| 123 | + if (/watches/.test(p)) return 'watches'; | |
| 124 | + if (/handbag|bags|wallets|fashion/.test(p)) return 'fashion'; | |
| 125 | + if (/tableware|barware|serveware/.test(p)) return 'tableware'; | |
| 126 | + if (/decor|mirrors|accents|vessels|statues/.test(p)) return 'decor'; | |
| 127 | + if (/rugs|textiles|pillows|wallpaper/.test(p)) return 'rugs'; | |
| 128 | + if (/furniture|seating|tables|casegoods|desks|beds|sofas|style\//.test(p)) return 'furniture'; | |
| 129 | + return 'unknown'; | |
| 130 | +} | |
| 131 | + | |
| 132 | +export class ChairishConnector extends BaseConnector { | |
| 133 | + readonly version = '1.0.0'; | |
| 134 | + readonly parserVersion = PARSER_VERSION; | |
| 135 | + protected override minIntervalMs = 3000; | |
| 136 | + override readonly urlPatterns = [/^https?:\/\/(www\.)?chairish\.com\/product\/(\d+)\//i]; | |
| 137 | + private readonly cfg: z.infer<typeof ConfigSchema>; | |
| 138 | + | |
| 139 | + constructor(meta: ConnectorMeta) { | |
| 140 | + super(meta); | |
| 141 | + this.cfg = ConfigSchema.parse(meta.config); | |
| 142 | + } | |
| 143 | + | |
| 144 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 145 | + const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds; | |
| 146 | + const backfill = ctx.options.mode === 'backfill'; | |
| 147 | + const maxPages = backfill ? this.policy.backfillMaxPages : this.cfg.pagesPerSeed; | |
| 148 | + const start = readSeedCursor(ctx.options.cursor, seeds.length); | |
| 149 | + const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun); | |
| 150 | + let count = 0; | |
| 151 | + let items = 0; | |
| 152 | + for (let k = 0; k < seedsThisRun; k++) { | |
| 153 | + const seedIndex = (start.seedIndex + k) % seeds.length; | |
| 154 | + const seed = seeds[seedIndex]!; | |
| 155 | + let page = k === 0 ? start.page : 1; | |
| 156 | + for (; page <= maxPages; page++) { | |
| 157 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 158 | + const url = pageUrl(seed.path, page); | |
| 159 | + await this.throttle(url); | |
| 160 | + const res = await ctx.fetch(url, { | |
| 161 | + engines: ['api'], | |
| 162 | + responseType: 'text', | |
| 163 | + expect: ['title', 'price', 'currency'], | |
| 164 | + parse: (r) => { | |
| 165 | + const list = r.html ? parseBrowseHtml(r.html) : []; | |
| 166 | + const priced = list.find((i) => i.price); | |
| 167 | + return list.length ? { title: list[0]!.name, price: priced?.price ?? null, currency: priced?.currency ?? null } : null; | |
| 168 | + }, | |
| 169 | + minQuality: 0.3, | |
| 170 | + }); | |
| 171 | + const list = res.success && res.html ? parseBrowseHtml(res.html) : null; | |
| 172 | + if (!list) { | |
| 173 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 174 | + break; | |
| 175 | + } | |
| 176 | + if (!list.length) { | |
| 177 | + if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no JSON-LD products`); | |
| 178 | + break; | |
| 179 | + } | |
| 180 | + count++; | |
| 181 | + items += list.length; | |
| 182 | + const payload: PagePayload = { kind: 'listing_page', url, seed, page, items: list }; | |
| 183 | + yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 184 | + await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() }); | |
| 185 | + await ctx.progress({ page, itemsProcessed: items }); | |
| 186 | + if (list.length < PAGE_SIZE || !hasNextPage(res.html!)) break; | |
| 187 | + } | |
| 188 | + const nextSeed = (seedIndex + 1) % seeds.length; | |
| 189 | + await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) }); | |
| 190 | + } | |
| 191 | + } | |
| 192 | + | |
| 193 | + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { | |
| 194 | + if (!this.urlPatterns[0]!.test(url)) return []; | |
| 195 | + await this.throttle(url); | |
| 196 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 }); | |
| 197 | + const list = res.success && res.html ? parseBrowseHtml(res.html) : []; | |
| 198 | + if (!list.length) return []; | |
| 199 | + const seed: Seed = { path: new URL(url).pathname, slug: null, vertical: 'unknown' }; | |
| 200 | + const payload: PagePayload = { kind: 'listing_page', url, seed, page: 1, items: list.slice(0, 1) }; | |
| 201 | + return [{ url, externalId: `product:${list[0]!.id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; | |
| 202 | + } | |
| 203 | + | |
| 204 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 205 | + const p = PagePayloadSchema.parse(raw.payload); | |
| 206 | + const vertical = verticalFor(p.seed); | |
| 207 | + const out: NormalizedRecord[] = []; | |
| 208 | + for (const it of p.items) { | |
| 209 | + const categorySlug = p.seed.slug ?? designSlug(it.category, it.name, vertical); | |
| 210 | + if (!categorySlug) continue; | |
| 211 | + const { year, decade } = yearOrDecade(it.name); | |
| 212 | + const brand = it.brand && !/^(unknown|unbranded|n\/a|none)$/i.test(it.brand) ? it.brand : makerFromTitle(it.name); | |
| 213 | + const attributes = AssetAttributesSchema.parse({ | |
| 214 | + categorySlug, | |
| 215 | + brand, | |
| 216 | + name: it.name, | |
| 217 | + year, | |
| 218 | + material: it.material, | |
| 219 | + color: it.color, | |
| 220 | + size: it.dimensions, | |
| 221 | + identifiers: { chairish_product_id: it.id }, | |
| 222 | + metadata: { source_category: it.category, decade, seller_location: [it.sellerCity, it.sellerCountry].filter(Boolean).join(', ') || null }, | |
| 223 | + }); | |
| 224 | + out.push( | |
| 225 | + NormalizedListingSchema.parse({ | |
| 226 | + kind: 'listing', | |
| 227 | + connectorId: this.meta.id, | |
| 228 | + sourceId: this.meta.sourceId, | |
| 229 | + sourceUrl: it.url, | |
| 230 | + externalId: it.id, | |
| 231 | + rawTitle: it.name, | |
| 232 | + description: it.description, | |
| 233 | + imageUrls: it.images, | |
| 234 | + attributes, | |
| 235 | + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, | |
| 236 | + condition: { condition: null, conditionRaw: it.condition, completeness: null }, | |
| 237 | + observedAt: raw.fetchedAt, | |
| 238 | + confidence: 0.8, | |
| 239 | + parserVersion: PARSER_VERSION, | |
| 240 | + listingType: 'fixed_price', | |
| 241 | + price: it.price, | |
| 242 | + currency: it.currency && /^[A-Z]{3}$/.test(it.currency) ? it.currency : it.price ? 'USD' : null, | |
| 243 | + seller: null, | |
| 244 | + location: [it.sellerCity, it.sellerCountry].filter(Boolean).join(', ') || null, | |
| 245 | + availability: it.availability, | |
| 246 | + }), | |
| 247 | + ); | |
| 248 | + } | |
| 249 | + return out; | |
| 250 | + } | |
| 251 | +} | |
| 252 | + | |
| 253 | +export default function createConnector(meta: ConnectorMeta): ChairishConnector { | |
| 254 | + return new ChairishConnector(meta); | |
| 255 | +} | |
added
connectors/api/chairish/meta.json
+46 −0
@@ -0,0 +1,46 @@ | ||
| 1 | +{ | |
| 2 | + "id": "chairish", | |
| 3 | + "displayName": "Chairish (vintage furniture, decor, art & jewelry listings)", | |
| 4 | + "sourceId": "chairish", | |
| 5 | + "sourceName": "Chairish", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://www.chairish.com", | |
| 8 | + "module": "api/chairish", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["design_furniture", "antiques", "porcelain", "glass_crystal", "silver", "clocks", "art", "contemporary_art", "photography", "jewelry", "other_watches", "luxury_handbags"], | |
| 11 | + "regions": ["US"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["USD"], | |
| 14 | + "supportsListings": true, | |
| 15 | + "supportsSold": false, | |
| 16 | + "supportsAuctions": false, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": true, | |
| 21 | + "refreshFrequencyMinutes": 720, | |
| 22 | + "priority": "medium", | |
| 23 | + "trustScore": 0.75, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.chairish.com/legal", | |
| 26 | + "acquisitionMethod": "public HTML — schema.org Product JSON-LD on browse pages", | |
| 27 | + "historicalDepth": "none", | |
| 28 | + "accessNotes": "Chairish browse pages (/collection/<slug> and /style/<slug>, ?page=N, 48 listings per page) are server-rendered and embed one schema.org Product per listing in a JSON-LD array: name, description, images, brand (designer/maker), color, material, width/depth/height, Offer {price, priceCurrency USD, availability, itemCondition, category path such as 'Furniture > Chairs > Chaises', seller address locality/country}. We read only that JSON-LD block — no search endpoint, no /product/list, /product/grid or /product/data XHRs (all disallowed by robots.txt), no account or cart paths. robots.txt (User-agent: * and named AI bots) disallows /search, /saved-search, /product/list|grid|data|id and account paths; /collection/ and /style/ are allowed. The terms-of-service URL is JS-only (help centre) and could not be reviewed automatically; no anti-bot challenge is served to the honest bot UA (200 OK). Asking prices only (listings, never sales); the seller's business location (city/country shown on every card) is kept as the listing location, no personal data. Category mapping: seed vertical + JSON-LD category + title keywords → design_furniture (default for furniture/lighting), antiques (period/antique evidence), porcelain, glass_crystal, silver, clocks, art/photography, jewelry, watch brands, luxury handbags; rugs/textiles/pillows/wallpaper are skipped. 3 s between requests, 1 in flight; 4 seeds × 2 pages per incremental run, seed rotation via cursor {seedIndex, page}.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "pagesPerSeed": 2, | |
| 33 | + "seedsPerRun": 4, | |
| 34 | + "seeds": [ | |
| 35 | + { "path": "/collection/vintage-furniture", "vertical": "furniture" }, | |
| 36 | + { "path": "/style/mid-century-modern", "vertical": "furniture" }, | |
| 37 | + { "path": "/collection/lighting", "vertical": "lighting" }, | |
| 38 | + { "path": "/collection/decor", "vertical": "decor" }, | |
| 39 | + { "path": "/collection/tableware-and-barware", "vertical": "tableware" }, | |
| 40 | + { "path": "/collection/art", "vertical": "art" }, | |
| 41 | + { "path": "/collection/jewelry", "vertical": "jewelry" }, | |
| 42 | + { "path": "/collection/watches", "vertical": "watches" }, | |
| 43 | + { "path": "/collection/handbags", "slug": "luxury_handbags", "vertical": "fashion" } | |
| 44 | + ] | |
| 45 | + } | |
| 46 | +} | |
added
connectors/api/chatterley-luxuries/README.md
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +# chatterley-luxuries — Chatterley Luxuries (pens & lighters dealer) | |
| 2 | + | |
| 3 | +Metadata-only connector on the framework `WooCommerceStoreConnector`: public Store API | |
| 4 | +`/wp-json/wc/store/v1/products?category=<id>&per_page=100&page=N`, categories resolved by slug. | |
| 5 | +Collections `pens`, `vintage`, `consignments` → `pens`; `lighters` (+ title rule) → `lighters`; ink, | |
| 6 | +leather, nibs and accessories excluded. USD asking prices → listings with `sku`. | |
| 7 | + | |
| 8 | +Host policy (connectors/domains.d/g10-global-apis-misc.json): `userAgent: browser` because the bot UA | |
| 9 | +receives a query-stripping 301 (see meta.json accessNotes); Crawl-delay 10 s honoured. | |
added
connectors/api/chatterley-luxuries/index.test.ts
+32 −0
@@ -0,0 +1,32 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 3 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 4 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const connector = createConnector(localMeta(metaJson)); | |
| 8 | + | |
| 9 | +describe('chatterley-luxuries', () => { | |
| 10 | + runFixtureSuite(connector, it, expect); | |
| 11 | + | |
| 12 | + it('normalises WooCommerce Store API products into USD pen listings with SKU', async () => { | |
| 13 | + const fx = loadFixture('chatterley-luxuries', 'pens-p1'); | |
| 14 | + const out = await connector.normalize(fx.raw); | |
| 15 | + expect(out.length).toBe(1); | |
| 16 | + const r = out[0]!; | |
| 17 | + if (r.kind !== 'listing') throw new Error('expected listing'); | |
| 18 | + expect(r.currency).toBe('USD'); | |
| 19 | + expect(r.price).toBeGreaterThan(0); | |
| 20 | + expect(r.attributes.categorySlug).toBe('pens'); | |
| 21 | + expect(r.attributes.identifiers.sku).toBeTruthy(); | |
| 22 | + expect(r.seller).toBe('Chatterley Luxuries'); | |
| 23 | + expect(r.sourceUrl).toMatch(/^https:\/\/chatterleyluxuries\.com\/product\//); | |
| 24 | + }); | |
| 25 | + | |
| 26 | + it('maps lighters through the category and title rules', async () => { | |
| 27 | + const fx = loadFixture('chatterley-luxuries', 'lighters-p1'); | |
| 28 | + const out = await connector.normalize(fx.raw); | |
| 29 | + expect(out.length).toBe(1); | |
| 30 | + expect((out[0] as { attributes: { categorySlug: string } }).attributes.categorySlug).toBe('lighters'); | |
| 31 | + }); | |
| 32 | +}); | |
added
connectors/api/chatterley-luxuries/index.ts
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +import { WooCommerceStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Chatterley Luxuries (US) — luxury and limited-edition fountain pens, pen lighters and S.T. Dupont. | |
| 5 | + * WooCommerce shop: the public Store API (/wp-json/wc/store/v1/products?category=<id>&per_page=100&page=N) | |
| 6 | + * is read through the framework adapter; collection → taxonomy mapping lives in meta.json config. | |
| 7 | + * Asking prices only (listings). See meta.json accessNotes for the UA/redirect observation. | |
| 8 | + */ | |
| 9 | +export class ChatterleyLuxuriesConnector extends WooCommerceStoreConnector { | |
| 10 | + override readonly version = '1.0.0'; | |
| 11 | + override readonly parserVersion = '1.0.0'; | |
| 12 | + protected override minIntervalMs = 10_000; | |
| 13 | +} | |
| 14 | + | |
| 15 | +export default function createConnector(meta: ConnectorMeta): ChatterleyLuxuriesConnector { | |
| 16 | + return new ChatterleyLuxuriesConnector(meta); | |
| 17 | +} | |
added
connectors/api/chatterley-luxuries/meta.json
+49 −0
@@ -0,0 +1,49 @@ | ||
| 1 | +{ | |
| 2 | + "id": "chatterley-luxuries", | |
| 3 | + "displayName": "Chatterley Luxuries (luxury pens & lighters dealer)", | |
| 4 | + "sourceId": "chatterley-luxuries", | |
| 5 | + "sourceName": "Chatterley Luxuries", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://chatterleyluxuries.com", | |
| 8 | + "module": "api/chatterley-luxuries", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["pens", "lighters"], | |
| 11 | + "regions": ["US"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["USD"], | |
| 14 | + "supportsListings": true, | |
| 15 | + "supportsSold": false, | |
| 16 | + "supportsAuctions": false, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "low", | |
| 23 | + "trustScore": 0.75, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://chatterleyluxuries.com/privacy-policy/", | |
| 26 | + "acquisitionMethod": "WooCommerce Store API (public JSON)", | |
| 27 | + "historicalDepth": "none", | |
| 28 | + "accessNotes": "Chatterley Luxuries is a WooCommerce shop (limited-edition and luxury fountain pens: Montblanc, Pelikan, Pilot/Namiki maki-e, Visconti, Montegrappa, Omas, Scribo, S.T. Dupont pens and lighters, consignments). We read the public, unauthenticated WooCommerce Blocks Store API — /wp-json/wc/store/v1/products/categories and /wp-json/wc/store/v1/products?category=<id>&per_page=100&page=N — exactly what the shop's own front end calls; product pages, cart, account and admin-ajax are never fetched. robots.txt (User-agent: *) allows /wp-json/ and asks for Crawl-delay: 10, which we honour (10 s between requests, one connection). Observation 2026-09-08: with the RareIndexBot UA the server answers the Store API with a 301 to the plain-http URL and drops the query string (pagination impossible), while a standard browser UA receives 200 JSON on HTTPS — the site serves bots a degraded response; connectors/domains.d sets userAgent=browser for this host, the request volume stays tiny (a few pages per day) and the privacy policy (the only legal page published) has no clause on automated access. Asking prices in USD (prices.currency_code, minor units) → listings; only the pens, vintage, consignments and lighters categories are crawled (the 6,900-item 'sold-out editions' archive is intentionally skipped — those are historical asking prices, not sales); ink, leather goods, nibs and accessories are excluded by rule. Category ids resolved by slug at run time.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "currency": "USD", | |
| 33 | + "perPage": 100, | |
| 34 | + "seller": "Chatterley Luxuries", | |
| 35 | + "location": "United States", | |
| 36 | + "keepOutOfStock": true, | |
| 37 | + "exclude": "gift card|\\bink\\b|inks\\b|inkwell|ink well|leather|notebook|paper|\\bnibs?\\b|nib unit|converter|refill|pouch|pen case|pen roll|accessor|bottle|cartridge", | |
| 38 | + "collections": [ | |
| 39 | + { "handle": "pens", "categorySlug": "pens", "pages": 3 }, | |
| 40 | + { "handle": "vintage", "categorySlug": "pens", "pages": 1 }, | |
| 41 | + { "handle": "consignments", "categorySlug": "pens", "pages": 2 }, | |
| 42 | + { "handle": "lighters", "categorySlug": "lighters", "pages": 1 } | |
| 43 | + ], | |
| 44 | + "rules": [ | |
| 45 | + { "match": "\\blighter\\b", "categorySlug": "lighters" }, | |
| 46 | + { "match": "fountain pen|rollerball|ballpoint|mechanical pencil|pen set", "categorySlug": "pens" } | |
| 47 | + ] | |
| 48 | + } | |
| 49 | +} | |
added
connectors/api/cherrycollectables/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Cherry Collectables connector (`cherrycollectables`) | |
| 2 | + | |
| 3 | +- Source: https://www.cherrycollectables.com.au · Australia's largest card shop (Melbourne; official Panini NBA distributor): ~390k singles across NBA/NFL/soccer/MLB/NHL/AFL/UFC/F1 sports cards, Pokémon (incl. PSA/CGC/BGS graded slabs), Yu-Gi-Oh!, MTG, One Piece, Digimon, Flesh and Blood, plus sealed boxes. Group breaks are excluded. | |
| 4 | +- Country/currency: AU / AUD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (21): `pokemon-singles` → pokemon, `graded-pkm` → pokemon, `psa-10` → pokemon, `pokemon-booster-boxes` → pokemon, `magic-the-gathering-singles` → magic_the_gathering, `ygo-singles` → yugioh, `one-piece-singles` → one_piece_card_game, `digimon-singles` → digimon_tcg, `flesh-and-blood-singles` → flesh_and_blood, `lorcana` → disney_lorcana, `nba-singles` → basketball_cards, `graded-nba-cards` → basketball_cards, `nfl-singles` → football_cards, `soccer-singles` → soccer_cards, `mlb-singles` → baseball_cards, `nhl-singles` → hockey_cards, `all-f1` → f1_cards, `afl-singles` → other_sports_cards, `ufc-singles` → other_sports_cards, `wrestling-singles` → other_sports_cards, `marvel-singles` → non_sport_cards. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/cherrycollectables/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts cherrycollectables`. | |
added
connectors/api/cherrycollectables/index.test.ts
+88 −0
@@ -0,0 +1,88 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('cherrycollectables', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "PSA 10 Kai'sa - Daughter of the Void 299* - Origins Signature Overnumber 424", | |
| 18 | + "collection": "psa-10", | |
| 19 | + "type": "Singles", | |
| 20 | + "tags": [ | |
| 21 | + "Condition_Graded", | |
| 22 | + "grade 10", | |
| 23 | + "Grader_PSA" | |
| 24 | + ], | |
| 25 | + "expect": "pokemon", | |
| 26 | + "grader": "psa", | |
| 27 | + "grade": "10" | |
| 28 | + }, | |
| 29 | + { | |
| 30 | + "title": "JAPANESE CGC 8 Machoke - 177/165 sv2a - Art Rare Pokemon 151 (189)", | |
| 31 | + "collection": "pokemon-singles", | |
| 32 | + "type": "Singles", | |
| 33 | + "tags": [ | |
| 34 | + "CGC 8", | |
| 35 | + "japanese-pokemon", | |
| 36 | + "pokemon singles" | |
| 37 | + ], | |
| 38 | + "expect": "pokemon", | |
| 39 | + "grader": "cgc", | |
| 40 | + "grade": "8" | |
| 41 | + }, | |
| 42 | + { | |
| 43 | + "title": "2025 Donruss WNBA RICKEA JACKSON Net Marvels Orange Fireworks 46/75", | |
| 44 | + "collection": "nba-singles", | |
| 45 | + "type": "Singles", | |
| 46 | + "tags": [ | |
| 47 | + "Collection_Donruss", | |
| 48 | + "Sport_NBA" | |
| 49 | + ], | |
| 50 | + "expect": "basketball_cards", | |
| 51 | + "year": 2025 | |
| 52 | + }, | |
| 53 | + { | |
| 54 | + "title": "2026 Topps Chrome Premier League EPL ERLING HAALAND Hidden Gems SSP #3", | |
| 55 | + "collection": "soccer-singles", | |
| 56 | + "type": "Singles", | |
| 57 | + "tags": [ | |
| 58 | + "Sport_Soccer" | |
| 59 | + ], | |
| 60 | + "expect": "soccer_cards" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "title": "[FOIL] Mountain (Chocobo Track Foil) #481 - LAND FIC - Commander: FINAL FANTASY", | |
| 64 | + "collection": "magic-the-gathering-singles", | |
| 65 | + "type": "Singles", | |
| 66 | + "tags": [ | |
| 67 | + "Brand_Magic the Gathering", | |
| 68 | + "foil" | |
| 69 | + ], | |
| 70 | + "variant": "Near Mint", | |
| 71 | + "expect": "magic_the_gathering", | |
| 72 | + "conditionRaw": "Near Mint" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "title": "Derby・Daily MLB Team Based Opening - #33362 (Sep 09 4pm)", | |
| 76 | + "collection": "mlb-singles", | |
| 77 | + "type": "Breaks", | |
| 78 | + "expect": null | |
| 79 | + }, | |
| 80 | + { | |
| 81 | + "title": "2024 Prizm Basketball Hobby Box Break - Random Team Spot", | |
| 82 | + "collection": "nba-singles", | |
| 83 | + "type": "Breaks", | |
| 84 | + "expect": null | |
| 85 | + } | |
| 86 | + ]); | |
| 87 | + }); | |
| 88 | +}); | |
added
connectors/api/cherrycollectables/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Cherry Collectables — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/cherrycollectables/meta.json
+235 −0
@@ -0,0 +1,235 @@ | ||
| 1 | +{ | |
| 2 | + "id": "cherrycollectables", | |
| 3 | + "displayName": "Cherry Collectables", | |
| 4 | + "sourceId": "cherrycollectables", | |
| 5 | + "sourceName": "Cherry Collectables", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.cherrycollectables.com.au", | |
| 8 | + "module": "api/cherrycollectables", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "pokemon", | |
| 14 | + "magic_the_gathering", | |
| 15 | + "yugioh", | |
| 16 | + "one_piece_card_game", | |
| 17 | + "digimon_tcg", | |
| 18 | + "flesh_and_blood", | |
| 19 | + "disney_lorcana", | |
| 20 | + "basketball_cards", | |
| 21 | + "football_cards", | |
| 22 | + "soccer_cards", | |
| 23 | + "baseball_cards", | |
| 24 | + "hockey_cards", | |
| 25 | + "f1_cards", | |
| 26 | + "other_sports_cards", | |
| 27 | + "non_sport_cards", | |
| 28 | + "star_wars_tcg", | |
| 29 | + "dragon_ball_tcg", | |
| 30 | + "weiss_schwarz", | |
| 31 | + "final_fantasy_tcg", | |
| 32 | + "other_tcg" | |
| 33 | + ], | |
| 34 | + "regions": [ | |
| 35 | + "AU" | |
| 36 | + ], | |
| 37 | + "country": "AU", | |
| 38 | + "languages": [ | |
| 39 | + "en" | |
| 40 | + ], | |
| 41 | + "currency": [ | |
| 42 | + "AUD" | |
| 43 | + ], | |
| 44 | + "supportsListings": true, | |
| 45 | + "supportsSold": false, | |
| 46 | + "supportsAuctions": false, | |
| 47 | + "supportsImages": true, | |
| 48 | + "supportsCatalog": false, | |
| 49 | + "supportsPopulation": false, | |
| 50 | + "supportsLookup": true, | |
| 51 | + "refreshFrequencyMinutes": 720, | |
| 52 | + "priority": "medium", | |
| 53 | + "trustScore": 0.7, | |
| 54 | + "attributionRequired": true, | |
| 55 | + "termsUrl": "https://www.cherrycollectables.com.au/policies/terms-of-service", | |
| 56 | + "accessNotes": "Cherry Collectables (cherrycollectables.com.au) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (pokemon-singles, graded-pkm, psa-10, pokemon-booster-boxes, magic-the-gathering-singles, ygo-singles, one-piece-singles, digimon-singles, flesh-and-blood-singles, lorcana, nba-singles, graded-nba-cards, nfl-singles, soccer-singles, mlb-singles, nhl-singles, all-f1, afl-singles, ufc-singles, wrestling-singles, marvel-singles). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency AUD (/meta.json currency AUD, Shopify.currency rate 1.0), GST included. Graded slabs carry the grader and grade in the title (parsed by parseGradeFromTitle) and in tags (Grader_PSA, grade 10). Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 57 | + "enabled": true, | |
| 58 | + "schemaVersion": "1.0", | |
| 59 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 60 | + "historicalDepth": "none", | |
| 61 | + "requires": [], | |
| 62 | + "config": { | |
| 63 | + "currency": "AUD", | |
| 64 | + "seller": "Cherry Collectables", | |
| 65 | + "location": null, | |
| 66 | + "collections": [ | |
| 67 | + { | |
| 68 | + "handle": "pokemon-singles", | |
| 69 | + "categorySlug": "pokemon", | |
| 70 | + "franchise": "Pokémon" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "graded-pkm", | |
| 74 | + "categorySlug": "pokemon", | |
| 75 | + "franchise": "Pokémon" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "handle": "psa-10", | |
| 79 | + "categorySlug": "pokemon", | |
| 80 | + "franchise": "Pokémon" | |
| 81 | + }, | |
| 82 | + { | |
| 83 | + "handle": "pokemon-booster-boxes", | |
| 84 | + "categorySlug": "pokemon", | |
| 85 | + "franchise": "Pokémon" | |
| 86 | + }, | |
| 87 | + { | |
| 88 | + "handle": "magic-the-gathering-singles", | |
| 89 | + "categorySlug": "magic_the_gathering", | |
| 90 | + "franchise": "Magic: The Gathering" | |
| 91 | + }, | |
| 92 | + { | |
| 93 | + "handle": "ygo-singles", | |
| 94 | + "categorySlug": "yugioh", | |
| 95 | + "franchise": "Yu-Gi-Oh!" | |
| 96 | + }, | |
| 97 | + { | |
| 98 | + "handle": "one-piece-singles", | |
| 99 | + "categorySlug": "one_piece_card_game", | |
| 100 | + "franchise": "One Piece" | |
| 101 | + }, | |
| 102 | + { | |
| 103 | + "handle": "digimon-singles", | |
| 104 | + "categorySlug": "digimon_tcg", | |
| 105 | + "franchise": "Digimon" | |
| 106 | + }, | |
| 107 | + { | |
| 108 | + "handle": "flesh-and-blood-singles", | |
| 109 | + "categorySlug": "flesh_and_blood", | |
| 110 | + "franchise": "Flesh and Blood" | |
| 111 | + }, | |
| 112 | + { | |
| 113 | + "handle": "lorcana", | |
| 114 | + "categorySlug": "disney_lorcana", | |
| 115 | + "franchise": "Disney Lorcana" | |
| 116 | + }, | |
| 117 | + { | |
| 118 | + "handle": "nba-singles", | |
| 119 | + "categorySlug": "basketball_cards" | |
| 120 | + }, | |
| 121 | + { | |
| 122 | + "handle": "graded-nba-cards", | |
| 123 | + "categorySlug": "basketball_cards" | |
| 124 | + }, | |
| 125 | + { | |
| 126 | + "handle": "nfl-singles", | |
| 127 | + "categorySlug": "football_cards" | |
| 128 | + }, | |
| 129 | + { | |
| 130 | + "handle": "soccer-singles", | |
| 131 | + "categorySlug": "soccer_cards" | |
| 132 | + }, | |
| 133 | + { | |
| 134 | + "handle": "mlb-singles", | |
| 135 | + "categorySlug": "baseball_cards" | |
| 136 | + }, | |
| 137 | + { | |
| 138 | + "handle": "nhl-singles", | |
| 139 | + "categorySlug": "hockey_cards" | |
| 140 | + }, | |
| 141 | + { | |
| 142 | + "handle": "all-f1", | |
| 143 | + "categorySlug": "f1_cards" | |
| 144 | + }, | |
| 145 | + { | |
| 146 | + "handle": "afl-singles", | |
| 147 | + "categorySlug": "other_sports_cards" | |
| 148 | + }, | |
| 149 | + { | |
| 150 | + "handle": "ufc-singles", | |
| 151 | + "categorySlug": "other_sports_cards" | |
| 152 | + }, | |
| 153 | + { | |
| 154 | + "handle": "wrestling-singles", | |
| 155 | + "categorySlug": "other_sports_cards" | |
| 156 | + }, | |
| 157 | + { | |
| 158 | + "handle": "marvel-singles", | |
| 159 | + "categorySlug": "non_sport_cards", | |
| 160 | + "franchise": "Marvel" | |
| 161 | + } | |
| 162 | + ], | |
| 163 | + "rules": [ | |
| 164 | + { | |
| 165 | + "match": "psa-10|graded-pkm", | |
| 166 | + "categorySlug": "pokemon", | |
| 167 | + "franchise": "Pokémon" | |
| 168 | + }, | |
| 169 | + { | |
| 170 | + "match": "star wars:? unlimited", | |
| 171 | + "categorySlug": "star_wars_tcg", | |
| 172 | + "franchise": "Star Wars" | |
| 173 | + }, | |
| 174 | + { | |
| 175 | + "match": "lorcana", | |
| 176 | + "categorySlug": "disney_lorcana", | |
| 177 | + "franchise": "Disney Lorcana" | |
| 178 | + }, | |
| 179 | + { | |
| 180 | + "match": "one piece (card|tcg|ccg|single|promo|sealed|booster|starter)|\\bop\\d{2}\\b|one-piece-(single|tcg|card)|\\| one piece single", | |
| 181 | + "categorySlug": "one_piece_card_game", | |
| 182 | + "franchise": "One Piece" | |
| 183 | + }, | |
| 184 | + { | |
| 185 | + "match": "dragon ?ball (super |z )?(card|tcg|ccg|fusion world)|fusion world|dragonball-super|dragon-ball-super", | |
| 186 | + "categorySlug": "dragon_ball_tcg", | |
| 187 | + "franchise": "Dragon Ball" | |
| 188 | + }, | |
| 189 | + { | |
| 190 | + "match": "digimon", | |
| 191 | + "categorySlug": "digimon_tcg", | |
| 192 | + "franchise": "Digimon" | |
| 193 | + }, | |
| 194 | + { | |
| 195 | + "match": "flesh (and|&) blood|\\bfab\\b", | |
| 196 | + "categorySlug": "flesh_and_blood", | |
| 197 | + "franchise": "Flesh and Blood" | |
| 198 | + }, | |
| 199 | + { | |
| 200 | + "match": "wei(ss|ß) schwarz", | |
| 201 | + "categorySlug": "weiss_schwarz" | |
| 202 | + }, | |
| 203 | + { | |
| 204 | + "match": "final fantasy (tcg|trading card|opus)|\\bopus (i{1,3}|iv|v|vi{0,3}|ix|x{1,2}|xi{1,3}|xiv|xv)\\b|\\bfftcg\\b", | |
| 205 | + "categorySlug": "final_fantasy_tcg", | |
| 206 | + "franchise": "Final Fantasy" | |
| 207 | + }, | |
| 208 | + { | |
| 209 | + "match": "yu-?gi-?oh|\\bygo\\b|yugioh", | |
| 210 | + "categorySlug": "yugioh", | |
| 211 | + "franchise": "Yu-Gi-Oh!" | |
| 212 | + }, | |
| 213 | + { | |
| 214 | + "match": "pok[eé]mon|\\bpkm\\b", | |
| 215 | + "categorySlug": "pokemon", | |
| 216 | + "franchise": "Pokémon" | |
| 217 | + }, | |
| 218 | + { | |
| 219 | + "match": "cardfight|vanguard|battle spirits|union arena|riftbound|grand archive|gundam (card|tcg)|hololive (official )?card|shadowverse|sorcery:? contested|\\baltered\\b|elestrals|alpha clash|universus|my hero academia (ccg|tcg|single)|akora|kryptik|cyberpunk tcg|neuroscape|beyblade x (tcg|card)", | |
| 220 | + "categorySlug": "other_tcg" | |
| 221 | + }, | |
| 222 | + { | |
| 223 | + "match": "magic:? the gathering|\\bmtg\\b|magic the gathering singles", | |
| 224 | + "categorySlug": "magic_the_gathering", | |
| 225 | + "franchise": "Magic: The Gathering" | |
| 226 | + } | |
| 227 | + ], | |
| 228 | + "defaultCategory": null, | |
| 229 | + "exclude": "gift card|\\bsleeves?\\b|deck box|deck case|deck holder|binder|playmat|play mat|toploader|top loader|storage box|card case|portfolio|\\balbum\\b|\\bdice\\b|spindown|life counter|tokens? only|bundle of|mystery (box|pack|bundle|bag)|repack|\\bticket|event entry|entry fee|weekly league|display case|acrylic (stand|display|case)|card holder|card protector|magnetic holder|semi-rigid|perfect fit|penny sleeve|team bag|card saver|\\baccessor|dice set|d20|d6\\b|\\bpaint|brush|primer|glue|hobby tool|cutter|tweezers|sticker|keyring|key ring|lanyard|badge|coaster|mug\\b|t-shirt|hoodie|\\bcap\\b|socks|towel|blanket|poster|wall scroll|calendar|advent|\\bbreak\\b|breaks\\b|opening\\b|\\bspot\\b|random team|\\bslot\\b|filler|group break|personal break|\\bpyt\\b|ebay\\b", | |
| 230 | + "keepOutOfStock": true, | |
| 231 | + "pageSize": 250, | |
| 232 | + "fetchBarcodes": false, | |
| 233 | + "wholeShop": false | |
| 234 | + } | |
| 235 | +} | |
added
connectors/api/chiswick-auctions/README.md
+8 −0
@@ -0,0 +1,8 @@ | ||
| 1 | +# chiswick-auctions — Chiswick Auctions (London) results | |
| 2 | + | |
| 3 | +goauction platform, same structure as Sworders (/results/ → /auction/details/<slug>?au=<id>&pn=N&g=1). | |
| 4 | + | |
| 5 | +- Built on `_g8-auctions-eu-apac-lib/sale-results.ts` (shared crawl/normalize/backfill): `sale` for lots with a | |
| 6 | + published result, `auction_lot` (ended) for unsold lots; native currency; premium basis as labelled by the source | |
| 7 | + (see meta.json accessNotes). Identifiers: `<house>_lot = <sale id>/<lot number>`. | |
| 8 | +- Fixtures: `pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts chiswick-auctions --save`. | |
added
connectors/api/chiswick-auctions/index.test.ts
+38 −0
@@ -0,0 +1,38 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 3 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 4 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | +import { parseGoauctionCalendar } from '../_g8-auctions-eu-apac-lib/goauction-platform.js'; | |
| 7 | + | |
| 8 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 9 | +const connector = createConnector(meta); | |
| 10 | + | |
| 11 | +const CAL = `<div class="auction-calendar-item calendar-third "> <a href='https://chiswickauctions.co.uk/auction/details/4sept26-affordable-jewellery-watches-silver--coins---timed-online?au=1385'target='blank'> <div class="auction-calendar-image" style="background-image: url("x");"> </div> </a> <div class="auction-calendar-text "> <a href='https://chiswickauctions.co.uk/auction/details/4sept26-affordable-jewellery-watches-silver--coins---timed-online?au=1385'target='blank'> <H6>Affordable Jewellery, Watches, Silver & Coins - Timed Online</H6> </a> <div><h5>Friday 4 September 2026</h5></div></div></div>`; | |
| 12 | + | |
| 13 | +describe('chiswick-auctions', () => { | |
| 14 | + runFixtureSuite(connector, it, expect); | |
| 15 | + | |
| 16 | + it('fixtures: GBP results from the goauction grid', async () => { | |
| 17 | + let sales = 0; | |
| 18 | + for (const name of listFixtures('chiswick-auctions')) { | |
| 19 | + for (const r of await connector.normalize(loadFixture('chiswick-auctions', name).raw)) { | |
| 20 | + if (!('attributes' in r)) continue; | |
| 21 | + expect(r.attributes.identifiers.chiswick_lot).toMatch(/^\d+\/\S+$/); | |
| 22 | + if (r.kind === 'sale') { | |
| 23 | + sales++; | |
| 24 | + expect(r.currency).toBe('GBP'); | |
| 25 | + expect(r.auctionHouse).toBe('Chiswick Auctions'); | |
| 26 | + expect(r.buyerPremiumIncluded).toBeNull(); | |
| 27 | + } | |
| 28 | + } | |
| 29 | + } | |
| 30 | + expect(sales).toBeGreaterThan(10); | |
| 31 | + }); | |
| 32 | + | |
| 33 | + it('parses the calendar with absolute (non-www) links and H6 titles', () => { | |
| 34 | + const sales = parseGoauctionCalendar(CAL, { base: 'https://www.chiswickauctions.co.uk', currency: 'GBP', listStyle: 'details' }); | |
| 35 | + expect(sales).toHaveLength(1); | |
| 36 | + expect(sales[0]).toMatchObject({ id: '1385', title: 'Affordable Jewellery, Watches, Silver & Coins - Timed Online', date: '2026-09-04T00:00:00.000Z', url: 'https://chiswickauctions.co.uk/auction/details/4sept26-affordable-jewellery-watches-silver--coins---timed-online?au=1385' }); | |
| 37 | + }); | |
| 38 | +}); | |
added
connectors/api/chiswick-auctions/index.ts
+40 −0
@@ -0,0 +1,40 @@ | ||
| 1 | +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors'; | |
| 2 | +import type { ExtractionResult } from '@rareindex/shared'; | |
| 3 | +import { goauctionPageUrl, parseGoauctionCalendar, parseGoauctionLots, type GoauctionHouse } from '../_g8-auctions-eu-apac-lib/goauction-platform.js'; | |
| 4 | +import { SaleResultsConnector, type HouseConfig, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Chiswick Auctions (London) — goauction platform, same structure as Sworders: /results/ calendar → | |
| 8 | + * /auction/details/<slug>?au=<id>&pn=N&g=1 lot grid with "Sold for £…". | |
| 9 | + */ | |
| 10 | +const HOUSE: GoauctionHouse = { base: 'https://www.chiswickauctions.co.uk', currency: 'GBP', listStyle: 'details' }; | |
| 11 | + | |
| 12 | +export class ChiswickConnector extends SaleResultsConnector { | |
| 13 | + readonly version = '1.0.0'; | |
| 14 | + readonly house: HouseConfig = { houseName: 'Chiswick Auctions', defaultCurrency: 'GBP', location: 'London, United Kingdom', idKey: 'chiswick_lot', premiumIncluded: null, fallbackSlug: 'antiques', minIntervalMs: 2500, maxPagesPerSale: 20 }; | |
| 15 | + protected override minIntervalMs = 2500; | |
| 16 | + | |
| 17 | + async listSales(ctx: CrawlContext): Promise<SaleRef[]> { | |
| 18 | + const url = String(this.meta.config.resultsUrl ?? `${HOUSE.base}/results/`); | |
| 19 | + await this.throttle(url); | |
| 20 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 21 | + if (!res.success || !res.html) { | |
| 22 | + ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 23 | + return []; | |
| 24 | + } | |
| 25 | + // the calendar links to chiswickauctions.co.uk without www → normalise to the canonical host | |
| 26 | + return parseGoauctionCalendar(res.html, HOUSE).map((s) => ({ ...s, url: s.url.replace('https://chiswickauctions.co.uk', HOUSE.base) })); | |
| 27 | + } | |
| 28 | + | |
| 29 | + salePageUrl(sale: SaleRef, page: number): string { | |
| 30 | + return goauctionPageUrl(sale, page, HOUSE); | |
| 31 | + } | |
| 32 | + | |
| 33 | + parseSalePage(res: ExtractionResult, sale: SaleRef): ParsedSalePage | null { | |
| 34 | + return res.html ? parseGoauctionLots(res.html, sale, HOUSE) : null; | |
| 35 | + } | |
| 36 | +} | |
| 37 | + | |
| 38 | +export default function createConnector(meta: ConnectorMeta) { | |
| 39 | + return new ChiswickConnector(meta); | |
| 40 | +} | |
added
connectors/api/chiswick-auctions/meta.json
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +{ | |
| 2 | + "id": "chiswick-auctions", | |
| 3 | + "displayName": "Chiswick Auctions (London) — results", | |
| 4 | + "sourceId": "chiswick-auctions", | |
| 5 | + "sourceName": "Chiswick Auctions", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.chiswickauctions.co.uk", | |
| 8 | + "module": "api/chiswick-auctions", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["jewelry", "other_watches", "rolex", "omega", "silver", "coins", "antiques", "art", "contemporary_art", "photography", "design_furniture", "porcelain", "glass_crystal", "books", "maps", "wine", "whisky", "luxury_handbags", "fashion_streetwear", "vintage_toys", "cameras", "movie_posters", "music_memorabilia"], | |
| 11 | + "regions": ["GB"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["GBP"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "low", | |
| 23 | + "trustScore": 0.85, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.chiswickauctions.co.uk/terms-and-conditions/", | |
| 26 | + "acquisitionMethod": "server-rendered HTML (goauction platform)", | |
| 27 | + "historicalDepth": "years", | |
| 28 | + "accessNotes": "Chiswick Auctions (London; jewellery, watches, silver, Asian art, photographs, design, wine) runs on the goauction platform shared with Sworders. Public pages read: /results/ (calendar of past sales with title, date, au id) and /auction/details/<slug>?au=<id>&pn=N&g=1 (lot grid: lot number, title, image, 'Sold for £…'; unsold lots carry no price). The grid does not state hammer vs premium → buyer_premium_included null. robots.txt disallows /account, /admin, /cms/lotdetailspdf, /elmah, /imagebrowser, /language only. 2.5 s politeness; salesPerRun caps incremental runs; resumable backfill over the calendar. No login, no bidder data.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "resultsUrl": "https://www.chiswickauctions.co.uk/results/", | |
| 33 | + "salesPerRun": 2 | |
| 34 | + } | |
| 35 | +} | |
added
connectors/api/chronext/README.md
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# chronext — CHRONEXT (certified pre-owned watches, DE) | |
| 2 | + | |
| 3 | +Dealer asking prices from CHRONEXT's server-rendered category pages. | |
| 4 | + | |
| 5 | +- Seeds are category paths (`/rolex/submariner`, `/omega/speedmaster`, `/patek-philippe/nautilus`, …, all verified 200). Each page (~3.7 MB) holds 24 `.product-tile` elements: brand, model, reference, displayed price (`USD 4,380` from North America, EUR from the EU — parsed with `parsePrice`, stored in the shown currency), condition label, image, `/brand/model/reference/V<id>` URL → `listing`. | |
| 6 | +- Pagination follows the page's own `…[offset]=N` links (`pagesPerSeed`, backfill bounded by `backfillMaxPages`). | |
| 7 | +- `lookup()` reads the product page's schema.org Product (sku `A…`, mpn = reference, brand, offer price/currency/availability/condition). | |
| 8 | +- Identity: `identifiers.chronext_id` (V-code), `chronext_sku`, `reference`. | |
| 9 | + | |
| 10 | +Smoke: `pnpm tsx connectors/api/_g9-asia-watch-sneaker-lib/capture.ts chronext --limit 1 --seeds /rolex/submariner`. | |
added
connectors/api/chronext/index.test.ts
+72 −0
@@ -0,0 +1,72 @@ | ||
| 1 | +import { readFileSync } from 'node:fs'; | |
| 2 | +import path from 'node:path'; | |
| 3 | +import { describe, expect, it } from 'vitest'; | |
| 4 | +import { getConnectorMeta } from '@rareindex/connectors'; | |
| 5 | +import { fixtureDir, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import createConnector, { parseCategoryPage, parseProductPage } from './index.js'; | |
| 7 | + | |
| 8 | +const connector = createConnector(getConnectorMeta('chronext')); | |
| 9 | +const sample = (name: string) => readFileSync(path.join(fixtureDir('chronext'), name), 'utf8'); | |
| 10 | + | |
| 11 | +describe('chronext', () => { | |
| 12 | + runFixtureSuite(connector, it, expect); | |
| 13 | + | |
| 14 | + it('parses product tiles (brand, model, reference, price text, condition, image, V-id) and the next-page offset link', async () => { | |
| 15 | + const p = parseCategoryPage(sample('category-tiles.sample.html'), 'https://www.chronext.com/rolex/datejust-pre-owned', '/rolex/datejust-pre-owned', 1); | |
| 16 | + expect(p.tiles).toHaveLength(3); | |
| 17 | + const t = p.tiles[0]!; | |
| 18 | + expect(t.id).toBe('V01800358'); | |
| 19 | + expect(t.brand).toBe('Rolex'); | |
| 20 | + expect(t.model).toBe('Lady-Datejust'); | |
| 21 | + expect(t.reference).toBe('79160'); | |
| 22 | + expect(t.priceText).toBe('USD 4,380'); | |
| 23 | + expect(t.condition).toBe('Like New'); | |
| 24 | + expect(t.href).toBe('https://www.chronext.com/rolex/lady-datejust/79160/V01800358'); | |
| 25 | + expect(t.image).toMatch(/^https:\/\/chronexttime\.imgix\.net\//); | |
| 26 | + expect(p.nextUrl).toContain('offset%5D=24'); | |
| 27 | + expect(p.title).toBe('Rolex Datejust Pre-Owned Watches'); | |
| 28 | + const out = await connector.normalize({ url: p.url, externalId: null, kind: 'listing', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload: p }); | |
| 29 | + const l = out[0]; | |
| 30 | + if (l?.kind !== 'listing') throw new Error('expected listing'); | |
| 31 | + expect(l.price).toBe(4380); | |
| 32 | + expect(l.currency).toBe('USD'); | |
| 33 | + expect(l.attributes.categorySlug).toBe('rolex'); | |
| 34 | + expect(l.attributes.reference).toBe('79160'); | |
| 35 | + expect(l.attributes.identifiers).toEqual({ chronext_id: 'V01800358', reference: '79160' }); | |
| 36 | + expect(l.condition.conditionRaw).toBe('Excellent'); | |
| 37 | + expect(l.seller).toBe('CHRONEXT'); | |
| 38 | + }); | |
| 39 | + | |
| 40 | + it('parses the product page JSON-LD (sku, mpn, brand, USD offer)', async () => { | |
| 41 | + const p = parseProductPage(sample('product-jsonld.sample.html'), 'https://www.chronext.com/rolex/datejust/116234/V66464'); | |
| 42 | + if (p?.kind !== 'product_page') throw new Error('expected product page'); | |
| 43 | + expect(p.product.id).toBe('V66464'); | |
| 44 | + expect(p.product.sku).toBe('A20325'); | |
| 45 | + expect(p.product.mpn).toBe('116234'); | |
| 46 | + expect(p.product.brand).toBe('Rolex'); | |
| 47 | + expect(p.product.price).toBe(8850); | |
| 48 | + expect(p.product.currency).toBe('USD'); | |
| 49 | + expect(p.product.condition).toBe('RefurbishedCondition'); | |
| 50 | + const out = await connector.normalize({ url: p.url, externalId: null, kind: 'listing', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload: p }); | |
| 51 | + const l = out[0]; | |
| 52 | + if (l?.kind !== 'listing') throw new Error('expected listing'); | |
| 53 | + expect(l.attributes.model).toBe('Datejust 36'); | |
| 54 | + expect(l.attributes.identifiers.chronext_sku).toBe('A20325'); | |
| 55 | + expect(l.attributes.reference).toBe('116234'); | |
| 56 | + expect(l.availability).toBe('available'); | |
| 57 | + }); | |
| 58 | + | |
| 59 | + it('category fixture normalises to Rolex listings with references; EUR price text is kept in EUR', async () => { | |
| 60 | + const fx = loadFixture('chronext', 'rolex-datejust-p1'); | |
| 61 | + const out = await connector.normalize(fx.raw); | |
| 62 | + expect(out.length).toBeGreaterThan(5); | |
| 63 | + expect(out.every((r) => r.kind === 'listing' && r.attributes.categorySlug === 'rolex' && r.currency === 'USD')).toBe(true); | |
| 64 | + const eur = await connector.normalize({ ...fx.raw, payload: { ...(fx.raw.payload as object), tiles: [{ id: 'V1', href: 'https://www.chronext.com/omega/speedmaster/311.30.42.30.01.005/V1', brand: 'Omega', model: 'Speedmaster', reference: '311.30.42.30.01.005', priceText: 'EUR 5.850', condition: 'Very Good', image: null }] } }); | |
| 65 | + const l = eur[0]; | |
| 66 | + if (l?.kind !== 'listing') throw new Error('expected listing'); | |
| 67 | + expect(l.currency).toBe('EUR'); | |
| 68 | + expect(l.price).toBe(5850); | |
| 69 | + expect(l.attributes.categorySlug).toBe('omega'); | |
| 70 | + expect(l.condition.conditionRaw).toBe('Very good'); | |
| 71 | + }); | |
| 72 | +}); | |
added
connectors/api/chronext/index.ts
+210 −0
@@ -0,0 +1,210 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { normalizeCondition } from '@rareindex/taxonomy'; | |
| 4 | +import { AssetAttributesSchema, CurrencySchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared'; | |
| 5 | +import { watchCategory, watchMaterial, caseSize } from '../_luxury-lib/index.js'; | |
| 6 | +import { extractYear } from '@rareindex/shared'; | |
| 7 | + | |
| 8 | +/** | |
| 9 | + * CHRONEXT (Cologne, DE) — certified pre-owned/new watch retailer. Category pages (`/rolex/submariner` …) are | |
| 10 | + * server-rendered with one `.product-tile` per watch (brand, model, reference, price "USD 4,380", condition label, | |
| 11 | + * image, product URL `/brand/model/reference/V<id>`); pagination links carry a stream id + `offset`. Product pages | |
| 12 | + * expose a schema.org Product (sku, mpn, brand, offers) used for URL lookup. Asking prices → `listing`. | |
| 13 | + */ | |
| 14 | + | |
| 15 | +const SITE = 'https://www.chronext.com'; | |
| 16 | +const PARSER_VERSION = '1.0.0'; | |
| 17 | +const PAGE_SIZE = 24; | |
| 18 | + | |
| 19 | +export const TileSchema = z.object({ id: z.string(), href: z.string(), brand: z.string().nullable(), model: z.string().nullable(), reference: z.string().nullable(), priceText: z.string().nullable(), condition: z.string().nullable(), image: z.string().nullable() }); | |
| 20 | +export type Tile = z.infer<typeof TileSchema>; | |
| 21 | +export const ProductSchema = z.object({ id: z.string(), href: z.string(), name: z.string(), sku: z.string().nullable(), mpn: z.string().nullable(), brand: z.string().nullable(), price: z.number().nullable(), currency: z.string().nullable(), availability: z.string().nullable(), condition: z.string().nullable(), image: z.string().nullable() }); | |
| 22 | +export const PagePayloadSchema = z.discriminatedUnion('kind', [ | |
| 23 | + z.object({ kind: z.literal('category_page'), url: z.string(), seed: z.string(), page: z.number(), tiles: z.array(TileSchema), nextUrl: z.string().nullable(), title: z.string().nullable() }), | |
| 24 | + z.object({ kind: z.literal('product_page'), url: z.string(), product: ProductSchema }), | |
| 25 | +]); | |
| 26 | +export type PagePayload = z.infer<typeof PagePayloadSchema>; | |
| 27 | +export type CategoryPagePayload = Extract<PagePayload, { kind: 'category_page' }>; | |
| 28 | + | |
| 29 | +export function parseCategoryPage(htmlText: string, url: string, seed: string, page: number): CategoryPagePayload { | |
| 30 | + const $ = H.load(htmlText); | |
| 31 | + const tiles: Tile[] = []; | |
| 32 | + const seen = new Set<string>(); | |
| 33 | + $('.product-tile').each((_, el) => { | |
| 34 | + const $t = $(el); | |
| 35 | + const href = $t.find('a[href]').first().attr('href') ?? null; | |
| 36 | + const id = href?.match(/\/(V\d+)\/?$/)?.[1] ?? null; | |
| 37 | + if (!href || !id || seen.has(id)) return; | |
| 38 | + seen.add(id); | |
| 39 | + tiles.push({ | |
| 40 | + id, | |
| 41 | + href: href.startsWith('http') ? href : `${SITE}${href}`, | |
| 42 | + brand: H.text($t.find('.product-tile__brand').first()), | |
| 43 | + model: H.text($t.find('.product-tile__model').first()), | |
| 44 | + reference: H.text($t.find('.product-tile__reference').first()), | |
| 45 | + priceText: H.text($t.find('.product-tile__price .price').first()) ?? H.text($t.find('.price').first()), | |
| 46 | + condition: H.text($t.find('.condition-with-icon__text').first()), | |
| 47 | + image: $t.find('img').first().attr('src') ?? null, | |
| 48 | + }); | |
| 49 | + }); | |
| 50 | + // Pagination links look like "<seed>?s[<stream>][offset]=24&nodeId=…"; the next page is the one whose offset = page × 24. | |
| 51 | + let nextUrl: string | null = null; | |
| 52 | + $('a[href*="offset"]').each((_, a) => { | |
| 53 | + const href = $(a).attr('href') ?? ''; | |
| 54 | + const off = decodeURIComponent(href).match(/\[offset\]=(\d+)/)?.[1]; | |
| 55 | + if (off && Number(off) === page * PAGE_SIZE) nextUrl = href.startsWith('http') ? href.replace(/&/g, '&') : `${SITE}${href.replace(/&/g, '&')}`; | |
| 56 | + }); | |
| 57 | + const title = H.text($('h1').first()); | |
| 58 | + return { kind: 'category_page', url, seed, page, tiles, nextUrl, title }; | |
| 59 | +} | |
| 60 | + | |
| 61 | +export function parseProductPage(htmlText: string, url: string): PagePayload | null { | |
| 62 | + const prod = H.jsonLd(htmlText, 'Product')[0]; | |
| 63 | + if (!prod) return null; | |
| 64 | + const offers = (Array.isArray(prod.offers) ? prod.offers[0] : prod.offers) as Record<string, unknown> | undefined; | |
| 65 | + const brand = prod.brand && typeof prod.brand === 'object' ? String((prod.brand as { name?: string }).name ?? '') : prod.brand ? String(prod.brand) : null; | |
| 66 | + const price = offers?.price !== undefined ? Number(offers.price) : NaN; | |
| 67 | + const id = url.match(/\/(V\d+)\/?$/)?.[1] ?? String(prod.sku ?? ''); | |
| 68 | + return { | |
| 69 | + kind: 'product_page', | |
| 70 | + url, | |
| 71 | + product: { id, href: url, name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(), sku: prod.sku ? String(prod.sku) : null, mpn: prod.mpn ? String(prod.mpn) : null, brand: brand || null, price: Number.isFinite(price) && price > 0 ? price : null, currency: offers?.priceCurrency ? String(offers.priceCurrency) : null, availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null, condition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null, image: typeof prod.image === 'string' ? prod.image : Array.isArray(prod.image) ? String(prod.image[0] ?? '') || null : null }, | |
| 72 | + }; | |
| 73 | +} | |
| 74 | + | |
| 75 | +function conditionFromLabel(label: string | null): string | null { | |
| 76 | + if (!label) return null; | |
| 77 | + const l = label.toLowerCase(); | |
| 78 | + if (/like new|mint|excellent/.test(l)) return 'Excellent'; | |
| 79 | + if (/unworn|\bnew\b/.test(l)) return 'Unworn'; | |
| 80 | + if (/very good/.test(l)) return 'Very good'; | |
| 81 | + if (/good/.test(l)) return 'Good'; | |
| 82 | + if (/fair|vintage/.test(l)) return 'Fair'; | |
| 83 | + return label; | |
| 84 | +} | |
| 85 | + | |
| 86 | +export class ChronextConnector extends BaseConnector { | |
| 87 | + readonly version = '1.0.0'; | |
| 88 | + readonly parserVersion = PARSER_VERSION; | |
| 89 | + protected override minIntervalMs = 4000; | |
| 90 | + override readonly urlPatterns = [/^https?:\/\/(?:www\.)?chronext\.com\/([a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9.-]+\/V\d+)/i]; | |
| 91 | + | |
| 92 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 93 | + const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []); | |
| 94 | + const pages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 1); | |
| 95 | + const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number }; | |
| 96 | + let count = 0; | |
| 97 | + for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) { | |
| 98 | + const seed = seeds[si]!; | |
| 99 | + let url: string | null = `${SITE}${seed.startsWith('/') ? seed : `/${seed}`}`; | |
| 100 | + for (let page = 1; page <= pages && url; page++) { | |
| 101 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 102 | + const pageUrl: string = url; | |
| 103 | + await this.throttle(pageUrl); | |
| 104 | + const res = await ctx.fetch(pageUrl, { engines: ['api', 'firecrawl'], responseType: 'text', timeoutMs: 90_000, headers: { accept: 'text/html,application/xhtml+xml' }, expect: ['title', 'price', 'identifiers'], parse: (r) => { | |
| 105 | + const p = r.html ? parseCategoryPage(r.html, pageUrl, seed, page) : null; | |
| 106 | + const t = p?.tiles[0]; | |
| 107 | + return t ? { title: `${t.brand} ${t.model}`, price: t.priceText, identifiers: t.reference ? { reference: t.reference } : null } : null; | |
| 108 | + } }); | |
| 109 | + if (!res.success || !res.html) { | |
| 110 | + ctx.anomaly('page_fetch_failed', `${pageUrl}: ${res.error ?? res.httpStatus}`); | |
| 111 | + break; | |
| 112 | + } | |
| 113 | + const payload = parseCategoryPage(res.html, pageUrl, seed, page); | |
| 114 | + if (!payload.tiles.length) { | |
| 115 | + if (page === 1) ctx.anomaly('selector_missing', `${pageUrl}: no product tiles`); | |
| 116 | + break; | |
| 117 | + } | |
| 118 | + count++; | |
| 119 | + yield { url: pageUrl, externalId: `${seed}#${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 120 | + await ctx.progress({ page, itemsProcessed: count }); | |
| 121 | + url = payload.nextUrl; | |
| 122 | + } | |
| 123 | + await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() }); | |
| 124 | + } | |
| 125 | + await ctx.setCursor({ done: true, at: new Date().toISOString() }); | |
| 126 | + } | |
| 127 | + | |
| 128 | + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { | |
| 129 | + const path = url.match(this.urlPatterns[0]!)?.[1]; | |
| 130 | + if (!path) return []; | |
| 131 | + const target = `${SITE}/${path}`; | |
| 132 | + await this.throttle(target); | |
| 133 | + const res = await ctx.fetch(target, { engines: ['api', 'firecrawl'], responseType: 'text', headers: { accept: 'text/html,application/xhtml+xml' }, minQuality: 0.2 }); | |
| 134 | + const payload = res.success && res.html ? parseProductPage(res.html, target) : null; | |
| 135 | + if (!payload) return []; | |
| 136 | + return [{ url: target, externalId: `product:${path.split('/').pop()}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; | |
| 137 | + } | |
| 138 | + | |
| 139 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 140 | + const p = PagePayloadSchema.parse(raw.payload); | |
| 141 | + const out: NormalizedRecord[] = []; | |
| 142 | + if (p.kind === 'product_page') { | |
| 143 | + const pr = p.product; | |
| 144 | + const categorySlug = watchCategory(pr.brand); | |
| 145 | + const conditionRaw = pr.condition === 'NewCondition' ? 'Unworn' : pr.condition === 'RefurbishedCondition' || pr.condition === 'UsedCondition' ? 'Pre-owned' : null; | |
| 146 | + const cur = CurrencySchema.safeParse(pr.currency ?? ''); | |
| 147 | + out.push( | |
| 148 | + NormalizedListingSchema.parse({ | |
| 149 | + kind: 'listing', | |
| 150 | + connectorId: this.meta.id, | |
| 151 | + sourceId: this.meta.sourceId, | |
| 152 | + sourceUrl: pr.href, | |
| 153 | + externalId: pr.id, | |
| 154 | + rawTitle: pr.name, | |
| 155 | + imageUrls: pr.image ? [pr.image] : [], | |
| 156 | + attributes: AssetAttributesSchema.parse({ categorySlug, brand: pr.brand, name: pr.name, model: pr.name.replace(new RegExp(`^${(pr.brand ?? '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*`, 'i'), '') || null, reference: pr.mpn, year: extractYear(pr.name), material: watchMaterial(pr.name), size: caseSize(pr.name), identifiers: { chronext_id: pr.id, ...(pr.sku ? { chronext_sku: pr.sku } : {}), ...(pr.mpn ? { reference: pr.mpn } : {}) }, metadata: { availability_raw: pr.availability, condition_raw: pr.condition } }), | |
| 157 | + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, | |
| 158 | + condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null }, | |
| 159 | + observedAt: raw.fetchedAt, | |
| 160 | + confidence: 0.85, | |
| 161 | + parserVersion: PARSER_VERSION, | |
| 162 | + listingType: 'fixed_price', | |
| 163 | + price: cur.success ? pr.price : null, | |
| 164 | + currency: cur.success && pr.price ? cur.data : null, | |
| 165 | + seller: 'CHRONEXT', | |
| 166 | + location: 'DE', | |
| 167 | + quantity: 1, | |
| 168 | + availability: pr.availability === 'InStock' ? 'available' : pr.availability === 'OutOfStock' ? 'ended' : 'unknown', | |
| 169 | + }), | |
| 170 | + ); | |
| 171 | + return out; | |
| 172 | + } | |
| 173 | + for (const t of p.tiles) { | |
| 174 | + const parsed = t.priceText ? parsePrice(t.priceText) : null; | |
| 175 | + const cur = parsed?.currency ? CurrencySchema.safeParse(parsed.currency) : null; | |
| 176 | + const categorySlug = watchCategory(t.brand); | |
| 177 | + const name = [t.brand, t.model, t.reference].filter(Boolean).join(' '); | |
| 178 | + const conditionRaw = conditionFromLabel(t.condition); | |
| 179 | + out.push( | |
| 180 | + NormalizedListingSchema.parse({ | |
| 181 | + kind: 'listing', | |
| 182 | + connectorId: this.meta.id, | |
| 183 | + sourceId: this.meta.sourceId, | |
| 184 | + sourceUrl: t.href, | |
| 185 | + externalId: t.id, | |
| 186 | + rawTitle: name, | |
| 187 | + imageUrls: t.image ? [t.image] : [], | |
| 188 | + attributes: AssetAttributesSchema.parse({ categorySlug, brand: t.brand, name, model: t.model, reference: t.reference, identifiers: { chronext_id: t.id, ...(t.reference ? { reference: t.reference } : {}) }, metadata: { category_page: p.url.split('?')[0], condition_label: t.condition, price_text: t.priceText } }), | |
| 189 | + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, | |
| 190 | + condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null }, | |
| 191 | + observedAt: raw.fetchedAt, | |
| 192 | + confidence: 0.8, | |
| 193 | + parserVersion: PARSER_VERSION, | |
| 194 | + listingType: 'fixed_price', | |
| 195 | + price: cur?.success && parsed ? parsed.amount : null, | |
| 196 | + currency: cur?.success ? cur.data : null, | |
| 197 | + seller: 'CHRONEXT', | |
| 198 | + location: 'DE', | |
| 199 | + quantity: 1, | |
| 200 | + availability: 'available', | |
| 201 | + }), | |
| 202 | + ); | |
| 203 | + } | |
| 204 | + return out; | |
| 205 | + } | |
| 206 | +} | |
| 207 | + | |
| 208 | +export default function createConnector(meta: ConnectorMeta) { | |
| 209 | + return new ChronextConnector(meta); | |
| 210 | +} | |
added
connectors/api/chronext/meta.json
+43 −0
@@ -0,0 +1,43 @@ | ||
| 1 | +{ | |
| 2 | + "id": "chronext", | |
| 3 | + "displayName": "CHRONEXT (certified pre-owned watches — listings)", | |
| 4 | + "sourceId": "chronext", | |
| 5 | + "sourceName": "CHRONEXT", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.chronext.com", | |
| 8 | + "module": "api/chronext", | |
| 9 | + "enginePriority": ["api", "firecrawl"], | |
| 10 | + "categories": ["watches", "rolex", "patek_philippe", "audemars_piguet", "omega", "other_watches"], | |
| 11 | + "regions": ["DE", "EU", "global"], | |
| 12 | + "country": "DE", | |
| 13 | + "languages": ["en"], | |
| 14 | + "currency": ["USD", "EUR", "GBP", "CHF"], | |
| 15 | + "supportsListings": true, | |
| 16 | + "supportsSold": false, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": true, | |
| 22 | + "refreshFrequencyMinutes": 720, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.8, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.chronext.com/terms-and-conditions", | |
| 27 | + "acquisitionMethod": "server-rendered category tiles + schema.org Product JSON-LD, direct HTTP", | |
| 28 | + "historicalDepth": "none", | |
| 29 | + "accessNotes": "Public category pages (`/rolex/submariner`, `/omega/speedmaster`, … from config.seeds; ≈3.7 MB each, server-rendered) parsed with cheerio: one `.product-tile` per watch with brand, model, reference, displayed price (currency follows the requesting region, e.g. 'USD 4,380' from North America, EUR from the EU — stored as parsed, never converted), condition label (Like New / Very Good / Good), image and product URL `/brand/model/reference/V<id>`. Pagination follows the page's own `offset` links (24 tiles per page, pagesPerSeed cap). Product pages (schema.org Product: sku, mpn, brand, offers price/currency/availability/itemCondition) are fetched only for URL lookup. robots.txt (verified 2026-09-08) allows these paths and disallows checkout, cart, campaign and the two-letter country mirrors (/de/, /fr/ … which we never request). Asking prices are listings, never sales. No account, cart or checkout traffic; honest RareIndexBot UA over plain HTTPS, 4 s between requests, concurrency 1.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "requires": [], | |
| 33 | + "config": { | |
| 34 | + "seeds": [ | |
| 35 | + "/rolex/submariner", "/rolex/datejust", "/rolex/cosmograph-daytona", "/rolex/gmt-master-ii", "/rolex/day-date", "/rolex/explorer", | |
| 36 | + "/omega/speedmaster", "/omega/seamaster", | |
| 37 | + "/patek-philippe/nautilus", "/patek-philippe/aquanaut", | |
| 38 | + "/audemars-piguet/royal-oak", | |
| 39 | + "/tudor/black-bay", "/cartier/santos", "/breitling/navitimer", "/iwc/portugieser", "/jaeger-lecoultre/reverso", "/panerai/luminor", "/grand-seiko" | |
| 40 | + ], | |
| 41 | + "pagesPerSeed": 1 | |
| 42 | + } | |
| 43 | +} | |
added
connectors/api/clean-sweep/README.md
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +# clean-sweep — Clean Sweep Auctions (past results) | |
| 2 | + | |
| 3 | +Sports cards & memorabilia house (Port Washington, NY). Source: the public archive on www.cleansweepauctions.com. | |
| 4 | + | |
| 5 | +- `/past-auctions` → one page per closed monthly auction (April 2008 → present). | |
| 6 | +- `/past-auctions/<month-year>?page=N` → 50 lot links per page. | |
| 7 | +- `/past-auctions/<month-year>/<slug>` → `Winning bid: $X` (hammer; a 22 % buyer's premium is added by the house → `buyerPremiumIncluded: false`). | |
| 8 | + | |
| 9 | +One request per lot (2 s politeness). `saleDate` = first day of the auction month with `metadata.sale_date_precision = "month"` because the archive gives no closing date. Cursor `{ doneMonths, current: { month, page, index } }` resumes mid-month; incremental runs finish the newest pending month, backfill walks newest → oldest and ends with `done: true`. The live auction host `marketplace.cleansweepauctions.com` is not crawled. | |
| 10 | + | |
| 11 | +Tests: `pnpm vitest run connectors/api/clean-sweep`. Live smoke / fixture capture: `pnpm tsx connectors/api/_g7-auctions-na-lib/smoke.ts api/clean-sweep --capture lot --limit 3`. | |
added
connectors/api/clean-sweep/index.test.ts
+83 −0
@@ -0,0 +1,83 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import { fileURLToPath } from 'node:url'; | |
| 5 | +import { runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 7 | +import { monthYearDate } from '../_g7-auctions-na-lib/index.js'; | |
| 8 | +import createConnector, { cleanSweepCategory, isBundle, monthKey, parseLotPage, parseMonthPage, parsePastIndex } from './index.js'; | |
| 9 | + | |
| 10 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 11 | +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 12 | +const connector = createConnector(meta); | |
| 13 | + | |
| 14 | +// Real fragments captured live from www.cleansweepauctions.com on 2026-09-08. | |
| 15 | +const INDEX = `<div class="col-md-3"> <div class="auction-list-item"> <div class="image-container"> <a href="/past-auctions/may-2025"> <img src="http://marketplace.cleansweepauctions.com/images/main/entertainmentlot705986.jpg" alt="May 2025" /> </a> </div> <a href="/past-auctions/may-2025"> May 2025 </a> </div> </div> | |
| 16 | +<div class="col-md-3"> <div class="auction-list-item"> <div class="image-container"> <a href="/past-auctions/april-2025"> <img src="http://marketplace.cleansweepauctions.com/images/main/flick678144b.jpg" alt="April 2025" /> </a> </div> <a href="/past-auctions/april-2025"> April 2025 </a> </div> </div> | |
| 17 | +<div class="col-md-3"> <div class="auction-list-item"> <a href="/past-auctions/december-2025"> December 2025 </a> </div> </div> | |
| 18 | +<div class="col-md-3"> <div class="auction-list-item"> <a href="/past-auctions/april-2008"> April 2008 </a> </div> </div> | |
| 19 | +<a href="/past-auctions">Past Auctions</a>`; | |
| 20 | + | |
| 21 | +const MONTH_PAGE = `<h1 style="font-weight:600;">April 2025</h1> | |
| 22 | +<div class="row"> <div class="col-md-3"> <div class="auction-list-item"> <div class="image-container"> <img src="http://marketplace.cleansweepauctions.com/images/main/mcclothlin713341.jpg" alt="1971 Topps 556 Jim McClothlin 8 " /> </div> <!-- 🔥 CORRECT ROUTE --> <a href="/past-auctions/april-2025/1971-topps-556-jim-mcclothlin-8"> 1971 Topps 556 Jim McClothlin 8 </a> </div> </div> | |
| 23 | +<div class="col-md-3"> <div class="auction-list-item"> <div class="image-container"> <img src="http://marketplace.cleansweepauctions.com/images/main/dodgers713283.jpg" alt="1947 Exhibit 294 Dodgers Team 1956 PSA 3.5 " /> </div> <a href="/past-auctions/april-2025/1947-exhibit-294-dodgers-team-1956-psa-35"> 1947 Exhibit 294 Dodgers Team 1956 PSA 3.5 </a> </div> </div> </div> | |
| 24 | +<div class="pagination" style="text-align:center;"> <a href="?page=1&q=&category=&auction=&sort=" class="active"> 1 </a> <a href="?page=2&q=&category=&auction=&sort=" class=""> 2 </a> <a href="?page=3&q=&category=&auction=&sort=" class=""> 3 </a> <a href="?page=12&q=" class="">Next →</a> </div>`; | |
| 25 | + | |
| 26 | +const LOT_PAGE = `<section id="subheader"> <div class="container"> <h1 style="font-weight:600;">1971 Topps 556 Jim McClothlin 8 </h1> <ul class="crumb"> <li><a href="/">Home</a></li> <li class="sep">/</li> <li><a href="april-2025">April 2025</a></li> <li class="sep">/</li> <li>Auction Item Details</li> </ul> </div> </section> | |
| 27 | +<section> <div class="container"> <div class="row"> <div class="col-md-4"> <div class='card'> <img src='http://marketplace.cleansweepauctions.com/images/main/mcclothlin713341.jpg' /> </div> </div> <div class="col-md-8"> <h2 style="padding-top: 10px;"> 1971 Topps 556 Jim McClothlin 8 </h2> <div> </div> <div> <strong style="display:block; padding-top:10px; color: #993B34;">Winning bid: $70.00</strong> </div> </div> </div> <a href="/past-auctions/april-2025" class="back-to-auction">< Back to April 2025</a> </div> </section>`; | |
| 28 | + | |
| 29 | +describe('clean-sweep', () => { | |
| 30 | + runFixtureSuite(connector, it, expect); | |
| 31 | + | |
| 32 | + it('parses the past-auction index newest first', () => { | |
| 33 | + expect(parsePastIndex(INDEX)).toEqual(['december-2025', 'may-2025', 'april-2025', 'april-2008']); | |
| 34 | + expect(monthKey('april-2025')).toBe(202504); | |
| 35 | + expect(monthKey('not-a-month')).toBeNull(); | |
| 36 | + }); | |
| 37 | + | |
| 38 | + it('parses a month page (lots + page count)', () => { | |
| 39 | + const mp = parseMonthPage(MONTH_PAGE, 'april-2025', 1); | |
| 40 | + expect(mp.monthLabel).toBe('April 2025'); | |
| 41 | + expect(mp.totalPages).toBe(12); | |
| 42 | + expect(mp.lots.length).toBe(2); | |
| 43 | + expect(mp.lots[0]).toEqual({ slug: '1971-topps-556-jim-mcclothlin-8', url: 'https://www.cleansweepauctions.com/past-auctions/april-2025/1971-topps-556-jim-mcclothlin-8', title: '1971 Topps 556 Jim McClothlin 8', image: 'https://marketplace.cleansweepauctions.com/images/main/mcclothlin713341.jpg' }); | |
| 44 | + }); | |
| 45 | + | |
| 46 | + it('parses the lot page winning bid and month date', () => { | |
| 47 | + const p = parseLotPage(LOT_PAGE, 'https://www.cleansweepauctions.com/past-auctions/april-2025/1971-topps-556-jim-mcclothlin-8', 'april-2025', '1971-topps-556-jim-mcclothlin-8', 'https://www.cleansweepauctions.com/past-auctions/april-2025')!; | |
| 48 | + expect(p).toMatchObject({ title: '1971 Topps 556 Jim McClothlin 8', winningBidText: '$70.00', price: 70, monthLabel: 'April 2025' }); | |
| 49 | + expect(p.image).toBe('https://marketplace.cleansweepauctions.com/images/main/mcclothlin713341.jpg'); | |
| 50 | + expect(monthYearDate('april-2025')?.toISOString()).toBe('2025-04-01T00:00:00.000Z'); | |
| 51 | + }); | |
| 52 | + | |
| 53 | + it('normalises to a hammer sale with month precision and no inferred grader', async () => { | |
| 54 | + const p = parseLotPage(LOT_PAGE, 'https://www.cleansweepauctions.com/past-auctions/april-2025/1971-topps-556-jim-mcclothlin-8', 'april-2025', '1971-topps-556-jim-mcclothlin-8', 'https://www.cleansweepauctions.com/past-auctions/april-2025')!; | |
| 55 | + const out = await connector.normalize({ url: p.url, externalId: 'april-2025/1971-topps-556-jim-mcclothlin-8', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload: p }); | |
| 56 | + expect(out.length).toBe(1); | |
| 57 | + const s = out[0]!; | |
| 58 | + if (s.kind !== 'sale') throw new Error('expected sale'); | |
| 59 | + expect(s).toMatchObject({ price: 70, currency: 'USD', buyerPremiumIncluded: false, auctionHouse: 'Clean Sweep Auctions', confidence: 0.7, isBundle: false }); | |
| 60 | + expect(s.saleDate.toISOString()).toBe('2025-04-01T00:00:00.000Z'); | |
| 61 | + expect(s.attributes.categorySlug).toBe('baseball_cards'); | |
| 62 | + expect(s.attributes.metadata).toMatchObject({ sale_date_precision: 'month', buyers_premium_pct: 22, hammer_price: 70 }); | |
| 63 | + expect(s.attributes.identifiers.clean_sweep_lot_slug).toBe('april-2025/1971-topps-556-jim-mcclothlin-8'); | |
| 64 | + expect(s.grade.grader).toBeNull(); | |
| 65 | + expect(s.attributes.year).toBe(1971); | |
| 66 | + }); | |
| 67 | + | |
| 68 | + it('keeps graders when the title names one and flags bundles', async () => { | |
| 69 | + const graded = { ...parseLotPage(LOT_PAGE, 'https://www.cleansweepauctions.com/x', 'april-2025', 'x', 'y')!, title: '1947 Exhibit 294 Dodgers Team 1956 PSA 3.5', price: 120 }; | |
| 70 | + const [s] = await connector.normalize({ url: 'https://www.cleansweepauctions.com/x', externalId: 'x', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload: graded }); | |
| 71 | + if (!s || s.kind !== 'sale') throw new Error('expected sale'); | |
| 72 | + expect(s.grade).toMatchObject({ grader: 'psa', grade: '3.5' }); | |
| 73 | + expect(s.confidence).toBe(0.75); | |
| 74 | + expect(isBundle('1970s Reds Signed Dinner Programs w/Bench, Rose (3 pcs) 9')).toBe(true); | |
| 75 | + expect(isBundle('1971 Topps 556 Jim McClothlin 8')).toBe(false); | |
| 76 | + expect(cleanSweepCategory('2018 Topps Update 285 Ohtani RC Ex')).toBe('baseball_cards'); | |
| 77 | + expect(cleanSweepCategory('1986 Fleer 57 Michael Jordan RC PSA 7')).toBe('basketball_cards'); | |
| 78 | + expect(cleanSweepCategory('1970 NLCS At Cincinnati Program NM')).toBe('sports_memorabilia'); | |
| 79 | + expect(cleanSweepCategory('Big Red Machine Single Signed Ball Lot (27 different) 8')).toBe('sports_memorabilia'); | |
| 80 | + const unsold = await connector.normalize({ url: 'https://www.cleansweepauctions.com/x', externalId: 'x', kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { ...graded, price: null, winningBidText: null } }); | |
| 81 | + expect(unsold).toEqual([]); | |
| 82 | + }); | |
| 83 | +}); | |
added
connectors/api/clean-sweep/index.ts
+269 −0
@@ -0,0 +1,269 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { amount, certFromTitle, isBundleTitle, lotAttributes, makeSale, monthYearDate, safeYear, saleGrade, sportsCategory } from '../_g7-auctions-na-lib/index.js'; | |
| 5 | + | |
| 6 | +const BASE = 'https://www.cleansweepauctions.com'; | |
| 7 | +const HOUSE = 'Clean Sweep Auctions'; | |
| 8 | +const PARSER_VERSION = '1.0.0'; | |
| 9 | +const BUYERS_PREMIUM_PCT = 22; | |
| 10 | + | |
| 11 | +export const LotPayloadSchema = z.object({ | |
| 12 | + kind: z.literal('cs_lot'), | |
| 13 | + month: z.string(), | |
| 14 | + monthLabel: z.string().nullable(), | |
| 15 | + listUrl: z.string(), | |
| 16 | + url: z.string(), | |
| 17 | + slug: z.string(), | |
| 18 | + title: z.string(), | |
| 19 | + winningBidText: z.string().nullable(), | |
| 20 | + price: z.number().nullable(), | |
| 21 | + image: z.string().nullable(), | |
| 22 | + description: z.string().nullable(), | |
| 23 | +}); | |
| 24 | +export type LotPayload = z.infer<typeof LotPayloadSchema>; | |
| 25 | + | |
| 26 | +const MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december']; | |
| 27 | + | |
| 28 | +/** "april-2025" → sortable key 202504; null when not a month slug. */ | |
| 29 | +export function monthKey(slug: string): number | null { | |
| 30 | + const m = slug.match(/^([a-z]+)-(\d{4})$/i); | |
| 31 | + if (!m) return null; | |
| 32 | + const mo = MONTHS.indexOf(m[1]!.toLowerCase()); | |
| 33 | + return mo < 0 ? null : Number(m[2]) * 100 + mo + 1; | |
| 34 | +} | |
| 35 | + | |
| 36 | +/** /past-auctions index → month slugs, newest first. */ | |
| 37 | +export function parsePastIndex(htmlText: string): string[] { | |
| 38 | + const $ = H.load(htmlText); | |
| 39 | + const set = new Set<string>(); | |
| 40 | + $('a[href^="/past-auctions/"]').each((_, a) => { | |
| 41 | + const slug = ($(a).attr('href') ?? '').replace(/^\/past-auctions\//, '').split(/[?#/]/)[0] ?? ''; | |
| 42 | + if (monthKey(slug) !== null) set.add(slug.toLowerCase()); | |
| 43 | + }); | |
| 44 | + return [...set].sort((a, b) => monthKey(b)! - monthKey(a)!); | |
| 45 | +} | |
| 46 | + | |
| 47 | +export interface MonthPage { | |
| 48 | + month: string; | |
| 49 | + monthLabel: string | null; | |
| 50 | + page: number; | |
| 51 | + totalPages: number | null; | |
| 52 | + lots: Array<{ slug: string; url: string; title: string; image: string | null }>; | |
| 53 | +} | |
| 54 | + | |
| 55 | +/** /past-auctions/<month-year>?page=N → lot links (50 per page) + page count. */ | |
| 56 | +export function parseMonthPage(htmlText: string, month: string, page: number): MonthPage { | |
| 57 | + const $ = H.load(htmlText); | |
| 58 | + const lots: MonthPage['lots'] = []; | |
| 59 | + const seen = new Set<string>(); | |
| 60 | + const prefix = `/past-auctions/${month}/`; | |
| 61 | + $('.auction-list-item').each((_, el) => { | |
| 62 | + const it = $(el); | |
| 63 | + const a = it.find(`a[href^="${prefix}"]`).first(); | |
| 64 | + const href = a.attr('href'); | |
| 65 | + const title = H.text(a); | |
| 66 | + if (!href || !title) return; | |
| 67 | + const slug = href.slice(prefix.length).split(/[?#]/)[0]!; | |
| 68 | + if (!slug || seen.has(slug)) return; | |
| 69 | + seen.add(slug); | |
| 70 | + const img = it.find('img').attr('src') ?? null; | |
| 71 | + lots.push({ slug, url: `${BASE}${prefix}${slug}`, title, image: img ? img.replace(/^http:/, 'https:') : null }); | |
| 72 | + }); | |
| 73 | + const pages = $('.pagination a[href*="page="]') | |
| 74 | + .map((_, a) => Number(($(a).attr('href') ?? '').match(/page=(\d+)/)?.[1] ?? 0)) | |
| 75 | + .get() | |
| 76 | + .filter((n) => n > 0); | |
| 77 | + return { month, monthLabel: H.text($('h1').first()), page, totalPages: pages.length ? Math.max(...pages) : null, lots }; | |
| 78 | +} | |
| 79 | + | |
| 80 | +/** Lot page → title, "Winning bid: $X", image. */ | |
| 81 | +export function parseLotPage(htmlText: string, url: string, month: string, slug: string, listUrl: string): LotPayload | null { | |
| 82 | + const $ = H.load(htmlText); | |
| 83 | + const title = H.text($('h1').first()) ?? H.text($('h2').first()); | |
| 84 | + if (!title) return null; | |
| 85 | + const text = $('body').text().replace(/\s+/g, ' '); | |
| 86 | + const bid = text.match(/Winning bid:\s*(\$[\d,]+(?:\.\d{1,2})?)/i)?.[1] ?? null; | |
| 87 | + const image = $('.card img').first().attr('src') ?? null; | |
| 88 | + const monthLabel = H.text($('ul.crumb a[href$="' + month + '"]').first()) ?? null; | |
| 89 | + const desc = H.text($('.col-md-8 > div p').first()); | |
| 90 | + return { kind: 'cs_lot', month, monthLabel, listUrl, url, slug, title, winningBidText: bid, price: amount(bid), image: image ? image.replace(/^http:/, 'https:') : null, description: desc }; | |
| 91 | +} | |
| 92 | + | |
| 93 | +const CARD_BRANDS = /\b(topps|bowman|fleer|donruss|panini|upper deck|leaf|goudey|exhibit|play ball|t206|t205|e\d{2,3}|w\d{3}|score|pro set|skybox|hoops|o-pee-chee|opc|parkhurst|red man|kellogg'?s|hostess|post cereal|bazooka|nu-card|philadelphia|sportscaster|tcma|sgc|psa|bgs|beckett)\b/i; | |
| 94 | +const CARD_HINTS = /\b(rc|rookie|refractor|parallel|auto|autograph card|wax|unopened|pack|set break|cello|rack pack|checklist|#\d+|\d{4} .* \d{1,3}\b)\b/i; | |
| 95 | +const MEMORABILIA_WORDS = /\b(program|ticket|stub|pennant|photo|photograph|ball|bat|jersey|glove|helmet|button|pin|magazine|yearbook|guide|book|press|schedule|poster|scorecard|contract|check|letter|menu|plate|bobble|figure|puck|stick|medal|ring|patch|cap|hat|shirt|uniform|display|sign|dinner)\b/i; | |
| 96 | + | |
| 97 | +/** | |
| 98 | + * Clean Sweep titles are terse ("1971 Topps 556 Jim McClothlin 8", "2018 Topps Update 285 Ohtani RC Ex"): | |
| 99 | + * a year + card brand + number without memorabilia words is a card even when the shared mapper cannot tell. | |
| 100 | + */ | |
| 101 | +export function cleanSweepCategory(title: string): string { | |
| 102 | + const generic = sportsCategory(title); | |
| 103 | + if (generic.endsWith('_cards') || generic === 'pokemon' || generic === 'magic_the_gathering') return generic; | |
| 104 | + const cardLike = CARD_BRANDS.test(title) && (CARD_HINTS.test(title) || /\b(19|20)\d{2}\b/.test(title)) && !MEMORABILIA_WORDS.test(title); | |
| 105 | + if (!cardLike) return generic; | |
| 106 | + const sport = /\b(basketball|nba|jordan|lebron|kobe|hoops|skybox)\b/i.test(title) ? 'basketball_cards' : /\b(football|nfl|pro set|brady|mahomes|montana|payton)\b/i.test(title) ? 'football_cards' : /\b(hockey|nhl|o-pee-chee|opc|parkhurst|gretzky|orr|howe)\b/i.test(title) ? 'hockey_cards' : /\b(boxing|golf|tennis|wrestling|nascar|racing|olympic|soccer)\b/i.test(title) ? 'other_sports_cards' : 'baseball_cards'; | |
| 107 | + return sport; | |
| 108 | +} | |
| 109 | + | |
| 110 | +/** Bundles: "(3 pcs)", "Lot of 12", "(27 different)". */ | |
| 111 | +export function isBundle(title: string): boolean { | |
| 112 | + return isBundleTitle(title) || /\(\s*\d+\s*(?:pcs?|pieces|different|cards|items)\s*\)/i.test(title) || /\b\d+\s*(?:pcs?|different)\b/i.test(title); | |
| 113 | +} | |
| 114 | + | |
| 115 | +interface Cursor { | |
| 116 | + doneMonths: string[]; | |
| 117 | + current: { month: string; page: number; index: number } | null; | |
| 118 | + done?: boolean; | |
| 119 | +} | |
| 120 | + | |
| 121 | +/** | |
| 122 | + * Clean Sweep Auctions — past-auction archive (2008 → present). Public pages only: | |
| 123 | + * /past-auctions (one entry per closed monthly auction) → /past-auctions/<month-year>?page=N (50 lots) | |
| 124 | + * → /past-auctions/<month-year>/<slug> ("Winning bid: $X", hammer before the 22 % buyer's premium). | |
| 125 | + * The source only dates lots by the auction month, so saleDate is the first of that month (precision flagged). | |
| 126 | + */ | |
| 127 | +export class CleanSweepConnector extends BaseConnector { | |
| 128 | + readonly version = '1.0.0'; | |
| 129 | + readonly parserVersion = PARSER_VERSION; | |
| 130 | + protected override minIntervalMs = 2000; | |
| 131 | + | |
| 132 | + private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> { | |
| 133 | + await this.throttle(url); | |
| 134 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 45_000 }); | |
| 135 | + if (!res.success || !res.html) { | |
| 136 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 137 | + return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 138 | + } | |
| 139 | + return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 140 | + } | |
| 141 | + | |
| 142 | + private readCursor(ctx: CrawlContext): Cursor { | |
| 143 | + const c = ctx.options.cursor ?? {}; | |
| 144 | + const cur = c.current as Cursor['current'] | undefined; | |
| 145 | + return { doneMonths: Array.isArray(c.doneMonths) ? (c.doneMonths as string[]) : [], current: cur && typeof cur === 'object' && cur.month ? { month: cur.month, page: Number(cur.page ?? 1), index: Number(cur.index ?? 0) } : null }; | |
| 146 | + } | |
| 147 | + | |
| 148 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 149 | + const mode = ctx.options.mode; | |
| 150 | + const cfg = this.meta.config; | |
| 151 | + const lotsPerRun = mode === 'probe' ? (ctx.options.limit ?? 3) : Number(cfg.lotsPerRun ?? 150); | |
| 152 | + const pagesPerRun = mode === 'probe' ? 1 : Number(cfg.pagesPerRun ?? 4); | |
| 153 | + const cursor = this.readCursor(ctx); | |
| 154 | + const done = new Set(cursor.doneMonths); | |
| 155 | + | |
| 156 | + // Seeds: month slugs / month URLs / lot URLs. | |
| 157 | + const seedLots: Array<{ month: string; slug: string }> = []; | |
| 158 | + const seedMonths: string[] = []; | |
| 159 | + for (const s of ctx.options.seeds ?? []) { | |
| 160 | + const m = s.match(/past-auctions\/([a-z]+-\d{4})(?:\/([^/?#]+))?/i) ?? s.match(/^([a-z]+-\d{4})$/i); | |
| 161 | + if (!m) continue; | |
| 162 | + if (m[2]) seedLots.push({ month: m[1]!.toLowerCase(), slug: m[2] }); | |
| 163 | + else seedMonths.push(m[1]!.toLowerCase()); | |
| 164 | + } | |
| 165 | + let count = 0; | |
| 166 | + let items = 0; | |
| 167 | + for (const sl of seedLots) { | |
| 168 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 169 | + const listUrl = `${BASE}/past-auctions/${sl.month}`; | |
| 170 | + const url = `${listUrl}/${sl.slug}`; | |
| 171 | + const r = await this.html(ctx, url); | |
| 172 | + const payload = r.html ? parseLotPage(r.html, url, sl.month, sl.slug, listUrl) : null; | |
| 173 | + if (!payload) continue; | |
| 174 | + count++; | |
| 175 | + yield { url, externalId: `${sl.month}/${sl.slug}`, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt }; | |
| 176 | + } | |
| 177 | + if (seedLots.length && !seedMonths.length) return; | |
| 178 | + | |
| 179 | + let months = seedMonths; | |
| 180 | + if (!months.length) { | |
| 181 | + const idx = await this.html(ctx, `${BASE}/past-auctions`); | |
| 182 | + if (!idx.html) return; | |
| 183 | + months = parsePastIndex(idx.html); | |
| 184 | + if (!months.length) { | |
| 185 | + ctx.anomaly('selector_missing', '/past-auctions: no month links found'); | |
| 186 | + return; | |
| 187 | + } | |
| 188 | + } | |
| 189 | + // Backfill and incremental both walk newest → oldest; incremental stops after the first (newest) pending month. | |
| 190 | + const pending = months.filter((m) => !done.has(m)); | |
| 191 | + const queue = cursor.current && pending.includes(cursor.current.month) ? [cursor.current.month, ...pending.filter((m) => m !== cursor.current!.month)] : pending; | |
| 192 | + if (!queue.length) { | |
| 193 | + if (mode === 'backfill') await ctx.setCursor({ doneMonths: [...done], current: null, done: true, updatedAt: new Date().toISOString() }); | |
| 194 | + return; | |
| 195 | + } | |
| 196 | + const monthsThisRun = mode === 'incremental' ? queue.slice(0, 1) : queue; | |
| 197 | + let pagesFetched = 0; | |
| 198 | + let reachedDate: Date | null = null; | |
| 199 | + for (const month of monthsThisRun) { | |
| 200 | + if (ctx.signal?.aborted || this.reached(ctx, count) || items >= lotsPerRun || pagesFetched >= pagesPerRun) break; | |
| 201 | + const listUrl = `${BASE}/past-auctions/${month}`; | |
| 202 | + let page = cursor.current?.month === month ? cursor.current.page : 1; | |
| 203 | + let index = cursor.current?.month === month ? cursor.current.index : 0; | |
| 204 | + let monthComplete = false; | |
| 205 | + while (!ctx.signal?.aborted && pagesFetched < pagesPerRun) { | |
| 206 | + const r = await this.html(ctx, page > 1 ? `${listUrl}?page=${page}` : listUrl); | |
| 207 | + pagesFetched++; | |
| 208 | + if (!r.html) break; | |
| 209 | + const mp = parseMonthPage(r.html, month, page); | |
| 210 | + if (mp.lots.length === 0 && page === 1) ctx.anomaly('selector_missing', `${listUrl}: no lots found`); | |
| 211 | + let stop = false; | |
| 212 | + for (let i = index; i < mp.lots.length; i++) { | |
| 213 | + if (ctx.signal?.aborted || this.reached(ctx, count) || items >= lotsPerRun) { | |
| 214 | + stop = true; | |
| 215 | + break; | |
| 216 | + } | |
| 217 | + const lot = mp.lots[i]!; | |
| 218 | + const r2 = await this.html(ctx, lot.url); | |
| 219 | + index = i + 1; | |
| 220 | + await ctx.setCursor({ doneMonths: [...done], current: { month, page, index }, updatedAt: new Date().toISOString() }); | |
| 221 | + if (!r2.html) continue; | |
| 222 | + const payload = parseLotPage(r2.html, lot.url, month, lot.slug, listUrl); | |
| 223 | + if (!payload) { | |
| 224 | + ctx.anomaly('parse_failure_page', lot.url); | |
| 225 | + continue; | |
| 226 | + } | |
| 227 | + if (!payload.image && lot.image) payload.image = lot.image; | |
| 228 | + if (payload.price === null) ctx.anomaly('price_parse_failure', `${lot.url}: ${payload.winningBidText ?? 'no winning bid'}`); | |
| 229 | + count++; | |
| 230 | + items++; | |
| 231 | + yield { url: lot.url, externalId: `${month}/${lot.slug}`, kind: 'sale', engine: 'api', httpStatus: r2.status, payload, fetchedAt: r2.fetchedAt }; | |
| 232 | + } | |
| 233 | + if (stop) break; | |
| 234 | + const last = mp.totalPages !== null ? page >= mp.totalPages : mp.lots.length < 50; | |
| 235 | + if (last) { | |
| 236 | + monthComplete = true; | |
| 237 | + break; | |
| 238 | + } | |
| 239 | + page++; | |
| 240 | + index = 0; | |
| 241 | + await ctx.setCursor({ doneMonths: [...done], current: { month, page, index }, updatedAt: new Date().toISOString() }); | |
| 242 | + } | |
| 243 | + if (monthComplete) { | |
| 244 | + done.add(month); | |
| 245 | + reachedDate = monthYearDate(month); | |
| 246 | + await ctx.setCursor({ doneMonths: [...done], current: null, updatedAt: new Date().toISOString() }); | |
| 247 | + } | |
| 248 | + await ctx.progress({ page: months.filter((m) => done.has(m)).length, totalPages: months.length, itemsProcessed: items, reachedDate, cursor: { doneMonths: [...done], current: monthComplete ? null : { month, page, index } } }); | |
| 249 | + } | |
| 250 | + if (mode === 'backfill' && months.every((m) => done.has(m))) await ctx.setCursor({ doneMonths: [...done], current: null, done: true, updatedAt: new Date().toISOString() }); | |
| 251 | + } | |
| 252 | + | |
| 253 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 254 | + const p = LotPayloadSchema.parse(raw.payload); | |
| 255 | + if (p.price === null || p.price <= 0) return []; | |
| 256 | + const saleDate = monthYearDate(p.month); | |
| 257 | + if (!saleDate) return []; | |
| 258 | + const g = saleGrade(p.title); | |
| 259 | + const graded = Boolean(g.grader && g.grade); | |
| 260 | + const slug = /\b(signed|autograph)/i.test(p.title) && !/\b(baseball|football|basketball|hockey|boxing|golf|tennis|nascar|wrestling|topps|bowman|fleer|card|ball|bat|jersey|program|ticket|photo)\b/i.test(p.title) ? 'autographs' : cleanSweepCategory(p.title); | |
| 261 | + const attributes = lotAttributes({ categorySlug: slug, name: p.title, year: safeYear(p.title), identifiers: { clean_sweep_lot_slug: `${p.month}/${p.slug}` }, metadata: { auction_month: p.month, auction_label: p.monthLabel, sale_date_precision: 'month', buyers_premium_pct: BUYERS_PREMIUM_PCT, hammer_price: p.price } }); | |
| 262 | + const sale = makeSale({ meta: this.meta, sourceUrl: p.url, externalId: `${p.month}/${p.slug}`, rawTitle: p.title, description: p.description, attributes, price: p.price, currency: 'USD', saleDate, buyerPremiumIncluded: false, auctionHouse: HOUSE, imageUrls: p.image ? [p.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'US', isBundle: isBundle(p.title), confidence: graded ? 0.75 : 0.7 }); | |
| 263 | + sale.grade.qualifier = g.qualifier; | |
| 264 | + sale.grade.certificationNumber = certFromTitle(p.title); | |
| 265 | + return [sale]; | |
| 266 | + } | |
| 267 | +} | |
| 268 | + | |
| 269 | +export default (meta: ConnectorMeta) => new CleanSweepConnector(meta); | |
added
connectors/api/clean-sweep/meta.json
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +{ | |
| 2 | + "id": "clean-sweep", | |
| 3 | + "displayName": "Clean Sweep Auctions (past results)", | |
| 4 | + "sourceId": "clean-sweep", | |
| 5 | + "sourceName": "Clean Sweep Auctions", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.cleansweepauctions.com", | |
| 8 | + "module": "api/clean-sweep", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "other_sports_cards", "sports_memorabilia", "autographs"], | |
| 11 | + "regions": ["US"], | |
| 12 | + "country": "US", | |
| 13 | + "languages": ["en"], | |
| 14 | + "currency": ["USD"], | |
| 15 | + "supportsListings": false, | |
| 16 | + "supportsSold": true, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 1440, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.8, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.cleansweepauctions.com/auction-rules", | |
| 27 | + "acquisitionMethod": "server-rendered past-auction pages (HTML)", | |
| 28 | + "historicalDepth": "decades", | |
| 29 | + "accessNotes": "Plain HTTPS with the RareIndex user agent on www.cleansweepauctions.com (robots.txt: Disallow only account/cart/checkout/forum/customer paths, no Crawl-delay; we keep 2 s between requests). Pages read: /past-auctions (one entry per closed monthly auction, April 2008 → present, ~200 months), /past-auctions/<month-year>?page=N (50 lot links per page, server-rendered) and each lot page /past-auctions/<month-year>/<slug>, whose 'Winning bid: $X' is the hammer price; the auction rules add a 22 % buyer's premium on top, so sales are stored with buyerPremiumIncluded=false and buyers_premium_pct=22 in metadata. The archive dates lots only by auction month: saleDate is the first day of that month (UTC) with metadata.sale_date_precision='month' and confidence 0.7. Titles often end in a bare grade number without a grading company; no grader is inferred in that case. The live-auction host marketplace.cleansweepauctions.com (Crawl-delay 15, AJAX price grid, bidder accounts) is never crawled — only its public image URLs are referenced. 0 credits.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "config": { | |
| 33 | + "lotsPerRun": 150, | |
| 34 | + "pagesPerRun": 4 | |
| 35 | + } | |
| 36 | +} | |
added
connectors/api/colonial-acres/README.md
+58 −0
@@ -0,0 +1,58 @@ | ||
| 1 | +# Colonial Acres Coins connector (`colonial-acres`) | |
| 2 | + | |
| 3 | +- Source: https://www.colonialacres.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Kitchener numismatic dealer (Canadian decimal coins, ICCS-certified coins, Bank of Canada / Dominion notes, tokens). Shopify storefront; product_type distinguishes coins, ICCS slabs and paper money. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `1-cent` | `coins` | — | | |
| 14 | +| `5-cents` | `coins` | — | | |
| 15 | +| `10-cents` | `coins` | — | | |
| 16 | +| `25-cents` | `coins` | — | | |
| 17 | +| `50-cents` | `coins` | — | | |
| 18 | +| `dollar` | `coins` | — | | |
| 19 | +| `canada-deals-on-decimal` | `coins` | — | | |
| 20 | +| `usa-deals-on-decimal` | `coins` | — | | |
| 21 | +| `world-deals-on-decimal` | `coins` | — | | |
| 22 | +| `canada-proof-silver-dollars-1971-date` | `coins` | — | | |
| 23 | +| `bank-tokens` | `coins` | — | | |
| 24 | +| `province-of-canada` | `coins` | — | | |
| 25 | +| `lower-canada` | `coins` | — | | |
| 26 | +| `canada-tokens-medallions` | `medals` | — | | |
| 27 | +| `deals-on-paper-money` | `banknotes` | — | | |
| 28 | +| `1-notes-1935-1937` | `banknotes` | — | | |
| 29 | +| `1-notes-1973` | `banknotes` | — | | |
| 30 | +| `2-notes-1935-1954` | `banknotes` | — | | |
| 31 | +| `2-notes-1974` | `banknotes` | — | | |
| 32 | +| `2-notes-1986` | `banknotes` | — | | |
| 33 | +| `5-notes-1935-1954` | `banknotes` | — | | |
| 34 | +| `5-notes-2001-date` | `banknotes` | — | | |
| 35 | +| `10-notes-1935-1954` | `banknotes` | — | | |
| 36 | +| `10-notes-1971-1989` | `banknotes` | — | | |
| 37 | +| `20-notes-1935-1979` | `banknotes` | — | | |
| 38 | +| `20-notes-1991` | `banknotes` | — | | |
| 39 | +| `50-notes-1935-date` | `banknotes` | — | | |
| 40 | +| `100-notes-1935-date` | `banknotes` | — | | |
| 41 | +| `bc-37` | `banknotes` | — | | |
| 42 | +| `bc-56a` | `banknotes` | — | | |
| 43 | +| `bc-56b` | `banknotes` | — | | |
| 44 | +| `bc-56c` | `banknotes` | — | | |
| 45 | + | |
| 46 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 47 | +- `\| (Paper Money|Deals on Paper|Certified Paper Money|Paper Money Sets) \||\bBC-\d|\$\d+ Notes? \d{4}|\bbanknotes?\b|\bpaper money\b` → `banknotes` | |
| 48 | +- `\bmedal(lion)?s?\b` → `medals` | |
| 49 | +- `\btokens?\b` → `coins` | |
| 50 | + | |
| 51 | +Excluded (regex): `gift card|supplies|\balbums?\b|\bholders?\b|\bcapsules?\b|catalogue|catalog|magnifier|cleaning|\bflips?\b|\bpages?\b|\brolls?\b|\bbooks?\b|\btubes?\b|\bcases?\b|storage|\bbinder|\bloupe\b|\bgloves?\b|\bscale\b|\bmaps?\b|\bframe` | |
| 52 | + | |
| 53 | +## Access & compliance | |
| 54 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: bullion (silver/copper rounds, bars, bank rolls), RCM year/complete sets and gift sets, coin supplies, catalogues. | |
| 55 | + | |
| 56 | +## Fixtures & tests | |
| 57 | +`data/fixtures/colonial-acres/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts colonial-acres`), trimmed single-product payloads incl. a sold-out variant and a graded item. | |
| 58 | +`pnpm vitest run connectors/api/colonial-acres` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts colonial-acres`. | |
added
connectors/api/colonial-acres/index.test.ts
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Colonial Acres Coins — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/colonial-acres/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "BC-1 1935 Canada $1 Osborne-Towers, English, Series A, Fine (F12)", | |
| 15 | + "productType": "Paper Money", | |
| 16 | + "collection": "1-notes-1935-1937", | |
| 17 | + "categorySlug": "banknotes" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "1858 Broken Stem Canada 1-cent ICCS Certified VG10", | |
| 21 | + "productType": "ICCS", | |
| 22 | + "collection": "1-cent", | |
| 23 | + "categorySlug": "coins" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "title": "Canada 1-cent Roll (50 pcs) 1967", | |
| 27 | + "collection": "1-cent", | |
| 28 | + "categorySlug": null | |
| 29 | + }, | |
| 30 | + { | |
| 31 | + "title": "Coin Capsule 19mm - 10 pack", | |
| 32 | + "collection": "1-cent", | |
| 33 | + "categorySlug": null | |
| 34 | + }, | |
| 35 | + { | |
| 36 | + "title": "1820 Lower Canada Bust & Harp Token", | |
| 37 | + "collection": "lower-canada", | |
| 38 | + "categorySlug": "coins" | |
| 39 | + } | |
| 40 | + ] | |
| 41 | +}); | |
added
connectors/api/colonial-acres/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Colonial Acres Coins — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/colonial-acres/meta.json
+200 −0
@@ -0,0 +1,200 @@ | ||
| 1 | +{ | |
| 2 | + "id": "colonial-acres", | |
| 3 | + "displayName": "Colonial Acres Coins (Canadian coin & banknote store, CAD)", | |
| 4 | + "sourceId": "colonial-acres", | |
| 5 | + "sourceName": "Colonial Acres Coins", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.colonialacres.com", | |
| 8 | + "module": "api/colonial-acres", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "coins", | |
| 14 | + "medals", | |
| 15 | + "banknotes" | |
| 16 | + ], | |
| 17 | + "regions": [ | |
| 18 | + "CA" | |
| 19 | + ], | |
| 20 | + "languages": [ | |
| 21 | + "en" | |
| 22 | + ], | |
| 23 | + "currency": [ | |
| 24 | + "CAD" | |
| 25 | + ], | |
| 26 | + "supportsListings": true, | |
| 27 | + "supportsSold": false, | |
| 28 | + "supportsAuctions": false, | |
| 29 | + "supportsImages": true, | |
| 30 | + "supportsCatalog": false, | |
| 31 | + "supportsPopulation": false, | |
| 32 | + "supportsLookup": true, | |
| 33 | + "refreshFrequencyMinutes": 720, | |
| 34 | + "priority": "medium", | |
| 35 | + "trustScore": 0.75, | |
| 36 | + "attributionRequired": true, | |
| 37 | + "termsUrl": "https://www.colonialacres.com/policies/terms-of-service", | |
| 38 | + "accessNotes": "Colonial Acres Coins (colonialacres.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 32 configured collections (1-cent, 5-cents, 10-cents, 25-cents, 50-cents, dollar, canada-deals-on-decimal, usa-deals-on-decimal … (+24 more, see config.collections)) and /products/<handle>.json for URL lookups (~15k Canadian decimal coins by denomination, ~2.5k Bank of Canada notes). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed; extra merchant rules only target Nutch (Disallow: /) and Ahrefs/MJ12 (Crawl-delay: 10), not User-agent *. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Third-party grades in titles (PSA/BGS/CGC/ICCS/PMG…) are parsed by parseGradeFromTitle; cert numbers are not extracted. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — bullion (silver/copper rounds, bars, bank rolls), RCM year/complete sets and gift sets, coin supplies, catalogues. No personal data is collected; seller = the store itself.", | |
| 39 | + "enabled": true, | |
| 40 | + "schemaVersion": "1.0", | |
| 41 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 42 | + "historicalDepth": "none", | |
| 43 | + "requires": [], | |
| 44 | + "config": { | |
| 45 | + "currency": "CAD", | |
| 46 | + "market": "CA", | |
| 47 | + "seller": "Colonial Acres Coins", | |
| 48 | + "location": "Kitchener, ON, Canada", | |
| 49 | + "collections": [ | |
| 50 | + { | |
| 51 | + "handle": "1-cent", | |
| 52 | + "categorySlug": "coins" | |
| 53 | + }, | |
| 54 | + { | |
| 55 | + "handle": "5-cents", | |
| 56 | + "categorySlug": "coins" | |
| 57 | + }, | |
| 58 | + { | |
| 59 | + "handle": "10-cents", | |
| 60 | + "categorySlug": "coins" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "25-cents", | |
| 64 | + "categorySlug": "coins" | |
| 65 | + }, | |
| 66 | + { | |
| 67 | + "handle": "50-cents", | |
| 68 | + "categorySlug": "coins" | |
| 69 | + }, | |
| 70 | + { | |
| 71 | + "handle": "dollar", | |
| 72 | + "categorySlug": "coins" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "handle": "canada-deals-on-decimal", | |
| 76 | + "categorySlug": "coins" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "usa-deals-on-decimal", | |
| 80 | + "categorySlug": "coins" | |
| 81 | + }, | |
| 82 | + { | |
| 83 | + "handle": "world-deals-on-decimal", | |
| 84 | + "categorySlug": "coins" | |
| 85 | + }, | |
| 86 | + { | |
| 87 | + "handle": "canada-proof-silver-dollars-1971-date", | |
| 88 | + "categorySlug": "coins" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "handle": "bank-tokens", | |
| 92 | + "categorySlug": "coins" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "handle": "province-of-canada", | |
| 96 | + "categorySlug": "coins" | |
| 97 | + }, | |
| 98 | + { | |
| 99 | + "handle": "lower-canada", | |
| 100 | + "categorySlug": "coins" | |
| 101 | + }, | |
| 102 | + { | |
| 103 | + "handle": "canada-tokens-medallions", | |
| 104 | + "categorySlug": "medals" | |
| 105 | + }, | |
| 106 | + { | |
| 107 | + "handle": "deals-on-paper-money", | |
| 108 | + "categorySlug": "banknotes" | |
| 109 | + }, | |
| 110 | + { | |
| 111 | + "handle": "1-notes-1935-1937", | |
| 112 | + "categorySlug": "banknotes" | |
| 113 | + }, | |
| 114 | + { | |
| 115 | + "handle": "1-notes-1973", | |
| 116 | + "categorySlug": "banknotes" | |
| 117 | + }, | |
| 118 | + { | |
| 119 | + "handle": "2-notes-1935-1954", | |
| 120 | + "categorySlug": "banknotes" | |
| 121 | + }, | |
| 122 | + { | |
| 123 | + "handle": "2-notes-1974", | |
| 124 | + "categorySlug": "banknotes" | |
| 125 | + }, | |
| 126 | + { | |
| 127 | + "handle": "2-notes-1986", | |
| 128 | + "categorySlug": "banknotes" | |
| 129 | + }, | |
| 130 | + { | |
| 131 | + "handle": "5-notes-1935-1954", | |
| 132 | + "categorySlug": "banknotes" | |
| 133 | + }, | |
| 134 | + { | |
| 135 | + "handle": "5-notes-2001-date", | |
| 136 | + "categorySlug": "banknotes" | |
| 137 | + }, | |
| 138 | + { | |
| 139 | + "handle": "10-notes-1935-1954", | |
| 140 | + "categorySlug": "banknotes" | |
| 141 | + }, | |
| 142 | + { | |
| 143 | + "handle": "10-notes-1971-1989", | |
| 144 | + "categorySlug": "banknotes" | |
| 145 | + }, | |
| 146 | + { | |
| 147 | + "handle": "20-notes-1935-1979", | |
| 148 | + "categorySlug": "banknotes" | |
| 149 | + }, | |
| 150 | + { | |
| 151 | + "handle": "20-notes-1991", | |
| 152 | + "categorySlug": "banknotes" | |
| 153 | + }, | |
| 154 | + { | |
| 155 | + "handle": "50-notes-1935-date", | |
| 156 | + "categorySlug": "banknotes" | |
| 157 | + }, | |
| 158 | + { | |
| 159 | + "handle": "100-notes-1935-date", | |
| 160 | + "categorySlug": "banknotes" | |
| 161 | + }, | |
| 162 | + { | |
| 163 | + "handle": "bc-37", | |
| 164 | + "categorySlug": "banknotes" | |
| 165 | + }, | |
| 166 | + { | |
| 167 | + "handle": "bc-56a", | |
| 168 | + "categorySlug": "banknotes" | |
| 169 | + }, | |
| 170 | + { | |
| 171 | + "handle": "bc-56b", | |
| 172 | + "categorySlug": "banknotes" | |
| 173 | + }, | |
| 174 | + { | |
| 175 | + "handle": "bc-56c", | |
| 176 | + "categorySlug": "banknotes" | |
| 177 | + } | |
| 178 | + ], | |
| 179 | + "rules": [ | |
| 180 | + { | |
| 181 | + "match": "\\| (Paper Money|Deals on Paper|Certified Paper Money|Paper Money Sets) \\||\\bBC-\\d|\\$\\d+ Notes? \\d{4}|\\bbanknotes?\\b|\\bpaper money\\b", | |
| 182 | + "categorySlug": "banknotes" | |
| 183 | + }, | |
| 184 | + { | |
| 185 | + "match": "\\bmedal(lion)?s?\\b", | |
| 186 | + "categorySlug": "medals" | |
| 187 | + }, | |
| 188 | + { | |
| 189 | + "match": "\\btokens?\\b", | |
| 190 | + "categorySlug": "coins" | |
| 191 | + } | |
| 192 | + ], | |
| 193 | + "defaultCategory": null, | |
| 194 | + "exclude": "gift card|supplies|\\balbums?\\b|\\bholders?\\b|\\bcapsules?\\b|catalogue|catalog|magnifier|cleaning|\\bflips?\\b|\\bpages?\\b|\\brolls?\\b|\\bbooks?\\b|\\btubes?\\b|\\bcases?\\b|storage|\\bbinder|\\bloupe\\b|\\bgloves?\\b|\\bscale\\b|\\bmaps?\\b|\\bframe", | |
| 195 | + "keepOutOfStock": true, | |
| 196 | + "fetchBarcodes": false, | |
| 197 | + "wholeShop": false, | |
| 198 | + "pageSize": 250 | |
| 199 | + } | |
| 200 | +} | |
added
connectors/api/corinphila/README.md
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +# corinphila — Corinphila Auctions stamp hammer prices | |
| 2 | + | |
| 3 | +Zürich stamp auction house; static archive (c4ms) of auctions 12–28 (2016–2024) with hammer prices in CHF. | |
| 4 | + | |
| 5 | +- **Engine**: `api` (static HTML), 10 s politeness (robots Crawl-Delay 10), ≤ 20 pages per run. | |
| 6 | +- **Flow**: archive list → auction overview (`c4msEnv.auctionData` JSON: currency, dates, status; catalogue parts with lot counts) → lot lists `…&action=showLots&auctionID=N&catalogPart=P&show_all_lots=1&page=K` (100 lots/page). | |
| 7 | +- **Records**: `sale` per sold lot — hammer CHF (`buyerPremiumIncluded=false`), starting bid in metadata, sale date = auction start date, country label → ISO where known, `identifiers.corinphila_lot = <auctionId>/<lotNo>`; "not sold" lots skipped. | |
| 8 | +- **Cursor**: `{ doneAuctions, inProgress: { auction, parts, partIndex, page }, done }`. | |
| 9 | +- **Not fetched**: live catalogue on auction.corinphila.ch (JS/ajax), PDF result lists. | |
added
connectors/api/corinphila/_smoke.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import path from 'node:path'; | |
| 2 | +import { fileURLToPath } from 'node:url'; | |
| 3 | +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js'; | |
| 4 | + | |
| 5 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 6 | +if (process.argv[2] === '--capture') await captureFixture(dir, process.argv[3] ?? 'category-page', Number(process.argv[4] ?? 6)); | |
| 7 | +else await runSmoke(dir, Number(process.argv[2] ?? 2)); | |
added
connectors/api/corinphila/index.test.ts
+59 −0
@@ -0,0 +1,59 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import { fileURLToPath } from 'node:url'; | |
| 5 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 7 | +import createConnector, { lotsUrl, parseArchive, parseLotsPage, parseOverview } from './index.js'; | |
| 8 | + | |
| 9 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 10 | +const connector = createConnector(localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')))); | |
| 11 | + | |
| 12 | +const ARCHIVE = `<div class="auctionBox"><div class="pic"><a class="noGraphic" href="/en/_auctions/&action=showAuctionOverview&auctionID=28"><img src="c.jpg"></a></div><div class="infos"><h4>Auction 321-332</h4><div class="date">June 3,2024 - June 8,2024</div><div class="links"><a href="/en/_auctions/&action=showAuctionOverview&auctionID=28">Show Auction Catalogue</a></div></div></div> | |
| 13 | +<div class="auctionBox"><div class="infos"><h4>Auction 291-297</h4><div class="date">November 28,2022 - December 3,2022</div><div class="links"><a href="/en/_auctions/&action=showAuctionOverview&auctionID=24">Show Auction Catalogue</a></div></div></div>`; | |
| 14 | +const OVERVIEW = `<script>c4msEnv.auctionData = {"id":"24","currency":"CHF","no":"0","startDate":"2022-11-28 14:34:07","endDate":"2022-12-03 14:34:07","status":"closed","name":"Auction 291-297","description":null};</script><h1>Auction 291-297</h1> | |
| 15 | +<div class="bookmark_left_red"><h3>Catalogue 291: Europe & Overseas</h3></div><div class="countryBox"><table><tr class="row_even groupTR"><td><a href="/en/_auctions/&action=showLots&auctionID=24&catalogPart=199&show_all_lots=1" class="noIconLink">Show all lots</a></td><td class="lotCounterWrapper"><a href="/en/_auctions/&action=showLots&auctionID=24&catalogPart=199&show_all_lots=1" class="noIconLink lotCounter">1636</a></td></tr></table></div> | |
| 16 | +<div class="bookmark_left_red"><h3>Catalogue 294: Ceylon</h3></div><div class="countryBox"><table><tr><td><a href="/en/_auctions/&action=showLots&auctionID=24&catalogPart=194&show_all_lots=1" class="noIconLink">Show all lots</a></td><td><a href="/en/_auctions/&action=showLots&auctionID=24&catalogPart=194&show_all_lots=1" class="noIconLink lotCounter">744</a></td></tr></table></div>`; | |
| 17 | +const LOTS = `<div class="pagination"><div class="pageCounter">pages (<span class="pageCounterLabel">8</span>):</div></div> | |
| 18 | +<div class="lot" data-lotNo="6001"><div class="bookmark_left_red"><h3>Lot# : <a href="/en/_auctions/&action=showLot&auctionID=24&lotno=6001"><span class="lotno lotTitle">6001</span></a><span class="lotCountry">Ceylon</span></h3></div><div class="lotBox"><div class="lotPic"><div class="picContainer"><ul><li><a href="https://d2xqn5t7wr4wg1.cloudfront.net/modules/auctions/24/pics/big/a.jpg"><img data-src="https://d2xqn5t7wr4wg1.cloudfront.net/modules/auctions/24/pics/small/a.jpg" class="lazyload"></a></li></ul></div><span class="lot-cond lot-cond-8"></span><span class="lot-cond lot-cond-9"></span></div><div class="lotDesc"><div class="datatext"><span class="text">Recess by Perkins Bacon 1856/57: Proof for the initial issue, 6 d. black imperforate, a sheet marginal block of four (SG 1).</span></div><div class="lotAttr"><div class="prices"><div class="start">Starting bid : <span class="value"> 200.00 CHF </span></div><div class="bid">Hammer price : <span class="value">260.00 CHF</span></div></div></div></div></div></div> | |
| 19 | +<div class="lot" data-lotNo="6015"><div class="bookmark_left_red"><h3>Lot# : <span class="lotno lotTitle">6015</span><span class="lotCountry">Ceylon</span></h3></div><div class="lotDesc"><div class="datatext"><span class="text">1857 4 d. dull rose, unused without gum, a fine example. Cert. RPSL (1973).</span></div><div class="lotAttr"><div class="prices"><div class="start">Starting bid : <span class="value">200.00 CHF</span></div><div class="bid">Hammer price : <span class="value">not sold </span></div></div></div></div></div>`; | |
| 20 | + | |
| 21 | +describe('corinphila', () => { | |
| 22 | + runFixtureSuite(connector, it, expect); | |
| 23 | + | |
| 24 | + it('parses the archive list and the auction overview (auctionData JSON + catalogue parts)', () => { | |
| 25 | + expect(parseArchive(ARCHIVE)).toEqual([ | |
| 26 | + { id: '28', name: 'Auction 321-332', dateText: 'June 3,2024 - June 8,2024' }, | |
| 27 | + { id: '24', name: 'Auction 291-297', dateText: 'November 28,2022 - December 3,2022' }, | |
| 28 | + ]); | |
| 29 | + const ov = parseOverview(OVERVIEW, '24')!; | |
| 30 | + expect(ov.auction).toEqual({ id: '24', name: 'Auction 291-297', currency: 'CHF', startDate: '2022-11-28 14:34:07', endDate: '2022-12-03 14:34:07', status: 'closed' }); | |
| 31 | + expect(ov.parts).toEqual([{ catalogPart: '199', title: 'Catalogue 291: Europe & Overseas', lotCount: 1636 }, { catalogPart: '194', title: 'Catalogue 294: Ceylon', lotCount: 744 }]); | |
| 32 | + expect(lotsUrl('24', '194', 2)).toBe('https://corinphila.ch/en/_auctions/&action=showLots&auctionID=24&catalogPart=194&show_all_lots=1&page=2'); | |
| 33 | + }); | |
| 34 | + | |
| 35 | + it('parses lot blocks with hammer / not sold and normalises CHF hammer sales', async () => { | |
| 36 | + const ov = parseOverview(OVERVIEW, '24')!; | |
| 37 | + const p = parseLotsPage(LOTS, ov.auction, ov.parts[1]!, lotsUrl('24', '194', 1)); | |
| 38 | + expect(p.totalPages).toBe(8); | |
| 39 | + expect(p.lots.length).toBe(2); | |
| 40 | + expect(p.lots[0]).toMatchObject({ lotNo: '6001', country: 'Ceylon', startText: '200.00 CHF', hammerText: '260.00 CHF', conditionCodes: ['8', '9'], images: ['https://d2xqn5t7wr4wg1.cloudfront.net/modules/auctions/24/pics/small/a.jpg'] }); | |
| 41 | + expect(p.lots[1]!.hammerText).toBe('not sold'); | |
| 42 | + const out = await connector.normalize({ url: p.url, externalId: 'x', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload: p }); | |
| 43 | + expect(out.length).toBe(1); | |
| 44 | + const s = out[0]!; | |
| 45 | + if (s.kind !== 'sale') throw new Error('sale'); | |
| 46 | + expect(s).toMatchObject({ price: 260, currency: 'CHF', buyerPremiumIncluded: false, lotNumber: '6001', auctionHouse: 'Corinphila Auctions', location: 'CH', sourceUrl: 'https://corinphila.ch/en/_auctions/&action=showLot&auctionID=24&lotno=6001' }); | |
| 47 | + expect(s.saleDate.toISOString()).toBe('2022-11-28T00:00:00.000Z'); | |
| 48 | + expect(s.attributes).toMatchObject({ categorySlug: 'stamps', country: 'LK', set: 'Catalogue 294: Ceylon' }); | |
| 49 | + expect(s.attributes.identifiers.corinphila_lot).toBe('24/6001'); | |
| 50 | + expect(s.attributes.metadata).toMatchObject({ starting_bid: 200, michel_or_sg: 'SG 1' }); | |
| 51 | + }); | |
| 52 | + | |
| 53 | + it('fixture normalises to CHF hammer sales and skips not-sold lots', async () => { | |
| 54 | + const fx = loadFixture('corinphila', 'lots-page-auction-24-part-194'); | |
| 55 | + const out = await connector.normalize(fx.raw); | |
| 56 | + expect(out.length).toBe(8); | |
| 57 | + for (const r of out) if (r.kind === 'sale') expect(r).toMatchObject({ currency: 'CHF', buyerPremiumIncluded: false }); | |
| 58 | + }); | |
| 59 | +}); | |
added
connectors/api/corinphila/index.ts
+232 −0
@@ -0,0 +1,232 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared'; | |
| 4 | +import { makeSale } from '../../firecrawl/_carlib/index.js'; | |
| 5 | +import { clean, isNumisBundle, numisAttributes, parseAuctionDate, realized } from '../../firecrawl/_g5-numismatics-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Corinphila Auctions (Zürich) — Switzerland's oldest stamp auction house. The archive on corinphila.ch | |
| 9 | + * (c4ms platform) is static HTML: auction overview (catalogue parts with lot counts, c4msEnv.auctionData | |
| 10 | + * JSON with currency and dates) → lot list pages of 100 lots with country, description, starting bid and | |
| 11 | + * "Hammer price : 260.00 CHF" (or "not sold"). Live catalogues (auction.corinphila.ch) are JS/ajax and | |
| 12 | + * are not used. robots: Crawl-Delay 10 → 10 s between requests. | |
| 13 | + */ | |
| 14 | + | |
| 15 | +const SITE = 'https://corinphila.ch'; | |
| 16 | +const PARSER_VERSION = '1.0.0'; | |
| 17 | + | |
| 18 | +export const AuctionSchema = z.object({ id: z.string(), name: z.string(), currency: z.string().nullable(), startDate: z.string().nullable(), endDate: z.string().nullable(), status: z.string().nullable() }); | |
| 19 | +export const PartSchema = z.object({ catalogPart: z.string(), title: z.string().nullable(), lotCount: z.number().int().nullable() }); | |
| 20 | +export const LotSchema = z.object({ | |
| 21 | + lotNo: z.string(), | |
| 22 | + country: z.string().nullable(), | |
| 23 | + description: z.string(), | |
| 24 | + startText: z.string().nullable(), | |
| 25 | + hammerText: z.string().nullable(), | |
| 26 | + conditionCodes: z.array(z.string()).default([]), | |
| 27 | + images: z.array(z.string()).default([]), | |
| 28 | +}); | |
| 29 | +export const PayloadSchema = z.object({ kind: z.literal('lots_page'), auction: AuctionSchema, part: PartSchema, url: z.string(), page: z.number().int(), totalPages: z.number().int().nullable(), lots: z.array(LotSchema) }); | |
| 30 | +export type Payload = z.infer<typeof PayloadSchema>; | |
| 31 | + | |
| 32 | +/** Archive page (…&action=show&id=211) → auction ids with printed names/dates. */ | |
| 33 | +export function parseArchive(htmlText: string): Array<{ id: string; name: string; dateText: string | null }> { | |
| 34 | + const $ = H.load(htmlText); | |
| 35 | + const out: Array<{ id: string; name: string; dateText: string | null }> = []; | |
| 36 | + $('.auctionBox').each((_, box) => { | |
| 37 | + const e = $(box); | |
| 38 | + const id = e.find('a[href*="showAuctionOverview"]').first().attr('href')?.match(/auctionID=(\d+)/)?.[1]; | |
| 39 | + if (!id || out.some((a) => a.id === id)) return; | |
| 40 | + const name = clean(e.find('h4').first().text()); | |
| 41 | + const dateText = clean(e.find('.date').first().text()) || null; | |
| 42 | + if (name) out.push({ id, name, dateText }); | |
| 43 | + }); | |
| 44 | + return out; | |
| 45 | +} | |
| 46 | + | |
| 47 | +/** Auction overview → auctionData JSON + catalogue parts ("Show all lots" links carry the lot count). */ | |
| 48 | +export function parseOverview(htmlText: string, id: string): { auction: z.infer<typeof AuctionSchema>; parts: z.infer<typeof PartSchema>[] } | null { | |
| 49 | + const m = htmlText.match(/c4msEnv\.auctionData\s*=\s*(\{[\s\S]*?\});/); | |
| 50 | + let data: { currency?: string; startDate?: string; endDate?: string; status?: string; name?: string } = {}; | |
| 51 | + if (m) { | |
| 52 | + try { | |
| 53 | + data = JSON.parse(m[1]!) as typeof data; | |
| 54 | + } catch { | |
| 55 | + data = {}; | |
| 56 | + } | |
| 57 | + } | |
| 58 | + const $ = H.load(htmlText); | |
| 59 | + const name = clean(data.name ?? '') || clean($('h1').first().text()); | |
| 60 | + if (!name) return null; | |
| 61 | + const parts: z.infer<typeof PartSchema>[] = []; | |
| 62 | + $('.bookmark_left_red').each((_, hdr) => { | |
| 63 | + const title = clean($(hdr).find('h3').first().text()) || null; | |
| 64 | + const box = $(hdr).nextAll('.countryBox').first(); | |
| 65 | + const link = box.find(`a[href*="action=showLots"][href*="auctionID=${id}"][href*="show_all_lots=1"]`).first(); | |
| 66 | + const cp = link.attr('href')?.match(/catalogPart=(\d+)/)?.[1]; | |
| 67 | + if (!cp || parts.some((p) => p.catalogPart === cp)) return; | |
| 68 | + const count = Number(clean(box.find('a.lotCounter').first().text()).replace(/\D/g, '')); | |
| 69 | + parts.push({ catalogPart: cp, title, lotCount: Number.isFinite(count) && count > 0 ? count : null }); | |
| 70 | + }); | |
| 71 | + return { auction: { id, name, currency: data.currency ?? null, startDate: data.startDate ?? null, endDate: data.endDate ?? null, status: data.status ?? null }, parts }; | |
| 72 | +} | |
| 73 | + | |
| 74 | +export function lotsUrl(auctionId: string, catalogPart: string, page: number): string { | |
| 75 | + return `${SITE}/en/_auctions/&action=showLots&auctionID=${auctionId}&catalogPart=${catalogPart}&show_all_lots=1&page=${page}`; | |
| 76 | +} | |
| 77 | + | |
| 78 | +/** Lot list page → lots + page count. */ | |
| 79 | +export function parseLotsPage(htmlText: string, auction: z.infer<typeof AuctionSchema>, part: z.infer<typeof PartSchema>, url: string): Payload { | |
| 80 | + const $ = H.load(htmlText); | |
| 81 | + const page = Number(url.match(/[?&]page=(\d+)/)?.[1] ?? 1); | |
| 82 | + const totalPagesText = clean($('.pageCounterLabel').first().text()); | |
| 83 | + const lots: z.infer<typeof LotSchema>[] = []; | |
| 84 | + $('div.lot[data-lotno], div.lot[data-lotNo]').each((_, el) => { | |
| 85 | + const e = $(el); | |
| 86 | + const lotNo = e.attr('data-lotno') ?? e.attr('data-lotNo') ?? clean(e.find('.lotno').first().text()); | |
| 87 | + const description = clean(e.find('.lotDesc .text').first().text()); | |
| 88 | + if (!lotNo || !description) return; | |
| 89 | + const start = clean(e.find('.prices .start .value').first().text()) || null; | |
| 90 | + const hammer = clean(e.find('.prices .bid .value').first().text()) || null; | |
| 91 | + const images = e.find('.picContainer img[data-src]').map((__, img) => $(img).attr('data-src') ?? '').get().filter(Boolean); | |
| 92 | + const conditionCodes = e.find('.lot-cond').map((__, c) => ($(c).attr('class') ?? '').match(/lot-cond-(\d+)/)?.[1] ?? '').get().filter(Boolean); | |
| 93 | + lots.push({ lotNo, country: clean(e.find('.lotCountry').first().text()) || null, description, startText: start, hammerText: hammer, conditionCodes, images: images.slice(0, 3) }); | |
| 94 | + }); | |
| 95 | + return { kind: 'lots_page', auction, part, url, page, totalPages: totalPagesText ? Number(totalPagesText) : null, lots }; | |
| 96 | +} | |
| 97 | + | |
| 98 | +interface Cursor { | |
| 99 | + doneAuctions?: string[]; | |
| 100 | + inProgress?: { auction: z.infer<typeof AuctionSchema>; parts: z.infer<typeof PartSchema>[]; partIndex: number; page: number } | null; | |
| 101 | + done?: boolean; | |
| 102 | + updatedAt?: string; | |
| 103 | +} | |
| 104 | + | |
| 105 | +export class CorinphilaConnector extends BaseConnector { | |
| 106 | + readonly version = '1.0.0'; | |
| 107 | + readonly parserVersion = PARSER_VERSION; | |
| 108 | + protected override minIntervalMs = 10_000; | |
| 109 | + | |
| 110 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 111 | + const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 20); | |
| 112 | + const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 1); | |
| 113 | + const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) }; | |
| 114 | + const done = new Set(cursor.doneAuctions ?? []); | |
| 115 | + let pages = 0; | |
| 116 | + let yielded = 0; | |
| 117 | + let finished = 0; | |
| 118 | + const text = (url: string, expect: Parameters<CrawlContext['fetch']>[1] = {}) => ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0, ...expect }); | |
| 119 | + | |
| 120 | + await this.throttle(); | |
| 121 | + const archive = await text(`${SITE}/en/_pages/&action=show&id=211`); | |
| 122 | + pages++; | |
| 123 | + const auctions = archive.success && archive.html ? parseArchive(archive.html) : []; | |
| 124 | + if (!auctions.length) { | |
| 125 | + ctx.anomaly(archive.success ? 'selector_missing' : 'page_fetch_failed', `archive: ${archive.error ?? archive.httpStatus ?? 'no auction boxes'}`); | |
| 126 | + return; | |
| 127 | + } | |
| 128 | + auctions.sort((a, b) => Number(b.id) - Number(a.id)); | |
| 129 | + const queue = [...(cursor.inProgress ? [cursor.inProgress.auction.id] : []), ...auctions.map((a) => a.id).filter((id) => !done.has(id) && id !== cursor.inProgress?.auction.id)]; | |
| 130 | + for (const id of queue) { | |
| 131 | + if (ctx.signal?.aborted || finished >= auctionsPerRun || pages >= pagesPerRun || this.reached(ctx, yielded)) break; | |
| 132 | + let state = cursor.inProgress?.auction.id === id ? cursor.inProgress : null; | |
| 133 | + if (!state) { | |
| 134 | + await this.throttle(); | |
| 135 | + const ov = await text(`${SITE}/en/_auctions/&action=showAuctionOverview&auctionID=${id}`); | |
| 136 | + pages++; | |
| 137 | + const parsed = ov.success && ov.html ? parseOverview(ov.html, id) : null; | |
| 138 | + if (!parsed || !parsed.parts.length) { | |
| 139 | + ctx.anomaly(ov.success ? 'parse_failure_page' : 'page_fetch_failed', `overview ${id}: ${ov.error ?? ov.httpStatus ?? 'no catalogue parts'}`); | |
| 140 | + if (ov.success) done.add(id); | |
| 141 | + continue; | |
| 142 | + } | |
| 143 | + if (parsed.auction.status && parsed.auction.status !== 'closed') continue; // running sale | |
| 144 | + state = { auction: parsed.auction, parts: parsed.parts, partIndex: 0, page: 1 }; | |
| 145 | + } | |
| 146 | + while (state.partIndex < state.parts.length && pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) { | |
| 147 | + const part = state.parts[state.partIndex]!; | |
| 148 | + const url = lotsUrl(id, part.catalogPart, state.page); | |
| 149 | + await this.throttle(); | |
| 150 | + const res = await text(url, { | |
| 151 | + expect: ['title', 'price'], | |
| 152 | + parse: (r) => { | |
| 153 | + const p = r.html ? parseLotsPage(r.html, state!.auction, part, url) : null; | |
| 154 | + return p?.lots.length ? { title: p.lots[0]!.description, price: p.lots.find((l) => l.hammerText && !/not sold/i.test(l.hammerText))?.hammerText ?? null } : null; | |
| 155 | + }, | |
| 156 | + }); | |
| 157 | + pages++; | |
| 158 | + if (!res.success || !res.html) { | |
| 159 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 160 | + break; | |
| 161 | + } | |
| 162 | + const payload = parseLotsPage(res.html, state.auction, part, url); | |
| 163 | + if (!payload.lots.length && state.page === 1) ctx.anomaly('parse_failure_page', `${url}: no lot blocks`); | |
| 164 | + if (payload.lots.some((l) => l.hammerText && !/not sold/i.test(l.hammerText))) { | |
| 165 | + yielded++; | |
| 166 | + yield { url, externalId: `auction:${id}:part:${part.catalogPart}:p${payload.page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 167 | + } | |
| 168 | + const more = payload.lots.length > 0 && payload.totalPages !== null && payload.page < payload.totalPages; | |
| 169 | + if (more) state.page++; | |
| 170 | + else { | |
| 171 | + state.partIndex++; | |
| 172 | + state.page = 1; | |
| 173 | + } | |
| 174 | + cursor.inProgress = state; | |
| 175 | + await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-100), updatedAt: new Date().toISOString() }); | |
| 176 | + } | |
| 177 | + if (state.partIndex >= state.parts.length) { | |
| 178 | + done.add(id); | |
| 179 | + finished++; | |
| 180 | + cursor.inProgress = null; | |
| 181 | + if (ctx.options.mode === 'backfill') await ctx.progress({ page: auctions.findIndex((a) => a.id === id) + 1, totalPages: auctions.length, itemsProcessed: yielded, reachedDate: parseAuctionDate(state.auction.startDate) }); | |
| 182 | + } | |
| 183 | + await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-100), updatedAt: new Date().toISOString() }); | |
| 184 | + } | |
| 185 | + if (ctx.options.mode === 'backfill' && auctions.every((a) => done.has(a.id))) await ctx.setCursor({ ...cursor, done: true, doneAuctions: [...done].slice(-100), updatedAt: new Date().toISOString() }); | |
| 186 | + } | |
| 187 | + | |
| 188 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 189 | + const p = PayloadSchema.parse(raw.payload); | |
| 190 | + const saleDate = parseAuctionDate(p.auction.startDate); | |
| 191 | + if (!saleDate) return []; | |
| 192 | + const out: NormalizedSale[] = []; | |
| 193 | + for (const lot of p.lots) { | |
| 194 | + if (!lot.hammerText || /not sold|withdrawn/i.test(lot.hammerText)) continue; | |
| 195 | + const price = realized(lot.hammerText, (p.auction.currency as 'CHF' | null) ?? 'CHF'); | |
| 196 | + if (!price) continue; | |
| 197 | + const start = realized(lot.startText, price.currency); | |
| 198 | + const title = lot.country ? `${lot.country}: ${lot.description}` : lot.description; | |
| 199 | + const attributes = numisAttributes({ | |
| 200 | + categorySlug: 'stamps', | |
| 201 | + title, | |
| 202 | + section: p.part.title, | |
| 203 | + identifiers: { corinphila_lot: `${p.auction.id}/${lot.lotNo}` }, | |
| 204 | + metadata: { auction_id: p.auction.id, auction_name: p.auction.name, catalogue_part: p.part.title, country_label: lot.country, starting_bid: start?.amount ?? null, hammer_price: price.amount, buyer_premium: "excluded — page label 'Hammer price'; Corinphila's premium is stated in the conditions of sale", condition_codes: lot.conditionCodes, michel_or_sg: lot.description.match(/\b(?:SG|Mi\.?|Michel|Zumstein|Yv\.?|Scott)\s*\d+[a-z]?/i)?.[0] ?? null }, | |
| 205 | + }); | |
| 206 | + const sale = makeSale({ | |
| 207 | + meta: this.meta, | |
| 208 | + sourceUrl: `${SITE}/en/_auctions/&action=showLot&auctionID=${p.auction.id}&lotno=${lot.lotNo}`, | |
| 209 | + externalId: `${p.auction.id}-${lot.lotNo}`, | |
| 210 | + rawTitle: title.length > 240 ? `${title.slice(0, 239)}…` : title, | |
| 211 | + description: lot.description, | |
| 212 | + attributes, | |
| 213 | + price: price.amount, | |
| 214 | + currency: price.currency, | |
| 215 | + saleDate, | |
| 216 | + buyerPremiumIncluded: false, | |
| 217 | + auctionHouse: 'Corinphila Auctions', | |
| 218 | + lotNumber: lot.lotNo, | |
| 219 | + imageUrls: lot.images, | |
| 220 | + observedAt: raw.fetchedAt, | |
| 221 | + parserVersion: PARSER_VERSION, | |
| 222 | + confidence: 0.85, | |
| 223 | + isBundle: isNumisBundle(lot.description) || /\b(collection|accumulation|lot of|group of|balance)\b/i.test(lot.description), | |
| 224 | + location: 'CH', | |
| 225 | + }); | |
| 226 | + out.push(sale); | |
| 227 | + } | |
| 228 | + return out; | |
| 229 | + } | |
| 230 | +} | |
| 231 | + | |
| 232 | +export default (meta: ConnectorMeta) => new CorinphilaConnector(meta); | |
added
connectors/api/corinphila/meta.json
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +{ | |
| 2 | + "id": "corinphila", | |
| 3 | + "displayName": "Corinphila Auctions (Zürich) — stamp hammer prices, archive 2016–2024", | |
| 4 | + "sourceId": "corinphila", | |
| 5 | + "sourceName": "Corinphila Auktionen AG", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://corinphila.ch", | |
| 8 | + "module": "api/corinphila", | |
| 9 | + "enginePriority": ["api", "firecrawl"], | |
| 10 | + "categories": ["stamps"], | |
| 11 | + "regions": ["CH"], | |
| 12 | + "languages": ["en", "de"], | |
| 13 | + "currency": ["CHF"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": false, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 10080, | |
| 22 | + "priority": "low", | |
| 23 | + "trustScore": 0.9, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://corinphila.ch/en/versteigerungsbedingungen/", | |
| 26 | + "accessNotes": "Plain HTTPS with the RareIndex user agent, 0 credits, no login, no anti-bot. robots.txt (corinphila.ch): Disallow /geheim, *ajax=1, *admin=1; Crawl-Delay 10 → the connector waits 10 s between requests and reads ≤ 20 pages per run. Pages read: the archive list (/en/_pages/&action=show&id=211 → auction ids 12–28, 2016–2024), the auction overview (/en/_auctions/&action=showAuctionOverview&auctionID=N: catalogue parts with lot counts and the c4msEnv.auctionData JSON giving currency CHF, start/end dates and status) and the lot lists (/en/_auctions/&action=showLots&auctionID=N&catalogPart=P&show_all_lots=1&page=K, 100 lots per page: lot number, country, description, 'Starting bid', 'Hammer price : 260.00 CHF' or 'not sold', condition icons, CloudFront thumbnails). Sales are hammer prices in CHF (buyer_premium_included=false); unsold lots are skipped; sale date = auction start date. Not fetched: the live catalogue on auction.corinphila.ch (WordPress + admin-ajax JS listing), the PDF result lists, bidding or account pages.", | |
| 27 | + "enabled": true, | |
| 28 | + "schemaVersion": "1.0", | |
| 29 | + "historicalDepth": "years", | |
| 30 | + "acquisitionMethod": "Static HTML archive lot lists (c4ms)", | |
| 31 | + "config": { | |
| 32 | + "auctionsPerRun": 1, | |
| 33 | + "pagesPerRun": 20 | |
| 34 | + } | |
| 35 | +} | |
added
connectors/api/crepslocker/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Crepslocker connector (`crepslocker`) | |
| 2 | + | |
| 3 | +- Source: https://www.crepslocker.com · Blackburn (UK) luxury sneaker and streetwear reseller: Air Jordan, Nike Dunk, Yeezy, designer trainers (Dior, Balenciaga, Hermès), a "Preloved" pre-owned range with condition tags, Swatch × AP watches, Pokémon sealed and Pop Mart Labubu. | |
| 4 | +- Country/currency: GB / GBP · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (8): `footwear` → sneakers, `air-jordan` → nike_jordan, `nike-dunk` → nike_jordan, `yeezy-trainers` → adidas_yeezy, `preloved-footwear` → sneakers, `watches` → other_watches, `pokemon` → pokemon, `labubu-by-pop-mart` → designer_toys. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/crepslocker/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts crepslocker`. | |
added
connectors/api/crepslocker/index.test.ts
+78 −0
@@ -0,0 +1,78 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('crepslocker', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Preloved - Air Jordan 1 Retro High OG Bred Toe", | |
| 18 | + "collection": "air-jordan", | |
| 19 | + "type": "Trainers", | |
| 20 | + "tags": [ | |
| 21 | + "Air Jordan", | |
| 22 | + "Condition: Very good", | |
| 23 | + "preloved" | |
| 24 | + ], | |
| 25 | + "variant": "UK 9", | |
| 26 | + "expect": "nike_jordan", | |
| 27 | + "brand": "Jordan" | |
| 28 | + }, | |
| 29 | + { | |
| 30 | + "title": "Nike Dunk Low Retro Black White Panda 2021", | |
| 31 | + "collection": "nike-dunk", | |
| 32 | + "type": "Trainers", | |
| 33 | + "variant": "UK 6 | EU 39 / Black", | |
| 34 | + "expect": "nike_jordan", | |
| 35 | + "brand": "Nike" | |
| 36 | + }, | |
| 37 | + { | |
| 38 | + "title": "Preloved - Adidas Yeezy Boost 350 Pirate Black (2015)", | |
| 39 | + "collection": "preloved-footwear", | |
| 40 | + "type": "Trainers", | |
| 41 | + "expect": "adidas_yeezy", | |
| 42 | + "brand": "adidas Yeezy", | |
| 43 | + "year": 2015 | |
| 44 | + }, | |
| 45 | + { | |
| 46 | + "title": "Dior B22 Sneakers White Blue", | |
| 47 | + "collection": "footwear", | |
| 48 | + "type": "Trainers", | |
| 49 | + "expect": "new_balance_asics_other" | |
| 50 | + }, | |
| 51 | + { | |
| 52 | + "title": "Swatch x Audemars Piguet Royal Pop Ocho Negro With Rubber Strap", | |
| 53 | + "collection": "watches", | |
| 54 | + "type": "Watches", | |
| 55 | + "expect": "other_watches", | |
| 56 | + "brand": "Swatch" | |
| 57 | + }, | |
| 58 | + { | |
| 59 | + "title": "Pokemon Mega Evolution Pitch Black Booster Pack (Random Pack Art)", | |
| 60 | + "collection": "pokemon", | |
| 61 | + "type": "Collectables", | |
| 62 | + "expect": "pokemon" | |
| 63 | + }, | |
| 64 | + { | |
| 65 | + "title": "Supreme x Jordan Black Track Jacket (SS26)", | |
| 66 | + "collection": "air-jordan", | |
| 67 | + "type": "Coats & Jackets", | |
| 68 | + "expect": null | |
| 69 | + }, | |
| 70 | + { | |
| 71 | + "title": "Cartier Santos Sunglasses Gold", | |
| 72 | + "collection": "footwear", | |
| 73 | + "type": "Sunglasses", | |
| 74 | + "expect": null | |
| 75 | + } | |
| 76 | + ]); | |
| 77 | + }); | |
| 78 | +}); | |
added
connectors/api/crepslocker/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Crepslocker — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/crepslocker/meta.json
+154 −0
@@ -0,0 +1,154 @@ | ||
| 1 | +{ | |
| 2 | + "id": "crepslocker", | |
| 3 | + "displayName": "Crepslocker", | |
| 4 | + "sourceId": "crepslocker", | |
| 5 | + "sourceName": "Crepslocker", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.crepslocker.com", | |
| 8 | + "module": "api/crepslocker", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "sneakers", | |
| 14 | + "nike_jordan", | |
| 15 | + "adidas_yeezy", | |
| 16 | + "other_watches", | |
| 17 | + "pokemon", | |
| 18 | + "designer_toys", | |
| 19 | + "rolex", | |
| 20 | + "omega", | |
| 21 | + "patek_philippe", | |
| 22 | + "audemars_piguet", | |
| 23 | + "new_balance_asics_other" | |
| 24 | + ], | |
| 25 | + "regions": [ | |
| 26 | + "GB" | |
| 27 | + ], | |
| 28 | + "country": "GB", | |
| 29 | + "languages": [ | |
| 30 | + "en" | |
| 31 | + ], | |
| 32 | + "currency": [ | |
| 33 | + "GBP" | |
| 34 | + ], | |
| 35 | + "supportsListings": true, | |
| 36 | + "supportsSold": false, | |
| 37 | + "supportsAuctions": false, | |
| 38 | + "supportsImages": true, | |
| 39 | + "supportsCatalog": false, | |
| 40 | + "supportsPopulation": false, | |
| 41 | + "supportsLookup": true, | |
| 42 | + "refreshFrequencyMinutes": 720, | |
| 43 | + "priority": "medium", | |
| 44 | + "trustScore": 0.65, | |
| 45 | + "attributionRequired": true, | |
| 46 | + "termsUrl": "https://www.crepslocker.com/policies/terms-of-service", | |
| 47 | + "accessNotes": "Crepslocker (crepslocker.com) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (footwear, air-jordan, nike-dunk, yeezy-trainers, preloved-footwear, watches, pokemon, labubu-by-pop-mart). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency GBP (/meta.json currency GBP, Shopify.currency rate 1.0). \"Preloved -\" titles are pre-owned pairs; their condition tag (e.g. \"Condition: Very good\") is kept in metadata.tags. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 48 | + "enabled": true, | |
| 49 | + "schemaVersion": "1.0", | |
| 50 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 51 | + "historicalDepth": "none", | |
| 52 | + "requires": [], | |
| 53 | + "config": { | |
| 54 | + "currency": "GBP", | |
| 55 | + "seller": "Crepslocker", | |
| 56 | + "location": null, | |
| 57 | + "collections": [ | |
| 58 | + { | |
| 59 | + "handle": "footwear", | |
| 60 | + "categorySlug": "sneakers" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "air-jordan", | |
| 64 | + "categorySlug": "nike_jordan", | |
| 65 | + "brand": "Jordan" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "handle": "nike-dunk", | |
| 69 | + "categorySlug": "nike_jordan", | |
| 70 | + "brand": "Nike" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "yeezy-trainers", | |
| 74 | + "categorySlug": "adidas_yeezy", | |
| 75 | + "brand": "adidas Yeezy" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "handle": "preloved-footwear", | |
| 79 | + "categorySlug": "sneakers" | |
| 80 | + }, | |
| 81 | + { | |
| 82 | + "handle": "watches", | |
| 83 | + "categorySlug": "other_watches" | |
| 84 | + }, | |
| 85 | + { | |
| 86 | + "handle": "pokemon", | |
| 87 | + "categorySlug": "pokemon", | |
| 88 | + "franchise": "Pokémon" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "handle": "labubu-by-pop-mart", | |
| 92 | + "categorySlug": "designer_toys", | |
| 93 | + "brand": "Pop Mart" | |
| 94 | + } | |
| 95 | + ], | |
| 96 | + "rules": [ | |
| 97 | + { | |
| 98 | + "match": "swatch|moonswatch|scuba fifty", | |
| 99 | + "categorySlug": "other_watches", | |
| 100 | + "brand": "Swatch" | |
| 101 | + }, | |
| 102 | + { | |
| 103 | + "match": "rolex", | |
| 104 | + "categorySlug": "rolex", | |
| 105 | + "brand": "Rolex" | |
| 106 | + }, | |
| 107 | + { | |
| 108 | + "match": "omega", | |
| 109 | + "categorySlug": "omega", | |
| 110 | + "brand": "Omega" | |
| 111 | + }, | |
| 112 | + { | |
| 113 | + "match": "patek", | |
| 114 | + "categorySlug": "patek_philippe", | |
| 115 | + "brand": "Patek Philippe" | |
| 116 | + }, | |
| 117 | + { | |
| 118 | + "match": "audemars|royal oak", | |
| 119 | + "categorySlug": "audemars_piguet", | |
| 120 | + "brand": "Audemars Piguet" | |
| 121 | + }, | |
| 122 | + { | |
| 123 | + "match": "jordan", | |
| 124 | + "categorySlug": "nike_jordan", | |
| 125 | + "brand": "Jordan" | |
| 126 | + }, | |
| 127 | + { | |
| 128 | + "match": "\\bnike\\b|air max|air force|\\bdunk\\b|\\bsb\\b|blazer|vapormax|huarache|cortez|p-6000|vomero|pegasus|\\bshox\\b|\\bkobe\\b|lebron|\\bnocta\\b|\\bacg\\b|sacai|off-white|travis scott", | |
| 129 | + "categorySlug": "nike_jordan", | |
| 130 | + "brand": "Nike" | |
| 131 | + }, | |
| 132 | + { | |
| 133 | + "match": "yeezy", | |
| 134 | + "categorySlug": "adidas_yeezy", | |
| 135 | + "brand": "adidas Yeezy" | |
| 136 | + }, | |
| 137 | + { | |
| 138 | + "match": "adidas|\\bsamba\\b|gazelle|spezial|superstar|\\bcampus\\b|\\bforum\\b|ultraboost|\\bnmd\\b|stan smith|adizero|\\by-3\\b|climacool|\\bzx\\b|\\beqt\\b|\\bsl ?72\\b|adistar|megaride|taekwondo|wales bonner", | |
| 139 | + "categorySlug": "adidas_yeezy", | |
| 140 | + "brand": "adidas" | |
| 141 | + }, | |
| 142 | + { | |
| 143 | + "match": "new balance|asics|puma|reebok|saucony|salomon|converse|\\bvans\\b|\\bhoka\\b|karhu|mizuno|autry|diadora|\\bveja\\b|clarks|timberland|dr\\.? ?martens|\\bugg\\b|mallet|cleens|umbro|le coq|kangaroos|\\bfila\\b|k-swiss|lacoste|golden goose|balenciaga|\\bdior\\b|gucci|louis vuitton|prada|amiri|herm[eè]s|rick owens|margiela|mcqueen|common projects|axel arigato|represent|loewe|bottega|valentino|givenchy|fendi|burberry|chanel|celine|saint laurent|versace|moncler|birkenstock|\\bon\\b (cloud|running)|cloudmonster|cloudtilt|merrell|keen\\b|ewing|ellesse|kappa|hi-tec|etnies|\\bdc shoes|gola|onitsuka|norda|novesta|stepney|sunnei|camper|mschf|maison mihara|\\bbape\\b|\\bsta\\b", | |
| 144 | + "categorySlug": "new_balance_asics_other" | |
| 145 | + } | |
| 146 | + ], | |
| 147 | + "defaultCategory": null, | |
| 148 | + "exclude": "t-shirt|\\btee\\b|\\btees\\b|hoodie|sweatshirt|crewneck|jacket|\\bcoat\\b|tracksuit|track pants|track jacket|sweatpants|joggers|shorts|jeans|trousers|\\bpants\\b|cargo|\\bcap\\b|\\bhat\\b|beanie|balaclava|socks|\\bbag\\b|backpack|tote|wallet|card holder|\\bbelt\\b|sunglasses|fragrance|perfume|cologne|keyring|lanyard|cleaner|crep protect|protector spray|laces|insole|shoe tree|gift card|jersey|\\bpolo\\b|\\bshirt\\b|knitwear|cardigan|\\bvest\\b|gilet|puffer|scarf|gloves|jewellery|jewelry|\\bchain\\b|bracelet|necklace|\\bring\\b|umbrella|towel|doormat|\\brug\\b|candle|\\bmug\\b|phone case|airpods|dress\\b|skirt|leggings|bodysuit|swim|bikini|underwear|boxers|slippers|flip[- ]?flops?|sandals?\\b|\\bslides?\\b|\\bmules?\\b|crocs|clog|\\| (clothing|coats & jackets|pants|hoodies|t-shirts|accessories|designer hats|caps|bags|fragrance|sunglasses|jewellery|home) \\|", | |
| 149 | + "keepOutOfStock": true, | |
| 150 | + "pageSize": 250, | |
| 151 | + "fetchBarcodes": false, | |
| 152 | + "wholeShop": false | |
| 153 | + } | |
| 154 | +} | |
added
connectors/api/dlrc/README.md
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# dlrc — David Lawrence Rare Coins sold lots | |
| 2 | + | |
| 3 | +US coin dealer/auction house; weekly internet auctions ("no buyer's fee"). Public JSON API of the site's SPA (Collectibles Showcase, tenant header `x-client-id` resolved from `/consumer/client?url=`). | |
| 4 | + | |
| 5 | +- **Engine**: `api` (JSON), 0 credits. | |
| 6 | +- **Flow (incremental)**: `/consumer/auction/past` → newest closed auctions → `/consumer/inventory/past/auction?auctions=<id>&pageNumber=N&pageSize=100`. | |
| 7 | +- **Flow (backfill)**: unfiltered `/consumer/inventory/past/auction?pageNumber=N` (162k sold lots incl. legacy 2016+ sales), cursor `archivePage`. | |
| 8 | +- **Records**: `sale` per sold lot — `salePrice` cents → USD, `buyerPremiumIncluded=true` (no buyer's fee, note in metadata), sale date = lot closing day, grade from `gradingService`/`fullGrade`/`isPlus`/`isCac`, identifiers `dlrc_inventory_id`, `pcgs_number`, `<grader>_cert`, optional `numista_id`. | |
| 9 | +- **Config**: `clientId` (default 3), `auctionsPerRun`, `pagesPerRun`. | |
| 10 | +- **Smoke**: `pnpm tsx connectors/api/dlrc/_smoke.ts 1` | |
added
connectors/api/dlrc/_smoke.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import path from 'node:path'; | |
| 2 | +import { fileURLToPath } from 'node:url'; | |
| 3 | +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js'; | |
| 4 | + | |
| 5 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 6 | +if (process.argv[2] === '--capture') await captureFixture(dir, process.argv[3] ?? 'category-page', Number(process.argv[4] ?? 6)); | |
| 7 | +else await runSmoke(dir, Number(process.argv[2] ?? 2)); | |
added
connectors/api/dlrc/index.test.ts
+52 −0
@@ -0,0 +1,52 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import { fileURLToPath } from 'node:url'; | |
| 5 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 7 | +import createConnector, { dlrcGrade, LotSchema } from './index.js'; | |
| 8 | + | |
| 9 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 10 | +const connector = createConnector(localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')))); | |
| 11 | + | |
| 12 | +const LOT = { | |
| 13 | + auctionId: 24, lotId: 9580, lotNumber: 7001, totalBids: 2, lotCurrentBid: 265500, lotClosesAt: '2023-09-18T00:00:00Z', soldInAuction: true, id: 752042, itemType: 'US_COIN', | |
| 14 | + name: '1793 1/2C NGC/CAC Poor 01', shortDescription: 'First Year Half Cent', description: 'A rare coin in exciting, lowball condition!', seriesName: 'Liberty Cap Half Cent', groupName: 'Half-Cents and Cents', categoryName: 'U.S. Coins', | |
| 15 | + certificationNumber: '6460687001', gradingService: 'NGC', grade: 1, fullGrade: 'PO1BN', isPlus: false, isCac: true, isEPQ: false, price: null, salePrice: 265500, status: 'SOLD', | |
| 16 | + catalogEntry: { id: 3915, pcgsNumber: 1000, title: '1793 1/2C, BN', denomination: '1/2C', coinDate: 1793, mintMark: 'P', designation: 'BN', majorVariety: '', dieVariety: '', strikeType: 'MS', mintage: 35334, numistaTypeId: null, kmNumber: null, categoryName: 'Liberty Cap Half Cent' }, | |
| 17 | + images: [{ thumbnailUrl: 'https://d2lj9qe62mju4a.cloudfront.net/inventory/3/752042/a.jpg?width=256&height=256', url: 'https://d2lj9qe62mju4a.cloudfront.net/inventory/3/752042/a.jpg' }], | |
| 18 | +}; | |
| 19 | +const UNSOLD = { ...LOT, id: 1, lotId: 2, salePrice: null, lotCurrentBid: 1000, soldInAuction: false, status: 'AVAILABLE' }; | |
| 20 | + | |
| 21 | +describe('dlrc', () => { | |
| 22 | + runFixtureSuite(connector, it, expect); | |
| 23 | + | |
| 24 | + it('maps grading service / full grade / CAC / details into the grade block', () => { | |
| 25 | + expect(dlrcGrade(LotSchema.parse(LOT))).toEqual({ grader: 'ngc', grade: 'PO1BN', qualifier: 'CAC', certificationNumber: '6460687001' }); | |
| 26 | + expect(dlrcGrade({ gradingService: 'ANACS', fullGrade: 'VG8BN', grade: 8, isPlus: false, isCac: false, certificationNumber: null, name: '1804 1/2C ANACS VG Details (Plain 4, No Stems, Damaged)' })).toMatchObject({ grader: 'anacs', grade: 'VG8BN', qualifier: 'VG Details (Plain 4, No Stems, Damaged)' }); | |
| 27 | + expect(dlrcGrade({ gradingService: 'PCGS', fullGrade: 'MS65', grade: 65, isPlus: true, isCac: false, certificationNumber: '12345678', name: '1881-S $1 PCGS MS65+' }).grade).toBe('MS65+'); | |
| 28 | + }); | |
| 29 | + | |
| 30 | + it('normalises sold lots: cents → dollars, no buyer premium, cert + PCGS number identifiers, unsold skipped', async () => { | |
| 31 | + const payload = { kind: 'past_auction_page' as const, url: 'https://api.collectiblesshowcase.com/consumer/inventory/past/auction?auctions=24&pageNumber=1&pageSize=100', auctionId: 24, auctionTitle: 'Test', pageNumber: 1, totalPages: 1, totalItems: 2, lots: [LOT, UNSOLD] }; | |
| 32 | + const out = await connector.normalize({ url: payload.url, externalId: 'x', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload }); | |
| 33 | + expect(out.length).toBe(1); | |
| 34 | + const s = out[0]!; | |
| 35 | + if (s.kind !== 'sale') throw new Error('sale'); | |
| 36 | + expect(s).toMatchObject({ price: 2655, currency: 'USD', buyerPremiumIncluded: true, lotNumber: '7001', auctionHouse: 'David Lawrence Rare Coins', sourceUrl: 'https://www.davidlawrence.com/inventory/752042', isBundle: false }); | |
| 37 | + expect(s.saleDate.toISOString()).toBe('2023-09-18T00:00:00.000Z'); | |
| 38 | + expect(s.grade).toEqual({ grader: 'ngc', grade: 'PO1BN', qualifier: 'CAC', certificationNumber: '6460687001' }); | |
| 39 | + expect(s.attributes).toMatchObject({ categorySlug: 'coins', year: 1793, country: 'US', series: 'Liberty Cap Half Cent', variant: 'P mint' }); | |
| 40 | + expect(s.attributes.identifiers).toEqual({ dlrc_inventory_id: '752042', pcgs_number: '1000', ngc_cert: '6460687001' }); | |
| 41 | + expect(s.attributes.metadata).toMatchObject({ mintage: 35334, denomination: '1/2C', designation: 'BN' }); | |
| 42 | + expect(s.imageUrls).toEqual(['https://d2lj9qe62mju4a.cloudfront.net/inventory/3/752042/a.jpg']); | |
| 43 | + }); | |
| 44 | + | |
| 45 | + it('fixture pages normalise to USD sales with grades', async () => { | |
| 46 | + const out = await connector.normalize(loadFixture('dlrc', 'auction-337-page-1').raw); | |
| 47 | + expect(out.length).toBeGreaterThan(3); | |
| 48 | + for (const r of out) if (r.kind === 'sale') expect(r).toMatchObject({ currency: 'USD', buyerPremiumIncluded: true }); | |
| 49 | + const legacy = await connector.normalize(loadFixture('dlrc', 'archive-page-legacy-lots').raw); | |
| 50 | + expect(legacy.some((r) => r.kind === 'sale' && r.saleDate.getUTCFullYear() < 2020)).toBe(true); | |
| 51 | + }); | |
| 52 | +}); | |
added
connectors/api/dlrc/index.ts
+294 −0
@@ -0,0 +1,294 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared'; | |
| 4 | +import { makeSale } from '../../firecrawl/_carlib/index.js'; | |
| 5 | +import { isNumisBundle, numisAttributes, numisCategory } from '../../firecrawl/_g5-numismatics-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * David Lawrence Rare Coins — sold auction lots from the public Collectibles Showcase JSON API that the | |
| 9 | + * davidlawrence.com SPA itself calls (no key: the tenant id is resolved from /consumer/client?url=…). | |
| 10 | + * Every lot carries grading service, grade, certification number, PCGS number and sale price in cents. | |
| 11 | + * DLRC states "DLRC has no buyer's fee" → the sale price is the full amount paid. | |
| 12 | + */ | |
| 13 | + | |
| 14 | +const API = 'https://api.collectiblesshowcase.com'; | |
| 15 | +const SITE = 'https://www.davidlawrence.com'; | |
| 16 | +const PARSER_VERSION = '1.0.0'; | |
| 17 | +const PAGE_SIZE = 100; | |
| 18 | + | |
| 19 | +export const LotSchema = z.object({ | |
| 20 | + id: z.number(), | |
| 21 | + auctionId: z.number().nullable(), | |
| 22 | + lotId: z.number().nullable(), | |
| 23 | + lotNumber: z.number().nullable(), | |
| 24 | + totalBids: z.number().nullable().optional(), | |
| 25 | + lotCurrentBid: z.number().nullable().optional(), | |
| 26 | + lotClosesAt: z.string().nullable(), | |
| 27 | + soldInAuction: z.boolean().nullable().optional(), | |
| 28 | + itemType: z.string().nullable().optional(), | |
| 29 | + name: z.string(), | |
| 30 | + shortDescription: z.string().nullable().optional(), | |
| 31 | + description: z.string().nullable().optional(), | |
| 32 | + seriesName: z.string().nullable().optional(), | |
| 33 | + groupName: z.string().nullable().optional(), | |
| 34 | + categoryName: z.string().nullable().optional(), | |
| 35 | + certificationNumber: z.string().nullable().optional(), | |
| 36 | + gradingService: z.string().nullable().optional(), | |
| 37 | + grade: z.number().nullable().optional(), | |
| 38 | + fullGrade: z.string().nullable().optional(), | |
| 39 | + isPlus: z.boolean().nullable().optional(), | |
| 40 | + isCac: z.boolean().nullable().optional(), | |
| 41 | + isEPQ: z.boolean().nullable().optional(), | |
| 42 | + price: z.number().nullable().optional(), | |
| 43 | + salePrice: z.number().nullable().optional(), | |
| 44 | + status: z.string().nullable().optional(), | |
| 45 | + catalogEntry: z | |
| 46 | + .object({ | |
| 47 | + id: z.number().nullable().optional(), | |
| 48 | + pcgsNumber: z.number().nullable().optional(), | |
| 49 | + title: z.string().nullable().optional(), | |
| 50 | + denomination: z.string().nullable().optional(), | |
| 51 | + coinDate: z.number().nullable().optional(), | |
| 52 | + mintMark: z.string().nullable().optional(), | |
| 53 | + designation: z.string().nullable().optional(), | |
| 54 | + majorVariety: z.string().nullable().optional(), | |
| 55 | + dieVariety: z.string().nullable().optional(), | |
| 56 | + strikeType: z.string().nullable().optional(), | |
| 57 | + mintage: z.number().nullable().optional(), | |
| 58 | + numistaTypeId: z.union([z.number(), z.string()]).nullable().optional(), | |
| 59 | + kmNumber: z.string().nullable().optional(), | |
| 60 | + categoryName: z.string().nullable().optional(), | |
| 61 | + }) | |
| 62 | + .nullable() | |
| 63 | + .optional(), | |
| 64 | + images: z.array(z.object({ url: z.string().nullable().optional(), thumbnailUrl: z.string().nullable().optional() })).default([]), | |
| 65 | +}); | |
| 66 | +export type Lot = z.infer<typeof LotSchema>; | |
| 67 | +export const PayloadSchema = z.object({ | |
| 68 | + kind: z.literal('past_auction_page'), | |
| 69 | + url: z.string(), | |
| 70 | + auctionId: z.number().nullable(), | |
| 71 | + auctionTitle: z.string().nullable(), | |
| 72 | + pageNumber: z.number().int(), | |
| 73 | + totalPages: z.number().int().nullable(), | |
| 74 | + totalItems: z.number().int().nullable(), | |
| 75 | + lots: z.array(LotSchema), | |
| 76 | +}); | |
| 77 | +export type Payload = z.infer<typeof PayloadSchema>; | |
| 78 | + | |
| 79 | +const ApiPage = z.object({ pagination: z.object({ totalItemsCount: z.number(), pageSize: z.number(), pageNumber: z.number(), totalPages: z.number() }).nullable(), payload: z.array(z.unknown()) }); | |
| 80 | +const PastAuctions = z.object({ payload: z.array(z.object({ id: z.number(), title: z.string(), estimatedClosesAt: z.string().nullable(), totalLotCount: z.number().nullable() })) }); | |
| 81 | + | |
| 82 | +/** "1793 1/2C NGC/CAC Poor 01" + structured fields → grader slug, grade token. */ | |
| 83 | +export function dlrcGrade(lot: Pick<Lot, 'gradingService' | 'fullGrade' | 'grade' | 'isPlus' | 'isCac' | 'certificationNumber' | 'name'>): { grader: string | null; grade: string | null; qualifier: string | null; certificationNumber: string | null } { | |
| 84 | + const svc = lot.gradingService?.toUpperCase() ?? null; | |
| 85 | + const grader = svc && /^(PCGS|NGC|ANACS|ICCS|PMG|ICG|CAC)$/.test(svc) ? svc.toLowerCase() : svc ? svc.toLowerCase() : null; | |
| 86 | + let grade = lot.fullGrade ?? (lot.grade ? String(lot.grade) : null); | |
| 87 | + if (grade && lot.isPlus && !grade.includes('+')) grade = grade.replace(/^([A-Z]+\d{1,2})/, '$1+'); | |
| 88 | + const details = /details/i.test(lot.name) ? lot.name.match(/\b(\w+ Details(?: \([^)]*\))?)/i)?.[1] ?? 'Details' : null; | |
| 89 | + const qualifier = [lot.isCac ? 'CAC' : null, details].filter(Boolean).join(' · ') || null; | |
| 90 | + return { grader, grade, qualifier, certificationNumber: lot.certificationNumber ?? null }; | |
| 91 | +} | |
| 92 | + | |
| 93 | +interface Cursor { | |
| 94 | + doneAuctions?: number[]; | |
| 95 | + inProgress?: { auctionId: number | null; title: string | null; pageNumber: number; totalPages: number | null } | null; | |
| 96 | + /** backfill over the unfiltered archive (includes pre-2023 legacy lots without an auction id) */ | |
| 97 | + archivePage?: number; | |
| 98 | + done?: boolean; | |
| 99 | + updatedAt?: string; | |
| 100 | +} | |
| 101 | + | |
| 102 | +export class DlrcConnector extends BaseConnector { | |
| 103 | + readonly version = '1.0.0'; | |
| 104 | + readonly parserVersion = PARSER_VERSION; | |
| 105 | + protected override minIntervalMs = 1500; | |
| 106 | + override readonly urlPatterns = [/davidlawrence\.com\//i]; | |
| 107 | + private clientId: string | null = null; | |
| 108 | + | |
| 109 | + private async headers(ctx: CrawlContext): Promise<Record<string, string> | null> { | |
| 110 | + if (this.clientId) return { 'x-client-id': this.clientId }; | |
| 111 | + const configured = this.meta.config.clientId; | |
| 112 | + if (configured) this.clientId = String(configured); | |
| 113 | + else { | |
| 114 | + await this.throttle(); | |
| 115 | + const res = await ctx.fetch(`${API}/consumer/client?url=${encodeURIComponent(SITE)}`, { engines: ['api'], minQuality: 0, force: true }); | |
| 116 | + const id = (res.json as { payload?: { id?: number } } | null)?.payload?.id; | |
| 117 | + if (!res.success || !id) { | |
| 118 | + ctx.anomaly('page_fetch_failed', `client id lookup: ${res.error ?? res.httpStatus}`); | |
| 119 | + return null; | |
| 120 | + } | |
| 121 | + this.clientId = String(id); | |
| 122 | + } | |
| 123 | + return { 'x-client-id': this.clientId }; | |
| 124 | + } | |
| 125 | + | |
| 126 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 127 | + const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2); | |
| 128 | + const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 40); | |
| 129 | + const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) }; | |
| 130 | + const headers = await this.headers(ctx); | |
| 131 | + if (!headers) return; | |
| 132 | + let pages = 0; | |
| 133 | + let yielded = 0; | |
| 134 | + | |
| 135 | + const fetchLots = async (params: Record<string, string | number>) => { | |
| 136 | + const q = new URLSearchParams(Object.entries(params).map(([k, v]) => [k, String(v)] as [string, string])); | |
| 137 | + const url = `${API}/consumer/inventory/past/auction?${q}`; | |
| 138 | + await this.throttle(); | |
| 139 | + const res = await ctx.fetch(url, { engines: ['api'], headers, expect: ['title', 'price', 'identifiers'], parse: (r) => { | |
| 140 | + const first = (r.json as { payload?: Array<Record<string, unknown>> } | null)?.payload?.[0]; | |
| 141 | + return first ? { title: first.name, price: first.salePrice, identifiers: first.certificationNumber ? { cert: first.certificationNumber } : null } : null; | |
| 142 | + } }); | |
| 143 | + pages++; | |
| 144 | + if (!res.success || !res.json) { | |
| 145 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 146 | + return null; | |
| 147 | + } | |
| 148 | + const parsed = ApiPage.safeParse(res.json); | |
| 149 | + if (!parsed.success) { | |
| 150 | + ctx.anomaly('schema_drift', `${url}: ${parsed.error.issues[0]?.message}`); | |
| 151 | + return null; | |
| 152 | + } | |
| 153 | + return { url, res, page: parsed.data }; | |
| 154 | + }; | |
| 155 | + | |
| 156 | + if (ctx.options.mode === 'backfill') { | |
| 157 | + // Whole archive, catalogue order (includes legacy 2016+ sales without an auction id). | |
| 158 | + let pageNumber = Math.max(1, cursor.archivePage ?? 1); | |
| 159 | + let totalPages: number | null = null; | |
| 160 | + while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) { | |
| 161 | + const got = await fetchLots({ pageNumber, pageSize: PAGE_SIZE }); | |
| 162 | + if (!got) break; | |
| 163 | + totalPages = got.page.pagination?.totalPages ?? totalPages; | |
| 164 | + const lots = got.page.payload; | |
| 165 | + if (lots.length) { | |
| 166 | + yielded++; | |
| 167 | + const payload: Payload = { kind: 'past_auction_page', url: got.url, auctionId: null, auctionTitle: null, pageNumber, totalPages, totalItems: got.page.pagination?.totalItemsCount ?? null, lots: lots.map((l) => LotSchema.parse(l)) }; | |
| 168 | + yield { url: got.url, externalId: `archive:p${pageNumber}`, kind: 'sale', engine: got.res.engine, httpStatus: got.res.httpStatus, payload, fetchedAt: got.res.fetchedAt }; | |
| 169 | + } | |
| 170 | + const finished = !lots.length || (totalPages !== null && pageNumber >= totalPages); | |
| 171 | + pageNumber++; | |
| 172 | + await ctx.progress({ page: pageNumber - 1, totalPages, itemsProcessed: yielded }); | |
| 173 | + await ctx.setCursor({ ...cursor, archivePage: pageNumber, done: finished, updatedAt: new Date().toISOString() }); | |
| 174 | + if (finished) return; | |
| 175 | + } | |
| 176 | + return; | |
| 177 | + } | |
| 178 | + | |
| 179 | + // Incremental: newest closed auctions not yet done, lots filtered per auction. | |
| 180 | + await this.throttle(); | |
| 181 | + const list = await ctx.fetch(`${API}/consumer/auction/past`, { engines: ['api'], headers, minQuality: 0 }); | |
| 182 | + pages++; | |
| 183 | + const parsedList = list.success ? PastAuctions.safeParse(list.json) : null; | |
| 184 | + if (!parsedList?.success) { | |
| 185 | + ctx.anomaly('page_fetch_failed', `/consumer/auction/past: ${list.error ?? list.httpStatus}`); | |
| 186 | + return; | |
| 187 | + } | |
| 188 | + const done = new Set(cursor.doneAuctions ?? []); | |
| 189 | + const auctions = [...parsedList.data.payload].sort((a, b) => (b.estimatedClosesAt ?? '').localeCompare(a.estimatedClosesAt ?? '')); | |
| 190 | + const queue = [...(cursor.inProgress ? [cursor.inProgress] : []), ...auctions.filter((a) => !done.has(a.id) && a.id !== cursor.inProgress?.auctionId).map((a) => ({ auctionId: a.id as number | null, title: a.title as string | null, pageNumber: 1, totalPages: null as number | null }))]; | |
| 191 | + let finished = 0; | |
| 192 | + for (const state of queue) { | |
| 193 | + if (ctx.signal?.aborted || finished >= auctionsPerRun || pages >= pagesPerRun || this.reached(ctx, yielded)) break; | |
| 194 | + let complete = false; | |
| 195 | + while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) { | |
| 196 | + const got = await fetchLots({ auctions: state.auctionId ?? '', pageNumber: state.pageNumber, pageSize: PAGE_SIZE }); | |
| 197 | + if (!got) break; | |
| 198 | + state.totalPages = got.page.pagination?.totalPages ?? state.totalPages; | |
| 199 | + const lots = got.page.payload; | |
| 200 | + if (lots.length) { | |
| 201 | + yielded++; | |
| 202 | + const payload: Payload = { kind: 'past_auction_page', url: got.url, auctionId: state.auctionId, auctionTitle: state.title, pageNumber: state.pageNumber, totalPages: state.totalPages, totalItems: got.page.pagination?.totalItemsCount ?? null, lots: lots.map((l) => LotSchema.parse(l)) }; | |
| 203 | + yield { url: got.url, externalId: `auction:${state.auctionId}:p${state.pageNumber}`, kind: 'sale', engine: got.res.engine, httpStatus: got.res.httpStatus, payload, fetchedAt: got.res.fetchedAt }; | |
| 204 | + } | |
| 205 | + if (!lots.length || (state.totalPages !== null && state.pageNumber >= state.totalPages)) { | |
| 206 | + complete = true; | |
| 207 | + break; | |
| 208 | + } | |
| 209 | + state.pageNumber++; | |
| 210 | + cursor.inProgress = state; | |
| 211 | + await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-400), updatedAt: new Date().toISOString() }); | |
| 212 | + } | |
| 213 | + if (complete) { | |
| 214 | + if (state.auctionId !== null) done.add(state.auctionId); | |
| 215 | + finished++; | |
| 216 | + cursor.inProgress = null; | |
| 217 | + } else cursor.inProgress = state; | |
| 218 | + await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-400), updatedAt: new Date().toISOString() }); | |
| 219 | + } | |
| 220 | + } | |
| 221 | + | |
| 222 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 223 | + const p = PayloadSchema.parse(raw.payload); | |
| 224 | + const out: NormalizedSale[] = []; | |
| 225 | + for (const lot of p.lots) { | |
| 226 | + const cents = lot.salePrice ?? (lot.soldInAuction ? lot.lotCurrentBid ?? null : null); | |
| 227 | + if (!cents || cents <= 0 || (lot.status && lot.status !== 'SOLD' && !lot.soldInAuction)) continue; | |
| 228 | + const closes = lot.lotClosesAt ? new Date(lot.lotClosesAt) : null; | |
| 229 | + if (!closes || Number.isNaN(closes.getTime())) continue; | |
| 230 | + const saleDate = new Date(Date.UTC(closes.getUTCFullYear(), closes.getUTCMonth(), closes.getUTCDate())); | |
| 231 | + const ce = lot.catalogEntry ?? null; | |
| 232 | + const categorySlug = lot.itemType === 'CURRENCY' || /currency|banknote|paper money/i.test(`${lot.categoryName ?? ''} ${lot.groupName ?? ''}`) ? 'banknotes' : numisCategory(lot.name, lot.categoryName ?? lot.groupName, 'coins'); | |
| 233 | + const g = dlrcGrade(lot); | |
| 234 | + const identifiers: Record<string, string> = { dlrc_inventory_id: String(lot.id) }; | |
| 235 | + if (ce?.pcgsNumber) identifiers.pcgs_number = String(ce.pcgsNumber); | |
| 236 | + if (g.certificationNumber && g.grader) identifiers[`${g.grader}_cert`] = g.certificationNumber; | |
| 237 | + if (ce?.numistaTypeId) identifiers.numista_id = String(ce.numistaTypeId); | |
| 238 | + const attributes = numisAttributes({ | |
| 239 | + categorySlug, | |
| 240 | + title: lot.name, | |
| 241 | + section: lot.seriesName ?? lot.groupName ?? null, | |
| 242 | + series: lot.seriesName ?? null, | |
| 243 | + country: lot.itemType === 'US_COIN' || lot.categoryName === 'U.S. Coins' ? 'US' : undefined, | |
| 244 | + identifiers, | |
| 245 | + metadata: { | |
| 246 | + auction_id: lot.auctionId, | |
| 247 | + auction_title: p.auctionTitle, | |
| 248 | + lot_id: lot.lotId, | |
| 249 | + item_type: lot.itemType, | |
| 250 | + group: lot.groupName, | |
| 251 | + category: lot.categoryName, | |
| 252 | + catalog_title: ce?.title ?? null, | |
| 253 | + designation: ce?.designation ?? null, | |
| 254 | + strike_type: ce?.strikeType ?? null, | |
| 255 | + major_variety: ce?.majorVariety || null, | |
| 256 | + die_variety: ce?.dieVariety || null, | |
| 257 | + mintage: ce?.mintage ?? null, | |
| 258 | + total_bids: lot.totalBids ?? null, | |
| 259 | + is_cac: lot.isCac ?? null, | |
| 260 | + buyer_premium: "none — 'DLRC has no buyer's fee' (site notice); sale price is the total paid", | |
| 261 | + }, | |
| 262 | + }); | |
| 263 | + if (ce?.coinDate && !attributes.year) attributes.year = ce.coinDate; | |
| 264 | + if (ce?.mintMark && !attributes.variant) attributes.variant = `${ce.mintMark} mint`; | |
| 265 | + if (ce?.mintMark) attributes.metadata.mint_mark = ce.mintMark; | |
| 266 | + if (ce?.denomination) attributes.metadata.denomination = ce.denomination; | |
| 267 | + const sale = makeSale({ | |
| 268 | + meta: this.meta, | |
| 269 | + sourceUrl: `${SITE}/inventory/${lot.id}`, | |
| 270 | + externalId: String(lot.id), | |
| 271 | + rawTitle: lot.name, | |
| 272 | + description: [lot.shortDescription, lot.description].filter(Boolean).join(' — ') || null, | |
| 273 | + attributes, | |
| 274 | + price: cents / 100, | |
| 275 | + currency: 'USD', | |
| 276 | + saleDate, | |
| 277 | + buyerPremiumIncluded: true, | |
| 278 | + auctionHouse: 'David Lawrence Rare Coins', | |
| 279 | + lotNumber: lot.lotNumber !== null ? String(lot.lotNumber) : null, | |
| 280 | + imageUrls: lot.images.map((i) => i.url ?? i.thumbnailUrl).filter((u): u is string => Boolean(u)).slice(0, 4), | |
| 281 | + observedAt: raw.fetchedAt, | |
| 282 | + parserVersion: PARSER_VERSION, | |
| 283 | + confidence: g.certificationNumber ? 0.95 : 0.85, | |
| 284 | + isBundle: isNumisBundle(lot.name), | |
| 285 | + location: 'US', | |
| 286 | + }); | |
| 287 | + sale.grade = g; | |
| 288 | + out.push(sale); | |
| 289 | + } | |
| 290 | + return out; | |
| 291 | + } | |
| 292 | +} | |
| 293 | + | |
| 294 | +export default (meta: ConnectorMeta) => new DlrcConnector(meta); | |
added
connectors/api/dlrc/meta.json
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +{ | |
| 2 | + "id": "dlrc", | |
| 3 | + "displayName": "David Lawrence Rare Coins — sold auction lots (Collectibles Showcase API)", | |
| 4 | + "sourceId": "dlrc", | |
| 5 | + "sourceName": "David Lawrence Rare Coins", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.davidlawrence.com", | |
| 8 | + "module": "api/dlrc", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["coins", "banknotes"], | |
| 11 | + "regions": ["US"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["USD"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": false, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "high", | |
| 23 | + "trustScore": 0.92, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.davidlawrence.com/terms-and-conditions", | |
| 26 | + "accessNotes": "Public JSON API of the davidlawrence.com single-page app (api.collectiblesshowcase.com), no key or login: the tenant header x-client-id is the public value the site resolves from GET /consumer/client?url=https://www.davidlawrence.com (id 3, also configurable). Endpoints read: /consumer/auction/past (closed weekly auctions since May 2023) and /consumer/inventory/past/auction?auctions=<id>&pageNumber=N&pageSize=100 (sold lots: name, series, grading service, grade, certification number, PCGS number, sale price in cents, closing time, images). The unfiltered listing (162 000 lots, including legacy sales back to 2016 without an auction id) is walked page by page in backfill mode. DLRC displays 'DLRC has no buyer's fee' on every lot → salePrice is the total paid (buyer_premium_included=true, note in metadata). Sale date = the lot's own closing date (UTC day). Robots: the SPA has no robots.txt; the API host answers 404 for /robots.txt; nothing else (account, bids, cart, Auth0) is called. 1.5 s politeness, concurrency 1. Lot URLs are www.davidlawrence.com/inventory/<id>.", | |
| 27 | + "enabled": true, | |
| 28 | + "schemaVersion": "1.0", | |
| 29 | + "historicalDepth": "years", | |
| 30 | + "acquisitionMethod": "Public JSON API (Collectibles Showcase, tenant header)", | |
| 31 | + "config": { | |
| 32 | + "clientId": "3", | |
| 33 | + "auctionsPerRun": 2, | |
| 34 | + "pagesPerRun": 40 | |
| 35 | + } | |
| 36 | +} | |
added
connectors/api/dominic-winter/README.md
+8 −0
@@ -0,0 +1,8 @@ | ||
| 1 | +# dominic-winter — Dominic Winter Auctioneers results (books, maps, documents) | |
| 2 | + | |
| 3 | +goauction platform with capitalised routes: /auction-results/ → /Auction/Search?au=<id>&sd=2&pn=N&g=1 (48 lots/page). | |
| 4 | + | |
| 5 | +- Built on `_g8-auctions-eu-apac-lib/sale-results.ts` (shared crawl/normalize/backfill): `sale` for lots with a | |
| 6 | + published result, `auction_lot` (ended) for unsold lots; native currency; premium basis as labelled by the source | |
| 7 | + (see meta.json accessNotes). Identifiers: `<house>_lot = <sale id>/<lot number>`. | |
| 8 | +- Fixtures: `pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts dominic-winter --save`. | |
added
connectors/api/dominic-winter/index.test.ts
+51 −0
@@ -0,0 +1,51 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 3 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 6 | +import createConnector from './index.js'; | |
| 7 | +import { goauctionPageUrl, parseGoauctionCalendar, parseGoauctionLots } from '../_g8-auctions-eu-apac-lib/goauction-platform.js'; | |
| 8 | + | |
| 9 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 10 | +const connector = createConnector(meta); | |
| 11 | +const attrsOf = (r: NormalizedRecord) => { | |
| 12 | + if (!('attributes' in r)) throw new Error(`record kind ${r.kind} has no attributes`); | |
| 13 | + return r.attributes; | |
| 14 | +}; | |
| 15 | +const HOUSE = { base: 'https://www.dominicwinter.co.uk', currency: 'GBP' as const, listStyle: 'search' as const }; | |
| 16 | + | |
| 17 | +const CAL = `<div class="auction-calendar-item calendar-standard "> <a href='/Auction/Search?au=885&sd=2'> <div class="auction-calendar-image"> <img src="x" alt="Printed Books, Maps & Documents, Ornithology, Music Manuscripts, British & European Bookplates" /> </div> </a> <div class="auction-calendar-text "> <a href='/Auction/Search?au=885&sd=2'><H4>Printed Books, Maps & Documents</H4></a> <div><h5>Wednesday 28 January 2026</h5> <strong>Results: <a href="https://dominicwinter.bidpath.cloud/results-pdf/x.pdf">Download PDF results</a></strong></div></div></div>`; | |
| 18 | +const GRID = `<link rel="next" href="/Auction/Search/?au=885&sd=2&pn=2&g=1"/><div class='auction-grid agh'><div class='auction-grid-lot col-sm-6 col-md-4 col-lg-3'> <div class="auction-lot"> <span class="corner-flash corner-flash-sold auction-lot-sold">SOLD</span> <div class="auction-lot-image"> <a href="/Auction/Lot/lot-1---amedeo-luigi-on-the-polar-star-2-volumes-1903/?lot=425001&au=885&sd=1&pp=48&pn=1&g=1"> <img src="https://dominicwinter.goauction.co.uk/stock/722474-0-small.jpg?v=1" alt="Lot 1 - Amedeo (Luigi). On the "Polar Star", 2 volumes, 1903" /> </a> </div> <div class="auction-lot-text "> <p class="auction-lot-title"> <a href="/Auction/Lot/lot-1---amedeo-luigi-on-the-polar-star-2-volumes-1903/?lot=425001&au=885&sd=1"> <span class='lot-title cat-113'>Lot 1 - <span class="req-tag"></span>Amedeo (Luigi). On the "Polar Star", 2 volumes, 1903</span> </a> </p> <p></p> <p> <strong>Sold for £480</strong> </p> </div> </div> </div></div>`; | |
| 19 | + | |
| 20 | +describe('dominic-winter', () => { | |
| 21 | + runFixtureSuite(connector, it, expect); | |
| 22 | + | |
| 23 | + it('fixtures: GBP book results', async () => { | |
| 24 | + let sales = 0; | |
| 25 | + for (const name of listFixtures('dominic-winter')) { | |
| 26 | + for (const r of await connector.normalize(loadFixture('dominic-winter', name).raw)) { | |
| 27 | + if (!('attributes' in r)) continue; | |
| 28 | + expect(r.attributes.identifiers.dominic_winter_lot).toMatch(/^\d+\/\S+$/); | |
| 29 | + if (r.kind === 'sale') { | |
| 30 | + sales++; | |
| 31 | + expect(r.currency).toBe('GBP'); | |
| 32 | + expect(r.auctionHouse).toBe('Dominic Winter Auctioneers'); | |
| 33 | + } | |
| 34 | + } | |
| 35 | + } | |
| 36 | + expect(sales).toBeGreaterThan(10); | |
| 37 | + }); | |
| 38 | + | |
| 39 | + it('parses the /Auction/Search style calendar + grid ("Lot 1 - title")', async () => { | |
| 40 | + const sales = parseGoauctionCalendar(CAL, HOUSE); | |
| 41 | + expect(sales).toHaveLength(1); | |
| 42 | + expect(sales[0]).toMatchObject({ id: '885', title: 'Printed Books, Maps & Documents', date: '2026-01-28T00:00:00.000Z', url: 'https://www.dominicwinter.co.uk/Auction/Search?au=885&sd=2' }); | |
| 43 | + expect(goauctionPageUrl(sales[0]!, 3, HOUSE)).toBe('https://www.dominicwinter.co.uk/Auction/Search?au=885&sd=2&pn=3&g=1'); | |
| 44 | + const p = parseGoauctionLots(GRID, sales[0]!, HOUSE)!; | |
| 45 | + expect(p.hasMore).toBe(true); | |
| 46 | + expect(p.lots[0]).toMatchObject({ lotNo: '1', title: 'Amedeo (Luigi). On the "Polar Star", 2 volumes, 1903', price: 480, currency: 'GBP', sold: true }); | |
| 47 | + const out = await connector.normalize({ url: sales[0]!.url, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'sale_results', url: sales[0]!.url, sale: sales[0]!, page: 1, totalLots: null, lots: p.lots } }); | |
| 48 | + expect(attrsOf(out[0]!).categorySlug).toBe('books'); | |
| 49 | + expect(attrsOf(out[0]!).year).toBe(1903); | |
| 50 | + }); | |
| 51 | +}); | |
added
connectors/api/dominic-winter/index.ts
+40 −0
@@ -0,0 +1,40 @@ | ||
| 1 | +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors'; | |
| 2 | +import type { ExtractionResult } from '@rareindex/shared'; | |
| 3 | +import { goauctionPageUrl, parseGoauctionCalendar, parseGoauctionLots, type GoauctionHouse } from '../_g8-auctions-eu-apac-lib/goauction-platform.js'; | |
| 4 | +import { SaleResultsConnector, type HouseConfig, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Dominic Winter Auctioneers (Cirencester — printed books, maps, documents, photographs, Ornithology) — goauction | |
| 8 | + * platform with capitalised routes: /auction-results/ calendar → /Auction/Search?au=<id>&sd=2&pn=N&g=1 grid | |
| 9 | + * (48 lots per page, "Sold for £480"). | |
| 10 | + */ | |
| 11 | +const HOUSE: GoauctionHouse = { base: 'https://www.dominicwinter.co.uk', currency: 'GBP', listStyle: 'search' }; | |
| 12 | + | |
| 13 | +export class DominicWinterConnector extends SaleResultsConnector { | |
| 14 | + readonly version = '1.0.0'; | |
| 15 | + readonly house: HouseConfig = { houseName: 'Dominic Winter Auctioneers', defaultCurrency: 'GBP', location: 'Cirencester, United Kingdom', idKey: 'dominic_winter_lot', premiumIncluded: null, fallbackSlug: 'books', minIntervalMs: 2500, maxPagesPerSale: 25 }; | |
| 16 | + protected override minIntervalMs = 2500; | |
| 17 | + | |
| 18 | + async listSales(ctx: CrawlContext): Promise<SaleRef[]> { | |
| 19 | + const url = String(this.meta.config.resultsUrl ?? `${HOUSE.base}/auction-results/`); | |
| 20 | + await this.throttle(url); | |
| 21 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 22 | + if (!res.success || !res.html) { | |
| 23 | + ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 24 | + return []; | |
| 25 | + } | |
| 26 | + return parseGoauctionCalendar(res.html, HOUSE); | |
| 27 | + } | |
| 28 | + | |
| 29 | + salePageUrl(sale: SaleRef, page: number): string { | |
| 30 | + return goauctionPageUrl(sale, page, HOUSE); | |
| 31 | + } | |
| 32 | + | |
| 33 | + parseSalePage(res: ExtractionResult, sale: SaleRef): ParsedSalePage | null { | |
| 34 | + return res.html ? parseGoauctionLots(res.html, sale, HOUSE) : null; | |
| 35 | + } | |
| 36 | +} | |
| 37 | + | |
| 38 | +export default function createConnector(meta: ConnectorMeta) { | |
| 39 | + return new DominicWinterConnector(meta); | |
| 40 | +} | |
added
connectors/api/dominic-winter/meta.json
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +{ | |
| 2 | + "id": "dominic-winter", | |
| 3 | + "displayName": "Dominic Winter Auctioneers (books, maps, documents) — results", | |
| 4 | + "sourceId": "dominic-winter", | |
| 5 | + "sourceName": "Dominic Winter Auctioneers", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.dominicwinter.co.uk", | |
| 8 | + "module": "api/dominic-winter", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["books", "maps", "historical_documents", "autographs", "photography", "art", "postcards", "stamps", "coins", "medals", "militaria", "vintage_toys", "sports_memorabilia", "music_memorabilia", "movie_posters", "antiques"], | |
| 11 | + "regions": ["GB"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["GBP"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "medium", | |
| 23 | + "trustScore": 0.88, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.dominicwinter.co.uk/terms-and-conditions/", | |
| 26 | + "acquisitionMethod": "server-rendered HTML (goauction platform)", | |
| 27 | + "historicalDepth": "years", | |
| 28 | + "accessNotes": "Dominic Winter Auctioneers (South Cerney, Cirencester; printed books, maps, manuscripts, photographs, ornithology, medals, toys) runs on the goauction platform. Public pages read: /auction-results/ (calendar of past sales: title, date, PDF results, au id) and /Auction/Search?au=<id>&sd=2&pn=N&g=1 (grid of 48 lots per page: lot number, title, image, 'Sold for £480'; unsold lots carry no price). The grid does not state hammer vs premium → buyer_premium_included null. robots.txt disallows /account, /admin, /cms/lotdetailspdf, /elmah, /imagebrowser, /language only. 2.5 s politeness; salesPerRun caps incremental runs; resumable backfill over the calendar. No login, no bidder data.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "resultsUrl": "https://www.dominicwinter.co.uk/auction-results/", | |
| 33 | + "salesPerRun": 2 | |
| 34 | + } | |
| 35 | +} | |
added
connectors/api/doyle/README.md
+8 −0
@@ -0,0 +1,8 @@ | ||
| 1 | +# doyle — Doyle (New York) auction results | |
| 2 | + | |
| 3 | +- **Source**: https://www.doyle.com — `/past-auctions/` (closed sales with date) → `/auction/search/?au=<id>&pp=96&pn=N&g=1` (server-rendered result cards). | |
| 4 | +- **Engine**: `api` (plain HTTPS, honest UA, 10 s crawl-delay per robots.txt). 0 credits. | |
| 5 | +- **Records**: `sale` — price = "Sold for $X" (USD, buyer's premium included per the lot pages), saleDate = sale date from the calendar, lot number, estimate in metadata, `identifiers.doyle_lot_id`. | |
| 6 | +- **Categories**: sale name → department hint, refined by title keywords (`doyleCategory`). Unmapped lots are skipped. | |
| 7 | +- **Cursor**: `{ doneAuctions: string[] }`; backfill walks the calendar oldest→newest and reports progress; `done: true` when every listed sale is processed. | |
| 8 | +- **Smoke**: `set -a; . ./.env; set +a; pnpm tsx connectors/api/_g7-auctions-na-lib/smoke.ts api/doyle --capture results --limit 2` | |
added
connectors/api/doyle/index.test.ts
+83 −0
@@ -0,0 +1,83 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import { fileURLToPath } from 'node:url'; | |
| 5 | +import { runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 7 | +import { parseUsDate } from '../_g7-auctions-na-lib/index.js'; | |
| 8 | +import createConnector, { doyleCategory, parseAuctionHeaderDate, parsePastAuctions, parseResultsPage } from './index.js'; | |
| 9 | + | |
| 10 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 11 | +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 12 | +const connector = createConnector(meta); | |
| 13 | + | |
| 14 | +// Real fragments of https://www.doyle.com/past-auctions/ (captured 2026-09-08): a live sale ("Date:") and an online sale ("Ends:"). | |
| 15 | +const CALENDAR = `<div class="auction-calendar-item calendar-third nv"> <a href='/auction/26bp01-rare-books-autographs--maps?au=9180'> <div class="auction-calendar-image" style=""> <img src="https://goauctiondoyle.blob.core.windows.net/26bp01/26BP01-post-sale-placard-1.png" alt="Rare Books, Autographs & Maps" /> </div> </a> <div class="auction-calendar-text "> <a href='/auction/26bp01-rare-books-autographs--maps?au=9180'> <H3>Rare Books, Autographs & Maps</H3> </a> <div> <a href='/auction/26bp01-rare-books-autographs--maps?au=9180'> <strong> Date: Apr 16, 2026 10:00 EST </strong> <br /> </a> </div> <div class="auction-calendar-buttons-left"> <div class="auction-calendar-buttons-left"> <a href="/auction/26bp01-rare-books-autographs--maps?au=9180" class="btn">View Results</a> </div> </div> </div> </div> | |
| 16 | +<div class="auction-calendar-item calendar-third nv"> <a href='/auction/26kh01-80s-new-york-featuring-the-keith-haring-pop-shop-collection-of-julia-markus?au=9241'> <div class="auction-calendar-image" style=""> <img src="https://goauctiondoyle.goauction.co.uk/placards/26KH01-Placa.jpg" alt="x" /> </div> </a> <div class="auction-calendar-text "> <a href='/auction/26kh01-80s-new-york-featuring-the-keith-haring-pop-shop-collection-of-julia-markus?au=9241'> <H3>'80s New York Featuring the Keith Haring Pop Shop Collection of Julia Markus </H3> </a> <div> <a href='/auction/26kh01-80s-new-york-featuring-the-keith-haring-pop-shop-collection-of-julia-markus?au=9241'> <strong> Ends: Aug 20, 2026 10:00 EST </strong> <br /> </a> </div> </div> </div> | |
| 17 | +<div class="auction-calendar-item calendar-third nv"> <a href='/section-detail/property-from-the-estate-of-jorie-butler-kent?i=62'> <H3>Section without an auction id</H3> </a> </div>`; | |
| 18 | + | |
| 19 | +// Real fragment of https://www.doyle.com/auction/search/?au=9180 (page 1 of 5): a sold lot and an unsold lot, plus the pager. | |
| 20 | +const RESULTS = `<nav class="pagination"><select class='form-control pull-left active-select'><option value='/auction/search/?au=9180&g=1' selected>1</option><option value='/auction/search/?au=9180&pn=2&g=1' >2</option></select><span class='pull-left'> of 5</span><a href='/auction/search/?au=9180&pn=2&g=1' class='btn btn-sm pull-right'><i class='fal fa-chevron-right fa-fw'></i></a></nav> | |
| 21 | +<div class='auction-grid agc agh'><div class='auction-grid-lot col-sm-6 col-md-4 col-lg-4'> <div class="auction-lot"> <div class="auction-lot-image" style="position: relative;"> <a href="/auction/lot/lot-1---clutch-of-five-sauropod-eggs/?lot=1461680&so=0&st=&sto=0&au=9180&ef=&et=&ic=False&sd=1&mc=409&pp=96&pn=1&g=1"> <img src="https://goauctiondoyle.goauction.co.uk/stock/2079710-1-small.jpg?v=63907825355487" alt="Lot 1 - Clutch of five sauropod eggs" /> </a> </div> <div class="auction-lot-text " style="position: relative;"> <a class="anchor-offset" name="1461680"></a> <p class="auction-lot-title"> <a href="/auction/lot/lot-1---clutch-of-five-sauropod-eggs/?lot=1461680&so=0&st=&sto=0&au=9180&ef=&et=&ic=False&sd=1&mc=409&pp=96&pn=1&g=1"> <span class='lot-title cat-146'>Lot 1<br />Clutch of five sauropod eggs</span> </a> </p> <p></p> <p> <strong>Sold for $2,048</strong> <br /> <strong>Estimated at $1,500 - $3,000</strong> </p> </div> <div class="clearfix"></div> </div> </div> | |
| 22 | +<div class='auction-grid-lot col-sm-6 col-md-4 col-lg-4'> <div class="auction-lot"> <div class="auction-lot-image" style="position: relative;"> <a href="/auction/lot/lot-6---a-complete-set-of-the-first-aldine-edition-of-livy/?lot=1468947&so=0&au=9180&pp=96&pn=1&g=1"> <img src="https://goauctiondoyle.goauction.co.uk/stock/2102486-2-small.jpg?v=63909982187970" alt="Lot 6 - A complete set of the first Aldine edition of Livy" /> </a> </div> <div class="auction-lot-text " style="position: relative;"> <a class="anchor-offset" name="1468947"></a> <p class="auction-lot-title"> <a href="/auction/lot/lot-6---a-complete-set-of-the-first-aldine-edition-of-livy/?lot=1468947&so=0&au=9180&pp=96&pn=1&g=1"> <span class='lot-title cat-4'>Lot 6<br />A complete set of the first Aldine edition of Livy</span> </a> </p> <p><p class="stockfields-list">Property sold to benefit Albertus Magnus High School</p></p> <p> <strong>Estimated at $6,000 - $9,000</strong> </p> </div> <div class="clearfix"></div> </div> </div></div>`; | |
| 23 | + | |
| 24 | +describe('doyle', () => { | |
| 25 | + runFixtureSuite(connector, it, expect); | |
| 26 | + | |
| 27 | + it('parses the past-auctions calendar (Date: and Ends: labels, skips entries without an auction id)', () => { | |
| 28 | + const list = parsePastAuctions(CALENDAR); | |
| 29 | + expect(list.length).toBe(2); | |
| 30 | + expect(list[0]).toEqual({ auId: '9180', code: '26BP01', name: 'Rare Books, Autographs & Maps', dateText: 'Apr 16, 2026 10:00 EST', url: 'https://www.doyle.com/auction/26bp01-rare-books-autographs--maps?au=9180' }); | |
| 31 | + expect(list[1]).toMatchObject({ auId: '9241', code: '26KH01', dateText: 'Aug 20, 2026 10:00 EST' }); | |
| 32 | + expect(list[1]!.name.startsWith("'80s New York")).toBe(true); | |
| 33 | + // 10:00 US Eastern (EST label) → 15:00 UTC | |
| 34 | + expect(parseUsDate(list[0]!.dateText)?.toISOString()).toBe('2026-04-16T15:00:00.000Z'); | |
| 35 | + expect(parseUsDate('Apr 16, 2026')?.toISOString()).toBe('2026-04-16T00:00:00.000Z'); | |
| 36 | + }); | |
| 37 | + | |
| 38 | + it('parses result cards: lot number/title split, price, estimate, lot id, canonical url, pagination', () => { | |
| 39 | + const auction = parsePastAuctions(CALENDAR)[0]!; | |
| 40 | + const p = parseResultsPage(RESULTS, auction, 1); | |
| 41 | + expect(p.kind).toBe('doyle_results_page'); | |
| 42 | + expect(p.hasNext).toBe(true); | |
| 43 | + expect(p.lots.length).toBe(2); | |
| 44 | + expect(p.lots[0]).toMatchObject({ lotId: '1461680', lotNumber: '1', title: 'Clutch of five sauropod eggs', soldPrice: 2048, estimateLow: 1500, estimateHigh: 3000, categoryId: '146', url: 'https://www.doyle.com/auction/lot/lot-1---clutch-of-five-sauropod-eggs/?lot=1461680&au=9180' }); | |
| 45 | + expect(p.lots[0]!.image).toContain('/stock/2079710-1-small.jpg'); | |
| 46 | + expect(p.lots[1]).toMatchObject({ lotId: '1468947', lotNumber: '6', soldPrice: null, soldText: null, estimateLow: 6000, estimateHigh: 9000, categoryId: '4' }); | |
| 47 | + // last page: no pn=2 link | |
| 48 | + expect(parseResultsPage(RESULTS.replace(/pn=2/g, 'pn=9'), auction, 1).hasNext).toBe(false); | |
| 49 | + expect(parseAuctionHeaderDate('<html><body><div>Request an Auction Estimate Apr 16, 2026 10:00 EST Rare Books</div></body></html>')).toBe('Apr 16, 2026 10:00 EST'); | |
| 50 | + }); | |
| 51 | + | |
| 52 | + it('normalises only sold lots, dated by the calendar, USD with buyer premium included', async () => { | |
| 53 | + const auction = parsePastAuctions(CALENDAR)[0]!; | |
| 54 | + const payload = parseResultsPage(RESULTS, auction, 1); | |
| 55 | + const out = await connector.normalize({ url: 'https://www.doyle.com/auction/search/?au=9180&pp=96&g=1', externalId: 'auction:9180:page:1', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload }); | |
| 56 | + expect(out.length).toBe(1); | |
| 57 | + const s = out[0]!; | |
| 58 | + if (s.kind !== 'sale') throw new Error('expected sale'); | |
| 59 | + expect(s).toMatchObject({ price: 2048, currency: 'USD', buyerPremiumIncluded: true, auctionHouse: 'Doyle', lotNumber: '1', location: 'US', externalId: '1461680' }); | |
| 60 | + expect(s.saleDate.toISOString()).toBe('2026-04-16T15:00:00.000Z'); | |
| 61 | + expect(s.attributes.categorySlug).toBe('fossils'); | |
| 62 | + expect(s.attributes.identifiers).toEqual({ doyle_lot_id: '1461680' }); | |
| 63 | + expect(s.attributes.metadata).toMatchObject({ auction_id: '9180', auction_code: '26BP01', estimate_low: 1500, estimate_high: 3000, doyle_category_id: '146' }); | |
| 64 | + // no calendar date → nothing is emitted (never guess a sale date) | |
| 65 | + expect(await connector.normalize({ url: 'x', externalId: 'y', kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { ...payload, auction: { ...auction, dateText: null } } })).toEqual([]); | |
| 66 | + }); | |
| 67 | + | |
| 68 | + it('maps sale departments and titles to taxonomy slugs', () => { | |
| 69 | + expect(doyleCategory('Rare Books, Autographs & Maps', 'A complete set of the first Aldine edition of Livy')).toBe('books'); | |
| 70 | + expect(doyleCategory('Rare Books, Autographs & Maps', 'Map of the Americas, engraved, 1650')).toBe('maps'); | |
| 71 | + expect(doyleCategory('Rare Books, Autographs & Maps', 'Thomas Jefferson autograph letter signed to Charles Willson Peale')).toBe('historical_documents'); | |
| 72 | + expect(doyleCategory('Fine Jewelry', 'Platinum, diamond and sapphire ring')).toBe('jewelry'); | |
| 73 | + expect(doyleCategory('Important Watches', 'Rolex stainless steel Submariner wristwatch Ref. 5513')).toBe('rolex'); | |
| 74 | + expect(doyleCategory('Coins, Bank Notes, Stamps & Collectibles', '1921 Morgan silver dollar MS65')).toBe('coins'); | |
| 75 | + expect(doyleCategory('Coins, Bank Notes, Stamps & Collectibles', 'United States $20 legal tender banknote 1880')).toBe('banknotes'); | |
| 76 | + expect(doyleCategory('English & Continental Furniture, Old Master Paintings, Silver', 'George III mahogany chest of drawers')).toBe('antiques'); | |
| 77 | + expect(doyleCategory('English & Continental Furniture, Old Master Paintings, Silver', 'Sterling silver tea service, Tiffany & Co.')).toBe('silver'); | |
| 78 | + expect(doyleCategory('Post-War & Contemporary Art', 'Keith Haring, Untitled, screenprint, edition of 100')).toBe('contemporary_art'); | |
| 79 | + expect(doyleCategory('Doyle at Home', 'Meissen porcelain figure group')).toBe('porcelain'); | |
| 80 | + expect(doyleCategory('The Collection of a Florida Bibliophile', 'Hemingway, first edition, dust jacket')).toBe('books'); | |
| 81 | + expect(doyleCategory('Entertaining with Style', 'Untitled mixed lot')).toBeNull(); | |
| 82 | + }); | |
| 83 | +}); | |
added
connectors/api/doyle/index.ts
+279 −0
@@ -0,0 +1,279 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { hintFromLabel, slugFromTitle } from '../_auction-lib/categories.js'; | |
| 5 | +import { amount, decodeEntities, houseCategory, isBundleTitle, lotAttributes, makeSale, parseUsDate, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js'; | |
| 6 | + | |
| 7 | +const SITE = 'https://www.doyle.com'; | |
| 8 | +const HOUSE = 'Doyle'; | |
| 9 | +const PARSER_VERSION = '1.0.0'; | |
| 10 | +const PAGE_SIZE = 96; | |
| 11 | + | |
| 12 | +export const AuctionSchema = z.object({ auId: z.string(), code: z.string().nullable(), name: z.string(), dateText: z.string().nullable(), url: z.string() }); | |
| 13 | +export const LotSchema = z.object({ | |
| 14 | + lotId: z.string(), | |
| 15 | + lotNumber: z.string().nullable(), | |
| 16 | + title: z.string(), | |
| 17 | + url: z.string(), | |
| 18 | + image: z.string().nullable(), | |
| 19 | + soldText: z.string().nullable(), | |
| 20 | + soldPrice: z.number().nullable(), | |
| 21 | + estimateText: z.string().nullable(), | |
| 22 | + estimateLow: z.number().nullable(), | |
| 23 | + estimateHigh: z.number().nullable(), | |
| 24 | + categoryId: z.string().nullable(), | |
| 25 | +}); | |
| 26 | +export const PayloadSchema = z.object({ kind: z.literal('doyle_results_page'), auction: AuctionSchema, page: z.number(), hasNext: z.boolean(), lots: z.array(LotSchema) }); | |
| 27 | +export type Payload = z.infer<typeof PayloadSchema>; | |
| 28 | +export type DoyleAuction = z.infer<typeof AuctionSchema>; | |
| 29 | + | |
| 30 | +/** /past-auctions/ → calendar items (newest first, as displayed). */ | |
| 31 | +export function parsePastAuctions(htmlText: string): DoyleAuction[] { | |
| 32 | + const $ = H.load(htmlText); | |
| 33 | + const out: DoyleAuction[] = []; | |
| 34 | + const seen = new Set<string>(); | |
| 35 | + $('.auction-calendar-item').each((_, el) => { | |
| 36 | + const item = $(el); | |
| 37 | + const href = item.find('a[href*="/auction/"][href*="au="]').first().attr('href'); | |
| 38 | + const auId = href?.match(/[?&]au=(\d+)/)?.[1]; | |
| 39 | + if (!href || !auId || seen.has(auId)) return; | |
| 40 | + seen.add(auId); | |
| 41 | + const name = H.text(item.find('h3, H3').first()) ?? ''; | |
| 42 | + // Live sales print "Date: …"; online-only sales print "Ends: …" (the close date = sale date). | |
| 43 | + const dateText = H.text(item.find('strong').filter((_, s) => /^\s*(Date|Ends?):/i.test($(s).text())).first())?.replace(/^(Date|Ends?):\s*/i, '') ?? null; | |
| 44 | + const code = href.match(/\/auction\/([0-9a-z]+)-/i)?.[1]?.toUpperCase() ?? null; | |
| 45 | + out.push({ auId, code, name: decodeEntities(name), dateText, url: `${SITE}${href.split('#')[0]}` }); | |
| 46 | + }); | |
| 47 | + return out; | |
| 48 | +} | |
| 49 | + | |
| 50 | +/** "Mon d, yyyy hh:mm EST" header on an auction page (fallback when the calendar date is missing, e.g. seeds). */ | |
| 51 | +export function parseAuctionHeaderDate(htmlText: string): string | null { | |
| 52 | + const text = htmlText.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/g, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' '); | |
| 53 | + return text.match(/\b((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2}, 20\d{2}(?: \d{1,2}:\d{2} [A-Z]{2,4})?)\b/)?.[1] ?? null; | |
| 54 | +} | |
| 55 | + | |
| 56 | +function lotUrl(href: string): string { | |
| 57 | + const u = new URL(href.replace(/&/g, '&'), SITE); | |
| 58 | + const lot = u.searchParams.get('lot'); | |
| 59 | + const au = u.searchParams.get('au'); | |
| 60 | + const out = new URL(u.pathname, SITE); | |
| 61 | + if (lot) out.searchParams.set('lot', lot); | |
| 62 | + if (au) out.searchParams.set('au', au); | |
| 63 | + return out.toString(); | |
| 64 | +} | |
| 65 | + | |
| 66 | +/** One results page (/auction/search/?au=…&pp=96&pn=N&g=1) → lot cards. */ | |
| 67 | +export function parseResultsPage(htmlText: string, auction: DoyleAuction, page: number): Payload { | |
| 68 | + const $ = H.load(htmlText); | |
| 69 | + const lots: Payload['lots'] = []; | |
| 70 | + const seen = new Set<string>(); | |
| 71 | + $('div.auction-lot').each((_, el) => { | |
| 72 | + const card = $(el); | |
| 73 | + const link = card.find('p.auction-lot-title a').first(); | |
| 74 | + const href = link.attr('href') ?? card.find('a[href*="/auction/lot/"]').first().attr('href'); | |
| 75 | + const lotId = href?.match(/[?&]lot=(\d+)/)?.[1]; | |
| 76 | + if (!href || !lotId || seen.has(lotId)) return; | |
| 77 | + seen.add(lotId); | |
| 78 | + const span = card.find('span.lot-title').first(); | |
| 79 | + const parts = (span.html() ?? '').split(/<br\s*\/?>/i); | |
| 80 | + const lotNumber = H.text(H.load(parts[0] ?? '')('body'))?.replace(/^Lot\s*/i, '').trim() || null; | |
| 81 | + const title = decodeEntities(H.text(H.load(parts.slice(1).join(' '))('body')) ?? '') || decodeEntities(H.text(span) ?? ''); | |
| 82 | + if (!title) return; | |
| 83 | + const categoryId = (span.attr('class') ?? '').match(/\bcat-(\d+)/)?.[1] ?? null; | |
| 84 | + let soldText: string | null = null; | |
| 85 | + let estimateText: string | null = null; | |
| 86 | + card.find('strong').each((_, s) => { | |
| 87 | + const t = H.text($(s)) ?? ''; | |
| 88 | + if (/^Sold for/i.test(t)) soldText = t; | |
| 89 | + else if (/^Estimate/i.test(t)) estimateText = t; | |
| 90 | + }); | |
| 91 | + const est = (estimateText ?? '').match(/\$([\d,]+)\s*-\s*\$([\d,]+)/); | |
| 92 | + const img = card.find('img[src*="/stock/"]').first().attr('src') ?? null; | |
| 93 | + lots.push({ | |
| 94 | + lotId, | |
| 95 | + lotNumber, | |
| 96 | + title, | |
| 97 | + url: lotUrl(href), | |
| 98 | + image: img, | |
| 99 | + soldText, | |
| 100 | + soldPrice: soldText ? amount((soldText as string).replace(/^Sold for/i, '')) : null, | |
| 101 | + estimateText, | |
| 102 | + estimateLow: est ? amount(est[1]) : null, | |
| 103 | + estimateHigh: est ? amount(est[2]) : null, | |
| 104 | + categoryId, | |
| 105 | + }); | |
| 106 | + }); | |
| 107 | + const hasNext = new RegExp(`[?&](?:amp;)?pn=${page + 1}(?:&|'|"|$)`).test(htmlText); | |
| 108 | + return { kind: 'doyle_results_page', auction, page, hasNext, lots }; | |
| 109 | +} | |
| 110 | + | |
| 111 | +/** Doyle sale name + lot title → taxonomy slug (null when nothing confident). */ | |
| 112 | +export function doyleCategory(saleName: string, title: string): string | null { | |
| 113 | + const s = saleName.toLowerCase(); | |
| 114 | + // Natural history lots appear inside book/decorative sales; decide them before department fallbacks | |
| 115 | + // (and before the handbag keyword "clutch" can misfire on "clutch of eggs"). | |
| 116 | + if (/\b(meteorite|pallasite|chondrite)\b/i.test(title)) return 'meteorites'; | |
| 117 | + if (/\b(fossil|ammonite|trilobite|dinosaur|sauropod|megalodon|mammoth|mosasaur|petrified|fossilized|coprolite)\b/i.test(title)) return 'fossils'; | |
| 118 | + if (/\b(mineral specimen|geode|quartz cluster|amethyst|tourmaline|fluorite|azurite|malachite|crystal cluster|agate slice)\b/i.test(title)) return 'minerals'; | |
| 119 | + if (/couture|handbag|fashion|luxury accessor/.test(s)) return slugFromTitle(title, 'fashion') ?? 'fashion_streetwear'; | |
| 120 | + if (/book|autograph|map|manuscript|bibliophil|librar|print(ed)? & manuscript/.test(s)) return slugFromTitle(title, 'books') ?? 'books'; | |
| 121 | + if (/photograph/.test(s)) return slugFromTitle(title, 'photographs') ?? 'photography'; | |
| 122 | + if (/jewel|gem/.test(s) && !/watch/.test(s)) return slugFromTitle(title, 'jewelry') ?? 'jewelry'; | |
| 123 | + if (/watch/.test(s)) return slugFromTitle(title, /\b(watch|wristwatch|chronograph|pocket watch)\b/i.test(title) ? 'watches' : 'jewelry') ?? 'jewelry'; | |
| 124 | + if (/coin|bank ?note|stamp|currency|numismat/.test(s)) return slugFromTitle(title, 'coins') ?? 'coins'; | |
| 125 | + // Silver-only sales; mixed sales ("… Furniture, Old Master Paintings, Silver") fall through to the title sweep below. | |
| 126 | + if (/silver|vertu/.test(s) && !/furniture|painting|decorative|works of art/.test(s)) return slugFromTitle(title, 'silver') ?? 'silver'; | |
| 127 | + // Mixed sales ("English & Continental Furniture, Old Master Paintings, Silver"): let the title decide, default antiques. | |
| 128 | + if (/furniture|decorative|works of art|at home|estate|collects/.test(s) && /painting|silver|art\b/.test(s)) { | |
| 129 | + if (/\b(oil on|acrylic|watercolou?r|gouache|lithograph|etching|engraving|screenprint|woodcut|drawing|pastel|bronze|sculpture|mixed media)\b/i.test(title)) return slugFromTitle(title, 'art') ?? 'art'; | |
| 130 | + return slugFromTitle(title, 'furniture') ?? 'antiques'; | |
| 131 | + } | |
| 132 | + if (/contemporary|post-war|modern art/.test(s)) return slugFromTitle(title, 'contemporary') ?? 'contemporary_art'; | |
| 133 | + if (/painting|prints|drawing|american art|european art|impressionist|old master|fine art|sculpture|works on paper|artist/.test(s)) return slugFromTitle(title, 'art') ?? 'art'; | |
| 134 | + if (/design|mid-century|20th century decorative/.test(s)) return slugFromTitle(title, 'design') ?? 'design_furniture'; | |
| 135 | + if (/asian|chinese|japanese|russian|furniture|decorative|english|continental|at home|estate|collects|interior|american story|americana|works of art|antique|rug|carpet/.test(s)) return slugFromTitle(title, 'furniture') ?? 'antiques'; | |
| 136 | + const generic = houseCategory(saleName, title, null); | |
| 137 | + if (generic) return generic; | |
| 138 | + return hintFromLabel(saleName) === 'unknown' ? null : 'antiques'; | |
| 139 | +} | |
| 140 | + | |
| 141 | +/** | |
| 142 | + * Doyle (New York) — auction results. Public server-rendered pages over plain HTTPS: /past-auctions/ (calendar of | |
| 143 | + * closed sales with date) and /auction/search/?au=<id>&pp=96&pn=N&g=1 (result cards "Sold for $X", estimate). | |
| 144 | + * robots.txt asks for crawl-delay 10, honoured. See meta.json accessNotes. | |
| 145 | + */ | |
| 146 | +export class DoyleConnector extends BaseConnector { | |
| 147 | + readonly version = '1.0.0'; | |
| 148 | + readonly parserVersion = PARSER_VERSION; | |
| 149 | + protected override minIntervalMs = 10_000; // robots.txt crawl-delay: 10 | |
| 150 | + | |
| 151 | + private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> { | |
| 152 | + await this.throttle(url); | |
| 153 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 154 | + if (!res.success || !res.html) { | |
| 155 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 156 | + return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 157 | + } | |
| 158 | + return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 159 | + } | |
| 160 | + | |
| 161 | + private async listAuctions(ctx: CrawlContext): Promise<DoyleAuction[]> { | |
| 162 | + if (ctx.options.seeds?.length) { | |
| 163 | + const out: DoyleAuction[] = []; | |
| 164 | + for (const s of ctx.options.seeds) { | |
| 165 | + const auId = s.match(/[?&]au=(\d+)/)?.[1]; | |
| 166 | + if (!auId) continue; | |
| 167 | + const code = s.match(/\/auction\/([0-9a-z]+)-/i)?.[1]?.toUpperCase() ?? null; | |
| 168 | + out.push({ auId, code, name: '', dateText: null, url: s.startsWith('http') ? s : `${SITE}${s}` }); | |
| 169 | + } | |
| 170 | + return out; | |
| 171 | + } | |
| 172 | + const r = await this.html(ctx, `${SITE}/past-auctions/`); | |
| 173 | + if (!r.html) return []; | |
| 174 | + const list = parsePastAuctions(r.html); | |
| 175 | + if (!list.length) ctx.anomaly('selector_missing', 'past-auctions: no .auction-calendar-item found'); | |
| 176 | + return list; | |
| 177 | + } | |
| 178 | + | |
| 179 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 180 | + const auctionsPerRun = ctx.options.mode === 'probe' ? 1 : Number(this.meta.config.auctionsPerRun ?? 2); | |
| 181 | + const maxPages = ctx.options.mode === 'probe' ? 1 : Number(this.meta.config.pagesPerAuction ?? 15); | |
| 182 | + const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []); | |
| 183 | + const all = await this.listAuctions(ctx); | |
| 184 | + // Incremental: newest first (list order). Backfill: oldest first so progress walks the archive forward. | |
| 185 | + const ordered = ctx.options.mode === 'backfill' ? [...all].reverse() : all; | |
| 186 | + const pending = ordered.filter((a) => !done.has(a.auId)); | |
| 187 | + let processed = 0; | |
| 188 | + let count = 0; | |
| 189 | + let items = 0; | |
| 190 | + for (const auction of pending) { | |
| 191 | + if (ctx.signal?.aborted || processed >= auctionsPerRun || this.reached(ctx, count)) break; | |
| 192 | + if (!auction.dateText) { | |
| 193 | + // Seeds carry no calendar date: read the auction page header once. | |
| 194 | + const page = await this.html(ctx, auction.url); | |
| 195 | + if (page.html) { | |
| 196 | + auction.dateText = parseAuctionHeaderDate(page.html); | |
| 197 | + if (!auction.name) auction.name = decodeEntities(H.text(H.load(page.html)('h1').first()) ?? H.text(H.load(page.html)('title')) ?? ''); | |
| 198 | + } | |
| 199 | + if (!auction.dateText) ctx.anomaly('missing_auction_date', auction.url); | |
| 200 | + } | |
| 201 | + let complete = true; | |
| 202 | + for (let page = 1; page <= maxPages; page++) { | |
| 203 | + if (ctx.signal?.aborted || this.reached(ctx, count)) { | |
| 204 | + complete = false; | |
| 205 | + break; | |
| 206 | + } | |
| 207 | + const url = `${SITE}/auction/search/?au=${auction.auId}&pp=${PAGE_SIZE}${page > 1 ? `&pn=${page}` : ''}&g=1`; | |
| 208 | + const r = await this.html(ctx, url); | |
| 209 | + if (!r.html) { | |
| 210 | + complete = false; | |
| 211 | + break; | |
| 212 | + } | |
| 213 | + const payload = parseResultsPage(r.html, auction, page); | |
| 214 | + if (payload.lots.length === 0) { | |
| 215 | + if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no lot cards`); | |
| 216 | + break; | |
| 217 | + } | |
| 218 | + count++; | |
| 219 | + items += payload.lots.length; | |
| 220 | + yield { url, externalId: `auction:${auction.auId}:page:${page}`, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt }; | |
| 221 | + if (!payload.hasNext) break; | |
| 222 | + if (page === maxPages) complete = false; | |
| 223 | + } | |
| 224 | + processed++; | |
| 225 | + if (complete) done.add(auction.auId); | |
| 226 | + const doneList = [...done].slice(-500); | |
| 227 | + await ctx.setCursor({ doneAuctions: doneList, updatedAt: new Date().toISOString() }); | |
| 228 | + await ctx.progress({ page: all.filter((a) => done.has(a.auId)).length, totalPages: all.length, itemsProcessed: items, cursor: { doneAuctions: doneList } }); | |
| 229 | + } | |
| 230 | + if (ctx.options.mode === 'backfill' && all.length && all.every((a) => done.has(a.auId))) await ctx.setCursor({ doneAuctions: [...done].slice(-500), done: true, updatedAt: new Date().toISOString() }); | |
| 231 | + } | |
| 232 | + | |
| 233 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 234 | + const p = PayloadSchema.parse(raw.payload); | |
| 235 | + const saleDate = parseUsDate(p.auction.dateText); | |
| 236 | + if (!saleDate || saleDate.getTime() > Date.now() + 86_400_000) return []; | |
| 237 | + const out: NormalizedRecord[] = []; | |
| 238 | + for (const l of p.lots) { | |
| 239 | + if (!l.soldPrice) continue; // unsold / withdrawn / passed: no realised price | |
| 240 | + const slug = doyleCategory(p.auction.name, l.title); | |
| 241 | + if (!slug) continue; | |
| 242 | + const g = saleGrade(l.title); | |
| 243 | + const attributes = lotAttributes({ | |
| 244 | + categorySlug: slug, | |
| 245 | + name: l.title, | |
| 246 | + year: safeYear(l.title), | |
| 247 | + identifiers: { doyle_lot_id: l.lotId }, | |
| 248 | + metadata: { auction_id: p.auction.auId, auction_code: p.auction.code, auction_name: p.auction.name, estimate_low: l.estimateLow, estimate_high: l.estimateHigh, doyle_category_id: l.categoryId }, | |
| 249 | + }); | |
| 250 | + out.push( | |
| 251 | + makeSale({ | |
| 252 | + meta: this.meta, | |
| 253 | + sourceUrl: l.url, | |
| 254 | + externalId: l.lotId, | |
| 255 | + rawTitle: l.title, | |
| 256 | + attributes, | |
| 257 | + price: l.soldPrice, | |
| 258 | + currency: 'USD', | |
| 259 | + saleDate, | |
| 260 | + // Doyle lot pages print "Includes Buyer's Premium" under "Sold for". | |
| 261 | + buyerPremiumIncluded: true, | |
| 262 | + auctionHouse: HOUSE, | |
| 263 | + lotNumber: l.lotNumber, | |
| 264 | + imageUrls: l.image ? [l.image] : [], | |
| 265 | + location: 'US', | |
| 266 | + observedAt: raw.fetchedAt, | |
| 267 | + parserVersion: PARSER_VERSION, | |
| 268 | + grader: g.grader, | |
| 269 | + grade: g.grade, | |
| 270 | + isBundle: isBundleTitle(l.title), | |
| 271 | + confidence: 0.85, | |
| 272 | + }), | |
| 273 | + ); | |
| 274 | + } | |
| 275 | + return out; | |
| 276 | + } | |
| 277 | +} | |
| 278 | + | |
| 279 | +export default (meta: ConnectorMeta) => new DoyleConnector(meta); | |
added
connectors/api/doyle/meta.json
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +{ | |
| 2 | + "id": "doyle", | |
| 3 | + "displayName": "Doyle (auction results)", | |
| 4 | + "sourceId": "doyle", | |
| 5 | + "sourceName": "Doyle", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://www.doyle.com", | |
| 8 | + "module": "api/doyle", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["books", "maps", "historical_documents", "autographs", "photography", "art", "contemporary_art", "jewelry", "gemstones", "other_watches", "rolex", "coins", "banknotes", "stamps", "silver", "antiques", "porcelain", "glass_crystal", "clocks", "design_furniture", "luxury_handbags", "fashion_streetwear", "fossils", "minerals"], | |
| 11 | + "regions": ["US"], | |
| 12 | + "country": "US", | |
| 13 | + "languages": ["en"], | |
| 14 | + "currency": ["USD"], | |
| 15 | + "supportsListings": false, | |
| 16 | + "supportsSold": true, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 1440, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.9, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.doyle.com/terms/", | |
| 27 | + "acquisitionMethod": "server-rendered results pages (HTML)", | |
| 28 | + "historicalDepth": "years", | |
| 29 | + "accessNotes": "Plain HTTPS with the RareIndex user agent on public pages of www.doyle.com. robots.txt (User-agent *) disallows only /account/*, /admin/*, /cms/lotdetailspdf/, /elmah/*, /error/*, /imagebrowser/*, /language/*, /profile/*, /seo/* and asks crawl-delay 10 → 10 s between requests (minIntervalMs 10000). Pages read: /past-auctions/ (calendar of closed sales — 72 entries covering roughly the last 8–9 months, each with name, code and 'Date: Mon d, yyyy hh:mm EST', which is the sale date we store) and the per-sale results pages /auction/search/?au=<id>&pp=96&pn=N&g=1 (96 server-rendered cards per page: lot number, title, 'Sold for $X', 'Estimated at $A - $B', image, lot URL). Prices are USD; the lot detail pages state 'Includes Buyer's Premium' under 'Sold for' (verified live 2026-09-08) → buyerPremiumIncluded=true. Lots without 'Sold for' (unsold/withdrawn) are skipped. Not fetched: lot detail pages (all needed fields are on the results grid), live bidding (live.doyle.com), account/registration pages, PDFs. 0 credits, no JS rendering.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "config": { | |
| 33 | + "auctionsPerRun": 2, | |
| 34 | + "pagesPerAuction": 15 | |
| 35 | + } | |
| 36 | +} | |
added
connectors/api/dunbar-sloane/README.md
+8 −0
@@ -0,0 +1,8 @@ | ||
| 1 | +# dunbar-sloane — Dunbar Sloane (NZ) realised prices | |
| 2 | + | |
| 3 | +Tandem Auctions platform: /previous-auctions → /<id>/catalogue?page=N ('Realised: $140 + premium' = hammer, NZD). | |
| 4 | + | |
| 5 | +- Built on `_g8-auctions-eu-apac-lib/sale-results.ts` (shared crawl/normalize/backfill): `sale` for lots with a | |
| 6 | + published result, `auction_lot` (ended) for unsold lots; native currency; premium basis as labelled by the source | |
| 7 | + (see meta.json accessNotes). Identifiers: `<house>_lot = <sale id>/<lot number>`. | |
| 8 | +- Fixtures: `pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts dunbar-sloane --save`. | |
added
connectors/api/dunbar-sloane/index.test.ts
+63 −0
@@ -0,0 +1,63 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 3 | +import { ConnectorMetaSchema } from '@rareindex/connectors'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 6 | +import createConnector, { parseCataloguePage, parseNzDate, parsePreviousAuctions } from './index.js'; | |
| 7 | + | |
| 8 | +const meta = ConnectorMetaSchema.parse(metaJson); | |
| 9 | +const connector = createConnector(meta); | |
| 10 | +const attrsOf = (r: NormalizedRecord) => { | |
| 11 | + if (!('attributes' in r)) throw new Error(`record kind ${r.kind} has no attributes`); | |
| 12 | + return r.attributes; | |
| 13 | +}; | |
| 14 | + | |
| 15 | +const INDEX = `<div class="card auction-card mb-3"> <div class="card-body"> <span class="badge bg-secondary">Timed Auction</span> <h2 class="card-title">Vintage Toy Auction</h2> <h3 class="card-subtitle text-muted mb-2">Thursday, 2 - Wednesday, 15 October 2025</h3> <a class="btn btn-dark" href="/1858">Auction Details</a> </div></div> | |
| 16 | +<div class="card auction-card mb-3"><div class="card-body"><h2 class="card-title">Sterling Silver & Accoutrements</h2><h3 class="card-subtitle text-muted mb-2">Wednesday, 21 June 2023</h3><a class="btn btn-dark" href="/1745">Auction Details</a></div></div>`; | |
| 17 | + | |
| 18 | +const CAT = `<h1 class="page-header">Sterling Silver & Accoutrements <small class="text-muted">Wednesday, 21 June 2023 / 10:00 am start</small></h1><button type="button" class="btn dropdown-toggle"> Page 1 of 6 </button> | |
| 19 | +<div class="lot-grid row"> <div class="col mb-4"> <div id="0001" class="card h-100" data-ln="0001"> <a href="/1745/catalogue/0001"> <img class="card-img-top" src="/img/loading600.gif" data-src="https://d1v8q4.tandemauctions.com/1745/0001/sq/875763a3.webp" alt="Geo II S/S Sugar Caster"> </a> <div class="card-body"> <p class="card-subtitle mb-2 text-muted"> <a href="/1745/catalogue/0001"> 0001 </a> </p> <p class="card-text"> Geo II S/S Sugar Caster<br /> <small class="estimate">Estimate: $200 - $350</small> </p> </div> <div class="card-footer realised"> Realised: $140 <small>+ premium</small> </div> </div> </div> | |
| 20 | +<div class="col mb-4"> <div id="0002" class="card h-100" data-ln="0002"> <a href="/1745/catalogue/0002"><img data-src="https://d1v8q4.tandemauctions.com/1745/0002/sq/x.webp" alt="Rolex Oyster Perpetual wristwatch"></a> <div class="card-body"> <p class="card-text"> Rolex Oyster Perpetual wristwatch<br /> <small class="estimate">Estimate: $3,000 - $5,000</small> </p> </div> </div> </div></div>`; | |
| 21 | + | |
| 22 | +describe('dunbar-sloane', () => { | |
| 23 | + runFixtureSuite(connector, it, expect); | |
| 24 | + | |
| 25 | + it('fixtures: NZD hammer ("plus premium") results', async () => { | |
| 26 | + let sales = 0; | |
| 27 | + for (const name of listFixtures('dunbar-sloane')) { | |
| 28 | + for (const r of await connector.normalize(loadFixture('dunbar-sloane', name).raw)) { | |
| 29 | + if (!('attributes' in r)) continue; | |
| 30 | + expect(r.attributes.identifiers.dunbar_sloane_lot).toMatch(/^\d+\/\S+$/); | |
| 31 | + expect(r.sourceUrl).toMatch(/^https:\/\/auctions\.dunbarsloane\.co\.nz\/\d+\/catalogue\//); | |
| 32 | + if (r.kind === 'sale') { | |
| 33 | + sales++; | |
| 34 | + expect(r.currency).toBe('NZD'); | |
| 35 | + expect(r.buyerPremiumIncluded).toBe(false); | |
| 36 | + expect(r.auctionHouse).toBe('Dunbar Sloane'); | |
| 37 | + } | |
| 38 | + } | |
| 39 | + } | |
| 40 | + expect(sales).toBeGreaterThan(10); | |
| 41 | + }); | |
| 42 | + | |
| 43 | + it('parses NZ dates, the previous-auctions list and a catalogue page', async () => { | |
| 44 | + expect(parseNzDate('Thursday, 2 - Wednesday, 15 October 2025')).toBe('2025-10-02T00:00:00.000Z'); | |
| 45 | + expect(parseNzDate('Wednesday, 21 June 2023 / 10:00 am start')).toBe('2023-06-21T00:00:00.000Z'); | |
| 46 | + expect(parseNzDate('Friday, 17 - Tuesday, 28 October 2025')).toBe('2025-10-17T00:00:00.000Z'); | |
| 47 | + const sales = parsePreviousAuctions(INDEX); | |
| 48 | + expect(sales.map((s) => s.id)).toEqual(['1858', '1745']); | |
| 49 | + expect(sales[0]).toMatchObject({ title: 'Vintage Toy Auction', url: 'https://auctions.dunbarsloane.co.nz/1858/catalogue', date: '2025-10-02T00:00:00.000Z' }); | |
| 50 | + const p = parseCataloguePage(CAT, sales[1]!, 1)!; | |
| 51 | + expect(p.hasMore).toBe(true); | |
| 52 | + expect(parseCataloguePage(CAT, sales[1]!, 6)!.hasMore).toBe(false); | |
| 53 | + expect(p.sale).toMatchObject({ title: 'Sterling Silver & Accoutrements', date: '2023-06-21T00:00:00.000Z' }); | |
| 54 | + expect(p.lots[0]).toMatchObject({ lotNo: '0001', title: 'Geo II S/S Sugar Caster', price: 140, currency: 'NZD', premiumIncluded: false, estimateLow: 200, estimateHigh: 350, sold: true, url: 'https://auctions.dunbarsloane.co.nz/1745/catalogue/0001', image: 'https://d1v8q4.tandemauctions.com/1745/0001/sq/875763a3.webp' }); | |
| 55 | + expect(p.lots[1]).toMatchObject({ lotNo: '0002', price: null, sold: false, estimateLow: 3000, estimateHigh: 5000 }); | |
| 56 | + const out = await connector.normalize({ url: sales[1]!.url, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'sale_results', url: sales[1]!.url, sale: { ...sales[1]!, ...p.sale, extra: sales[1]!.extra }, page: 1, totalLots: null, lots: p.lots } }); | |
| 57 | + expect(out.map((r) => r.kind)).toEqual(['sale', 'auction_lot']); | |
| 58 | + expect(attrsOf(out[0]!).categorySlug).toBe('silver'); | |
| 59 | + expect(attrsOf(out[1]!).categorySlug).toBe('rolex'); | |
| 60 | + expect(connector.salePageUrl(sales[1]!, 2)).toBe('https://auctions.dunbarsloane.co.nz/1745/catalogue?page=2'); | |
| 61 | + expect(parseCataloguePage('<html>x</html>', sales[1]!, 1)).toBeNull(); | |
| 62 | + }); | |
| 63 | +}); | |
added
connectors/api/dunbar-sloane/index.ts
+114 −0
@@ -0,0 +1,114 @@ | ||
| 1 | +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors'; | |
| 2 | +import type { ExtractionResult } from '@rareindex/shared'; | |
| 3 | +import { parseEuMoney } from '../_g8-auctions-eu-apac-lib/index.js'; | |
| 4 | +import { SaleResultsConnector, absolute, chunksBetween, pick, textOf, type HouseConfig, type ParsedLot, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Dunbar Sloane (Wellington / Auckland, NZ) — Tandem Auctions platform (auctions.dunbarsloane.co.nz). | |
| 8 | + * /previous-auctions lists past sales (title, "Thursday, 2 - Wednesday, 15 October 2025", /<id>); | |
| 9 | + * /<id>/catalogue?page=N shows lot cards (lot number, title, "Estimate: $200 - $350", "Realised: $140 + premium"). | |
| 10 | + * "Realised … plus premium" = HAMMER in NZD. | |
| 11 | + */ | |
| 12 | +const BASE = 'https://auctions.dunbarsloane.co.nz'; | |
| 13 | + | |
| 14 | +/** "Wednesday, 21 June 2023" | "Thursday, 2 - Wednesday, 15 October 2025" | "Friday, 17 - Tuesday, 28 October 2025" → first day. */ | |
| 15 | +export function parseNzDate(text: string | null | undefined): string | null { | |
| 16 | + if (!text) return null; | |
| 17 | + const t = textOf(text).replace(/\s+/g, ' '); | |
| 18 | + const range = t.match(/(\d{1,2})\s*(?:([A-Za-z]{3,9})\s*)?-\s*(?:[A-Za-z]+,\s*)?(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})/); | |
| 19 | + const single = t.match(/(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})/); | |
| 20 | + const parts = range ? { d: range[1]!, mon: range[2] ?? range[4]!, y: range[5]! } : single ? { d: single[1]!, mon: single[2]!, y: single[3]! } : null; | |
| 21 | + if (!parts) return null; | |
| 22 | + const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; | |
| 23 | + const mo = months.indexOf(parts.mon.slice(0, 3).toLowerCase()); | |
| 24 | + if (mo < 0) return null; | |
| 25 | + const d = new Date(Date.UTC(Number(parts.y), mo, Number(parts.d))); | |
| 26 | + return Number.isNaN(d.getTime()) ? null : d.toISOString(); | |
| 27 | +} | |
| 28 | + | |
| 29 | +export function parsePreviousAuctions(htmlText: string): SaleRef[] { | |
| 30 | + const out: SaleRef[] = []; | |
| 31 | + const seen = new Set<string>(); | |
| 32 | + for (const chunk of chunksBetween(htmlText, /<div class="card auction-card[^"]*">/)) { | |
| 33 | + const id = chunk.match(/href="\/(\d+)"/)?.[1]; | |
| 34 | + if (!id || seen.has(id)) continue; | |
| 35 | + seen.add(id); | |
| 36 | + const title = pick(chunk, /<h2 class="card-title">([\s\S]*?)<\/h2>/) ?? `Auction ${id}`; | |
| 37 | + const dateText = pick(chunk, /<h3 class="card-subtitle[^"]*">([\s\S]*?)<\/h3>/); | |
| 38 | + const badge = pick(chunk, /<span class="badge[^"]*">([\s\S]*?)<\/span>/); | |
| 39 | + // timed sales run for two weeks: keep the closing day so the crawler waits for results | |
| 40 | + const endText = dateText?.match(/-\s*([A-Za-z]+,\s*\d{1,2}\s+[A-Za-z]{3,9}\s+\d{4})/)?.[1] ?? null; | |
| 41 | + out.push({ id, title, url: `${BASE}/${id}/catalogue`, date: parseNzDate(dateText), location: null, extra: { date_text: dateText, auction_type: badge, end_date: endText ? parseNzDate(endText) : null } }); | |
| 42 | + } | |
| 43 | + return out; | |
| 44 | +} | |
| 45 | + | |
| 46 | +export function parseCataloguePage(htmlText: string, sale: SaleRef, page: number): ParsedSalePage | null { | |
| 47 | + if (!/class="lot-grid/.test(htmlText) && !/data-ln="/.test(htmlText)) return null; | |
| 48 | + const lots: ParsedLot[] = []; | |
| 49 | + for (const chunk of chunksBetween(htmlText, /<div id="\d+[A-Za-z]?" class="card h-100" data-ln="/)) { | |
| 50 | + const lotNo = chunk.match(/data-ln="([^"]+)"/)?.[1]; | |
| 51 | + const href = chunk.match(/href="(\/\d+\/catalogue\/[^"]+)"/)?.[1] ?? null; | |
| 52 | + if (!lotNo || !href) continue; | |
| 53 | + const image = chunk.match(/data-src="([^"]+)"/)?.[1] ?? null; | |
| 54 | + const alt = chunk.match(/alt="([^"]*)"/)?.[1] ?? null; | |
| 55 | + const textBlock = chunk.match(/<p class="card-text">([\s\S]*?)<small class="estimate">/)?.[1] ?? chunk.match(/<p class="card-text">([\s\S]*?)<\/p>/)?.[1] ?? ''; | |
| 56 | + const title = textOf(textBlock) || (alt ? textOf(alt) : ''); | |
| 57 | + if (!title) continue; | |
| 58 | + const est = pick(chunk, /<small class="estimate">\s*Estimate:\s*([^<]*)</); | |
| 59 | + const estM = est?.match(/\$\s*([\d,]+)\s*-\s*\$?\s*([\d,]+)/); | |
| 60 | + const realised = chunk.match(/card-footer realised">\s*Realised:\s*\$?\s*([\d,]+)\s*(<small>[^<]*<\/small>)?/); | |
| 61 | + const price = realised ? parseEuMoney(`NZ$${realised[1]}`, 'NZD', 'en') : null; | |
| 62 | + const premiumNote = realised?.[2] ? textOf(realised[2]) : null; | |
| 63 | + lots.push({ | |
| 64 | + lotNo, | |
| 65 | + title, | |
| 66 | + subtitle: null, | |
| 67 | + description: null, | |
| 68 | + url: absolute(BASE, href)!, | |
| 69 | + image, | |
| 70 | + price: price?.amount ?? null, | |
| 71 | + currency: 'NZD', | |
| 72 | + premiumIncluded: premiumNote ? !/\+|plus/i.test(premiumNote) : null, | |
| 73 | + estimateLow: estM ? Number(estM[1]!.replace(/,/g, '')) : null, | |
| 74 | + estimateHigh: estM ? Number(estM[2]!.replace(/,/g, '')) : null, | |
| 75 | + date: null, | |
| 76 | + sold: price !== null, | |
| 77 | + extra: { premium_note: premiumNote }, | |
| 78 | + }); | |
| 79 | + } | |
| 80 | + const pageOf = htmlText.match(/Page\s+(\d+)\s+of\s+(\d+)/); | |
| 81 | + const total = pageOf ? Number(pageOf[2]) : null; | |
| 82 | + const header = pick(htmlText, /<h1 class="page-header">([\s\S]*?)<small/); | |
| 83 | + const dateText = pick(htmlText, /<small class="text-muted">([\s\S]*?)<\/small>/); | |
| 84 | + return { lots, hasMore: total !== null ? page < total : false, totalLots: null, sale: { title: header ?? undefined, date: parseNzDate(dateText) ?? undefined, extra: { pages: total } } }; | |
| 85 | +} | |
| 86 | + | |
| 87 | +export class DunbarSloaneConnector extends SaleResultsConnector { | |
| 88 | + readonly version = '1.0.0'; | |
| 89 | + readonly house: HouseConfig = { houseName: 'Dunbar Sloane', defaultCurrency: 'NZD', location: 'Wellington, New Zealand', idKey: 'dunbar_sloane_lot', premiumIncluded: false, fallbackSlug: 'antiques', minIntervalMs: 2500, maxPagesPerSale: 25 }; | |
| 90 | + protected override minIntervalMs = 2500; | |
| 91 | + | |
| 92 | + async listSales(ctx: CrawlContext): Promise<SaleRef[]> { | |
| 93 | + const url = String(this.meta.config.previousUrl ?? `${BASE}/previous-auctions`); | |
| 94 | + await this.throttle(url); | |
| 95 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 96 | + if (!res.success || !res.html) { | |
| 97 | + ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 98 | + return []; | |
| 99 | + } | |
| 100 | + return parsePreviousAuctions(res.html); | |
| 101 | + } | |
| 102 | + | |
| 103 | + salePageUrl(sale: SaleRef, page: number): string { | |
| 104 | + return page > 1 ? `${sale.url}?page=${page}` : sale.url; | |
| 105 | + } | |
| 106 | + | |
| 107 | + parseSalePage(res: ExtractionResult, sale: SaleRef, page: number): ParsedSalePage | null { | |
| 108 | + return res.html ? parseCataloguePage(res.html, sale, page) : null; | |
| 109 | + } | |
| 110 | +} | |
| 111 | + | |
| 112 | +export default function createConnector(meta: ConnectorMeta) { | |
| 113 | + return new DunbarSloaneConnector(meta); | |
| 114 | +} | |
added
connectors/api/dunbar-sloane/meta.json
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +{ | |
| 2 | + "id": "dunbar-sloane", | |
| 3 | + "displayName": "Dunbar Sloane (New Zealand) — realised prices", | |
| 4 | + "sourceId": "dunbar-sloane", | |
| 5 | + "sourceName": "Dunbar Sloane", | |
| 6 | + "sourceType": "auction_house", | |
| 7 | + "sourceUrl": "https://auctions.dunbarsloane.co.nz", | |
| 8 | + "module": "api/dunbar-sloane", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["antiques", "silver", "porcelain", "glass_crystal", "art", "contemporary_art", "jewelry", "other_watches", "stamps", "coins", "banknotes", "vintage_toys", "model_trains", "model_cars", "books", "maps", "militaria", "medals", "wine", "cameras", "musical_instruments", "design_furniture"], | |
| 11 | + "regions": ["NZ"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["NZD"], | |
| 14 | + "supportsListings": false, | |
| 15 | + "supportsSold": true, | |
| 16 | + "supportsAuctions": true, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": false, | |
| 21 | + "refreshFrequencyMinutes": 1440, | |
| 22 | + "priority": "low", | |
| 23 | + "trustScore": 0.85, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.dunbarsloane.co.nz/buying", | |
| 26 | + "acquisitionMethod": "server-rendered HTML (Tandem Auctions platform)", | |
| 27 | + "historicalDepth": "years", | |
| 28 | + "accessNotes": "Dunbar Sloane (Wellington & Auckland, NZ; silver, decorative arts, stamps, coins, toys, art, jewellery) publishes catalogues and realised prices on auctions.dunbarsloane.co.nz (Tandem Auctions). Public pages read: /previous-auctions (69 past sales: title, 'Thursday, 2 - Wednesday, 15 October 2025', /<id>) and /<id>/catalogue?page=N (lot cards: lot number, title, image, 'Estimate: $200 - $350', 'Realised: $140 + premium'). 'Realised … plus premium' is explicitly the HAMMER price → buyer_premium_included=false, NZD. robots.txt on the auctions host returns 404 (no rules); no anti-automation clause found. 2.5 s politeness; salesPerRun caps incremental runs; resumable backfill over the previous-auctions list. No login, no bidder data.", | |
| 29 | + "enabled": true, | |
| 30 | + "schemaVersion": "1.0", | |
| 31 | + "config": { | |
| 32 | + "previousUrl": "https://auctions.dunbarsloane.co.nz/previous-auctions", | |
| 33 | + "salesPerRun": 2 | |
| 34 | + } | |
| 35 | +} | |
added
connectors/api/ebay-browse/README.md
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +# ebay-browse — eBay Browse API (gated) | |
| 2 | + | |
| 3 | +Official Buy → Browse API, live listings only (no sold data: Marketplace Insights is limited release). | |
| 4 | + | |
| 5 | +- Auth: OAuth2 client credentials (`EBAY_CLIENT_ID` = App ID, `EBAY_CLIENT_SECRET` = Cert ID; optional | |
| 6 | + `EBAY_ENV=sandbox`). Token cached in-process (`_g10-lib cachedToken`), refreshed 60 s before expiry. | |
| 7 | +- `GET /buy/browse/v1/item_summary/search?q&category_ids&filter&sort=newlyListed&limit=200&offset` per | |
| 8 | + (query × marketplace × page); header `X-EBAY-C-MARKETPLACE-ID` ∈ EBAY_US/CA/GB/DE/FR/AU; offset+limit ≤ 10,000. | |
| 9 | +- Normalised `listing`: native marketplace currency, listingType from `buyingOptions`, seller username + | |
| 10 | + feedback, shipping cost, item location, `listedAt`/`endsAt`, `bidCount`, grade from title, identifiers | |
| 11 | + `ebay_item_id`, `ebay_legacy_item_id`, `ebay_epid`. | |
| 12 | +- `lookup()` resolves `ebay.*/itm/<legacyId>` through `get_item_by_legacy_id`. | |
| 13 | +- Fixtures are built from the official OpenAPI contract (buy_browse_v1 v1.20.4) field definitions, not | |
| 14 | + live captures — see `note` in each fixture. | |
| 15 | + | |
| 16 | +Credentials: https://developer.ebay.com → My Account → Application Keys (production keyset) → Buy API | |
| 17 | +access application. | |
added
connectors/api/ebay-browse/index.test.ts
+82 −0
@@ -0,0 +1,82 @@ | ||
| 1 | +import { afterEach, describe, expect, it } from 'vitest'; | |
| 2 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 3 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 4 | +import { missingRequirements } from '@rareindex/connectors'; | |
| 5 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 6 | +import { clearTokenCache } from '../_g10-lib/index.js'; | |
| 7 | +import createConnector, { apiBase, getAppToken, parseSearchResponse, searchUrl } from './index.js'; | |
| 8 | + | |
| 9 | +const meta = localMeta(metaJson); | |
| 10 | +const connector = createConnector(meta); | |
| 11 | + | |
| 12 | +describe('ebay-browse (gated)', () => { | |
| 13 | + runFixtureSuite(connector, it, expect); | |
| 14 | + afterEach(() => clearTokenCache()); | |
| 15 | + | |
| 16 | + it('is reported as gated until EBAY_CLIENT_ID / EBAY_CLIENT_SECRET exist', () => { | |
| 17 | + expect(meta.requires).toEqual(['EBAY_CLIENT_ID', 'EBAY_CLIENT_SECRET']); | |
| 18 | + expect(missingRequirements(meta, {})).toEqual(['EBAY_CLIENT_ID', 'EBAY_CLIENT_SECRET']); | |
| 19 | + expect(missingRequirements(meta, { EBAY_CLIENT_ID: 'a', EBAY_CLIENT_SECRET: 'b' })).toEqual([]); | |
| 20 | + expect(apiBase(undefined).token).toBe('https://api.ebay.com/identity/v1/oauth2/token'); | |
| 21 | + expect(apiBase('sandbox').api).toBe('https://api.sandbox.ebay.com'); | |
| 22 | + }); | |
| 23 | + | |
| 24 | + it('builds search URLs with marketplace-independent params and filters', () => { | |
| 25 | + const url = searchUrl('https://api.ebay.com', { q: 'pokemon psa 10', categoryIds: ['183454'], categorySlug: 'pokemon', filter: 'price:[100..],priceCurrency:USD' }, { limit: 200, offset: 400, filter: 'buyingOptions:{FIXED_PRICE|AUCTION}', sort: 'newlyListed' }); | |
| 26 | + const u = new URL(url); | |
| 27 | + expect(u.pathname).toBe('/buy/browse/v1/item_summary/search'); | |
| 28 | + expect(u.searchParams.get('q')).toBe('pokemon psa 10'); | |
| 29 | + expect(u.searchParams.get('category_ids')).toBe('183454'); | |
| 30 | + expect(u.searchParams.get('filter')).toBe('buyingOptions:{FIXED_PRICE|AUCTION},price:[100..],priceCurrency:USD'); | |
| 31 | + expect(u.searchParams.get('limit')).toBe('200'); | |
| 32 | + expect(u.searchParams.get('offset')).toBe('400'); | |
| 33 | + }); | |
| 34 | + | |
| 35 | + it('requests and caches the client-credentials token (Basic auth, form body, api_scope)', async () => { | |
| 36 | + let calls = 0; | |
| 37 | + const fakeFetch = (async (url: string | URL | Request, init?: RequestInit) => { | |
| 38 | + calls++; | |
| 39 | + expect(String(url)).toBe('https://api.ebay.com/identity/v1/oauth2/token'); | |
| 40 | + expect((init?.headers as Record<string, string>).authorization).toBe(`Basic ${Buffer.from('id:secret').toString('base64')}`); | |
| 41 | + expect((init?.headers as Record<string, string>)['content-type']).toBe('application/x-www-form-urlencoded'); | |
| 42 | + expect(String(init?.body)).toBe('grant_type=client_credentials&scope=https%3A%2F%2Fapi.ebay.com%2Foauth%2Fapi_scope'); | |
| 43 | + return new Response(JSON.stringify({ access_token: 'v^1.1#tok', expires_in: 7200, token_type: 'Application Access Token' }), { status: 200 }); | |
| 44 | + }) as typeof fetch; | |
| 45 | + const t1 = await getAppToken('id', 'secret', 'https://api.ebay.com/identity/v1/oauth2/token', fakeFetch); | |
| 46 | + const t2 = await getAppToken('id', 'secret', 'https://api.ebay.com/identity/v1/oauth2/token', fakeFetch); | |
| 47 | + expect(t1).toBe('v^1.1#tok'); | |
| 48 | + expect(t2).toBe(t1); | |
| 49 | + expect(calls).toBe(1); | |
| 50 | + }); | |
| 51 | + | |
| 52 | + it('parses the documented search response shape and normalises listings in native currency', async () => { | |
| 53 | + const fx = loadFixture('ebay-browse', 'docs-search-ebay-us'); | |
| 54 | + const payload = fx.raw.payload as { items: unknown[] }; | |
| 55 | + const parsed = parseSearchResponse({ itemSummaries: payload.items, total: 12345, next: 'https://api.ebay.com/buy/browse/v1/item_summary/search?q=x&limit=2&offset=2' }); | |
| 56 | + expect(parsed?.items.length).toBe(payload.items.length); | |
| 57 | + expect(parsed?.next).toContain('offset=2'); | |
| 58 | + expect(parseSearchResponse({ errors: [{ errorId: 12000 }] })).toBeNull(); | |
| 59 | + const out = await connector.normalize(fx.raw); | |
| 60 | + expect(out.length).toBeGreaterThan(0); | |
| 61 | + const first = out[0]!; | |
| 62 | + if (first.kind !== 'listing') throw new Error('expected listing'); | |
| 63 | + expect(first.currency).toBe('USD'); | |
| 64 | + expect(first.attributes.identifiers.ebay_item_id).toMatch(/^v1\|\d+\|\d+$/); | |
| 65 | + expect(first.attributes.identifiers.ebay_epid).toBeTruthy(); | |
| 66 | + expect(first.seller).toBeTruthy(); | |
| 67 | + expect(first.sellerReputation).toMatch(/%/); | |
| 68 | + expect(first.shippingCost).not.toBeNull(); | |
| 69 | + expect(first.listingType).toBe('best_offer'); // FIXED_PRICE + BEST_OFFER → best_offer | |
| 70 | + const auction = out.find((r) => r.kind === 'listing' && r.listingType === 'auction'); | |
| 71 | + expect(auction && auction.kind === 'listing' ? auction.bidCount : null).not.toBeNull(); | |
| 72 | + const gb = loadFixture('ebay-browse', 'docs-search-ebay-gb-converted'); | |
| 73 | + const gbOut = await connector.normalize(gb.raw); | |
| 74 | + expect(gbOut.length).toBe(1); | |
| 75 | + const g = gbOut[0]!; | |
| 76 | + if (g.kind !== 'listing') throw new Error('expected listing'); | |
| 77 | + expect(g.currency).toBe('GBP'); | |
| 78 | + expect(g.attributes.metadata.price_converted_by_ebay).toBe(true); | |
| 79 | + expect(g.grade.grader).toBe('psa'); | |
| 80 | + expect(g.grade.grade).toBe('10'); | |
| 81 | + }); | |
| 82 | +}); | |
added
connectors/api/ebay-browse/index.ts
+313 −0
@@ -0,0 +1,313 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 5 | +import { cachedToken, isBundleTitle, isoDate, safeYear, slugFromTitle } from '../_g10-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * eBay Browse API (official, OAuth2 client-credentials, scope https://api.ebay.com/oauth/api_scope). | |
| 9 | + * GET /buy/browse/v1/item_summary/search?q=&category_ids=&filter=&limit=200&offset=N with the | |
| 10 | + * X-EBAY-C-MARKETPLACE-ID header selects the marketplace (EBAY_US, EBAY_CA, EBAY_GB, EBAY_DE, EBAY_FR, | |
| 11 | + * EBAY_AU…). The Browse API returns LIVE listings only — no sold/completed items (Marketplace Insights | |
| 12 | + * is a limited-release API, see meta.json). Gated: requires EBAY_CLIENT_ID + EBAY_CLIENT_SECRET. | |
| 13 | + */ | |
| 14 | +const PARSER_VERSION = '1.0.0'; | |
| 15 | +const MAX_LIMIT = 200; | |
| 16 | +const MAX_RESULTS = 10_000; // Browse caps offset + limit at 10,000 per query | |
| 17 | + | |
| 18 | +export function apiBase(env: string | undefined): { api: string; token: string } { | |
| 19 | + const sandbox = (env ?? '').toLowerCase() === 'sandbox'; | |
| 20 | + return sandbox ? { api: 'https://api.sandbox.ebay.com', token: 'https://api.sandbox.ebay.com/identity/v1/oauth2/token' } : { api: 'https://api.ebay.com', token: 'https://api.ebay.com/identity/v1/oauth2/token' }; | |
| 21 | +} | |
| 22 | + | |
| 23 | +export const MarketplaceSchema = z.object({ id: z.string().regex(/^EBAY_[A-Z]{2,4}$/), currency: z.string().length(3), country: z.string().length(2), locale: z.string().default('en-US') }); | |
| 24 | +export type Marketplace = z.infer<typeof MarketplaceSchema>; | |
| 25 | +export const QuerySchema = z.object({ q: z.string().optional(), categoryIds: z.array(z.string()).default([]), categorySlug: z.string().nullable().default(null), filter: z.string().optional(), marketplaces: z.array(z.string()).optional() }); | |
| 26 | +export type Query = z.infer<typeof QuerySchema>; | |
| 27 | + | |
| 28 | +const ConfigSchema = z.object({ | |
| 29 | + marketplaces: z.array(MarketplaceSchema).min(1), | |
| 30 | + queries: z.array(QuerySchema).min(1), | |
| 31 | + limit: z.number().int().min(1).max(MAX_LIMIT).default(MAX_LIMIT), | |
| 32 | + pagesPerQuery: z.number().int().min(1).default(1), | |
| 33 | + backfillPages: z.number().int().min(1).default(10), | |
| 34 | + /** default eBay field filter appended to every query (buying options, conditions, price range…) */ | |
| 35 | + defaultFilter: z.string().default('buyingOptions:{FIXED_PRICE|AUCTION|BEST_OFFER}'), | |
| 36 | + sort: z.string().default('newlyListed'), | |
| 37 | + queriesPerRun: z.number().int().min(1).default(6), | |
| 38 | +}); | |
| 39 | + | |
| 40 | +const Amount = z.object({ value: z.string(), currency: z.string(), convertedFromValue: z.string().optional(), convertedFromCurrency: z.string().optional() }); | |
| 41 | +export const ItemSummarySchema = z.object({ | |
| 42 | + itemId: z.string(), | |
| 43 | + legacyItemId: z.string().optional(), | |
| 44 | + title: z.string(), | |
| 45 | + shortDescription: z.string().optional(), | |
| 46 | + price: Amount.optional(), | |
| 47 | + currentBidPrice: Amount.optional(), | |
| 48 | + bidCount: z.number().int().optional(), | |
| 49 | + buyingOptions: z.array(z.string()).default([]), | |
| 50 | + condition: z.string().optional(), | |
| 51 | + conditionId: z.string().optional(), | |
| 52 | + itemWebUrl: z.string(), | |
| 53 | + itemAffiliateWebUrl: z.string().optional(), | |
| 54 | + image: z.object({ imageUrl: z.string() }).optional(), | |
| 55 | + thumbnailImages: z.array(z.object({ imageUrl: z.string() })).default([]), | |
| 56 | + additionalImages: z.array(z.object({ imageUrl: z.string() })).default([]), | |
| 57 | + seller: z.object({ username: z.string().optional(), feedbackPercentage: z.string().optional(), feedbackScore: z.number().int().optional(), sellerAccountType: z.string().optional() }).optional(), | |
| 58 | + shippingOptions: z.array(z.object({ shippingCost: Amount.optional(), shippingCostType: z.string().optional(), guaranteedDelivery: z.boolean().optional() })).default([]), | |
| 59 | + itemLocation: z.object({ city: z.string().optional(), stateOrProvince: z.string().optional(), postalCode: z.string().optional(), country: z.string().optional() }).optional(), | |
| 60 | + epid: z.string().optional(), | |
| 61 | + categories: z.array(z.object({ categoryId: z.string(), categoryName: z.string().optional() })).default([]), | |
| 62 | + leafCategoryIds: z.array(z.string()).default([]), | |
| 63 | + itemCreationDate: z.string().optional(), | |
| 64 | + itemEndDate: z.string().optional(), | |
| 65 | + itemGroupHref: z.string().optional(), | |
| 66 | + itemGroupType: z.string().optional(), | |
| 67 | + listingMarketplaceId: z.string().optional(), | |
| 68 | + adultOnly: z.boolean().optional(), | |
| 69 | + topRatedBuyingExperience: z.boolean().optional(), | |
| 70 | + watchCount: z.number().int().optional(), | |
| 71 | +}); | |
| 72 | +export type ItemSummary = z.infer<typeof ItemSummarySchema>; | |
| 73 | + | |
| 74 | +export const SearchPayloadSchema = z.object({ | |
| 75 | + kind: z.literal('search_page'), | |
| 76 | + url: z.string(), | |
| 77 | + marketplace: MarketplaceSchema, | |
| 78 | + query: QuerySchema, | |
| 79 | + offset: z.number().int(), | |
| 80 | + limit: z.number().int(), | |
| 81 | + total: z.number().int().nullable(), | |
| 82 | + next: z.string().nullable(), | |
| 83 | + items: z.array(ItemSummarySchema), | |
| 84 | + warnings: z.array(z.unknown()).default([]), | |
| 85 | +}); | |
| 86 | +export type SearchPayload = z.infer<typeof SearchPayloadSchema>; | |
| 87 | + | |
| 88 | +/** Trim an API item summary to the fields we persist (drops marketing/compat noise; never invents). */ | |
| 89 | +export function trimItem(raw: unknown): ItemSummary | null { | |
| 90 | + const r = ItemSummarySchema.safeParse(raw); | |
| 91 | + return r.success ? r.data : null; | |
| 92 | +} | |
| 93 | + | |
| 94 | +export function parseSearchResponse(json: unknown): { items: ItemSummary[]; total: number | null; next: string | null; warnings: unknown[]; rejected: number } | null { | |
| 95 | + const j = json as { itemSummaries?: unknown[]; total?: number; next?: string; warnings?: unknown[] } | null; | |
| 96 | + if (!j || typeof j !== 'object' || (!Array.isArray(j.itemSummaries) && typeof j.total !== 'number')) return null; | |
| 97 | + const items: ItemSummary[] = []; | |
| 98 | + let rejected = 0; | |
| 99 | + for (const it of j.itemSummaries ?? []) { | |
| 100 | + const t = trimItem(it); | |
| 101 | + if (t) items.push(t); | |
| 102 | + else rejected++; | |
| 103 | + } | |
| 104 | + return { items, total: typeof j.total === 'number' ? j.total : null, next: typeof j.next === 'string' ? j.next : null, warnings: j.warnings ?? [], rejected }; | |
| 105 | +} | |
| 106 | + | |
| 107 | +export function searchUrl(base: string, q: Query, opts: { limit: number; offset: number; filter: string; sort: string }): string { | |
| 108 | + const u = new URL(`${base}/buy/browse/v1/item_summary/search`); | |
| 109 | + if (q.q) u.searchParams.set('q', q.q); | |
| 110 | + if (q.categoryIds.length) u.searchParams.set('category_ids', q.categoryIds.join(',')); | |
| 111 | + const filters = [opts.filter, q.filter].filter(Boolean).join(','); | |
| 112 | + if (filters) u.searchParams.set('filter', filters); | |
| 113 | + if (opts.sort) u.searchParams.set('sort', opts.sort); | |
| 114 | + u.searchParams.set('limit', String(opts.limit)); | |
| 115 | + u.searchParams.set('offset', String(opts.offset)); | |
| 116 | + return u.toString(); | |
| 117 | +} | |
| 118 | + | |
| 119 | +/** Client-credentials token (cached process-wide until 60 s before expiry). Direct fetch: the token endpoint needs a form body + Basic auth. */ | |
| 120 | +export async function getAppToken(clientId: string, clientSecret: string, tokenUrl: string, fetchImpl: typeof fetch = fetch): Promise<string> { | |
| 121 | + return cachedToken(`ebay:${tokenUrl}:${clientId}`, async () => { | |
| 122 | + const res = await fetchImpl(tokenUrl, { | |
| 123 | + method: 'POST', | |
| 124 | + headers: { authorization: `Basic ${Buffer.from(`${clientId}:${clientSecret}`).toString('base64')}`, 'content-type': 'application/x-www-form-urlencoded' }, | |
| 125 | + body: new URLSearchParams({ grant_type: 'client_credentials', scope: 'https://api.ebay.com/oauth/api_scope' }).toString(), | |
| 126 | + }); | |
| 127 | + const text = await res.text(); | |
| 128 | + if (!res.ok) throw new Error(`eBay token HTTP ${res.status}: ${text.slice(0, 200)}`); | |
| 129 | + const j = JSON.parse(text) as { access_token?: string; expires_in?: number; token_type?: string }; | |
| 130 | + if (!j.access_token) throw new Error('eBay token response without access_token'); | |
| 131 | + return { token: j.access_token, expiresInSeconds: Number(j.expires_in ?? 7200) }; | |
| 132 | + }); | |
| 133 | +} | |
| 134 | + | |
| 135 | +const num = (a: { value: string } | undefined): number | null => { | |
| 136 | + if (!a) return null; | |
| 137 | + const n = Number.parseFloat(a.value); | |
| 138 | + return Number.isFinite(n) && n >= 0 ? n : null; | |
| 139 | +}; | |
| 140 | + | |
| 141 | +export class EbayBrowseConnector extends BaseConnector { | |
| 142 | + readonly version = '1.0.0'; | |
| 143 | + readonly parserVersion = PARSER_VERSION; | |
| 144 | + protected override minIntervalMs = 600; | |
| 145 | + override readonly urlPatterns = [/^https?:\/\/(www\.)?ebay\.(com|ca|co\.uk|de|fr|com\.au|it|es)\/itm\/(?:[^/]+\/)?(\d+)/i]; | |
| 146 | + private readonly cfg: z.infer<typeof ConfigSchema>; | |
| 147 | + | |
| 148 | + constructor(meta: ConnectorMeta) { | |
| 149 | + super(meta); | |
| 150 | + this.cfg = ConfigSchema.parse(meta.config); | |
| 151 | + } | |
| 152 | + | |
| 153 | + private credentials(ctx: CrawlContext): { id: string; secret: string; base: ReturnType<typeof apiBase> } | null { | |
| 154 | + const id = process.env.EBAY_CLIENT_ID?.trim(); | |
| 155 | + const secret = process.env.EBAY_CLIENT_SECRET?.trim(); | |
| 156 | + if (!id || !secret) { | |
| 157 | + ctx.anomaly('missing_credentials', 'EBAY_CLIENT_ID / EBAY_CLIENT_SECRET not set — connector is gated (DISABLED)'); | |
| 158 | + return null; | |
| 159 | + } | |
| 160 | + return { id, secret, base: apiBase(process.env.EBAY_ENV) }; | |
| 161 | + } | |
| 162 | + | |
| 163 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 164 | + const cred = this.credentials(ctx); | |
| 165 | + if (!cred) return; | |
| 166 | + let token: string; | |
| 167 | + try { | |
| 168 | + token = await getAppToken(cred.id, cred.secret, cred.base.token); | |
| 169 | + } catch (err) { | |
| 170 | + ctx.anomaly('auth_failed', err instanceof Error ? err.message : String(err)); | |
| 171 | + return; | |
| 172 | + } | |
| 173 | + const backfill = ctx.options.mode === 'backfill'; | |
| 174 | + const pages = backfill ? this.cfg.backfillPages : this.cfg.pagesPerQuery; | |
| 175 | + const queries = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => QuerySchema.parse({ q: s })) : this.cfg.queries; | |
| 176 | + const cursor = (ctx.options.cursor ?? {}) as { queryIndex?: number }; | |
| 177 | + const startQ = backfill ? 0 : Math.min(cursor.queryIndex ?? 0, queries.length - 1); | |
| 178 | + const perRun = backfill ? queries.length : Math.min(queries.length, this.cfg.queriesPerRun); | |
| 179 | + let count = 0; | |
| 180 | + let items = 0; | |
| 181 | + for (let k = 0; k < perRun; k++) { | |
| 182 | + const qi = (startQ + k) % queries.length; | |
| 183 | + const q = queries[qi]!; | |
| 184 | + const markets = this.cfg.marketplaces.filter((m) => !q.marketplaces || q.marketplaces.includes(m.id)); | |
| 185 | + for (const market of markets) { | |
| 186 | + for (let page = 0; page < pages; page++) { | |
| 187 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 188 | + const offset = page * this.cfg.limit; | |
| 189 | + if (offset + this.cfg.limit > MAX_RESULTS) break; | |
| 190 | + const url = searchUrl(cred.base.api, q, { limit: this.cfg.limit, offset, filter: this.cfg.defaultFilter, sort: this.cfg.sort }); | |
| 191 | + await this.throttle(url); | |
| 192 | + const res = await ctx.fetch(url, { | |
| 193 | + engines: ['api'], | |
| 194 | + responseType: 'json', | |
| 195 | + headers: { authorization: `Bearer ${token}`, 'x-ebay-c-marketplace-id': market.id, 'accept-language': market.locale, accept: 'application/json' }, | |
| 196 | + expect: ['title', 'price', 'currency'], | |
| 197 | + parse: (r) => { | |
| 198 | + const p = parseSearchResponse(r.json); | |
| 199 | + return p ? { title: p.items[0]?.title ?? (p.total === 0 ? 'empty' : null), price: p.items[0]?.price?.value ?? null, currency: p.items[0]?.price?.currency ?? null } : null; | |
| 200 | + }, | |
| 201 | + minQuality: 0.2, | |
| 202 | + }); | |
| 203 | + const parsed = res.success ? parseSearchResponse(res.json) : null; | |
| 204 | + if (!parsed) { | |
| 205 | + const errs = (res.json as { errors?: Array<{ errorId?: number; message?: string }> } | null)?.errors; | |
| 206 | + ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${market.id} ${q.q ?? q.categoryIds.join(',')} offset ${offset}: ${res.httpStatus} ${errs?.map((e) => `${e.errorId} ${e.message}`).join('; ') ?? res.error ?? ''}`); | |
| 207 | + break; | |
| 208 | + } | |
| 209 | + if (parsed.rejected) ctx.anomaly('schema_drift', `${parsed.rejected} item summaries rejected by schema`); | |
| 210 | + if (!parsed.items.length) break; | |
| 211 | + count++; | |
| 212 | + items += parsed.items.length; | |
| 213 | + const payload: SearchPayload = { kind: 'search_page', url, marketplace: market, query: q, offset, limit: this.cfg.limit, total: parsed.total, next: parsed.next, items: parsed.items, warnings: parsed.warnings }; | |
| 214 | + yield { url, externalId: `${market.id}:${q.q ?? ''}:${q.categoryIds.join('+')}:${offset}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 215 | + await ctx.progress({ page: page + 1, totalPages: parsed.total ? Math.ceil(Math.min(parsed.total, MAX_RESULTS) / this.cfg.limit) : null, itemsProcessed: items }); | |
| 216 | + if (!parsed.next || parsed.items.length < this.cfg.limit) break; | |
| 217 | + } | |
| 218 | + } | |
| 219 | + await ctx.setCursor({ queryIndex: (qi + 1) % queries.length, at: new Date().toISOString() }); | |
| 220 | + } | |
| 221 | + if (backfill) await ctx.setCursor({ queryIndex: 0, done: true, at: new Date().toISOString() }); | |
| 222 | + } | |
| 223 | + | |
| 224 | + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { | |
| 225 | + const legacy = url.match(this.urlPatterns[0]!)?.[3]; | |
| 226 | + const cred = legacy ? this.credentials(ctx) : null; | |
| 227 | + if (!legacy || !cred) return []; | |
| 228 | + const token = await getAppToken(cred.id, cred.secret, cred.base.token); | |
| 229 | + const tld = url.match(/ebay\.([a-z.]+)\//i)?.[1] ?? 'com'; | |
| 230 | + const market = this.cfg.marketplaces.find((m) => ({ com: 'EBAY_US', ca: 'EBAY_CA', 'co.uk': 'EBAY_GB', de: 'EBAY_DE', fr: 'EBAY_FR', 'com.au': 'EBAY_AU', it: 'EBAY_IT', es: 'EBAY_ES' })[tld] === m.id) ?? this.cfg.marketplaces[0]!; | |
| 231 | + const apiUrl = `${cred.base.api}/buy/browse/v1/item/get_item_by_legacy_id?legacy_item_id=${encodeURIComponent(legacy)}`; | |
| 232 | + await this.throttle(apiUrl); | |
| 233 | + const res = await ctx.fetch(apiUrl, { engines: ['api'], responseType: 'json', headers: { authorization: `Bearer ${token}`, 'x-ebay-c-marketplace-id': market.id }, minQuality: 0 }); | |
| 234 | + const item = res.success ? trimItem({ ...(res.json as object), buyingOptions: (res.json as { buyingOptions?: string[] })?.buyingOptions ?? [] }) : null; | |
| 235 | + if (!item) return []; | |
| 236 | + const payload: SearchPayload = { kind: 'search_page', url: apiUrl, marketplace: market, query: { categoryIds: [], categorySlug: null }, offset: 0, limit: 1, total: 1, next: null, items: [item], warnings: [] }; | |
| 237 | + return [{ url: apiUrl, externalId: `item:${item.itemId}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; | |
| 238 | + } | |
| 239 | + | |
| 240 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 241 | + const p = SearchPayloadSchema.parse(raw.payload); | |
| 242 | + const out: NormalizedRecord[] = []; | |
| 243 | + for (const it of p.items) { | |
| 244 | + if (it.adultOnly) continue; | |
| 245 | + const categorySlug = p.query.categorySlug ?? slugFromTitle(it.title); | |
| 246 | + if (!categorySlug) continue; | |
| 247 | + const priceAmt = it.price ?? it.currentBidPrice; | |
| 248 | + const price = num(priceAmt); | |
| 249 | + const currency = priceAmt?.currency ?? p.marketplace.currency; | |
| 250 | + const converted = Boolean(priceAmt?.convertedFromCurrency && priceAmt.convertedFromCurrency !== priceAmt.currency); | |
| 251 | + const opts = it.buyingOptions; | |
| 252 | + const listingType = opts.includes('AUCTION') ? 'auction' : opts.includes('BEST_OFFER') ? 'best_offer' : opts.includes('FIXED_PRICE') ? 'fixed_price' : 'unknown'; | |
| 253 | + const ship = it.shippingOptions.find((s) => s.shippingCost && s.shippingCost.currency === currency); | |
| 254 | + const grade = parseGradeFromTitle(it.title); | |
| 255 | + const images = [...new Set([it.image?.imageUrl, ...it.additionalImages.map((i) => i.imageUrl), ...it.thumbnailImages.map((i) => i.imageUrl)].filter((x): x is string => Boolean(x)))].slice(0, 8); | |
| 256 | + const attributes = AssetAttributesSchema.parse({ | |
| 257 | + categorySlug, | |
| 258 | + name: it.title, | |
| 259 | + year: safeYear(it.title), | |
| 260 | + identifiers: { ebay_item_id: it.itemId, ...(it.legacyItemId ? { ebay_legacy_item_id: it.legacyItemId } : {}), ...(it.epid ? { ebay_epid: it.epid } : {}) }, | |
| 261 | + metadata: { | |
| 262 | + marketplace: it.listingMarketplaceId ?? p.marketplace.id, | |
| 263 | + buying_options: opts, | |
| 264 | + condition_id: it.conditionId ?? null, | |
| 265 | + categories: it.categories, | |
| 266 | + leaf_category_ids: it.leafCategoryIds, | |
| 267 | + item_group_type: it.itemGroupType ?? null, | |
| 268 | + price_converted_by_ebay: converted, | |
| 269 | + original_price: converted ? { value: priceAmt?.convertedFromValue, currency: priceAmt?.convertedFromCurrency } : null, | |
| 270 | + shipping_cost_type: ship?.shippingCostType ?? null, | |
| 271 | + watch_count: it.watchCount ?? null, | |
| 272 | + top_rated_buying_experience: it.topRatedBuyingExperience ?? null, | |
| 273 | + query: p.query.q ?? null, | |
| 274 | + is_bundle_title: isBundleTitle(it.title), | |
| 275 | + }, | |
| 276 | + }); | |
| 277 | + out.push( | |
| 278 | + NormalizedListingSchema.parse({ | |
| 279 | + kind: 'listing', | |
| 280 | + connectorId: this.meta.id, | |
| 281 | + sourceId: this.meta.sourceId, | |
| 282 | + sourceUrl: it.itemWebUrl, | |
| 283 | + externalId: it.itemId, | |
| 284 | + rawTitle: it.title, | |
| 285 | + description: it.shortDescription ?? null, | |
| 286 | + imageUrls: images, | |
| 287 | + attributes, | |
| 288 | + grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null }, | |
| 289 | + condition: { condition: null, conditionRaw: it.condition ?? null, completeness: null }, | |
| 290 | + observedAt: raw.fetchedAt, | |
| 291 | + confidence: it.epid ? 0.85 : p.query.categorySlug ? 0.78 : 0.65, | |
| 292 | + parserVersion: PARSER_VERSION, | |
| 293 | + listingType, | |
| 294 | + price, | |
| 295 | + currency: currency && /^[A-Z]{3}$/.test(currency) ? currency : null, | |
| 296 | + seller: it.seller?.username ?? null, | |
| 297 | + sellerReputation: it.seller ? [it.seller.feedbackPercentage ? `${it.seller.feedbackPercentage}%` : null, typeof it.seller.feedbackScore === 'number' ? `${it.seller.feedbackScore} feedback` : null].filter(Boolean).join(' · ') || null : null, | |
| 298 | + location: [it.itemLocation?.city, it.itemLocation?.stateOrProvince, it.itemLocation?.country].filter(Boolean).join(', ') || null, | |
| 299 | + shippingCost: num(ship?.shippingCost), | |
| 300 | + listedAt: isoDate(it.itemCreationDate), | |
| 301 | + endsAt: isoDate(it.itemEndDate), | |
| 302 | + availability: 'available', | |
| 303 | + bidCount: it.bidCount ?? null, | |
| 304 | + }), | |
| 305 | + ); | |
| 306 | + } | |
| 307 | + return out; | |
| 308 | + } | |
| 309 | +} | |
| 310 | + | |
| 311 | +export default function createConnector(meta: ConnectorMeta): EbayBrowseConnector { | |
| 312 | + return new EbayBrowseConnector(meta); | |
| 313 | +} | |
added
connectors/api/ebay-browse/meta.json
+67 −0
@@ -0,0 +1,67 @@ | ||
| 1 | +{ | |
| 2 | + "id": "ebay-browse", | |
| 3 | + "displayName": "eBay Browse API (live listings, 6 marketplaces) — gated", | |
| 4 | + "sourceId": "ebay", | |
| 5 | + "sourceName": "eBay", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://www.ebay.com", | |
| 8 | + "module": "api/ebay-browse", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["pokemon", "magic_the_gathering", "yugioh", "sports_cards", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "comics", "video_games", "sneakers", "watches", "rolex", "omega", "other_watches", "lego_sets", "funko", "coins", "banknotes", "stamps", "music", "cameras", "pens", "whisky", "action_figures", "vintage_toys", "luxury_handbags"], | |
| 11 | + "regions": ["US", "CA", "GB", "DE", "FR", "AU"], | |
| 12 | + "languages": ["en", "de", "fr"], | |
| 13 | + "currency": ["USD", "CAD", "GBP", "EUR", "AUD"], | |
| 14 | + "supportsListings": true, | |
| 15 | + "supportsSold": false, | |
| 16 | + "supportsAuctions": false, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": true, | |
| 21 | + "refreshFrequencyMinutes": 60, | |
| 22 | + "priority": "high", | |
| 23 | + "trustScore": 0.85, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://developer.ebay.com/join/api-license-agreement", | |
| 26 | + "acquisitionMethod": "official API (OAuth2 client credentials)", | |
| 27 | + "historicalDepth": "none", | |
| 28 | + "requires": ["EBAY_CLIENT_ID", "EBAY_CLIENT_SECRET"], | |
| 29 | + "accessNotes": "Official eBay Buy → Browse API v1 (developer.ebay.com/api-docs/buy/browse). Auth: OAuth2 client-credentials grant — POST https://api.ebay.com/identity/v1/oauth2/token with Basic base64(client_id:client_secret), body grant_type=client_credentials&scope=https://api.ebay.com/oauth/api_scope; the application token (expires_in ≈ 7200 s) is cached inside the connector and refreshed 60 s before expiry. Data: GET /buy/browse/v1/item_summary/search with q / category_ids / filter (buyingOptions:{FIXED_PRICE|AUCTION|BEST_OFFER}, conditions, price:[..],priceCurrency, itemLocationCountry…) / sort=newlyListed / limit ≤ 200 / offset (offset+limit ≤ 10,000 per query), one call per (query × marketplace × page); the X-EBAY-C-MARKETPLACE-ID header selects EBAY_US, EBAY_CA, EBAY_GB, EBAY_DE, EBAY_FR or EBAY_AU and prices come back in the marketplace currency (convertedFromCurrency is recorded when eBay converted). Fields kept: itemId (v1|…|0), legacyItemId, epid (eBay product id → identifiers.ebay_epid), title, price/currentBidPrice, bidCount, buyingOptions, condition/conditionId, images, seller username + feedback percentage/score (public marketplace reputation, no personal data), shippingOptions[0].shippingCost, itemLocation city/state/country, itemCreationDate, itemEndDate, categories, watchCount. Limits: Browse is a 'Buy' API — default 5,000 calls/day per application for new keysets, more after eBay's Application Growth Check; we use 1 page per query per hourly run (≤ 6 queries × 6 marketplaces) and back off on 429 through the HTTP engine. IMPORTANT: the Browse API exposes LIVE listings only — there is no sold/completed-item data; eBay's sold data lives in the Marketplace Insights API (limited release, business approval required) and in the deprecated Finding API's findCompletedItems (retired) — documented, not implemented. Credentials: create a developer account at https://developer.ebay.com, an application keyset (production), and for Buy APIs complete the 'Buy API access' application; set EBAY_CLIENT_ID (App ID) and EBAY_CLIENT_SECRET (Cert ID); EBAY_ENV=sandbox switches to api.sandbox.ebay.com. Until the keys exist the framework reports this connector DISABLED (requires) and the crawler exits with a missing_credentials anomaly. Category ids in config are the public eBay leaf categories used in the query seeds; verify/adjust them with the Taxonomy API once a key is available (invalid ids raise error 12000 → page_fetch_failed anomaly). eBay API License Agreement: attribution + no storing of data beyond the permitted caching windows for listing data — RareIndex stores asking prices as listings and links back to itemWebUrl.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "config": { | |
| 33 | + "limit": 200, | |
| 34 | + "pagesPerQuery": 1, | |
| 35 | + "backfillPages": 10, | |
| 36 | + "queriesPerRun": 6, | |
| 37 | + "sort": "newlyListed", | |
| 38 | + "defaultFilter": "buyingOptions:{FIXED_PRICE|AUCTION|BEST_OFFER}", | |
| 39 | + "marketplaces": [ | |
| 40 | + { "id": "EBAY_US", "currency": "USD", "country": "US", "locale": "en-US" }, | |
| 41 | + { "id": "EBAY_CA", "currency": "CAD", "country": "CA", "locale": "en-CA" }, | |
| 42 | + { "id": "EBAY_GB", "currency": "GBP", "country": "GB", "locale": "en-GB" }, | |
| 43 | + { "id": "EBAY_DE", "currency": "EUR", "country": "DE", "locale": "de-DE" }, | |
| 44 | + { "id": "EBAY_FR", "currency": "EUR", "country": "FR", "locale": "fr-FR" }, | |
| 45 | + { "id": "EBAY_AU", "currency": "AUD", "country": "AU", "locale": "en-AU" } | |
| 46 | + ], | |
| 47 | + "queries": [ | |
| 48 | + { "q": "pokemon psa 10", "categoryIds": ["183454"], "categorySlug": "pokemon" }, | |
| 49 | + { "q": "pokemon charizard", "categoryIds": ["183454"], "categorySlug": "pokemon" }, | |
| 50 | + { "q": "magic the gathering foil", "categoryIds": ["183454"], "categorySlug": "magic_the_gathering" }, | |
| 51 | + { "q": "yu-gi-oh 1st edition", "categoryIds": ["183454"], "categorySlug": "yugioh" }, | |
| 52 | + { "q": "rookie psa", "categoryIds": ["261328"], "categorySlug": "sports_cards" }, | |
| 53 | + { "q": "cgc 9.8", "categoryIds": ["63"], "categorySlug": "comics" }, | |
| 54 | + { "q": "sealed wata", "categoryIds": ["139973"], "categorySlug": "video_games" }, | |
| 55 | + { "q": "air jordan 1 retro high og", "categoryIds": ["15709"], "categorySlug": "sneakers" }, | |
| 56 | + { "q": "rolex submariner", "categoryIds": ["31387"], "categorySlug": "rolex" }, | |
| 57 | + { "q": "omega speedmaster", "categoryIds": ["31387"], "categorySlug": "omega" }, | |
| 58 | + { "q": "lego sealed retired", "categoryIds": ["19006"], "categorySlug": "lego_sets" }, | |
| 59 | + { "q": "funko pop chase", "categoryIds": ["149372"], "categorySlug": "funko" }, | |
| 60 | + { "q": "pcgs ms", "categoryIds": ["11116"], "categorySlug": "coins" }, | |
| 61 | + { "q": "leica m3", "categoryIds": ["15230"], "categorySlug": "cameras" }, | |
| 62 | + { "q": "montblanc 149", "categoryIds": ["7275"], "categorySlug": "pens" }, | |
| 63 | + { "q": "first pressing vinyl", "categoryIds": ["176985"], "categorySlug": "music" }, | |
| 64 | + { "q": "hermes birkin", "categoryIds": ["169291"], "categorySlug": "luxury_handbags" } | |
| 65 | + ] | |
| 66 | + } | |
| 67 | +} | |
added
connectors/api/etsy/README.md
+14 −0
@@ -0,0 +1,14 @@ | ||
| 1 | +# etsy — Etsy Open API v3 (gated) | |
| 2 | + | |
| 3 | +`GET https://openapi.etsy.com/v3/application/listings/active?keywords&taxonomy_id&limit=100&offset&sort_on=created` | |
| 4 | +with `x-api-key: $ETSY_API_KEY`; images + shop via `/listings/batch?listing_ids=…&includes=Images,Shop`; | |
| 5 | +taxonomy ids resolved from `/seller-taxonomy/nodes` by node-name path (cached in the cursor). | |
| 6 | + | |
| 7 | +- Money `{amount, divisor, currency_code}` → native-currency `listing` (never a sale); supplies, | |
| 8 | + made-to-order and inactive listings skipped; `when_made` era in metadata. | |
| 9 | +- Cursor `{queryIndex, taxonomy}` rotates the keyword/taxonomy seeds; `lookup()` for `etsy.com/listing/<id>`. | |
| 10 | +- Identifier: `etsy_listing_id`. | |
| 11 | +- Fixture synthesised from the official OpenAPI schema (not a live capture — no key yet). | |
| 12 | + | |
| 13 | +Credentials: https://www.etsy.com/developers/register → create app → keystring → `ETSY_API_KEY` | |
| 14 | +(commercial use requires Etsy API Terms approval). | |
added
connectors/api/etsy/index.test.ts
+56 −0
@@ -0,0 +1,56 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import metaJson from './meta.json' with { type: 'json' }; | |
| 3 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 4 | +import { missingRequirements } from '@rareindex/connectors'; | |
| 5 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 6 | +import createConnector, { parseSearchResponse, priceOf, resolveTaxonomyPath, searchUrl } from './index.js'; | |
| 7 | + | |
| 8 | +const meta = localMeta(metaJson); | |
| 9 | +const connector = createConnector(meta); | |
| 10 | + | |
| 11 | +describe('etsy (gated)', () => { | |
| 12 | + runFixtureSuite(connector, it, expect); | |
| 13 | + | |
| 14 | + it('is gated on ETSY_API_KEY', () => { | |
| 15 | + expect(meta.requires).toEqual(['ETSY_API_KEY']); | |
| 16 | + expect(missingRequirements(meta, {})).toEqual(['ETSY_API_KEY']); | |
| 17 | + expect(missingRequirements(meta, { ETSY_API_KEY: 'k' })).toEqual([]); | |
| 18 | + }); | |
| 19 | + | |
| 20 | + it('builds findAllListingsActive URLs and resolves taxonomy paths', () => { | |
| 21 | + const url = new URL(searchUrl({ keywords: 'vintage fountain pen', categorySlug: 'pens', minPrice: 50 }, { limit: 100, offset: 200, sortOn: 'created', sortOrder: 'desc' }, 1234)); | |
| 22 | + expect(url.origin + url.pathname).toBe('https://openapi.etsy.com/v3/application/listings/active'); | |
| 23 | + expect(url.searchParams.get('keywords')).toBe('vintage fountain pen'); | |
| 24 | + expect(url.searchParams.get('taxonomy_id')).toBe('1234'); | |
| 25 | + expect(url.searchParams.get('min_price')).toBe('50'); | |
| 26 | + expect(url.searchParams.get('limit')).toBe('100'); | |
| 27 | + expect(url.searchParams.get('offset')).toBe('200'); | |
| 28 | + const tree = { count: 1, results: [{ id: 1, name: 'Toys & Games', children: [{ id: 10, name: 'Games & Puzzles', children: [{ id: 100, name: 'Card Games', children: [] }] }] }] }; | |
| 29 | + expect(resolveTaxonomyPath(tree, ['Toys & Games', 'Games & Puzzles', 'Card Games'])).toBe(100); | |
| 30 | + expect(resolveTaxonomyPath(tree, ['toys & games'])).toBe(1); | |
| 31 | + expect(resolveTaxonomyPath(tree, ['Nope'])).toBeNull(); | |
| 32 | + }); | |
| 33 | + | |
| 34 | + it('parses Money prices (amount/divisor) and normalises native-currency listings', async () => { | |
| 35 | + expect(priceOf({ amount: 12999, divisor: 100, currency_code: 'USD' })).toBe(129.99); | |
| 36 | + expect(priceOf({ amount: 0, divisor: 100, currency_code: 'USD' })).toBeNull(); | |
| 37 | + const fx = loadFixture('etsy', 'docs-listings-active'); | |
| 38 | + const payload = fx.raw.payload as { listings: unknown[] }; | |
| 39 | + const parsed = parseSearchResponse({ count: 3, results: payload.listings }); | |
| 40 | + expect(parsed?.listings.length).toBe(payload.listings.length); | |
| 41 | + expect(parseSearchResponse({ error: 'Invalid API key' })).toBeNull(); | |
| 42 | + const out = await connector.normalize(fx.raw); | |
| 43 | + // one made_to_order listing is skipped | |
| 44 | + expect(out.length).toBe(payload.listings.length - 1); | |
| 45 | + const first = out[0]!; | |
| 46 | + if (first.kind !== 'listing') throw new Error('expected listing'); | |
| 47 | + expect(first.currency).toBe('USD'); | |
| 48 | + expect(first.price).toBe(129.99); | |
| 49 | + expect(first.attributes.identifiers.etsy_listing_id).toMatch(/^\d+$/); | |
| 50 | + expect(first.imageUrls[0]).toMatch(/etsystatic/); | |
| 51 | + expect(first.seller).toBeTruthy(); | |
| 52 | + expect(first.listedAt?.getUTCFullYear()).toBe(2026); | |
| 53 | + const gbp = out.find((r) => r.kind === 'listing' && r.currency === 'GBP'); | |
| 54 | + expect(gbp).toBeTruthy(); | |
| 55 | + }); | |
| 56 | +}); | |
added
connectors/api/etsy/index.ts
+307 −0
@@ -0,0 +1,307 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { parseGradeFromTitle } from '@rareindex/taxonomy'; | |
| 5 | +import { epochSeconds, isBundleTitle, plainText, safeYear } from '../_g10-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Etsy Open API v3 (official; x-api-key header = the app keystring). findAllListingsActive — | |
| 9 | + * GET /v3/application/listings/active?keywords=&taxonomy_id=&limit=100&offset=&sort_on=created&sort_order=desc — | |
| 10 | + * returns active listings with Money prices {amount, divisor, currency_code} in the shop's currency. | |
| 11 | + * Images/shop names are not part of that response; they are attached with one | |
| 12 | + * GET /v3/application/listings/batch?listing_ids=…&includes=Images,Shop call per page (≤ 100 ids). | |
| 13 | + * Taxonomy ids are resolved at run time from /v3/application/seller-taxonomy/nodes by node-name path. | |
| 14 | + * Gated: requires ETSY_API_KEY. Listings only (Etsy exposes no sold prices publicly). | |
| 15 | + */ | |
| 16 | +const API = 'https://openapi.etsy.com/v3/application'; | |
| 17 | +const PARSER_VERSION = '1.0.0'; | |
| 18 | +const MAX_LIMIT = 100; | |
| 19 | + | |
| 20 | +export const QuerySchema = z.object({ | |
| 21 | + keywords: z.string().optional(), | |
| 22 | + taxonomyId: z.number().int().optional(), | |
| 23 | + /** node names from the seller taxonomy root, e.g. ["Art & Collectibles", "Collectibles"] */ | |
| 24 | + taxonomyPath: z.array(z.string()).optional(), | |
| 25 | + categorySlug: z.string(), | |
| 26 | + minPrice: z.number().optional(), | |
| 27 | + maxPrice: z.number().optional(), | |
| 28 | +}); | |
| 29 | +export type Query = z.infer<typeof QuerySchema>; | |
| 30 | + | |
| 31 | +const ConfigSchema = z.object({ | |
| 32 | + queries: z.array(QuerySchema).min(1), | |
| 33 | + limit: z.number().int().min(1).max(MAX_LIMIT).default(MAX_LIMIT), | |
| 34 | + pagesPerQuery: z.number().int().min(1).default(1), | |
| 35 | + backfillPages: z.number().int().min(1).default(5), | |
| 36 | + queriesPerRun: z.number().int().min(1).default(8), | |
| 37 | + fetchImages: z.boolean().default(true), | |
| 38 | + sortOn: z.enum(['created', 'price', 'updated', 'score']).default('created'), | |
| 39 | + sortOrder: z.enum(['asc', 'desc']).default('desc'), | |
| 40 | +}); | |
| 41 | + | |
| 42 | +export const MoneySchema = z.object({ amount: z.number(), divisor: z.number().positive(), currency_code: z.string() }); | |
| 43 | +export const ListingSchema = z.object({ | |
| 44 | + listing_id: z.number().int(), | |
| 45 | + shop_id: z.number().int().nullable().optional(), | |
| 46 | + title: z.string(), | |
| 47 | + description: z.string().nullable().optional(), | |
| 48 | + state: z.string().nullable().optional(), | |
| 49 | + url: z.string(), | |
| 50 | + quantity: z.number().int().nullable().optional(), | |
| 51 | + price: MoneySchema, | |
| 52 | + taxonomy_id: z.number().int().nullable().optional(), | |
| 53 | + tags: z.array(z.string()).default([]), | |
| 54 | + materials: z.array(z.string()).default([]), | |
| 55 | + when_made: z.string().nullable().optional(), | |
| 56 | + who_made: z.string().nullable().optional(), | |
| 57 | + is_supply: z.boolean().nullable().optional(), | |
| 58 | + is_customizable: z.boolean().nullable().optional(), | |
| 59 | + has_variations: z.boolean().nullable().optional(), | |
| 60 | + listing_type: z.string().nullable().optional(), | |
| 61 | + language: z.string().nullable().optional(), | |
| 62 | + creation_timestamp: z.number().nullable().optional(), | |
| 63 | + original_creation_timestamp: z.number().nullable().optional(), | |
| 64 | + ending_timestamp: z.number().nullable().optional(), | |
| 65 | + last_modified_timestamp: z.number().nullable().optional(), | |
| 66 | + num_favorers: z.number().int().nullable().optional(), | |
| 67 | + views: z.number().int().nullable().optional(), | |
| 68 | + /** attached from /listings/batch?includes=Images,Shop */ | |
| 69 | + images: z.array(z.object({ url_570xN: z.string().optional(), url_fullxfull: z.string().optional(), listing_image_id: z.number().optional() })).default([]), | |
| 70 | + shop: z.object({ shop_name: z.string().optional(), url: z.string().optional(), shop_location_country_iso: z.string().nullable().optional(), review_average: z.number().nullable().optional(), review_count: z.number().nullable().optional() }).nullable().optional(), | |
| 71 | +}); | |
| 72 | +export type Listing = z.infer<typeof ListingSchema>; | |
| 73 | + | |
| 74 | +export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), query: QuerySchema, offset: z.number().int(), limit: z.number().int(), count: z.number().int().nullable(), listings: z.array(ListingSchema) }); | |
| 75 | +export type PagePayload = z.infer<typeof PagePayloadSchema>; | |
| 76 | + | |
| 77 | +/** Trim one API listing to the persisted shape (schema-drift tolerant: unknown → dropped). */ | |
| 78 | +export function trimListing(raw: unknown): Listing | null { | |
| 79 | + if (!raw || typeof raw !== 'object') return null; | |
| 80 | + const r = raw as Record<string, unknown>; | |
| 81 | + const parsed = ListingSchema.safeParse({ ...r, description: typeof r.description === 'string' ? plainText(r.description, 800) : null, tags: Array.isArray(r.tags) ? r.tags.slice(0, 20) : [], materials: Array.isArray(r.materials) ? r.materials.slice(0, 10) : [] }); | |
| 82 | + return parsed.success ? parsed.data : null; | |
| 83 | +} | |
| 84 | + | |
| 85 | +export function parseSearchResponse(json: unknown): { count: number | null; listings: Listing[]; rejected: number } | null { | |
| 86 | + const j = json as { count?: number; results?: unknown[] } | null; | |
| 87 | + if (!j || !Array.isArray(j.results)) return null; | |
| 88 | + const listings: Listing[] = []; | |
| 89 | + let rejected = 0; | |
| 90 | + for (const r of j.results) { | |
| 91 | + const t = trimListing(r); | |
| 92 | + if (t) listings.push(t); | |
| 93 | + else rejected++; | |
| 94 | + } | |
| 95 | + return { count: typeof j.count === 'number' ? j.count : null, listings, rejected }; | |
| 96 | +} | |
| 97 | + | |
| 98 | +export function searchUrl(q: Query, opts: { limit: number; offset: number; sortOn: string; sortOrder: string }, taxonomyId: number | null): string { | |
| 99 | + const u = new URL(`${API}/listings/active`); | |
| 100 | + if (q.keywords) u.searchParams.set('keywords', q.keywords); | |
| 101 | + if (taxonomyId) u.searchParams.set('taxonomy_id', String(taxonomyId)); | |
| 102 | + if (q.minPrice !== undefined) u.searchParams.set('min_price', String(q.minPrice)); | |
| 103 | + if (q.maxPrice !== undefined) u.searchParams.set('max_price', String(q.maxPrice)); | |
| 104 | + u.searchParams.set('sort_on', opts.sortOn); | |
| 105 | + u.searchParams.set('sort_order', opts.sortOrder); | |
| 106 | + u.searchParams.set('limit', String(opts.limit)); | |
| 107 | + u.searchParams.set('offset', String(opts.offset)); | |
| 108 | + return u.toString(); | |
| 109 | +} | |
| 110 | + | |
| 111 | +/** Walk the seller taxonomy tree by node names (case-insensitive). */ | |
| 112 | +export function resolveTaxonomyPath(nodes: unknown, path: string[]): number | null { | |
| 113 | + let level = (nodes as { results?: unknown[] } | null)?.results ?? (Array.isArray(nodes) ? nodes : null); | |
| 114 | + let id: number | null = null; | |
| 115 | + for (const name of path) { | |
| 116 | + if (!Array.isArray(level)) return null; | |
| 117 | + const hit = (level as Array<{ id?: number; name?: string; children?: unknown[] }>).find((n) => (n.name ?? '').toLowerCase() === name.toLowerCase()); | |
| 118 | + if (!hit || typeof hit.id !== 'number') return null; | |
| 119 | + id = hit.id; | |
| 120 | + level = hit.children ?? []; | |
| 121 | + } | |
| 122 | + return id; | |
| 123 | +} | |
| 124 | + | |
| 125 | +export function priceOf(m: z.infer<typeof MoneySchema>): number | null { | |
| 126 | + const v = m.amount / m.divisor; | |
| 127 | + return Number.isFinite(v) && v > 0 ? v : null; | |
| 128 | +} | |
| 129 | + | |
| 130 | +export class EtsyConnector extends BaseConnector { | |
| 131 | + readonly version = '1.0.0'; | |
| 132 | + readonly parserVersion = PARSER_VERSION; | |
| 133 | + protected override minIntervalMs = 400; | |
| 134 | + override readonly urlPatterns = [/^https?:\/\/(www\.)?etsy\.com\/(?:[a-z]{2}\/)?listing\/(\d+)/i]; | |
| 135 | + private readonly cfg: z.infer<typeof ConfigSchema>; | |
| 136 | + | |
| 137 | + constructor(meta: ConnectorMeta) { | |
| 138 | + super(meta); | |
| 139 | + this.cfg = ConfigSchema.parse(meta.config); | |
| 140 | + } | |
| 141 | + | |
| 142 | + private apiKey(ctx: CrawlContext): string | null { | |
| 143 | + const key = process.env.ETSY_API_KEY?.trim(); | |
| 144 | + if (!key) ctx.anomaly('missing_credentials', 'ETSY_API_KEY not set — connector is gated (DISABLED)'); | |
| 145 | + return key ?? null; | |
| 146 | + } | |
| 147 | + | |
| 148 | + private headers(key: string): Record<string, string> { | |
| 149 | + return { 'x-api-key': key, accept: 'application/json' }; | |
| 150 | + } | |
| 151 | + | |
| 152 | + private async taxonomyId(ctx: CrawlContext, key: string, q: Query, cache: Record<string, number>): Promise<number | null> { | |
| 153 | + if (q.taxonomyId) return q.taxonomyId; | |
| 154 | + if (!q.taxonomyPath?.length) return null; | |
| 155 | + const k = q.taxonomyPath.join(' > '); | |
| 156 | + if (cache[k]) return cache[k]!; | |
| 157 | + await this.throttle(); | |
| 158 | + const res = await ctx.fetch(`${API}/seller-taxonomy/nodes`, { engines: ['api'], responseType: 'json', headers: this.headers(key), minQuality: 0, force: true }); | |
| 159 | + const id = res.success ? resolveTaxonomyPath(res.json, q.taxonomyPath) : null; | |
| 160 | + if (!id) ctx.anomaly('selector_missing', `Etsy taxonomy path not found: ${k}`); | |
| 161 | + else cache[k] = id; | |
| 162 | + return id; | |
| 163 | + } | |
| 164 | + | |
| 165 | + private async attachImages(ctx: CrawlContext, key: string, listings: Listing[]): Promise<void> { | |
| 166 | + if (!this.cfg.fetchImages || !listings.length) return; | |
| 167 | + const ids = listings.map((l) => l.listing_id); | |
| 168 | + const url = `${API}/listings/batch?listing_ids=${ids.join(',')}&includes=Images,Shop`; | |
| 169 | + await this.throttle(url); | |
| 170 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', headers: this.headers(key), minQuality: 0 }); | |
| 171 | + const results = (res.json as { results?: Array<Record<string, unknown>> } | null)?.results; | |
| 172 | + if (!res.success || !Array.isArray(results)) { | |
| 173 | + ctx.anomaly('page_fetch_failed', `listings/batch: ${res.error ?? res.httpStatus}`); | |
| 174 | + return; | |
| 175 | + } | |
| 176 | + const byId = new Map(results.map((r) => [Number(r.listing_id), r] as const)); | |
| 177 | + for (const l of listings) { | |
| 178 | + const full = byId.get(l.listing_id); | |
| 179 | + if (!full) continue; | |
| 180 | + const imgs = Array.isArray(full.images) ? (full.images as Array<Record<string, unknown>>).slice(0, 6).map((i) => ({ url_570xN: typeof i.url_570xN === 'string' ? i.url_570xN : undefined, url_fullxfull: typeof i.url_fullxfull === 'string' ? i.url_fullxfull : undefined, listing_image_id: typeof i.listing_image_id === 'number' ? i.listing_image_id : undefined })) : []; | |
| 181 | + const shop = full.shop && typeof full.shop === 'object' ? (full.shop as Record<string, unknown>) : null; | |
| 182 | + l.images = imgs; | |
| 183 | + l.shop = shop ? { shop_name: typeof shop.shop_name === 'string' ? shop.shop_name : undefined, url: typeof shop.url === 'string' ? shop.url : undefined, shop_location_country_iso: typeof shop.shop_location_country_iso === 'string' ? shop.shop_location_country_iso : null, review_average: typeof shop.review_average === 'number' ? shop.review_average : null, review_count: typeof shop.review_count === 'number' ? shop.review_count : null } : null; | |
| 184 | + } | |
| 185 | + } | |
| 186 | + | |
| 187 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 188 | + const key = this.apiKey(ctx); | |
| 189 | + if (!key) return; | |
| 190 | + const backfill = ctx.options.mode === 'backfill'; | |
| 191 | + const pages = backfill ? this.cfg.backfillPages : this.cfg.pagesPerQuery; | |
| 192 | + const queries = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => QuerySchema.parse({ keywords: s, categorySlug: 'trading_cards' })) : this.cfg.queries; | |
| 193 | + const cursor = (ctx.options.cursor ?? {}) as { queryIndex?: number; taxonomy?: Record<string, number> }; | |
| 194 | + const taxonomyCache: Record<string, number> = { ...(cursor.taxonomy ?? {}) }; | |
| 195 | + const startQ = backfill ? 0 : Math.min(cursor.queryIndex ?? 0, queries.length - 1); | |
| 196 | + const perRun = backfill ? queries.length : Math.min(queries.length, this.cfg.queriesPerRun); | |
| 197 | + let count = 0; | |
| 198 | + let items = 0; | |
| 199 | + for (let k = 0; k < perRun; k++) { | |
| 200 | + const qi = (startQ + k) % queries.length; | |
| 201 | + const q = queries[qi]!; | |
| 202 | + const taxId = await this.taxonomyId(ctx, key, q, taxonomyCache); | |
| 203 | + if (q.taxonomyPath?.length && !taxId) continue; | |
| 204 | + for (let page = 0; page < pages; page++) { | |
| 205 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 206 | + const offset = page * this.cfg.limit; | |
| 207 | + const url = searchUrl(q, { limit: this.cfg.limit, offset, sortOn: this.cfg.sortOn, sortOrder: this.cfg.sortOrder }, taxId); | |
| 208 | + await this.throttle(url); | |
| 209 | + const res = await ctx.fetch(url, { | |
| 210 | + engines: ['api'], | |
| 211 | + responseType: 'json', | |
| 212 | + headers: this.headers(key), | |
| 213 | + expect: ['title', 'price', 'currency'], | |
| 214 | + parse: (r) => { | |
| 215 | + const p = parseSearchResponse(r.json); | |
| 216 | + return p ? { title: p.listings[0]?.title ?? (p.count === 0 ? 'empty' : null), price: p.listings[0]?.price.amount ?? null, currency: p.listings[0]?.price.currency_code ?? null } : null; | |
| 217 | + }, | |
| 218 | + minQuality: 0.2, | |
| 219 | + }); | |
| 220 | + const parsed = res.success ? parseSearchResponse(res.json) : null; | |
| 221 | + if (!parsed) { | |
| 222 | + const err = (res.json as { error?: string } | null)?.error; | |
| 223 | + ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${q.keywords ?? q.taxonomyPath?.join('>')} offset ${offset}: ${res.httpStatus} ${err ?? res.error ?? ''}`); | |
| 224 | + break; | |
| 225 | + } | |
| 226 | + if (parsed.rejected) ctx.anomaly('schema_drift', `${parsed.rejected} listings rejected by schema`); | |
| 227 | + if (!parsed.listings.length) break; | |
| 228 | + await this.attachImages(ctx, key, parsed.listings); | |
| 229 | + count++; | |
| 230 | + items += parsed.listings.length; | |
| 231 | + const payload: PagePayload = { kind: 'search_page', url, query: q, offset, limit: this.cfg.limit, count: parsed.count, listings: parsed.listings }; | |
| 232 | + yield { url, externalId: `${q.keywords ?? ''}:${taxId ?? ''}:${offset}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 233 | + await ctx.progress({ page: page + 1, totalPages: parsed.count ? Math.ceil(parsed.count / this.cfg.limit) : null, itemsProcessed: items }); | |
| 234 | + if (parsed.listings.length < this.cfg.limit || (parsed.count !== null && offset + this.cfg.limit >= parsed.count)) break; | |
| 235 | + } | |
| 236 | + await ctx.setCursor({ queryIndex: (qi + 1) % queries.length, taxonomy: taxonomyCache, at: new Date().toISOString() }); | |
| 237 | + } | |
| 238 | + if (backfill) await ctx.setCursor({ queryIndex: 0, taxonomy: taxonomyCache, done: true, at: new Date().toISOString() }); | |
| 239 | + } | |
| 240 | + | |
| 241 | + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { | |
| 242 | + const id = url.match(this.urlPatterns[0]!)?.[2]; | |
| 243 | + const key = id ? this.apiKey(ctx) : null; | |
| 244 | + if (!id || !key) return []; | |
| 245 | + const apiUrl = `${API}/listings/${id}?includes=Images,Shop`; | |
| 246 | + await this.throttle(apiUrl); | |
| 247 | + const res = await ctx.fetch(apiUrl, { engines: ['api'], responseType: 'json', headers: this.headers(key), minQuality: 0 }); | |
| 248 | + const l = res.success ? trimListing(res.json) : null; | |
| 249 | + if (!l) return []; | |
| 250 | + const payload: PagePayload = { kind: 'search_page', url: apiUrl, query: { categorySlug: 'trading_cards' }, offset: 0, limit: 1, count: 1, listings: [l] }; | |
| 251 | + return [{ url: apiUrl, externalId: `listing:${id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; | |
| 252 | + } | |
| 253 | + | |
| 254 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 255 | + const p = PagePayloadSchema.parse(raw.payload); | |
| 256 | + const out: NormalizedRecord[] = []; | |
| 257 | + for (const l of p.listings) { | |
| 258 | + if (l.is_supply || l.when_made === 'made_to_order' || (l.state && l.state !== 'active')) continue; | |
| 259 | + const price = priceOf(l.price); | |
| 260 | + if (!price) continue; | |
| 261 | + const grade = parseGradeFromTitle(l.title); | |
| 262 | + const decade = l.when_made?.match(/^(1[6-9]\d0|20[0-2]0)s$/)?.[0] ?? null; | |
| 263 | + const attributes = AssetAttributesSchema.parse({ | |
| 264 | + categorySlug: p.query.categorySlug, | |
| 265 | + name: l.title, | |
| 266 | + year: safeYear(l.title), | |
| 267 | + material: l.materials[0] ?? null, | |
| 268 | + language: l.language ?? null, | |
| 269 | + identifiers: { etsy_listing_id: String(l.listing_id) }, | |
| 270 | + metadata: { when_made: l.when_made ?? null, decade, who_made: l.who_made ?? null, tags: l.tags, materials: l.materials, taxonomy_id: l.taxonomy_id ?? null, num_favorers: l.num_favorers ?? null, has_variations: l.has_variations ?? null, shop_id: l.shop_id ?? null, shop_country: l.shop?.shop_location_country_iso ?? null, keywords: p.query.keywords ?? null, is_bundle_title: isBundleTitle(l.title) }, | |
| 271 | + }); | |
| 272 | + out.push( | |
| 273 | + NormalizedListingSchema.parse({ | |
| 274 | + kind: 'listing', | |
| 275 | + connectorId: this.meta.id, | |
| 276 | + sourceId: this.meta.sourceId, | |
| 277 | + sourceUrl: l.url, | |
| 278 | + externalId: String(l.listing_id), | |
| 279 | + rawTitle: l.title, | |
| 280 | + description: l.description ?? null, | |
| 281 | + imageUrls: l.images.map((i) => i.url_fullxfull ?? i.url_570xN).filter((x): x is string => Boolean(x)), | |
| 282 | + attributes, | |
| 283 | + grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null }, | |
| 284 | + condition: { condition: null, conditionRaw: null, completeness: null }, | |
| 285 | + observedAt: raw.fetchedAt, | |
| 286 | + confidence: 0.7, | |
| 287 | + parserVersion: PARSER_VERSION, | |
| 288 | + listingType: 'fixed_price', | |
| 289 | + price, | |
| 290 | + currency: /^[A-Z]{3}$/.test(l.price.currency_code) ? l.price.currency_code : null, | |
| 291 | + seller: l.shop?.shop_name ?? null, | |
| 292 | + sellerReputation: l.shop?.review_average != null && l.shop.review_count != null ? `${l.shop.review_average.toFixed(1)}★ (${l.shop.review_count} reviews)` : null, | |
| 293 | + location: l.shop?.shop_location_country_iso ?? null, | |
| 294 | + quantity: l.quantity ?? null, | |
| 295 | + listedAt: epochSeconds(l.original_creation_timestamp ?? l.creation_timestamp), | |
| 296 | + endsAt: epochSeconds(l.ending_timestamp), | |
| 297 | + availability: 'available', | |
| 298 | + }), | |
| 299 | + ); | |
| 300 | + } | |
| 301 | + return out; | |
| 302 | + } | |
| 303 | +} | |
| 304 | + | |
| 305 | +export default function createConnector(meta: ConnectorMeta): EtsyConnector { | |
| 306 | + return new EtsyConnector(meta); | |
| 307 | +} | |
added
connectors/api/etsy/meta.json
+62 −0
@@ -0,0 +1,62 @@ | ||
| 1 | +{ | |
| 2 | + "id": "etsy", | |
| 3 | + "displayName": "Etsy Open API v3 (vintage & collectibles listings) — gated", | |
| 4 | + "sourceId": "etsy", | |
| 5 | + "sourceName": "Etsy", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://www.etsy.com", | |
| 8 | + "module": "api/etsy", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["vintage_toys", "cameras", "pens", "clocks", "design_furniture", "antiques", "porcelain", "glass_crystal", "silver", "jewelry", "other_watches", "movie_posters", "music", "books", "postcards", "advertising", "pokemon", "sports_cards", "comics", "typewriters", "scientific_instruments"], | |
| 11 | + "regions": ["US", "GB", "CA", "DE", "FR", "AU"], | |
| 12 | + "languages": ["en"], | |
| 13 | + "currency": ["USD", "GBP", "EUR", "CAD", "AUD"], | |
| 14 | + "supportsListings": true, | |
| 15 | + "supportsSold": false, | |
| 16 | + "supportsAuctions": false, | |
| 17 | + "supportsImages": true, | |
| 18 | + "supportsCatalog": false, | |
| 19 | + "supportsPopulation": false, | |
| 20 | + "supportsLookup": true, | |
| 21 | + "refreshFrequencyMinutes": 360, | |
| 22 | + "priority": "medium", | |
| 23 | + "trustScore": 0.7, | |
| 24 | + "attributionRequired": true, | |
| 25 | + "termsUrl": "https://www.etsy.com/legal/api", | |
| 26 | + "acquisitionMethod": "official API (x-api-key)", | |
| 27 | + "historicalDepth": "none", | |
| 28 | + "requires": ["ETSY_API_KEY"], | |
| 29 | + "accessNotes": "Official Etsy Open API v3 (developers.etsy.com/documentation, OpenAPI contract at etsy.com/openapi/generated/oas/3.0.0.json). Auth: the application keystring in the x-api-key header (Etsy's request guide shows the 'keystring:shared_secret' form; either value works in ETSY_API_KEY — we send it verbatim). No OAuth is needed for the application-scoped read endpoints we use: GET /v3/application/listings/active (findAllListingsActive: keywords, taxonomy_id, min_price/max_price, sort_on=created, sort_order=desc, limit ≤ 100, offset), GET /v3/application/listings/batch?listing_ids=…&includes=Images,Shop (one call per page to attach images and the public shop name/country/review score) and GET /v3/application/seller-taxonomy/nodes (taxonomy ids resolved by node-name path and cached in the cursor). Prices are Money {amount, divisor, currency_code} in each shop's own currency → native currency listings; images url_fullxfull; created/ending epoch timestamps → listedAt/endsAt. Etsy publishes no sold prices, so this is a listings-only source; supplies, made-to-order and non-active listings are skipped and 'when_made' eras are kept as metadata. Rate limits (Etsy default): 10 requests/second and 10,000 requests/day per app — a run uses ≤ 8 queries × (1 search + 1 batch) calls every 6 h. Credentials: register at https://www.etsy.com/developers/register, create an app (personal-access apps are approved instantly; commercial access requires Etsy's review under the API Terms of Use https://www.etsy.com/legal/api) and copy the keystring into ETSY_API_KEY. Until then the framework reports the connector DISABLED (requires) and the crawler exits with a missing_credentials anomaly. Fixtures are synthesised from the official OpenAPI schema (ShopListing, Money, ListingImage), not live captures — see the fixture note. Category slugs come from the configured query seeds (keywords + taxonomy path), confidence 0.7 (title-level identification only).", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "config": { | |
| 33 | + "limit": 100, | |
| 34 | + "pagesPerQuery": 1, | |
| 35 | + "backfillPages": 5, | |
| 36 | + "queriesPerRun": 8, | |
| 37 | + "fetchImages": true, | |
| 38 | + "sortOn": "created", | |
| 39 | + "sortOrder": "desc", | |
| 40 | + "queries": [ | |
| 41 | + { "keywords": "vintage pokemon card", "taxonomyPath": ["Toys & Games", "Games & Puzzles", "Card Games"], "categorySlug": "pokemon" }, | |
| 42 | + { "keywords": "vintage baseball card", "taxonomyPath": ["Art & Collectibles", "Collectibles"], "categorySlug": "sports_cards" }, | |
| 43 | + { "keywords": "vintage comic book", "taxonomyPath": ["Books, Movies & Music", "Books", "Comics & Graphic Novels"], "categorySlug": "comics" }, | |
| 44 | + { "keywords": "vintage film camera", "taxonomyPath": ["Electronics & Accessories", "Cameras & Equipment"], "categorySlug": "cameras" }, | |
| 45 | + { "keywords": "vintage fountain pen", "categorySlug": "pens" }, | |
| 46 | + { "keywords": "antique mantel clock", "categorySlug": "clocks" }, | |
| 47 | + { "keywords": "mid century modern chair", "taxonomyPath": ["Home & Living", "Furniture"], "categorySlug": "design_furniture" }, | |
| 48 | + { "keywords": "antique porcelain vase", "categorySlug": "porcelain" }, | |
| 49 | + { "keywords": "murano glass vase", "categorySlug": "glass_crystal" }, | |
| 50 | + { "keywords": "sterling silver antique", "categorySlug": "silver" }, | |
| 51 | + { "keywords": "vintage wristwatch", "taxonomyPath": ["Accessories", "Watches"], "categorySlug": "other_watches" }, | |
| 52 | + { "keywords": "original movie poster", "categorySlug": "movie_posters" }, | |
| 53 | + { "keywords": "vintage vinyl record first pressing", "categorySlug": "music" }, | |
| 54 | + { "keywords": "first edition book", "taxonomyPath": ["Books, Movies & Music", "Books"], "categorySlug": "books" }, | |
| 55 | + { "keywords": "antique postcard", "categorySlug": "postcards" }, | |
| 56 | + { "keywords": "vintage advertising sign", "categorySlug": "advertising" }, | |
| 57 | + { "keywords": "vintage typewriter", "categorySlug": "typewriters" }, | |
| 58 | + { "keywords": "antique microscope", "categorySlug": "scientific_instruments" }, | |
| 59 | + { "keywords": "vintage tin toy", "taxonomyPath": ["Toys & Games", "Toys"], "categorySlug": "vintage_toys" } | |
| 60 | + ] | |
| 61 | + } | |
| 62 | +} | |
added
connectors/api/european-watch-company/README.md
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# european-watch-company — European Watch Company (Boston) | |
| 2 | + | |
| 3 | +USD dealer asking prices for ~750 pre-owned luxury watches. | |
| 4 | + | |
| 5 | +- One request per run: `/all` (server-rendered grid) → cards with brand + model title, price, badge (In Stock / New Arrival / Sale Pending), image and `/watch/<slug>-<id>` URL → `listing`. | |
| 6 | +- `lookup()` reads the product page's schema.org Product (sku, mpn = reference, price, availability, condition). | |
| 7 | +- Identity: `identifiers.ewc_sku` (the trailing id) + `reference` (Rolex/Patek/AP numeric refs and letter-prefixed codes such as SBGE201G). Category from the brand (`watchCategory`). | |
| 8 | +- Never sales: 'Sale Pending' stays a listing with `metadata.sale_pending = true`. | |
| 9 | + | |
| 10 | +Smoke: `pnpm tsx connectors/api/_g9-asia-watch-sneaker-lib/capture.ts european-watch-company --limit 1`. | |
added
connectors/api/european-watch-company/index.test.ts
+63 −0
@@ -0,0 +1,63 @@ | ||
| 1 | +import { readFileSync } from 'node:fs'; | |
| 2 | +import path from 'node:path'; | |
| 3 | +import { describe, expect, it } from 'vitest'; | |
| 4 | +import { getConnectorMeta } from '@rareindex/connectors'; | |
| 5 | +import { fixtureDir, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import createConnector, { parseInventoryPage, parseProductPage } from './index.js'; | |
| 7 | + | |
| 8 | +const connector = createConnector(getConnectorMeta('european-watch-company')); | |
| 9 | +const sample = (name: string) => readFileSync(path.join(fixtureDir('european-watch-company'), name), 'utf8'); | |
| 10 | + | |
| 11 | +describe('european-watch-company', () => { | |
| 12 | + runFixtureSuite(connector, it, expect); | |
| 13 | + | |
| 14 | + it('parses inventory cards (title, USD price, badge, image, product url) and dedupes desktop/mobile layouts', () => { | |
| 15 | + const p = parseInventoryPage(sample('inventory-cards.sample.html'), 'https://www.europeanwatch.com/all'); | |
| 16 | + expect(p.cards.length).toBeGreaterThanOrEqual(1); | |
| 17 | + const c = p.cards[0]!; | |
| 18 | + expect(c.id).toBe('71732'); | |
| 19 | + expect(c.title).toBe('Patek Philippe 7010R-013 Nautilus Luce Diamonds 18K Rose Gold Purple Wave Dial'); | |
| 20 | + expect(c.price).toBe(79900); | |
| 21 | + expect(c.currency).toBe('USD'); | |
| 22 | + expect(c.badge).toBe('Sale Pending'); | |
| 23 | + expect(c.href).toBe('https://www.europeanwatch.com/watch/patek-philippe-7010r-013-7010r-013-nautilus-luce-diamonds-18k-ros-71732'); | |
| 24 | + expect(c.image).toBe('https://images.europeanwatch.com/images/71/71732-1.jpg'); | |
| 25 | + expect(new Set(p.cards.map((x) => x.id)).size).toBe(p.cards.length); | |
| 26 | + }); | |
| 27 | + | |
| 28 | + it('parses the product page JSON-LD (sku, mpn = reference, offer)', async () => { | |
| 29 | + const p = parseProductPage(sample('product-jsonld.sample.html'), 'https://www.europeanwatch.com/watch/patek-philippe-7010r-013-7010r-013-nautilus-luce-diamonds-18k-ros-71732'); | |
| 30 | + expect(p?.kind).toBe('product_page'); | |
| 31 | + if (p?.kind !== 'product_page') throw new Error(); | |
| 32 | + expect(p.product.sku).toBe('71732'); | |
| 33 | + expect(p.product.mpn).toBe('7010R-013'); | |
| 34 | + expect(p.product.price).toBe(79900); | |
| 35 | + expect(p.product.currency).toBe('USD'); | |
| 36 | + expect(p.product.availability).toBe('OutOfStock'); | |
| 37 | + expect(p.product.condition).toBe('UsedCondition'); | |
| 38 | + const out = await connector.normalize({ url: p.url, externalId: null, kind: 'listing', engine: 'api', fetchedAt: new Date('2026-09-08T00:00:00Z'), payload: p }); | |
| 39 | + const l = out[0]; | |
| 40 | + if (l?.kind !== 'listing') throw new Error('expected listing'); | |
| 41 | + expect(l.attributes.reference).toBe('7010R-013'); | |
| 42 | + expect(l.attributes.brand).toBe('Patek Philippe'); | |
| 43 | + expect(l.attributes.categorySlug).toBe('patek_philippe'); | |
| 44 | + expect(l.attributes.identifiers).toEqual({ ewc_sku: '71732', reference: '7010R-013' }); | |
| 45 | + expect(l.availability).toBe('ended'); | |
| 46 | + expect(l.price).toBe(79900); | |
| 47 | + }); | |
| 48 | + | |
| 49 | + it('inventory fixture normalises to USD dealer listings with brand-based categories', async () => { | |
| 50 | + const fx = loadFixture('european-watch-company', 'inventory-all'); | |
| 51 | + const out = await connector.normalize(fx.raw); | |
| 52 | + expect(out.length).toBeGreaterThan(5); | |
| 53 | + for (const r of out) { | |
| 54 | + if (r.kind !== 'listing') throw new Error('expected listing'); | |
| 55 | + expect(r.currency).toBe('USD'); | |
| 56 | + expect(r.seller).toBe('European Watch Company'); | |
| 57 | + expect(r.listingType).toBe('fixed_price'); | |
| 58 | + } | |
| 59 | + expect(out.some((r) => r.kind === 'listing' && r.attributes.categorySlug === 'patek_philippe')).toBe(true); | |
| 60 | + const gs = out.find((r) => r.kind === 'listing' && /SBGE201G/.test(r.rawTitle)); | |
| 61 | + expect(gs && gs.kind === 'listing' ? gs.attributes.reference : null).toBe('SBGE201G'); | |
| 62 | + }); | |
| 63 | +}); | |
added
connectors/api/european-watch-company/index.ts
+208 −0
@@ -0,0 +1,208 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { normalizeCondition } from '@rareindex/taxonomy'; | |
| 4 | +import { AssetAttributesSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared'; | |
| 5 | +import { watchFromTitle } from '../_g9-asia-watch-sneaker-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * European Watch Company (Boston dealer) — the server-rendered inventory grid `/all` lists every watch in stock | |
| 9 | + * (brand + model title, USD asking price, status badge, image, product URL). Product pages carry a schema.org | |
| 10 | + * Product (sku, mpn = reference, price, availability, condition) used for URL lookup. Asking prices → `listing`. | |
| 11 | + */ | |
| 12 | + | |
| 13 | +const SITE = 'https://www.europeanwatch.com'; | |
| 14 | +const PARSER_VERSION = '1.0.0'; | |
| 15 | + | |
| 16 | +export const CardSchema = z.object({ id: z.string(), href: z.string(), title: z.string(), price: z.number().nullable(), currency: z.string().nullable(), badge: z.string().nullable(), image: z.string().nullable() }); | |
| 17 | +export type Card = z.infer<typeof CardSchema>; | |
| 18 | +export const ProductSchema = z.object({ id: z.string(), href: z.string(), name: z.string(), sku: z.string().nullable(), mpn: z.string().nullable(), brand: z.string().nullable(), price: z.number().nullable(), currency: z.string().nullable(), availability: z.string().nullable(), condition: z.string().nullable(), description: z.string().nullable(), images: z.array(z.string()) }); | |
| 19 | +export type Product = z.infer<typeof ProductSchema>; | |
| 20 | +export const PagePayloadSchema = z.discriminatedUnion('kind', [ | |
| 21 | + z.object({ kind: z.literal('inventory_page'), url: z.string(), cards: z.array(CardSchema) }), | |
| 22 | + z.object({ kind: z.literal('product_page'), url: z.string(), product: ProductSchema }), | |
| 23 | +]); | |
| 24 | +export type PagePayload = z.infer<typeof PagePayloadSchema>; | |
| 25 | +export type InventoryPagePayload = Extract<PagePayload, { kind: 'inventory_page' }>; | |
| 26 | + | |
| 27 | +function decodeNextImage(src: string | undefined): string | null { | |
| 28 | + if (!src) return null; | |
| 29 | + const m = src.match(/[?&]url=([^&]+)/); | |
| 30 | + if (m) { | |
| 31 | + try { | |
| 32 | + return decodeURIComponent(m[1]!); | |
| 33 | + } catch { | |
| 34 | + return null; | |
| 35 | + } | |
| 36 | + } | |
| 37 | + return src.startsWith('http') ? src : null; | |
| 38 | +} | |
| 39 | + | |
| 40 | +/** Inventory grid: each desktop card is a <div> holding exactly one <h3> (title) and one <p> (price); the link sits in the sibling image block. */ | |
| 41 | +export function parseInventoryPage(htmlText: string, url: string): InventoryPagePayload { | |
| 42 | + const $ = H.load(htmlText); | |
| 43 | + const cards: Card[] = []; | |
| 44 | + const seen = new Set<string>(); | |
| 45 | + $('div').each((_, d) => { | |
| 46 | + const $d = $(d); | |
| 47 | + if ($d.children('h3').length !== 1 || $d.children('p').length !== 1) return; | |
| 48 | + const card = $d.parent(); | |
| 49 | + const href = card.find('a[href^="/watch/"]').first().attr('href') ?? null; | |
| 50 | + if (!href) return; | |
| 51 | + const id = href.match(/-(\d+)\/?$/)?.[1] ?? href.replace(/^\/watch\//, ''); | |
| 52 | + if (seen.has(id)) return; | |
| 53 | + const title = H.text($d.children('h3').first()); | |
| 54 | + const priceText = H.text($d.children('p').first()); | |
| 55 | + if (!title) return; | |
| 56 | + const price = parsePrice(priceText ?? '', 'USD'); | |
| 57 | + const badge = H.text($d.children('div').first()); | |
| 58 | + const img = card.find('img').first(); | |
| 59 | + seen.add(id); | |
| 60 | + cards.push({ id, href: `${SITE}${href}`, title, price: price && price.amount > 0 ? price.amount : null, currency: price?.currency ?? null, badge, image: decodeNextImage(img.attr('src') ?? img.attr('srcset')?.split(' ')[0]) }); | |
| 61 | + }); | |
| 62 | + return { kind: 'inventory_page', url, cards }; | |
| 63 | +} | |
| 64 | + | |
| 65 | +export function parseProductPage(htmlText: string, url: string): PagePayload | null { | |
| 66 | + const prod = H.jsonLd(htmlText, 'Product')[0]; | |
| 67 | + if (!prod) return null; | |
| 68 | + const offers = (Array.isArray(prod.offers) ? prod.offers[0] : prod.offers) as Record<string, unknown> | undefined; | |
| 69 | + const brand = prod.brand && typeof prod.brand === 'object' ? String((prod.brand as { name?: string }).name ?? '') : prod.brand ? String(prod.brand) : null; | |
| 70 | + const img = prod.image; | |
| 71 | + const id = url.match(/-(\d+)\/?$/)?.[1] ?? String(prod.sku ?? ''); | |
| 72 | + const price = offers?.price !== undefined ? Number(offers.price) : NaN; | |
| 73 | + return { | |
| 74 | + kind: 'product_page', | |
| 75 | + url, | |
| 76 | + product: { | |
| 77 | + id, | |
| 78 | + href: url, | |
| 79 | + name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(), | |
| 80 | + sku: prod.sku ? String(prod.sku) : null, | |
| 81 | + mpn: prod.mpn ? String(prod.mpn) : null, | |
| 82 | + brand: brand || null, | |
| 83 | + price: Number.isFinite(price) && price > 0 ? price : null, | |
| 84 | + currency: offers?.priceCurrency ? String(offers.priceCurrency) : null, | |
| 85 | + availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null, | |
| 86 | + condition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null, | |
| 87 | + description: prod.description ? String(prod.description).slice(0, 1200) : null, | |
| 88 | + images: Array.isArray(img) ? img.slice(0, 4).map(String) : typeof img === 'string' ? [img] : [], | |
| 89 | + }, | |
| 90 | + }; | |
| 91 | +} | |
| 92 | + | |
| 93 | +export class EuropeanWatchCompanyConnector extends BaseConnector { | |
| 94 | + readonly version = '1.0.0'; | |
| 95 | + readonly parserVersion = PARSER_VERSION; | |
| 96 | + protected override minIntervalMs = 3000; | |
| 97 | + override readonly urlPatterns = [/^https?:\/\/(?:www\.)?europeanwatch\.com\/watch\/([a-z0-9-]+)/i]; | |
| 98 | + | |
| 99 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 100 | + const pages = (this.meta.config.pages as string[] | undefined) ?? ['/all']; | |
| 101 | + let count = 0; | |
| 102 | + for (const path of pages) { | |
| 103 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 104 | + const url = `${SITE}${path}`; | |
| 105 | + await this.throttle(url); | |
| 106 | + const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', timeoutMs: 90_000, expect: ['title', 'price', 'currency'], parse: (r) => { | |
| 107 | + const p = r.html ? parseInventoryPage(r.html, url) : null; | |
| 108 | + const c = p?.cards.find((x) => x.price); | |
| 109 | + return c ? { title: c.title, price: c.price, currency: c.currency } : null; | |
| 110 | + } }); | |
| 111 | + if (!res.success || !res.html) { | |
| 112 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 113 | + continue; | |
| 114 | + } | |
| 115 | + const payload = parseInventoryPage(res.html, url); | |
| 116 | + if (!payload.cards.length) { | |
| 117 | + ctx.anomaly('selector_missing', `${url}: no inventory cards parsed`); | |
| 118 | + continue; | |
| 119 | + } | |
| 120 | + count++; | |
| 121 | + yield { url, externalId: `inventory:${path}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 122 | + await ctx.setCursor({ page: path, at: new Date().toISOString() }); | |
| 123 | + } | |
| 124 | + await ctx.setCursor({ done: true, at: new Date().toISOString() }); | |
| 125 | + } | |
| 126 | + | |
| 127 | + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { | |
| 128 | + const slug = url.match(this.urlPatterns[0]!)?.[1]; | |
| 129 | + if (!slug) return []; | |
| 130 | + const target = `${SITE}/watch/${slug}`; | |
| 131 | + await this.throttle(target); | |
| 132 | + const res = await ctx.fetch(target, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0.2 }); | |
| 133 | + const payload = res.success && res.html ? parseProductPage(res.html, target) : null; | |
| 134 | + if (!payload) return []; | |
| 135 | + return [{ url: target, externalId: `product:${slug}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; | |
| 136 | + } | |
| 137 | + | |
| 138 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 139 | + const p = PagePayloadSchema.parse(raw.payload); | |
| 140 | + const out: NormalizedRecord[] = []; | |
| 141 | + if (p.kind === 'product_page') { | |
| 142 | + const pr = p.product; | |
| 143 | + const w = watchFromTitle(pr.name, pr.brand); | |
| 144 | + const reference = pr.mpn ?? w.reference; | |
| 145 | + const conditionRaw = pr.condition === 'NewCondition' ? 'Unworn' : w.conditionRaw ?? (pr.condition ? 'Pre-owned' : null); | |
| 146 | + out.push( | |
| 147 | + NormalizedListingSchema.parse({ | |
| 148 | + kind: 'listing', | |
| 149 | + connectorId: this.meta.id, | |
| 150 | + sourceId: this.meta.sourceId, | |
| 151 | + sourceUrl: pr.href, | |
| 152 | + externalId: pr.id, | |
| 153 | + rawTitle: pr.name, | |
| 154 | + description: pr.description, | |
| 155 | + imageUrls: pr.images, | |
| 156 | + attributes: AssetAttributesSchema.parse({ categorySlug: w.categorySlug, brand: w.brand, name: pr.name, reference, year: w.year, material: w.material, size: w.size, identifiers: { ewc_sku: pr.sku ?? pr.id, ...(reference ? { reference } : {}) }, metadata: { availability_raw: pr.availability, condition_raw: pr.condition } }), | |
| 157 | + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, | |
| 158 | + condition: { condition: normalizeCondition(w.categorySlug, conditionRaw), conditionRaw, completeness: w.completeness }, | |
| 159 | + observedAt: raw.fetchedAt, | |
| 160 | + confidence: 0.85, | |
| 161 | + parserVersion: PARSER_VERSION, | |
| 162 | + listingType: 'fixed_price', | |
| 163 | + price: pr.price, | |
| 164 | + currency: pr.currency === 'USD' ? 'USD' : pr.price ? 'USD' : null, | |
| 165 | + seller: 'European Watch Company', | |
| 166 | + location: 'Boston, MA, US', | |
| 167 | + quantity: 1, | |
| 168 | + availability: pr.availability === 'InStock' ? 'available' : pr.availability === 'OutOfStock' ? 'ended' : 'unknown', | |
| 169 | + }), | |
| 170 | + ); | |
| 171 | + return out; | |
| 172 | + } | |
| 173 | + for (const c of p.cards) { | |
| 174 | + const w = watchFromTitle(c.title); | |
| 175 | + const pending = /sale pending/i.test(c.badge ?? ''); | |
| 176 | + out.push( | |
| 177 | + NormalizedListingSchema.parse({ | |
| 178 | + kind: 'listing', | |
| 179 | + connectorId: this.meta.id, | |
| 180 | + sourceId: this.meta.sourceId, | |
| 181 | + sourceUrl: c.href, | |
| 182 | + externalId: c.id, | |
| 183 | + rawTitle: c.title, | |
| 184 | + description: null, | |
| 185 | + imageUrls: c.image ? [c.image] : [], | |
| 186 | + attributes: AssetAttributesSchema.parse({ categorySlug: w.categorySlug, brand: w.brand, name: c.title, reference: w.reference, year: w.year, material: w.material, size: w.size, identifiers: { ewc_sku: c.id, ...(w.reference ? { reference: w.reference } : {}) }, metadata: { badge: c.badge, sale_pending: pending } }), | |
| 187 | + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, | |
| 188 | + condition: { condition: normalizeCondition(w.categorySlug, w.conditionRaw), conditionRaw: w.conditionRaw, completeness: w.completeness }, | |
| 189 | + observedAt: raw.fetchedAt, | |
| 190 | + confidence: 0.8, | |
| 191 | + parserVersion: PARSER_VERSION, | |
| 192 | + listingType: 'fixed_price', | |
| 193 | + price: c.price, | |
| 194 | + currency: c.price ? ((c.currency as 'USD' | null) ?? 'USD') : null, | |
| 195 | + seller: 'European Watch Company', | |
| 196 | + location: 'Boston, MA, US', | |
| 197 | + quantity: 1, | |
| 198 | + availability: 'available', | |
| 199 | + }), | |
| 200 | + ); | |
| 201 | + } | |
| 202 | + return out; | |
| 203 | + } | |
| 204 | +} | |
| 205 | + | |
| 206 | +export default function createConnector(meta: ConnectorMeta) { | |
| 207 | + return new EuropeanWatchCompanyConnector(meta); | |
| 208 | +} | |
added
connectors/api/european-watch-company/meta.json
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +{ | |
| 2 | + "id": "european-watch-company", | |
| 3 | + "displayName": "European Watch Company (Boston dealer — inventory)", | |
| 4 | + "sourceId": "european-watch-company", | |
| 5 | + "sourceName": "European Watch Company", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.europeanwatch.com", | |
| 8 | + "module": "api/european-watch-company", | |
| 9 | + "enginePriority": ["api", "firecrawl"], | |
| 10 | + "categories": ["watches", "rolex", "patek_philippe", "audemars_piguet", "omega", "other_watches"], | |
| 11 | + "regions": ["US"], | |
| 12 | + "country": "US", | |
| 13 | + "languages": ["en"], | |
| 14 | + "currency": ["USD"], | |
| 15 | + "supportsListings": true, | |
| 16 | + "supportsSold": false, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": true, | |
| 22 | + "refreshFrequencyMinutes": 720, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.8, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.europeanwatch.com/terms-and-conditions", | |
| 27 | + "acquisitionMethod": "server-rendered inventory grid + schema.org Product JSON-LD, direct HTTP", | |
| 28 | + "historicalDepth": "none", | |
| 29 | + "accessNotes": "One request per run: the public inventory grid `/all` (≈5 MB, server-rendered, ~750 watches) gives brand + model title, USD asking price, status badge (In Stock / New Arrival / Sale Pending), image and product URL (`/watch/<slug>-<id>`); parsed with cheerio from the card markup, no JS. Product pages (schema.org Product with sku, mpn = reference, price, availability, condition) are fetched only for URL lookup. robots.txt disallows only /private/, /api/, /app/, /profile/ (verified 2026-09-08); the sitemap `sitemap-products.xml` mirrors the same 750 URLs. Asking prices are listings, never sales; 'Sale Pending' is kept as metadata. No account, cart or customer pages are touched. 3 s between requests, concurrency 1.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "requires": [], | |
| 33 | + "config": { | |
| 34 | + "pages": ["/all"] | |
| 35 | + } | |
| 36 | +} | |
added
connectors/api/everything-games/README.md
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +# Everything Games connector (`everything-games`) | |
| 2 | + | |
| 3 | +- Source: https://www.everythinggames.ca · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Winnipeg TCG/wargame store. Shopify storefront; singles titled 'Name [SET - number]' with condition/language/finish variants. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `mtg-singles` | `magic_the_gathering` | Magic: The Gathering | | |
| 14 | +| `pokemon-singles-instock` | `pokemon` | Pokémon | | |
| 15 | +| `grand-archive-1` | `other_tcg` | Grand Archive | | |
| 16 | +| `flesh-and-blood-singles` | `flesh_and_blood` | Flesh and Blood | | |
| 17 | +| `weiss-singles` | `weiss_schwarz` | Weiß Schwarz | | |
| 18 | +| `riftbound-singles` | `other_tcg` | Riftbound | | |
| 19 | +| `shadowverse` | `other_tcg` | Shadowverse Evolve | | |
| 20 | +| `board-games` | `board_games` | — | | |
| 21 | + | |
| 22 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live` | |
| 23 | + | |
| 24 | +Title pattern: `^(?<name>.+?) \[(?<set>[A-Z0-9]+)(?: - (?<number>\S+))?\]$` (name / set / number) | |
| 25 | + | |
| 26 | +## Access & compliance | |
| 27 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: Warhammer/miniature ranges, paints, dice, RPG books, supplies. | |
| 28 | + | |
| 29 | +## Fixtures & tests | |
| 30 | +`data/fixtures/everything-games/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts everything-games`), trimmed single-product payloads incl. a sold-out variant. | |
| 31 | +`pnpm vitest run connectors/api/everything-games` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts everything-games`. | |
added
connectors/api/everything-games/index.test.ts
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Everything Games — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/everything-games/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "Survival of the Fittest [EXO - 129]", | |
| 15 | + "productType": "Magic: The Gathering Singles", | |
| 16 | + "collection": "mtg-singles", | |
| 17 | + "categorySlug": "magic_the_gathering" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "Dragon Shield Sleeves Matte Blue (100)", | |
| 21 | + "collection": "mtg-singles", | |
| 22 | + "categorySlug": null | |
| 23 | + } | |
| 24 | + ] | |
| 25 | +}); | |
added
connectors/api/everything-games/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Everything Games — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/everything-games/meta.json
+102 −0
@@ -0,0 +1,102 @@ | ||
| 1 | +{ | |
| 2 | + "id": "everything-games", | |
| 3 | + "displayName": "Everything Games (Canadian TCG & board-game store, CAD)", | |
| 4 | + "sourceId": "everything-games", | |
| 5 | + "sourceName": "Everything Games", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.everythinggames.ca", | |
| 8 | + "module": "api/everything-games", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "pokemon", | |
| 15 | + "other_tcg", | |
| 16 | + "flesh_and_blood", | |
| 17 | + "weiss_schwarz", | |
| 18 | + "board_games" | |
| 19 | + ], | |
| 20 | + "regions": [ | |
| 21 | + "CA" | |
| 22 | + ], | |
| 23 | + "languages": [ | |
| 24 | + "en" | |
| 25 | + ], | |
| 26 | + "currency": [ | |
| 27 | + "CAD" | |
| 28 | + ], | |
| 29 | + "supportsListings": true, | |
| 30 | + "supportsSold": false, | |
| 31 | + "supportsAuctions": false, | |
| 32 | + "supportsImages": true, | |
| 33 | + "supportsCatalog": false, | |
| 34 | + "supportsPopulation": false, | |
| 35 | + "supportsLookup": true, | |
| 36 | + "refreshFrequencyMinutes": 720, | |
| 37 | + "priority": "medium", | |
| 38 | + "trustScore": 0.75, | |
| 39 | + "attributionRequired": true, | |
| 40 | + "termsUrl": "https://www.everythinggames.ca/policies/terms-of-service", | |
| 41 | + "accessNotes": "Everything Games (everythinggames.ca) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 8 configured collections (mtg-singles, pokemon-singles-instock, grand-archive-1, flesh-and-blood-singles, weiss-singles, riftbound-singles, shadowverse, board-games) and /products/<handle>.json for URL lookups (~20k MTG singles, 3k Pokémon, Grand Archive/FaB/Weiß singles). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — Warhammer/miniature ranges, paints, dice, RPG books, supplies. No personal data is collected; seller = the store itself.", | |
| 42 | + "enabled": true, | |
| 43 | + "schemaVersion": "1.0", | |
| 44 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 45 | + "historicalDepth": "none", | |
| 46 | + "requires": [], | |
| 47 | + "config": { | |
| 48 | + "currency": "CAD", | |
| 49 | + "market": "CA", | |
| 50 | + "seller": "Everything Games", | |
| 51 | + "location": "Winnipeg, MB, Canada", | |
| 52 | + "collections": [ | |
| 53 | + { | |
| 54 | + "handle": "mtg-singles", | |
| 55 | + "categorySlug": "magic_the_gathering", | |
| 56 | + "franchise": "Magic: The Gathering" | |
| 57 | + }, | |
| 58 | + { | |
| 59 | + "handle": "pokemon-singles-instock", | |
| 60 | + "categorySlug": "pokemon", | |
| 61 | + "franchise": "Pokémon" | |
| 62 | + }, | |
| 63 | + { | |
| 64 | + "handle": "grand-archive-1", | |
| 65 | + "categorySlug": "other_tcg", | |
| 66 | + "franchise": "Grand Archive" | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "handle": "flesh-and-blood-singles", | |
| 70 | + "categorySlug": "flesh_and_blood", | |
| 71 | + "franchise": "Flesh and Blood" | |
| 72 | + }, | |
| 73 | + { | |
| 74 | + "handle": "weiss-singles", | |
| 75 | + "categorySlug": "weiss_schwarz", | |
| 76 | + "franchise": "Weiß Schwarz" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "riftbound-singles", | |
| 80 | + "categorySlug": "other_tcg", | |
| 81 | + "franchise": "Riftbound" | |
| 82 | + }, | |
| 83 | + { | |
| 84 | + "handle": "shadowverse", | |
| 85 | + "categorySlug": "other_tcg", | |
| 86 | + "franchise": "Shadowverse Evolve" | |
| 87 | + }, | |
| 88 | + { | |
| 89 | + "handle": "board-games", | |
| 90 | + "categorySlug": "board_games" | |
| 91 | + } | |
| 92 | + ], | |
| 93 | + "rules": [], | |
| 94 | + "defaultCategory": null, | |
| 95 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live", | |
| 96 | + "keepOutOfStock": true, | |
| 97 | + "fetchBarcodes": false, | |
| 98 | + "wholeShop": false, | |
| 99 | + "pageSize": 250, | |
| 100 | + "titlePattern": "^(?<name>.+?) \\[(?<set>[A-Z0-9]+)(?: - (?<number>\\S+))?\\]$" | |
| 101 | + } | |
| 102 | +} | |
added
connectors/api/face-to-face-games/README.md
+35 −0
@@ -0,0 +1,35 @@ | ||
| 1 | +# Face to Face Games connector (`face-to-face-games`) | |
| 2 | + | |
| 3 | +- Source: https://www.facetofacegames.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Major Canadian MTG/Pokémon/Lorcana retailer (Toronto + Montréal). Shopify storefront with clean per-game singles/sealed/high-end collections; titles embed number and set code. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `magic-the-gathering-singles` | `magic_the_gathering` | Magic: The Gathering | | |
| 14 | +| `magic-the-gathering-high-end` | `magic_the_gathering` | Magic: The Gathering | | |
| 15 | +| `magic-the-gathering-sealed` | `magic_the_gathering` | Magic: The Gathering | | |
| 16 | +| `pokemon-singles` | `pokemon` | Pokémon | | |
| 17 | +| `pokemon-high-end` | `pokemon` | Pokémon | | |
| 18 | +| `pokemon-sealed` | `pokemon` | Pokémon | | |
| 19 | +| `lorcana-singles` | `disney_lorcana` | Disney Lorcana | | |
| 20 | +| `lorcana-sealed` | `disney_lorcana` | Disney Lorcana | | |
| 21 | +| `yu-gi-oh-premium` | `yugioh` | Yu-Gi-Oh! | | |
| 22 | +| `one-piece` | `one_piece_card_game` | One Piece | | |
| 23 | +| `star-wars-sealed` | `star_wars_tcg` | Star Wars Unlimited | | |
| 24 | +| `riftbound-singles` | `other_tcg` | Riftbound | | |
| 25 | + | |
| 26 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live` | |
| 27 | + | |
| 28 | +Title pattern: `^(?<name>.+?) - (?<number>[^\s\[]+) - [^\[]*\[(?<set>[a-z0-9]+)(?:-[^\]]*)?\]` (name / set / number) | |
| 29 | + | |
| 30 | +## Access & compliance | |
| 31 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: events/WPN tickets, supplies (sleeves, deck boxes, playmats), the giant "single"/"archive"/flash-sale umbrella collections (duplicates of the per-game ones). | |
| 32 | + | |
| 33 | +## Fixtures & tests | |
| 34 | +`data/fixtures/face-to-face-games/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts face-to-face-games`), trimmed single-product payloads incl. a sold-out variant and a graded item. | |
| 35 | +`pnpm vitest run connectors/api/face-to-face-games` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts face-to-face-games`. | |
added
connectors/api/face-to-face-games/index.test.ts
+30 −0
@@ -0,0 +1,30 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Face to Face Games — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/face-to-face-games/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "Mew - 011/025 - Rare Holo [cel25-011] [Holo]", | |
| 15 | + "productType": "Singles", | |
| 16 | + "collection": "pokemon-singles", | |
| 17 | + "categorySlug": "pokemon" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "Dragon Shield Sleeves - Matte Black", | |
| 21 | + "collection": "magic-the-gathering-sealed", | |
| 22 | + "categorySlug": null | |
| 23 | + }, | |
| 24 | + { | |
| 25 | + "title": "Store Credit Top-Up", | |
| 26 | + "collection": "magic-the-gathering-singles", | |
| 27 | + "categorySlug": null | |
| 28 | + } | |
| 29 | + ] | |
| 30 | +}); | |
added
connectors/api/face-to-face-games/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Face to Face Games — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/face-to-face-games/meta.json
+125 −0
@@ -0,0 +1,125 @@ | ||
| 1 | +{ | |
| 2 | + "id": "face-to-face-games", | |
| 3 | + "displayName": "Face to Face Games (Canadian TCG store, CAD)", | |
| 4 | + "sourceId": "face-to-face-games", | |
| 5 | + "sourceName": "Face to Face Games", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.facetofacegames.com", | |
| 8 | + "module": "api/face-to-face-games", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "pokemon", | |
| 15 | + "disney_lorcana", | |
| 16 | + "yugioh", | |
| 17 | + "one_piece_card_game", | |
| 18 | + "star_wars_tcg", | |
| 19 | + "other_tcg" | |
| 20 | + ], | |
| 21 | + "regions": [ | |
| 22 | + "CA" | |
| 23 | + ], | |
| 24 | + "languages": [ | |
| 25 | + "en", | |
| 26 | + "fr" | |
| 27 | + ], | |
| 28 | + "currency": [ | |
| 29 | + "CAD" | |
| 30 | + ], | |
| 31 | + "supportsListings": true, | |
| 32 | + "supportsSold": false, | |
| 33 | + "supportsAuctions": false, | |
| 34 | + "supportsImages": true, | |
| 35 | + "supportsCatalog": false, | |
| 36 | + "supportsPopulation": false, | |
| 37 | + "supportsLookup": true, | |
| 38 | + "refreshFrequencyMinutes": 720, | |
| 39 | + "priority": "medium", | |
| 40 | + "trustScore": 0.75, | |
| 41 | + "attributionRequired": true, | |
| 42 | + "termsUrl": "https://www.facetofacegames.com/policies/terms-of-service", | |
| 43 | + "accessNotes": "Face to Face Games (facetofacegames.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 12 configured collections (magic-the-gathering-singles, magic-the-gathering-high-end, magic-the-gathering-sealed, pokemon-singles, pokemon-high-end, pokemon-sealed, lorcana-singles, lorcana-sealed … (+4 more, see config.collections)) and /products/<handle>.json for URL lookups (~290k singles; MTG 168k, Pokémon 40k, high-end/graded 15k). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Third-party grades in titles (PSA/BGS/CGC/ICCS/PMG…) are parsed by parseGradeFromTitle; cert numbers are not extracted. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — events/WPN tickets, supplies (sleeves, deck boxes, playmats), the giant \"single\"/\"archive\"/flash-sale umbrella collections (duplicates of the per-game ones). No personal data is collected; seller = the store itself.", | |
| 44 | + "enabled": true, | |
| 45 | + "schemaVersion": "1.0", | |
| 46 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 47 | + "historicalDepth": "none", | |
| 48 | + "requires": [], | |
| 49 | + "config": { | |
| 50 | + "currency": "CAD", | |
| 51 | + "market": "CA", | |
| 52 | + "seller": "Face to Face Games", | |
| 53 | + "location": "Toronto / Montréal, Canada", | |
| 54 | + "collections": [ | |
| 55 | + { | |
| 56 | + "handle": "magic-the-gathering-singles", | |
| 57 | + "categorySlug": "magic_the_gathering", | |
| 58 | + "franchise": "Magic: The Gathering" | |
| 59 | + }, | |
| 60 | + { | |
| 61 | + "handle": "magic-the-gathering-high-end", | |
| 62 | + "categorySlug": "magic_the_gathering", | |
| 63 | + "franchise": "Magic: The Gathering" | |
| 64 | + }, | |
| 65 | + { | |
| 66 | + "handle": "magic-the-gathering-sealed", | |
| 67 | + "categorySlug": "magic_the_gathering", | |
| 68 | + "franchise": "Magic: The Gathering" | |
| 69 | + }, | |
| 70 | + { | |
| 71 | + "handle": "pokemon-singles", | |
| 72 | + "categorySlug": "pokemon", | |
| 73 | + "franchise": "Pokémon" | |
| 74 | + }, | |
| 75 | + { | |
| 76 | + "handle": "pokemon-high-end", | |
| 77 | + "categorySlug": "pokemon", | |
| 78 | + "franchise": "Pokémon" | |
| 79 | + }, | |
| 80 | + { | |
| 81 | + "handle": "pokemon-sealed", | |
| 82 | + "categorySlug": "pokemon", | |
| 83 | + "franchise": "Pokémon" | |
| 84 | + }, | |
| 85 | + { | |
| 86 | + "handle": "lorcana-singles", | |
| 87 | + "categorySlug": "disney_lorcana", | |
| 88 | + "franchise": "Disney Lorcana" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "handle": "lorcana-sealed", | |
| 92 | + "categorySlug": "disney_lorcana", | |
| 93 | + "franchise": "Disney Lorcana" | |
| 94 | + }, | |
| 95 | + { | |
| 96 | + "handle": "yu-gi-oh-premium", | |
| 97 | + "categorySlug": "yugioh", | |
| 98 | + "franchise": "Yu-Gi-Oh!" | |
| 99 | + }, | |
| 100 | + { | |
| 101 | + "handle": "one-piece", | |
| 102 | + "categorySlug": "one_piece_card_game", | |
| 103 | + "franchise": "One Piece" | |
| 104 | + }, | |
| 105 | + { | |
| 106 | + "handle": "star-wars-sealed", | |
| 107 | + "categorySlug": "star_wars_tcg", | |
| 108 | + "franchise": "Star Wars Unlimited" | |
| 109 | + }, | |
| 110 | + { | |
| 111 | + "handle": "riftbound-singles", | |
| 112 | + "categorySlug": "other_tcg", | |
| 113 | + "franchise": "Riftbound" | |
| 114 | + } | |
| 115 | + ], | |
| 116 | + "rules": [], | |
| 117 | + "defaultCategory": null, | |
| 118 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live", | |
| 119 | + "keepOutOfStock": true, | |
| 120 | + "fetchBarcodes": false, | |
| 121 | + "wholeShop": false, | |
| 122 | + "pageSize": 250, | |
| 123 | + "titlePattern": "^(?<name>.+?) - (?<number>[^\\s\\[]+) - [^\\[]*\\[(?<set>[a-z0-9]+)(?:-[^\\]]*)?\\]" | |
| 124 | + } | |
| 125 | +} | |
added
connectors/api/fanatics-collect/README.md
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +# fanatics-collect | |
| 2 | + | |
| 3 | +Fanatics Collect (ex-PWCC) sold results — weekly/premier auctions and fixed-price sales of sports cards, TCG and memorabilia. | |
| 4 | + | |
| 5 | +- **Discovery**: `/sitemap.xml` → `sitemap/sales-history-{weekly-auction,fixed-price}-N.xml.gz` (gzip url sets with `lastmod`, newest = highest N). | |
| 6 | +- **Item pages**: `/weekly/<uuid>`, `/fixed/<uuid>/<slug>` — the Next.js RSC payload carries `prefetchedItemData` (CollectListing) with `collectSales[{soldDate, soldFor}]`. | |
| 7 | +- **Records**: `sale` only when `collectSales` is non-empty. `soldFor` includes the 20 % buyer's premium (`buyerPremiumIncluded: true` for auctions, hammer = `currentBid` in metadata). Fixed-price/best-offer → `saleType: fixed_price`, premium `null`. | |
| 8 | +- **Cursor**: incremental `{ since }` (newest lastmod processed, newest-first walk); backfill `{ sitemap, offset, urlsDone }` oldest→newest with `ctx.progress`, ends with `{ done: true }`. | |
| 9 | +- **Lookup**: item URLs are supported (`urlPatterns`). | |
| 10 | +- Smoke: `set -a; . ./.env; set +a; pnpm tsx connectors/api/_g7-auctions-na-lib/smoke.ts api/fanatics-collect --limit 3`. | |
added
connectors/api/fanatics-collect/index.test.ts
+137 −0
@@ -0,0 +1,137 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import { fileURLToPath } from 'node:url'; | |
| 5 | +import { runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 7 | +import createConnector, { parseItemPage, salesHistorySitemaps } from './index.js'; | |
| 8 | + | |
| 9 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 10 | +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 11 | +const connector = createConnector(meta); | |
| 12 | + | |
| 13 | +/** Real prefetchedItemData captured live from https://www.fanaticscollect.com/weekly/1e389fce-6f23-11ed-a610-0a58a9feac02 (2026-09-08), trimmed. */ | |
| 14 | +const SOLD_LISTING = { | |
| 15 | + __typename: 'CollectListing', | |
| 16 | + id: '1e389fce-6f23-11ed-a610-0a58a9feac02', | |
| 17 | + title: '2003 Fleer Tradition LeBron James ROOKIE #261 PSA 9 MINT', | |
| 18 | + listingType: 'WEEKLY', | |
| 19 | + isManageable: false, | |
| 20 | + certifiedSeller: 'Fanatics Collect', | |
| 21 | + currentBid: { __typename: 'Money', amountInCents: 23000, currency: 'USD' }, | |
| 22 | + auction: { __typename: 'CollectWeeklyAuction', id: '8d27de96-6f1d-11ed-ab20-0a58a9feac02', auctionIntegerId: 30, integerId: 172, name: '2022 Weekly Auction #1', shortName: 'WA1', payoutDate: '2022-02-16', startsAt: '2022-01-14T03:00:00Z', endsAt: '2022-01-24T03:00:00Z', status: 'CLOSED' }, | |
| 23 | + integerId: 3093498, | |
| 24 | + insertedAt: '2021-12-22T13:41:22Z', | |
| 25 | + updatedAt: '2023-01-09T18:38:25Z', | |
| 26 | + soldDate: null, | |
| 27 | + lotString: 'WA1 Lot: 1100', | |
| 28 | + imageSets: [{ __typename: 'CollectImageSet', large: 'https://dilxwvfkfup17.cloudfront.net/abc-large', medium: 'https://dilxwvfkfup17.cloudfront.net/abc-medium' }], | |
| 29 | + slug: null, | |
| 30 | + soldFor: null, | |
| 31 | + startingPrice: { __typename: 'Money', amountInCents: 500, currency: 'USD' }, | |
| 32 | + description: 'Welcome to our first Weekly Sunday Auction! This is one of over 3,600 items spanning all sports and genres, closing on the evening of Sunday, January 23rd.', | |
| 33 | + status: 'STATUS_SOLD', | |
| 34 | + bidCount: 27, | |
| 35 | + collectSales: [{ __typename: 'CollectSale', soldDate: '2022-01-24T04:10:06Z', soldFor: { __typename: 'Money', amountInCents: 27600, currency: 'USD' } }], | |
| 36 | + vaultItem: { __typename: 'CollectVaultItem', id: '0bf62e5c-6f1f-11ed-a6c4-0a58a9feac02', integerId: 732786 }, | |
| 37 | +}; | |
| 38 | + | |
| 39 | +/** Real fixed-price (best offer) listing from /fixed/eb86cdda-a912-11f0-8fa5-0a58a9feac02/… — sold via offer, but no CollectSale exposed. */ | |
| 40 | +const FIXED_LISTING = { | |
| 41 | + __typename: 'CollectListing', | |
| 42 | + id: '178984d1-fd50-473c-8e6d-3e96d63b5ba3', | |
| 43 | + title: '2000 Pokemon Insert Card Ancient Mew CGC 5.5 EX+', | |
| 44 | + listingType: 'BO', | |
| 45 | + status: 'BO_STATUS_SOLD', | |
| 46 | + soldDate: null, | |
| 47 | + soldFor: null, | |
| 48 | + startingPrice: { __typename: 'Money', amountInCents: 0, currency: 'USD' }, | |
| 49 | + collectSales: [], | |
| 50 | + vaultItem: { __typename: 'CollectVaultItem', id: '60ecaa72-89d8-11f0-85e1-02f5d6e69f97' }, | |
| 51 | + imageSets: [{ __typename: 'CollectImageSet', large: 'https://cdn-vault.fanaticscollect.com/2025/9/4/rm2/large/v1503807_2020090306325159R_23.jpg' }], | |
| 52 | + slug: '2000-pokemon-insert-card-ancient-mew-cgc-55-ex', | |
| 53 | + description: '', | |
| 54 | +}; | |
| 55 | + | |
| 56 | +/** Wrap an object in a Next.js flight chunk the way the live page does (JS-string escaped JSON). */ | |
| 57 | +function flightHtml(listing: unknown): string { | |
| 58 | + const line = `2f:["$","$L42",null,{"prefetchedItemData":${JSON.stringify(listing)},"foo":1}]\n`; | |
| 59 | + return `<html><head><title>x</title></head><body><script>self.__next_f.push([1,"1:\\"$Sreact.fragment\\"\\n"])</script><script>self.__next_f.push([1,${JSON.stringify(line)}])</script></body></html>`; | |
| 60 | +} | |
| 61 | + | |
| 62 | +const SITEMAP_INDEX = `<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> | |
| 63 | +<sitemap><loc>https://www.fanaticscollect.com/sitemap/pages.xml.gz</loc></sitemap> | |
| 64 | +<sitemap><loc>https://www.fanaticscollect.com/sitemap/weekly-auction-1.xml.gz</loc></sitemap> | |
| 65 | +<sitemap><loc>https://www.fanaticscollect.com/sitemap/sales-history-weekly-auction-1.xml.gz</loc></sitemap> | |
| 66 | +<sitemap><loc>https://www.fanaticscollect.com/sitemap/sales-history-weekly-auction-2.xml.gz</loc></sitemap> | |
| 67 | +<sitemap><loc>https://www.fanaticscollect.com/sitemap/sales-history-fixed-price-1.xml.gz</loc></sitemap> | |
| 68 | +<sitemap><loc>https://www.fanaticscollect.com/sitemap/sales-history-fixed-price-7.xml.gz</loc></sitemap> | |
| 69 | +</sitemapindex>`; | |
| 70 | + | |
| 71 | +describe('fanatics-collect', () => { | |
| 72 | + runFixtureSuite(connector, it, expect); | |
| 73 | + | |
| 74 | + it('parses the RSC flight payload of a sold weekly lot', () => { | |
| 75 | + const url = 'https://www.fanaticscollect.com/weekly/1e389fce-6f23-11ed-a610-0a58a9feac02'; | |
| 76 | + const p = parseItemPage(flightHtml(SOLD_LISTING), url, '2023-01-09')!; | |
| 77 | + expect(p.kind).toBe('fc_item'); | |
| 78 | + expect(p.listing).toMatchObject({ id: '1e389fce-6f23-11ed-a610-0a58a9feac02', listingType: 'WEEKLY', status: 'STATUS_SOLD', bidCount: 27, lotString: 'WA1 Lot: 1100' }); | |
| 79 | + expect(p.listing.collectSales[0]).toEqual({ soldDate: '2022-01-24T04:10:06Z', soldFor: { amountInCents: 27600, currency: 'USD' } }); | |
| 80 | + expect(p.listing.images).toEqual(['https://dilxwvfkfup17.cloudfront.net/abc-large']); | |
| 81 | + expect(p.listing.auction).toMatchObject({ shortName: 'WA1', endsAt: '2022-01-24T03:00:00Z' }); | |
| 82 | + expect((p.listing as Record<string, unknown>).__typename).toBeUndefined(); | |
| 83 | + }); | |
| 84 | + | |
| 85 | + it('normalises a sale with buyer premium included, hammer in metadata, PSA grade and lot number', async () => { | |
| 86 | + const url = 'https://www.fanaticscollect.com/weekly/1e389fce-6f23-11ed-a610-0a58a9feac02'; | |
| 87 | + const p = parseItemPage(flightHtml(SOLD_LISTING), url, '2023-01-09')!; | |
| 88 | + const out = await connector.normalize({ url, externalId: p.listing.id, kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T10:00:00Z'), payload: p }); | |
| 89 | + expect(out.length).toBe(1); | |
| 90 | + const s = out[0]!; | |
| 91 | + if (s.kind !== 'sale') throw new Error('expected sale'); | |
| 92 | + expect(s).toMatchObject({ price: 276, currency: 'USD', buyerPremiumIncluded: true, auctionHouse: 'Fanatics Collect', lotNumber: '1100', saleType: 'auction', location: 'US', externalId: '1e389fce-6f23-11ed-a610-0a58a9feac02' }); | |
| 93 | + expect(s.saleDate.toISOString()).toBe('2022-01-24T04:10:06.000Z'); | |
| 94 | + expect(s.attributes.categorySlug).toBe('basketball_cards'); | |
| 95 | + expect(s.attributes.year).toBe(2003); | |
| 96 | + expect(s.attributes.metadata).toMatchObject({ hammer_price: 230, buyer_premium_pct: 20, auction_short_name: 'WA1', listing_type: 'WEEKLY' }); | |
| 97 | + expect(s.attributes.identifiers).toEqual({ fanatics_listing_id: '1e389fce-6f23-11ed-a610-0a58a9feac02', fanatics_vault_item_id: '0bf62e5c-6f1f-11ed-a6c4-0a58a9feac02' }); | |
| 98 | + expect(s.grade).toMatchObject({ grader: 'psa', grade: '9' }); | |
| 99 | + expect(s.confidence).toBe(0.9); | |
| 100 | + }); | |
| 101 | + | |
| 102 | + it('skips fixed-price pages that expose no CollectSale', async () => { | |
| 103 | + const url = 'https://www.fanaticscollect.com/fixed/eb86cdda-a912-11f0-8fa5-0a58a9feac02/2000-pokemon-insert-card-ancient-mew-cgc-55-ex'; | |
| 104 | + const p = parseItemPage(flightHtml(FIXED_LISTING), url)!; | |
| 105 | + expect(p.listing.listingType).toBe('BO'); | |
| 106 | + expect(p.listing.collectSales).toEqual([]); | |
| 107 | + expect(await connector.normalize({ url, externalId: p.listing.id, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: p })).toEqual([]); | |
| 108 | + }); | |
| 109 | + | |
| 110 | + it('records fixed-price sales without a premium flag when a CollectSale exists', async () => { | |
| 111 | + const url = 'https://www.fanaticscollect.com/fixed/178984d1-fd50-473c-8e6d-3e96d63b5ba3/x'; | |
| 112 | + const p = parseItemPage(flightHtml({ ...FIXED_LISTING, collectSales: [{ __typename: 'CollectSale', soldDate: '2025-10-22T09:59:16.000Z', soldFor: { __typename: 'Money', amountInCents: 500, currency: 'USD' } }] }), url)!; | |
| 113 | + const out = await connector.normalize({ url, externalId: p.listing.id, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: p }); | |
| 114 | + expect(out.length).toBe(1); | |
| 115 | + const s = out[0]!; | |
| 116 | + if (s.kind !== 'sale') throw new Error('expected sale'); | |
| 117 | + expect(s).toMatchObject({ price: 5, saleType: 'fixed_price', buyerPremiumIncluded: null, lotNumber: null }); | |
| 118 | + expect(s.attributes.categorySlug).toBe('pokemon'); | |
| 119 | + expect(s.grade).toMatchObject({ grader: 'cgc', grade: '5.5' }); | |
| 120 | + }); | |
| 121 | + | |
| 122 | + it('returns null for pages without listing JSON and orders sales-history sitemaps newest first', () => { | |
| 123 | + expect(parseItemPage('<html><body>nothing</body></html>', 'https://www.fanaticscollect.com/weekly/x')).toBeNull(); | |
| 124 | + expect(salesHistorySitemaps(SITEMAP_INDEX)).toEqual([ | |
| 125 | + 'https://www.fanaticscollect.com/sitemap/sales-history-fixed-price-7.xml.gz', | |
| 126 | + 'https://www.fanaticscollect.com/sitemap/sales-history-weekly-auction-2.xml.gz', | |
| 127 | + 'https://www.fanaticscollect.com/sitemap/sales-history-fixed-price-1.xml.gz', | |
| 128 | + 'https://www.fanaticscollect.com/sitemap/sales-history-weekly-auction-1.xml.gz', | |
| 129 | + ]); | |
| 130 | + }); | |
| 131 | + | |
| 132 | + it('recognises item URLs for lookup', () => { | |
| 133 | + expect(connector.urlPatterns![0]!.test('https://www.fanaticscollect.com/weekly/1e389fce-6f23-11ed-a610-0a58a9feac02')).toBe(true); | |
| 134 | + expect(connector.urlPatterns![0]!.test('https://www.fanaticscollect.com/fixed/eb86cdda-a912-11f0-8fa5-0a58a9feac02/slug')).toBe(true); | |
| 135 | + expect(connector.urlPatterns![0]!.test('https://www.fanaticscollect.com/marketplace?type=WEEKLY')).toBe(false); | |
| 136 | + }); | |
| 137 | +}); | |
added
connectors/api/fanatics-collect/index.ts
+276 −0
@@ -0,0 +1,276 @@ | ||
| 1 | +import { gunzipSync } from 'node:zlib'; | |
| 2 | +import { z } from 'zod'; | |
| 3 | +import { adapters, BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 4 | +import type { NormalizedRecord } from '@rareindex/shared'; | |
| 5 | +import { amount, cardHouseCategory, certFromTitle, isBundleTitle, isCurrency, isoDate, jsonAfterKey, lotAttributes, makeSale, nextFlightText, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js'; | |
| 6 | + | |
| 7 | +const SITE = 'https://www.fanaticscollect.com'; | |
| 8 | +const HOUSE = 'Fanatics Collect'; | |
| 9 | +const PARSER_VERSION = '1.0.0'; | |
| 10 | + | |
| 11 | +const MoneySchema = z.object({ amountInCents: z.number().nullable().optional(), currency: z.string().nullable().optional() }).nullable().optional(); | |
| 12 | +export const ListingSchema = z.object({ | |
| 13 | + id: z.string(), | |
| 14 | + title: z.string(), | |
| 15 | + listingType: z.string().nullable().optional(), | |
| 16 | + status: z.string().nullable().optional(), | |
| 17 | + bidCount: z.number().nullable().optional(), | |
| 18 | + currentBid: MoneySchema, | |
| 19 | + startingPrice: MoneySchema, | |
| 20 | + collectSales: z.array(z.object({ soldDate: z.string().nullable().optional(), soldFor: MoneySchema })).default([]), | |
| 21 | + auction: z.object({ id: z.string().nullable().optional(), name: z.string().nullable().optional(), shortName: z.string().nullable().optional(), startsAt: z.string().nullable().optional(), endsAt: z.string().nullable().optional(), status: z.string().nullable().optional() }).nullable().optional(), | |
| 22 | + lotString: z.string().nullable().optional(), | |
| 23 | + slug: z.string().nullable().optional(), | |
| 24 | + description: z.string().nullable().optional(), | |
| 25 | + integerId: z.number().nullable().optional(), | |
| 26 | + insertedAt: z.string().nullable().optional(), | |
| 27 | + updatedAt: z.string().nullable().optional(), | |
| 28 | + vaultItem: z.object({ id: z.string().nullable().optional(), integerId: z.number().nullable().optional() }).nullable().optional(), | |
| 29 | + images: z.array(z.string()).default([]), | |
| 30 | +}); | |
| 31 | +export type Listing = z.infer<typeof ListingSchema>; | |
| 32 | +export const PayloadSchema = z.object({ kind: z.literal('fc_item'), url: z.string(), lastmod: z.string().nullable(), listing: ListingSchema }); | |
| 33 | +export type Payload = z.infer<typeof PayloadSchema>; | |
| 34 | + | |
| 35 | +type RawListing = Record<string, unknown> & { imageSets?: Array<{ large?: string | null; medium?: string | null; small?: string | null }> | null; description?: string | null; vaultItem?: Record<string, unknown> | null; auction?: Record<string, unknown> | null }; | |
| 36 | + | |
| 37 | +/** Item page HTML → trimmed CollectListing (RSC flight payload, JSON-LD Product as fallback for title/images). */ | |
| 38 | +export function parseItemPage(html: string, url: string, lastmod: string | null = null): Payload | null { | |
| 39 | + const text = nextFlightText(html); | |
| 40 | + const raw = jsonAfterKey<RawListing>(text, '"prefetchedItemData":'); | |
| 41 | + const ld = H.jsonLd(html, 'Product')[0]; | |
| 42 | + if (!raw && !ld) return null; | |
| 43 | + const images: string[] = []; | |
| 44 | + for (const s of raw?.imageSets ?? []) { | |
| 45 | + const u = s?.large ?? s?.medium ?? s?.small; | |
| 46 | + if (u && !images.includes(u)) images.push(u); | |
| 47 | + } | |
| 48 | + if (!images.length && ld?.image) for (const u of Array.isArray(ld.image) ? ld.image : [ld.image]) if (typeof u === 'string') images.push(u); | |
| 49 | + const pick = (o: Record<string, unknown> | null | undefined, keys: string[]) => (o ? Object.fromEntries(keys.filter((k) => k in o).map((k) => [k, o[k]])) : o ?? null); | |
| 50 | + const listing = { | |
| 51 | + ...pick(raw ?? {}, ['id', 'title', 'listingType', 'status', 'bidCount', 'currentBid', 'startingPrice', 'collectSales', 'lotString', 'slug', 'integerId', 'insertedAt', 'updatedAt']), | |
| 52 | + id: (raw?.id as string | undefined) ?? (typeof ld?.sku === 'string' ? ld.sku : url.match(/\/(?:weekly|fixed|premier)\/([0-9a-f-]{36})/i)?.[1]), | |
| 53 | + title: (raw?.title as string | undefined) ?? (typeof ld?.name === 'string' ? ld.name : undefined), | |
| 54 | + auction: pick(raw?.auction ?? null, ['id', 'name', 'shortName', 'startsAt', 'endsAt', 'status']), | |
| 55 | + vaultItem: pick(raw?.vaultItem ?? null, ['id', 'integerId']), | |
| 56 | + description: typeof raw?.description === 'string' ? raw.description.slice(0, 2000) : null, | |
| 57 | + images: images.slice(0, 6), | |
| 58 | + }; | |
| 59 | + const parsed = ListingSchema.safeParse(listing); | |
| 60 | + if (!parsed.success) return null; | |
| 61 | + return { kind: 'fc_item', url, lastmod, listing: parsed.data }; | |
| 62 | +} | |
| 63 | + | |
| 64 | +/** Sales-history children from the sitemap index, newest first (higher N = newer; fixed-price after weekly of the same N). */ | |
| 65 | +export function salesHistorySitemaps(indexXml: string): string[] { | |
| 66 | + const locs = adapters.parseSitemapIndex(indexXml).map((e) => e.loc).filter((l) => /sales-history/.test(l)); | |
| 67 | + const n = (l: string) => Number(l.match(/-(\d+)\.xml/)?.[1] ?? 0); | |
| 68 | + const kind = (l: string) => (/fixed-price/.test(l) ? 1 : 0); | |
| 69 | + return locs.sort((a, b) => n(b) - n(a) || kind(b) - kind(a)); | |
| 70 | +} | |
| 71 | + | |
| 72 | +function gunzipMaybe(buf: Uint8Array | null | undefined, text: string | null | undefined): string | null { | |
| 73 | + if (buf && buf.byteLength) { | |
| 74 | + const b = Buffer.from(buf); | |
| 75 | + return (b.length >= 2 && b[0] === 0x1f && b[1] === 0x8b ? gunzipSync(b) : b).toString('utf8'); | |
| 76 | + } | |
| 77 | + return text ?? null; | |
| 78 | +} | |
| 79 | + | |
| 80 | +/** | |
| 81 | + * Fanatics Collect (formerly PWCC) — sold results of the weekly/premier auctions and fixed-price sales. | |
| 82 | + * Discovery through the public sitemap index (sales-history-*.xml.gz children with lastmod); each public | |
| 83 | + * item page embeds the listing (incl. collectSales) in its Next.js RSC payload. See meta.json accessNotes. | |
| 84 | + */ | |
| 85 | +export class FanaticsCollectConnector extends BaseConnector { | |
| 86 | + readonly version = '1.0.0'; | |
| 87 | + readonly parserVersion = PARSER_VERSION; | |
| 88 | + override readonly urlPatterns = [/^https?:\/\/(www\.)?fanaticscollect\.com\/(weekly|fixed|premier)\/[0-9a-f-]{36}/i]; | |
| 89 | + protected override minIntervalMs = 1500; | |
| 90 | + | |
| 91 | + private async fetchText(ctx: CrawlContext, url: string, binary = false): Promise<{ text: string | null; status: number | null; fetchedAt: Date }> { | |
| 92 | + await this.throttle(url); | |
| 93 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: binary ? 'binary' : 'text', minQuality: 0, force: binary, timeoutMs: 45_000 }); | |
| 94 | + if (!res.success) { | |
| 95 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 96 | + return { text: null, status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 97 | + } | |
| 98 | + return { text: gunzipMaybe(res.buffer, res.html), status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 99 | + } | |
| 100 | + | |
| 101 | + private async childEntries(ctx: CrawlContext, url: string): Promise<Array<{ loc: string; lastmod: string | null }>> { | |
| 102 | + const r = await this.fetchText(ctx, url, true); | |
| 103 | + if (!r.text) return []; | |
| 104 | + const entries = adapters.parseUrlset(r.text).map((e) => ({ loc: e.loc, lastmod: e.lastmod })); | |
| 105 | + if (!entries.length) ctx.anomaly('pagination_failure', `${url}: empty sitemap`); | |
| 106 | + return entries; | |
| 107 | + } | |
| 108 | + | |
| 109 | + private async itemRecord(ctx: CrawlContext, url: string, lastmod: string | null): Promise<RawRecordInput | null> { | |
| 110 | + const r = await this.fetchText(ctx, url); | |
| 111 | + if (!r.text) return null; | |
| 112 | + const payload = parseItemPage(r.text, url, lastmod); | |
| 113 | + if (!payload) { | |
| 114 | + ctx.anomaly('parse_failure_page', `${url}: no prefetchedItemData / Product JSON-LD`); | |
| 115 | + return null; | |
| 116 | + } | |
| 117 | + return { url, externalId: payload.listing.id, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt }; | |
| 118 | + } | |
| 119 | + | |
| 120 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 121 | + const mode = ctx.options.mode; | |
| 122 | + const cfg = this.meta.config; | |
| 123 | + const pagesPerRun = mode === 'probe' ? Number(cfg.probePages ?? 3) : Number(cfg.pagesPerRun ?? 300); | |
| 124 | + let fetched = 0; | |
| 125 | + let noSale = 0; | |
| 126 | + let count = 0; | |
| 127 | + const track = (rec: RawRecordInput | null) => { | |
| 128 | + fetched++; | |
| 129 | + if (rec && (rec.payload as Payload).listing.collectSales.length === 0) noSale++; | |
| 130 | + }; | |
| 131 | + if (ctx.options.seeds?.length) { | |
| 132 | + for (const url of ctx.options.seeds) { | |
| 133 | + if (ctx.signal?.aborted || this.reached(ctx, count)) break; | |
| 134 | + const rec = await this.itemRecord(ctx, url, null); | |
| 135 | + track(rec); | |
| 136 | + if (rec) { | |
| 137 | + count++; | |
| 138 | + yield rec; | |
| 139 | + } | |
| 140 | + } | |
| 141 | + return; | |
| 142 | + } | |
| 143 | + const idx = await this.fetchText(ctx, `${SITE}/sitemap.xml`); | |
| 144 | + if (!idx.text) return; | |
| 145 | + const children = salesHistorySitemaps(idx.text); | |
| 146 | + if (!children.length) { | |
| 147 | + ctx.anomaly('pagination_failure', 'sitemap index has no sales-history children'); | |
| 148 | + return; | |
| 149 | + } | |
| 150 | + const cursor = ctx.options.cursor ?? {}; | |
| 151 | + if (mode === 'backfill') { | |
| 152 | + // Oldest → newest, resumable by child url + offset. | |
| 153 | + const order = [...children].reverse(); | |
| 154 | + let startIdx = typeof cursor.sitemap === 'string' ? Math.max(0, order.indexOf(cursor.sitemap)) : 0; | |
| 155 | + let offset = startIdx === order.indexOf(cursor.sitemap as string) && typeof cursor.offset === 'number' ? cursor.offset : 0; | |
| 156 | + let urlsDone = typeof cursor.urlsDone === 'number' ? cursor.urlsDone : 0; | |
| 157 | + let items = typeof cursor.itemsProcessed === 'number' ? cursor.itemsProcessed : 0; | |
| 158 | + for (let ci = startIdx; ci < order.length; ci++) { | |
| 159 | + if (ctx.signal?.aborted || fetched >= pagesPerRun) break; | |
| 160 | + const child = order[ci]!; | |
| 161 | + const entries = await this.childEntries(ctx, child); | |
| 162 | + for (let i = offset; i < entries.length; i++) { | |
| 163 | + if (ctx.signal?.aborted || fetched >= pagesPerRun || this.reached(ctx, count)) { | |
| 164 | + await ctx.setCursor({ sitemap: child, offset: i, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() }); | |
| 165 | + await ctx.progress({ page: urlsDone, totalPages: null, itemsProcessed: items, cursor: { sitemap: child, offset: i, urlsDone, itemsProcessed: items } }); | |
| 166 | + this.reportNoSale(ctx, noSale, fetched); | |
| 167 | + return; | |
| 168 | + } | |
| 169 | + const e = entries[i]!; | |
| 170 | + const rec = await this.itemRecord(ctx, e.loc, e.lastmod); | |
| 171 | + track(rec); | |
| 172 | + urlsDone++; | |
| 173 | + if (rec) { | |
| 174 | + items++; | |
| 175 | + count++; | |
| 176 | + yield rec; | |
| 177 | + } | |
| 178 | + if (urlsDone % 25 === 0) { | |
| 179 | + await ctx.setCursor({ sitemap: child, offset: i + 1, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() }); | |
| 180 | + await ctx.progress({ page: urlsDone, totalPages: null, itemsProcessed: items }); | |
| 181 | + } | |
| 182 | + } | |
| 183 | + offset = 0; | |
| 184 | + startIdx = ci + 1; | |
| 185 | + await ctx.setCursor({ sitemap: order[ci + 1] ?? child, offset: 0, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() }); | |
| 186 | + } | |
| 187 | + if (startIdx >= order.length) await ctx.setCursor({ done: true, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() }); | |
| 188 | + this.reportNoSale(ctx, noSale, fetched); | |
| 189 | + return; | |
| 190 | + } | |
| 191 | + // Incremental / probe: newest children first, only entries newer than the last fully processed lastmod. | |
| 192 | + const since = typeof cursor.since === 'string' ? cursor.since : ''; | |
| 193 | + let newest = since; | |
| 194 | + const maxChildren = mode === 'probe' ? 1 : children.length; | |
| 195 | + for (const child of children.slice(0, maxChildren)) { | |
| 196 | + if (ctx.signal?.aborted || fetched >= pagesPerRun) break; | |
| 197 | + const entries = await this.childEntries(ctx, child); | |
| 198 | + const fresh = entries.filter((e) => !since || (e.lastmod ?? '') > since).sort((a, b) => (b.lastmod ?? '').localeCompare(a.lastmod ?? '')); | |
| 199 | + if (!fresh.length) break; // whole child older than the cursor → stop descending | |
| 200 | + for (const e of fresh) { | |
| 201 | + if (ctx.signal?.aborted || fetched >= pagesPerRun || this.reached(ctx, count)) break; | |
| 202 | + const rec = await this.itemRecord(ctx, e.loc, e.lastmod); | |
| 203 | + track(rec); | |
| 204 | + if (e.lastmod && e.lastmod > newest) newest = e.lastmod; | |
| 205 | + if (rec) { | |
| 206 | + count++; | |
| 207 | + yield rec; | |
| 208 | + } | |
| 209 | + } | |
| 210 | + } | |
| 211 | + if (mode !== 'probe' && newest && newest !== since) await ctx.setCursor({ since: newest, updatedAt: new Date().toISOString() }); | |
| 212 | + this.reportNoSale(ctx, noSale, fetched); | |
| 213 | + } | |
| 214 | + | |
| 215 | + private reportNoSale(ctx: CrawlContext, noSale: number, fetched: number): void { | |
| 216 | + if (fetched > 0 && noSale / fetched > 0.3) ctx.anomaly('no_sale_on_page', `${noSale} of ${fetched} item pages carried no collectSales`); | |
| 217 | + } | |
| 218 | + | |
| 219 | + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { | |
| 220 | + const rec = await this.itemRecord(ctx, url, null); | |
| 221 | + return rec ? [rec] : []; | |
| 222 | + } | |
| 223 | + | |
| 224 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 225 | + const p = PayloadSchema.parse(raw.payload); | |
| 226 | + const l = p.listing; | |
| 227 | + const sale = l.collectSales.find((s) => s.soldFor?.amountInCents && s.soldDate) ?? l.collectSales[0]; | |
| 228 | + if (!sale) return []; | |
| 229 | + const price = amount((sale.soldFor?.amountInCents ?? 0) / 100); | |
| 230 | + const saleDate = isoDate(sale.soldDate); | |
| 231 | + if (!price || !saleDate || saleDate.getTime() > Date.now() + 86_400_000) return []; | |
| 232 | + const currency = isCurrency(sale.soldFor?.currency) ? sale.soldFor!.currency! : 'USD'; | |
| 233 | + const type = (l.listingType ?? '').toUpperCase(); | |
| 234 | + const isAuction = type === 'WEEKLY' || type === 'PREMIER' || type === 'FLASH' || type === 'AUCTION'; | |
| 235 | + const hammer = isAuction ? amount((l.currentBid?.amountInCents ?? 0) / 100) : null; | |
| 236 | + const g = saleGrade(l.title); | |
| 237 | + const cert = certFromTitle(l.title) ?? (l.description ? certFromTitle(l.description) : null); | |
| 238 | + const identifiers: Record<string, string> = { fanatics_listing_id: l.id }; | |
| 239 | + if (l.vaultItem?.id) identifiers.fanatics_vault_item_id = l.vaultItem.id; | |
| 240 | + const attributes = lotAttributes({ | |
| 241 | + categorySlug: cardHouseCategory(l.title), | |
| 242 | + name: l.title, | |
| 243 | + year: safeYear(l.title), | |
| 244 | + identifiers, | |
| 245 | + metadata: { auction_name: l.auction?.name ?? null, auction_short_name: l.auction?.shortName ?? null, auction_ends_at: l.auction?.endsAt ?? null, listing_type: l.listingType ?? null, bid_count: l.bidCount ?? null, fanatics_status: l.status ?? null, hammer_price: hammer, buyer_premium_pct: isAuction && hammer && price > hammer ? Math.round(((price / hammer) - 1) * 1000) / 10 : null, sitemap_lastmod: p.lastmod }, | |
| 246 | + }); | |
| 247 | + const record = makeSale({ | |
| 248 | + meta: this.meta, | |
| 249 | + sourceUrl: p.url, | |
| 250 | + externalId: l.id, | |
| 251 | + rawTitle: l.title, | |
| 252 | + description: l.description ?? null, | |
| 253 | + attributes, | |
| 254 | + price, | |
| 255 | + currency, | |
| 256 | + saleDate, | |
| 257 | + buyerPremiumIncluded: isAuction ? true : null, | |
| 258 | + auctionHouse: HOUSE, | |
| 259 | + lotNumber: l.lotString?.match(/Lot:?\s*([A-Za-z0-9-]+)/i)?.[1] ?? null, | |
| 260 | + imageUrls: l.images, | |
| 261 | + location: 'US', | |
| 262 | + observedAt: raw.fetchedAt, | |
| 263 | + parserVersion: PARSER_VERSION, | |
| 264 | + grader: g.grader, | |
| 265 | + grade: g.grade, | |
| 266 | + isBundle: isBundleTitle(l.title), | |
| 267 | + saleType: isAuction ? 'auction' : 'fixed_price', | |
| 268 | + confidence: g.grader ? 0.9 : 0.85, | |
| 269 | + }); | |
| 270 | + record.grade.qualifier = g.qualifier; | |
| 271 | + record.grade.certificationNumber = cert; | |
| 272 | + return [record]; | |
| 273 | + } | |
| 274 | +} | |
| 275 | + | |
| 276 | +export default (meta: ConnectorMeta) => new FanaticsCollectConnector(meta); | |
added
connectors/api/fanatics-collect/meta.json
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +{ | |
| 2 | + "id": "fanatics-collect", | |
| 3 | + "displayName": "Fanatics Collect (sold results)", | |
| 4 | + "sourceId": "fanatics-collect", | |
| 5 | + "sourceName": "Fanatics Collect", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://www.fanaticscollect.com", | |
| 8 | + "module": "api/fanatics-collect", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "other_sports_cards", "pokemon", "magic_the_gathering", "yugioh", "one_piece_card_game", "non_sport_cards", "sports_memorabilia"], | |
| 11 | + "regions": ["US"], | |
| 12 | + "country": "US", | |
| 13 | + "languages": ["en"], | |
| 14 | + "currency": ["USD"], | |
| 15 | + "supportsListings": false, | |
| 16 | + "supportsSold": true, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": true, | |
| 22 | + "refreshFrequencyMinutes": 360, | |
| 23 | + "priority": "high", | |
| 24 | + "trustScore": 0.9, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://help.fanaticscollect.com/hc/en-us/articles/18433319094045-Terms-of-Use", | |
| 27 | + "acquisitionMethod": "sitemap + embedded RSC JSON on public item pages", | |
| 28 | + "historicalDepth": "years", | |
| 29 | + "accessNotes": "Plain HTTPS with the RareIndex user agent (site is behind Cloudflare but serves the bot UA normally; no challenge seen). robots.txt (User-agent *) disallows only /login, /join, /i-am-not-a-robot, /share/ and /email/verify/* — none of which we touch. Discovery: /sitemap.xml (index) → sitemap/sales-history-weekly-auction-N.xml.gz and sitemap/sales-history-fixed-price-N.xml.gz (gzip url sets of ~50 000 URLs each with <lastmod>; 63 children in total, higher N = newer). Each public item page (/weekly/<uuid>, /fixed/<uuid>/<slug>) embeds the listing in its Next.js RSC payload (prefetchedItemData: title, listingType, status, bidCount, currentBid, collectSales[{soldDate, soldFor}], auction name/shortName/endsAt, lotString, images) plus a Product JSON-LD block; we read that HTML only — no GraphQL/API calls, no login, no bidding pages. A sale is recorded only when collectSales is non-empty; its soldFor INCLUDES the buyer's premium (observed: winning bid $230.00 → soldFor $276.00, +20 %), so buyerPremiumIncluded=true for auction listings with the hammer (currentBid) and derived premium % kept in metadata; fixed-price/best-offer sales have no premium concept (null). saleDate = collectSales.soldDate (never fetch time). Pages without a sale are skipped (anomaly if >30 % of a run). The Terms of Use are published on a client-rendered Zendesk page whose text is not readable over plain HTTP, so no automated-access clause could be verified — flagged here for legal review. 1.5 s politeness delay, 0 credits.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "config": { | |
| 33 | + "pagesPerRun": 300, | |
| 34 | + "probePages": 3 | |
| 35 | + } | |
| 36 | +} | |
added
connectors/api/flagshipgames/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Flagship Games connector (`flagshipgames`) | |
| 2 | + | |
| 3 | +- Source: https://www.flagshipgames.sg · Singapore MTG specialist on BinderPOS: singles for most Magic sets back to Ice Age/Fourth Edition, Mystery Booster playtest cards, Final Fantasy and Marvel Universes Beyond, plus English and Japanese Pokémon singles, sealed MTG, One Piece and Lorcana. | |
| 4 | +- Country/currency: SG / SGD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, a `titlePattern` that extracts name/set/number, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (22): `english-pokemon-1` → pokemon, `japanese-pokemon-1` → pokemon, `pokemon-25th-anniversary-celebrations` → pokemon, `mystery-booster` → magic_the_gathering, `final-fantasy` (rules decide), `final-fantasy-commander` → magic_the_gathering, `marvel-super-heroes-commander` → magic_the_gathering, `foundations` → magic_the_gathering, `aetherdrift` → magic_the_gathering, `edge-of-eternities` → magic_the_gathering, `lorwyn-eclipsed` → magic_the_gathering, `bloomburrow` → magic_the_gathering, `duskmourn-house-of-horror` → magic_the_gathering, `modern-horizons-3` → magic_the_gathering, `commander-masters` → magic_the_gathering, `avatar-the-last-airbender` → magic_the_gathering, `ice-age` → magic_the_gathering, `fourth-edition` → magic_the_gathering, `alpha-limited-edition` → magic_the_gathering, `mtg-sealed-products` → magic_the_gathering, `one-piece-card-game` → one_piece_card_game, `lorcana-sealed-products` → disney_lorcana. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/flagshipgames/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts flagshipgames`. | |
added
connectors/api/flagshipgames/index.test.ts
+76 −0
@@ -0,0 +1,76 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('flagshipgames', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Accursed Spirit [Magic 2015]", | |
| 18 | + "collection": "magic-2015", | |
| 19 | + "type": "MTG Single", | |
| 20 | + "variant": "Near Mint Foil", | |
| 21 | + "expect": "magic_the_gathering", | |
| 22 | + "name": "Accursed Spirit", | |
| 23 | + "set": "Magic 2015", | |
| 24 | + "conditionRaw": "Near Mint" | |
| 25 | + }, | |
| 26 | + { | |
| 27 | + "title": "Accelgor (094/086) [Scarlet & Violet: White Flare]", | |
| 28 | + "collection": "english-pokemon-1", | |
| 29 | + "type": "Pokemon Single", | |
| 30 | + "tags": [ | |
| 31 | + "Illustration Rare" | |
| 32 | + ], | |
| 33 | + "variant": "Near Mint Holofoil", | |
| 34 | + "expect": "pokemon", | |
| 35 | + "name": "Accelgor", | |
| 36 | + "number": "094/086", | |
| 37 | + "set": "Scarlet & Violet: White Flare" | |
| 38 | + }, | |
| 39 | + { | |
| 40 | + "title": "A Good Thing [Mystery Booster Playtest Cards]", | |
| 41 | + "collection": "mystery-booster", | |
| 42 | + "type": "MTG Single", | |
| 43 | + "variant": "Near Mint", | |
| 44 | + "expect": "magic_the_gathering" | |
| 45 | + }, | |
| 46 | + { | |
| 47 | + "title": "Clive, Ifrit's Dominant [Final Fantasy]", | |
| 48 | + "collection": "final-fantasy", | |
| 49 | + "type": "MTG Single", | |
| 50 | + "variant": "Near Mint", | |
| 51 | + "expect": "magic_the_gathering" | |
| 52 | + }, | |
| 53 | + { | |
| 54 | + "title": "Charizard / リザードン 001/025", | |
| 55 | + "collection": "japanese-pokemon-1", | |
| 56 | + "type": "Japanese Pokemon", | |
| 57 | + "variant": "Near Mint", | |
| 58 | + "expect": "pokemon", | |
| 59 | + "set": null | |
| 60 | + }, | |
| 61 | + { | |
| 62 | + "title": "Avatar: The Last Airbender - Collector Booster Display", | |
| 63 | + "collection": "mtg-sealed-products", | |
| 64 | + "type": "MTG Sealed", | |
| 65 | + "variant": "New", | |
| 66 | + "expect": "magic_the_gathering" | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "title": "1 Card Mystery Pack", | |
| 70 | + "collection": "english-pokemon-1", | |
| 71 | + "type": "Pokemon Single", | |
| 72 | + "expect": null | |
| 73 | + } | |
| 74 | + ]); | |
| 75 | + }); | |
| 76 | +}); | |
added
connectors/api/flagshipgames/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Flagship Games — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/flagshipgames/meta.json
+182 −0
@@ -0,0 +1,182 @@ | ||
| 1 | +{ | |
| 2 | + "id": "flagshipgames", | |
| 3 | + "displayName": "Flagship Games", | |
| 4 | + "sourceId": "flagshipgames", | |
| 5 | + "sourceName": "Flagship Games", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.flagshipgames.sg", | |
| 8 | + "module": "api/flagshipgames", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "pokemon", | |
| 14 | + "magic_the_gathering", | |
| 15 | + "one_piece_card_game", | |
| 16 | + "disney_lorcana", | |
| 17 | + "final_fantasy_tcg" | |
| 18 | + ], | |
| 19 | + "regions": [ | |
| 20 | + "SG" | |
| 21 | + ], | |
| 22 | + "country": "SG", | |
| 23 | + "languages": [ | |
| 24 | + "en" | |
| 25 | + ], | |
| 26 | + "currency": [ | |
| 27 | + "SGD" | |
| 28 | + ], | |
| 29 | + "supportsListings": true, | |
| 30 | + "supportsSold": false, | |
| 31 | + "supportsAuctions": false, | |
| 32 | + "supportsImages": true, | |
| 33 | + "supportsCatalog": false, | |
| 34 | + "supportsPopulation": false, | |
| 35 | + "supportsLookup": true, | |
| 36 | + "refreshFrequencyMinutes": 720, | |
| 37 | + "priority": "medium", | |
| 38 | + "trustScore": 0.65, | |
| 39 | + "attributionRequired": true, | |
| 40 | + "termsUrl": "https://www.flagshipgames.sg/policies/terms-of-service", | |
| 41 | + "accessNotes": "Flagship Games (flagshipgames.sg) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (english-pokemon-1, japanese-pokemon-1, pokemon-25th-anniversary-celebrations, mystery-booster, final-fantasy, final-fantasy-commander, marvel-super-heroes-commander, foundations, aetherdrift, edge-of-eternities, lorwyn-eclipsed, bloomburrow, duskmourn-house-of-horror, modern-horizons-3, commander-masters, avatar-the-last-airbender, ice-age, fourth-edition, alpha-limited-edition, mtg-sealed-products, one-piece-card-game, lorcana-sealed-products). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency SGD (/meta.json currency SGD, country SG, Shopify.currency rate 1.0), GST included. Variants are condition + finish (Near Mint Foil…); a 0.00 price marks an unstocked condition and is dropped. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 42 | + "enabled": true, | |
| 43 | + "schemaVersion": "1.0", | |
| 44 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 45 | + "historicalDepth": "none", | |
| 46 | + "requires": [], | |
| 47 | + "config": { | |
| 48 | + "currency": "SGD", | |
| 49 | + "seller": "Flagship Games", | |
| 50 | + "location": null, | |
| 51 | + "collections": [ | |
| 52 | + { | |
| 53 | + "handle": "english-pokemon-1", | |
| 54 | + "categorySlug": "pokemon", | |
| 55 | + "franchise": "Pokémon" | |
| 56 | + }, | |
| 57 | + { | |
| 58 | + "handle": "japanese-pokemon-1", | |
| 59 | + "categorySlug": "pokemon", | |
| 60 | + "franchise": "Pokémon" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "pokemon-25th-anniversary-celebrations", | |
| 64 | + "categorySlug": "pokemon", | |
| 65 | + "franchise": "Pokémon" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "handle": "mystery-booster", | |
| 69 | + "categorySlug": "magic_the_gathering", | |
| 70 | + "franchise": "Magic: The Gathering" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "final-fantasy", | |
| 74 | + "categorySlug": null | |
| 75 | + }, | |
| 76 | + { | |
| 77 | + "handle": "final-fantasy-commander", | |
| 78 | + "categorySlug": "magic_the_gathering", | |
| 79 | + "franchise": "Magic: The Gathering" | |
| 80 | + }, | |
| 81 | + { | |
| 82 | + "handle": "marvel-super-heroes-commander", | |
| 83 | + "categorySlug": "magic_the_gathering", | |
| 84 | + "franchise": "Magic: The Gathering" | |
| 85 | + }, | |
| 86 | + { | |
| 87 | + "handle": "foundations", | |
| 88 | + "categorySlug": "magic_the_gathering", | |
| 89 | + "franchise": "Magic: The Gathering" | |
| 90 | + }, | |
| 91 | + { | |
| 92 | + "handle": "aetherdrift", | |
| 93 | + "categorySlug": "magic_the_gathering", | |
| 94 | + "franchise": "Magic: The Gathering" | |
| 95 | + }, | |
| 96 | + { | |
| 97 | + "handle": "edge-of-eternities", | |
| 98 | + "categorySlug": "magic_the_gathering", | |
| 99 | + "franchise": "Magic: The Gathering" | |
| 100 | + }, | |
| 101 | + { | |
| 102 | + "handle": "lorwyn-eclipsed", | |
| 103 | + "categorySlug": "magic_the_gathering", | |
| 104 | + "franchise": "Magic: The Gathering" | |
| 105 | + }, | |
| 106 | + { | |
| 107 | + "handle": "bloomburrow", | |
| 108 | + "categorySlug": "magic_the_gathering", | |
| 109 | + "franchise": "Magic: The Gathering" | |
| 110 | + }, | |
| 111 | + { | |
| 112 | + "handle": "duskmourn-house-of-horror", | |
| 113 | + "categorySlug": "magic_the_gathering", | |
| 114 | + "franchise": "Magic: The Gathering" | |
| 115 | + }, | |
| 116 | + { | |
| 117 | + "handle": "modern-horizons-3", | |
| 118 | + "categorySlug": "magic_the_gathering", | |
| 119 | + "franchise": "Magic: The Gathering" | |
| 120 | + }, | |
| 121 | + { | |
| 122 | + "handle": "commander-masters", | |
| 123 | + "categorySlug": "magic_the_gathering", | |
| 124 | + "franchise": "Magic: The Gathering" | |
| 125 | + }, | |
| 126 | + { | |
| 127 | + "handle": "avatar-the-last-airbender", | |
| 128 | + "categorySlug": "magic_the_gathering", | |
| 129 | + "franchise": "Magic: The Gathering" | |
| 130 | + }, | |
| 131 | + { | |
| 132 | + "handle": "ice-age", | |
| 133 | + "categorySlug": "magic_the_gathering", | |
| 134 | + "franchise": "Magic: The Gathering" | |
| 135 | + }, | |
| 136 | + { | |
| 137 | + "handle": "fourth-edition", | |
| 138 | + "categorySlug": "magic_the_gathering", | |
| 139 | + "franchise": "Magic: The Gathering" | |
| 140 | + }, | |
| 141 | + { | |
| 142 | + "handle": "alpha-limited-edition", | |
| 143 | + "categorySlug": "magic_the_gathering", | |
| 144 | + "franchise": "Magic: The Gathering" | |
| 145 | + }, | |
| 146 | + { | |
| 147 | + "handle": "mtg-sealed-products", | |
| 148 | + "categorySlug": "magic_the_gathering", | |
| 149 | + "franchise": "Magic: The Gathering" | |
| 150 | + }, | |
| 151 | + { | |
| 152 | + "handle": "one-piece-card-game", | |
| 153 | + "categorySlug": "one_piece_card_game", | |
| 154 | + "franchise": "One Piece" | |
| 155 | + }, | |
| 156 | + { | |
| 157 | + "handle": "lorcana-sealed-products", | |
| 158 | + "categorySlug": "disney_lorcana", | |
| 159 | + "franchise": "Disney Lorcana" | |
| 160 | + } | |
| 161 | + ], | |
| 162 | + "rules": [ | |
| 163 | + { | |
| 164 | + "match": "\\| mtg single \\||\\| mtg sealed \\||magic: the gathering", | |
| 165 | + "categorySlug": "magic_the_gathering", | |
| 166 | + "franchise": "Magic: The Gathering" | |
| 167 | + }, | |
| 168 | + { | |
| 169 | + "match": "final fantasy (tcg|trading card)|\\| ff-?tcg|\\bopus\\b", | |
| 170 | + "categorySlug": "final_fantasy_tcg", | |
| 171 | + "franchise": "Final Fantasy" | |
| 172 | + } | |
| 173 | + ], | |
| 174 | + "defaultCategory": null, | |
| 175 | + "exclude": "gift card|\\bsleeves?\\b|deck box|deck case|deck holder|binder|playmat|play mat|toploader|top loader|storage box|card case|portfolio|\\balbum\\b|\\bdice\\b|spindown|life counter|tokens? only|bundle of|mystery (box|pack|bundle|bag)|repack|\\bticket|event entry|entry fee|weekly league|display case|acrylic (stand|display|case)|card holder|card protector|magnetic holder|semi-rigid|perfect fit|penny sleeve|team bag|card saver|\\baccessor|dice set|d20|d6\\b|\\bpaint|brush|primer|glue|hobby tool|cutter|tweezers|sticker|keyring|key ring|lanyard|badge|coaster|mug\\b|t-shirt|hoodie|\\bcap\\b|socks|towel|blanket|poster|wall scroll|calendar|advent|mystery pack|1 card mystery|\\btoken\\b|art series|art card|singapore shipping only", | |
| 176 | + "keepOutOfStock": true, | |
| 177 | + "titlePattern": "^(?<name>.+?)(?:\\s*\\((?<number>\\d+/\\d+)\\))?\\s*\\[(?<set>[^\\]]+)\\]\\s*$", | |
| 178 | + "pageSize": 250, | |
| 179 | + "fetchBarcodes": false, | |
| 180 | + "wholeShop": false | |
| 181 | + } | |
| 182 | +} | |
added
connectors/api/footdistrict/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# FOOTDISTRICT connector (`footdistrict`) | |
| 2 | + | |
| 3 | +- Source: https://footdistrict.com · Spanish authorised sneaker & streetwear retailer (Almería): Nike/Jordan/adidas/ASICS/New Balance releases with EU sizes and style-code SKUs (e.g. IQ6573-100), plus Medicom Be@rbrick art toys. Spanish-language product types ("Calzado"). | |
| 4 | +- Country/currency: ES / EUR · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (6): `sneakers` → sneakers, `sneakers-jordan` → nike_jordan, `sneakers-nike-dunk` → nike_jordan, `sneakers-adidas-samba` → adidas_yeezy, `sneakers-new-balance-made-in-usa` → new_balance_asics_other, `art-toys` → designer_toys. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/footdistrict/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts footdistrict`. | |
added
connectors/api/footdistrict/index.test.ts
+63 −0
@@ -0,0 +1,63 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('footdistrict', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Air Jordan 7 Retro \"Miró\"", | |
| 18 | + "collection": "sneakers-jordan", | |
| 19 | + "type": "Calzado", | |
| 20 | + "vendor": "Jordan", | |
| 21 | + "variant": "42", | |
| 22 | + "expect": "nike_jordan", | |
| 23 | + "brand": "Jordan" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "title": "Nike Dunk Low Retro Premium \"Corduroy\"", | |
| 27 | + "collection": "sneakers", | |
| 28 | + "type": "Calzado", | |
| 29 | + "vendor": "Nike", | |
| 30 | + "expect": "nike_jordan", | |
| 31 | + "brand": "Nike" | |
| 32 | + }, | |
| 33 | + { | |
| 34 | + "title": "Samba OG \"Cloud White\"", | |
| 35 | + "collection": "sneakers-adidas-samba", | |
| 36 | + "type": "Calzado", | |
| 37 | + "vendor": "adidas", | |
| 38 | + "expect": "adidas_yeezy", | |
| 39 | + "brand": "adidas" | |
| 40 | + }, | |
| 41 | + { | |
| 42 | + "title": "Gel-Kayano 14 \"Cream\"", | |
| 43 | + "collection": "sneakers", | |
| 44 | + "type": "Calzado", | |
| 45 | + "vendor": "ASICS", | |
| 46 | + "expect": "sneakers" | |
| 47 | + }, | |
| 48 | + { | |
| 49 | + "title": "Medicom Toy Be@rbrick CANOTWAIT William Chan 400%", | |
| 50 | + "collection": "art-toys", | |
| 51 | + "type": "Lifestyle", | |
| 52 | + "vendor": "Medicom Toy", | |
| 53 | + "expect": "designer_toys" | |
| 54 | + }, | |
| 55 | + { | |
| 56 | + "title": "Gorra Nike ACG Club Unisex", | |
| 57 | + "collection": "sneakers", | |
| 58 | + "type": "Accesorios", | |
| 59 | + "expect": null | |
| 60 | + } | |
| 61 | + ]); | |
| 62 | + }); | |
| 63 | +}); | |
added
connectors/api/footdistrict/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Footdistrict — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/footdistrict/meta.json
+116 −0
@@ -0,0 +1,116 @@ | ||
| 1 | +{ | |
| 2 | + "id": "footdistrict", | |
| 3 | + "displayName": "FOOTDISTRICT", | |
| 4 | + "sourceId": "footdistrict", | |
| 5 | + "sourceName": "FOOTDISTRICT", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://footdistrict.com", | |
| 8 | + "module": "api/footdistrict", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "sneakers", | |
| 14 | + "nike_jordan", | |
| 15 | + "adidas_yeezy", | |
| 16 | + "new_balance_asics_other", | |
| 17 | + "designer_toys" | |
| 18 | + ], | |
| 19 | + "regions": [ | |
| 20 | + "ES" | |
| 21 | + ], | |
| 22 | + "country": "ES", | |
| 23 | + "languages": [ | |
| 24 | + "es", | |
| 25 | + "en" | |
| 26 | + ], | |
| 27 | + "currency": [ | |
| 28 | + "EUR" | |
| 29 | + ], | |
| 30 | + "supportsListings": true, | |
| 31 | + "supportsSold": false, | |
| 32 | + "supportsAuctions": false, | |
| 33 | + "supportsImages": true, | |
| 34 | + "supportsCatalog": false, | |
| 35 | + "supportsPopulation": false, | |
| 36 | + "supportsLookup": true, | |
| 37 | + "refreshFrequencyMinutes": 720, | |
| 38 | + "priority": "medium", | |
| 39 | + "trustScore": 0.7, | |
| 40 | + "attributionRequired": true, | |
| 41 | + "termsUrl": "https://footdistrict.com/policies/terms-of-service", | |
| 42 | + "accessNotes": "FOOTDISTRICT (footdistrict.com) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (sneakers, sneakers-jordan, sneakers-nike-dunk, sneakers-adidas-samba, sneakers-new-balance-made-in-usa, art-toys). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency EUR (/meta.json currency EUR, Shopify.currency rate 1.0), Spanish VAT included. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 43 | + "enabled": true, | |
| 44 | + "schemaVersion": "1.0", | |
| 45 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 46 | + "historicalDepth": "none", | |
| 47 | + "requires": [], | |
| 48 | + "config": { | |
| 49 | + "currency": "EUR", | |
| 50 | + "seller": "FOOTDISTRICT", | |
| 51 | + "location": null, | |
| 52 | + "collections": [ | |
| 53 | + { | |
| 54 | + "handle": "sneakers", | |
| 55 | + "categorySlug": "sneakers" | |
| 56 | + }, | |
| 57 | + { | |
| 58 | + "handle": "sneakers-jordan", | |
| 59 | + "categorySlug": "nike_jordan", | |
| 60 | + "brand": "Jordan" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "sneakers-nike-dunk", | |
| 64 | + "categorySlug": "nike_jordan", | |
| 65 | + "brand": "Nike" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "handle": "sneakers-adidas-samba", | |
| 69 | + "categorySlug": "adidas_yeezy", | |
| 70 | + "brand": "adidas" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "sneakers-new-balance-made-in-usa", | |
| 74 | + "categorySlug": "new_balance_asics_other", | |
| 75 | + "brand": "New Balance" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "handle": "art-toys", | |
| 79 | + "categorySlug": "designer_toys", | |
| 80 | + "brand": "Medicom Toy" | |
| 81 | + } | |
| 82 | + ], | |
| 83 | + "rules": [ | |
| 84 | + { | |
| 85 | + "match": "jordan", | |
| 86 | + "categorySlug": "nike_jordan", | |
| 87 | + "brand": "Jordan" | |
| 88 | + }, | |
| 89 | + { | |
| 90 | + "match": "\\bnike\\b|air max|air force|\\bdunk\\b|\\bsb\\b|blazer|vapormax|huarache|cortez|p-6000|vomero|pegasus|\\bshox\\b|\\bkobe\\b|lebron|\\bnocta\\b|\\bacg\\b|sacai|off-white|travis scott", | |
| 91 | + "categorySlug": "nike_jordan", | |
| 92 | + "brand": "Nike" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "match": "yeezy", | |
| 96 | + "categorySlug": "adidas_yeezy", | |
| 97 | + "brand": "adidas Yeezy" | |
| 98 | + }, | |
| 99 | + { | |
| 100 | + "match": "adidas|\\bsamba\\b|gazelle|spezial|superstar|\\bcampus\\b|\\bforum\\b|ultraboost|\\bnmd\\b|stan smith|adizero|\\by-3\\b|climacool|\\bzx\\b|\\beqt\\b|\\bsl ?72\\b|adistar|megaride|taekwondo|wales bonner", | |
| 101 | + "categorySlug": "adidas_yeezy", | |
| 102 | + "brand": "adidas" | |
| 103 | + }, | |
| 104 | + { | |
| 105 | + "match": "new balance|asics|puma|reebok|saucony|salomon|converse|\\bvans\\b|\\bhoka\\b|karhu|mizuno|autry|diadora|\\bveja\\b|clarks|timberland|dr\\.? ?martens|\\bugg\\b|mallet|cleens|umbro|le coq|kangaroos|\\bfila\\b|k-swiss|lacoste|golden goose|balenciaga|\\bdior\\b|gucci|louis vuitton|prada|amiri|herm[eè]s|rick owens|margiela|mcqueen|common projects|axel arigato|represent|loewe|bottega|valentino|givenchy|fendi|burberry|chanel|celine|saint laurent|versace|moncler|birkenstock|\\bon\\b (cloud|running)|cloudmonster|cloudtilt|merrell|keen\\b|ewing|ellesse|kappa|hi-tec|etnies|\\bdc shoes|gola|onitsuka|norda|novesta|stepney|sunnei|camper|mschf|maison mihara|\\bbape\\b|\\bsta\\b", | |
| 106 | + "categorySlug": "new_balance_asics_other" | |
| 107 | + } | |
| 108 | + ], | |
| 109 | + "defaultCategory": null, | |
| 110 | + "exclude": "t-shirt|\\btee\\b|\\btees\\b|hoodie|sweatshirt|crewneck|jacket|\\bcoat\\b|tracksuit|track pants|track jacket|sweatpants|joggers|shorts|jeans|trousers|\\bpants\\b|cargo|\\bcap\\b|\\bhat\\b|beanie|balaclava|socks|\\bbag\\b|backpack|tote|wallet|card holder|\\bbelt\\b|sunglasses|fragrance|perfume|cologne|keyring|lanyard|cleaner|crep protect|protector spray|laces|insole|shoe tree|gift card|jersey|\\bpolo\\b|\\bshirt\\b|knitwear|cardigan|\\bvest\\b|gilet|puffer|scarf|gloves|jewellery|jewelry|\\bchain\\b|bracelet|necklace|\\bring\\b|umbrella|towel|doormat|\\brug\\b|candle|\\bmug\\b|phone case|airpods|dress\\b|skirt|leggings|bodysuit|swim|bikini|underwear|boxers|slippers|flip[- ]?flops?|sandals?\\b|\\bslides?\\b|\\bmules?\\b|crocs|clog|camiseta|sudadera|chaqueta|abrigo|pantal|gorra|calcet|bolso|cartera|cintur|mochila|sandalia|chancla|zapatilla de casa|\\bropa\\b|riñonera|bufanda|guantes|vestido|falda|plumas|chaleco|cardigan|polo\\b|camisa|jersey|\\| ropa|\\| accesorios", | |
| 111 | + "keepOutOfStock": true, | |
| 112 | + "pageSize": 250, | |
| 113 | + "fetchBarcodes": false, | |
| 114 | + "wholeShop": false | |
| 115 | + } | |
| 116 | +} | |
added
connectors/api/fusion-gaming/README.md
+33 −0
@@ -0,0 +1,33 @@ | ||
| 1 | +# Fusion Gaming connector (`fusion-gaming`) | |
| 2 | + | |
| 3 | +- Source: https://www.fusiongamingonline.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Winnipeg TCG retailer. Shopify storefront; singles titles are 'Name [Set]' with condition variants. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `mtg-singles` | `magic_the_gathering` | Magic: The Gathering | | |
| 14 | +| `mtg-sealed-products` | `magic_the_gathering` | Magic: The Gathering | | |
| 15 | +| `pokemon-singles` | `pokemon` | Pokémon | | |
| 16 | +| `pokemon-all-sealed` | `pokemon` | Pokémon | | |
| 17 | +| `star-wars-unlimited-singles` | `star_wars_tcg` | Star Wars Unlimited | | |
| 18 | +| `lorcana-singles-1` | `disney_lorcana` | Disney Lorcana | | |
| 19 | +| `one-piece-singles` | `one_piece_card_game` | One Piece | | |
| 20 | +| `riftbound-singles-in-stock` | `other_tcg` | Riftbound | | |
| 21 | +| `riftbound-sealed-product` | `other_tcg` | Riftbound | | |
| 22 | +| `board-games-all` | `board_games` | — | | |
| 23 | + | |
| 24 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live` | |
| 25 | + | |
| 26 | +Title pattern: `^(?<name>.+?) \[(?<set>[^\]]+)\]$` (name / set / number) | |
| 27 | + | |
| 28 | +## Access & compliance | |
| 29 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: accessories/playmats/sleeves/dice collections, event tickets, "items-with-noprice" collection. | |
| 30 | + | |
| 31 | +## Fixtures & tests | |
| 32 | +`data/fixtures/fusion-gaming/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts fusion-gaming`), trimmed single-product payloads incl. a sold-out variant. | |
| 33 | +`pnpm vitest run connectors/api/fusion-gaming` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts fusion-gaming`. | |
added
connectors/api/fusion-gaming/index.test.ts
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Fusion Gaming — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/fusion-gaming/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "Ephemerate [Modern Horizons]", | |
| 15 | + "productType": "MTG Single", | |
| 16 | + "collection": "mtg-singles", | |
| 17 | + "categorySlug": "magic_the_gathering" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "Ultra Pro Playmat - Black", | |
| 21 | + "collection": "mtg-sealed-products", | |
| 22 | + "categorySlug": null | |
| 23 | + } | |
| 24 | + ] | |
| 25 | +}); | |
added
connectors/api/fusion-gaming/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Fusion Gaming — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/fusion-gaming/meta.json
+113 −0
@@ -0,0 +1,113 @@ | ||
| 1 | +{ | |
| 2 | + "id": "fusion-gaming", | |
| 3 | + "displayName": "Fusion Gaming (Canadian TCG & board-game store, CAD)", | |
| 4 | + "sourceId": "fusion-gaming", | |
| 5 | + "sourceName": "Fusion Gaming", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.fusiongamingonline.com", | |
| 8 | + "module": "api/fusion-gaming", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "pokemon", | |
| 15 | + "star_wars_tcg", | |
| 16 | + "disney_lorcana", | |
| 17 | + "one_piece_card_game", | |
| 18 | + "other_tcg", | |
| 19 | + "board_games" | |
| 20 | + ], | |
| 21 | + "regions": [ | |
| 22 | + "CA" | |
| 23 | + ], | |
| 24 | + "languages": [ | |
| 25 | + "en" | |
| 26 | + ], | |
| 27 | + "currency": [ | |
| 28 | + "CAD" | |
| 29 | + ], | |
| 30 | + "supportsListings": true, | |
| 31 | + "supportsSold": false, | |
| 32 | + "supportsAuctions": false, | |
| 33 | + "supportsImages": true, | |
| 34 | + "supportsCatalog": false, | |
| 35 | + "supportsPopulation": false, | |
| 36 | + "supportsLookup": true, | |
| 37 | + "refreshFrequencyMinutes": 720, | |
| 38 | + "priority": "medium", | |
| 39 | + "trustScore": 0.75, | |
| 40 | + "attributionRequired": true, | |
| 41 | + "termsUrl": "https://www.fusiongamingonline.com/policies/terms-of-service", | |
| 42 | + "accessNotes": "Fusion Gaming (fusiongamingonline.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 10 configured collections (mtg-singles, mtg-sealed-products, pokemon-singles, pokemon-all-sealed, star-wars-unlimited-singles, lorcana-singles-1, one-piece-singles, riftbound-singles-in-stock … (+2 more, see config.collections)) and /products/<handle>.json for URL lookups (~256k products; MTG singles 52k, Pokémon singles 9k+). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — accessories/playmats/sleeves/dice collections, event tickets, \"items-with-noprice\" collection. No personal data is collected; seller = the store itself.", | |
| 43 | + "enabled": true, | |
| 44 | + "schemaVersion": "1.0", | |
| 45 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 46 | + "historicalDepth": "none", | |
| 47 | + "requires": [], | |
| 48 | + "config": { | |
| 49 | + "currency": "CAD", | |
| 50 | + "market": "CA", | |
| 51 | + "seller": "Fusion Gaming", | |
| 52 | + "location": "Winnipeg, MB, Canada", | |
| 53 | + "collections": [ | |
| 54 | + { | |
| 55 | + "handle": "mtg-singles", | |
| 56 | + "categorySlug": "magic_the_gathering", | |
| 57 | + "franchise": "Magic: The Gathering" | |
| 58 | + }, | |
| 59 | + { | |
| 60 | + "handle": "mtg-sealed-products", | |
| 61 | + "categorySlug": "magic_the_gathering", | |
| 62 | + "franchise": "Magic: The Gathering" | |
| 63 | + }, | |
| 64 | + { | |
| 65 | + "handle": "pokemon-singles", | |
| 66 | + "categorySlug": "pokemon", | |
| 67 | + "franchise": "Pokémon" | |
| 68 | + }, | |
| 69 | + { | |
| 70 | + "handle": "pokemon-all-sealed", | |
| 71 | + "categorySlug": "pokemon", | |
| 72 | + "franchise": "Pokémon" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "handle": "star-wars-unlimited-singles", | |
| 76 | + "categorySlug": "star_wars_tcg", | |
| 77 | + "franchise": "Star Wars Unlimited" | |
| 78 | + }, | |
| 79 | + { | |
| 80 | + "handle": "lorcana-singles-1", | |
| 81 | + "categorySlug": "disney_lorcana", | |
| 82 | + "franchise": "Disney Lorcana" | |
| 83 | + }, | |
| 84 | + { | |
| 85 | + "handle": "one-piece-singles", | |
| 86 | + "categorySlug": "one_piece_card_game", | |
| 87 | + "franchise": "One Piece" | |
| 88 | + }, | |
| 89 | + { | |
| 90 | + "handle": "riftbound-singles-in-stock", | |
| 91 | + "categorySlug": "other_tcg", | |
| 92 | + "franchise": "Riftbound" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "handle": "riftbound-sealed-product", | |
| 96 | + "categorySlug": "other_tcg", | |
| 97 | + "franchise": "Riftbound" | |
| 98 | + }, | |
| 99 | + { | |
| 100 | + "handle": "board-games-all", | |
| 101 | + "categorySlug": "board_games" | |
| 102 | + } | |
| 103 | + ], | |
| 104 | + "rules": [], | |
| 105 | + "defaultCategory": null, | |
| 106 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live", | |
| 107 | + "keepOutOfStock": true, | |
| 108 | + "fetchBarcodes": false, | |
| 109 | + "wholeShop": false, | |
| 110 | + "pageSize": 250, | |
| 111 | + "titlePattern": "^(?<name>.+?) \\[(?<set>[^\\]]+)\\]$" | |
| 112 | + } | |
| 113 | +} | |
added
connectors/api/gameology/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Gameology connector (`gameology`) | |
| 2 | + | |
| 3 | +- Source: https://www.gameology.com.au · Melbourne online game store: 22k TCG singles (MTG, Pokémon English/Japanese, One Piece, Lorcana, Yu-Gi-Oh!), PSA-graded cards, Funko Pop!, statues & anime figures, Topps/Panini sports cards. Tags carry Number_/Printing_/Set_ attributes. | |
| 4 | +- Country/currency: AU / AUD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (17): `magic-the-gathering-singles` → magic_the_gathering, `pokemon-tcg-single-cards` → pokemon, `japanese-pokemon-singles` → pokemon, `pokemon-collection` → pokemon, `one-piece-tcg-single-cards` → one_piece_card_game, `lorcana-tcg-singles` → disney_lorcana, `yugioh` → yugioh, `weiss-schwarz` → weiss_schwarz, `digimon-card-game` → digimon_tcg, `star-wars-unlimited` → star_wars_tcg, `flesh-and-blood-tcg` → flesh_and_blood, `dragon-ball-super` → dragon_ball_tcg, `graded-card` → other_tcg, `pop-vinyl-figures` → funko, `statues-figures` → action_figures, `sports-collectables` → other_sports_cards, `topps-sports-cards` → other_sports_cards. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/gameology/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts gameology`. | |
added
connectors/api/gameology/index.test.ts
+77 −0
@@ -0,0 +1,77 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('gameology', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Umbreon VMAX 215/203 - Pokemon Evolving Skies Holofoil", | |
| 18 | + "collection": "pokemon-tcg-single-cards", | |
| 19 | + "type": "Single Cards", | |
| 20 | + "tags": [ | |
| 21 | + "Number_215", | |
| 22 | + "Printing_Holofoil", | |
| 23 | + "Single Card Game: Pokemon TCG" | |
| 24 | + ], | |
| 25 | + "expect": "pokemon" | |
| 26 | + }, | |
| 27 | + { | |
| 28 | + "title": "Aang, at the Crossroads (Showcase) (TLA 346) - Avatar: The Last Airbender - Magic The Gathering", | |
| 29 | + "collection": "magic-the-gathering-singles", | |
| 30 | + "type": "Single Cards", | |
| 31 | + "expect": "magic_the_gathering" | |
| 32 | + }, | |
| 33 | + { | |
| 34 | + "title": "MONKEY D. LUFFY (ONE PIECE MAGAZINE VOL.20) #14 2025 PSA 10", | |
| 35 | + "collection": "graded-card", | |
| 36 | + "type": "Graded Cards", | |
| 37 | + "tags": [ | |
| 38 | + "Graded Card Game: One Piece", | |
| 39 | + "Graded Cards" | |
| 40 | + ], | |
| 41 | + "expect": "one_piece_card_game", | |
| 42 | + "grader": "psa", | |
| 43 | + "grade": "10" | |
| 44 | + }, | |
| 45 | + { | |
| 46 | + "title": "Pokemon Charizard Pop Vinyl", | |
| 47 | + "collection": "pop-vinyl-figures", | |
| 48 | + "type": "Statues & Figures", | |
| 49 | + "vendor": "Funko", | |
| 50 | + "expect": "funko", | |
| 51 | + "brand": "Funko" | |
| 52 | + }, | |
| 53 | + { | |
| 54 | + "title": "Dragon Ball Z Shen Long 7 Star Ball Statue with Mini Mountain", | |
| 55 | + "collection": "statues-figures", | |
| 56 | + "type": "Statues & Figures", | |
| 57 | + "expect": "action_figures" | |
| 58 | + }, | |
| 59 | + { | |
| 60 | + "title": "Topps Premier League 2026 Trading Cards Booster Box", | |
| 61 | + "collection": "topps-sports-cards", | |
| 62 | + "type": "Sports Cards", | |
| 63 | + "tags": [ | |
| 64 | + "Sport Cards Sport: Soccer/Football" | |
| 65 | + ], | |
| 66 | + "expect": "soccer_cards", | |
| 67 | + "brand": "Topps" | |
| 68 | + }, | |
| 69 | + { | |
| 70 | + "title": "Ultimate Guard Boulder Deck Case 100+ Standard Size Poppy Topaz Deck Box", | |
| 71 | + "collection": "pop-vinyl-figures", | |
| 72 | + "type": "Deck Boxes", | |
| 73 | + "expect": null | |
| 74 | + } | |
| 75 | + ]); | |
| 76 | + }); | |
| 77 | +}); | |
added
connectors/api/gameology/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Gameology — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/gameology/meta.json
+283 −0
@@ -0,0 +1,283 @@ | ||
| 1 | +{ | |
| 2 | + "id": "gameology", | |
| 3 | + "displayName": "Gameology", | |
| 4 | + "sourceId": "gameology", | |
| 5 | + "sourceName": "Gameology", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.gameology.com.au", | |
| 8 | + "module": "api/gameology", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "pokemon", | |
| 15 | + "one_piece_card_game", | |
| 16 | + "disney_lorcana", | |
| 17 | + "yugioh", | |
| 18 | + "weiss_schwarz", | |
| 19 | + "digimon_tcg", | |
| 20 | + "star_wars_tcg", | |
| 21 | + "flesh_and_blood", | |
| 22 | + "dragon_ball_tcg", | |
| 23 | + "other_tcg", | |
| 24 | + "funko", | |
| 25 | + "action_figures", | |
| 26 | + "other_sports_cards", | |
| 27 | + "basketball_cards", | |
| 28 | + "final_fantasy_tcg", | |
| 29 | + "football_cards", | |
| 30 | + "soccer_cards", | |
| 31 | + "baseball_cards", | |
| 32 | + "hockey_cards", | |
| 33 | + "f1_cards" | |
| 34 | + ], | |
| 35 | + "regions": [ | |
| 36 | + "AU" | |
| 37 | + ], | |
| 38 | + "country": "AU", | |
| 39 | + "languages": [ | |
| 40 | + "en" | |
| 41 | + ], | |
| 42 | + "currency": [ | |
| 43 | + "AUD" | |
| 44 | + ], | |
| 45 | + "supportsListings": true, | |
| 46 | + "supportsSold": false, | |
| 47 | + "supportsAuctions": false, | |
| 48 | + "supportsImages": true, | |
| 49 | + "supportsCatalog": false, | |
| 50 | + "supportsPopulation": false, | |
| 51 | + "supportsLookup": true, | |
| 52 | + "refreshFrequencyMinutes": 720, | |
| 53 | + "priority": "medium", | |
| 54 | + "trustScore": 0.65, | |
| 55 | + "attributionRequired": true, | |
| 56 | + "termsUrl": "https://www.gameology.com.au/policies/terms-of-service", | |
| 57 | + "accessNotes": "Gameology (gameology.com.au) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (magic-the-gathering-singles, pokemon-tcg-single-cards, japanese-pokemon-singles, pokemon-collection, one-piece-tcg-single-cards, lorcana-tcg-singles, yugioh, weiss-schwarz, digimon-card-game, star-wars-unlimited, flesh-and-blood-tcg, dragon-ball-super, graded-card, pop-vinyl-figures, statues-figures, sports-collectables, topps-sports-cards). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency AUD (/meta.json currency AUD, Shopify.currency rate 1.0), GST included. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 58 | + "enabled": true, | |
| 59 | + "schemaVersion": "1.0", | |
| 60 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 61 | + "historicalDepth": "none", | |
| 62 | + "requires": [], | |
| 63 | + "config": { | |
| 64 | + "currency": "AUD", | |
| 65 | + "seller": "Gameology", | |
| 66 | + "location": null, | |
| 67 | + "collections": [ | |
| 68 | + { | |
| 69 | + "handle": "magic-the-gathering-singles", | |
| 70 | + "categorySlug": "magic_the_gathering", | |
| 71 | + "franchise": "Magic: The Gathering" | |
| 72 | + }, | |
| 73 | + { | |
| 74 | + "handle": "pokemon-tcg-single-cards", | |
| 75 | + "categorySlug": "pokemon", | |
| 76 | + "franchise": "Pokémon" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "japanese-pokemon-singles", | |
| 80 | + "categorySlug": "pokemon", | |
| 81 | + "franchise": "Pokémon" | |
| 82 | + }, | |
| 83 | + { | |
| 84 | + "handle": "pokemon-collection", | |
| 85 | + "categorySlug": "pokemon", | |
| 86 | + "franchise": "Pokémon" | |
| 87 | + }, | |
| 88 | + { | |
| 89 | + "handle": "one-piece-tcg-single-cards", | |
| 90 | + "categorySlug": "one_piece_card_game", | |
| 91 | + "franchise": "One Piece" | |
| 92 | + }, | |
| 93 | + { | |
| 94 | + "handle": "lorcana-tcg-singles", | |
| 95 | + "categorySlug": "disney_lorcana", | |
| 96 | + "franchise": "Disney Lorcana" | |
| 97 | + }, | |
| 98 | + { | |
| 99 | + "handle": "yugioh", | |
| 100 | + "categorySlug": "yugioh", | |
| 101 | + "franchise": "Yu-Gi-Oh!" | |
| 102 | + }, | |
| 103 | + { | |
| 104 | + "handle": "weiss-schwarz", | |
| 105 | + "categorySlug": "weiss_schwarz" | |
| 106 | + }, | |
| 107 | + { | |
| 108 | + "handle": "digimon-card-game", | |
| 109 | + "categorySlug": "digimon_tcg", | |
| 110 | + "franchise": "Digimon" | |
| 111 | + }, | |
| 112 | + { | |
| 113 | + "handle": "star-wars-unlimited", | |
| 114 | + "categorySlug": "star_wars_tcg", | |
| 115 | + "franchise": "Star Wars" | |
| 116 | + }, | |
| 117 | + { | |
| 118 | + "handle": "flesh-and-blood-tcg", | |
| 119 | + "categorySlug": "flesh_and_blood", | |
| 120 | + "franchise": "Flesh and Blood" | |
| 121 | + }, | |
| 122 | + { | |
| 123 | + "handle": "dragon-ball-super", | |
| 124 | + "categorySlug": "dragon_ball_tcg", | |
| 125 | + "franchise": "Dragon Ball" | |
| 126 | + }, | |
| 127 | + { | |
| 128 | + "handle": "graded-card", | |
| 129 | + "categorySlug": "other_tcg" | |
| 130 | + }, | |
| 131 | + { | |
| 132 | + "handle": "pop-vinyl-figures", | |
| 133 | + "categorySlug": "funko", | |
| 134 | + "brand": "Funko" | |
| 135 | + }, | |
| 136 | + { | |
| 137 | + "handle": "statues-figures", | |
| 138 | + "categorySlug": "action_figures" | |
| 139 | + }, | |
| 140 | + { | |
| 141 | + "handle": "sports-collectables", | |
| 142 | + "categorySlug": "other_sports_cards" | |
| 143 | + }, | |
| 144 | + { | |
| 145 | + "handle": "topps-sports-cards", | |
| 146 | + "categorySlug": "other_sports_cards", | |
| 147 | + "brand": "Topps" | |
| 148 | + } | |
| 149 | + ], | |
| 150 | + "rules": [ | |
| 151 | + { | |
| 152 | + "match": "graded card game: one piece", | |
| 153 | + "categorySlug": "one_piece_card_game", | |
| 154 | + "franchise": "One Piece" | |
| 155 | + }, | |
| 156 | + { | |
| 157 | + "match": "graded card game: (pokemon|japanese pokemon)", | |
| 158 | + "categorySlug": "pokemon", | |
| 159 | + "franchise": "Pokémon" | |
| 160 | + }, | |
| 161 | + { | |
| 162 | + "match": "graded card game: (magic|mtg)", | |
| 163 | + "categorySlug": "magic_the_gathering", | |
| 164 | + "franchise": "Magic: The Gathering" | |
| 165 | + }, | |
| 166 | + { | |
| 167 | + "match": "graded card game: yu-?gi-?oh", | |
| 168 | + "categorySlug": "yugioh", | |
| 169 | + "franchise": "Yu-Gi-Oh!" | |
| 170 | + }, | |
| 171 | + { | |
| 172 | + "match": "graded card game: lorcana", | |
| 173 | + "categorySlug": "disney_lorcana", | |
| 174 | + "franchise": "Disney Lorcana" | |
| 175 | + }, | |
| 176 | + { | |
| 177 | + "match": "graded card game: (nba|basketball)", | |
| 178 | + "categorySlug": "basketball_cards" | |
| 179 | + }, | |
| 180 | + { | |
| 181 | + "match": "\\| graded cards \\|", | |
| 182 | + "categorySlug": "other_tcg" | |
| 183 | + }, | |
| 184 | + { | |
| 185 | + "match": "funko|pop! ?vinyl|pop vinyl|\\bpop!\\b|pop! games|pop! animation|pop! movies", | |
| 186 | + "categorySlug": "funko", | |
| 187 | + "brand": "Funko" | |
| 188 | + }, | |
| 189 | + { | |
| 190 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:star wars:? unlimited))", | |
| 191 | + "categorySlug": "star_wars_tcg", | |
| 192 | + "franchise": "Star Wars" | |
| 193 | + }, | |
| 194 | + { | |
| 195 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:lorcana))", | |
| 196 | + "categorySlug": "disney_lorcana", | |
| 197 | + "franchise": "Disney Lorcana" | |
| 198 | + }, | |
| 199 | + { | |
| 200 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:one piece (card|tcg|ccg|single|promo|sealed|booster|starter)|\\bop\\d{2}\\b|one-piece-(single|tcg|card)|\\| one piece single))", | |
| 201 | + "categorySlug": "one_piece_card_game", | |
| 202 | + "franchise": "One Piece" | |
| 203 | + }, | |
| 204 | + { | |
| 205 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:dragon ?ball (super |z )?(card|tcg|ccg|fusion world)|fusion world|dragonball-super|dragon-ball-super))", | |
| 206 | + "categorySlug": "dragon_ball_tcg", | |
| 207 | + "franchise": "Dragon Ball" | |
| 208 | + }, | |
| 209 | + { | |
| 210 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:digimon))", | |
| 211 | + "categorySlug": "digimon_tcg", | |
| 212 | + "franchise": "Digimon" | |
| 213 | + }, | |
| 214 | + { | |
| 215 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:flesh (and|&) blood|\\bfab\\b))", | |
| 216 | + "categorySlug": "flesh_and_blood", | |
| 217 | + "franchise": "Flesh and Blood" | |
| 218 | + }, | |
| 219 | + { | |
| 220 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:wei(ss|ß) schwarz))", | |
| 221 | + "categorySlug": "weiss_schwarz" | |
| 222 | + }, | |
| 223 | + { | |
| 224 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:final fantasy (tcg|trading card|opus)|\\bopus (i{1,3}|iv|v|vi{0,3}|ix|x{1,2}|xi{1,3}|xiv|xv)\\b|\\bfftcg\\b))", | |
| 225 | + "categorySlug": "final_fantasy_tcg", | |
| 226 | + "franchise": "Final Fantasy" | |
| 227 | + }, | |
| 228 | + { | |
| 229 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:yu-?gi-?oh|\\bygo\\b|yugioh))", | |
| 230 | + "categorySlug": "yugioh", | |
| 231 | + "franchise": "Yu-Gi-Oh!" | |
| 232 | + }, | |
| 233 | + { | |
| 234 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:pok[eé]mon|\\bpkm\\b))", | |
| 235 | + "categorySlug": "pokemon", | |
| 236 | + "franchise": "Pokémon" | |
| 237 | + }, | |
| 238 | + { | |
| 239 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:magic:? the gathering|\\bmtg\\b|magic single|secret lair|\\bcommander\\b|universes beyond|planeswalker))", | |
| 240 | + "categorySlug": "magic_the_gathering", | |
| 241 | + "franchise": "Magic: The Gathering" | |
| 242 | + }, | |
| 243 | + { | |
| 244 | + "match": "^(?!.*\\| statues & figures \\|)(?=.*(?:cardfight|vanguard|battle spirits|union arena|riftbound|grand archive|gundam (card|tcg)|hololive (official )?card|shadowverse|sorcery:? contested|\\baltered\\b|elestrals|alpha clash|universus|my hero academia (ccg|tcg|single)|akora|kryptik|cyberpunk tcg|neuroscape|beyblade x (tcg|card)))", | |
| 245 | + "categorySlug": "other_tcg" | |
| 246 | + }, | |
| 247 | + { | |
| 248 | + "match": "\\bnba\\b|basketball|wnba", | |
| 249 | + "categorySlug": "basketball_cards" | |
| 250 | + }, | |
| 251 | + { | |
| 252 | + "match": "\\bnfl\\b|american football", | |
| 253 | + "categorySlug": "football_cards" | |
| 254 | + }, | |
| 255 | + { | |
| 256 | + "match": "soccer|premier league|uefa|champions league|world cup|\\bepl\\b|bundesliga|la liga|serie a|match attax|\\bfifa\\b|road to (world cup|euro)", | |
| 257 | + "categorySlug": "soccer_cards" | |
| 258 | + }, | |
| 259 | + { | |
| 260 | + "match": "\\bmlb\\b|baseball", | |
| 261 | + "categorySlug": "baseball_cards" | |
| 262 | + }, | |
| 263 | + { | |
| 264 | + "match": "\\bnhl\\b|hockey", | |
| 265 | + "categorySlug": "hockey_cards" | |
| 266 | + }, | |
| 267 | + { | |
| 268 | + "match": "\\bf1\\b|formula (1|one)|grand prix", | |
| 269 | + "categorySlug": "f1_cards" | |
| 270 | + }, | |
| 271 | + { | |
| 272 | + "match": "\\bufc\\b|wrestling|\\bwwe\\b|\\baew\\b|\\bafl\\b|\\bnrl\\b|tennis|\\bgolf\\b|cricket|rugby|boxing|nascar|racing|olympic", | |
| 273 | + "categorySlug": "other_sports_cards" | |
| 274 | + } | |
| 275 | + ], | |
| 276 | + "defaultCategory": null, | |
| 277 | + "exclude": "gift card|\\bsleeves?\\b|deck box|deck case|deck holder|binder|playmat|play mat|toploader|top loader|storage box|card case|portfolio|\\balbum\\b|\\bdice\\b|spindown|life counter|tokens? only|bundle of|mystery (box|pack|bundle|bag)|repack|\\bticket|event entry|entry fee|weekly league|display case|acrylic (stand|display|case)|card holder|card protector|magnetic holder|semi-rigid|perfect fit|penny sleeve|team bag|card saver|\\baccessor|dice set|d20|d6\\b|\\bpaint|brush|primer|glue|hobby tool|cutter|tweezers|sticker|keyring|key ring|lanyard|badge|coaster|mug\\b|t-shirt|hoodie|\\bcap\\b|socks|towel|blanket|poster|wall scroll|calendar|advent|deck case|puzzle|jigsaw|board game|crystal ball|\\| deck boxes \\||\\| board games \\|", | |
| 278 | + "keepOutOfStock": true, | |
| 279 | + "pageSize": 250, | |
| 280 | + "fetchBarcodes": false, | |
| 281 | + "wholeShop": false | |
| 282 | + } | |
| 283 | +} | |
added
connectors/api/gamesportal/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Games Portal connector (`gamesportal`) | |
| 2 | + | |
| 3 | +- Source: https://gamesportal.com.au · Mitcham (Victoria) game store on BinderPOS: ~47k in-stock MTG singles plus One Piece, Lorcana, Digimon and Flesh and Blood singles with "Name [Set]" titles and Near Mint / Lightly Played / Moderately Played / Heavily Played (Foil) condition variants; sealed MTG, Lorcana, One Piece, Star Wars Unlimited, Gundam and hololive products. | |
| 4 | +- Country/currency: AU / AUD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, a `titlePattern` that extracts name/set/number, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (12): `magic-singles-instock` → magic_the_gathering, `sealed-magic-products` → magic_the_gathering, `one-piece-ccg-singles` → one_piece_card_game, `one-piece-card-game` → one_piece_card_game, `disney-lorcana-singles` → disney_lorcana, `disney-lorcana-sealed` → disney_lorcana, `digimon-singles` → digimon_tcg, `flesh-and-blood-singles` → flesh_and_blood, `star-wars-unlimited-sealed` → star_wars_tcg, `dragon-ball-super-fusion-world-sealed` → dragon_ball_tcg, `gundam-card-game-sealed` → other_tcg, `hololive-card-game-sealed` → other_tcg. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/gamesportal/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts gamesportal`. | |
added
connectors/api/gamesportal/index.test.ts
+63 −0
@@ -0,0 +1,63 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('gamesportal', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Kaya's Ghostform [War of the Spark]", | |
| 18 | + "collection": "magic-singles-instock", | |
| 19 | + "type": "MTG Single", | |
| 20 | + "variant": "Near Mint", | |
| 21 | + "expect": "magic_the_gathering", | |
| 22 | + "name": "Kaya's Ghostform", | |
| 23 | + "set": "War of the Spark", | |
| 24 | + "number": null, | |
| 25 | + "conditionRaw": "Near Mint" | |
| 26 | + }, | |
| 27 | + { | |
| 28 | + "title": "Belle - Snowfield Strategist (Enchanted) (236/204) [Winterspell]", | |
| 29 | + "collection": "disney-lorcana-singles", | |
| 30 | + "type": "Lorcana Single", | |
| 31 | + "variant": "Near Mint Holofoil", | |
| 32 | + "expect": "disney_lorcana", | |
| 33 | + "name": "Belle - Snowfield Strategist (Enchanted)", | |
| 34 | + "set": "Winterspell", | |
| 35 | + "number": "236/204" | |
| 36 | + }, | |
| 37 | + { | |
| 38 | + "title": "Roronoa Zoro (OP-06 Pre-Release Tournament) [Winner] [One Piece Promotion Cards]", | |
| 39 | + "collection": "one-piece-ccg-singles", | |
| 40 | + "type": "One Piece Single", | |
| 41 | + "variant": "Lightly Played Foil", | |
| 42 | + "expect": "one_piece_card_game", | |
| 43 | + "conditionRaw": "Lightly Played" | |
| 44 | + }, | |
| 45 | + { | |
| 46 | + "title": "Ultimate Masters Booster Box", | |
| 47 | + "collection": "sealed-magic-products", | |
| 48 | + "type": "MTG Sealed", | |
| 49 | + "tags": [ | |
| 50 | + "Booster Box", | |
| 51 | + "Magic the Gathering" | |
| 52 | + ], | |
| 53 | + "expect": "magic_the_gathering" | |
| 54 | + }, | |
| 55 | + { | |
| 56 | + "title": "Magic Lorwyn Eclipsed Prerelease Ticket - Saturday 10am", | |
| 57 | + "collection": "sealed-magic-products", | |
| 58 | + "type": "Event", | |
| 59 | + "expect": null | |
| 60 | + } | |
| 61 | + ]); | |
| 62 | + }); | |
| 63 | +}); | |
added
connectors/api/gamesportal/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Games Portal — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/gamesportal/meta.json
+124 −0
@@ -0,0 +1,124 @@ | ||
| 1 | +{ | |
| 2 | + "id": "gamesportal", | |
| 3 | + "displayName": "Games Portal", | |
| 4 | + "sourceId": "gamesportal", | |
| 5 | + "sourceName": "Games Portal", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://gamesportal.com.au", | |
| 8 | + "module": "api/gamesportal", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "one_piece_card_game", | |
| 15 | + "disney_lorcana", | |
| 16 | + "digimon_tcg", | |
| 17 | + "flesh_and_blood", | |
| 18 | + "star_wars_tcg", | |
| 19 | + "dragon_ball_tcg", | |
| 20 | + "other_tcg" | |
| 21 | + ], | |
| 22 | + "regions": [ | |
| 23 | + "AU" | |
| 24 | + ], | |
| 25 | + "country": "AU", | |
| 26 | + "languages": [ | |
| 27 | + "en" | |
| 28 | + ], | |
| 29 | + "currency": [ | |
| 30 | + "AUD" | |
| 31 | + ], | |
| 32 | + "supportsListings": true, | |
| 33 | + "supportsSold": false, | |
| 34 | + "supportsAuctions": false, | |
| 35 | + "supportsImages": true, | |
| 36 | + "supportsCatalog": false, | |
| 37 | + "supportsPopulation": false, | |
| 38 | + "supportsLookup": true, | |
| 39 | + "refreshFrequencyMinutes": 720, | |
| 40 | + "priority": "medium", | |
| 41 | + "trustScore": 0.65, | |
| 42 | + "attributionRequired": true, | |
| 43 | + "termsUrl": "https://gamesportal.com.au/policies/terms-of-service", | |
| 44 | + "accessNotes": "Games Portal (gamesportal.com.au) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (magic-singles-instock, sealed-magic-products, one-piece-ccg-singles, one-piece-card-game, disney-lorcana-singles, disney-lorcana-sealed, digimon-singles, flesh-and-blood-singles, star-wars-unlimited-sealed, dragon-ball-super-fusion-world-sealed, gundam-card-game-sealed, hololive-card-game-sealed). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency AUD (/meta.json currency AUD, Shopify.currency rate 1.0), GST included. SKUs follow BinderPOS \"SET-number-lang-finish-condition\" (e.g. WAR-94-EN-NF-1). Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 45 | + "enabled": true, | |
| 46 | + "schemaVersion": "1.0", | |
| 47 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 48 | + "historicalDepth": "none", | |
| 49 | + "requires": [], | |
| 50 | + "config": { | |
| 51 | + "currency": "AUD", | |
| 52 | + "seller": "Games Portal", | |
| 53 | + "location": null, | |
| 54 | + "collections": [ | |
| 55 | + { | |
| 56 | + "handle": "magic-singles-instock", | |
| 57 | + "categorySlug": "magic_the_gathering", | |
| 58 | + "franchise": "Magic: The Gathering" | |
| 59 | + }, | |
| 60 | + { | |
| 61 | + "handle": "sealed-magic-products", | |
| 62 | + "categorySlug": "magic_the_gathering", | |
| 63 | + "franchise": "Magic: The Gathering" | |
| 64 | + }, | |
| 65 | + { | |
| 66 | + "handle": "one-piece-ccg-singles", | |
| 67 | + "categorySlug": "one_piece_card_game", | |
| 68 | + "franchise": "One Piece" | |
| 69 | + }, | |
| 70 | + { | |
| 71 | + "handle": "one-piece-card-game", | |
| 72 | + "categorySlug": "one_piece_card_game", | |
| 73 | + "franchise": "One Piece" | |
| 74 | + }, | |
| 75 | + { | |
| 76 | + "handle": "disney-lorcana-singles", | |
| 77 | + "categorySlug": "disney_lorcana", | |
| 78 | + "franchise": "Disney Lorcana" | |
| 79 | + }, | |
| 80 | + { | |
| 81 | + "handle": "disney-lorcana-sealed", | |
| 82 | + "categorySlug": "disney_lorcana", | |
| 83 | + "franchise": "Disney Lorcana" | |
| 84 | + }, | |
| 85 | + { | |
| 86 | + "handle": "digimon-singles", | |
| 87 | + "categorySlug": "digimon_tcg", | |
| 88 | + "franchise": "Digimon" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "handle": "flesh-and-blood-singles", | |
| 92 | + "categorySlug": "flesh_and_blood", | |
| 93 | + "franchise": "Flesh and Blood" | |
| 94 | + }, | |
| 95 | + { | |
| 96 | + "handle": "star-wars-unlimited-sealed", | |
| 97 | + "categorySlug": "star_wars_tcg", | |
| 98 | + "franchise": "Star Wars" | |
| 99 | + }, | |
| 100 | + { | |
| 101 | + "handle": "dragon-ball-super-fusion-world-sealed", | |
| 102 | + "categorySlug": "dragon_ball_tcg", | |
| 103 | + "franchise": "Dragon Ball" | |
| 104 | + }, | |
| 105 | + { | |
| 106 | + "handle": "gundam-card-game-sealed", | |
| 107 | + "categorySlug": "other_tcg", | |
| 108 | + "franchise": "Gundam" | |
| 109 | + }, | |
| 110 | + { | |
| 111 | + "handle": "hololive-card-game-sealed", | |
| 112 | + "categorySlug": "other_tcg" | |
| 113 | + } | |
| 114 | + ], | |
| 115 | + "rules": [], | |
| 116 | + "defaultCategory": null, | |
| 117 | + "exclude": "gift card|\\bsleeves?\\b|deck box|deck case|deck holder|binder|playmat|play mat|toploader|top loader|storage box|card case|portfolio|\\balbum\\b|\\bdice\\b|spindown|life counter|tokens? only|bundle of|mystery (box|pack|bundle|bag)|repack|\\bticket|event entry|entry fee|weekly league|display case|acrylic (stand|display|case)|card holder|card protector|magnetic holder|semi-rigid|perfect fit|penny sleeve|team bag|card saver|\\baccessor|dice set|d20|d6\\b|\\bpaint|brush|primer|glue|hobby tool|cutter|tweezers|sticker|keyring|key ring|lanyard|badge|coaster|mug\\b|t-shirt|hoodie|\\bcap\\b|socks|towel|blanket|poster|wall scroll|calendar|advent|prerelease ticket|event ticket|\\| accessories \\||ultimate guard|gamegenic|dragon shield", | |
| 118 | + "keepOutOfStock": true, | |
| 119 | + "titlePattern": "^(?<name>.+?)(?:\\s*\\((?<number>\\d+/\\d+)\\))?\\s*\\[(?<set>[^\\]]+)\\]\\s*$", | |
| 120 | + "pageSize": 250, | |
| 121 | + "fetchBarcodes": false, | |
| 122 | + "wholeShop": false | |
| 123 | + } | |
| 124 | +} | |
added
connectors/api/gamezilla/README.md
+49 −0
@@ -0,0 +1,49 @@ | ||
| 1 | +# GameZilla connector (`gamezilla`) | |
| 2 | + | |
| 3 | +- Source: https://www.gamezilla.ca · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +New Brunswick TCG/anime shop chain. Shopify storefront with per-set MTG collections ('Name [Set]', store-location variants), Gunpla grades, anime figures/statues, Lorcana. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `mtg-commander-masters` | `magic_the_gathering` | Magic: The Gathering | | |
| 14 | +| `mtg-commander-legends-battle-for-baldurs-gat` | `magic_the_gathering` | Magic: The Gathering | | |
| 15 | +| `mtg-avatar-the-last-airbender` | `magic_the_gathering` | Magic: The Gathering | | |
| 16 | +| `mtg-foundations` | `magic_the_gathering` | Magic: The Gathering | | |
| 17 | +| `mtg-final-fantasy-commander` | `magic_the_gathering` | Magic: The Gathering | | |
| 18 | +| `mtg-commander-legends` | `magic_the_gathering` | Magic: The Gathering | | |
| 19 | +| `mtg-modern-horizons-3-commander` | `magic_the_gathering` | Magic: The Gathering | | |
| 20 | +| `mtg-edge-of-eternities` | `magic_the_gathering` | Magic: The Gathering | | |
| 21 | +| `mtg-aetherdrift` | `magic_the_gathering` | Magic: The Gathering | | |
| 22 | +| `mtg-final-fantasy` | `magic_the_gathering` | Magic: The Gathering | | |
| 23 | +| `mtg-dominaria-united` | `magic_the_gathering` | Magic: The Gathering | | |
| 24 | +| `mtg-double-masters-2022` | `magic_the_gathering` | Magic: The Gathering | | |
| 25 | +| `mtg-bloomburrow` | `magic_the_gathering` | Magic: The Gathering | | |
| 26 | +| `mtg-duskmourn-house-of-horror` | `magic_the_gathering` | Magic: The Gathering | | |
| 27 | +| `gundam` | `gundam` | Bandai / Gundam | | |
| 28 | +| `gundam-high-grade` | `gundam` | Bandai / Gundam / High Grade | | |
| 29 | +| `gundam-master-grade` | `gundam` | Bandai / Gundam / Master Grade | | |
| 30 | +| `gundam-real-grade` | `gundam` | Bandai / Gundam / Real Grade | | |
| 31 | +| `gundam-perfect-grade` | `gundam` | Bandai / Gundam / Perfect Grade | | |
| 32 | +| `anime-statue` | `action_figures` | — | | |
| 33 | +| `one-piece-figures` | `action_figures` | One Piece | | |
| 34 | +| `dragon-ball-figures` | `action_figures` | Dragon Ball | | |
| 35 | +| `naruto-figures` | `action_figures` | Naruto | | |
| 36 | +| `demon-slayer-figures` | `action_figures` | Demon Slayer | | |
| 37 | +| `disney-lorcana` | `disney_lorcana` | Disney Lorcana | | |
| 38 | +| `gundam-tcg` | `other_tcg` | Gundam Card Game | | |
| 39 | + | |
| 40 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live|\bglue\b|\btools?\b|\bnippers?\b|\bmarkers?\b|\bdecals?\b|action base|display base` | |
| 41 | + | |
| 42 | +Title pattern: `^(?<name>.+?) \[(?<set>[^\]]+)\]$` (name / set / number) | |
| 43 | + | |
| 44 | +## Access & compliance | |
| 45 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: model-kit tools/markers/decals, D&D unpainted minis, sale collections, the many small per-set MTG collections not listed (added on demand). | |
| 46 | + | |
| 47 | +## Fixtures & tests | |
| 48 | +`data/fixtures/gamezilla/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts gamezilla`), trimmed single-product payloads incl. a sold-out variant. | |
| 49 | +`pnpm vitest run connectors/api/gamezilla` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts gamezilla`. | |
added
connectors/api/gamezilla/index.test.ts
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * GameZilla — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/gamezilla/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "Agate Assault [Bloomburrow]", | |
| 15 | + "productType": "MTG Single", | |
| 16 | + "collection": "mtg-bloomburrow", | |
| 17 | + "categorySlug": "magic_the_gathering" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "HG 1/144 RX-78-2 Gundam (Beyond Global)", | |
| 21 | + "collection": "gundam-high-grade", | |
| 22 | + "categorySlug": "gundam", | |
| 23 | + "brand": "Bandai" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "title": "Gundam Marker Set - Basic 6", | |
| 27 | + "collection": "gundam", | |
| 28 | + "categorySlug": null | |
| 29 | + } | |
| 30 | + ] | |
| 31 | +}); | |
added
connectors/api/gamezilla/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * GameZilla — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/gamezilla/meta.json
+200 −0
@@ -0,0 +1,200 @@ | ||
| 1 | +{ | |
| 2 | + "id": "gamezilla", | |
| 3 | + "displayName": "GameZilla (Canadian TCG & collector-toy store, CAD)", | |
| 4 | + "sourceId": "gamezilla", | |
| 5 | + "sourceName": "GameZilla", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.gamezilla.ca", | |
| 8 | + "module": "api/gamezilla", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "gundam", | |
| 15 | + "action_figures", | |
| 16 | + "disney_lorcana", | |
| 17 | + "other_tcg" | |
| 18 | + ], | |
| 19 | + "regions": [ | |
| 20 | + "CA" | |
| 21 | + ], | |
| 22 | + "languages": [ | |
| 23 | + "en" | |
| 24 | + ], | |
| 25 | + "currency": [ | |
| 26 | + "CAD" | |
| 27 | + ], | |
| 28 | + "supportsListings": true, | |
| 29 | + "supportsSold": false, | |
| 30 | + "supportsAuctions": false, | |
| 31 | + "supportsImages": true, | |
| 32 | + "supportsCatalog": false, | |
| 33 | + "supportsPopulation": false, | |
| 34 | + "supportsLookup": true, | |
| 35 | + "refreshFrequencyMinutes": 720, | |
| 36 | + "priority": "medium", | |
| 37 | + "trustScore": 0.75, | |
| 38 | + "attributionRequired": true, | |
| 39 | + "termsUrl": "https://www.gamezilla.ca/policies/terms-of-service", | |
| 40 | + "accessNotes": "GameZilla (gamezilla.ca) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 26 configured collections (mtg-commander-masters, mtg-commander-legends-battle-for-baldurs-gat, mtg-avatar-the-last-airbender, mtg-foundations, mtg-final-fantasy-commander, mtg-commander-legends, mtg-modern-horizons-3-commander, mtg-edge-of-eternities … (+18 more, see config.collections)) and /products/<handle>.json for URL lookups (~10k MTG singles over per-set collections, 1.4k Gunpla, 600 anime statues). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — model-kit tools/markers/decals, D&D unpainted minis, sale collections, the many small per-set MTG collections not listed (added on demand). No personal data is collected; seller = the store itself.", | |
| 41 | + "enabled": true, | |
| 42 | + "schemaVersion": "1.0", | |
| 43 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 44 | + "historicalDepth": "none", | |
| 45 | + "requires": [], | |
| 46 | + "config": { | |
| 47 | + "currency": "CAD", | |
| 48 | + "market": "CA", | |
| 49 | + "seller": "GameZilla", | |
| 50 | + "location": "Moncton / Bathurst / Saint John, NB, Canada", | |
| 51 | + "collections": [ | |
| 52 | + { | |
| 53 | + "handle": "mtg-commander-masters", | |
| 54 | + "categorySlug": "magic_the_gathering", | |
| 55 | + "franchise": "Magic: The Gathering" | |
| 56 | + }, | |
| 57 | + { | |
| 58 | + "handle": "mtg-commander-legends-battle-for-baldurs-gat", | |
| 59 | + "categorySlug": "magic_the_gathering", | |
| 60 | + "franchise": "Magic: The Gathering" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "mtg-avatar-the-last-airbender", | |
| 64 | + "categorySlug": "magic_the_gathering", | |
| 65 | + "franchise": "Magic: The Gathering" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "handle": "mtg-foundations", | |
| 69 | + "categorySlug": "magic_the_gathering", | |
| 70 | + "franchise": "Magic: The Gathering" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "mtg-final-fantasy-commander", | |
| 74 | + "categorySlug": "magic_the_gathering", | |
| 75 | + "franchise": "Magic: The Gathering" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "handle": "mtg-commander-legends", | |
| 79 | + "categorySlug": "magic_the_gathering", | |
| 80 | + "franchise": "Magic: The Gathering" | |
| 81 | + }, | |
| 82 | + { | |
| 83 | + "handle": "mtg-modern-horizons-3-commander", | |
| 84 | + "categorySlug": "magic_the_gathering", | |
| 85 | + "franchise": "Magic: The Gathering" | |
| 86 | + }, | |
| 87 | + { | |
| 88 | + "handle": "mtg-edge-of-eternities", | |
| 89 | + "categorySlug": "magic_the_gathering", | |
| 90 | + "franchise": "Magic: The Gathering" | |
| 91 | + }, | |
| 92 | + { | |
| 93 | + "handle": "mtg-aetherdrift", | |
| 94 | + "categorySlug": "magic_the_gathering", | |
| 95 | + "franchise": "Magic: The Gathering" | |
| 96 | + }, | |
| 97 | + { | |
| 98 | + "handle": "mtg-final-fantasy", | |
| 99 | + "categorySlug": "magic_the_gathering", | |
| 100 | + "franchise": "Magic: The Gathering" | |
| 101 | + }, | |
| 102 | + { | |
| 103 | + "handle": "mtg-dominaria-united", | |
| 104 | + "categorySlug": "magic_the_gathering", | |
| 105 | + "franchise": "Magic: The Gathering" | |
| 106 | + }, | |
| 107 | + { | |
| 108 | + "handle": "mtg-double-masters-2022", | |
| 109 | + "categorySlug": "magic_the_gathering", | |
| 110 | + "franchise": "Magic: The Gathering" | |
| 111 | + }, | |
| 112 | + { | |
| 113 | + "handle": "mtg-bloomburrow", | |
| 114 | + "categorySlug": "magic_the_gathering", | |
| 115 | + "franchise": "Magic: The Gathering" | |
| 116 | + }, | |
| 117 | + { | |
| 118 | + "handle": "mtg-duskmourn-house-of-horror", | |
| 119 | + "categorySlug": "magic_the_gathering", | |
| 120 | + "franchise": "Magic: The Gathering" | |
| 121 | + }, | |
| 122 | + { | |
| 123 | + "handle": "gundam", | |
| 124 | + "categorySlug": "gundam", | |
| 125 | + "brand": "Bandai", | |
| 126 | + "franchise": "Gundam" | |
| 127 | + }, | |
| 128 | + { | |
| 129 | + "handle": "gundam-high-grade", | |
| 130 | + "categorySlug": "gundam", | |
| 131 | + "brand": "Bandai", | |
| 132 | + "franchise": "Gundam", | |
| 133 | + "series": "High Grade" | |
| 134 | + }, | |
| 135 | + { | |
| 136 | + "handle": "gundam-master-grade", | |
| 137 | + "categorySlug": "gundam", | |
| 138 | + "brand": "Bandai", | |
| 139 | + "franchise": "Gundam", | |
| 140 | + "series": "Master Grade" | |
| 141 | + }, | |
| 142 | + { | |
| 143 | + "handle": "gundam-real-grade", | |
| 144 | + "categorySlug": "gundam", | |
| 145 | + "brand": "Bandai", | |
| 146 | + "franchise": "Gundam", | |
| 147 | + "series": "Real Grade" | |
| 148 | + }, | |
| 149 | + { | |
| 150 | + "handle": "gundam-perfect-grade", | |
| 151 | + "categorySlug": "gundam", | |
| 152 | + "brand": "Bandai", | |
| 153 | + "franchise": "Gundam", | |
| 154 | + "series": "Perfect Grade" | |
| 155 | + }, | |
| 156 | + { | |
| 157 | + "handle": "anime-statue", | |
| 158 | + "categorySlug": "action_figures" | |
| 159 | + }, | |
| 160 | + { | |
| 161 | + "handle": "one-piece-figures", | |
| 162 | + "categorySlug": "action_figures", | |
| 163 | + "franchise": "One Piece" | |
| 164 | + }, | |
| 165 | + { | |
| 166 | + "handle": "dragon-ball-figures", | |
| 167 | + "categorySlug": "action_figures", | |
| 168 | + "franchise": "Dragon Ball" | |
| 169 | + }, | |
| 170 | + { | |
| 171 | + "handle": "naruto-figures", | |
| 172 | + "categorySlug": "action_figures", | |
| 173 | + "franchise": "Naruto" | |
| 174 | + }, | |
| 175 | + { | |
| 176 | + "handle": "demon-slayer-figures", | |
| 177 | + "categorySlug": "action_figures", | |
| 178 | + "franchise": "Demon Slayer" | |
| 179 | + }, | |
| 180 | + { | |
| 181 | + "handle": "disney-lorcana", | |
| 182 | + "categorySlug": "disney_lorcana", | |
| 183 | + "franchise": "Disney Lorcana" | |
| 184 | + }, | |
| 185 | + { | |
| 186 | + "handle": "gundam-tcg", | |
| 187 | + "categorySlug": "other_tcg", | |
| 188 | + "franchise": "Gundam Card Game" | |
| 189 | + } | |
| 190 | + ], | |
| 191 | + "rules": [], | |
| 192 | + "defaultCategory": null, | |
| 193 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live|\\bglue\\b|\\btools?\\b|\\bnippers?\\b|\\bmarkers?\\b|\\bdecals?\\b|action base|display base", | |
| 194 | + "keepOutOfStock": true, | |
| 195 | + "fetchBarcodes": false, | |
| 196 | + "wholeShop": false, | |
| 197 | + "pageSize": 250, | |
| 198 | + "titlePattern": "^(?<name>.+?) \\[(?<set>[^\\]]+)\\]$" | |
| 199 | + } | |
| 200 | +} | |
added
connectors/api/gcd/README.md
+15 −0
@@ -0,0 +1,15 @@ | ||
| 1 | +# Grand Comics Database connector (`gcd`) | |
| 2 | + | |
| 3 | +- Source: https://www.comics.org (comics.org) · type: catalog · country: US (global coverage) · currency: USD (cover prices when ISO-labelled) | |
| 4 | +- Acquisition: official JSON REST API, no key — `/api/series/?page=N` (50/page, 232k series), `/api/series/<id>/`, `/api/issue/<id>/`, `/api/publisher/<id>/` | |
| 5 | +- Records: `catalog_item` per issue with `identifiers.comics_org_id` (+ `gcd_series_id`, `gcd_publisher_id`, upc/ean/isbn from the barcode) | |
| 6 | +- Data licence: CC-BY-SA — kept in `metadata.license`, every record links to its comics.org issue page. | |
| 7 | + | |
| 8 | +## Behaviour | |
| 9 | +- Resumable page walk of the series list (cursor `{page, index}`); `issuesPerSeries` issue details per series; publisher names cached per run. | |
| 10 | +- Rate limit: the API answers 429 (Retry-After ≈ 29 min) after ~50 requests → `maxRequestsPerRun` (30) per hourly run, 2.5 s spacing, clean stop with anomaly `rate_limited`; domain policy `maxRetries: 0`. | |
| 11 | +- `lookup(https://www.comics.org/issue/<id>/ | /series/<id>/)` and seeds `series:<id>` / `issue:<id>`. | |
| 12 | +- Dates with GCD `00` placeholders stay year-only (`releaseDate` null); cover price → `originalMsrp` only with an ISO currency. | |
| 13 | + | |
| 14 | +## Fixtures / tests | |
| 15 | +`data/fixtures/gcd/*.json` are live API captures (trimmed). `pnpm vitest run connectors/api/gcd`. Capture: `pnpm tsx connectors/api/gcd/_capture.ts [seriesId…]`. | |
added
connectors/api/gcd/_capture.ts
+45 −0
@@ -0,0 +1,45 @@ | ||
| 1 | +/** | |
| 2 | + * Live fixture capture for the gcd connector (real API responses, trimmed). | |
| 3 | + * Usage: pnpm tsx connectors/api/gcd/_capture.ts [seriesId...] | |
| 4 | + */ | |
| 5 | +import { createCrawlContext, createRouter } from '@rareindex/connectors'; | |
| 6 | +import { saveFixture } from '@rareindex/connectors/testing'; | |
| 7 | +import { childLogger } from '@rareindex/shared'; | |
| 8 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 9 | +import { useGroupDomains } from '../_g6-comics-toys-games-lib/capture-env.js'; | |
| 10 | +import meta from './meta.json' with { type: 'json' }; | |
| 11 | +import createConnector, { type Payload } from './index.js'; | |
| 12 | + | |
| 13 | +useGroupDomains(); | |
| 14 | +const connector = createConnector(localMeta(meta)); | |
| 15 | +const router = createRouter({}); | |
| 16 | +const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: 2 }, log: childLogger({ connector: 'gcd', level: 'warn' }) }); | |
| 17 | + | |
| 18 | +// 1. First page of the series walk (probe: 2 series, issues trimmed to 3 each) | |
| 19 | +let n = 0; | |
| 20 | +for await (const raw of connector.crawl(ctx)) { | |
| 21 | + const p = raw.payload as Payload; | |
| 22 | + p.issues = p.issues.slice(0, 3); | |
| 23 | + saveFixture('gcd', `series-page1-${n++}-${p.series.id}`, { | |
| 24 | + raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload: raw.payload, fetchedAt: raw.fetchedAt ?? new Date() }, | |
| 25 | + expect: { minCount: 1, kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.comics_org_id', 'attributes.set', 'attributes.brand'] }, | |
| 26 | + note: `Live capture of https://www.comics.org/api/series/?format=json&page=1 (series ${p.series.id}) + /api/issue/<id>/ details, trimmed to 3 issues.`, | |
| 27 | + }); | |
| 28 | +} | |
| 29 | + | |
| 30 | +// 2. Lookups of well-known series (ids passed on the command line; defaults probe a few candidates) | |
| 31 | +const ids = process.argv.slice(2).length ? process.argv.slice(2) : ['1571', '2001', '12345']; | |
| 32 | +for (const id of ids) { | |
| 33 | + const recs = await connector.lookup(`https://www.comics.org/series/${id}/`, ctx); | |
| 34 | + for (const raw of recs) { | |
| 35 | + const p = raw.payload as Payload; | |
| 36 | + console.log(`series ${id}: ${p.series.name} (${p.series.year_began}) publisher=${p.publisher.name} issues=${p.issues.length} first=#${p.issues[0]?.number} key_date=${p.issues[0]?.key_date} price=${p.issues[0]?.price} barcode=${p.issues[0]?.barcode}`); | |
| 37 | + p.issues = p.issues.slice(0, 4); | |
| 38 | + saveFixture('gcd', `series-lookup-${id}`, { | |
| 39 | + raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload: raw.payload, fetchedAt: raw.fetchedAt ?? new Date() }, | |
| 40 | + expect: { minCount: 1, kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.comics_org_id', 'attributes.number', 'attributes.year'] }, | |
| 41 | + note: `Live capture of https://www.comics.org/api/series/${id}/?format=json + first issues (trimmed to 4).`, | |
| 42 | + }); | |
| 43 | + } | |
| 44 | +} | |
| 45 | +console.log('gcd fixtures written; engineStats=', JSON.stringify(ctx.engineStats), 'anomalies=', ctx.anomalies); | |
added
connectors/api/gcd/index.test.ts
+73 −0
@@ -0,0 +1,73 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 5 | +import createConnector, { barcodeIds, idFromApiUrl, parseCoverPrice, parseKeyDate, trimIssue, trimSeries } from './index.js'; | |
| 6 | +import { publisherCategory } from '../_g6-comics-toys-games-lib/comics.js'; | |
| 7 | + | |
| 8 | +const connector = createConnector(localMeta(meta)); | |
| 9 | + | |
| 10 | +describe('gcd', () => { | |
| 11 | + runFixtureSuite(connector, it, expect); | |
| 12 | + | |
| 13 | + it('parses GCD key dates without inventing missing day/month', () => { | |
| 14 | + expect(parseKeyDate('1988-05-00')).toEqual({ year: 1988, date: null }); | |
| 15 | + expect(parseKeyDate('1988-02-23')).toEqual({ year: 1988, date: new Date(Date.UTC(1988, 1, 23)) }); | |
| 16 | + expect(parseKeyDate('1867-00-00')).toEqual({ year: 1867, date: null }); | |
| 17 | + expect(parseKeyDate('')).toEqual({ year: null, date: null }); | |
| 18 | + expect(parseKeyDate(null)).toEqual({ year: null, date: null }); | |
| 19 | + }); | |
| 20 | + | |
| 21 | + it('parses cover prices only when an ISO currency is printed', () => { | |
| 22 | + expect(parseCoverPrice('0.12 USD')).toEqual({ amount: 0.12, currency: 'USD' }); | |
| 23 | + expect(parseCoverPrice('1.95 USD; 2.60 CAD; 0.50 GBP')).toEqual({ amount: 1.95, currency: 'USD' }); | |
| 24 | + expect(parseCoverPrice('0.00 FREE')).toBeNull(); | |
| 25 | + expect(parseCoverPrice('[none]')).toBeNull(); | |
| 26 | + expect(parseCoverPrice('')).toBeNull(); | |
| 27 | + }); | |
| 28 | + | |
| 29 | + it('derives deterministic identifiers from api urls and barcodes', () => { | |
| 30 | + expect(idFromApiUrl('https://www.comics.org/api/issue/127538/?format=json')).toBe('127538'); | |
| 31 | + expect(idFromApiUrl('https://www.comics.org/api/publisher/78/?format=json')).toBe('78'); | |
| 32 | + expect(idFromApiUrl(null)).toBeNull(); | |
| 33 | + expect(barcodeIds('759606024605')).toEqual({ upc: '759606024605' }); | |
| 34 | + expect(barcodeIds('75960602460500111')).toEqual({ upc: '759606024605', upc_supplement: '00111' }); | |
| 35 | + expect(barcodeIds('9781302900000')).toEqual({ isbn: '9781302900000', ean: '9781302900000' }); | |
| 36 | + expect(barcodeIds('')).toEqual({}); | |
| 37 | + }); | |
| 38 | + | |
| 39 | + it('maps publishers to taxonomy families', () => { | |
| 40 | + expect(publisherCategory('Marvel')).toBe('marvel_comics'); | |
| 41 | + expect(publisherCategory('DC')).toBe('dc_comics'); | |
| 42 | + expect(publisherCategory('Harrier')).toBe('independent_comics'); | |
| 43 | + expect(publisherCategory('Shueisha', 'ja')).toBe('manga'); | |
| 44 | + expect(publisherCategory(null)).toBe('comics'); | |
| 45 | + }); | |
| 46 | + | |
| 47 | + it('trims live API shapes and normalises fixtures into catalog items with comics_org_id', async () => { | |
| 48 | + const issue = trimIssue({ api_url: 'https://www.comics.org/api/issue/1/?format=json', number: '[nn]', key_date: '1867-00-00', price: '[none]', story_set: [{ type: 'cover', pencils: 'Gustave Doré' }] }); | |
| 49 | + expect(issue.id).toBe('1'); | |
| 50 | + expect(issue.cover_pencils).toBe('Gustave Doré'); | |
| 51 | + expect(issue.story_count).toBe(1); | |
| 52 | + const series = trimSeries({ api_url: 'https://www.comics.org/api/series/10814/?format=json', name: '!Gag!', country: 'gb', language: 'en', year_began: 1987, year_ended: 1989, active_issues: ['a', 'b'] }); | |
| 53 | + expect(series).toMatchObject({ id: '10814', country: 'gb', issue_count: 2 }); | |
| 54 | + for (const name of listFixtures('gcd')) { | |
| 55 | + const fx = loadFixture('gcd', name); | |
| 56 | + const out = await connector.normalize(fx.raw); | |
| 57 | + expect(out.length).toBeGreaterThan(0); | |
| 58 | + for (const r of out) { | |
| 59 | + expect(r.kind).toBe('catalog_item'); | |
| 60 | + if (r.kind !== 'catalog_item') continue; | |
| 61 | + expect(r.attributes.identifiers.comics_org_id).toMatch(/^\d+$/); | |
| 62 | + expect(r.sourceUrl).toBe(`https://www.comics.org/issue/${r.attributes.identifiers.comics_org_id}/`); | |
| 63 | + expect(r.attributes.metadata.license).toContain('CC-BY-SA'); | |
| 64 | + expect(r.confidence).toBeLessThan(1); | |
| 65 | + } | |
| 66 | + } | |
| 67 | + const gag = await connector.normalize(loadFixture('gcd', 'series-page1-0-10814').raw); | |
| 68 | + const first = gag[0]; | |
| 69 | + if (first?.kind !== 'catalog_item') throw new Error('expected catalog item'); | |
| 70 | + expect(first.attributes).toMatchObject({ categorySlug: 'independent_comics', brand: 'Harrier', set: '!Gag!', number: '1', year: 1987, country: 'GB', originalMsrp: 1.95, originalMsrpCurrency: 'USD' }); | |
| 71 | + expect(first.releaseDate).toBeNull(); | |
| 72 | + }); | |
| 73 | +}); | |
added
connectors/api/gcd/index.ts
+369 −0
@@ -0,0 +1,369 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { NormalizedCatalogItemSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { attrs } from '../_lib/shared.js'; | |
| 5 | +import { publisherCategory } from '../_g6-comics-toys-games-lib/comics.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Grand Comics Database (comics.org) — public, unauthenticated JSON REST API | |
| 9 | + * (https://www.comics.org/api/ — Django REST framework): /api/series/?page=N (50 per page, 232k series), | |
| 10 | + * /api/series/<id>/, /api/issue/<id>/, /api/publisher/<id>/. Data is CC-BY-SA (attribution kept on every record). | |
| 11 | + * One raw record per series (with up to `issuesPerSeries` issue details); normalize → one catalog_item per | |
| 12 | + * issue keyed by `comics_org_id`. | |
| 13 | + */ | |
| 14 | +const API = 'https://www.comics.org/api'; | |
| 15 | +const SITE = 'https://www.comics.org'; | |
| 16 | +const PARSER_VERSION = '1.0.0'; | |
| 17 | +const HEADERS = { accept: 'application/json' }; | |
| 18 | + | |
| 19 | +export const IssueSchema = z.object({ | |
| 20 | + id: z.string(), | |
| 21 | + number: z.string().nullable().default(null), | |
| 22 | + volume: z.string().nullable().default(null), | |
| 23 | + variant_name: z.string().nullable().default(null), | |
| 24 | + variant_of: z.string().nullable().default(null), | |
| 25 | + title: z.string().nullable().default(null), | |
| 26 | + publication_date: z.string().nullable().default(null), | |
| 27 | + key_date: z.string().nullable().default(null), | |
| 28 | + on_sale_date: z.string().nullable().default(null), | |
| 29 | + price: z.string().nullable().default(null), | |
| 30 | + page_count: z.string().nullable().default(null), | |
| 31 | + isbn: z.string().nullable().default(null), | |
| 32 | + barcode: z.string().nullable().default(null), | |
| 33 | + indicia_publisher: z.string().nullable().default(null), | |
| 34 | + brand_emblem: z.string().nullable().default(null), | |
| 35 | + cover: z.string().nullable().default(null), | |
| 36 | + cover_pencils: z.string().nullable().default(null), | |
| 37 | + cover_title: z.string().nullable().default(null), | |
| 38 | + story_count: z.number().int().nullable().default(null), | |
| 39 | + notes: z.string().nullable().default(null), | |
| 40 | +}); | |
| 41 | +export type Issue = z.infer<typeof IssueSchema>; | |
| 42 | +export const SeriesSchema = z.object({ | |
| 43 | + id: z.string(), | |
| 44 | + name: z.string(), | |
| 45 | + country: z.string().nullable().default(null), | |
| 46 | + language: z.string().nullable().default(null), | |
| 47 | + year_began: z.number().int().nullable().default(null), | |
| 48 | + year_ended: z.number().int().nullable().default(null), | |
| 49 | + binding: z.string().nullable().default(null), | |
| 50 | + publishing_format: z.string().nullable().default(null), | |
| 51 | + color: z.string().nullable().default(null), | |
| 52 | + issue_count: z.number().int().nullable().default(null), | |
| 53 | +}); | |
| 54 | +export type Series = z.infer<typeof SeriesSchema>; | |
| 55 | +export const PublisherSchema = z.object({ id: z.string().nullable(), name: z.string().nullable(), country: z.string().nullable().default(null) }); | |
| 56 | +export const PayloadSchema = z.object({ kind: z.literal('series_issues'), series: SeriesSchema, publisher: PublisherSchema, issues: z.array(IssueSchema) }); | |
| 57 | +export type Payload = z.infer<typeof PayloadSchema>; | |
| 58 | + | |
| 59 | +type Json = Record<string, any>; | |
| 60 | + | |
| 61 | +export function idFromApiUrl(url: string | null | undefined): string | null { | |
| 62 | + return url?.match(/\/api\/(?:issue|series|publisher)\/(\d+)\//)?.[1] ?? null; | |
| 63 | +} | |
| 64 | + | |
| 65 | +const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null); | |
| 66 | + | |
| 67 | +export function trimIssue(j: Json): Issue { | |
| 68 | + const stories: Json[] = Array.isArray(j.story_set) ? j.story_set : []; | |
| 69 | + const cover = stories.find((s) => s.type === 'cover'); | |
| 70 | + return IssueSchema.parse({ | |
| 71 | + id: idFromApiUrl(j.api_url) ?? String(j.id ?? ''), | |
| 72 | + number: str(j.number), | |
| 73 | + volume: str(j.volume), | |
| 74 | + variant_name: str(j.variant_name), | |
| 75 | + variant_of: idFromApiUrl(j.variant_of), | |
| 76 | + title: str(j.title), | |
| 77 | + publication_date: str(j.publication_date), | |
| 78 | + key_date: str(j.key_date), | |
| 79 | + on_sale_date: str(j.on_sale_date), | |
| 80 | + price: str(j.price), | |
| 81 | + page_count: str(j.page_count), | |
| 82 | + isbn: str(j.isbn), | |
| 83 | + barcode: str(j.barcode), | |
| 84 | + indicia_publisher: str(j.indicia_publisher), | |
| 85 | + brand_emblem: str(j.brand_emblem), | |
| 86 | + cover: str(j.cover), | |
| 87 | + cover_pencils: str(cover?.pencils), | |
| 88 | + cover_title: str(cover?.title), | |
| 89 | + story_count: stories.length, | |
| 90 | + notes: str(j.notes)?.slice(0, 400) ?? null, | |
| 91 | + }); | |
| 92 | +} | |
| 93 | + | |
| 94 | +export function trimSeries(j: Json): Series { | |
| 95 | + return SeriesSchema.parse({ | |
| 96 | + id: idFromApiUrl(j.api_url) ?? String(j.id ?? ''), | |
| 97 | + name: String(j.name ?? ''), | |
| 98 | + country: str(j.country), | |
| 99 | + language: str(j.language), | |
| 100 | + year_began: typeof j.year_began === 'number' ? j.year_began : null, | |
| 101 | + year_ended: typeof j.year_ended === 'number' ? j.year_ended : null, | |
| 102 | + binding: str(j.binding), | |
| 103 | + publishing_format: str(j.publishing_format), | |
| 104 | + color: str(j.color), | |
| 105 | + issue_count: Array.isArray(j.active_issues) ? j.active_issues.length : null, | |
| 106 | + }); | |
| 107 | +} | |
| 108 | + | |
| 109 | +/** "1988-05-00" → { year: 1988, date: null } · "1988-02-23" → full UTC date. GCD uses 00 for unknown month/day. */ | |
| 110 | +export function parseKeyDate(s: string | null | undefined): { year: number | null; date: Date | null } { | |
| 111 | + if (!s) return { year: null, date: null }; | |
| 112 | + const m = s.match(/^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?/); | |
| 113 | + if (!m) return { year: null, date: null }; | |
| 114 | + const year = Number(m[1]); | |
| 115 | + if (!year || year < 1500) return { year: null, date: null }; | |
| 116 | + const mo = m[2] ? Number(m[2]) : 0; | |
| 117 | + const d = m[3] ? Number(m[3]) : 0; | |
| 118 | + if (mo >= 1 && mo <= 12 && d >= 1 && d <= 31) { | |
| 119 | + const date = new Date(Date.UTC(year, mo - 1, d)); | |
| 120 | + return { year, date: Number.isNaN(date.getTime()) ? null : date }; | |
| 121 | + } | |
| 122 | + return { year, date: null }; | |
| 123 | +} | |
| 124 | + | |
| 125 | +/** "0.12 USD" · "$0.12 USD; 0.15 CAD" → first amount with an ISO currency we support. */ | |
| 126 | +export function parseCoverPrice(s: string | null | undefined): { amount: number; currency: 'USD' | 'CAD' | 'GBP' | 'EUR' | 'JPY' } | null { | |
| 127 | + if (!s) return null; | |
| 128 | + const m = s.match(/(\d+(?:\.\d+)?)\s*(USD|CAD|GBP|EUR|JPY)\b/); | |
| 129 | + if (!m) return null; | |
| 130 | + const amount = Number(m[1]); | |
| 131 | + return Number.isFinite(amount) && amount > 0 ? { amount, currency: m[2] as 'USD' } : null; | |
| 132 | +} | |
| 133 | + | |
| 134 | +export function barcodeIds(barcode: string | null | undefined): Record<string, string> { | |
| 135 | + if (!barcode) return {}; | |
| 136 | + const d = barcode.replace(/[^0-9]/g, ''); | |
| 137 | + if (d.length === 12) return { upc: d }; | |
| 138 | + if (d.length === 13) return /^97[89]/.test(d) ? { isbn: d, ean: d } : { ean: d }; | |
| 139 | + if (d.length >= 17 && d.length <= 18) return { upc: d.slice(0, 12), upc_supplement: d.slice(12) }; | |
| 140 | + return {}; | |
| 141 | +} | |
| 142 | + | |
| 143 | +export class GcdConnector extends BaseConnector { | |
| 144 | + readonly version = '1.0.0'; | |
| 145 | + readonly parserVersion = PARSER_VERSION; | |
| 146 | + protected override minIntervalMs = 2500; | |
| 147 | + override readonly urlPatterns = [/^https?:\/\/(?:www\.)?comics\.org\/(issue|series)\/(\d+)\/?/i]; | |
| 148 | + private publishers = new Map<string, z.infer<typeof PublisherSchema>>(); | |
| 149 | + /** Set when the API answers 429 (GCD throttles at roughly 50 requests per ~30 min); the run then stops at its last checkpoint. */ | |
| 150 | + private rateLimited = false; | |
| 151 | + private requests = 0; | |
| 152 | + | |
| 153 | + private async getJson(ctx: CrawlContext, url: string): Promise<Json | null> { | |
| 154 | + if (this.rateLimited) return null; | |
| 155 | + await this.throttle(url); | |
| 156 | + this.requests++; | |
| 157 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', headers: HEADERS, minQuality: 0 }); | |
| 158 | + if (res.httpStatus === 429) { | |
| 159 | + this.rateLimited = true; | |
| 160 | + ctx.anomaly('rate_limited', `${url}: HTTP 429 after ${this.requests} requests this run — stopping at the last checkpoint`); | |
| 161 | + return null; | |
| 162 | + } | |
| 163 | + if (!res.success || !res.json || typeof res.json !== 'object') { | |
| 164 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 165 | + return null; | |
| 166 | + } | |
| 167 | + return res.json as Json; | |
| 168 | + } | |
| 169 | + | |
| 170 | + private budgetLeft(): boolean { | |
| 171 | + return !this.rateLimited && this.requests < Number(this.meta.config.maxRequestsPerRun ?? 40); | |
| 172 | + } | |
| 173 | + | |
| 174 | + private async publisher(ctx: CrawlContext, apiUrl: string | null): Promise<z.infer<typeof PublisherSchema>> { | |
| 175 | + const id = idFromApiUrl(apiUrl); | |
| 176 | + if (!id) return { id: null, name: null, country: null }; | |
| 177 | + const hit = this.publishers.get(id); | |
| 178 | + if (hit) return hit; | |
| 179 | + const j = await this.getJson(ctx, `${API}/publisher/${id}/?format=json`); | |
| 180 | + const p = { id, name: str(j?.name), country: str(j?.country) }; | |
| 181 | + this.publishers.set(id, p); | |
| 182 | + return p; | |
| 183 | + } | |
| 184 | + | |
| 185 | + /** Fetch a series JSON (already loaded list entry or by id) plus up to `max` of its issues. */ | |
| 186 | + private async seriesRecord(ctx: CrawlContext, seriesJson: Json, max: number, onlyIssueIds?: string[]): Promise<RawRecordInput | null> { | |
| 187 | + const series = trimSeries(seriesJson); | |
| 188 | + if (!series.id) return null; | |
| 189 | + const publisher = await this.publisher(ctx, seriesJson.publisher); | |
| 190 | + const issueUrls: string[] = Array.isArray(seriesJson.active_issues) ? seriesJson.active_issues : []; | |
| 191 | + const wanted = onlyIssueIds ? issueUrls.filter((u) => onlyIssueIds.includes(idFromApiUrl(u) ?? '')) : issueUrls.slice(0, max); | |
| 192 | + const issues: Issue[] = []; | |
| 193 | + for (const u of wanted) { | |
| 194 | + if (ctx.signal?.aborted || !this.budgetLeft()) break; | |
| 195 | + const j = await this.getJson(ctx, u.includes('format=') ? u : `${u}${u.includes('?') ? '&' : '?'}format=json`); | |
| 196 | + if (!j) continue; | |
| 197 | + try { | |
| 198 | + issues.push(trimIssue(j)); | |
| 199 | + } catch (err) { | |
| 200 | + ctx.anomaly('schema_drift', `${u}: ${err instanceof Error ? err.message : String(err)}`); | |
| 201 | + } | |
| 202 | + } | |
| 203 | + if (!issues.length) return null; | |
| 204 | + const payload: Payload = { kind: 'series_issues', series, publisher, issues }; | |
| 205 | + return { url: `${SITE}/series/${series.id}/`, externalId: `series:${series.id}`, kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() }; | |
| 206 | + } | |
| 207 | + | |
| 208 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 209 | + const backfill = ctx.options.mode === 'backfill'; | |
| 210 | + const issuesPerSeries = Number(this.meta.config.issuesPerSeries ?? 12); | |
| 211 | + const seriesPerRun = ctx.options.mode === 'probe' ? Math.max(1, ctx.options.limit ?? 3) : Number(backfill ? this.meta.config.backfillSeriesPerRun ?? 200 : this.meta.config.seriesPerRun ?? 25); | |
| 212 | + let count = 0; | |
| 213 | + | |
| 214 | + // Manual seeds: "series:<id>", "issue:<id>" or comics.org URLs. | |
| 215 | + if (ctx.options.seeds?.length) { | |
| 216 | + for (const seed of ctx.options.seeds) { | |
| 217 | + if (ctx.signal?.aborted || this.reached(ctx, count)) return; | |
| 218 | + const m = seed.match(/(series|issue)[:/](\d+)/i); | |
| 219 | + if (!m) continue; | |
| 220 | + const recs = await this.lookup(`${SITE}/${m[1]!.toLowerCase()}/${m[2]}/`, ctx); | |
| 221 | + for (const r of recs) { | |
| 222 | + count++; | |
| 223 | + yield r; | |
| 224 | + } | |
| 225 | + } | |
| 226 | + return; | |
| 227 | + } | |
| 228 | + | |
| 229 | + const cursor = (ctx.options.cursor ?? {}) as { page?: number; index?: number; done?: boolean }; | |
| 230 | + let page = Math.max(1, Number(cursor.page ?? 1)); | |
| 231 | + let index = Math.max(0, Number(cursor.index ?? 0)); | |
| 232 | + let processed = 0; | |
| 233 | + this.rateLimited = false; | |
| 234 | + this.requests = 0; | |
| 235 | + while (processed < seriesPerRun) { | |
| 236 | + if (ctx.signal?.aborted || this.reached(ctx, count) || !this.budgetLeft()) return; | |
| 237 | + const listUrl = `${API}/series/?format=json&page=${page}`; | |
| 238 | + const list = await this.getJson(ctx, listUrl); | |
| 239 | + const results: Json[] = Array.isArray(list?.results) ? list!.results : []; | |
| 240 | + if (!list || !results.length) { | |
| 241 | + if (list && list.next === null) { | |
| 242 | + await ctx.setCursor({ done: true, page: 1, index: 0, updatedAt: new Date().toISOString() }); | |
| 243 | + return; | |
| 244 | + } | |
| 245 | + ctx.anomaly('pagination_failure', listUrl); | |
| 246 | + return; | |
| 247 | + } | |
| 248 | + const totalPages = typeof list.count === 'number' ? Math.ceil(list.count / 50) : null; | |
| 249 | + for (; index < results.length && processed < seriesPerRun; index++) { | |
| 250 | + if (ctx.signal?.aborted || this.reached(ctx, count) || !this.budgetLeft()) break; | |
| 251 | + const s = results[index]!; | |
| 252 | + if (!Array.isArray(s.active_issues) || !s.active_issues.length) { | |
| 253 | + processed++; | |
| 254 | + continue; | |
| 255 | + } | |
| 256 | + const rec = await this.seriesRecord(ctx, s, issuesPerSeries); | |
| 257 | + processed++; | |
| 258 | + if (rec) { | |
| 259 | + count++; | |
| 260 | + yield rec; | |
| 261 | + } | |
| 262 | + await ctx.setCursor({ page, index: index + 1, updatedAt: new Date().toISOString() }); | |
| 263 | + } | |
| 264 | + if (!this.budgetLeft()) return; | |
| 265 | + if (index >= results.length) { | |
| 266 | + if (list.next === null) { | |
| 267 | + await ctx.setCursor({ done: true, page: 1, index: 0, updatedAt: new Date().toISOString() }); | |
| 268 | + return; | |
| 269 | + } | |
| 270 | + page++; | |
| 271 | + index = 0; | |
| 272 | + await ctx.setCursor({ page, index: 0, updatedAt: new Date().toISOString() }); | |
| 273 | + if (backfill) await ctx.progress({ page, totalPages, itemsProcessed: count }); | |
| 274 | + } | |
| 275 | + } | |
| 276 | + } | |
| 277 | + | |
| 278 | + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> { | |
| 279 | + const m = url.match(this.urlPatterns[0]!); | |
| 280 | + if (!m) return []; | |
| 281 | + const [, kind, id] = m; | |
| 282 | + if (kind!.toLowerCase() === 'issue') { | |
| 283 | + const issue = await this.getJson(ctx, `${API}/issue/${id}/?format=json`); | |
| 284 | + const seriesUrl = issue?.series; | |
| 285 | + const sid = idFromApiUrl(seriesUrl); | |
| 286 | + if (!issue || !sid) return []; | |
| 287 | + const series = await this.getJson(ctx, `${API}/series/${sid}/?format=json`); | |
| 288 | + if (!series) return []; | |
| 289 | + const rec = await this.seriesRecord(ctx, series, 1, [id!]); | |
| 290 | + return rec ? [rec] : []; | |
| 291 | + } | |
| 292 | + const series = await this.getJson(ctx, `${API}/series/${id}/?format=json`); | |
| 293 | + if (!series) return []; | |
| 294 | + const rec = await this.seriesRecord(ctx, series, Number(this.meta.config.issuesPerSeries ?? 12)); | |
| 295 | + return rec ? [rec] : []; | |
| 296 | + } | |
| 297 | + | |
| 298 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 299 | + const p = PayloadSchema.parse(raw.payload); | |
| 300 | + const categorySlug = publisherCategory(p.publisher.name, p.series.language); | |
| 301 | + const out: NormalizedRecord[] = []; | |
| 302 | + for (const it of p.issues) { | |
| 303 | + const kd = parseKeyDate(it.key_date); | |
| 304 | + const onSale = parseKeyDate(it.on_sale_date); | |
| 305 | + const year = kd.year ?? onSale.year ?? (p.series.year_began && p.series.year_began === p.series.year_ended ? p.series.year_began : null); | |
| 306 | + const price = parseCoverPrice(it.price); | |
| 307 | + const number = it.number && it.number !== '[nn]' ? it.number : null; | |
| 308 | + const seriesLabel = p.series.year_began ? `${p.series.name} (${p.series.year_began}${p.series.year_ended && p.series.year_ended !== p.series.year_began ? `-${p.series.year_ended}` : ''} series)` : p.series.name; | |
| 309 | + const name = number ? `${p.series.name} #${number}${it.variant_name ? ` (${it.variant_name})` : ''}` : it.title || p.series.name; | |
| 310 | + const identifiers: Record<string, string> = { comics_org_id: it.id, gcd_series_id: p.series.id, ...barcodeIds(it.barcode) }; | |
| 311 | + if (it.isbn) identifiers.isbn = it.isbn.replace(/[^0-9Xx]/g, ''); | |
| 312 | + if (p.publisher.id) identifiers.gcd_publisher_id = p.publisher.id; | |
| 313 | + const attributes = attrs({ | |
| 314 | + categorySlug, | |
| 315 | + brand: p.publisher.name, | |
| 316 | + series: seriesLabel, | |
| 317 | + set: p.series.name, | |
| 318 | + name, | |
| 319 | + number, | |
| 320 | + year, | |
| 321 | + variant: it.variant_name, | |
| 322 | + language: p.series.language, | |
| 323 | + country: p.series.country ? p.series.country.toUpperCase() : null, | |
| 324 | + originalMsrp: price?.amount ?? null, | |
| 325 | + originalMsrpCurrency: price?.currency ?? null, | |
| 326 | + identifiers, | |
| 327 | + metadata: { | |
| 328 | + key_date: it.key_date, | |
| 329 | + publication_date: it.publication_date, | |
| 330 | + on_sale_date: it.on_sale_date, | |
| 331 | + cover_price_raw: it.price, | |
| 332 | + page_count: it.page_count, | |
| 333 | + indicia_publisher: it.indicia_publisher, | |
| 334 | + brand_emblem: it.brand_emblem, | |
| 335 | + variant_of: it.variant_of, | |
| 336 | + cover_pencils: it.cover_pencils, | |
| 337 | + story_count: it.story_count, | |
| 338 | + binding: p.series.binding, | |
| 339 | + publishing_format: p.series.publishing_format, | |
| 340 | + series_year_began: p.series.year_began, | |
| 341 | + series_year_ended: p.series.year_ended, | |
| 342 | + license: 'CC-BY-SA (Grand Comics Database)', | |
| 343 | + }, | |
| 344 | + }); | |
| 345 | + out.push( | |
| 346 | + NormalizedCatalogItemSchema.parse({ | |
| 347 | + kind: 'catalog_item', | |
| 348 | + connectorId: this.meta.id, | |
| 349 | + sourceId: this.meta.sourceId, | |
| 350 | + sourceUrl: `${SITE}/issue/${it.id}/`, | |
| 351 | + externalId: it.id, | |
| 352 | + rawTitle: `${p.series.name} #${number ?? it.title ?? ''} (${it.publication_date ?? p.series.year_began ?? '?'})`.trim(), | |
| 353 | + description: it.notes, | |
| 354 | + imageUrls: it.cover && /^https?:\/\//.test(it.cover) ? [it.cover] : [], | |
| 355 | + attributes, | |
| 356 | + grade: {}, | |
| 357 | + condition: {}, | |
| 358 | + observedAt: raw.fetchedAt, | |
| 359 | + confidence: 0.92, | |
| 360 | + parserVersion: PARSER_VERSION, | |
| 361 | + releaseDate: onSale.date ?? kd.date, | |
| 362 | + }), | |
| 363 | + ); | |
| 364 | + } | |
| 365 | + return out; | |
| 366 | + } | |
| 367 | +} | |
| 368 | + | |
| 369 | +export default (meta: ConnectorMeta) => new GcdConnector(meta); | |
added
connectors/api/gcd/meta.json
+70 −0
@@ -0,0 +1,70 @@ | ||
| 1 | +{ | |
| 2 | + "id": "gcd", | |
| 3 | + "displayName": "Grand Comics Database (comics.org catalog)", | |
| 4 | + "sourceId": "gcd", | |
| 5 | + "sourceName": "Grand Comics Database", | |
| 6 | + "sourceType": "catalog", | |
| 7 | + "sourceUrl": "https://www.comics.org", | |
| 8 | + "module": "api/gcd", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "comics", | |
| 14 | + "marvel_comics", | |
| 15 | + "dc_comics", | |
| 16 | + "independent_comics", | |
| 17 | + "manga" | |
| 18 | + ], | |
| 19 | + "regions": [ | |
| 20 | + "US", | |
| 21 | + "GB", | |
| 22 | + "CA", | |
| 23 | + "FR", | |
| 24 | + "DE", | |
| 25 | + "IT", | |
| 26 | + "ES", | |
| 27 | + "NL", | |
| 28 | + "SE", | |
| 29 | + "JP", | |
| 30 | + "BR" | |
| 31 | + ], | |
| 32 | + "languages": [ | |
| 33 | + "en", | |
| 34 | + "fr", | |
| 35 | + "de", | |
| 36 | + "it", | |
| 37 | + "es", | |
| 38 | + "nl", | |
| 39 | + "sv", | |
| 40 | + "ja", | |
| 41 | + "pt" | |
| 42 | + ], | |
| 43 | + "currency": [ | |
| 44 | + "USD" | |
| 45 | + ], | |
| 46 | + "supportsListings": false, | |
| 47 | + "supportsSold": false, | |
| 48 | + "supportsAuctions": false, | |
| 49 | + "supportsImages": false, | |
| 50 | + "supportsCatalog": true, | |
| 51 | + "supportsPopulation": false, | |
| 52 | + "supportsLookup": true, | |
| 53 | + "refreshFrequencyMinutes": 60, | |
| 54 | + "priority": "high", | |
| 55 | + "trustScore": 0.9, | |
| 56 | + "attributionRequired": true, | |
| 57 | + "termsUrl": "https://www.comics.org/about/", | |
| 58 | + "accessNotes": "Reads only the public, unauthenticated JSON REST API of the Grand Comics Database: https://www.comics.org/api/series/?format=json&page=N (50 series per page, 232,658 series → ~4,650 pages), /api/series/<id>/, /api/issue/<id>/ and /api/publisher/<id>/ (publisher names are cached per run). No login, no cookies; the API root advertises only `series` and `publisher` list endpoints and ignores filter parameters, so discovery is a resumable page walk (cursor = {page, index}). robots.txt (Cloudflare managed): `User-agent: *` Allow: / with Content-Signal search=yes, ai-train=no — we index catalog facts and link back; the HTML pages and the /download/ SQL dumps sit behind a Cloudflare managed challenge and a GCD account and are NOT fetched. GCD data is CC-BY-SA: every record carries the licence in metadata and links to its comics.org issue page. Rate limit (verified live 2026-09-08, twice): the API throttles per client at roughly 60 requests per hour (HTTP 429 with Retry-After up to ~1,741 s; a second burst of 7 requests 30 min later was refused again), so each hourly run is capped at maxRequestsPerRun (30) requests spaced 2.5 s apart, stops cleanly at its last checkpoint on the first 429 (anomaly 'rate_limited'; the comics.org domain policy sets maxRetries 0 so the engine never sleeps on Retry-After) and resumes from the cursor next run. Cover prices are stored as originalMsrp only when GCD prints an ISO currency (e.g. '0.12 USD'); dates with unknown day/month (GCD '00') are kept as year only, never completed.", | |
| 59 | + "enabled": true, | |
| 60 | + "schemaVersion": "1.0", | |
| 61 | + "acquisitionMethod": "official JSON API (comics.org/api, no key)", | |
| 62 | + "historicalDepth": "decades", | |
| 63 | + "requires": [], | |
| 64 | + "config": { | |
| 65 | + "seriesPerRun": 5, | |
| 66 | + "backfillSeriesPerRun": 5, | |
| 67 | + "issuesPerSeries": 6, | |
| 68 | + "maxRequestsPerRun": 30 | |
| 69 | + } | |
| 70 | +} | |
added
connectors/api/goblingames-nz/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Goblin Games NZ connector (`goblingames-nz`) | |
| 2 | + | |
| 3 | +- Source: https://goblingames.co.nz · New Zealand TCG specialist: ~190k MTG singles (incl. Alpha/Beta/Unlimited old-school and Secret Lair), 43k English + 38k Japanese Pokémon singles, Yu-Gi-Oh!, One Piece, Flesh and Blood, Star Wars Unlimited, Lorcana and Digimon singles with NM-Mint / Lightly Played / Moderately Played / Heavily Played variants; sealed MTG and Pokémon. | |
| 4 | +- Country/currency: NZ / NZD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, a `titlePattern` that extracts name/set/number, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (12): `mtg-singles-in-stock` → magic_the_gathering, `secret-lair-singles` → magic_the_gathering, `magic-the-gathering-sealed` → magic_the_gathering, `pokemon-tcg-single-cards` → pokemon, `japanese-pokemon-singles` → pokemon, `pokemon-tcg-sealed-packs` → pokemon, `yugioh-single-cards` → yugioh, `one-piece-single-cards` → one_piece_card_game, `flesh-and-blood-single-cards` → flesh_and_blood, `star-wars-unlimited-single-cards` → star_wars_tcg, `disney-lorcana-single-cards` → disney_lorcana, `digimon-single-cards` → digimon_tcg. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/goblingames-nz/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts goblingames-nz`. | |
added
connectors/api/goblingames-nz/index.test.ts
+73 −0
@@ -0,0 +1,73 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('goblingames-nz', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Misdirection [Mercadian Masques] Foil", | |
| 18 | + "collection": "mtg-singles-in-stock", | |
| 19 | + "type": "MTG Single", | |
| 20 | + "variant": "Lightly Played Foil", | |
| 21 | + "expect": "magic_the_gathering", | |
| 22 | + "name": "Misdirection", | |
| 23 | + "set": "Mercadian Masques", | |
| 24 | + "conditionRaw": "Lightly Played" | |
| 25 | + }, | |
| 26 | + { | |
| 27 | + "title": "Forcefield [Unlimited Edition]", | |
| 28 | + "collection": "mtg-singles-in-stock", | |
| 29 | + "type": "MTG Single", | |
| 30 | + "variant": "NM-Mint", | |
| 31 | + "expect": "magic_the_gathering", | |
| 32 | + "set": "Unlimited Edition" | |
| 33 | + }, | |
| 34 | + { | |
| 35 | + "title": "Aaron's Collection 088/111 - Rising Rivals", | |
| 36 | + "collection": "pokemon-tcg-single-cards", | |
| 37 | + "type": "Pokemon Single", | |
| 38 | + "tags": [ | |
| 39 | + "Brand_Pokemon", | |
| 40 | + "Number_88" | |
| 41 | + ], | |
| 42 | + "variant": "Lightly Played", | |
| 43 | + "expect": "pokemon", | |
| 44 | + "set": null, | |
| 45 | + "conditionRaw": "Lightly Played" | |
| 46 | + }, | |
| 47 | + { | |
| 48 | + "title": "Overwhelming Barrage | (092/252) [Spark of Rebellion]", | |
| 49 | + "collection": "star-wars-unlimited-single-cards", | |
| 50 | + "type": "Star Wars: Unlimited Single", | |
| 51 | + "variant": "NM-Mint", | |
| 52 | + "expect": "star_wars_tcg", | |
| 53 | + "name": "Overwhelming Barrage", | |
| 54 | + "set": "Spark of Rebellion", | |
| 55 | + "number": "092/252" | |
| 56 | + }, | |
| 57 | + { | |
| 58 | + "title": "Blue-Eyes White Dragon [SDKS-EN009] Common", | |
| 59 | + "collection": "yugioh-single-cards", | |
| 60 | + "type": "Yugioh Single", | |
| 61 | + "variant": "NM-Mint Unlimited", | |
| 62 | + "expect": "yugioh", | |
| 63 | + "name": "Blue-Eyes White Dragon" | |
| 64 | + }, | |
| 65 | + { | |
| 66 | + "title": "Dragon Shield Dual Matte Sleeves - Ruin", | |
| 67 | + "collection": "magic-the-gathering-sealed", | |
| 68 | + "type": "Accessories", | |
| 69 | + "expect": null | |
| 70 | + } | |
| 71 | + ]); | |
| 72 | + }); | |
| 73 | +}); | |
added
connectors/api/goblingames-nz/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Goblin Games NZ — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/goblingames-nz/meta.json
+125 −0
@@ -0,0 +1,125 @@ | ||
| 1 | +{ | |
| 2 | + "id": "goblingames-nz", | |
| 3 | + "displayName": "Goblin Games NZ", | |
| 4 | + "sourceId": "goblingames-nz", | |
| 5 | + "sourceName": "Goblin Games NZ", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://goblingames.co.nz", | |
| 8 | + "module": "api/goblingames-nz", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "pokemon", | |
| 15 | + "yugioh", | |
| 16 | + "one_piece_card_game", | |
| 17 | + "flesh_and_blood", | |
| 18 | + "star_wars_tcg", | |
| 19 | + "disney_lorcana", | |
| 20 | + "digimon_tcg" | |
| 21 | + ], | |
| 22 | + "regions": [ | |
| 23 | + "NZ" | |
| 24 | + ], | |
| 25 | + "country": "NZ", | |
| 26 | + "languages": [ | |
| 27 | + "en" | |
| 28 | + ], | |
| 29 | + "currency": [ | |
| 30 | + "NZD" | |
| 31 | + ], | |
| 32 | + "supportsListings": true, | |
| 33 | + "supportsSold": false, | |
| 34 | + "supportsAuctions": false, | |
| 35 | + "supportsImages": true, | |
| 36 | + "supportsCatalog": false, | |
| 37 | + "supportsPopulation": false, | |
| 38 | + "supportsLookup": true, | |
| 39 | + "refreshFrequencyMinutes": 720, | |
| 40 | + "priority": "medium", | |
| 41 | + "trustScore": 0.65, | |
| 42 | + "attributionRequired": true, | |
| 43 | + "termsUrl": "https://goblingames.co.nz/policies/terms-of-service", | |
| 44 | + "accessNotes": "Goblin Games NZ (goblingames.co.nz) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (mtg-singles-in-stock, secret-lair-singles, magic-the-gathering-sealed, pokemon-tcg-single-cards, japanese-pokemon-singles, pokemon-tcg-sealed-packs, yugioh-single-cards, one-piece-single-cards, flesh-and-blood-single-cards, star-wars-unlimited-single-cards, disney-lorcana-single-cards, digimon-single-cards). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency NZD (Shopify.currency active NZD with rate 1.0; the shop /meta.json endpoint returns 500), GST included. Titles mix BinderPOS \"Name [Set]\" and \"Name 088/111 - Set Printing\" styles; only the bracket style yields set extraction. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 45 | + "enabled": true, | |
| 46 | + "schemaVersion": "1.0", | |
| 47 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 48 | + "historicalDepth": "none", | |
| 49 | + "requires": [], | |
| 50 | + "config": { | |
| 51 | + "currency": "NZD", | |
| 52 | + "seller": "Goblin Games NZ", | |
| 53 | + "location": null, | |
| 54 | + "collections": [ | |
| 55 | + { | |
| 56 | + "handle": "mtg-singles-in-stock", | |
| 57 | + "categorySlug": "magic_the_gathering", | |
| 58 | + "franchise": "Magic: The Gathering" | |
| 59 | + }, | |
| 60 | + { | |
| 61 | + "handle": "secret-lair-singles", | |
| 62 | + "categorySlug": "magic_the_gathering", | |
| 63 | + "franchise": "Magic: The Gathering" | |
| 64 | + }, | |
| 65 | + { | |
| 66 | + "handle": "magic-the-gathering-sealed", | |
| 67 | + "categorySlug": "magic_the_gathering", | |
| 68 | + "franchise": "Magic: The Gathering" | |
| 69 | + }, | |
| 70 | + { | |
| 71 | + "handle": "pokemon-tcg-single-cards", | |
| 72 | + "categorySlug": "pokemon", | |
| 73 | + "franchise": "Pokémon" | |
| 74 | + }, | |
| 75 | + { | |
| 76 | + "handle": "japanese-pokemon-singles", | |
| 77 | + "categorySlug": "pokemon", | |
| 78 | + "franchise": "Pokémon" | |
| 79 | + }, | |
| 80 | + { | |
| 81 | + "handle": "pokemon-tcg-sealed-packs", | |
| 82 | + "categorySlug": "pokemon", | |
| 83 | + "franchise": "Pokémon" | |
| 84 | + }, | |
| 85 | + { | |
| 86 | + "handle": "yugioh-single-cards", | |
| 87 | + "categorySlug": "yugioh", | |
| 88 | + "franchise": "Yu-Gi-Oh!" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "handle": "one-piece-single-cards", | |
| 92 | + "categorySlug": "one_piece_card_game", | |
| 93 | + "franchise": "One Piece" | |
| 94 | + }, | |
| 95 | + { | |
| 96 | + "handle": "flesh-and-blood-single-cards", | |
| 97 | + "categorySlug": "flesh_and_blood", | |
| 98 | + "franchise": "Flesh and Blood" | |
| 99 | + }, | |
| 100 | + { | |
| 101 | + "handle": "star-wars-unlimited-single-cards", | |
| 102 | + "categorySlug": "star_wars_tcg", | |
| 103 | + "franchise": "Star Wars" | |
| 104 | + }, | |
| 105 | + { | |
| 106 | + "handle": "disney-lorcana-single-cards", | |
| 107 | + "categorySlug": "disney_lorcana", | |
| 108 | + "franchise": "Disney Lorcana" | |
| 109 | + }, | |
| 110 | + { | |
| 111 | + "handle": "digimon-single-cards", | |
| 112 | + "categorySlug": "digimon_tcg", | |
| 113 | + "franchise": "Digimon" | |
| 114 | + } | |
| 115 | + ], | |
| 116 | + "rules": [], | |
| 117 | + "defaultCategory": null, | |
| 118 | + "exclude": "gift card|\\bsleeves?\\b|deck box|deck case|deck holder|binder|playmat|play mat|toploader|top loader|storage box|card case|portfolio|\\balbum\\b|\\bdice\\b|spindown|life counter|tokens? only|bundle of|mystery (box|pack|bundle|bag)|repack|\\bticket|event entry|entry fee|weekly league|display case|acrylic (stand|display|case)|card holder|card protector|magnetic holder|semi-rigid|perfect fit|penny sleeve|team bag|card saver|\\baccessor|dice set|d20|d6\\b|\\bpaint|brush|primer|glue|hobby tool|cutter|tweezers|sticker|keyring|key ring|lanyard|badge|coaster|mug\\b|t-shirt|hoodie|\\bcap\\b|socks|towel|blanket|poster|wall scroll|calendar|advent|\\| accessories \\||plush|dragon shield|ultimate guard|gamegenic|chessex", | |
| 119 | + "keepOutOfStock": true, | |
| 120 | + "titlePattern": "^(?<name>.+?)\\s*(?:\\|\\s*)?(?:\\((?<number>\\d+/\\d+)\\)\\s*)?\\[(?<set>[^\\]]+)\\]", | |
| 121 | + "pageSize": 250, | |
| 122 | + "fetchBarcodes": false, | |
| 123 | + "wholeShop": false | |
| 124 | + } | |
| 125 | +} | |
added
connectors/api/goodgames/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Good Games connector (`goodgames`) | |
| 2 | + | |
| 3 | +- Source: https://www.goodgames.com.au · Australian games retailer chain (Sydney): sealed Pokémon, MTG, Yu-Gi-Oh!, One Piece, Digimon, Flesh and Blood, Lorcana, Star Wars Unlimited; Panini sports boxes; anime figures, action figures, Funko, blind boxes, Gunpla and plush. EAN barcodes appear in SKUs. | |
| 4 | +- Country/currency: AU / AUD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 24 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (17): `pokemon` → pokemon, `magic-the-gathering` → magic_the_gathering, `yu-gi-oh` → yugioh, `one-piece-card-game` → one_piece_card_game, `digimon` → digimon_tcg, `flesh-blood` → flesh_and_blood, `dragonball-super` → dragon_ball_tcg, `disney-lorcana-tcg` → disney_lorcana, `star-wars-unlimited` → star_wars_tcg, `gundam-card-game` → other_tcg, `sports-cards` → other_sports_cards, `funko` → funko, `anime-figure` → action_figures, `action-toy-figures` → action_figures, `blind-boxes` → designer_toys, `gundam` → gundam, `plush` → plush. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/goodgames/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts goodgames`. | |
added
connectors/api/goodgames/index.test.ts
+82 −0
@@ -0,0 +1,82 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('goodgames', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Pokemon TCG - Paldea Partners Tin", | |
| 18 | + "collection": "pokemon", | |
| 19 | + "type": "Trading Card Games", | |
| 20 | + "tags": [ | |
| 21 | + "TCG_PKM" | |
| 22 | + ], | |
| 23 | + "variant": "Meowscarada Ex", | |
| 24 | + "expect": "pokemon" | |
| 25 | + }, | |
| 26 | + { | |
| 27 | + "title": "Magic: The Gathering Final Fantasy Play Booster Box", | |
| 28 | + "collection": "magic-the-gathering", | |
| 29 | + "type": "Trading Card Games", | |
| 30 | + "tags": [ | |
| 31 | + "TCG_MTG" | |
| 32 | + ], | |
| 33 | + "expect": "magic_the_gathering" | |
| 34 | + }, | |
| 35 | + { | |
| 36 | + "title": "2022 - 2023 Donruss Basketball Booster Box", | |
| 37 | + "collection": "sports-cards", | |
| 38 | + "type": "Sports Cards", | |
| 39 | + "tags": [ | |
| 40 | + "Basketball", | |
| 41 | + "Sports Cards" | |
| 42 | + ], | |
| 43 | + "expect": "basketball_cards" | |
| 44 | + }, | |
| 45 | + { | |
| 46 | + "title": "PANINI 202223 Donruss Soccer Fat Pack", | |
| 47 | + "collection": "sports-cards", | |
| 48 | + "type": "Sports Cards", | |
| 49 | + "tags": [ | |
| 50 | + "Soccer" | |
| 51 | + ], | |
| 52 | + "expect": "soccer_cards" | |
| 53 | + }, | |
| 54 | + { | |
| 55 | + "title": "Spy Family PM Figure - Yor Forger (Plain Clothes)", | |
| 56 | + "collection": "anime-figure", | |
| 57 | + "type": "Anime Figures", | |
| 58 | + "vendor": "Sega", | |
| 59 | + "expect": "action_figures", | |
| 60 | + "brand": "Sega" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "title": "Gundam Card Game - Steel Requiem [GD03] Booster Box", | |
| 64 | + "collection": "gundam-card-game", | |
| 65 | + "type": "Trading Card Games", | |
| 66 | + "expect": "other_tcg" | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "title": "Dr Seuss - Merry Grinchmas Game", | |
| 70 | + "collection": "funko", | |
| 71 | + "type": "Board Games", | |
| 72 | + "expect": null | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "title": "Magic: The Gathering Repack Booster", | |
| 76 | + "collection": "magic-the-gathering", | |
| 77 | + "type": "ZZ - Other", | |
| 78 | + "expect": null | |
| 79 | + } | |
| 80 | + ]); | |
| 81 | + }); | |
| 82 | +}); | |
added
connectors/api/goodgames/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Good Games — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/goodgames/meta.json
+188 −0
@@ -0,0 +1,188 @@ | ||
| 1 | +{ | |
| 2 | + "id": "goodgames", | |
| 3 | + "displayName": "Good Games", | |
| 4 | + "sourceId": "goodgames", | |
| 5 | + "sourceName": "Good Games", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.goodgames.com.au", | |
| 8 | + "module": "api/goodgames", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "pokemon", | |
| 14 | + "magic_the_gathering", | |
| 15 | + "yugioh", | |
| 16 | + "one_piece_card_game", | |
| 17 | + "digimon_tcg", | |
| 18 | + "flesh_and_blood", | |
| 19 | + "dragon_ball_tcg", | |
| 20 | + "disney_lorcana", | |
| 21 | + "star_wars_tcg", | |
| 22 | + "other_tcg", | |
| 23 | + "other_sports_cards", | |
| 24 | + "funko", | |
| 25 | + "action_figures", | |
| 26 | + "designer_toys", | |
| 27 | + "gundam", | |
| 28 | + "plush", | |
| 29 | + "basketball_cards", | |
| 30 | + "football_cards", | |
| 31 | + "soccer_cards", | |
| 32 | + "baseball_cards", | |
| 33 | + "hockey_cards", | |
| 34 | + "f1_cards" | |
| 35 | + ], | |
| 36 | + "regions": [ | |
| 37 | + "AU" | |
| 38 | + ], | |
| 39 | + "country": "AU", | |
| 40 | + "languages": [ | |
| 41 | + "en" | |
| 42 | + ], | |
| 43 | + "currency": [ | |
| 44 | + "AUD" | |
| 45 | + ], | |
| 46 | + "supportsListings": true, | |
| 47 | + "supportsSold": false, | |
| 48 | + "supportsAuctions": false, | |
| 49 | + "supportsImages": true, | |
| 50 | + "supportsCatalog": false, | |
| 51 | + "supportsPopulation": false, | |
| 52 | + "supportsLookup": true, | |
| 53 | + "refreshFrequencyMinutes": 1440, | |
| 54 | + "priority": "medium", | |
| 55 | + "trustScore": 0.65, | |
| 56 | + "attributionRequired": true, | |
| 57 | + "termsUrl": "https://www.goodgames.com.au/policies/terms-of-service", | |
| 58 | + "accessNotes": "Good Games (goodgames.com.au) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (pokemon, magic-the-gathering, yu-gi-oh, one-piece-card-game, digimon, flesh-blood, dragonball-super, disney-lorcana-tcg, star-wars-unlimited, gundam-card-game, sports-cards, funko, anime-figure, action-toy-figures, blind-boxes, gundam, plush). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, once a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency AUD (/meta.json currency AUD, Shopify.currency rate 1.0), GST included. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 59 | + "enabled": true, | |
| 60 | + "schemaVersion": "1.0", | |
| 61 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 62 | + "historicalDepth": "none", | |
| 63 | + "requires": [], | |
| 64 | + "config": { | |
| 65 | + "currency": "AUD", | |
| 66 | + "seller": "Good Games", | |
| 67 | + "location": null, | |
| 68 | + "collections": [ | |
| 69 | + { | |
| 70 | + "handle": "pokemon", | |
| 71 | + "categorySlug": "pokemon", | |
| 72 | + "franchise": "Pokémon" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "handle": "magic-the-gathering", | |
| 76 | + "categorySlug": "magic_the_gathering", | |
| 77 | + "franchise": "Magic: The Gathering" | |
| 78 | + }, | |
| 79 | + { | |
| 80 | + "handle": "yu-gi-oh", | |
| 81 | + "categorySlug": "yugioh", | |
| 82 | + "franchise": "Yu-Gi-Oh!" | |
| 83 | + }, | |
| 84 | + { | |
| 85 | + "handle": "one-piece-card-game", | |
| 86 | + "categorySlug": "one_piece_card_game", | |
| 87 | + "franchise": "One Piece" | |
| 88 | + }, | |
| 89 | + { | |
| 90 | + "handle": "digimon", | |
| 91 | + "categorySlug": "digimon_tcg", | |
| 92 | + "franchise": "Digimon" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "handle": "flesh-blood", | |
| 96 | + "categorySlug": "flesh_and_blood", | |
| 97 | + "franchise": "Flesh and Blood" | |
| 98 | + }, | |
| 99 | + { | |
| 100 | + "handle": "dragonball-super", | |
| 101 | + "categorySlug": "dragon_ball_tcg", | |
| 102 | + "franchise": "Dragon Ball" | |
| 103 | + }, | |
| 104 | + { | |
| 105 | + "handle": "disney-lorcana-tcg", | |
| 106 | + "categorySlug": "disney_lorcana", | |
| 107 | + "franchise": "Disney Lorcana" | |
| 108 | + }, | |
| 109 | + { | |
| 110 | + "handle": "star-wars-unlimited", | |
| 111 | + "categorySlug": "star_wars_tcg", | |
| 112 | + "franchise": "Star Wars" | |
| 113 | + }, | |
| 114 | + { | |
| 115 | + "handle": "gundam-card-game", | |
| 116 | + "categorySlug": "other_tcg", | |
| 117 | + "franchise": "Gundam" | |
| 118 | + }, | |
| 119 | + { | |
| 120 | + "handle": "sports-cards", | |
| 121 | + "categorySlug": "other_sports_cards" | |
| 122 | + }, | |
| 123 | + { | |
| 124 | + "handle": "funko", | |
| 125 | + "categorySlug": "funko", | |
| 126 | + "brand": "Funko" | |
| 127 | + }, | |
| 128 | + { | |
| 129 | + "handle": "anime-figure", | |
| 130 | + "categorySlug": "action_figures" | |
| 131 | + }, | |
| 132 | + { | |
| 133 | + "handle": "action-toy-figures", | |
| 134 | + "categorySlug": "action_figures" | |
| 135 | + }, | |
| 136 | + { | |
| 137 | + "handle": "blind-boxes", | |
| 138 | + "categorySlug": "designer_toys" | |
| 139 | + }, | |
| 140 | + { | |
| 141 | + "handle": "gundam", | |
| 142 | + "categorySlug": "gundam", | |
| 143 | + "brand": "Bandai", | |
| 144 | + "franchise": "Gundam" | |
| 145 | + }, | |
| 146 | + { | |
| 147 | + "handle": "plush", | |
| 148 | + "categorySlug": "plush" | |
| 149 | + } | |
| 150 | + ], | |
| 151 | + "rules": [ | |
| 152 | + { | |
| 153 | + "match": "\\bnba\\b|basketball|wnba", | |
| 154 | + "categorySlug": "basketball_cards" | |
| 155 | + }, | |
| 156 | + { | |
| 157 | + "match": "\\bnfl\\b|american football", | |
| 158 | + "categorySlug": "football_cards" | |
| 159 | + }, | |
| 160 | + { | |
| 161 | + "match": "soccer|premier league|uefa|champions league|world cup|\\bepl\\b|bundesliga|la liga|serie a|match attax|\\bfifa\\b|road to (world cup|euro)", | |
| 162 | + "categorySlug": "soccer_cards" | |
| 163 | + }, | |
| 164 | + { | |
| 165 | + "match": "\\bmlb\\b|baseball", | |
| 166 | + "categorySlug": "baseball_cards" | |
| 167 | + }, | |
| 168 | + { | |
| 169 | + "match": "\\bnhl\\b|hockey", | |
| 170 | + "categorySlug": "hockey_cards" | |
| 171 | + }, | |
| 172 | + { | |
| 173 | + "match": "\\bf1\\b|formula (1|one)|grand prix", | |
| 174 | + "categorySlug": "f1_cards" | |
| 175 | + }, | |
| 176 | + { | |
| 177 | + "match": "\\bufc\\b|wrestling|\\bwwe\\b|\\baew\\b|\\bafl\\b|\\bnrl\\b|tennis|\\bgolf\\b|cricket|rugby|boxing|nascar|racing|olympic", | |
| 178 | + "categorySlug": "other_sports_cards" | |
| 179 | + } | |
| 180 | + ], | |
| 181 | + "defaultCategory": null, | |
| 182 | + "exclude": "gift card|\\bsleeves?\\b|deck box|deck case|deck holder|binder|playmat|play mat|toploader|top loader|storage box|card case|portfolio|\\balbum\\b|\\bdice\\b|spindown|life counter|tokens? only|bundle of|mystery (box|pack|bundle|bag)|repack|\\bticket|event entry|entry fee|weekly league|display case|acrylic (stand|display|case)|card holder|card protector|magnetic holder|semi-rigid|perfect fit|penny sleeve|team bag|card saver|\\baccessor|dice set|d20|d6\\b|\\bpaint|brush|primer|glue|hobby tool|cutter|tweezers|sticker|keyring|key ring|lanyard|badge|coaster|mug\\b|t-shirt|hoodie|\\bcap\\b|socks|towel|blanket|poster|wall scroll|calendar|advent|folder|board game|jigsaw|puzzle|\\| board games \\||\\| zz - other \\|", | |
| 183 | + "keepOutOfStock": true, | |
| 184 | + "pageSize": 250, | |
| 185 | + "fetchBarcodes": false, | |
| 186 | + "wholeShop": false | |
| 187 | + } | |
| 188 | +} | |
added
connectors/api/hareruya/README.md
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +# hareruya | |
| 2 | + | |
| 3 | +Hareruya (晴れる屋, hareruyamtg.com) — Japan's largest MTG shop, EC-CUBE storefront with an English UI. | |
| 4 | + | |
| 5 | +- Card sets: `<select name="cardset">` of the public search form `https://www.hareruyamtg.com/en/products/search` (no query string). | |
| 6 | +- Listings: the JSON endpoint the search page itself loads, `GET /en/products/search/unisearch_api?fq.category_id=1&fq.cardset=<id>&fq.price=1~*&rows=60&page=<n>` → docs with product id, `《Name》[SET]`, card_name, language code, JPY price, foil flag, stock, weekly_sales, product_class, condition code. | |
| 7 | + | |
| 8 | +robots.txt disallows the HTML result pages with `page=`/`sort=`/`order=` — we never request those; only the JSON endpoint, ≥ 4 s apart. | |
| 9 | + | |
| 10 | +Output: `listing` (JPY, seller Hareruya, `Foil` variant, language from the code table, condition left null — the numeric code goes to `metadata.condition_code`). Cursor `{ setIdx, page }`; incremental = `setsPerRun` newest sets, backfill = all (site caps a query at 4 000 results). | |
| 11 | +Fixtures: `pnpm tsx connectors/api/hareruya/_capture.ts`. | |
added
connectors/api/hareruya/_capture.ts
+37 −0
@@ -0,0 +1,37 @@ | ||
| 1 | +/** Live fixture capture for hareruya: search-form cardsets + two unisearch_api pages. Usage: pnpm tsx connectors/api/hareruya/_capture.ts */ | |
| 2 | +import { mkdirSync, writeFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import { fixtureDir, saveFixture } from '@rareindex/connectors/testing'; | |
| 5 | +import { DocSchema, apiUrl, parseCardsets, type HareruyaPayload } from './index.js'; | |
| 6 | +import { HTML_HEADERS, JSON_HEADERS } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 7 | + | |
| 8 | +const SITE = 'https://www.hareruyamtg.com'; | |
| 9 | +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)); | |
| 10 | +mkdirSync(fixtureDir('hareruya'), { recursive: true }); | |
| 11 | + | |
| 12 | +const form = await (await fetch(`${SITE}/en/products/search`, { headers: HTML_HEADERS })).text(); | |
| 13 | +writeFileSync(path.join(fixtureDir('hareruya'), 'search-form.html'), form); | |
| 14 | +const cardsets = parseCardsets(form); | |
| 15 | +console.log('cardsets', cardsets.length, cardsets.slice(0, 3)); | |
| 16 | + | |
| 17 | +async function page(cardset: { id: string; name: string }, pageNo: number, name: string, note: string, keep = 15) { | |
| 18 | + await wait(4000); | |
| 19 | + const url = apiUrl(cardset.id, pageNo, 60); | |
| 20 | + const json = (await (await fetch(url, { headers: { ...JSON_HEADERS, referer: `${SITE}/en/products/search?cardset=${cardset.id}` } })).json()) as { response: { numFound: number; page: number | string; docs: unknown[] } }; | |
| 21 | + const docs = json.response.docs.map((d) => DocSchema.parse(d)).slice(0, keep); | |
| 22 | + const payload: HareruyaPayload = { cardset, page: pageNo, numFound: json.response.numFound, docs }; | |
| 23 | + saveFixture('hareruya', name, { | |
| 24 | + raw: { url, externalId: `${cardset.id}:p${pageNo}`, kind: 'listing', engine: 'api', fetchedAt: new Date(), payload }, | |
| 25 | + expect: { minCount: Math.min(keep, docs.length), kinds: ['listing'], requiredFields: ['price', 'attributes.name', 'attributes.identifiers.hareruya_product_id'] }, | |
| 26 | + note: `Live capture ${new Date().toISOString().slice(0, 10)} — ${url} (${note}; numFound ${json.response.numFound}, first ${docs.length} docs kept)`, | |
| 27 | + }); | |
| 28 | + console.log(name, cardset, json.response.numFound, docs[0]); | |
| 29 | + return json.response.numFound; | |
| 30 | +} | |
| 31 | + | |
| 32 | +const newest = cardsets[0]!; | |
| 33 | +await page(newest, 1, 'newest-set-page1', `newest card set in the form: ${newest.name}`); | |
| 34 | +// a set with several languages/foils: find one with a large numFound among the first options | |
| 35 | +const big = cardsets.find((c) => /Foundations|Bloomburrow|Duskmourn|Aetherdrift|Tarkir/i.test(c.name)) ?? cardsets[3]!; | |
| 36 | +const n = await page(big, 1, 'multi-language-page1', `set ${big.name} with EN/JP/foil offers`); | |
| 37 | +if (n > 60) await page(big, 2, 'multi-language-page2', `page 2 of ${big.name} (pagination)`, 8); | |
added
connectors/api/hareruya/index.test.ts
+58 −0
@@ -0,0 +1,58 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import meta from './meta.json' with { type: 'json' }; | |
| 5 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 6 | +import { fixtureDir, listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 7 | +import createConnector, { LANGUAGES, apiUrl, parseCardsets, parseProductName } from './index.js'; | |
| 8 | + | |
| 9 | +const connector = createConnector(localMeta(meta)); | |
| 10 | + | |
| 11 | +describe('hareruya', () => { | |
| 12 | + runFixtureSuite(connector, it, expect); | |
| 13 | + | |
| 14 | + it('parses product names and builds the JSON endpoint URL the page uses', () => { | |
| 15 | + expect(parseProductName('《Black Lotus》[2ED]')).toEqual({ name: 'Black Lotus', setCode: '2ED', number: null }); | |
| 16 | + expect(parseProductName('《Black Lotus》[2ED] 茶R')).toEqual({ name: 'Black Lotus', setCode: '2ED', number: null }); | |
| 17 | + expect(parseProductName('(001)《Ugin, Eye of the Storms》[TDM]')).toEqual({ name: 'Ugin, Eye of the Storms', setCode: 'TDM', number: '001' }); | |
| 18 | + expect(parseProductName(null)).toEqual({ name: null, setCode: null, number: null }); | |
| 19 | + expect(apiUrl('429', 2, 60)).toBe('https://www.hareruyamtg.com/en/products/search/unisearch_api?fq.category_id=1&fq.cardset=429&fq.price=1%7E%2A&rows=60&page=2'); | |
| 20 | + expect(LANGUAGES['1']).toBe('Japanese'); | |
| 21 | + expect(LANGUAGES['2']).toBe('English'); | |
| 22 | + }); | |
| 23 | + | |
| 24 | + it('reads the card-set options from the saved public search form', () => { | |
| 25 | + const sets = parseCardsets(readFileSync(path.join(fixtureDir('hareruya'), 'search-form.html'), 'utf8')); | |
| 26 | + expect(sets.length).toBeGreaterThan(100); | |
| 27 | + expect(sets.every((s) => /^\d+$/.test(s.id))).toBe(true); | |
| 28 | + expect(sets[0]!.name.length).toBeGreaterThan(2); | |
| 29 | + }); | |
| 30 | + | |
| 31 | + it('emits one JPY listing per offer with language, foil variant and set', async () => { | |
| 32 | + for (const name of listFixtures('hareruya')) { | |
| 33 | + const fx = loadFixture('hareruya', name); | |
| 34 | + const out = await connector.normalize(fx.raw); | |
| 35 | + const docs = (fx.raw.payload as { docs: unknown[] }).docs; | |
| 36 | + if (!docs.length) { | |
| 37 | + // empty result page (a set announced but not yet stocked) → no records, no crash | |
| 38 | + expect(out).toEqual([]); | |
| 39 | + continue; | |
| 40 | + } | |
| 41 | + expect(out.length).toBeGreaterThan(0); | |
| 42 | + const ids = new Set<string>(); | |
| 43 | + for (const r of out) { | |
| 44 | + if (r.kind !== 'listing') continue; | |
| 45 | + expect(r.currency).toBe('JPY'); | |
| 46 | + expect(r.price).toBeGreaterThan(0); | |
| 47 | + expect(Number.isInteger(r.price)).toBe(true); | |
| 48 | + expect(r.seller).toBe('Hareruya'); | |
| 49 | + expect(r.attributes.set).toBe((fx.raw.payload as { cardset: { name: string } }).cardset.name); | |
| 50 | + expect(r.sourceUrl).toMatch(/^https:\/\/www\.hareruyamtg\.com\/en\/products\/detail\/\d+\?lang=[A-Z]{2}/); | |
| 51 | + expect(r.condition.condition).toBeNull(); // condition codes are never guessed | |
| 52 | + expect(ids.has(r.externalId!)).toBe(false); | |
| 53 | + ids.add(r.externalId!); | |
| 54 | + if (r.attributes.variant) expect(r.attributes.variant).toBe('Foil'); | |
| 55 | + } | |
| 56 | + } | |
| 57 | + }); | |
| 58 | +}); | |
added
connectors/api/hareruya/index.ts
+193 −0
@@ -0,0 +1,193 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { attrs, makeTitle } from '../_lib/shared.js'; | |
| 5 | +import { HTML_HEADERS, JSON_HEADERS, cleanText, yen } from '../_g1-cards-eu-jp-lib/index.js'; | |
| 6 | + | |
| 7 | +/** | |
| 8 | + * Hareruya (晴れる屋) — Japan's largest MTG shop. The public search page loads its results from a | |
| 9 | + * JSON endpoint (unisearch_api); we read that endpoint per card set. One raw record per API page; | |
| 10 | + * normalize emits one JPY listing per offer (product × language × condition × foil). | |
| 11 | + */ | |
| 12 | +const SITE = 'https://www.hareruyamtg.com'; | |
| 13 | +const API = `${SITE}/en/products/search/unisearch_api`; | |
| 14 | +const PARSER_VERSION = '1.0.0'; | |
| 15 | + | |
| 16 | +export const LANGUAGES: Record<string, string> = { '1': 'Japanese', '2': 'English', '3': 'Chinese (Simplified)', '4': 'Chinese (Traditional)', '5': 'French', '6': 'German', '7': 'Italian', '8': 'Korean', '9': 'Russian', '10': 'Spanish', '11': 'Portuguese' }; | |
| 17 | +const LANG_CODE: Record<string, string> = { '1': 'JP', '2': 'EN', '3': 'CS', '4': 'CT', '5': 'FR', '6': 'DE', '7': 'IT', '8': 'KO', '9': 'RU', '10': 'ES', '11': 'PT' }; | |
| 18 | + | |
| 19 | +const str = z.union([z.string(), z.number()]).transform((v) => String(v)); | |
| 20 | +export const DocSchema = z.object({ | |
| 21 | + product: str, | |
| 22 | + product_name: z.string().nullable().optional(), | |
| 23 | + product_name_en: z.string().nullable().optional(), | |
| 24 | + card_name: z.string().nullable().optional(), | |
| 25 | + language: str.nullable().optional(), | |
| 26 | + price: str.nullable().optional(), | |
| 27 | + image_url: z.string().nullable().optional(), | |
| 28 | + foil_flg: str.nullable().optional(), | |
| 29 | + stock: str.nullable().optional(), | |
| 30 | + weekly_sales: str.nullable().optional(), | |
| 31 | + product_class: str.nullable().optional(), | |
| 32 | + card_condition: str.nullable().optional(), | |
| 33 | + sale_flg: str.nullable().optional(), | |
| 34 | + high_price_code: str.nullable().optional(), | |
| 35 | +}); | |
| 36 | +export type HareruyaDoc = z.infer<typeof DocSchema>; | |
| 37 | +const RawPayloadSchema = z.object({ cardset: z.object({ id: z.string(), name: z.string() }), page: z.number().int(), numFound: z.number().int().nullable(), docs: z.array(DocSchema) }); | |
| 38 | +export type HareruyaPayload = z.infer<typeof RawPayloadSchema>; | |
| 39 | +const ApiResponseSchema = z.object({ response: z.object({ numFound: z.number().int().optional(), page: z.union([z.number(), z.string()]).optional(), docs: z.array(z.unknown()).default([]) }) }); | |
| 40 | + | |
| 41 | +/** Card-set options of the public search form. */ | |
| 42 | +export function parseCardsets(doc: string): Array<{ id: string; name: string }> { | |
| 43 | + const $ = html.load(doc); | |
| 44 | + const out: Array<{ id: string; name: string }> = []; | |
| 45 | + $('select[name="cardset"] option').each((_, el) => { | |
| 46 | + const id = $(el).attr('value')?.trim(); | |
| 47 | + const name = cleanText($(el).text()); | |
| 48 | + if (id && name) out.push({ id, name }); | |
| 49 | + }); | |
| 50 | + return out; | |
| 51 | +} | |
| 52 | + | |
| 53 | +/** "(001)《Ugin, Eye of the Storms》[TDM]" → { number: "001", name: "Ugin, Eye of the Storms", setCode: "TDM" }; "《Black Lotus》[2ED] 茶R" keeps only those parts. */ | |
| 54 | +export function parseProductName(s: string | null | undefined): { name: string | null; setCode: string | null; number: string | null } { | |
| 55 | + if (!s) return { name: null, setCode: null, number: null }; | |
| 56 | + const name = s.match(/《([^》]+)》/)?.[1]?.trim() ?? null; | |
| 57 | + const setCode = s.match(/\[([A-Za-z0-9_+-]+)\]/)?.[1]?.trim() ?? null; | |
| 58 | + const number = s.match(/^\s*\(([A-Za-z0-9]{1,4}[-/]?[A-Za-z0-9]{0,4})\)\s*《/)?.[1] ?? null; | |
| 59 | + return { name, setCode, number }; | |
| 60 | +} | |
| 61 | + | |
| 62 | +export function apiUrl(cardsetId: string, page: number, rows: number): string { | |
| 63 | + return `${API}?fq.category_id=1&fq.cardset=${encodeURIComponent(cardsetId)}&fq.price=1%7E%2A&rows=${rows}&page=${page}`; | |
| 64 | +} | |
| 65 | + | |
| 66 | +export class HareruyaConnector extends BaseConnector { | |
| 67 | + readonly version = '1.0.0'; | |
| 68 | + readonly parserVersion = PARSER_VERSION; | |
| 69 | + protected override minIntervalMs = 4000; | |
| 70 | + | |
| 71 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 72 | + const rows = Number(this.meta.config.rows ?? 60); | |
| 73 | + const backfill = ctx.options.mode === 'backfill'; | |
| 74 | + const setsPerRun = backfill ? Infinity : Number(this.meta.config.setsPerRun ?? 12); | |
| 75 | + const maxPages = Math.min(Number(this.meta.config.maxPagesPerSet ?? 67), Math.ceil(4000 / rows)); | |
| 76 | + let cardsets = (this.meta.config.cardsets as Array<{ id: string; name: string }> | undefined) ?? []; | |
| 77 | + if (!cardsets.length) { | |
| 78 | + const formUrl = `${SITE}/en/products/search`; | |
| 79 | + await this.throttle(formUrl); | |
| 80 | + const form = await ctx.fetch(formUrl, { engines: ['api'], headers: HTML_HEADERS, responseType: 'text', minQuality: 0 }); | |
| 81 | + if (!form.success || !form.html) throw new Error(`hareruya search form failed: ${form.error ?? form.httpStatus}`); | |
| 82 | + cardsets = parseCardsets(form.html); | |
| 83 | + if (!cardsets.length) throw new Error('hareruya: no cardset options found (selector_missing)'); | |
| 84 | + } | |
| 85 | + if (ctx.options.seeds?.length) cardsets = cardsets.filter((c) => ctx.options.seeds!.includes(c.id) || ctx.options.seeds!.includes(c.name)); | |
| 86 | + cardsets = cardsets.slice(0, Number.isFinite(setsPerRun) ? setsPerRun : cardsets.length); | |
| 87 | + let setIdx = Number(ctx.options.cursor?.setIdx ?? 0); | |
| 88 | + let page = Number(ctx.options.cursor?.page ?? 1); | |
| 89 | + let count = 0; | |
| 90 | + for (; setIdx < cardsets.length; setIdx++, page = 1) { | |
| 91 | + const cardset = cardsets[setIdx]!; | |
| 92 | + for (; page <= maxPages; page++) { | |
| 93 | + if (ctx.signal?.aborted) return; | |
| 94 | + if (this.reached(ctx, count)) { | |
| 95 | + await ctx.setCursor({ setIdx, page }); | |
| 96 | + return; | |
| 97 | + } | |
| 98 | + const url = apiUrl(cardset.id, page, rows); | |
| 99 | + await this.throttle(url); | |
| 100 | + const res = await ctx.fetch(url, { engines: ['api'], headers: { ...JSON_HEADERS, referer: `${SITE}/en/products/search?cardset=${cardset.id}` }, minQuality: 0 }); | |
| 101 | + const parsed = ApiResponseSchema.safeParse(res.json); | |
| 102 | + if (!res.success || !parsed.success) { | |
| 103 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? 'unexpected response shape'}`); | |
| 104 | + break; | |
| 105 | + } | |
| 106 | + const docs: HareruyaDoc[] = []; | |
| 107 | + let bad = 0; | |
| 108 | + for (const d of parsed.data.response.docs) { | |
| 109 | + const p = DocSchema.safeParse(d); | |
| 110 | + if (p.success) docs.push(p.data); | |
| 111 | + else bad++; | |
| 112 | + } | |
| 113 | + if (bad) ctx.anomaly('schema_drift', `${url}: ${bad} docs failed DocSchema`); | |
| 114 | + const numFound = parsed.data.response.numFound ?? null; | |
| 115 | + if (!docs.length) { | |
| 116 | + if (page === 1 && (numFound ?? 0) > 0) ctx.anomaly('parse_failure_page', url); | |
| 117 | + break; | |
| 118 | + } | |
| 119 | + count++; | |
| 120 | + const payload: HareruyaPayload = { cardset, page, numFound, docs }; | |
| 121 | + yield { url, externalId: `${cardset.id}:p${page}`, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; | |
| 122 | + await ctx.setCursor({ setIdx, page: page + 1 }); | |
| 123 | + const total = Math.min(numFound ?? 0, 4000); | |
| 124 | + await ctx.progress({ page, totalPages: total ? Math.ceil(total / rows) : null, itemsProcessed: count }); | |
| 125 | + if (numFound !== null && page * rows >= total) break; | |
| 126 | + } | |
| 127 | + } | |
| 128 | + await ctx.setCursor({ setIdx: 0, page: 1, completedAt: new Date().toISOString(), done: true }); | |
| 129 | + } | |
| 130 | + | |
| 131 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 132 | + const p = RawPayloadSchema.parse(raw.payload); | |
| 133 | + const out: NormalizedRecord[] = []; | |
| 134 | + const observedAt = raw.fetchedAt; | |
| 135 | + const seen = new Set<string>(); | |
| 136 | + for (const d of p.docs) { | |
| 137 | + const price = yen(d.price); | |
| 138 | + if (price === null) continue; | |
| 139 | + const pn = parseProductName(d.product_name_en ?? d.product_name); | |
| 140 | + const name = d.card_name?.trim() || pn.name; | |
| 141 | + if (!name) continue; | |
| 142 | + const langCode = d.language ?? null; | |
| 143 | + const language = langCode ? (LANGUAGES[langCode] ?? null) : null; | |
| 144 | + const foil = d.foil_flg === '1'; | |
| 145 | + const externalId = `${d.product}:${d.product_class ?? `${langCode ?? 'x'}-${d.card_condition ?? 'x'}-${foil ? 'f' : 'n'}`}`; | |
| 146 | + if (seen.has(externalId)) continue; | |
| 147 | + seen.add(externalId); | |
| 148 | + const stock = d.stock !== null && d.stock !== undefined && d.stock !== '' ? Number(d.stock) : null; | |
| 149 | + const url = `${SITE}/en/products/detail/${d.product}?lang=${LANG_CODE[langCode ?? ''] ?? 'EN'}${d.high_price_code && d.high_price_code !== '0' && d.product_class ? `&class=${d.product_class}` : ''}`; | |
| 150 | + const a = attrs({ | |
| 151 | + categorySlug: 'magic_the_gathering', | |
| 152 | + franchise: 'Magic: The Gathering', | |
| 153 | + brand: 'Wizards of the Coast', | |
| 154 | + set: p.cardset.name, | |
| 155 | + setCode: pn.setCode, | |
| 156 | + name, | |
| 157 | + number: pn.number, | |
| 158 | + variant: foil ? 'Foil' : null, | |
| 159 | + language, | |
| 160 | + identifiers: { hareruya_product_id: d.product, ...(d.product_class ? { hareruya_product_class: d.product_class } : {}) }, | |
| 161 | + metadata: { hareruya_cardset_id: p.cardset.id, product_name_ja: d.product_name ?? null, condition_code: d.card_condition ?? null, weekly_sales: d.weekly_sales !== undefined && d.weekly_sales !== null ? Number(d.weekly_sales) : null, on_sale: d.sale_flg === '1', high_price_code: d.high_price_code && d.high_price_code !== '0' ? d.high_price_code : null }, | |
| 162 | + }); | |
| 163 | + const rawTitle = `${makeTitle({ name, set: p.cardset.name, number: pn.number, variant: foil ? 'Foil' : null })}${language ? ` (${language})` : ''}${d.card_condition ? ` · cond ${d.card_condition}` : ''}`; | |
| 164 | + out.push( | |
| 165 | + NormalizedListingSchema.parse({ | |
| 166 | + kind: 'listing', | |
| 167 | + connectorId: this.meta.id, | |
| 168 | + sourceId: this.meta.sourceId, | |
| 169 | + sourceUrl: url, | |
| 170 | + externalId, | |
| 171 | + rawTitle, | |
| 172 | + imageUrls: d.image_url ? [d.image_url] : [], | |
| 173 | + attributes: a, | |
| 174 | + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, | |
| 175 | + condition: { condition: null, conditionRaw: null, completeness: null }, | |
| 176 | + observedAt, | |
| 177 | + confidence: 0.8, | |
| 178 | + parserVersion: PARSER_VERSION, | |
| 179 | + listingType: 'fixed_price', | |
| 180 | + price, | |
| 181 | + currency: 'JPY', | |
| 182 | + seller: 'Hareruya', | |
| 183 | + location: 'JP', | |
| 184 | + quantity: Number.isFinite(stock) ? stock : null, | |
| 185 | + availability: stock === 0 ? 'ended' : 'available', | |
| 186 | + }), | |
| 187 | + ); | |
| 188 | + } | |
| 189 | + return out; | |
| 190 | + } | |
| 191 | +} | |
| 192 | + | |
| 193 | +export default (meta: ConnectorMeta) => new HareruyaConnector(meta); | |
added
connectors/api/hareruya/meta.json
+38 −0
@@ -0,0 +1,38 @@ | ||
| 1 | +{ | |
| 2 | + "id": "hareruya", | |
| 3 | + "displayName": "Hareruya (晴れる屋) MTG singles", | |
| 4 | + "sourceId": "hareruya", | |
| 5 | + "sourceName": "Hareruya", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.hareruyamtg.com/en/", | |
| 8 | + "module": "api/hareruya", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["magic_the_gathering"], | |
| 11 | + "regions": ["JP"], | |
| 12 | + "country": "JP", | |
| 13 | + "languages": ["en", "ja"], | |
| 14 | + "currency": ["JPY"], | |
| 15 | + "supportsListings": true, | |
| 16 | + "supportsSold": false, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 1440, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.8, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://www.hareruyamtg.com/en/help/agreement", | |
| 27 | + "acquisitionMethod": "the storefront's own search JSON endpoint (unisearch_api) per card set", | |
| 28 | + "historicalDepth": "none", | |
| 29 | + "capabilities": ["live_listings", "images"], | |
| 30 | + "accessNotes": "Hareruya is Japan's largest Magic retailer with an English storefront (EC-CUBE). The HTML search page renders its results through a public, unauthenticated JSON endpoint — GET https://www.hareruyamtg.com/en/products/search/unisearch_api?fq.category_id=1&fq.cardset=<id>&fq.price=1~*&rows=60&page=<n> — which we call exactly as the page does (60 docs per page, capped by the site at 4 000 results per query): product id, English/Japanese product name (《Card》[SET]), card_name, language code (1 JP, 2 EN, 3 CS, 4 CT, 5 FR, 6 DE, 7 IT, 8 KO …), JPY price incl. tax, foil flag, stock, weekly_sales, product_class (per language/condition/foil offer), card_condition code and image. Card-set ids/names are read once per run from the <select name=cardset> of the public search form (https://www.hareruyamtg.com/en/products/search, no query parameters). robots.txt allows / and disallows only /en/products/search?*page=* | *sort=* | *order=* (the HTML result pages), /*.csv$, deck downloads, purchase forwards and news; we never request those HTML pages — only the JSON endpoint the page itself loads, ≥ 4 s apart, 1 concurrent, honest UA. Not fetched: buy-list (purchase) prices, product detail pages (only linked), carts, member areas. Prices are shop asking prices → listings, never sales. The numeric card_condition code is kept in metadata only (Hareruya's NM/EX/PL labels are not exposed by the endpoint, so condition is left null rather than guessed).", | |
| 31 | + "enabled": true, | |
| 32 | + "schemaVersion": "1.0", | |
| 33 | + "config": { | |
| 34 | + "setsPerRun": 12, | |
| 35 | + "maxPagesPerSet": 67, | |
| 36 | + "rows": 60 | |
| 37 | + } | |
| 38 | +} | |
added
connectors/api/hibid/README.md
+20 −0
@@ -0,0 +1,20 @@ | ||
| 1 | +# hibid — HiBid past catalogs (prices realized) | |
| 2 | + | |
| 3 | +HiBid hosts thousands of regional auctioneers. The public Angular pages embed their Apollo GraphQL state in | |
| 4 | +`<script id="hibid-state">`; this connector reads that JSON from two public page types only: | |
| 5 | + | |
| 6 | +- `/auctions/past?apage=N` — archived auctions of the current month window (25 per page); | |
| 7 | +- `/catalog/<id>/<slug>?apage=N` — the catalog (100 lots per page) with `lotState.priceRealized`. | |
| 8 | + | |
| 9 | +A **sale** is emitted for closed lots with a published price realized (hammer; premium charged separately → | |
| 10 | +`buyerPremiumIncluded=false`, premium text in metadata). Currency = the auction's `currencyAbbreviation`. | |
| 11 | +Lots are kept only when their HiBid category maps to the collectibles taxonomy (`categories.ts`); real estate, | |
| 12 | +vehicles, equipment, household goods are skipped and firearms/ammunition are never emitted. | |
| 13 | + | |
| 14 | +Cursor: `{ doneAuctions: number[], pastPage }`; backfill walks the past pages and ends with `done: true`. | |
| 15 | + | |
| 16 | +```bash | |
| 17 | +set -a; . ./.env; set +a | |
| 18 | +pnpm tsx connectors/api/_g7-auctions-na-lib/smoke.ts api/hibid --limit 3 --capture catalog | |
| 19 | +pnpm vitest run connectors/api/hibid | |
| 20 | +``` | |
added
connectors/api/hibid/categories.ts
+97 −0
@@ -0,0 +1,97 @@ | ||
| 1 | +import { slugFromTitle, watchBrand } from '../_auction-lib/categories.js'; | |
| 2 | +import { popCultureCategory, sportsCategory } from '../_memorabilia-lib/index.js'; | |
| 3 | + | |
| 4 | +/** Firearms / ammunition are never emitted (compliance). */ | |
| 5 | +export function isFirearm(text: string): boolean { | |
| 6 | + return /\b(firearms?|rifles?|pistols?|shotguns?|revolvers?|handguns?|carbines?|ammunition|ammo|cartridges|\d+\s?(?:ga|gauge)\b|\.\d{2,3}\s?cal(?:iber)?|9\s?mm|air ?gun|bb gun)\b/i.test(text); | |
| 7 | +} | |
| 8 | + | |
| 9 | +function tcg(title: string): string { | |
| 10 | + if (/pok[eé]mon|charizard|pikachu/i.test(title)) return 'pokemon'; | |
| 11 | + if (/magic:? the gathering|\bmtg\b/i.test(title)) return 'magic_the_gathering'; | |
| 12 | + if (/yu-?gi-?oh/i.test(title)) return 'yugioh'; | |
| 13 | + if (/\bone piece\b/i.test(title)) return 'one_piece_card_game'; | |
| 14 | + if (/lorcana/i.test(title)) return 'disney_lorcana'; | |
| 15 | + if (/\b(topps|panini|bowman|upper deck|fleer|donruss|rookie|psa|sgc|bgs)\b/i.test(title)) return sportsCategory(title); | |
| 16 | + return 'other_tcg'; | |
| 17 | +} | |
| 18 | + | |
| 19 | +/** | |
| 20 | + * HiBid CategoryTree.fullCategory ("Coins & Currency - Currency - Canada") + lot title → taxonomy slug, | |
| 21 | + * or null when the lot is outside the collectibles universe (real estate, vehicles, equipment, household…). | |
| 22 | + * Only slugs present in data/taxonomy/categories.json are returned. | |
| 23 | + */ | |
| 24 | +/** New merchandise / liquidation lots that surface under collectible categories (wholesale, bulk packs, notices). */ | |
| 25 | +const LIQUIDATION_TITLE = /\b(\d+\s?pcs?|\d+\s?pack|wholesale|liquidation|pallet|case of \d+|information only|do not bid|shelf pull|overstock|brand new in box|nib\b.*\bnew)\b/i; | |
| 26 | +/** Sub-categories of "Antiques & Collectibles - Collectibles" that are household goods, not collectibles. */ | |
| 27 | +const HOUSEHOLD_SUB = /bakeware|candle|ornaments?|kitchen|home d[eé]cor|linens?|bedding|lighting|office|garden|tools?|electronics|holiday|seasonal|party|crafts?|\bpet\b|baby|health|beauty|cleaning|storage|furniture|appliances?|cookware|tableware|dinnerware|drinkware|mugs?|frames?|wall art|rugs?|pillows?|blankets?|misc/; | |
| 28 | + | |
| 29 | +export function hibidCategory(fullCategory: string | null | undefined, title: string): string | null { | |
| 30 | + const c = (fullCategory ?? '').toLowerCase(); | |
| 31 | + if (!c) return null; | |
| 32 | + if (isFirearm(c) || LIQUIDATION_TITLE.test(title)) return null; | |
| 33 | + if (/^coins & currency/.test(c)) { | |
| 34 | + const sub = c.replace(/^coins & currency/, ''); | |
| 35 | + if (/currency|paper money|bank ?notes?/.test(sub) || /\b(bill|note|banknote|dollar bill)\b/i.test(title)) return 'banknotes'; | |
| 36 | + if (/stamps?/.test(c)) return 'stamps'; | |
| 37 | + return 'coins'; | |
| 38 | + } | |
| 39 | + if (/sports? (memorabilia|cards?)|sports? collectibles|^sports\b/.test(c)) return sportsCategory(title); | |
| 40 | + if (/trading cards?|collectible card|card games?|tcg|non-?sport cards?/.test(c)) return /non-?sport/.test(c) ? 'non_sport_cards' : tcg(title); | |
| 41 | + if (/comics?|comic books?/.test(c)) return popCultureCategory(title); | |
| 42 | + if (/video ?games?|gaming consoles?/.test(c)) return 'video_games'; | |
| 43 | + if (/toys? & hobbies|\btoys?\b|action figures?|dolls?|die-?cast|model (trains?|kits?|cars?)/.test(c)) { | |
| 44 | + if (/model trains?|railroad/.test(c)) return 'model_trains'; | |
| 45 | + if (/die-?cast|model cars?/.test(c)) return 'model_cars'; | |
| 46 | + if (/dolls?/.test(c)) return 'dolls'; | |
| 47 | + if (/action figures?/.test(c)) return 'action_figures'; | |
| 48 | + return slugFromTitle(title, 'toys') ?? popCultureCategory(title); | |
| 49 | + } | |
| 50 | + if (/watches?/.test(c)) return watchBrand(title).slug; | |
| 51 | + if (/jewelry|jewellery|gemstones?|diamonds?/.test(c)) return /\b(watch|wristwatch)\b/i.test(title) ? watchBrand(title).slug : /loose|unmounted|gia/i.test(title) ? 'gemstones' : 'jewelry'; | |
| 52 | + if (/advertising|signs?|petroliana|breweriana|soda/.test(c)) return 'advertising'; | |
| 53 | + if (/stamps?|philatel/.test(c)) return 'stamps'; | |
| 54 | + if (/militaria|military collectibles|medals?/.test(c)) return /\bmedal/i.test(title) ? 'medals' : 'militaria'; | |
| 55 | + if (/musical instruments?|music - instruments|guitars?/.test(c)) return 'musical_instruments'; | |
| 56 | + if (/records?|vinyl|\blps?\b/.test(c)) return 'music'; | |
| 57 | + if (/music memorabilia|entertainment memorabilia|movie memorabilia|movie posters?/.test(c)) return popCultureCategory(title); | |
| 58 | + if (/cameras?|photography equipment/.test(c)) return 'cameras'; | |
| 59 | + if (/clocks?/.test(c)) return 'clocks'; | |
| 60 | + if (/books?|manuscripts?|ephemera/.test(c)) return /\bmap\b|atlas/i.test(title) ? 'maps' : /postcard/i.test(title) ? 'postcards' : 'books'; | |
| 61 | + if (/maps?|atlases/.test(c)) return 'maps'; | |
| 62 | + if (/postcards?/.test(c)) return 'postcards'; | |
| 63 | + if (/glass(ware)?|crystal/.test(c)) return 'glass_crystal'; | |
| 64 | + if (/porcelain|china|pottery|ceramics?|stoneware/.test(c)) return 'porcelain'; | |
| 65 | + if (/silver|sterling/.test(c)) return 'silver'; | |
| 66 | + if (/fine art|paintings?|prints?|sculptures?|^art\b|art - |artwork/.test(c)) return slugFromTitle(title, 'art') ?? 'art'; | |
| 67 | + if (/fossils?|minerals?|meteorites?|natural history|rocks?/.test(c)) return /meteorite/i.test(title) ? 'meteorites' : /\b(fossil|ammonite|trilobite|tooth|skull|dinosaur)\b/i.test(title) ? 'fossils' : 'minerals'; | |
| 68 | + if (/pens?|writing instruments?/.test(c)) return 'pens'; | |
| 69 | + if (/lighters?/.test(c)) return 'lighters'; | |
| 70 | + if (/perfume|fragrance/.test(c)) return 'perfume'; | |
| 71 | + if (/pinball|arcade|slot machines?|coin-?op/.test(c)) return /slot machine/i.test(title) ? 'casino_memorabilia' : 'arcade_pinball'; | |
| 72 | + if (/casino|gaming chips?/.test(c)) return 'casino_memorabilia'; | |
| 73 | + if (/political/.test(c)) return 'political_memorabilia'; | |
| 74 | + if (/disney/.test(c)) return 'disney_collectibles'; | |
| 75 | + if (/star wars/.test(c)) return 'star_wars'; | |
| 76 | + if (/board games?|puzzles?/.test(c)) return 'board_games'; | |
| 77 | + if (/license plates?/.test(c)) return 'license_plates'; | |
| 78 | + if (/pins?|badges?|buttons?/.test(c)) return 'pins'; | |
| 79 | + if (/vintage computers?|apple/.test(c)) return /\bapple\b|macintosh/i.test(title) ? 'apple_collectibles' : 'vintage_computers'; | |
| 80 | + if (/typewriters?/.test(c)) return 'typewriters'; | |
| 81 | + if (/scientific instruments?/.test(c)) return 'scientific_instruments'; | |
| 82 | + if (/antiques?|collectibles|decorative arts?|primitives|folk art|americana|asian antiques|oriental/.test(c)) { | |
| 83 | + const sub = c.replace(/^antiques & collectibles(?: - collectibles)?/, ''); | |
| 84 | + const vintageSignal = /\b(antique|vintage|victorian|edwardian|georgian|art deco|art nouveau|mid-?century|19th|18th|17th|c\.?\s?1[89]\d\d|1[89]\d\ds|primitive|folk art|cast iron)\b/i.test(title) || /antique|primitive|folk art|americana/.test(sub); | |
| 85 | + if (HOUSEHOLD_SUB.test(sub) && !vintageSignal) return null; | |
| 86 | + if (/\b(lighter|zippo)\b/i.test(title)) return 'lighters'; | |
| 87 | + if (/\b(fountain pen|montblanc|parker 51)\b/i.test(title)) return 'pens'; | |
| 88 | + if (/\b(mechanical bank|still bank|tin toy|wind-?up|pedal car|cap gun)\b/i.test(title)) return 'vintage_toys'; | |
| 89 | + const bySignal = slugFromTitle(title, 'decorative'); | |
| 90 | + if (bySignal && bySignal !== 'antiques') return bySignal; | |
| 91 | + return vintageSignal || (bySignal === 'antiques' && !HOUSEHOLD_SUB.test(sub)) ? 'antiques' : null; | |
| 92 | + } | |
| 93 | + if (/automobilia|automotive memorabilia|petroliana|gas & oil/.test(c)) return 'automotive_memorabilia'; | |
| 94 | + if (/wine|whisk(e)?y|spirits/.test(c)) return slugFromTitle(title, 'wine'); | |
| 95 | + if (/sneakers?|streetwear|handbags?|luxury|designer/.test(c)) return slugFromTitle(title, 'fashion'); | |
| 96 | + return null; | |
| 97 | +} | |
added
connectors/api/hibid/fixtures.state.json
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +{"apollo.state":{"ROOT_QUERY":{"__typename":"Query","auction({\"countAsView\":true,\"id\":752746})":{"__ref":"Auction:752746"},"lotSearch({\"input\":{\"auctionId\":752746,\"category\":null,\"countAsView\":true,\"countryName\":\"\",\"filter\":\"ALL\",\"hideGoogle\":false,\"isArchive\":false,\"miles\":50,\"searchText\":null,\"shippingOffered\":false,\"sortOrder\":\"LOT_NUMBER\",\"status\":\"ALL\",\"zip\":\"\"},\"pageLength\":100,\"pageNumber\":1,\"sortDirection\":\"DESC\"})":{"__typename":"LotSearchResult","pagedResults":{"__typename":"LotPagedResult","pageLength":100,"pageNumber":1,"totalCount":3,"filteredCount":3,"results":[{"__ref":"Lot:308002774"},{"__ref":"Lot:308002776"},{"__ref":"Lot:308002775"}]}}},"Auction:752746":{"__typename":"Auction","id":752746,"altBiddingUrl":"","altBiddingUrlCaption":"","amexAccepted":false,"discoverAccepted":false,"mastercardAccepted":true,"visaAccepted":true,"regType":"CREDIT_CARD_EVERY_TIME","holdAmount":0,"auctioneer":{"__ref":"Auctioneer:136746"},"auctionNotice":"","auctionOptions":{"__typename":"AuctionOptionsType","bidding":true,"altBidding":false,"catalog":true,"liveCatalog":true,"shippingType":"SHIPPING_OFFERED_ALL","preview":false,"registration":true,"webcast":false,"useLotNumber":true,"useSaleOrder":false},"auctionState":{"__typename":"AuctionStateType","auctionStatus":"ARCHIVED","bidCardNumber":0,"isRegistered":false,"openLotCount":0,"timeToOpen":""},"bidAmountType":"MAX_BIDDING","bidOpenDateTime":"2026-06-18T17:00:00","bidCloseDateTime":"2026-09-07T19:00:00","bidType":"INTERNET_ONLY","buyerPremium":"Buyers Premium 18%","buyerPremiumRate":1,"previewDateInfo":"","currencyAbbreviation":"CAD","description":"","eventAddress":"219 Talbot Street West\r\n","eventCity":"Leamington","eventDateBegin":"2026-06-18T00:00:00","eventDateEnd":"2026-09-07T00:00:00","eventDateInfo":"Auction starts on 6/18/26 at 5PM est.\r\nEnds on 9/7/26 at 7PM est.","eventName":"Auction 200 Silver and Paper Money","eventState":"ON","eventZip":"N8H 1N8","lotCount":62,"showBuyerPremium":false,"audioVideoChatInfo":{"__typename":"AuctionAudioVideoChat","aVCEnabled":false,"blockChat":false},"hidden":false,"sourceType":"AFLEX","distanceMiles":null},"Auctioneer:136746":{"__typename":"Auctioneer","bidIncrementDisclaimer":"Your bid must adhere to the<br/>bid increment schedule.","buyerRegNotesCaption":"YOUR NOTES TO THE AUCTIONEER","city":"Leamington","countryId":178,"country":"Canada","cRMID":0,"id":136746,"internetAddress":"legacyauctionson.hibid.com","missingThumbnail":"https://cdn.hibid.com/cdn/images/hibid/missing/thumbnail_en.png","name":"Legacy Auctions","noMinimumCaption":"No Minimum","state":"ON","postalCode":"N8H 1N8"},"Lot:308002774":{"__typename":"Lot","bidAmount":123.45,"bidQuantity":"","description":"UNC 58","estimate":"","featuredPicture":{"__typename":"Picture","description":"1-1937 BANK OF CANADA 1 DOLLAR BILL S/N3599211","fullSizeLocation":"https://cdn.hibid.com/img.axd?id=8375062716&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrPiq%2f1ZMnee%2big%2bfxBltNny","height":0,"hdThumbnailLocation":"https://cdn.hibid.com/img.axd?id=8375062716&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrPiq%2f1ZMnee%2big%2bfxBltNny&h=400&w=400","thumbnailLocation":"https://cdn.hibid.com/img.axd?id=8375062716&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrPiq%2f1ZMnee%2big%2bfxBltNny&h=350&w=350","width":0},"forceLiveCatalog":true,"hideLeadWithDescription":false,"id":308002774,"itemId":13047,"lead":"1-1937 BANK OF CANADA 1 DOLLAR BILL S/N3599211","links":[],"linkTypes":[],"lotNavigator":null,"lotNumber":"91","lotState":{"__typename":"LotState","bidCount":0,"biddingExtended":false,"bidMax":0,"bidMaxTotal":0,"buyerBidStatus":"NO_BID","buyerHighBid":0,"buyerHighBidTotal":0,"buyNow":0,"choiceType":"SINGLE_LOT","highBid":0,"highBuyerId":"0","isArchived":true,"isClosed":true,"isHidden":false,"isLive":false,"isNotYetLive":false,"isOnLiveCatalog":false,"isPosted":false,"isPublicHidden":false,"isRegistered":false,"isWatching":false,"linkedSoftClose":"","mayHaveWonStatus":"","minBid":0,"priceRealized":0,"priceRealizedMessage":null,"priceRealizedPerEach":0,"productStatus":"BUY_NOW_SET","productUrl":null,"quantitySold":0,"reserveSatisfied":true,"sealed":false,"showBidStatus":true,"showReserveStatus":false,"softCloseMinutes":0,"softCloseSeconds":0,"status":"CLOSED","timeLeft":"","timeLeftLead":"","timeLeftSeconds":0,"timeLeftTitle":"","timeLeftWithLimboSeconds":0,"watchNotes":null},"pictureCount":2,"pictures":[],"quantity":1,"ringNumber":0,"rv":4,"category":[{"__ref":"CategoryTree:40137"},{"__ref":"CategoryTree:40135"},{"__ref":"CategoryTree:40118"}],"shippingOffered":true,"simulcastStatus":"PENDING","site":{"__typename":"Site","domain":null,"fr8StarUrl":"https://app.fr8star.com/transport-estimate?utm_channel=referral&utm_source=hibid&utm_medium=site-link&utm_campaign=shipping&utm_content=homepage","isDomainRequest":false,"isExtraWWWRequest":false,"siteType":"PUBLIC","subdomain":"WWW"},"saleOrder":1},"Lot:308002776":{"__typename":"Lot","bidAmount":123.45,"bidQuantity":"","description":"","estimate":"","featuredPicture":{"__typename":"Picture","description":"1-1954 DEVIL FACE 20 DOLLAR BILL C/E2791225","fullSizeLocation":"https://cdn.hibid.com/img.axd?id=8375062758&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrM9Y4Mvk5lg%2f6VgQ1%2fiLJ7X","height":0,"hdThumbnailLocation":"https://cdn.hibid.com/img.axd?id=8375062758&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrM9Y4Mvk5lg%2f6VgQ1%2fiLJ7X&h=400&w=400","thumbnailLocation":"https://cdn.hibid.com/img.axd?id=8375062758&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrM9Y4Mvk5lg%2f6VgQ1%2fiLJ7X&h=350&w=350","width":0},"forceLiveCatalog":true,"hideLeadWithDescription":false,"id":308002776,"itemId":13049,"lead":"1-1954 DEVIL FACE 20 DOLLAR BILL C/E2791225","links":[],"linkTypes":[],"lotNavigator":null,"lotNumber":"93","lotState":{"__typename":"LotState","bidCount":0,"biddingExtended":false,"bidMax":0,"bidMaxTotal":0,"buyerBidStatus":"NO_BID","buyerHighBid":0,"buyerHighBidTotal":0,"buyNow":0,"choiceType":"SINGLE_LOT","highBid":48,"highBuyerId":"25567483","isArchived":true,"isClosed":true,"isHidden":false,"isLive":false,"isNotYetLive":false,"isOnLiveCatalog":false,"isPosted":false,"isPublicHidden":false,"isRegistered":false,"isWatching":false,"linkedSoftClose":"","mayHaveWonStatus":"","minBid":0,"priceRealized":48,"priceRealizedMessage":null,"priceRealizedPerEach":48,"productStatus":"BUY_NOW_SET","productUrl":null,"quantitySold":1,"reserveSatisfied":true,"sealed":false,"showBidStatus":true,"showReserveStatus":false,"softCloseMinutes":0,"softCloseSeconds":0,"status":"CLOSED","timeLeft":"","timeLeftLead":"","timeLeftSeconds":0,"timeLeftTitle":"","timeLeftWithLimboSeconds":0,"watchNotes":null},"pictureCount":2,"pictures":[],"quantity":1,"ringNumber":0,"rv":6,"category":[{"__ref":"CategoryTree:40137"},{"__ref":"CategoryTree:40135"},{"__ref":"CategoryTree:40118"}],"shippingOffered":true,"simulcastStatus":"PENDING","site":{"__typename":"Site","domain":null,"fr8StarUrl":"https://app.fr8star.com/transport-estimate?utm_channel=referral&utm_source=hibid&utm_medium=site-link&utm_campaign=shipping&utm_content=homepage","isDomainRequest":false,"isExtraWWWRequest":false,"siteType":"PUBLIC","subdomain":"WWW"},"saleOrder":3},"Lot:308002775":{"__typename":"Lot","bidAmount":123.45,"bidQuantity":"","description":"CENTRE","estimate":"","featuredPicture":{"__typename":"Picture","description":"1-1867-1967 1 DOLLAR BILL G/P8122148 UNC CUT OFF","fullSizeLocation":"https://cdn.hibid.com/img.axd?id=8375062725&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrNN3hx%2bsdeRGqQ10RD4MeUb","height":0,"hdThumbnailLocation":"https://cdn.hibid.com/img.axd?id=8375062725&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrNN3hx%2bsdeRGqQ10RD4MeUb&h=400&w=400","thumbnailLocation":"https://cdn.hibid.com/img.axd?id=8375062725&wid=&rwl=false&p=&ext=&w=0&h=0&t=&lp=&c=true&wt=false&sz=MAX&checksum=RXeaJZmScrNN3hx%2bsdeRGqQ10RD4MeUb&h=350&w=350","width":0},"forceLiveCatalog":true,"hideLeadWithDescription":false,"id":308002775,"itemId":13048,"lead":"1-1867-1967 1 DOLLAR BILL G/P8122148 UNC CUT OFF","links":[],"linkTypes":[],"lotNavigator":null,"lotNumber":"92","lotState":{"__typename":"LotState","bidCount":0,"biddingExtended":false,"bidMax":0,"bidMaxTotal":0,"buyerBidStatus":"NO_BID","buyerHighBid":0,"buyerHighBidTotal":0,"buyNow":0,"choiceType":"SINGLE_LOT","highBid":0,"highBuyerId":"0","isArchived":true,"isClosed":true,"isHidden":false,"isLive":false,"isNotYetLive":false,"isOnLiveCatalog":false,"isPosted":false,"isPublicHidden":false,"isRegistered":false,"isWatching":false,"linkedSoftClose":"","mayHaveWonStatus":"","minBid":0,"priceRealized":0,"priceRealizedMessage":null,"priceRealizedPerEach":0,"productStatus":"BUY_NOW_SET","productUrl":null,"quantitySold":0,"reserveSatisfied":true,"sealed":false,"showBidStatus":true,"showReserveStatus":false,"softCloseMinutes":0,"softCloseSeconds":0,"status":"CLOSED","timeLeft":"","timeLeftLead":"","timeLeftSeconds":0,"timeLeftTitle":"","timeLeftWithLimboSeconds":0,"watchNotes":null},"pictureCount":2,"pictures":[],"quantity":1,"ringNumber":0,"rv":4,"category":[{"__ref":"CategoryTree:40137"},{"__ref":"CategoryTree:40135"},{"__ref":"CategoryTree:40118"}],"shippingOffered":true,"simulcastStatus":"PENDING","site":{"__typename":"Site","domain":null,"fr8StarUrl":"https://app.fr8star.com/transport-estimate?utm_channel=referral&utm_source=hibid&utm_medium=site-link&utm_campaign=shipping&utm_content=homepage","isDomainRequest":false,"isExtraWWWRequest":false,"siteType":"PUBLIC","subdomain":"WWW"},"saleOrder":2},"CategoryTree:40137":{"__typename":"CategoryTree","id":40137,"parentCategoryId":40135,"categoryName":"World","fullCategory":"Coins & Currency - Currency - World","hasChildren":false,"uRLPath":"coins-and-currency/currency/world","children":[],"baseCategoryId":40118,"header":null},"CategoryTree:40135":{"__typename":"CategoryTree","id":40135,"parentCategoryId":40118,"categoryName":"Currency","fullCategory":"Coins & Currency - Currency","hasChildren":true,"uRLPath":"coins-and-currency/currency","children":[],"baseCategoryId":40118,"header":null},"CategoryTree:40118":{"__typename":"CategoryTree","id":40118,"parentCategoryId":0,"baseCategoryId":40118,"categoryName":"Coins & Currency","fullCategory":"Coins & Currency","hasChildren":true,"uRLPath":"coins-and-currency","children":[],"header":null}}} | |
| \ No newline at end of file | ||
added
connectors/api/hibid/index.test.ts
+115 −0
@@ -0,0 +1,115 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { readFileSync } from 'node:fs'; | |
| 3 | +import path from 'node:path'; | |
| 4 | +import { fileURLToPath } from 'node:url'; | |
| 5 | +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing'; | |
| 6 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 7 | +import { hibidCategory, isFirearm } from './categories.js'; | |
| 8 | +import createConnector, { parseCatalog, parsePastAuctions, slugify } from './index.js'; | |
| 9 | + | |
| 10 | +const dir = path.dirname(fileURLToPath(import.meta.url)); | |
| 11 | +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); | |
| 12 | +const connector = createConnector(meta); | |
| 13 | + | |
| 14 | +/** Trimmed REAL Apollo state captured from https://hibid.com/catalog/752746/auction-200-silver-and-paper-money on 2026-09-08 (3 lots, contact details removed). */ | |
| 15 | +const STATE = readFileSync(path.join(dir, 'fixtures.state.json'), 'utf8'); | |
| 16 | +const CATALOG_HTML = `<!doctype html><html><head><title>Auction 200 Silver and Paper Money | HiBid</title></head><body><app-root></app-root><script id="hibid-state" type="application/json">${STATE}</script></body></html>`; | |
| 17 | + | |
| 18 | +function withPage(pageNumber: number, results: string): string { | |
| 19 | + return CATALOG_HTML.replace(/"pageNumber":1,"totalCount":3,"filteredCount":3,"results":\[[^\]]*\]/, `"pageNumber":${pageNumber},"totalCount":3,"filteredCount":3,"results":[${results}]`); | |
| 20 | +} | |
| 21 | + | |
| 22 | +describe('hibid', () => { | |
| 23 | + runFixtureSuite(connector, it, expect); | |
| 24 | + | |
| 25 | + it('parses the auction header and lots from the embedded Apollo state', () => { | |
| 26 | + const cat = parseCatalog(CATALOG_HTML, 1)!; | |
| 27 | + expect(cat.auction).toMatchObject({ id: '752746', eventName: 'Auction 200 Silver and Paper Money', currency: 'CAD', buyerPremium: 'Buyers Premium 18%', bidCloseDateTime: '2026-09-07T19:00:00' }); | |
| 28 | + expect(cat.auction.auctioneer).toMatchObject({ name: 'Legacy Auctions', city: 'Leamington', state: 'ON', country: 'Canada' }); | |
| 29 | + expect(cat.pageNumber).toBe(1); | |
| 30 | + expect(cat.totalCount).toBe(3); | |
| 31 | + expect(cat.lots.length).toBe(3); | |
| 32 | + const sold = cat.lots.find((l) => l.lotNumber === '93')!; | |
| 33 | + expect(sold).toMatchObject({ id: '308002776', title: '1-1954 DEVIL FACE 20 DOLLAR BILL C/E2791225', priceRealized: 48, quantitySold: 1, isClosed: true, categoryPath: 'Coins & Currency - Currency - World', mappable: true }); | |
| 34 | + expect(sold.url).toBe('https://hibid.com/lot/308002776/1-1954-devil-face-20-dollar-bill-c-e2791225'); | |
| 35 | + expect(cat.lots.find((l) => l.lotNumber === '91')!.priceRealized).toBeNull(); | |
| 36 | + expect(cat.lots[0]!.image).toMatch(/^https:\/\/cdn\.hibid\.com\/img\.axd/); | |
| 37 | + }); | |
| 38 | + | |
| 39 | + it('detects the end of a catalog (page 2 with no results)', () => { | |
| 40 | + const cat = parseCatalog(withPage(2, ''), 2)!; | |
| 41 | + expect(cat.pageNumber).toBe(2); | |
| 42 | + expect(cat.lots.length).toBe(0); | |
| 43 | + }); | |
| 44 | + | |
| 45 | + it('maps HiBid categories to taxonomy slugs and filters non-collectibles and firearms', () => { | |
| 46 | + expect(hibidCategory('Coins & Currency - Currency - Canada', '1-1937 BANK OF CANADA 1 DOLLAR BILL S/N3599211')).toBe('banknotes'); | |
| 47 | + expect(hibidCategory('Coins & Currency - Coins - Canada', '1967 Canada Silver Dollar')).toBe('coins'); | |
| 48 | + expect(hibidCategory('Sports Memorabilia & Cards - Cards', '1986 Fleer Michael Jordan Rookie PSA 8')).toBe('basketball_cards'); | |
| 49 | + expect(hibidCategory('Toys & Hobbies - Action Figures', 'Star Wars Kenner Luke Skywalker MOC')).toBe('action_figures'); | |
| 50 | + expect(hibidCategory('Jewelry, Watches & Gemstones - Watches', 'Rolex Submariner 16610 wristwatch')).toBe('rolex'); | |
| 51 | + expect(hibidCategory('Real Estate - Residential', '3 bedroom house')).toBeNull(); | |
| 52 | + expect(hibidCategory('Construction & Farm - Tractors', 'John Deere 4020')).toBeNull(); | |
| 53 | + expect(hibidCategory('Household - Kitchen', 'Kitchen aid mixer')).toBeNull(); | |
| 54 | + expect(hibidCategory('Firearms & Military - Rifles', 'Winchester Model 70')).toBeNull(); | |
| 55 | + // Liquidators file new merchandise under "Antiques & Collectibles" (seen live: ToyTexx wholesale sale) → filtered. | |
| 56 | + expect(hibidCategory('Antiques & Collectibles - Collectibles - Bakeware', '15PCS Nonstick Roasting Pan with Removable Flat Rack')).toBeNull(); | |
| 57 | + expect(hibidCategory('Antiques & Collectibles - Collectibles - Decorative - Ornaments', '10PCS Christmas Tree Building Sets, 730PC')).toBeNull(); | |
| 58 | + expect(hibidCategory('Antiques & Collectibles - Collectibles - Advertising', 'INFORMATION ONLY - DO NOT BID')).toBeNull(); | |
| 59 | + expect(hibidCategory('Antiques & Collectibles - Collectibles - Advertising', 'Coca-Cola porcelain sign 1950s')).toBe('advertising'); | |
| 60 | + expect(hibidCategory('Antiques & Collectibles - Collectibles - Kitchen', 'Vintage Pyrex Gooseberry bowl 1958')).toBe('antiques'); | |
| 61 | + expect(hibidCategory('Antiques & Collectibles - Collectibles', 'Vintage 1960s Zippo lighter')).toBe('lighters'); | |
| 62 | + expect(isFirearm('Antiques & Collectibles - Collectibles Remington .22 caliber rifle')).toBe(true); | |
| 63 | + expect(isFirearm('1-1954 DEVIL FACE 20 DOLLAR BILL')).toBe(false); | |
| 64 | + expect(slugify('Elite Auction SALE – Fine Jewelry, Rolex Watches, Art & Coll')).toBe('elite-auction-sale-fine-jewelry-rolex-watches-art-coll'); | |
| 65 | + }); | |
| 66 | + | |
| 67 | + it('parses the past-auction list (archived auctions + paging)', () => { | |
| 68 | + // Real page shape reduced to one auction entry. | |
| 69 | + const past = `<script id="hibid-state" type="application/json">${JSON.stringify({ | |
| 70 | + 'apollo.state': { | |
| 71 | + ROOT_QUERY: { __typename: 'Query', 'auctionSearch({"input":{"category":null,"isArchive":true},"pageLength":25,"pageNumber":2})': { __typename: 'AuctionSearchResult', pagedResults: { __typename: 'AuctionMatchPagedResult', pageLength: 25, pageNumber: 2, totalCount: 1324, filteredCount: 1324, results: [{ __typename: 'AuctionMatchType', matchinglotcount: 0, auction: { __ref: 'Auction:770300' } }] } } }, | |
| 72 | + 'Auction:770300': { __typename: 'Auction', id: 770300, eventName: '09/07/2026 @ 10am CT - Annual Labor Day: Fine Auction', bidCloseDateTime: '2026-09-07T23:00:00', eventDateEnd: '2026-09-07T00:00:00', lotCount: 455, auctionState: { __typename: 'AuctionStateType', auctionStatus: 'ARCHIVED' }, auctioneer: { __ref: 'Auctioneer:62991' } }, | |
| 73 | + 'Auctioneer:62991': { __typename: 'Auctioneer', id: 62991, name: 'Luther Auctions' }, | |
| 74 | + }, | |
| 75 | + })}</script>`; | |
| 76 | + const list = parsePastAuctions(past)!; | |
| 77 | + expect(list).toMatchObject({ pageNumber: 2, totalCount: 1324, pageLength: 25 }); | |
| 78 | + expect(list.auctions).toEqual([{ id: '770300', eventName: '09/07/2026 @ 10am CT - Annual Labor Day: Fine Auction', url: 'https://hibid.com/catalog/770300/09-07-2026-10am-ct-annual-labor-day-fine-auction', bidCloseDateTime: '2026-09-07T23:00:00', eventDateEnd: '2026-09-07T00:00:00', lotCount: 455, status: 'ARCHIVED' }]); | |
| 79 | + }); | |
| 80 | + | |
| 81 | + it('normalises only closed lots with a published price realized (hammer, CAD, premium not included)', async () => { | |
| 82 | + const cat = parseCatalog(CATALOG_HTML, 1)!; | |
| 83 | + const payload = { kind: 'hibid_catalog_page', auction: cat.auction, page: 1, totalCount: cat.totalCount, lots: cat.lots.filter((l) => l.mappable).map(({ mappable: _m, ...rest }) => rest) }; | |
| 84 | + const out = await connector.normalize({ url: 'https://hibid.com/catalog/752746/auction-200-silver-and-paper-money', externalId: 'catalog:752746:page:1', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-08T08:00:00Z'), payload }); | |
| 85 | + expect(out.length).toBe(1); | |
| 86 | + const s = out[0]!; | |
| 87 | + if (s.kind !== 'sale') throw new Error('expected sale'); | |
| 88 | + expect(s).toMatchObject({ price: 48, currency: 'CAD', buyerPremiumIncluded: false, auctionHouse: 'Legacy Auctions (HiBid)', lotNumber: '93', location: 'Leamington, ON, Canada', quantity: 1, isBundle: false }); | |
| 89 | + expect(s.saleDate.toISOString()).toBe('2026-09-07T19:00:00.000Z'); | |
| 90 | + expect(s.attributes.categorySlug).toBe('banknotes'); | |
| 91 | + expect(s.attributes.identifiers).toEqual({ hibid_lot_id: '308002776', hibid_auction_id: '752746' }); | |
| 92 | + expect(s.attributes.metadata).toMatchObject({ buyer_premium_text: 'Buyers Premium 18%', category_path: 'Coins & Currency - Currency - World' }); | |
| 93 | + expect(s.attributes.year).toBe(1954); | |
| 94 | + }); | |
| 95 | + | |
| 96 | + it('skips unsupported currencies and firearms at normalisation time', async () => { | |
| 97 | + const cat = parseCatalog(CATALOG_HTML, 1)!; | |
| 98 | + const lots = cat.lots.filter((l) => l.mappable).map(({ mappable: _m, ...rest }) => rest); | |
| 99 | + const gun = { ...lots[1]!, id: '1', title: 'Winchester Model 94 rifle .30-30', categoryPath: 'Antiques & Collectibles - Collectibles', priceRealized: 500 }; | |
| 100 | + const mxn = await connector.normalize({ url: 'x', externalId: 'e', kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'hibid_catalog_page', auction: { ...cat.auction, currency: 'MXN' }, page: 1, totalCount: 3, lots } }); | |
| 101 | + expect(mxn.length).toBe(0); | |
| 102 | + const withGun = await connector.normalize({ url: 'x', externalId: 'e', kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'hibid_catalog_page', auction: cat.auction, page: 1, totalCount: 3, lots: [...lots, gun] } }); | |
| 103 | + expect(withGun.map((r) => ('externalId' in r ? r.externalId : null))).toEqual(['308002776']); | |
| 104 | + }); | |
| 105 | + | |
| 106 | + it('fixture sales are CAD/USD hammer prices dated by the auction close', async () => { | |
| 107 | + const out = await connector.normalize(loadFixture('hibid', 'coins-1').raw); | |
| 108 | + expect(out.length).toBeGreaterThan(0); | |
| 109 | + for (const r of out) { | |
| 110 | + if (r.kind !== 'sale') throw new Error('expected sale'); | |
| 111 | + expect(['CAD', 'USD']).toContain(r.currency); | |
| 112 | + expect(r.buyerPremiumIncluded).toBe(false); | |
| 113 | + } | |
| 114 | + }); | |
| 115 | +}); | |
added
connectors/api/hibid/index.ts
+295 −0
@@ -0,0 +1,295 @@ | ||
| 1 | +import { z } from 'zod'; | |
| 2 | +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; | |
| 3 | +import type { CurrencyCode, NormalizedRecord } from '@rareindex/shared'; | |
| 4 | +import { amount, apolloRef, certFromTitle, hibidApolloState, isBundleTitle, isoDate, lotAttributes, makeSale, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js'; | |
| 5 | +import { hibidCategory, isFirearm } from './categories.js'; | |
| 6 | + | |
| 7 | +const BASE = 'https://hibid.com'; | |
| 8 | +const PARSER_VERSION = '1.0.0'; | |
| 9 | +const PAST_PAGE_LENGTH = 25; | |
| 10 | +const ALLOWED_CURRENCIES = new Set<CurrencyCode>(['USD', 'CAD', 'GBP', 'EUR', 'AUD']); | |
| 11 | + | |
| 12 | +export const AuctioneerSchema = z.object({ id: z.string().nullable(), name: z.string(), city: z.string().nullable(), state: z.string().nullable(), country: z.string().nullable() }); | |
| 13 | +export const AuctionSchema = z.object({ | |
| 14 | + id: z.string(), | |
| 15 | + eventName: z.string(), | |
| 16 | + url: z.string(), | |
| 17 | + bidCloseDateTime: z.string().nullable(), | |
| 18 | + eventDateEnd: z.string().nullable(), | |
| 19 | + currency: z.string().nullable(), | |
| 20 | + buyerPremium: z.string().nullable(), | |
| 21 | + buyerPremiumRate: z.number().nullable(), | |
| 22 | + auctioneer: AuctioneerSchema, | |
| 23 | +}); | |
| 24 | +export const LotSchema = z.object({ | |
| 25 | + id: z.string(), | |
| 26 | + lotNumber: z.string().nullable(), | |
| 27 | + title: z.string(), | |
| 28 | + description: z.string().nullable(), | |
| 29 | + estimateText: z.string().nullable(), | |
| 30 | + image: z.string().nullable(), | |
| 31 | + categoryPath: z.string().nullable(), | |
| 32 | + categoryName: z.string().nullable(), | |
| 33 | + priceRealized: z.number().nullable(), | |
| 34 | + quantitySold: z.number().nullable(), | |
| 35 | + quantity: z.number().nullable(), | |
| 36 | + bidCount: z.number().nullable(), | |
| 37 | + isClosed: z.boolean(), | |
| 38 | + url: z.string(), | |
| 39 | +}); | |
| 40 | +export const PayloadSchema = z.object({ kind: z.literal('hibid_catalog_page'), auction: AuctionSchema, page: z.number(), totalCount: z.number().nullable(), lots: z.array(LotSchema) }); | |
| 41 | +export type Payload = z.infer<typeof PayloadSchema>; | |
| 42 | +export type ParsedLot = z.infer<typeof LotSchema> & { mappable: boolean }; | |
| 43 | + | |
| 44 | +type Cache = Record<string, Record<string, unknown>>; | |
| 45 | + | |
| 46 | +export function slugify(s: string): string { | |
| 47 | + return s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 90); | |
| 48 | +} | |
| 49 | + | |
| 50 | +function str(v: unknown): string | null { | |
| 51 | + return typeof v === 'string' && v.trim() ? v.trim() : null; | |
| 52 | +} | |
| 53 | + | |
| 54 | +function rootEntries(cache: Cache, prefix: string): Array<Record<string, unknown>> { | |
| 55 | + const root = cache['ROOT_QUERY'] ?? {}; | |
| 56 | + return Object.entries(root) | |
| 57 | + .filter(([k]) => k.startsWith(prefix)) | |
| 58 | + .map(([, v]) => apolloRef<Record<string, unknown>>(cache, v)) | |
| 59 | + .filter((v): v is Record<string, unknown> => v !== null); | |
| 60 | +} | |
| 61 | + | |
| 62 | +export interface PastAuctionRef { | |
| 63 | + id: string; | |
| 64 | + eventName: string; | |
| 65 | + url: string; | |
| 66 | + bidCloseDateTime: string | null; | |
| 67 | + eventDateEnd: string | null; | |
| 68 | + lotCount: number | null; | |
| 69 | + status: string | null; | |
| 70 | +} | |
| 71 | + | |
| 72 | +/** /auctions/past?apage=N → archived auction references + paging. */ | |
| 73 | +export function parsePastAuctions(html: string): { auctions: PastAuctionRef[]; pageNumber: number | null; totalCount: number | null; pageLength: number | null } | null { | |
| 74 | + const cache = hibidApolloState(html); | |
| 75 | + if (!cache) return null; | |
| 76 | + const search = rootEntries(cache, 'auctionSearch(')[0]; | |
| 77 | + const paged = search ? apolloRef<Record<string, unknown>>(cache, search.pagedResults) : null; | |
| 78 | + const results = Array.isArray(paged?.results) ? (paged!.results as unknown[]) : []; | |
| 79 | + const auctions: PastAuctionRef[] = []; | |
| 80 | + for (const r of results) { | |
| 81 | + const match = apolloRef<Record<string, unknown>>(cache, r); | |
| 82 | + const a = apolloRef<Record<string, unknown>>(cache, match?.auction ?? match); | |
| 83 | + if (!a || typeof a.id !== 'number') continue; | |
| 84 | + const name = str(a.eventName) ?? `Auction ${a.id}`; | |
| 85 | + const state = apolloRef<Record<string, unknown>>(cache, a.auctionState); | |
| 86 | + auctions.push({ id: String(a.id), eventName: name, url: `${BASE}/catalog/${a.id}/${slugify(name)}`, bidCloseDateTime: str(a.bidCloseDateTime), eventDateEnd: str(a.eventDateEnd), lotCount: typeof a.lotCount === 'number' ? a.lotCount : null, status: str(state?.auctionStatus) }); | |
| 87 | + } | |
| 88 | + return { auctions, pageNumber: typeof paged?.pageNumber === 'number' ? paged.pageNumber : null, totalCount: typeof paged?.totalCount === 'number' ? paged.totalCount : null, pageLength: typeof paged?.pageLength === 'number' ? paged.pageLength : null }; | |
| 89 | +} | |
| 90 | + | |
| 91 | +/** /catalog/<id>/<slug>?apage=N → auction header + every lot on the page (with a `mappable` flag). */ | |
| 92 | +export function parseCatalog(html: string, page: number): { auction: Payload['auction']; lots: ParsedLot[]; pageNumber: number | null; totalCount: number | null; pageLength: number | null } | null { | |
| 93 | + const cache = hibidApolloState(html); | |
| 94 | + if (!cache) return null; | |
| 95 | + const auctionRaw = rootEntries(cache, 'auction(')[0] ?? Object.entries(cache).find(([k]) => k.startsWith('Auction:'))?.[1] ?? null; | |
| 96 | + if (!auctionRaw || typeof auctionRaw.id !== 'number') return null; | |
| 97 | + const auctioneer = apolloRef<Record<string, unknown>>(cache, auctionRaw.auctioneer); | |
| 98 | + const name = str(auctionRaw.eventName) ?? `Auction ${auctionRaw.id}`; | |
| 99 | + const auction: Payload['auction'] = { | |
| 100 | + id: String(auctionRaw.id), | |
| 101 | + eventName: name, | |
| 102 | + url: `${BASE}/catalog/${auctionRaw.id}/${slugify(name)}`, | |
| 103 | + bidCloseDateTime: str(auctionRaw.bidCloseDateTime), | |
| 104 | + eventDateEnd: str(auctionRaw.eventDateEnd), | |
| 105 | + currency: str(auctionRaw.currencyAbbreviation), | |
| 106 | + buyerPremium: str(auctionRaw.buyerPremium), | |
| 107 | + buyerPremiumRate: typeof auctionRaw.buyerPremiumRate === 'number' ? auctionRaw.buyerPremiumRate : null, | |
| 108 | + auctioneer: { id: auctioneer && auctioneer.id !== undefined ? String(auctioneer.id) : null, name: str(auctioneer?.name) ?? 'HiBid auctioneer', city: str(auctioneer?.city), state: str(auctioneer?.state), country: str(auctioneer?.country) }, | |
| 109 | + }; | |
| 110 | + const search = rootEntries(cache, 'lotSearch(')[0]; | |
| 111 | + const paged = search ? apolloRef<Record<string, unknown>>(cache, search.pagedResults) : null; | |
| 112 | + const refs = Array.isArray(paged?.results) ? (paged!.results as unknown[]) : Object.keys(cache).filter((k) => k.startsWith('Lot:')).map((k) => ({ __ref: k })); | |
| 113 | + const lots: ParsedLot[] = []; | |
| 114 | + const seen = new Set<string>(); | |
| 115 | + for (const r of refs) { | |
| 116 | + const l = apolloRef<Record<string, unknown>>(cache, r); | |
| 117 | + if (!l || typeof l.id !== 'number' || seen.has(String(l.id))) continue; | |
| 118 | + seen.add(String(l.id)); | |
| 119 | + const title = str(l.lead) ?? str((apolloRef<Record<string, unknown>>(cache, l.featuredPicture) ?? {}).description); | |
| 120 | + if (!title) continue; | |
| 121 | + const state = apolloRef<Record<string, unknown>>(cache, l.lotState) ?? {}; | |
| 122 | + const cats = (Array.isArray(l.category) ? l.category : []).map((c) => apolloRef<Record<string, unknown>>(cache, c)).filter((c): c is Record<string, unknown> => c !== null); | |
| 123 | + // Most specific category first (longest fullCategory path). | |
| 124 | + cats.sort((a, b) => String(b.fullCategory ?? '').length - String(a.fullCategory ?? '').length); | |
| 125 | + const categoryPath = str(cats[0]?.fullCategory); | |
| 126 | + const categoryName = str(cats[0]?.categoryName); | |
| 127 | + const picture = apolloRef<Record<string, unknown>>(cache, l.featuredPicture); | |
| 128 | + const description = str(l.description); | |
| 129 | + const slug = hibidCategory(categoryPath, title); | |
| 130 | + const mappable = slug !== null && !isFirearm(`${categoryPath ?? ''} ${title} ${description ?? ''}`); | |
| 131 | + lots.push({ | |
| 132 | + id: String(l.id), | |
| 133 | + lotNumber: str(l.lotNumber), | |
| 134 | + title, | |
| 135 | + description: description ? description.slice(0, 1500) : null, | |
| 136 | + estimateText: str(l.estimate), | |
| 137 | + image: str(picture?.fullSizeLocation) ?? str(picture?.hdThumbnailLocation), | |
| 138 | + categoryPath, | |
| 139 | + categoryName, | |
| 140 | + priceRealized: amount(state.priceRealized), | |
| 141 | + quantitySold: typeof state.quantitySold === 'number' ? state.quantitySold : null, | |
| 142 | + quantity: typeof l.quantity === 'number' ? l.quantity : null, | |
| 143 | + bidCount: typeof state.bidCount === 'number' ? state.bidCount : null, | |
| 144 | + isClosed: state.isClosed === true || state.isArchived === true, | |
| 145 | + url: `${BASE}/lot/${l.id}/${slugify(title)}`, | |
| 146 | + mappable, | |
| 147 | + }); | |
| 148 | + } | |
| 149 | + return { auction, lots, pageNumber: typeof paged?.pageNumber === 'number' ? paged.pageNumber : null, totalCount: typeof paged?.totalCount === 'number' ? paged.totalCount : null, pageLength: typeof paged?.pageLength === 'number' ? paged.pageLength : null }; | |
| 150 | +} | |
| 151 | + | |
| 152 | +/** | |
| 153 | + * HiBid — public past catalogs of thousands of regional auctioneers. Plain HTTPS on hibid.com; the lot | |
| 154 | + * data is read from the Apollo state the public Angular page embeds (`<script id="hibid-state">`). | |
| 155 | + * Sales = closed lots whose auctioneer publishes the price realized (hammer; premium charged separately). | |
| 156 | + */ | |
| 157 | +export class HibidConnector extends BaseConnector { | |
| 158 | + readonly version = '1.0.0'; | |
| 159 | + readonly parserVersion = PARSER_VERSION; | |
| 160 | + protected override minIntervalMs = 3000; | |
| 161 | + | |
| 162 | + private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> { | |
| 163 | + await this.throttle(url); | |
| 164 | + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); | |
| 165 | + if (!res.success || !res.html) { | |
| 166 | + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); | |
| 167 | + return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 168 | + } | |
| 169 | + return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt }; | |
| 170 | + } | |
| 171 | + | |
| 172 | + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> { | |
| 173 | + const cfg = this.meta.config; | |
| 174 | + const mode = ctx.options.mode; | |
| 175 | + const pastPagesPerRun = mode === 'probe' ? 1 : Number(cfg.pastPagesPerRun ?? 3); | |
| 176 | + const auctionsPerRun = mode === 'probe' ? 3 : Number(cfg.auctionsPerRun ?? 6); | |
| 177 | + const pagesPerAuction = mode === 'probe' ? 1 : Number(cfg.pagesPerAuction ?? 10); | |
| 178 | + const cursor = ctx.options.cursor ?? {}; | |
| 179 | + const doneAuctions = new Set<number>(Array.isArray(cursor.doneAuctions) ? (cursor.doneAuctions as number[]) : []); | |
| 180 | + let pastPage = mode === 'backfill' ? Number(cursor.pastPage ?? 1) : 1; | |
| 181 | + let filtered = 0; | |
| 182 | + let rawCount = 0; | |
| 183 | + let items = 0; | |
| 184 | + | |
| 185 | + // Seeds (catalog URLs) or the past-auction list decide which catalogs to visit. | |
| 186 | + const seeds = (ctx.options.seeds ?? []).map((s) => s.match(/\/catalog\/(\d+)/)?.[1]).filter((x): x is string => Boolean(x)); | |
| 187 | + const queue: PastAuctionRef[] = seeds.map((id) => ({ id, eventName: `Auction ${id}`, url: `${BASE}/catalog/${id}/x`, bidCloseDateTime: null, eventDateEnd: null, lotCount: null, status: null })); | |
| 188 | + let totalPastPages: number | null = null; | |
| 189 | + if (!seeds.length) { | |
| 190 | + for (let i = 0; i < pastPagesPerRun; i++) { | |
| 191 | + if (ctx.signal?.aborted) break; | |
| 192 | + const url = `${BASE}/auctions/past${pastPage > 1 ? `?apage=${pastPage}` : ''}`; | |
| 193 | + let r = await this.html(ctx, url); | |
| 194 | + if (!r.html) break; | |
| 195 | + let list = parsePastAuctions(r.html); | |
| 196 | + if (list && list.auctions.length === 0 && list.pageNumber === null) { | |
| 197 | + // The server occasionally renders the shell before the auction search resolves → one retry. | |
| 198 | + ctx.log.warn({ url }, 'past-auction list rendered without results; retrying once'); | |
| 199 | + r = await this.html(ctx, `${url}${url.includes('?') ? '&' : '?'}apage=${pastPage}`); | |
| 200 | + if (r.html) list = parsePastAuctions(r.html); | |
| 201 | + } | |
| 202 | + if (!list) { | |
| 203 | + ctx.anomaly('parse_failure_page', `${url}: no hibid-state`); | |
| 204 | + break; | |
| 205 | + } | |
| 206 | + if (list.totalCount !== null) totalPastPages = Math.max(1, Math.ceil(list.totalCount / (list.pageLength ?? PAST_PAGE_LENGTH))); | |
| 207 | + if (list.auctions.length === 0) ctx.anomaly('pagination_failure', `${url}: past-auction list rendered without results (pageNumber ${list.pageNumber}, total ${list.totalCount})`); | |
| 208 | + const fresh = list.auctions.filter((a) => !doneAuctions.has(Number(a.id)) && (a.status === null || a.status === 'ARCHIVED')); | |
| 209 | + queue.push(...fresh); | |
| 210 | + if (list.auctions.length === 0 || (totalPastPages !== null && pastPage >= totalPastPages)) { | |
| 211 | + if (mode === 'backfill' && totalPastPages !== null && pastPage >= totalPastPages) { | |
| 212 | + await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-3000), pastPage, done: true, updatedAt: new Date().toISOString() }); | |
| 213 | + pastPage = totalPastPages; | |
| 214 | + } | |
| 215 | + break; | |
| 216 | + } | |
| 217 | + pastPage++; | |
| 218 | + if (queue.length >= auctionsPerRun) break; | |
| 219 | + } | |
| 220 | + } | |
| 221 | + | |
| 222 | + let processed = 0; | |
| 223 | + for (const a of queue) { | |
| 224 | + if (ctx.signal?.aborted || processed >= auctionsPerRun || this.reached(ctx, rawCount)) break; | |
| 225 | + processed++; | |
| 226 | + const idNum = Number(a.id); | |
| 227 | + for (let page = 1; page <= pagesPerAuction; page++) { | |
| 228 | + if (ctx.signal?.aborted || this.reached(ctx, rawCount)) break; | |
| 229 | + const url = `${a.url}${page > 1 ? `?apage=${page}` : ''}`; | |
| 230 | + const r = await this.html(ctx, url); | |
| 231 | + if (!r.html) break; | |
| 232 | + const cat = parseCatalog(r.html, page); | |
| 233 | + if (!cat) { | |
| 234 | + ctx.anomaly('parse_failure_page', `${url}: no catalog state`); | |
| 235 | + break; | |
| 236 | + } | |
| 237 | + if (cat.lots.length === 0) break; | |
| 238 | + const mappable = cat.lots.filter((l) => l.mappable); | |
| 239 | + filtered += cat.lots.length - mappable.length; | |
| 240 | + if (page === 1 && mappable.length === 0) { | |
| 241 | + doneAuctions.add(idNum); // non-collectible sale (equipment, real estate, household…) — skip the rest | |
| 242 | + break; | |
| 243 | + } | |
| 244 | + if (mappable.length) { | |
| 245 | + const payload: Payload = { kind: 'hibid_catalog_page', auction: cat.auction, page, totalCount: cat.totalCount, lots: mappable.map(({ mappable: _m, ...rest }) => rest) }; | |
| 246 | + rawCount++; | |
| 247 | + items += mappable.length; | |
| 248 | + yield { url, externalId: `catalog:${a.id}:page:${page}`, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt }; | |
| 249 | + } | |
| 250 | + const pageLength = cat.pageLength ?? 100; | |
| 251 | + if (cat.totalCount !== null && page * pageLength >= cat.totalCount) { | |
| 252 | + doneAuctions.add(idNum); | |
| 253 | + break; | |
| 254 | + } | |
| 255 | + if (page === pagesPerAuction) doneAuctions.add(idNum); | |
| 256 | + } | |
| 257 | + await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-3000), pastPage, updatedAt: new Date().toISOString() }); | |
| 258 | + await ctx.progress({ page: pastPage, totalPages: totalPastPages, itemsProcessed: items, cursor: { doneAuctions: [...doneAuctions].slice(-3000), pastPage } }); | |
| 259 | + } | |
| 260 | + if (filtered) ctx.anomaly('filtered_non_collectible', `${filtered} lots outside the collectibles taxonomy skipped`); | |
| 261 | + if (!seeds.length) await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-3000), pastPage, updatedAt: new Date().toISOString(), ...(mode === 'backfill' && totalPastPages !== null && pastPage >= totalPastPages ? { done: true } : {}) }); | |
| 262 | + } | |
| 263 | + | |
| 264 | + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> { | |
| 265 | + const p = PayloadSchema.parse(raw.payload); | |
| 266 | + const out: NormalizedRecord[] = []; | |
| 267 | + const currency = p.auction.currency as CurrencyCode | null; | |
| 268 | + if (!currency || !ALLOWED_CURRENCIES.has(currency)) return out; | |
| 269 | + const saleDate = isoDate(p.auction.bidCloseDateTime) ?? isoDate(p.auction.eventDateEnd); | |
| 270 | + if (!saleDate || saleDate.getTime() > Date.now() + 86_400_000) return out; | |
| 271 | + const house = `${p.auction.auctioneer.name} (HiBid)`; | |
| 272 | + const location = [p.auction.auctioneer.city, p.auction.auctioneer.state, p.auction.auctioneer.country].filter(Boolean).join(', ') || null; | |
| 273 | + for (const l of p.lots) { | |
| 274 | + if (!l.isClosed || !l.priceRealized) continue; | |
| 275 | + const slug = hibidCategory(l.categoryPath, l.title); | |
| 276 | + if (!slug || isFirearm(`${l.categoryPath ?? ''} ${l.title} ${l.description ?? ''}`)) continue; | |
| 277 | + const g = saleGrade(l.title); | |
| 278 | + const attributes = lotAttributes({ | |
| 279 | + categorySlug: slug, | |
| 280 | + name: l.title, | |
| 281 | + year: safeYear(l.title), | |
| 282 | + identifiers: { hibid_lot_id: l.id, hibid_auction_id: p.auction.id }, | |
| 283 | + metadata: { auction_id: p.auction.id, event_name: p.auction.eventName, auctioneer_id: p.auction.auctioneer.id, estimate_text: l.estimateText, category_path: l.categoryPath, bid_count: l.bidCount, buyer_premium_text: p.auction.buyerPremium, buyer_premium_rate: p.auction.buyerPremiumRate, quantity_sold: l.quantitySold }, | |
| 284 | + }); | |
| 285 | + const sale = makeSale({ meta: this.meta, sourceUrl: l.url, externalId: l.id, rawTitle: l.title, description: l.description, attributes, price: l.priceRealized, currency, saleDate, buyerPremiumIncluded: false, auctionHouse: house, lotNumber: l.lotNumber, imageUrls: l.image ? [l.image] : [], location, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, isBundle: isBundleTitle(l.title) || (l.quantity ?? 1) > 1, confidence: g.grader ? 0.8 : 0.7 }); | |
| 286 | + sale.grade.qualifier = g.qualifier; | |
| 287 | + sale.grade.certificationNumber = certFromTitle(l.title); | |
| 288 | + sale.quantity = Math.max(1, l.quantitySold ?? l.quantity ?? 1); | |
| 289 | + out.push(sale); | |
| 290 | + } | |
| 291 | + return out; | |
| 292 | + } | |
| 293 | +} | |
| 294 | + | |
| 295 | +export default (meta: ConnectorMeta) => new HibidConnector(meta); | |
added
connectors/api/hibid/meta.json
+37 −0
@@ -0,0 +1,37 @@ | ||
| 1 | +{ | |
| 2 | + "id": "hibid", | |
| 3 | + "displayName": "HiBid (past catalogs, prices realized)", | |
| 4 | + "sourceId": "hibid", | |
| 5 | + "sourceName": "HiBid", | |
| 6 | + "sourceType": "marketplace", | |
| 7 | + "sourceUrl": "https://hibid.com", | |
| 8 | + "module": "api/hibid", | |
| 9 | + "enginePriority": ["api"], | |
| 10 | + "categories": ["coins", "banknotes", "stamps", "sports_memorabilia", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "other_sports_cards", "pokemon", "magic_the_gathering", "yugioh", "other_tcg", "non_sport_cards", "marvel_comics", "dc_comics", "independent_comics", "video_games", "vintage_toys", "action_figures", "dolls", "model_trains", "model_cars", "other_watches", "rolex", "omega", "jewelry", "gemstones", "advertising", "militaria", "medals", "musical_instruments", "music", "cameras", "clocks", "books", "maps", "postcards", "glass_crystal", "porcelain", "silver", "art", "antiques", "fossils", "minerals", "meteorites", "pens", "lighters", "arcade_pinball", "casino_memorabilia", "political_memorabilia", "disney_collectibles", "star_wars", "board_games", "license_plates", "pins", "automotive_memorabilia"], | |
| 11 | + "regions": ["US", "CA"], | |
| 12 | + "country": "US", | |
| 13 | + "languages": ["en"], | |
| 14 | + "currency": ["USD", "CAD"], | |
| 15 | + "supportsListings": false, | |
| 16 | + "supportsSold": true, | |
| 17 | + "supportsAuctions": false, | |
| 18 | + "supportsImages": true, | |
| 19 | + "supportsCatalog": false, | |
| 20 | + "supportsPopulation": false, | |
| 21 | + "supportsLookup": false, | |
| 22 | + "refreshFrequencyMinutes": 720, | |
| 23 | + "priority": "medium", | |
| 24 | + "trustScore": 0.75, | |
| 25 | + "attributionRequired": true, | |
| 26 | + "termsUrl": "https://hibid.com/terms-and-conditions", | |
| 27 | + "acquisitionMethod": "embedded Apollo state JSON on public catalog pages", | |
| 28 | + "historicalDepth": "months", | |
| 29 | + "accessNotes": "Plain HTTPS with the RareIndex user agent on hibid.com (Cloudflare in front; the bot UA is served normally). Pages read: the public past-auction list /auctions/past?apage=N (25 archived auctions per page, current month window, ~1 300 auctions) and catalog pages /catalog/<id>/<slug>?apage=N (100 lots per page). The lot data is taken from the Apollo state the public Angular page embeds in <script id=\"hibid-state\">; we never call hibid.com/graphql ourselves. robots.txt (User-agent *) disallows */livecatalog/, */webcast/, /auctioneer/, */account/, */catalog/print/, /error/, /auctions/current/map/, /hibiddemo/ and the keyword search /auctions/past/*?q=* — none of these are fetched; bingbot gets Crawl-delay 5, we wait 3 s between requests. The Terms & Conditions page is client-rendered and its text could not be read over plain HTTP (recorded here, to be re-checked). Sales: closed lots whose auctioneer publishes 'price realized' (many auctioneers hide it → 0 lots, skipped); the figure is the hammer/high bid — the buyer's premium (text like 'Buyers Premium 18%' kept in metadata) is charged separately → buyerPremiumIncluded=false. Sale date = auction bidCloseDateTime (zone-less local time treated as UTC, an approximation of a few hours). Currency = the auction's currencyAbbreviation (USD/CAD; others skipped). Only lots whose HiBid category maps to the collectibles taxonomy are kept — real estate, vehicles, farm/construction equipment, household, tools, business liquidations are filtered; firearms and ammunition are never emitted. Auction house name = the auctioneer's public business name + '(HiBid)'; auctioneer contact details (email, phone, address) are never stored. 0 credits.", | |
| 30 | + "enabled": true, | |
| 31 | + "schemaVersion": "1.0", | |
| 32 | + "config": { | |
| 33 | + "pastPagesPerRun": 3, | |
| 34 | + "auctionsPerRun": 6, | |
| 35 | + "pagesPerAuction": 10 | |
| 36 | + } | |
| 37 | +} | |
added
connectors/api/hobbiesville/README.md
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +# Hobbiesville connector (`hobbiesville`) | |
| 2 | + | |
| 3 | +- Source: https://www.hobbiesville.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Toronto TCG + toy retailer. Shopify storefront; MTG singles titles embed "(SET-number)"; also Funko, LEGO and Japanese Pokémon sealed. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `magic-singles` | `magic_the_gathering` | Magic: The Gathering | | |
| 14 | +| `fab-singles` | `flesh_and_blood` | Flesh and Blood | | |
| 15 | +| `disney-lorcana-singles` | `disney_lorcana` | Disney Lorcana | | |
| 16 | +| `gundam-card-game-singles` | `other_tcg` | Gundam Card Game | | |
| 17 | +| `japanese-pokemon` | `pokemon` | Pokémon | | |
| 18 | +| `bf-yugioh` | `yugioh` | Yu-Gi-Oh! | | |
| 19 | +| `funko-pop` | `funko` | Funko | | |
| 20 | +| `lego` | `lego_sets` | LEGO | | |
| 21 | +| `lego-star-wars` | `lego_sets` | LEGO / Star Wars | | |
| 22 | +| `banpresto-one-piece` | `action_figures` | Banpresto / One Piece | | |
| 23 | + | |
| 24 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 25 | +- `\blego\b[\s\S]*minifig|minifig[\s\S]*\blego\b` → `lego_minifigures` (brand LEGO) | |
| 26 | + | |
| 27 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live` | |
| 28 | + | |
| 29 | +Title pattern: `^(?<name>.+?) \((?<set>[A-Z0-9]{2,6})-(?<number>[A-Za-z0-9]+)\)` (name / set / number) | |
| 30 | + | |
| 31 | +## Access & compliance | |
| 32 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: gift-guide/sale umbrella collections, board games, blind boxes, supplies. | |
| 33 | + | |
| 34 | +## Fixtures & tests | |
| 35 | +`data/fixtures/hobbiesville/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts hobbiesville`), trimmed single-product payloads incl. a sold-out variant. | |
| 36 | +`pnpm vitest run connectors/api/hobbiesville` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts hobbiesville`. | |
added
connectors/api/hobbiesville/index.test.ts
+32 −0
@@ -0,0 +1,32 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Hobbiesville — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/hobbiesville/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "Pyrohemia (PLC-119): (colorshifted)", | |
| 15 | + "productType": "Single", | |
| 16 | + "collection": "magic-singles", | |
| 17 | + "categorySlug": "magic_the_gathering", | |
| 18 | + "franchise": "Magic: The Gathering" | |
| 19 | + }, | |
| 20 | + { | |
| 21 | + "title": "LEGO Star Wars Minifigure - Darth Vader", | |
| 22 | + "collection": "lego-star-wars", | |
| 23 | + "categorySlug": "lego_minifigures", | |
| 24 | + "brand": "LEGO" | |
| 25 | + }, | |
| 26 | + { | |
| 27 | + "title": "Ultimate Guard Binder 9-Pocket", | |
| 28 | + "collection": "magic-singles", | |
| 29 | + "categorySlug": null | |
| 30 | + } | |
| 31 | + ] | |
| 32 | +}); | |
added
connectors/api/hobbiesville/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Hobbiesville — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/hobbiesville/meta.json
+125 −0
@@ -0,0 +1,125 @@ | ||
| 1 | +{ | |
| 2 | + "id": "hobbiesville", | |
| 3 | + "displayName": "Hobbiesville (Canadian TCG & collector-toy store, CAD)", | |
| 4 | + "sourceId": "hobbiesville", | |
| 5 | + "sourceName": "Hobbiesville", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.hobbiesville.com", | |
| 8 | + "module": "api/hobbiesville", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "magic_the_gathering", | |
| 14 | + "flesh_and_blood", | |
| 15 | + "disney_lorcana", | |
| 16 | + "other_tcg", | |
| 17 | + "pokemon", | |
| 18 | + "yugioh", | |
| 19 | + "funko", | |
| 20 | + "lego_sets", | |
| 21 | + "action_figures", | |
| 22 | + "lego_minifigures" | |
| 23 | + ], | |
| 24 | + "regions": [ | |
| 25 | + "CA" | |
| 26 | + ], | |
| 27 | + "languages": [ | |
| 28 | + "en" | |
| 29 | + ], | |
| 30 | + "currency": [ | |
| 31 | + "CAD" | |
| 32 | + ], | |
| 33 | + "supportsListings": true, | |
| 34 | + "supportsSold": false, | |
| 35 | + "supportsAuctions": false, | |
| 36 | + "supportsImages": true, | |
| 37 | + "supportsCatalog": false, | |
| 38 | + "supportsPopulation": false, | |
| 39 | + "supportsLookup": true, | |
| 40 | + "refreshFrequencyMinutes": 720, | |
| 41 | + "priority": "medium", | |
| 42 | + "trustScore": 0.75, | |
| 43 | + "attributionRequired": true, | |
| 44 | + "termsUrl": "https://www.hobbiesville.com/policies/terms-of-service", | |
| 45 | + "accessNotes": "Hobbiesville (hobbiesville.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 10 configured collections (magic-singles, fab-singles, disney-lorcana-singles, gundam-card-game-singles, japanese-pokemon, bf-yugioh, funko-pop, lego … (+2 more, see config.collections)) and /products/<handle>.json for URL lookups (~200k products; 21k singles, 8.4k MTG singles). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed; extra merchant rules only target Nutch (Disallow: /) and Ahrefs/MJ12 (Crawl-delay: 10), not User-agent *. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — gift-guide/sale umbrella collections, board games, blind boxes, supplies. No personal data is collected; seller = the store itself.", | |
| 46 | + "enabled": true, | |
| 47 | + "schemaVersion": "1.0", | |
| 48 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 49 | + "historicalDepth": "none", | |
| 50 | + "requires": [], | |
| 51 | + "config": { | |
| 52 | + "currency": "CAD", | |
| 53 | + "market": "CA", | |
| 54 | + "seller": "Hobbiesville", | |
| 55 | + "location": "Toronto, ON, Canada", | |
| 56 | + "collections": [ | |
| 57 | + { | |
| 58 | + "handle": "magic-singles", | |
| 59 | + "categorySlug": "magic_the_gathering", | |
| 60 | + "franchise": "Magic: The Gathering" | |
| 61 | + }, | |
| 62 | + { | |
| 63 | + "handle": "fab-singles", | |
| 64 | + "categorySlug": "flesh_and_blood", | |
| 65 | + "franchise": "Flesh and Blood" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "handle": "disney-lorcana-singles", | |
| 69 | + "categorySlug": "disney_lorcana", | |
| 70 | + "franchise": "Disney Lorcana" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "gundam-card-game-singles", | |
| 74 | + "categorySlug": "other_tcg", | |
| 75 | + "franchise": "Gundam Card Game" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "handle": "japanese-pokemon", | |
| 79 | + "categorySlug": "pokemon", | |
| 80 | + "franchise": "Pokémon" | |
| 81 | + }, | |
| 82 | + { | |
| 83 | + "handle": "bf-yugioh", | |
| 84 | + "categorySlug": "yugioh", | |
| 85 | + "franchise": "Yu-Gi-Oh!" | |
| 86 | + }, | |
| 87 | + { | |
| 88 | + "handle": "funko-pop", | |
| 89 | + "categorySlug": "funko", | |
| 90 | + "brand": "Funko" | |
| 91 | + }, | |
| 92 | + { | |
| 93 | + "handle": "lego", | |
| 94 | + "categorySlug": "lego_sets", | |
| 95 | + "brand": "LEGO" | |
| 96 | + }, | |
| 97 | + { | |
| 98 | + "handle": "lego-star-wars", | |
| 99 | + "categorySlug": "lego_sets", | |
| 100 | + "brand": "LEGO", | |
| 101 | + "franchise": "Star Wars" | |
| 102 | + }, | |
| 103 | + { | |
| 104 | + "handle": "banpresto-one-piece", | |
| 105 | + "categorySlug": "action_figures", | |
| 106 | + "brand": "Banpresto", | |
| 107 | + "franchise": "One Piece" | |
| 108 | + } | |
| 109 | + ], | |
| 110 | + "rules": [ | |
| 111 | + { | |
| 112 | + "match": "\\blego\\b[\\s\\S]*minifig|minifig[\\s\\S]*\\blego\\b", | |
| 113 | + "categorySlug": "lego_minifigures", | |
| 114 | + "brand": "LEGO" | |
| 115 | + } | |
| 116 | + ], | |
| 117 | + "defaultCategory": null, | |
| 118 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live", | |
| 119 | + "keepOutOfStock": true, | |
| 120 | + "fetchBarcodes": false, | |
| 121 | + "wholeShop": false, | |
| 122 | + "pageSize": 250, | |
| 123 | + "titlePattern": "^(?<name>.+?) \\((?<set>[A-Z0-9]{2,6})-(?<number>[A-Za-z0-9]+)\\)" | |
| 124 | + } | |
| 125 | +} | |
added
connectors/api/hodinkee-shop/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# HODINKEE Shop connector (`hodinkee-shop`) | |
| 2 | + | |
| 3 | +- Source: https://shop.hodinkee.com · HODINKEE's authorised-retail and pre-owned/vintage watch shop (New York): pre-owned Rolex, Cartier, Omega, Grand Seiko, vintage chronographs (Universal Genève, Zenith, Heuer, Patek, AP) and limited editions for HODINKEE. Rich structured tags (approximate_age, case_size, materials, box/papers). Straps, accessories, books and Apple Watch are excluded. | |
| 4 | +- Country/currency: US / USD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 24 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, a `titlePattern` that extracts name/set/number, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (19): `vintage-watches` (rules decide), `vintage-rolex` (rules decide), `rolex-brand` (rules decide), `omega-brand` (rules decide), `vintage-omega-watches` (rules decide), `cartier-brand` (rules decide), `grand-seiko-brand` (rules decide), `vintage-patek-philippe` (rules decide), `vintage-audemars-piguet-watches` (rules decide), `vintage-tudor-watches` (rules decide), `vintage-tag-heuer-watches` (rules decide), `vintage-universal-geneve-watches` (rules decide), `vintage-zenith-watches` (rules decide), `vintage-vacheron-constantin-watches` (rules decide), `pre-owned-new-additions` (rules decide), `dress-watches` (rules decide), `chronograph-watches` (rules decide), `aviation-watches` (rules decide), `avant-garde-watches` (rules decide). | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/hodinkee-shop/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts hodinkee-shop`. | |
added
connectors/api/hodinkee-shop/index.test.ts
+90 −0
@@ -0,0 +1,90 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('hodinkee-shop', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "1979 Rolex Sea-Dweller Ref. 1665 'Great White'", | |
| 18 | + "collection": "vintage-watches", | |
| 19 | + "type": "Watches : Vintage : Vintage", | |
| 20 | + "vendor": "Rolex", | |
| 21 | + "tags": [ | |
| 22 | + "Rolex", | |
| 23 | + "vintage" | |
| 24 | + ], | |
| 25 | + "expect": "rolex", | |
| 26 | + "brand": "Rolex", | |
| 27 | + "name": "1979 Rolex Sea-Dweller", | |
| 28 | + "number": "1665", | |
| 29 | + "year": 1979 | |
| 30 | + }, | |
| 31 | + { | |
| 32 | + "title": "Rolex Explorer 14270", | |
| 33 | + "collection": "rolex-brand", | |
| 34 | + "type": "Watches : Pre-Owned : Pre-Owned", | |
| 35 | + "vendor": "Rolex", | |
| 36 | + "tags": [ | |
| 37 | + "approximate_age:1990s", | |
| 38 | + "brand_name:Rolex" | |
| 39 | + ], | |
| 40 | + "expect": "rolex", | |
| 41 | + "number": null | |
| 42 | + }, | |
| 43 | + { | |
| 44 | + "title": "1969 Omega Speedmaster Professional Ref. 145.022-69ST On Bracelet", | |
| 45 | + "collection": "omega-brand", | |
| 46 | + "type": "Watches : Vintage : Vintage", | |
| 47 | + "vendor": "OMEGA", | |
| 48 | + "expect": "omega", | |
| 49 | + "number": "145.022-69ST", | |
| 50 | + "year": 1969 | |
| 51 | + }, | |
| 52 | + { | |
| 53 | + "title": "1970s Audemars Piguet Royal Oak Reference 5402SA", | |
| 54 | + "collection": "vintage-watches", | |
| 55 | + "type": "Watches : Vintage : Vintage", | |
| 56 | + "vendor": "HODINKEE Shop", | |
| 57 | + "expect": "audemars_piguet", | |
| 58 | + "year": null | |
| 59 | + }, | |
| 60 | + { | |
| 61 | + "title": "Cartier Tank Americaine W2609156", | |
| 62 | + "collection": "cartier-brand", | |
| 63 | + "type": "Watches : Pre-Owned : Pre-Owned", | |
| 64 | + "vendor": "Cartier", | |
| 65 | + "expect": "other_watches" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "title": "Automatic GMT SBGM239 Limited Edition For Hodinkee", | |
| 69 | + "collection": "grand-seiko-brand", | |
| 70 | + "type": "Limited Editions : New Watches : New Watch", | |
| 71 | + "vendor": "Grand Seiko", | |
| 72 | + "expect": "other_watches" | |
| 73 | + }, | |
| 74 | + { | |
| 75 | + "title": "Rolex Datejust Two-Tone Leather Strap Brown", | |
| 76 | + "collection": "rolex-brand", | |
| 77 | + "type": "Accessories : Straps : Leather", | |
| 78 | + "vendor": "HODINKEE Shop", | |
| 79 | + "expect": null | |
| 80 | + }, | |
| 81 | + { | |
| 82 | + "title": "Modello Uno U1S-Carbon GMT Limited Edition For Hodinkee 2", | |
| 83 | + "collection": "vintage-watches", | |
| 84 | + "type": "", | |
| 85 | + "vendor": "HODINKEE Shop", | |
| 86 | + "expect": null | |
| 87 | + } | |
| 88 | + ]); | |
| 89 | + }); | |
| 90 | +}); | |
added
connectors/api/hodinkee-shop/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * HODINKEE Shop — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/hodinkee-shop/meta.json
+163 −0
@@ -0,0 +1,163 @@ | ||
| 1 | +{ | |
| 2 | + "id": "hodinkee-shop", | |
| 3 | + "displayName": "HODINKEE Shop", | |
| 4 | + "sourceId": "hodinkee-shop", | |
| 5 | + "sourceName": "HODINKEE Shop", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://shop.hodinkee.com", | |
| 8 | + "module": "api/hodinkee-shop", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "rolex", | |
| 14 | + "omega", | |
| 15 | + "patek_philippe", | |
| 16 | + "audemars_piguet", | |
| 17 | + "other_watches" | |
| 18 | + ], | |
| 19 | + "regions": [ | |
| 20 | + "US" | |
| 21 | + ], | |
| 22 | + "country": "US", | |
| 23 | + "languages": [ | |
| 24 | + "en" | |
| 25 | + ], | |
| 26 | + "currency": [ | |
| 27 | + "USD" | |
| 28 | + ], | |
| 29 | + "supportsListings": true, | |
| 30 | + "supportsSold": false, | |
| 31 | + "supportsAuctions": false, | |
| 32 | + "supportsImages": true, | |
| 33 | + "supportsCatalog": false, | |
| 34 | + "supportsPopulation": false, | |
| 35 | + "supportsLookup": true, | |
| 36 | + "refreshFrequencyMinutes": 1440, | |
| 37 | + "priority": "medium", | |
| 38 | + "trustScore": 0.75, | |
| 39 | + "attributionRequired": true, | |
| 40 | + "termsUrl": "https://shop.hodinkee.com/policies/terms-of-service", | |
| 41 | + "accessNotes": "HODINKEE Shop (shop.hodinkee.com) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (vintage-watches, vintage-rolex, rolex-brand, omega-brand, vintage-omega-watches, cartier-brand, grand-seiko-brand, vintage-patek-philippe, vintage-audemars-piguet-watches, vintage-tudor-watches, vintage-tag-heuer-watches, vintage-universal-geneve-watches, vintage-zenith-watches, vintage-vacheron-constantin-watches, pre-owned-new-additions, dress-watches, chronograph-watches, aviation-watches, avant-garde-watches). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, once a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency USD (/meta.json currency USD, Shopify.currency rate 1.0). Sold vintage/pre-owned pieces stay published as out-of-stock products, so most watch listings arrive as availability \"ended\" with their last asking price — useful price history, but never treated as a realised sale. Only products whose product_type is \"Watches : …\" or \"… : New Watch\" are mapped; everything else in a watch collection (straps, pouches, books) is skipped. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 42 | + "enabled": true, | |
| 43 | + "schemaVersion": "1.0", | |
| 44 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 45 | + "historicalDepth": "none", | |
| 46 | + "requires": [], | |
| 47 | + "config": { | |
| 48 | + "currency": "USD", | |
| 49 | + "seller": "HODINKEE Shop", | |
| 50 | + "location": null, | |
| 51 | + "collections": [ | |
| 52 | + { | |
| 53 | + "handle": "vintage-watches", | |
| 54 | + "categorySlug": null | |
| 55 | + }, | |
| 56 | + { | |
| 57 | + "handle": "vintage-rolex", | |
| 58 | + "categorySlug": null | |
| 59 | + }, | |
| 60 | + { | |
| 61 | + "handle": "rolex-brand", | |
| 62 | + "categorySlug": null | |
| 63 | + }, | |
| 64 | + { | |
| 65 | + "handle": "omega-brand", | |
| 66 | + "categorySlug": null | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "handle": "vintage-omega-watches", | |
| 70 | + "categorySlug": null | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "cartier-brand", | |
| 74 | + "categorySlug": null | |
| 75 | + }, | |
| 76 | + { | |
| 77 | + "handle": "grand-seiko-brand", | |
| 78 | + "categorySlug": null | |
| 79 | + }, | |
| 80 | + { | |
| 81 | + "handle": "vintage-patek-philippe", | |
| 82 | + "categorySlug": null | |
| 83 | + }, | |
| 84 | + { | |
| 85 | + "handle": "vintage-audemars-piguet-watches", | |
| 86 | + "categorySlug": null | |
| 87 | + }, | |
| 88 | + { | |
| 89 | + "handle": "vintage-tudor-watches", | |
| 90 | + "categorySlug": null | |
| 91 | + }, | |
| 92 | + { | |
| 93 | + "handle": "vintage-tag-heuer-watches", | |
| 94 | + "categorySlug": null | |
| 95 | + }, | |
| 96 | + { | |
| 97 | + "handle": "vintage-universal-geneve-watches", | |
| 98 | + "categorySlug": null | |
| 99 | + }, | |
| 100 | + { | |
| 101 | + "handle": "vintage-zenith-watches", | |
| 102 | + "categorySlug": null | |
| 103 | + }, | |
| 104 | + { | |
| 105 | + "handle": "vintage-vacheron-constantin-watches", | |
| 106 | + "categorySlug": null | |
| 107 | + }, | |
| 108 | + { | |
| 109 | + "handle": "pre-owned-new-additions", | |
| 110 | + "categorySlug": null | |
| 111 | + }, | |
| 112 | + { | |
| 113 | + "handle": "dress-watches", | |
| 114 | + "categorySlug": null | |
| 115 | + }, | |
| 116 | + { | |
| 117 | + "handle": "chronograph-watches", | |
| 118 | + "categorySlug": null | |
| 119 | + }, | |
| 120 | + { | |
| 121 | + "handle": "aviation-watches", | |
| 122 | + "categorySlug": null | |
| 123 | + }, | |
| 124 | + { | |
| 125 | + "handle": "avant-garde-watches", | |
| 126 | + "categorySlug": null | |
| 127 | + } | |
| 128 | + ], | |
| 129 | + "rules": [ | |
| 130 | + { | |
| 131 | + "match": "^(?=.*(?:Watches : |New Watch))(?=.*\\brolex\\b)", | |
| 132 | + "categorySlug": "rolex", | |
| 133 | + "brand": "Rolex" | |
| 134 | + }, | |
| 135 | + { | |
| 136 | + "match": "^(?=.*(?:Watches : |New Watch))(?=.*\\bomega\\b)", | |
| 137 | + "categorySlug": "omega", | |
| 138 | + "brand": "Omega" | |
| 139 | + }, | |
| 140 | + { | |
| 141 | + "match": "^(?=.*(?:Watches : |New Watch))(?=.*patek)", | |
| 142 | + "categorySlug": "patek_philippe", | |
| 143 | + "brand": "Patek Philippe" | |
| 144 | + }, | |
| 145 | + { | |
| 146 | + "match": "^(?=.*(?:Watches : |New Watch))(?=.*(audemars|royal oak))", | |
| 147 | + "categorySlug": "audemars_piguet", | |
| 148 | + "brand": "Audemars Piguet" | |
| 149 | + }, | |
| 150 | + { | |
| 151 | + "match": "Watches : |New Watch", | |
| 152 | + "categorySlug": "other_watches" | |
| 153 | + } | |
| 154 | + ], | |
| 155 | + "defaultCategory": null, | |
| 156 | + "exclude": "apple watch|smartwatch|gift card|\\bstraps?\\b(?!.*Watches :)|watch roll|pouch|travel case|storage|tool kit|spring bar|magazine|\\bbook\\b|assouline|\\bpen\\b|candle|umbrella|t-shirt|\\bcap\\b|\\bhat\\b|\\bbag\\b|wallet|notebook|puzzle|blanket|glass\\b|loupe|winder|cufflink|sunglasses|camera|leica|\\bfilm\\b|poster|\\bprint\\b|bundle", | |
| 157 | + "keepOutOfStock": true, | |
| 158 | + "titlePattern": "^(?<name>.+?) Ref\\.? (?<number>[\\w./-]+)", | |
| 159 | + "pageSize": 250, | |
| 160 | + "fetchBarcodes": false, | |
| 161 | + "wholeShop": false | |
| 162 | + } | |
| 163 | +} | |
added
connectors/api/kanzen-games/README.md
+47 −0
@@ -0,0 +1,47 @@ | ||
| 1 | +# KanZen Games connector (`kanzen-games`) | |
| 2 | + | |
| 3 | +- Source: https://www.kanzengames.com · dealer · CA · CAD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=CA` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Toronto TCG shop with a dedicated PSA "slabs" collection (Japanese Pokémon heavy), Lorcana/Gundam/Digimon singles and sealed product. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `slabs` | `pokemon` | Pokémon | | |
| 14 | +| `graded-card-in-stock` | `pokemon` | Pokémon | | |
| 15 | +| `ebay-pokemon-singles` | `pokemon` | Pokémon | | |
| 16 | +| `pokemon-tcg-mega-evolution` | `pokemon` | Pokémon | | |
| 17 | +| `japanese-sealed` | `pokemon` | Pokémon | | |
| 18 | +| `magic-the-gathering` | `magic_the_gathering` | Magic: The Gathering | | |
| 19 | +| `disney-lorcana-singles-1` | `disney_lorcana` | Disney Lorcana | | |
| 20 | +| `gundam-singles-all` | `other_tcg` | Gundam Card Game | | |
| 21 | +| `digimon-single` | `digimon_tcg` | Digimon | | |
| 22 | +| `digimon-sealed` | `digimon_tcg` | Digimon | | |
| 23 | +| `one-piece-tcg-singles-all` | `one_piece_card_game` | One Piece | | |
| 24 | +| `dragon-ball-tcg` | `dragon_ball_tcg` | Dragon Ball Super | | |
| 25 | +| `hololive-tcg-sealed` | `other_tcg` | hololive | | |
| 26 | +| `football-sealed` | `football_cards` | — | | |
| 27 | +| `basketball-sealed-all` | `basketball_cards` | — | | |
| 28 | + | |
| 29 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 30 | +- `\bone piece\b|\bOP\d{2}\b` → `one_piece_card_game` (franchise One Piece) | |
| 31 | +- `\bmagic\b|\bmtg\b` → `magic_the_gathering` (franchise Magic: The Gathering) | |
| 32 | +- `yu-?gi-?oh` → `yugioh` (franchise Yu-Gi-Oh!) | |
| 33 | +- `\blorcana\b` → `disney_lorcana` (franchise Disney Lorcana) | |
| 34 | +- `\bdigimon\b` → `digimon_tcg` (franchise Digimon) | |
| 35 | +- `\bdragon ball\b` → `dragon_ball_tcg` (franchise Dragon Ball Super) | |
| 36 | +- `\bweiss\b` → `weiss_schwarz` (franchise Weiß Schwarz) | |
| 37 | + | |
| 38 | +Excluded (regex): `gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\bentry\b|tournament|store credit|pre-?release event|supplies|card savers?|\brepack|bulk lot|\blot of\b|\bpaints?\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live` | |
| 39 | + | |
| 40 | +Title pattern: `^(?:PSA|BGS|CGC|TAG|ACE|SGC)\s*[\d.]+\s*-\s*(?<name>.+?)\s*-\s*(?<set>.+?)\s*-\s*#(?<number>\S+)` (name / set / number) | |
| 41 | + | |
| 42 | +## Access & compliance | |
| 43 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: deck-box/sleeve/binder collections, "all products" umbrella collections, tournament entries. | |
| 44 | + | |
| 45 | +## Fixtures & tests | |
| 46 | +`data/fixtures/kanzen-games/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts kanzen-games`), trimmed single-product payloads incl. a sold-out variant and a graded item. | |
| 47 | +`pnpm vitest run connectors/api/kanzen-games` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts kanzen-games`. | |
added
connectors/api/kanzen-games/index.test.ts
+30 −0
@@ -0,0 +1,30 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * KanZen Games — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/kanzen-games/ (runFixtureSuite invariants + CAD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "PSA 10 - Mew - Dream Shine Collection - #017 - 74542930 - Japanese", | |
| 15 | + "productType": "Graded Slab", | |
| 16 | + "collection": "slabs", | |
| 17 | + "categorySlug": "pokemon" | |
| 18 | + }, | |
| 19 | + { | |
| 20 | + "title": "PSA 9 - Monkey D. Luffy - One Piece Romance Dawn - #OP01-003", | |
| 21 | + "collection": "slabs", | |
| 22 | + "categorySlug": "one_piece_card_game" | |
| 23 | + }, | |
| 24 | + { | |
| 25 | + "title": "Dragon Shield Sleeve - Brushed Art", | |
| 26 | + "collection": "magic-the-gathering", | |
| 27 | + "categorySlug": null | |
| 28 | + } | |
| 29 | + ] | |
| 30 | +}); | |
added
connectors/api/kanzen-games/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * KanZen Games — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/kanzen-games/meta.json
+177 −0
@@ -0,0 +1,177 @@ | ||
| 1 | +{ | |
| 2 | + "id": "kanzen-games", | |
| 3 | + "displayName": "KanZen Games (Canadian TCG & sports-card store, CAD)", | |
| 4 | + "sourceId": "kanzen-games", | |
| 5 | + "sourceName": "KanZen Games", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.kanzengames.com", | |
| 8 | + "module": "api/kanzen-games", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "pokemon", | |
| 14 | + "magic_the_gathering", | |
| 15 | + "disney_lorcana", | |
| 16 | + "other_tcg", | |
| 17 | + "digimon_tcg", | |
| 18 | + "one_piece_card_game", | |
| 19 | + "dragon_ball_tcg", | |
| 20 | + "football_cards", | |
| 21 | + "basketball_cards", | |
| 22 | + "yugioh", | |
| 23 | + "weiss_schwarz" | |
| 24 | + ], | |
| 25 | + "regions": [ | |
| 26 | + "CA" | |
| 27 | + ], | |
| 28 | + "languages": [ | |
| 29 | + "en" | |
| 30 | + ], | |
| 31 | + "currency": [ | |
| 32 | + "CAD" | |
| 33 | + ], | |
| 34 | + "supportsListings": true, | |
| 35 | + "supportsSold": false, | |
| 36 | + "supportsAuctions": false, | |
| 37 | + "supportsImages": true, | |
| 38 | + "supportsCatalog": false, | |
| 39 | + "supportsPopulation": false, | |
| 40 | + "supportsLookup": true, | |
| 41 | + "refreshFrequencyMinutes": 720, | |
| 42 | + "priority": "medium", | |
| 43 | + "trustScore": 0.75, | |
| 44 | + "attributionRequired": true, | |
| 45 | + "termsUrl": "https://www.kanzengames.com/policies/terms-of-service", | |
| 46 | + "accessNotes": "KanZen Games (kanzengames.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 15 configured collections (slabs, graded-card-in-stock, ebay-pokemon-singles, pokemon-tcg-mega-evolution, japanese-sealed, magic-the-gathering, disney-lorcana-singles-1, gundam-singles-all … (+7 more, see config.collections)) and /products/<handle>.json for URL lookups (~63k products; 53k TCG singles, 750 PSA slabs). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Third-party grades in titles (PSA/BGS/CGC/ICCS/PMG…) are parsed by parseGradeFromTitle; cert numbers are not extracted. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — deck-box/sleeve/binder collections, \"all products\" umbrella collections, tournament entries. No personal data is collected; seller = the store itself.", | |
| 47 | + "enabled": true, | |
| 48 | + "schemaVersion": "1.0", | |
| 49 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 50 | + "historicalDepth": "none", | |
| 51 | + "requires": [], | |
| 52 | + "config": { | |
| 53 | + "currency": "CAD", | |
| 54 | + "market": "CA", | |
| 55 | + "seller": "KanZen Games", | |
| 56 | + "location": "Toronto, ON, Canada", | |
| 57 | + "collections": [ | |
| 58 | + { | |
| 59 | + "handle": "slabs", | |
| 60 | + "categorySlug": "pokemon", | |
| 61 | + "franchise": "Pokémon" | |
| 62 | + }, | |
| 63 | + { | |
| 64 | + "handle": "graded-card-in-stock", | |
| 65 | + "categorySlug": "pokemon", | |
| 66 | + "franchise": "Pokémon" | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "handle": "ebay-pokemon-singles", | |
| 70 | + "categorySlug": "pokemon", | |
| 71 | + "franchise": "Pokémon" | |
| 72 | + }, | |
| 73 | + { | |
| 74 | + "handle": "pokemon-tcg-mega-evolution", | |
| 75 | + "categorySlug": "pokemon", | |
| 76 | + "franchise": "Pokémon" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "japanese-sealed", | |
| 80 | + "categorySlug": "pokemon", | |
| 81 | + "franchise": "Pokémon" | |
| 82 | + }, | |
| 83 | + { | |
| 84 | + "handle": "magic-the-gathering", | |
| 85 | + "categorySlug": "magic_the_gathering", | |
| 86 | + "franchise": "Magic: The Gathering" | |
| 87 | + }, | |
| 88 | + { | |
| 89 | + "handle": "disney-lorcana-singles-1", | |
| 90 | + "categorySlug": "disney_lorcana", | |
| 91 | + "franchise": "Disney Lorcana" | |
| 92 | + }, | |
| 93 | + { | |
| 94 | + "handle": "gundam-singles-all", | |
| 95 | + "categorySlug": "other_tcg", | |
| 96 | + "franchise": "Gundam Card Game" | |
| 97 | + }, | |
| 98 | + { | |
| 99 | + "handle": "digimon-single", | |
| 100 | + "categorySlug": "digimon_tcg", | |
| 101 | + "franchise": "Digimon" | |
| 102 | + }, | |
| 103 | + { | |
| 104 | + "handle": "digimon-sealed", | |
| 105 | + "categorySlug": "digimon_tcg", | |
| 106 | + "franchise": "Digimon" | |
| 107 | + }, | |
| 108 | + { | |
| 109 | + "handle": "one-piece-tcg-singles-all", | |
| 110 | + "categorySlug": "one_piece_card_game", | |
| 111 | + "franchise": "One Piece" | |
| 112 | + }, | |
| 113 | + { | |
| 114 | + "handle": "dragon-ball-tcg", | |
| 115 | + "categorySlug": "dragon_ball_tcg", | |
| 116 | + "franchise": "Dragon Ball Super" | |
| 117 | + }, | |
| 118 | + { | |
| 119 | + "handle": "hololive-tcg-sealed", | |
| 120 | + "categorySlug": "other_tcg", | |
| 121 | + "franchise": "hololive" | |
| 122 | + }, | |
| 123 | + { | |
| 124 | + "handle": "football-sealed", | |
| 125 | + "categorySlug": "football_cards" | |
| 126 | + }, | |
| 127 | + { | |
| 128 | + "handle": "basketball-sealed-all", | |
| 129 | + "categorySlug": "basketball_cards" | |
| 130 | + } | |
| 131 | + ], | |
| 132 | + "rules": [ | |
| 133 | + { | |
| 134 | + "match": "\\bone piece\\b|\\bOP\\d{2}\\b", | |
| 135 | + "categorySlug": "one_piece_card_game", | |
| 136 | + "franchise": "One Piece" | |
| 137 | + }, | |
| 138 | + { | |
| 139 | + "match": "\\bmagic\\b|\\bmtg\\b", | |
| 140 | + "categorySlug": "magic_the_gathering", | |
| 141 | + "franchise": "Magic: The Gathering" | |
| 142 | + }, | |
| 143 | + { | |
| 144 | + "match": "yu-?gi-?oh", | |
| 145 | + "categorySlug": "yugioh", | |
| 146 | + "franchise": "Yu-Gi-Oh!" | |
| 147 | + }, | |
| 148 | + { | |
| 149 | + "match": "\\blorcana\\b", | |
| 150 | + "categorySlug": "disney_lorcana", | |
| 151 | + "franchise": "Disney Lorcana" | |
| 152 | + }, | |
| 153 | + { | |
| 154 | + "match": "\\bdigimon\\b", | |
| 155 | + "categorySlug": "digimon_tcg", | |
| 156 | + "franchise": "Digimon" | |
| 157 | + }, | |
| 158 | + { | |
| 159 | + "match": "\\bdragon ball\\b", | |
| 160 | + "categorySlug": "dragon_ball_tcg", | |
| 161 | + "franchise": "Dragon Ball Super" | |
| 162 | + }, | |
| 163 | + { | |
| 164 | + "match": "\\bweiss\\b", | |
| 165 | + "categorySlug": "weiss_schwarz", | |
| 166 | + "franchise": "Weiß Schwarz" | |
| 167 | + } | |
| 168 | + ], | |
| 169 | + "defaultCategory": null, | |
| 170 | + "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live", | |
| 171 | + "keepOutOfStock": true, | |
| 172 | + "fetchBarcodes": false, | |
| 173 | + "wholeShop": false, | |
| 174 | + "pageSize": 250, | |
| 175 | + "titlePattern": "^(?:PSA|BGS|CGC|TAG|ACE|SGC)\\s*[\\d.]+\\s*-\\s*(?<name>.+?)\\s*-\\s*(?<set>.+?)\\s*-\\s*#(?<number>\\S+)" | |
| 176 | + } | |
| 177 | +} | |
added
connectors/api/kapowtoys/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Kapow Toys connector (`kapowtoys`) | |
| 2 | + | |
| 3 | +- Source: https://www.kapowtoys.co.uk · UK action-figure specialist (WooCommerce): Hot Toys, NECA, Mezco One:12, Marvel Legends, Star Wars Black Series, Masters of the Universe, TMNT, G.I. Joe, McFarlane, Medicom MAFEX, Transformers Masterpiece and 3rd-party Transformers, Gunpla and busts/statues. | |
| 4 | +- Country/currency: GB / GBP · type: dealer · engine: api (WooCommerce Store API (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 24 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `WooCommerceStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (18): `hot-toys` → action_figures, `marvel-hot-toys` → action_figures, `neca` → action_figures, `mezco` → action_figures, `marvel-legends-1` → action_figures, `black-series-6-inch` → action_figures, `masters-of-the-universe-1` → action_figures, `teenage-mutant-ninja-turtles` → action_figures, `g-i-joe` → action_figures, `mcfarlane` → action_figures, `medicon-mafex` → action_figures, `masterpiece` → action_figures, `3rd-party-transformers` → action_figures, `busts-statues` → action_figures, `godzilla` → action_figures, `gundam` → gundam, `good-smile-company` → action_figures, `funko` → funko. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/kapowtoys/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts kapowtoys`. | |
added
connectors/api/kapowtoys/index.test.ts
+60 −0
@@ -0,0 +1,60 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('kapowtoys', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Marvel Legends MCU 10th Anniversary Red Skull Action Figure", | |
| 18 | + "collection": "marvel-legends-1", | |
| 19 | + "type": "Marvel Legends / New Arrival", | |
| 20 | + "expect": "action_figures", | |
| 21 | + "brand": "Hasbro", | |
| 22 | + "series": "Marvel Legends", | |
| 23 | + "franchise": "Marvel" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "title": "Hot Toys Iron Man Mark 85 Battle Damaged 1/6th Scale Figure", | |
| 27 | + "collection": "hot-toys", | |
| 28 | + "type": "Hot Toys", | |
| 29 | + "expect": "action_figures", | |
| 30 | + "brand": "Hot Toys" | |
| 31 | + }, | |
| 32 | + { | |
| 33 | + "title": "MAFEX No.185 Spider-Man (Classic Costume Ver.)", | |
| 34 | + "collection": "medicon-mafex", | |
| 35 | + "type": "Medicom MAFEX", | |
| 36 | + "expect": "action_figures", | |
| 37 | + "series": "MAFEX" | |
| 38 | + }, | |
| 39 | + { | |
| 40 | + "title": "Bandai MG 1/100 RX-78-2 Gundam Ver.3.0", | |
| 41 | + "collection": "gundam", | |
| 42 | + "type": "Gundam", | |
| 43 | + "expect": "gundam", | |
| 44 | + "franchise": "Gundam" | |
| 45 | + }, | |
| 46 | + { | |
| 47 | + "title": "Kapow Toys Logo T-Shirt Black L", | |
| 48 | + "collection": "hot-toys", | |
| 49 | + "type": "Gifts & Apparel", | |
| 50 | + "expect": null | |
| 51 | + }, | |
| 52 | + { | |
| 53 | + "title": "Hot Toys Pre-Order Deposit", | |
| 54 | + "collection": "hot-toys", | |
| 55 | + "type": "Hot Toys", | |
| 56 | + "expect": null | |
| 57 | + } | |
| 58 | + ]); | |
| 59 | + }); | |
| 60 | +}); | |
added
connectors/api/kapowtoys/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { WooCommerceStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Kapow Toys — woocommerce storefront read through the shared woocommerce adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new WooCommerceStoreConnector(meta); | |
added
connectors/api/kapowtoys/meta.json
+166 −0
@@ -0,0 +1,166 @@ | ||
| 1 | +{ | |
| 2 | + "id": "kapowtoys", | |
| 3 | + "displayName": "Kapow Toys", | |
| 4 | + "sourceId": "kapowtoys", | |
| 5 | + "sourceName": "Kapow Toys", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.kapowtoys.co.uk", | |
| 8 | + "module": "api/kapowtoys", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "action_figures", | |
| 14 | + "gundam", | |
| 15 | + "funko" | |
| 16 | + ], | |
| 17 | + "regions": [ | |
| 18 | + "GB" | |
| 19 | + ], | |
| 20 | + "country": "GB", | |
| 21 | + "languages": [ | |
| 22 | + "en" | |
| 23 | + ], | |
| 24 | + "currency": [ | |
| 25 | + "GBP" | |
| 26 | + ], | |
| 27 | + "supportsListings": true, | |
| 28 | + "supportsSold": false, | |
| 29 | + "supportsAuctions": false, | |
| 30 | + "supportsImages": true, | |
| 31 | + "supportsCatalog": false, | |
| 32 | + "supportsPopulation": false, | |
| 33 | + "supportsLookup": false, | |
| 34 | + "refreshFrequencyMinutes": 1440, | |
| 35 | + "priority": "medium", | |
| 36 | + "trustScore": 0.65, | |
| 37 | + "attributionRequired": true, | |
| 38 | + "termsUrl": "https://www.kapowtoys.co.uk/policies/terms-of-service", | |
| 39 | + "accessNotes": "Kapow Toys (kapowtoys.co.uk) is a WooCommerce shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /wp-json/wc/store/v1/products?per_page=100&page=N&category=<id> (WooCommerce Store API shipped with WooCommerce Blocks, no key) for the configured categories, resolved by slug through /wp-json/wc/store/v1/products/categories (hot-toys, marvel-hot-toys, neca, mezco, marvel-legends-1, black-series-6-inch, masters-of-the-universe-1, teenage-mutant-ninja-turtles, g-i-joe, mcfarlane, medicon-mafex, masterpiece, 3rd-party-transformers, busts-statues, godzilla, gundam, good-smile-company, funko). robots.txt (checked 2026-09-08) only disallows /wp-admin/ (except admin-ajax.php), add-to-cart query strings and the WooCommerce upload/log folders; /wp-json/wc/store/v1/ is not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, once a day, 3 pages of 100 per category per incremental run. Prices come from the Store API in minor units with an explicit currency_code (GBP, currency_minor_unit 2); the adapter uses that code when present. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 40 | + "enabled": true, | |
| 41 | + "schemaVersion": "1.0", | |
| 42 | + "acquisitionMethod": "WooCommerce Store API (public JSON)", | |
| 43 | + "historicalDepth": "none", | |
| 44 | + "requires": [], | |
| 45 | + "config": { | |
| 46 | + "currency": "GBP", | |
| 47 | + "seller": "Kapow Toys", | |
| 48 | + "location": null, | |
| 49 | + "collections": [ | |
| 50 | + { | |
| 51 | + "handle": "hot-toys", | |
| 52 | + "categorySlug": "action_figures", | |
| 53 | + "brand": "Hot Toys" | |
| 54 | + }, | |
| 55 | + { | |
| 56 | + "handle": "marvel-hot-toys", | |
| 57 | + "categorySlug": "action_figures", | |
| 58 | + "brand": "Hot Toys", | |
| 59 | + "franchise": "Marvel" | |
| 60 | + }, | |
| 61 | + { | |
| 62 | + "handle": "neca", | |
| 63 | + "categorySlug": "action_figures", | |
| 64 | + "brand": "NECA" | |
| 65 | + }, | |
| 66 | + { | |
| 67 | + "handle": "mezco", | |
| 68 | + "categorySlug": "action_figures", | |
| 69 | + "brand": "Mezco" | |
| 70 | + }, | |
| 71 | + { | |
| 72 | + "handle": "marvel-legends-1", | |
| 73 | + "categorySlug": "action_figures", | |
| 74 | + "brand": "Hasbro", | |
| 75 | + "series": "Marvel Legends", | |
| 76 | + "franchise": "Marvel" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "black-series-6-inch", | |
| 80 | + "categorySlug": "action_figures", | |
| 81 | + "brand": "Hasbro", | |
| 82 | + "series": "Black Series", | |
| 83 | + "franchise": "Star Wars" | |
| 84 | + }, | |
| 85 | + { | |
| 86 | + "handle": "masters-of-the-universe-1", | |
| 87 | + "categorySlug": "action_figures", | |
| 88 | + "franchise": "Masters of the Universe" | |
| 89 | + }, | |
| 90 | + { | |
| 91 | + "handle": "teenage-mutant-ninja-turtles", | |
| 92 | + "categorySlug": "action_figures", | |
| 93 | + "franchise": "Teenage Mutant Ninja Turtles" | |
| 94 | + }, | |
| 95 | + { | |
| 96 | + "handle": "g-i-joe", | |
| 97 | + "categorySlug": "action_figures", | |
| 98 | + "franchise": "G.I. Joe" | |
| 99 | + }, | |
| 100 | + { | |
| 101 | + "handle": "mcfarlane", | |
| 102 | + "categorySlug": "action_figures", | |
| 103 | + "brand": "McFarlane Toys" | |
| 104 | + }, | |
| 105 | + { | |
| 106 | + "handle": "medicon-mafex", | |
| 107 | + "categorySlug": "action_figures", | |
| 108 | + "brand": "Medicom Toy", | |
| 109 | + "series": "MAFEX" | |
| 110 | + }, | |
| 111 | + { | |
| 112 | + "handle": "masterpiece", | |
| 113 | + "categorySlug": "action_figures", | |
| 114 | + "brand": "Takara Tomy", | |
| 115 | + "series": "Masterpiece", | |
| 116 | + "franchise": "Transformers" | |
| 117 | + }, | |
| 118 | + { | |
| 119 | + "handle": "3rd-party-transformers", | |
| 120 | + "categorySlug": "action_figures", | |
| 121 | + "franchise": "Transformers" | |
| 122 | + }, | |
| 123 | + { | |
| 124 | + "handle": "busts-statues", | |
| 125 | + "categorySlug": "action_figures" | |
| 126 | + }, | |
| 127 | + { | |
| 128 | + "handle": "godzilla", | |
| 129 | + "categorySlug": "action_figures", | |
| 130 | + "franchise": "Godzilla" | |
| 131 | + }, | |
| 132 | + { | |
| 133 | + "handle": "gundam", | |
| 134 | + "categorySlug": "gundam", | |
| 135 | + "brand": "Bandai", | |
| 136 | + "franchise": "Gundam" | |
| 137 | + }, | |
| 138 | + { | |
| 139 | + "handle": "good-smile-company", | |
| 140 | + "categorySlug": "action_figures", | |
| 141 | + "brand": "Good Smile Company" | |
| 142 | + }, | |
| 143 | + { | |
| 144 | + "handle": "funko", | |
| 145 | + "categorySlug": "funko", | |
| 146 | + "brand": "Funko" | |
| 147 | + } | |
| 148 | + ], | |
| 149 | + "rules": [ | |
| 150 | + { | |
| 151 | + "match": "nendoroid", | |
| 152 | + "categorySlug": "action_figures", | |
| 153 | + "series": "Nendoroid" | |
| 154 | + }, | |
| 155 | + { | |
| 156 | + "match": "\\bfigma\\b", | |
| 157 | + "categorySlug": "action_figures", | |
| 158 | + "series": "figma" | |
| 159 | + } | |
| 160 | + ], | |
| 161 | + "defaultCategory": null, | |
| 162 | + "exclude": "gift card|t-shirt|hoodie|\\bcap\\b|apparel|mug\\b|poster|\\bprint\\b|keyring|lanyard|display stand|stand only|protective case|\\bshelf\\b|riser|led light|sticker|pin badge|wallet|\\bbag\\b|backpack|sleeve|\\bbook\\b|art book|puzzle|calendar|voucher|deposit|payment plan|pre-order deposit", | |
| 163 | + "keepOutOfStock": true, | |
| 164 | + "perPage": 100 | |
| 165 | + } | |
| 166 | +} | |
added
connectors/api/kickgame/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Kick Game connector (`kickgame`) | |
| 2 | + | |
| 3 | +- Source: https://www.kickgame.co.uk · London premium sneaker retailer/reseller (since 2013): Air Jordan, Nike Dunk/SB, Yeezy, New Balance, plus Bearbrick, KAWS and Labubu collectibles and Swatch × Omega/Blancpain watches. Size-level variants carry the style code in the SKU (e.g. DM7866-200.3UK). | |
| 4 | +- Country/currency: GB / GBP · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 12 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (10): `sneakers` → sneakers, `air-jordan-1` → nike_jordan, `air-jordan-4` → nike_jordan, `nike-dunk-low` → nike_jordan, `adidas-yeezy` → adidas_yeezy, `new-balance-550` → new_balance_asics_other, `bearbrick` → designer_toys, `kaws` → designer_toys, `labubu` → designer_toys, `watches` → other_watches. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/kickgame/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts kickgame`. | |
added
connectors/api/kickgame/index.test.ts
+91 −0
@@ -0,0 +1,91 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('kickgame', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "Air Jordan 1 Low OG x Travis Scott SP 'Medium Olive'", | |
| 18 | + "collection": "sneakers", | |
| 19 | + "type": "Shoes", | |
| 20 | + "tags": [ | |
| 21 | + "Air Jordan", | |
| 22 | + "Jordan" | |
| 23 | + ], | |
| 24 | + "variant": "UK 8 | EU 42.5 | US 9", | |
| 25 | + "expect": "nike_jordan", | |
| 26 | + "brand": "Jordan" | |
| 27 | + }, | |
| 28 | + { | |
| 29 | + "title": "Nike Dunk Low 'Grey Fog'", | |
| 30 | + "collection": "sneakers", | |
| 31 | + "type": "Nike Shoes", | |
| 32 | + "expect": "nike_jordan", | |
| 33 | + "brand": "Nike" | |
| 34 | + }, | |
| 35 | + { | |
| 36 | + "title": "Adidas Yeezy Boost 350 V2 Core Black-Red", | |
| 37 | + "collection": "sneakers", | |
| 38 | + "type": "Adidas Shoes", | |
| 39 | + "expect": "adidas_yeezy", | |
| 40 | + "brand": "adidas Yeezy" | |
| 41 | + }, | |
| 42 | + { | |
| 43 | + "title": "New Balance 2002R Protection Pack Rain Cloud", | |
| 44 | + "collection": "sneakers", | |
| 45 | + "type": "Shoes", | |
| 46 | + "expect": "new_balance_asics_other" | |
| 47 | + }, | |
| 48 | + { | |
| 49 | + "title": "Bearbrick x Kith Palette 10 Year Anniversary 100% & 400% Set", | |
| 50 | + "collection": "bearbrick", | |
| 51 | + "type": "Collectibles", | |
| 52 | + "expect": "designer_toys", | |
| 53 | + "brand": "Medicom Toy" | |
| 54 | + }, | |
| 55 | + { | |
| 56 | + "title": "Swatch x Omega Bioceramic Moonswatch Mission to the Moon", | |
| 57 | + "collection": "watches", | |
| 58 | + "type": "Accessories", | |
| 59 | + "tags": [ | |
| 60 | + "Omega", | |
| 61 | + "Swatch", | |
| 62 | + "Watch" | |
| 63 | + ], | |
| 64 | + "expect": "other_watches", | |
| 65 | + "brand": "Swatch" | |
| 66 | + }, | |
| 67 | + { | |
| 68 | + "title": "Pop Mart Labubu The Monsters Exciting Macarons (Mystery Box of 1)", | |
| 69 | + "collection": "labubu", | |
| 70 | + "type": "Accessories", | |
| 71 | + "tags": [ | |
| 72 | + "Collectibles", | |
| 73 | + "Pop Mart" | |
| 74 | + ], | |
| 75 | + "expect": "designer_toys" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "title": "KAWS x UNIQLO UT Graphic T-Shirt 'Black'", | |
| 79 | + "collection": "kaws", | |
| 80 | + "type": "Clothing", | |
| 81 | + "expect": null | |
| 82 | + }, | |
| 83 | + { | |
| 84 | + "title": "Fear of God Essentials Hoodie Dark Oatmeal", | |
| 85 | + "collection": "sneakers", | |
| 86 | + "type": "Clothing", | |
| 87 | + "expect": null | |
| 88 | + } | |
| 89 | + ]); | |
| 90 | + }); | |
| 91 | +}); | |
added
connectors/api/kickgame/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Kick Game — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/kickgame/meta.json
+164 −0
@@ -0,0 +1,164 @@ | ||
| 1 | +{ | |
| 2 | + "id": "kickgame", | |
| 3 | + "displayName": "Kick Game", | |
| 4 | + "sourceId": "kickgame", | |
| 5 | + "sourceName": "Kick Game", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.kickgame.co.uk", | |
| 8 | + "module": "api/kickgame", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "sneakers", | |
| 14 | + "nike_jordan", | |
| 15 | + "adidas_yeezy", | |
| 16 | + "new_balance_asics_other", | |
| 17 | + "designer_toys", | |
| 18 | + "other_watches", | |
| 19 | + "rolex", | |
| 20 | + "omega", | |
| 21 | + "patek_philippe", | |
| 22 | + "audemars_piguet" | |
| 23 | + ], | |
| 24 | + "regions": [ | |
| 25 | + "GB" | |
| 26 | + ], | |
| 27 | + "country": "GB", | |
| 28 | + "languages": [ | |
| 29 | + "en" | |
| 30 | + ], | |
| 31 | + "currency": [ | |
| 32 | + "GBP" | |
| 33 | + ], | |
| 34 | + "supportsListings": true, | |
| 35 | + "supportsSold": false, | |
| 36 | + "supportsAuctions": false, | |
| 37 | + "supportsImages": true, | |
| 38 | + "supportsCatalog": false, | |
| 39 | + "supportsPopulation": false, | |
| 40 | + "supportsLookup": true, | |
| 41 | + "refreshFrequencyMinutes": 720, | |
| 42 | + "priority": "medium", | |
| 43 | + "trustScore": 0.7, | |
| 44 | + "attributionRequired": true, | |
| 45 | + "termsUrl": "https://www.kickgame.co.uk/policies/terms-of-service", | |
| 46 | + "accessNotes": "Kick Game (kickgame.co.uk) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (sneakers, air-jordan-1, air-jordan-4, nike-dunk-low, adidas-yeezy, new-balance-550, bearbrick, kaws, labubu, watches). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, twice a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency GBP: the storefront shows visitors a geo-converted presentment currency (Shopify.currency active USD, rate 1.38 from a North-American IP) but products.json and /meta.json (currency GBP, primary domain www.kickgame.co.uk) are GBP. The .com host redirects to .co.uk. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 47 | + "enabled": true, | |
| 48 | + "schemaVersion": "1.0", | |
| 49 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 50 | + "historicalDepth": "none", | |
| 51 | + "requires": [], | |
| 52 | + "config": { | |
| 53 | + "currency": "GBP", | |
| 54 | + "seller": "Kick Game", | |
| 55 | + "location": null, | |
| 56 | + "collections": [ | |
| 57 | + { | |
| 58 | + "handle": "sneakers", | |
| 59 | + "categorySlug": "sneakers" | |
| 60 | + }, | |
| 61 | + { | |
| 62 | + "handle": "air-jordan-1", | |
| 63 | + "categorySlug": "nike_jordan", | |
| 64 | + "brand": "Jordan" | |
| 65 | + }, | |
| 66 | + { | |
| 67 | + "handle": "air-jordan-4", | |
| 68 | + "categorySlug": "nike_jordan", | |
| 69 | + "brand": "Jordan" | |
| 70 | + }, | |
| 71 | + { | |
| 72 | + "handle": "nike-dunk-low", | |
| 73 | + "categorySlug": "nike_jordan", | |
| 74 | + "brand": "Nike" | |
| 75 | + }, | |
| 76 | + { | |
| 77 | + "handle": "adidas-yeezy", | |
| 78 | + "categorySlug": "adidas_yeezy", | |
| 79 | + "brand": "adidas Yeezy" | |
| 80 | + }, | |
| 81 | + { | |
| 82 | + "handle": "new-balance-550", | |
| 83 | + "categorySlug": "new_balance_asics_other", | |
| 84 | + "brand": "New Balance" | |
| 85 | + }, | |
| 86 | + { | |
| 87 | + "handle": "bearbrick", | |
| 88 | + "categorySlug": "designer_toys", | |
| 89 | + "brand": "Medicom Toy" | |
| 90 | + }, | |
| 91 | + { | |
| 92 | + "handle": "kaws", | |
| 93 | + "categorySlug": "designer_toys", | |
| 94 | + "brand": "KAWS" | |
| 95 | + }, | |
| 96 | + { | |
| 97 | + "handle": "labubu", | |
| 98 | + "categorySlug": "designer_toys", | |
| 99 | + "brand": "Pop Mart" | |
| 100 | + }, | |
| 101 | + { | |
| 102 | + "handle": "watches", | |
| 103 | + "categorySlug": "other_watches" | |
| 104 | + } | |
| 105 | + ], | |
| 106 | + "rules": [ | |
| 107 | + { | |
| 108 | + "match": "swatch|moonswatch|scuba fifty", | |
| 109 | + "categorySlug": "other_watches", | |
| 110 | + "brand": "Swatch" | |
| 111 | + }, | |
| 112 | + { | |
| 113 | + "match": "rolex", | |
| 114 | + "categorySlug": "rolex", | |
| 115 | + "brand": "Rolex" | |
| 116 | + }, | |
| 117 | + { | |
| 118 | + "match": "omega", | |
| 119 | + "categorySlug": "omega", | |
| 120 | + "brand": "Omega" | |
| 121 | + }, | |
| 122 | + { | |
| 123 | + "match": "patek", | |
| 124 | + "categorySlug": "patek_philippe", | |
| 125 | + "brand": "Patek Philippe" | |
| 126 | + }, | |
| 127 | + { | |
| 128 | + "match": "audemars|royal oak", | |
| 129 | + "categorySlug": "audemars_piguet", | |
| 130 | + "brand": "Audemars Piguet" | |
| 131 | + }, | |
| 132 | + { | |
| 133 | + "match": "jordan", | |
| 134 | + "categorySlug": "nike_jordan", | |
| 135 | + "brand": "Jordan" | |
| 136 | + }, | |
| 137 | + { | |
| 138 | + "match": "\\bnike\\b|air max|air force|\\bdunk\\b|\\bsb\\b|blazer|vapormax|huarache|cortez|p-6000|vomero|pegasus|\\bshox\\b|\\bkobe\\b|lebron|\\bnocta\\b|\\bacg\\b|sacai|off-white|travis scott", | |
| 139 | + "categorySlug": "nike_jordan", | |
| 140 | + "brand": "Nike" | |
| 141 | + }, | |
| 142 | + { | |
| 143 | + "match": "yeezy", | |
| 144 | + "categorySlug": "adidas_yeezy", | |
| 145 | + "brand": "adidas Yeezy" | |
| 146 | + }, | |
| 147 | + { | |
| 148 | + "match": "adidas|\\bsamba\\b|gazelle|spezial|superstar|\\bcampus\\b|\\bforum\\b|ultraboost|\\bnmd\\b|stan smith|adizero|\\by-3\\b|climacool|\\bzx\\b|\\beqt\\b|\\bsl ?72\\b|adistar|megaride|taekwondo|wales bonner", | |
| 149 | + "categorySlug": "adidas_yeezy", | |
| 150 | + "brand": "adidas" | |
| 151 | + }, | |
| 152 | + { | |
| 153 | + "match": "new balance|asics|puma|reebok|saucony|salomon|converse|\\bvans\\b|\\bhoka\\b|karhu|mizuno|autry|diadora|\\bveja\\b|clarks|timberland|dr\\.? ?martens|\\bugg\\b|mallet|cleens|umbro|le coq|kangaroos|\\bfila\\b|k-swiss|lacoste|golden goose|balenciaga|\\bdior\\b|gucci|louis vuitton|prada|amiri|herm[eè]s|rick owens|margiela|mcqueen|common projects|axel arigato|represent|loewe|bottega|valentino|givenchy|fendi|burberry|chanel|celine|saint laurent|versace|moncler|birkenstock|\\bon\\b (cloud|running)|cloudmonster|cloudtilt|merrell|keen\\b|ewing|ellesse|kappa|hi-tec|etnies|\\bdc shoes|gola|onitsuka|norda|novesta|stepney|sunnei|camper|mschf|maison mihara|\\bbape\\b|\\bsta\\b", | |
| 154 | + "categorySlug": "new_balance_asics_other" | |
| 155 | + } | |
| 156 | + ], | |
| 157 | + "defaultCategory": null, | |
| 158 | + "exclude": "t-shirt|\\btee\\b|\\btees\\b|hoodie|sweatshirt|crewneck|jacket|\\bcoat\\b|tracksuit|track pants|track jacket|sweatpants|joggers|shorts|jeans|trousers|\\bpants\\b|cargo|\\bcap\\b|\\bhat\\b|beanie|balaclava|socks|\\bbag\\b|backpack|tote|wallet|card holder|\\bbelt\\b|sunglasses|fragrance|perfume|cologne|keyring|lanyard|cleaner|crep protect|protector spray|laces|insole|shoe tree|gift card|jersey|\\bpolo\\b|\\bshirt\\b|knitwear|cardigan|\\bvest\\b|gilet|puffer|scarf|gloves|jewellery|jewelry|\\bchain\\b|bracelet|necklace|\\bring\\b|umbrella|towel|doormat|\\brug\\b|candle|\\bmug\\b|phone case|airpods|dress\\b|skirt|leggings|bodysuit|swim|bikini|underwear|boxers|slippers|flip[- ]?flops?|sandals?\\b|\\bslides?\\b|\\bmules?\\b|crocs|clog|\\| clothing \\||\\| accessories \\| (?!.*(watch|labubu|bearbrick|pop mart))", | |
| 159 | + "keepOutOfStock": true, | |
| 160 | + "pageSize": 250, | |
| 161 | + "fetchBarcodes": false, | |
| 162 | + "wholeShop": false | |
| 163 | + } | |
| 164 | +} | |
added
connectors/api/kidrobot/README.md
+34 −0
@@ -0,0 +1,34 @@ | ||
| 1 | +# Kidrobot connector (`kidrobot`) | |
| 2 | + | |
| 3 | +- Source: https://www.kidrobot.com · dealer · US · USD · Shopify storefront | |
| 4 | +- Acquisition: public `/collections/<handle>/products.json?limit=250&page=N` (shared `ShopifyStoreConnector` adapter wrapped by `MarketPinnedShopifyConnector`, which adds a `localization=US` cookie so Shopify Markets never geo-converts prices; no store-specific code) | |
| 5 | +- Records: `listing` (asking prices, one per variant; out-of-stock variants → `ended`) | |
| 6 | +- Refresh: every 12 h (ACTIVE→NORMAL boundary, large catalogue) · history: none (listings only) | |
| 7 | + | |
| 8 | +Designer-toy brand store (Dunny, Labbit, art toys, Hello Kitty and Disney collabs). Shopify storefront; product_type distinguishes Dunny/Plush/Apparel; SKUs 'KRxxxxx-01'. | |
| 9 | + | |
| 10 | +## Collections → taxonomy | |
| 11 | +| collection handle | category | brand / franchise / series | | |
| 12 | +|---|---|---| | |
| 13 | +| `dunny` | `designer_toys` | Kidrobot / Dunny | | |
| 14 | +| `labbit` | `designer_toys` | Kidrobot / Labbit | | |
| 15 | +| `art-toys` | `designer_toys` | Kidrobot | | |
| 16 | +| `art-toys-pop-culture` | `designer_toys` | Kidrobot | | |
| 17 | +| `blind-boxes-mini-figures` | `designer_toys` | Kidrobot | | |
| 18 | +| `limited-edition` | `designer_toys` | Kidrobot | | |
| 19 | +| `kidrobot-com-exclusives` | `designer_toys` | Kidrobot | | |
| 20 | +| `hello-kitty-and-friends` | `sanrio` | Kidrobot / Hello Kitty | | |
| 21 | +| `disney` | `disney_collectibles` | Kidrobot / Disney | | |
| 22 | +| `hello-kitty-plush-toys-collectibles` | `plush` | Kidrobot / Hello Kitty | | |
| 23 | + | |
| 24 | +Rules (first match wins, applied to "title | product_type | tags | collection"): | |
| 25 | +- `\| Plush \||\bplush\b` → `plush` | |
| 26 | + | |
| 27 | +Excluded (regex): `gift card|mystery|\| (Apparel|T-Shirts?|Hats?|Accessories|Pins?|Enamel Pins?|Patches?|Socks|Hoodies?|Bags?|Stationery|Stickers?|Prints?|Posters?|Merch|Keychains?|Lanyards?|Pins? Lanyard Set|Books?|Toy Displays?|Home Goods|Drinkware) \||t-shirt|\btee\b|hoodie|sweatshirt|\bhat\b|\bcap\b|beanie|\bsocks?\b|enamel pin|\bpins? set\b|\bpatch\b|sticker|\bkeychain\b|lanyard|\bmug\b|\btote\b|tumbler|display case|acrylic case|pop protector only|art print|\bposter\b|\bzine\b|\bbook\b|\bgame\b|puzzle|blanket|towel|\bwallet\b|backpack|\bshirt\b` | |
| 28 | + | |
| 29 | +## Access & compliance | |
| 30 | +See `accessNotes` in meta.json: robots.txt verified 2026-09-08 (standard Shopify rules, products.json paths allowed), honest UA, 1 req / 1.5 s, listings only, no personal data. Not fetched: gift-guide/holiday umbrella collections, apparel, accessories, enamel pins, keychains, home goods. | |
| 31 | + | |
| 32 | +## Fixtures & tests | |
| 33 | +`data/fixtures/kidrobot/` — live captures (`pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts kidrobot`), trimmed single-product payloads incl. a sold-out variant. | |
| 34 | +`pnpm vitest run connectors/api/kidrobot` · live probe: `pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts kidrobot`. | |
added
connectors/api/kidrobot/index.test.ts
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { describeShopifyStore } from '../_g3-shops-na-lib/store-suite.js'; | |
| 4 | +import createConnector from './index.js'; | |
| 5 | + | |
| 6 | +/** | |
| 7 | + * Kidrobot — metadata-only Shopify store connector. The shared suite checks the taxonomy mapping of every | |
| 8 | + * configured collection/rule, the exclusion regex on real title shapes, and normalises the live-captured | |
| 9 | + * fixtures in data/fixtures/kidrobot/ (runFixtureSuite invariants + USD currency, seller, SKUs, ended variants). | |
| 10 | + */ | |
| 11 | +describeShopifyStore(meta, createConnector, { describe, it, expect }, { | |
| 12 | + "cases": [ | |
| 13 | + { | |
| 14 | + "title": "OREO® 8\" Resin Dunny (PRE-ORDER)", | |
| 15 | + "productType": "Dunny", | |
| 16 | + "collection": "dunny", | |
| 17 | + "categorySlug": "designer_toys", | |
| 18 | + "brand": "Kidrobot" | |
| 19 | + }, | |
| 20 | + { | |
| 21 | + "title": "2026 CON EXCLUSIVE: McDonald's - Grimace in Disguise Large Plush (Limited Edition of 500)", | |
| 22 | + "productType": "Plush", | |
| 23 | + "collection": "limited-edition", | |
| 24 | + "categorySlug": "plush" | |
| 25 | + }, | |
| 26 | + { | |
| 27 | + "title": "2026 CON EXCLUSIVE: Hello Kitty® & Mimmy Devilishly Cute T-Shirt", | |
| 28 | + "productType": "Apparel", | |
| 29 | + "collection": "limited-edition", | |
| 30 | + "categorySlug": null | |
| 31 | + }, | |
| 32 | + { | |
| 33 | + "title": "Hello Kitty® Ita Plush and Lanyard Set", | |
| 34 | + "productType": "Pins Lanyard Set", | |
| 35 | + "collection": "limited-edition", | |
| 36 | + "categorySlug": null | |
| 37 | + } | |
| 38 | + ] | |
| 39 | +}); | |
added
connectors/api/kidrobot/index.ts
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +import type { ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | +import { MarketPinnedShopifyConnector } from '../_g3-shops-na-lib/shopify-market.js'; | |
| 3 | + | |
| 4 | +/** | |
| 5 | + * Kidrobot — Shopify storefront read through the shared Shopify adapter, pinned to the shop's home market | |
| 6 | + * (localization cookie) so Shopify Markets never geo-converts prices. All source-specific knowledge lives in | |
| 7 | + * meta.json (collections → taxonomy mapping, rules, exclusions, currency, market). | |
| 8 | + */ | |
| 9 | +export default (meta: ConnectorMeta) => new MarketPinnedShopifyConnector(meta); | |
added
connectors/api/kidrobot/meta.json
+120 −0
@@ -0,0 +1,120 @@ | ||
| 1 | +{ | |
| 2 | + "id": "kidrobot", | |
| 3 | + "displayName": "Kidrobot (US collector-toy store, USD)", | |
| 4 | + "sourceId": "kidrobot", | |
| 5 | + "sourceName": "Kidrobot", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://www.kidrobot.com", | |
| 8 | + "module": "api/kidrobot", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "designer_toys", | |
| 14 | + "sanrio", | |
| 15 | + "disney_collectibles", | |
| 16 | + "plush" | |
| 17 | + ], | |
| 18 | + "regions": [ | |
| 19 | + "US" | |
| 20 | + ], | |
| 21 | + "languages": [ | |
| 22 | + "en" | |
| 23 | + ], | |
| 24 | + "currency": [ | |
| 25 | + "USD" | |
| 26 | + ], | |
| 27 | + "supportsListings": true, | |
| 28 | + "supportsSold": false, | |
| 29 | + "supportsAuctions": false, | |
| 30 | + "supportsImages": true, | |
| 31 | + "supportsCatalog": false, | |
| 32 | + "supportsPopulation": false, | |
| 33 | + "supportsLookup": true, | |
| 34 | + "refreshFrequencyMinutes": 720, | |
| 35 | + "priority": "medium", | |
| 36 | + "trustScore": 0.75, | |
| 37 | + "attributionRequired": true, | |
| 38 | + "termsUrl": "https://www.kidrobot.com/policies/terms-of-service", | |
| 39 | + "accessNotes": "Kidrobot (kidrobot.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 10 configured collections (dunny, labbit, art-toys, art-toys-pop-culture, blind-boxes-mini-figures, limited-edition, kidrobot-com-exclusives, hello-kitty-and-friends … (+2 more, see config.collections)) and /products/<handle>.json for URL lookups (~1.9k live products; Dunny 430, limited editions 690, blind boxes 230). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=US cookie — the shop's own home market, exactly what a US visitor gets — so prices are always the declared USD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in USD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — gift-guide/holiday umbrella collections, apparel, accessories, enamel pins, keychains, home goods. No personal data is collected; seller = the store itself.", | |
| 40 | + "enabled": true, | |
| 41 | + "schemaVersion": "1.0", | |
| 42 | + "acquisitionMethod": "Shopify storefront products.json", | |
| 43 | + "historicalDepth": "none", | |
| 44 | + "requires": [], | |
| 45 | + "config": { | |
| 46 | + "currency": "USD", | |
| 47 | + "market": "US", | |
| 48 | + "seller": "Kidrobot", | |
| 49 | + "location": "Boulder, CO, USA", | |
| 50 | + "collections": [ | |
| 51 | + { | |
| 52 | + "handle": "dunny", | |
| 53 | + "categorySlug": "designer_toys", | |
| 54 | + "brand": "Kidrobot", | |
| 55 | + "series": "Dunny" | |
| 56 | + }, | |
| 57 | + { | |
| 58 | + "handle": "labbit", | |
| 59 | + "categorySlug": "designer_toys", | |
| 60 | + "brand": "Kidrobot", | |
| 61 | + "series": "Labbit" | |
| 62 | + }, | |
| 63 | + { | |
| 64 | + "handle": "art-toys", | |
| 65 | + "categorySlug": "designer_toys", | |
| 66 | + "brand": "Kidrobot" | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "handle": "art-toys-pop-culture", | |
| 70 | + "categorySlug": "designer_toys", | |
| 71 | + "brand": "Kidrobot" | |
| 72 | + }, | |
| 73 | + { | |
| 74 | + "handle": "blind-boxes-mini-figures", | |
| 75 | + "categorySlug": "designer_toys", | |
| 76 | + "brand": "Kidrobot" | |
| 77 | + }, | |
| 78 | + { | |
| 79 | + "handle": "limited-edition", | |
| 80 | + "categorySlug": "designer_toys", | |
| 81 | + "brand": "Kidrobot" | |
| 82 | + }, | |
| 83 | + { | |
| 84 | + "handle": "kidrobot-com-exclusives", | |
| 85 | + "categorySlug": "designer_toys", | |
| 86 | + "brand": "Kidrobot" | |
| 87 | + }, | |
| 88 | + { | |
| 89 | + "handle": "hello-kitty-and-friends", | |
| 90 | + "categorySlug": "sanrio", | |
| 91 | + "brand": "Kidrobot", | |
| 92 | + "franchise": "Hello Kitty" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "handle": "disney", | |
| 96 | + "categorySlug": "disney_collectibles", | |
| 97 | + "brand": "Kidrobot", | |
| 98 | + "franchise": "Disney" | |
| 99 | + }, | |
| 100 | + { | |
| 101 | + "handle": "hello-kitty-plush-toys-collectibles", | |
| 102 | + "categorySlug": "plush", | |
| 103 | + "brand": "Kidrobot", | |
| 104 | + "franchise": "Hello Kitty" | |
| 105 | + } | |
| 106 | + ], | |
| 107 | + "rules": [ | |
| 108 | + { | |
| 109 | + "match": "\\| Plush \\||\\bplush\\b", | |
| 110 | + "categorySlug": "plush" | |
| 111 | + } | |
| 112 | + ], | |
| 113 | + "defaultCategory": null, | |
| 114 | + "exclude": "gift card|mystery|\\| (Apparel|T-Shirts?|Hats?|Accessories|Pins?|Enamel Pins?|Patches?|Socks|Hoodies?|Bags?|Stationery|Stickers?|Prints?|Posters?|Merch|Keychains?|Lanyards?|Pins? Lanyard Set|Books?|Toy Displays?|Home Goods|Drinkware) \\||t-shirt|\\btee\\b|hoodie|sweatshirt|\\bhat\\b|\\bcap\\b|beanie|\\bsocks?\\b|enamel pin|\\bpins? set\\b|\\bpatch\\b|sticker|\\bkeychain\\b|lanyard|\\bmug\\b|\\btote\\b|tumbler|display case|acrylic case|pop protector only|art print|\\bposter\\b|\\bzine\\b|\\bbook\\b|\\bgame\\b|puzzle|blanket|towel|\\bwallet\\b|backpack|\\bshirt\\b", | |
| 115 | + "keepOutOfStock": true, | |
| 116 | + "fetchBarcodes": false, | |
| 117 | + "wholeShop": false, | |
| 118 | + "pageSize": 250 | |
| 119 | + } | |
| 120 | +} | |
added
connectors/api/kingscomics/README.md
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +# Kings Comics connector (`kingscomics`) | |
| 2 | + | |
| 3 | +- Source: https://kingscomics.com · Sydney pop-culture store (since 1988): CGC/PSA graded comics and cards, new and back-issue comics (Marvel, DC, Image…), manga, trading-card singles (vintage NBA), action figures, statues, Perth Mint coins. Titles are upper-case; tags carry the publisher and REL-YYYYMMDD release dates. | |
| 4 | +- Country/currency: AU / AUD · type: dealer · engine: api (Shopify storefront products.json (public JSON)) | |
| 5 | +- Records: `listing` only (dealer asking prices; never sales — SPEC §111). One listing per variant (condition / size / finish), out-of-stock variants kept as `ended`. | |
| 6 | +- Refresh: every 24 h (NORMAL class) · historical depth: none (no sold archive). | |
| 7 | + | |
| 8 | +## How it works | |
| 9 | +Metadata-only connector: `index.ts` instantiates the shared `ShopifyStoreConnector` adapter; everything source-specific lives in `meta.json` `config`: | |
| 10 | +collections → taxonomy slug (with brand/franchise/series), title/tag `rules` (first match wins), an `exclude` regex for accessories/apparel/events, and the shop currency. Anything unmapped is skipped, never guessed. | |
| 11 | + | |
| 12 | +Collections read (12): `cgc` → comics, `psa` → comics, `all-graded-comics` → comics, `comics-black-white-color` → comics, `trading-card-singles` → non_sport_cards, `pokemon-tcg` → pokemon, `magic-the-gathering` → magic_the_gathering, `one-piece-card-game` → one_piece_card_game, `manga` → manga, `action-figures` → action_figures, `statues-busts` → action_figures, `coins` → coins. | |
| 13 | + | |
| 14 | +## Access & compliance | |
| 15 | +See `accessNotes` in meta.json (robots findings, endpoints, rate limit, what is not fetched). Host policy: `connectors/domains.d/g4-shops-intl.json`. | |
| 16 | + | |
| 17 | +## Fixtures & tests | |
| 18 | +`data/fixtures/kingscomics/*.json` are live captures made with `connectors/api/_g4-shops-intl-lib/capture.ts` (trimmed payloads, expectations derived from the live record). `index.test.ts` runs the framework fixture suite, the g4 store invariants and a mapping unit test on titles observed live. Live probe: `pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts kingscomics`. | |
added
connectors/api/kingscomics/index.test.ts
+127 −0
@@ -0,0 +1,127 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import meta from './meta.json' with { type: 'json' }; | |
| 3 | +import { localMeta } from '../_lib/local-meta.js'; | |
| 4 | +import { expectMapping, storeSuite } from '../_g4-shops-intl-lib/suite.js'; | |
| 5 | +import createConnector from './index.js'; | |
| 6 | + | |
| 7 | +const parsed = localMeta(meta); | |
| 8 | +const connector = createConnector(parsed); | |
| 9 | + | |
| 10 | +describe('kingscomics', () => { | |
| 11 | + // Framework fixture suite + store invariants (currency, categories, listings only, stable ids). | |
| 12 | + storeSuite(parsed, connector, it, expect); | |
| 13 | + | |
| 14 | + it('maps live-observed titles onto taxonomy slugs and skips accessories/apparel', async () => { | |
| 15 | + await expectMapping(parsed, connector, expect, [ | |
| 16 | + { | |
| 17 | + "title": "2005 POKEMON EX UNSEEN FORCES RAIKOU HOLO #114 CGC 10", | |
| 18 | + "collection": "cgc", | |
| 19 | + "type": "Graded Collectables", | |
| 20 | + "tags": [ | |
| 21 | + "CARD", | |
| 22 | + "CGC", | |
| 23 | + "GRADED", | |
| 24 | + "POKEMON" | |
| 25 | + ], | |
| 26 | + "expect": "pokemon", | |
| 27 | + "grader": "cgc", | |
| 28 | + "grade": "10", | |
| 29 | + "year": 2005 | |
| 30 | + }, | |
| 31 | + { | |
| 32 | + "title": "CGC DAREDEVIL #1 (8.5) FADED COVER", | |
| 33 | + "collection": "cgc", | |
| 34 | + "type": "Graded Collectables", | |
| 35 | + "tags": [ | |
| 36 | + "CGC", | |
| 37 | + "DAREDEVIL", | |
| 38 | + "MARVEL COMICS" | |
| 39 | + ], | |
| 40 | + "expect": "marvel_comics", | |
| 41 | + "grader": "cgc" | |
| 42 | + }, | |
| 43 | + { | |
| 44 | + "title": "SPIDER-MAN (1990) #1 DIRECT PSA 9.8", | |
| 45 | + "collection": "all-graded-comics", | |
| 46 | + "type": "Graded Collectables", | |
| 47 | + "tags": [ | |
| 48 | + "COMICS", | |
| 49 | + "GRADED", | |
| 50 | + "PSA" | |
| 51 | + ], | |
| 52 | + "expect": "marvel_comics", | |
| 53 | + "grader": "psa", | |
| 54 | + "grade": "9.8", | |
| 55 | + "year": 1990 | |
| 56 | + }, | |
| 57 | + { | |
| 58 | + "title": "1986 FLEER MAGIC JOHNSON #53 LAKERS PSA 10", | |
| 59 | + "collection": "psa", | |
| 60 | + "type": "Graded Collectables", | |
| 61 | + "tags": [ | |
| 62 | + "PSA", | |
| 63 | + "TC-GRADED" | |
| 64 | + ], | |
| 65 | + "expect": "other_sports_cards", | |
| 66 | + "grader": "psa", | |
| 67 | + "grade": "10" | |
| 68 | + }, | |
| 69 | + { | |
| 70 | + "title": "1991-92 UPPER DECK BASKETBALL #69 MICHAEL JORDAN EAST ALL-STAR", | |
| 71 | + "collection": "trading-card-singles", | |
| 72 | + "type": "Trading Cards", | |
| 73 | + "tags": [ | |
| 74 | + "BASKETBALL", | |
| 75 | + "TC SINGLE" | |
| 76 | + ], | |
| 77 | + "expect": "basketball_cards" | |
| 78 | + }, | |
| 79 | + { | |
| 80 | + "title": "ABSOLUTE BATMAN (2024) #1", | |
| 81 | + "collection": "comics-black-white-color", | |
| 82 | + "type": "Comics – Black & White/Color", | |
| 83 | + "tags": [ | |
| 84 | + "DC COMICS", | |
| 85 | + "GENRE-SH" | |
| 86 | + ], | |
| 87 | + "expect": "dc_comics" | |
| 88 | + }, | |
| 89 | + { | |
| 90 | + "title": "STAR WARS (2025) #2 JOHN TYLER CHRISTOPHER ACTION FIGURE VAR", | |
| 91 | + "collection": "action-figures", | |
| 92 | + "type": "Comics", | |
| 93 | + "tags": [ | |
| 94 | + "ACTION FIGURES", | |
| 95 | + "MARVEL PRH" | |
| 96 | + ], | |
| 97 | + "expect": "comics" | |
| 98 | + }, | |
| 99 | + { | |
| 100 | + "title": "MARVEL LEGENDS SERIES SECRET WARS WOLVERINE AF", | |
| 101 | + "collection": "action-figures", | |
| 102 | + "type": "Toys and Models", | |
| 103 | + "tags": [ | |
| 104 | + "ACTION FIGURES", | |
| 105 | + "HASBRO", | |
| 106 | + "MARVEL" | |
| 107 | + ], | |
| 108 | + "expect": "action_figures" | |
| 109 | + }, | |
| 110 | + { | |
| 111 | + "title": "ALIEN 40TH ANNIVERSARY 2019 2oz SILVER ANTIQUED COLOURED COIN", | |
| 112 | + "collection": "coins", | |
| 113 | + "type": "Coins & Precious Metals", | |
| 114 | + "tags": [ | |
| 115 | + "PERTH MINT" | |
| 116 | + ], | |
| 117 | + "expect": "coins" | |
| 118 | + }, | |
| 119 | + { | |
| 120 | + "title": "SPIDER-MAN LOGO T-SHIRT XL", | |
| 121 | + "collection": "comics-black-white-color", | |
| 122 | + "type": "Apparel", | |
| 123 | + "expect": null | |
| 124 | + } | |
| 125 | + ]); | |
| 126 | + }); | |
| 127 | +}); | |
added
connectors/api/kingscomics/index.ts
+7 −0
@@ -0,0 +1,7 @@ | ||
| 1 | +import { ShopifyStoreConnector, type ConnectorMeta } from '@rareindex/connectors'; | |
| 2 | + | |
| 3 | +/** | |
| 4 | + * Kings Comics — shopify storefront read through the shared shopify adapter. | |
| 5 | + * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency). | |
| 6 | + */ | |
| 7 | +export default (meta: ConnectorMeta) => new ShopifyStoreConnector(meta); | |
added
connectors/api/kingscomics/meta.json
+112 −0
@@ -0,0 +1,182 @@ | ||
| 1 | +{ | |
| 2 | + "id": "kingscomics", | |
| 3 | + "displayName": "Kings Comics", | |
| 4 | + "sourceId": "kingscomics", | |
| 5 | + "sourceName": "Kings Comics", | |
| 6 | + "sourceType": "dealer", | |
| 7 | + "sourceUrl": "https://kingscomics.com", | |
| 8 | + "module": "api/kingscomics", | |
| 9 | + "enginePriority": [ | |
| 10 | + "api" | |
| 11 | + ], | |
| 12 | + "categories": [ | |
| 13 | + "comics", | |
| 14 | + "non_sport_cards", | |
| 15 | + "pokemon", | |
| 16 | + "magic_the_gathering", | |
| 17 | + "one_piece_card_game", | |
| 18 | + "manga", | |
| 19 | + "action_figures", | |
| 20 | + "coins", | |
| 21 | + "basketball_cards", | |
| 22 | + "football_cards", | |
| 23 | + "soccer_cards", | |
| 24 | + "baseball_cards", | |
| 25 | + "hockey_cards", | |
| 26 | + "other_sports_cards", | |
| 27 | + "marvel_comics", | |
| 28 | + "dc_comics", | |
| 29 | + "independent_comics" | |
| 30 | + ], | |
| 31 | + "regions": [ | |
| 32 | + "AU" | |
| 33 | + ], | |
| 34 | + "country": "AU", | |
| 35 | + "languages": [ | |
| 36 | + "en" | |
| 37 | + ], | |
| 38 | + "currency": [ | |
| 39 | + "AUD" | |
| 40 | + ], | |
| 41 | + "supportsListings": true, | |
| 42 | + "supportsSold": false, | |
| 43 | + "supportsAuctions": false, | |
| 44 | + "supportsImages": true, | |
| 45 | + "supportsCatalog": false, | |
| 46 | + "supportsPopulation": false, | |
| 47 | + "supportsLookup": true, | |
| 48 | + "refreshFrequencyMinutes": 1440, | |
| 49 | + "priority": "medium", | |
| 50 | + "trustScore": 0.65, | |
| 51 | + "attributionRequired": true, | |
| 52 | + "termsUrl": "https://kingscomics.com/policies/terms-of-service", | |
| 53 | + "accessNotes": "Kings Comics (kingscomics.com) is a Shopify shop; we read only the public, unauthenticated storefront JSON the shop serves to every visitor: /collections/<handle>/products.json?limit=250&page=N for the configured collections and /products/<handle>.json for URL lookups (cgc, psa, all-graded-comics, comics-black-white-color, trading-card-singles, pokemon-tcg, magic-the-gathering, one-piece-card-game, manga, action-figures, statues-busts, coins). robots.txt (checked 2026-09-08) is the standard Shopify file: for User-agent * it disallows /admin, /cart, /orders, /checkout(s), /carts, /account, /search, /policies, /recommendations/products, sort_by / filter / \"+\" collection permutations, preview_theme_id, oseid, the -remote product variants and /cdn/wpm; it blocks AhrefsBot, AhrefsSiteAudit, MJ12bot, Bytespider, Nutch and Pinterest entirely. The /collections/<handle>/products.json and /products/<handle>.json endpoints we read are not disallowed. Rate: 1 request per 1.5 s, one connection (connectors/domains.d/g4-shops-intl.json), honest RareIndexBot UA, once a day, 3 pages per collection per incremental run (backfill bounded by backfillMaxPages). Prices are the base currency AUD (/meta.json currency AUD, Shopify.currency rate 1.0), GST included. CGC comic grades written as \"(8.5)\" after the title are not parsed by parseGradeFromTitle (grader is, grade stays null); PSA grades (\"PSA 9.8\") are parsed. Dealer asking prices are stored as listings, never as sales (SPEC §111); out-of-stock variants are kept as ended listings for price-history audit. Deliberately not fetched: cart/checkout/account pages, customer names, reviews or any personal data, search and sort/filter permutations (robots-disallowed), /recommendations, blog pages, and images beyond the URLs already present in the JSON.", | |
| 54 | + "enabled": true, | |
| 55 | + "schemaVersion": "1.0", | |
| 56 | + "acquisitionMethod": "Shopify storefront products.json (public JSON)", | |
| 57 | + "historicalDepth": "none", | |
| 58 | + "requires": [], | |
| 59 | + "config": { | |
| 60 | + "currency": "AUD", | |
| 61 | + "seller": "Kings Comics", | |
| 62 | + "location": null, | |
| 63 | + "collections": [ | |
| 64 | + { | |
| 65 | + "handle": "cgc", | |
| 66 | + "categorySlug": "comics" | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "handle": "psa", | |
| 70 | + "categorySlug": "comics" | |
| 71 | + }, | |
| 72 | + { | |
| 73 | + "handle": "all-graded-comics", | |
| 74 | + "categorySlug": "comics" | |
| 75 | + }, | |
| 76 | + { | |
| 77 | + "handle": "comics-black-white-color", | |
| 78 | + "categorySlug": "comics" | |
| 79 | + }, | |
| 80 | + { | |
| 81 | + "handle": "trading-card-singles", | |
| 82 | + "categorySlug": "non_sport_cards" | |
| 83 | + }, | |
| 84 | + { | |
| 85 | + "handle": "pokemon-tcg", | |
| 86 | + "categorySlug": "pokemon", | |
| 87 | + "franchise": "Pokémon" | |
| 88 | + }, | |
| 89 | + { | |
| 90 | + "handle": "magic-the-gathering", | |
| 91 | + "categorySlug": "magic_the_gathering", | |
| 92 | + "franchise": "Magic: The Gathering" | |
| 93 | + }, | |
| 94 | + { | |
| 95 | + "handle": "one-piece-card-game", | |
| 96 | + "categorySlug": "one_piece_card_game", | |
| 97 | + "franchise": "One Piece" | |
| 98 | + }, | |
| 99 | + { | |
| 100 | + "handle": "manga", | |
| 101 | + "categorySlug": "manga" | |
| 102 | + }, | |
| 103 | + { | |
| 104 | + "handle": "action-figures", | |
| 105 | + "categorySlug": "action_figures" | |
| 106 | + }, | |
| 107 | + { | |
| 108 | + "handle": "statues-busts", | |
| 109 | + "categorySlug": "action_figures" | |
| 110 | + }, | |
| 111 | + { | |
| 112 | + "handle": "coins", | |
Diff truncated — file too large.