import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { normalizeCondition } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; import { intOrNull, watchFromTitle } from '../_g9-asia-watch-sneaker-lib/index.js'; /** * Loupe This — online watch auction platform (US). Public JSON:API behind the site: * GET https://api.loupethis.com/api/v1/auctions?status=closed|live&per_page=100&page=N&include=brand * Closed lots with a sold price → `sale` (price includes the 10 % buyer's premium; hammer kept in metadata). * Live/upcoming lots → `auction_lot`. USD. */ const API = 'https://api.loupethis.com/api/v1'; const SITE = 'https://loupethis.com'; const PARSER_VERSION = '1.0.0'; const PAGE_SIZE = 100; export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), lot: z.string().nullable(), startsAt: z.string().nullable(), endsAt: z.string().nullable(), listedAt: z.string().nullable(), isClosed: z.boolean(), bidsCount: z.number().nullable(), currentBidCents: z.number().nullable(), soldPriceCents: z.number().nullable(), buyersPremiumPercent: z.number().nullable(), reserveMet: z.boolean().nullable(), brand: z.string().nullable(), image: z.string().nullable(), }); export type Auction = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('auction_page'), status: z.enum(['closed', 'live']), page: z.number(), totalPages: z.number().nullable(), totalCount: z.number().nullable(), auctions: z.array(AuctionSchema) }); export type PagePayload = z.infer; const Attr = z.object({ title: z.string(), slug: z.string(), lot: z.union([z.string(), z.number()]).nullable().optional(), starts_at: z.string().nullable().optional(), ends_at: z.string().nullable().optional(), listed_at: z.string().nullable().optional(), is_closed: z.boolean().optional(), bids_count: z.number().nullable().optional(), current_bid_price_cents: z.number().nullable().optional(), sold_price_cents: z.number().nullable().optional(), buyers_premium_percent: z.number().nullable().optional(), reserve_price_met: z.boolean().nullable().optional(), featured_image_url: z.string().nullable().optional(), }); const Resource = z.object({ id: z.string(), type: z.string(), attributes: Attr, relationships: z.object({ brand: z.object({ data: z.object({ id: z.string() }).nullable() }).optional() }).optional() }); const Included = z.object({ id: z.string(), type: z.string(), attributes: z.record(z.string(), z.unknown()) }); export const ListResponseSchema = z.object({ data: z.array(Resource), included: z.array(Included).optional(), meta: z.object({ pagination: z.object({ current_page: z.number().optional(), total_pages: z.number().optional(), total_count: z.number().optional() }).optional() }).optional() }); export const SingleResponseSchema = z.object({ data: Resource, included: z.array(Included).optional() }); export function toAuctions(data: z.infer[], included: z.infer[] = []): Auction[] { const brands = new Map(included.filter((i) => i.type === 'brands').map((i) => [i.id, String(i.attributes.name ?? '')])); return data .filter((r) => r.type === 'auctions') .map((r) => { const a = r.attributes; const brandId = r.relationships?.brand?.data?.id ?? null; return { id: r.id, slug: a.slug, title: a.title.replace(/\s+/g, ' ').trim(), lot: a.lot === null || a.lot === undefined ? null : String(a.lot), startsAt: a.starts_at ?? null, endsAt: a.ends_at ?? null, listedAt: a.listed_at ?? null, isClosed: Boolean(a.is_closed), bidsCount: a.bids_count ?? null, currentBidCents: a.current_bid_price_cents ?? null, soldPriceCents: a.sold_price_cents ?? null, buyersPremiumPercent: a.buyers_premium_percent ?? null, reserveMet: a.reserve_price_met ?? null, brand: brandId ? brands.get(brandId) || null : null, image: a.featured_image_url ?? null, }; }); } export function listUrl(status: 'closed' | 'live', page: number): string { return `${API}/auctions?status=${status}&per_page=${PAGE_SIZE}&page=${page}&include=brand`; } type Cursor = { status?: 'closed' | 'live'; page?: number; done?: boolean }; export class LoupeThisConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/(?:www\.)?loupethis\.com\/auctions\/([a-z0-9-]+)/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const backfill = ctx.options.mode === 'backfill'; const cur = (ctx.options.cursor ?? {}) as Cursor; if (backfill && cur.done) return; const plan: Array<{ status: 'closed' | 'live'; pages: number }> = backfill ? [{ status: 'closed', pages: this.policy.backfillMaxPages }] : [{ status: 'live', pages: Number(this.meta.config.livePages ?? 2) }, { status: 'closed', pages: Number(this.meta.config.closedPages ?? 2) }]; let count = 0; for (const step of plan) { if (cur.status && cur.status !== step.status && plan.findIndex((s) => s.status === cur.status) > plan.indexOf(step)) continue; let page = cur.status === step.status && cur.page ? cur.page : 1; for (; page <= step.pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = listUrl(step.status, page); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price', 'date'], parse: (r) => { const parsed = ListResponseSchema.safeParse(r.json); const first = parsed.success ? parsed.data.data[0]?.attributes : null; return first ? { title: first.title, price: first.sold_price_cents ?? first.current_bid_price_cents ?? 1, date: first.ends_at } : parsed.success ? { title: 'empty', price: 1, date: 'none' } : null; } }); const parsed = ListResponseSchema.safeParse(res.json); if (!res.success || !parsed.success) { ctx.anomaly(res.success ? 'schema_drift' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const all = toAuctions(parsed.data.data, parsed.data.included); // `status=live` also returns already-closed lots once the open ones are exhausted: keep the open ones and stop when a page has none. const auctions = step.status === 'live' ? all.filter((a) => !a.isClosed) : all; const pg = parsed.data.meta?.pagination; const payload: PagePayload = { kind: 'auction_page', status: step.status, page, totalPages: pg?.total_pages ?? null, totalCount: pg?.total_count ?? null, auctions }; if (!auctions.length) break; count++; yield { url, externalId: `${step.status}:${page}`, kind: step.status === 'closed' ? 'sale' : 'auction_lot', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ status: step.status, page: page + 1, at: new Date().toISOString() }); if (backfill) { const oldest = auctions.map((a) => (a.endsAt ? new Date(a.endsAt) : null)).filter((d): d is Date => Boolean(d && !Number.isNaN(d.getTime()))).sort((a, b) => a.getTime() - b.getTime())[0] ?? null; await ctx.progress({ page, totalPages: payload.totalPages, itemsProcessed: count, reachedDate: oldest }); } if (payload.totalPages !== null && page >= payload.totalPages) break; } await ctx.setCursor({ status: step.status, page: step.pages + 1, at: new Date().toISOString() }); } await ctx.setCursor({ done: true, at: new Date().toISOString() }); } async lookup(url: string, ctx: CrawlContext): Promise { const slug = url.match(this.urlPatterns[0]!)?.[1]; if (!slug) return []; const target = `${API}/auctions/${slug}?include=brand`; await this.throttle(target); const res = await ctx.fetch(target, { engines: ['api'], responseType: 'json', minQuality: 0 }); const parsed = SingleResponseSchema.safeParse(res.json); if (!res.success || !parsed.success) return []; const auctions = toAuctions([parsed.data.data], parsed.data.included); const a = auctions[0]; if (!a) return []; const payload: PagePayload = { kind: 'auction_page', status: a.isClosed ? 'closed' : 'live', page: 1, totalPages: 1, totalCount: 1, auctions }; return [{ url: `${SITE}/auctions/${slug}`, externalId: `auction:${a.id}`, kind: a.isClosed ? 'sale' : 'auction_lot', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const a of p.auctions) { const w = watchFromTitle(a.title, a.brand); const attributes = AssetAttributesSchema.parse({ categorySlug: w.categorySlug, brand: w.brand, name: a.title, reference: w.reference, year: w.year, material: w.material, size: w.size, identifiers: { loupe_this_id: a.id, ...(w.reference ? { reference: w.reference } : {}) }, metadata: { lot: a.lot, bids_count: a.bidsCount, buyers_premium_percent: a.buyersPremiumPercent, reserve_met: a.reserveMet, listed_at: a.listedAt, slug: a.slug }, }); const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${SITE}/auctions/${a.slug}`, externalId: a.id, rawTitle: a.title, description: null, imageUrls: a.image ? [a.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: normalizeCondition(w.categorySlug, w.conditionRaw), conditionRaw: w.conditionRaw, completeness: w.completeness }, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; const endsAt = a.endsAt ? new Date(a.endsAt) : null; if (a.isClosed) { if (!a.soldPriceCents || a.soldPriceCents <= 0 || !endsAt) continue; // closed without a sale (reserve not met / withdrawn) const hammer = a.currentBidCents ? a.currentBidCents / 100 : null; out.push( NormalizedSaleSchema.parse({ kind: 'sale', ...base, attributes: { ...attributes, metadata: { ...attributes.metadata, hammer_price: hammer, buyers_premium_percent: a.buyersPremiumPercent } }, confidence: 0.85, saleType: 'auction', saleDate: endsAt, price: a.soldPriceCents / 100, currency: 'USD', buyerPremiumIncluded: true, quantity: 1, isBundle: false, location: 'US', auctionHouse: 'Loupe This', lotNumber: a.lot, }), ); continue; } const startsAt = a.startsAt ? new Date(a.startsAt) : null; const now = raw.fetchedAt.getTime(); const status = startsAt && startsAt.getTime() > now ? 'upcoming' : endsAt && endsAt.getTime() < now ? 'ended' : 'live'; out.push( NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, confidence: 0.8, auctionHouse: 'Loupe This', auctionName: null, lotNumber: a.lot, startsAt, endsAt, estimateLow: null, estimateHigh: null, currentBid: a.currentBidCents && a.currentBidCents > 0 ? a.currentBidCents / 100 : null, currency: 'USD', status, location: 'US', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new LoupeThisConnector(meta); }