TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';5import { intOrNull, watchFromTitle } from '../_g9-asia-watch-sneaker-lib/index.js';67/**8 * Loupe This — online watch auction platform (US). Public JSON:API behind the site:9 * GET https://api.loupethis.com/api/v1/auctions?status=closed|live&per_page=100&page=N&include=brand10 * Closed lots with a sold price → `sale` (price includes the 10 % buyer's premium; hammer kept in metadata).11 * Live/upcoming lots → `auction_lot`. USD.12 */1314const API = 'https://api.loupethis.com/api/v1';15const SITE = 'https://loupethis.com';16const PARSER_VERSION = '1.0.0';17const PAGE_SIZE = 100;1819export const AuctionSchema = z.object({20 id: z.string(),21 slug: z.string(),22 title: z.string(),23 lot: z.string().nullable(),24 startsAt: z.string().nullable(),25 endsAt: z.string().nullable(),26 listedAt: z.string().nullable(),27 isClosed: z.boolean(),28 bidsCount: z.number().nullable(),29 currentBidCents: z.number().nullable(),30 soldPriceCents: z.number().nullable(),31 buyersPremiumPercent: z.number().nullable(),32 reserveMet: z.boolean().nullable(),33 brand: z.string().nullable(),34 image: z.string().nullable(),35});36export type Auction = z.infer<typeof AuctionSchema>;37export const PagePayloadSchema = z.object({ kind: z.literal('auction_page'), status: z.enum(['closed', 'live']), page: z.number(), totalPages: z.number().nullable(), totalCount: z.number().nullable(), auctions: z.array(AuctionSchema) });38export type PagePayload = z.infer<typeof PagePayloadSchema>;3940const Attr = z.object({41 title: z.string(),42 slug: z.string(),43 lot: z.union([z.string(), z.number()]).nullable().optional(),44 starts_at: z.string().nullable().optional(),45 ends_at: z.string().nullable().optional(),46 listed_at: z.string().nullable().optional(),47 is_closed: z.boolean().optional(),48 bids_count: z.number().nullable().optional(),49 current_bid_price_cents: z.number().nullable().optional(),50 sold_price_cents: z.number().nullable().optional(),51 buyers_premium_percent: z.number().nullable().optional(),52 reserve_price_met: z.boolean().nullable().optional(),53 featured_image_url: z.string().nullable().optional(),54});55const Resource = z.object({ id: z.string(), type: z.string(), attributes: Attr, relationships: z.object({ brand: z.object({ data: z.object({ id: z.string() }).nullable() }).optional() }).optional() });56const Included = z.object({ id: z.string(), type: z.string(), attributes: z.record(z.string(), z.unknown()) });57export const ListResponseSchema = z.object({ data: z.array(Resource), included: z.array(Included).optional(), meta: z.object({ pagination: z.object({ current_page: z.number().optional(), total_pages: z.number().optional(), total_count: z.number().optional() }).optional() }).optional() });58export const SingleResponseSchema = z.object({ data: Resource, included: z.array(Included).optional() });5960export function toAuctions(data: z.infer<typeof Resource>[], included: z.infer<typeof Included>[] = []): Auction[] {61 const brands = new Map(included.filter((i) => i.type === 'brands').map((i) => [i.id, String(i.attributes.name ?? '')]));62 return data63 .filter((r) => r.type === 'auctions')64 .map((r) => {65 const a = r.attributes;66 const brandId = r.relationships?.brand?.data?.id ?? null;67 return {68 id: r.id,69 slug: a.slug,70 title: a.title.replace(/\s+/g, ' ').trim(),71 lot: a.lot === null || a.lot === undefined ? null : String(a.lot),72 startsAt: a.starts_at ?? null,73 endsAt: a.ends_at ?? null,74 listedAt: a.listed_at ?? null,75 isClosed: Boolean(a.is_closed),76 bidsCount: a.bids_count ?? null,77 currentBidCents: a.current_bid_price_cents ?? null,78 soldPriceCents: a.sold_price_cents ?? null,79 buyersPremiumPercent: a.buyers_premium_percent ?? null,80 reserveMet: a.reserve_price_met ?? null,81 brand: brandId ? brands.get(brandId) || null : null,82 image: a.featured_image_url ?? null,83 };84 });85}8687export function listUrl(status: 'closed' | 'live', page: number): string {88 return `${API}/auctions?status=${status}&per_page=${PAGE_SIZE}&page=${page}&include=brand`;89}9091type Cursor = { status?: 'closed' | 'live'; page?: number; done?: boolean };9293export class LoupeThisConnector extends BaseConnector {94 readonly version = '1.0.0';95 readonly parserVersion = PARSER_VERSION;96 protected override minIntervalMs = 1500;97 override readonly urlPatterns = [/^https?:\/\/(?:www\.)?loupethis\.com\/auctions\/([a-z0-9-]+)/i];9899 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {100 const backfill = ctx.options.mode === 'backfill';101 const cur = (ctx.options.cursor ?? {}) as Cursor;102 if (backfill && cur.done) return;103 const plan: Array<{ status: 'closed' | 'live'; pages: number }> = backfill104 ? [{ status: 'closed', pages: this.policy.backfillMaxPages }]105 : [{ status: 'live', pages: Number(this.meta.config.livePages ?? 2) }, { status: 'closed', pages: Number(this.meta.config.closedPages ?? 2) }];106 let count = 0;107 for (const step of plan) {108 if (cur.status && cur.status !== step.status && plan.findIndex((s) => s.status === cur.status) > plan.indexOf(step)) continue;109 let page = cur.status === step.status && cur.page ? cur.page : 1;110 for (; page <= step.pages; page++) {111 if (ctx.signal?.aborted || this.reached(ctx, count)) return;112 const url = listUrl(step.status, page);113 await this.throttle(url);114 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price', 'date'], parse: (r) => {115 const parsed = ListResponseSchema.safeParse(r.json);116 const first = parsed.success ? parsed.data.data[0]?.attributes : null;117 return first ? { title: first.title, price: first.sold_price_cents ?? first.current_bid_price_cents ?? 1, date: first.ends_at } : parsed.success ? { title: 'empty', price: 1, date: 'none' } : null;118 } });119 const parsed = ListResponseSchema.safeParse(res.json);120 if (!res.success || !parsed.success) {121 ctx.anomaly(res.success ? 'schema_drift' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);122 break;123 }124 const all = toAuctions(parsed.data.data, parsed.data.included);125 // `status=live` also returns already-closed lots once the open ones are exhausted: keep the open ones and stop when a page has none.126 const auctions = step.status === 'live' ? all.filter((a) => !a.isClosed) : all;127 const pg = parsed.data.meta?.pagination;128 const payload: PagePayload = { kind: 'auction_page', status: step.status, page, totalPages: pg?.total_pages ?? null, totalCount: pg?.total_count ?? null, auctions };129 if (!auctions.length) break;130 count++;131 yield { url, externalId: `${step.status}:${page}`, kind: step.status === 'closed' ? 'sale' : 'auction_lot', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };132 await ctx.setCursor({ status: step.status, page: page + 1, at: new Date().toISOString() });133 if (backfill) {134 const oldest = auctions.map((a) => (a.endsAt ? new Date(a.endsAt) : null)).filter((d): d is Date => Boolean(d && !Number.isNaN(d.getTime()))).sort((a, b) => a.getTime() - b.getTime())[0] ?? null;135 await ctx.progress({ page, totalPages: payload.totalPages, itemsProcessed: count, reachedDate: oldest });136 }137 if (payload.totalPages !== null && page >= payload.totalPages) break;138 }139 await ctx.setCursor({ status: step.status, page: step.pages + 1, at: new Date().toISOString() });140 }141 await ctx.setCursor({ done: true, at: new Date().toISOString() });142 }143144 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {145 const slug = url.match(this.urlPatterns[0]!)?.[1];146 if (!slug) return [];147 const target = `${API}/auctions/${slug}?include=brand`;148 await this.throttle(target);149 const res = await ctx.fetch(target, { engines: ['api'], responseType: 'json', minQuality: 0 });150 const parsed = SingleResponseSchema.safeParse(res.json);151 if (!res.success || !parsed.success) return [];152 const auctions = toAuctions([parsed.data.data], parsed.data.included);153 const a = auctions[0];154 if (!a) return [];155 const payload: PagePayload = { kind: 'auction_page', status: a.isClosed ? 'closed' : 'live', page: 1, totalPages: 1, totalCount: 1, auctions };156 return [{ url: `${SITE}/auctions/${slug}`, externalId: `auction:${a.id}`, kind: a.isClosed ? 'sale' : 'auction_lot', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];157 }158159 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {160 const p = PagePayloadSchema.parse(raw.payload);161 const out: NormalizedRecord[] = [];162 for (const a of p.auctions) {163 const w = watchFromTitle(a.title, a.brand);164 const attributes = AssetAttributesSchema.parse({165 categorySlug: w.categorySlug,166 brand: w.brand,167 name: a.title,168 reference: w.reference,169 year: w.year,170 material: w.material,171 size: w.size,172 identifiers: { loupe_this_id: a.id, ...(w.reference ? { reference: w.reference } : {}) },173 metadata: { lot: a.lot, bids_count: a.bidsCount, buyers_premium_percent: a.buyersPremiumPercent, reserve_met: a.reserveMet, listed_at: a.listedAt, slug: a.slug },174 });175 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${SITE}/auctions/${a.slug}`, externalId: a.id, rawTitle: a.title, description: null, imageUrls: a.image ? [a.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: normalizeCondition(w.categorySlug, w.conditionRaw), conditionRaw: w.conditionRaw, completeness: w.completeness }, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };176 const endsAt = a.endsAt ? new Date(a.endsAt) : null;177 if (a.isClosed) {178 if (!a.soldPriceCents || a.soldPriceCents <= 0 || !endsAt) continue; // closed without a sale (reserve not met / withdrawn)179 const hammer = a.currentBidCents ? a.currentBidCents / 100 : null;180 out.push(181 NormalizedSaleSchema.parse({182 kind: 'sale',183 ...base,184 attributes: { ...attributes, metadata: { ...attributes.metadata, hammer_price: hammer, buyers_premium_percent: a.buyersPremiumPercent } },185 confidence: 0.85,186 saleType: 'auction',187 saleDate: endsAt,188 price: a.soldPriceCents / 100,189 currency: 'USD',190 buyerPremiumIncluded: true,191 quantity: 1,192 isBundle: false,193 location: 'US',194 auctionHouse: 'Loupe This',195 lotNumber: a.lot,196 }),197 );198 continue;199 }200 const startsAt = a.startsAt ? new Date(a.startsAt) : null;201 const now = raw.fetchedAt.getTime();202 const status = startsAt && startsAt.getTime() > now ? 'upcoming' : endsAt && endsAt.getTime() < now ? 'ended' : 'live';203 out.push(204 NormalizedAuctionLotSchema.parse({205 kind: 'auction_lot',206 ...base,207 confidence: 0.8,208 auctionHouse: 'Loupe This',209 auctionName: null,210 lotNumber: a.lot,211 startsAt,212 endsAt,213 estimateLow: null,214 estimateHigh: null,215 currentBid: a.currentBidCents && a.currentBidCents > 0 ? a.currentBidCents / 100 : null,216 currency: 'USD',217 status,218 location: 'US',219 }),220 );221 }222 return out;223 }224}225226export default function createConnector(meta: ConnectorMeta) {227 return new LoupeThisConnector(meta);228}229