TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { adapters, BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { slugFromTitle, watchBrand } from '../_auction-lib/categories.js';5import { popCultureCategory } from '../_memorabilia-lib/index.js';6import { amount, houseCategory, isBundleTitle, isCurrency, isoDate, jsonObjectsWithMarker, lotAttributes, makeLot, makeSale, safeYear, saleGrade, sportsCategory } from '../_g7-auctions-na-lib/index.js';78const BASE = 'https://live.millerandmillerauctions.com';9const HOUSE = 'Miller & Miller Auctions';10const PARSER_VERSION = '1.0.0';1112export const AuctionSchema = z.object({ id: z.string(), title: z.string(), url: z.string(), timeStart: z.string().nullable(), endTime: z.string().nullable(), auctionType: z.string().nullable(), buyersPremiumPct: z.number().nullable(), currency: z.string().nullable(), lotCount: z.number().nullable() });13export const LotSchema = z.object({14 id: z.string(),15 lotNumber: z.string().nullable(),16 status: z.string().nullable(),17 title: z.string(),18 description: z.string().nullable(),19 estimateLow: z.number().nullable(),20 estimateHigh: z.number().nullable(),21 startingPrice: z.number().nullable(),22 currency: z.string().nullable(),23 image: z.string().nullable(),24 /** winning timed bid (hammer, before buyer's premium) */25 hammer: z.number().nullable(),26 bidAt: z.string().nullable(),27 lastUpdated: z.string().nullable(),28 url: z.string(),29});30export const PayloadSchema = z.object({ kind: z.literal('am_catalog_page'), house: z.string(), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) });31export type Payload = z.infer<typeof PayloadSchema>;3233type AmAuction = { row_id?: string; title?: string; time_start?: string | null; effective_end_time?: string | null; extended_end_time?: string | null; auction_type?: string | null; default_buyers_premium?: string | number | null; currency_code?: string | null; lot_count?: number | null; _detail_url?: string | null; _slug?: string | null };34type AmLot = { row_id?: string; lot_number?: number | string | null; lot_number_extension?: string | null; status?: string | null; title?: string; truncated_description?: string | null; estimate_low?: string | null; estimate_high?: string | null; starting_price?: string | null; currency_code?: string | null; cover_thumbnail?: string | null; timed_auction_bid?: { amount?: string | null; updated_at?: string | null } | null; last_updated?: string | null; _detail_url?: string | null; _slug?: string | null; auction?: AmAuction | null };3536function slugify(s: string): string {37 return s.toLowerCase().replace(/&/g, ' and ').replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80);38}3940/** Auction Mobility server-rendered catalog page → auction header + lot summaries (36 per page). */41export function parseAmCatalog(html: string, auctionId: string, page: number): Payload | null {42 const auctions = jsonObjectsWithMarker<AmAuction>(html, '"type":"auction"');43 const lots = jsonObjectsWithMarker<AmLot>(html, '"type":"auction-lot-summary"');44 const a = auctions.find((x) => x.row_id === auctionId) ?? auctions[0] ?? lots[0]?.auction ?? null;45 if (!a && lots.length === 0) return null;46 const auction: z.infer<typeof AuctionSchema> = {47 id: a?.row_id ?? auctionId,48 title: a?.title ?? '',49 url: a?._detail_url ? `${BASE}${a._detail_url}` : `${BASE}/auctions/${auctionId}`,50 timeStart: a?.time_start ?? null,51 endTime: a?.extended_end_time ?? a?.effective_end_time ?? lots.find((l) => l.auction?.effective_end_time)?.auction?.effective_end_time ?? null,52 auctionType: a?.auction_type ?? null,53 buyersPremiumPct: amount(a?.default_buyers_premium ?? lots.find((l) => l.auction?.default_buyers_premium)?.auction?.default_buyers_premium),54 currency: a?.currency_code ?? lots.find((l) => l.currency_code)?.currency_code ?? null,55 lotCount: typeof a?.lot_count === 'number' ? a.lot_count : null,56 };57 const seen = new Set<string>();58 const items: Payload['lots'] = [];59 for (const l of lots) {60 if (!l.row_id || !l.title || seen.has(l.row_id)) continue;61 seen.add(l.row_id);62 items.push({63 id: l.row_id,64 lotNumber: l.lot_number !== null && l.lot_number !== undefined ? `${l.lot_number}${l.lot_number_extension ?? ''}` : null,65 status: l.status ?? null,66 title: l.title.trim(),67 description: l.truncated_description?.trim() || null,68 estimateLow: amount(l.estimate_low),69 estimateHigh: amount(l.estimate_high),70 startingPrice: amount(l.starting_price),71 currency: l.currency_code ?? auction.currency,72 image: l.cover_thumbnail ?? null,73 hammer: amount(l.timed_auction_bid?.amount),74 bidAt: l.timed_auction_bid?.updated_at ?? null,75 lastUpdated: l.last_updated ?? null,76 url: l._detail_url ? `${BASE}${l._detail_url}` : `${BASE}/lots/view/${l.row_id}/${slugify(l.title)}`,77 });78 }79 return { kind: 'am_catalog_page', house: HOUSE, auction, page, lots: items };80}8182/** Miller & Miller sale titles → taxonomy; the sale department is the strongest signal. */83export function millerCategory(saleTitle: string, lotTitle: string): string | null {84 const s = saleTitle.toLowerCase();85 if (/firearm|weapon|edged/.test(s) || /\b(rifle|pistol|revolver|shotgun|carbine|ammunition|cartridges?)\b/i.test(lotTitle)) return null;86 if (/sports? card|sports? memorabilia|hockey|baseball/.test(s)) return sportsCategory(lotTitle);87 if (/petroliana|advertising|soda|breweriana|general store|country store|signs?/.test(s)) return /\b(sign|clock|thermometer|display|calendar|poster|globe|tin\b|can\b|bottle|crate|tray|door push|cabinet|dispenser)\b/i.test(lotTitle) ? 'advertising' : slugFromTitle(lotTitle) ?? 'advertising';88 if (/coin[- ]?op|arcade|jukebox|slot/.test(s)) return /\b(slot machine|trade stimulator)\b/i.test(lotTitle) ? 'casino_memorabilia' : /\b(jukebox|arcade|pinball)\b/i.test(lotTitle) ? 'arcade_pinball' : 'vending_machines';89 if (/toys?|comics?|pop culture|star wars|movie|music memorabilia/.test(s)) return popCultureCategory(lotTitle);90 if (/watch|jewel/.test(s)) return /watch|chronograph/i.test(lotTitle) ? watchBrand(lotTitle).slug : slugFromTitle(lotTitle, 'jewelry');91 if (/coins?|currency|banknote|numismatic/.test(s)) return /\b(note|bill|banknote|dollar bill)\b/i.test(lotTitle) ? 'banknotes' : 'coins';92 if (/folk art|fine art|paintings?|canadian art|inuit|prints/.test(s)) return slugFromTitle(lotTitle, 'art') ?? 'art';93 if (/decoy|canadiana|historic|antiques?|furniture|pottery|glass|clocks?|decorative/.test(s)) return slugFromTitle(lotTitle, 'furniture') ?? 'antiques';94 if (/music|instrument|guitar/.test(s)) return slugFromTitle(lotTitle, 'music') ?? 'musical_instruments';95 if (/militaria|military|medal/.test(s)) return slugFromTitle(lotTitle, 'militaria') ?? 'militaria';96 if (/book|map|document|ephemera|postcard/.test(s)) return /\bpostcard/i.test(lotTitle) ? 'postcards' : slugFromTitle(lotTitle, 'books') ?? 'books';97 if (/automobil|motorcycle|cars?\b|automotive/.test(s)) return slugFromTitle(lotTitle, 'cars') ?? 'automotive_memorabilia';98 return houseCategory(saleTitle, lotTitle, null);99}100101/**102 * Miller & Miller Auctions (New Hamburg, Ontario) — Auction Mobility platform. The public catalog pages103 * live.millerandmillerauctions.com/auctions/<id>[/<slug>]?page=N embed the lot summaries (36 per page) as104 * JSON in the server-rendered HTML; past lots carry status "sold" and the winning timed bid (hammer, CAD).105 * Auction ids come from the platform's own sitemap (/sitemap/auctions.xml). No API calls, no login.106 */107export class MillerMillerConnector extends BaseConnector {108 readonly version = '1.0.0';109 readonly parserVersion = PARSER_VERSION;110 protected override minIntervalMs = 10_000; // robots.txt Crawl-delay: 10111112 private async page(ctx: CrawlContext, auctionId: string, page: number): Promise<{ payload: Payload | null; engine: 'api'; status: number | null; fetchedAt: Date; url: string }> {113 const url = `${BASE}/auctions/${auctionId}${page > 1 ? `?page=${page}` : ''}`;114 await this.throttle(url);115 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });116 if (!res.success || !res.html) {117 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);118 return { payload: null, engine: 'api', status: res.httpStatus, fetchedAt: res.fetchedAt, url };119 }120 const payload = parseAmCatalog(res.html, auctionId, page);121 if (!payload) ctx.anomaly('parse_failure_page', `${url}: no Auction Mobility JSON found`);122 return { payload, engine: 'api', status: res.httpStatus, fetchedAt: res.fetchedAt, url };123 }124125 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {126 const cfg = this.meta.config;127 const auctionsPerRun = ctx.options.mode === 'probe' ? 1 : Number(cfg.auctionsPerRun ?? (ctx.options.mode === 'backfill' ? 6 : 3));128 const maxPages = ctx.options.mode === 'probe' ? 1 : Number(cfg.pagesPerAuction ?? 40);129 const cursor = ctx.options.cursor ?? {};130 const finished: Record<string, string> = typeof cursor.finished === 'object' && cursor.finished ? { ...(cursor.finished as Record<string, string>) } : {};131 let ids: string[] = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => s.match(/auctions\/([A-Z0-9-]+)/i)?.[1] ?? s) : [];132 if (!ids.length) {133 const entries = await adapters.discoverFromSitemap(ctx, `${BASE}/sitemap/auctions.xml`, { maxSitemaps: 1, match: /sitemap\/auctions\// });134 // The index itself lists one child sitemap per auction; we only need the ids (no child fetch).135 ids = entries.map((e) => e.loc.match(/auctions\/([A-Z0-9-]+)\.xml$/i)?.[1] ?? '').filter(Boolean);136 if (!ids.length) {137 const res = await ctx.fetch(`${BASE}/sitemap/auctions.xml`, { engines: ['api'], responseType: 'text', minQuality: 0, force: true });138 ids = [...(res.html ?? '').matchAll(/auctions\/([A-Z0-9-]+)\.xml/gi)].map((m) => m[1]!);139 }140 if (!ids.length) {141 ctx.anomaly('pagination_failure', 'auction sitemap returned no auction ids');142 return;143 }144 ids.reverse(); // newest ids are appended last by the platform145 }146 const total = ids.length;147 const pending = ids.filter((id) => !finished[id]);148 let processed = 0;149 let count = 0;150 let items = 0;151 for (const id of pending) {152 if (ctx.signal?.aborted || processed >= auctionsPerRun || this.reached(ctx, count)) break;153 let complete = true;154 let endTime: string | null = null;155 for (let page = 1; page <= maxPages; page++) {156 if (ctx.signal?.aborted) break;157 const r = await this.page(ctx, id, page);158 if (!r.payload) {159 complete = false;160 break;161 }162 endTime = r.payload.auction.endTime ?? r.payload.auction.timeStart ?? endTime;163 if (r.payload.lots.length === 0) break;164 count++;165 items += r.payload.lots.length;166 yield { url: r.url, externalId: `auction:${id}:page:${page}`, kind: r.payload.lots.some((l) => l.status === 'sold') ? 'sale' : 'auction_lot', engine: r.engine, httpStatus: r.status, payload: r.payload, fetchedAt: r.fetchedAt };167 if (r.payload.lots.length < 36 || this.reached(ctx, count)) break;168 if (page === maxPages) complete = false;169 }170 processed++;171 // Closed auctions are crawled once; open/upcoming ones are revisited every run until they close.172 const ended = endTime ? new Date(endTime).getTime() < Date.now() - 2 * 86_400_000 : false;173 if (complete && ended) finished[id] = endTime ?? new Date().toISOString();174 const remaining = ids.filter((x) => !finished[x]).length;175 await ctx.setCursor({ finished, updatedAt: new Date().toISOString() });176 await ctx.progress({ page: total - remaining, totalPages: total, itemsProcessed: items, cursor: { finished } });177 }178 if (ctx.options.mode === 'backfill' && ids.every((id) => finished[id])) await ctx.setCursor({ finished, done: true, updatedAt: new Date().toISOString() });179 }180181 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {182 const p = PayloadSchema.parse(raw.payload);183 const out: NormalizedRecord[] = [];184 const auctionEnd = isoDate(p.auction.endTime);185 const auctionStart = isoDate(p.auction.timeStart);186 const now = Date.now();187 for (const l of p.lots) {188 const slug = millerCategory(p.auction.title, l.title);189 if (!slug) continue;190 const currency = isCurrency(l.currency) ? l.currency : isCurrency(p.auction.currency) ? p.auction.currency : 'CAD';191 const g = saleGrade(l.title);192 const attributes = lotAttributes({ categorySlug: slug, name: l.title, year: safeYear(l.title), identifiers: { miller_lot_id: l.id }, metadata: { auction_id: p.auction.id, auction_title: p.auction.title, auction_type: p.auction.auctionType, estimate_low: l.estimateLow, estimate_high: l.estimateHigh, starting_price: l.startingPrice, buyers_premium_pct: p.auction.buyersPremiumPct, hammer_price: l.hammer } });193 const common = { meta: this.meta, sourceUrl: l.url, externalId: l.id, rawTitle: l.title, description: l.description, attributes, imageUrls: l.image ? [l.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'CA' };194 if (l.status === 'sold' && l.hammer) {195 const saleDate = auctionEnd ?? isoDate(l.bidAt) ?? isoDate(l.lastUpdated);196 if (!saleDate || saleDate.getTime() > now + 86_400_000) continue;197 out.push(makeSale({ ...common, price: l.hammer, currency, saleDate, buyerPremiumIncluded: false, auctionHouse: HOUSE, lotNumber: l.lotNumber, isBundle: isBundleTitle(l.title), confidence: 0.85 }));198 } else if (l.status !== 'sold' && l.status !== 'expired' && (!auctionEnd || auctionEnd.getTime() > now)) {199 out.push(makeLot({ ...common, auctionHouse: HOUSE, auctionName: p.auction.title, lotNumber: l.lotNumber, startsAt: auctionStart, endsAt: auctionEnd, estimateLow: l.estimateLow, estimateHigh: l.estimateHigh, currentBid: l.hammer, currency, status: auctionStart && auctionStart.getTime() > now ? 'upcoming' : 'live', confidence: 0.8 }));200 }201 }202 return out;203 }204}205206export default (meta: ConnectorMeta) => new MillerMillerConnector(meta);207