import { z } from 'zod'; import { BaseConnector, adapters, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; import { brandFromSlug, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js'; import { fromUnix, isBundleMultilingual, isSupportedCurrency, parenCount, stripHtml, yearFromTitle } from '../_g8-auctions-eu-apac-lib/index.js'; /** * Auctionet — public JSON API (https://auctionet.com/api/v2/items.json). One raw record per API page * (trimmed items); normalise → `sale` (winning bid = hammer, native SEK/EUR/DKK/GBP) for sold lots and * `auction_lot` (status ended) for unsold ones. Titles are in the consigning house's language * (sv/de/da/fi/es/en); taxonomy comes from Auctionet's category tree + title keywords. */ const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ id: z.number(), catalogNumber: z.string().nullable(), auctionId: z.number().nullable(), currency: z.string(), estimate: z.number().nullable(), upperEstimate: z.number().nullable(), reserveMet: z.boolean().nullable(), state: z.string(), hammered: z.boolean().nullable(), title: z.string(), description: z.string().nullable(), condition: z.string().nullable(), companyId: z.number().nullable(), categoryId: z.number().nullable(), endsAt: z.number().nullable(), publishedAt: z.number().nullable(), type: z.string().nullable(), location: z.string().nullable(), house: z.string().nullable(), url: z.string(), images: z.array(z.string()), winningBid: z.number().nullable(), winningBidAt: z.number().nullable(), bidCount: z.number(), }); export type Item = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('items_page'), url: z.string(), page: z.number(), totalEntries: z.number().nullable(), slice: z.object({ categoryId: z.number().nullable(), countryCode: z.string().nullable() }), items: z.array(ItemSchema), }); export type PagePayload = z.infer; type ApiItem = Record; function n(v: unknown): number | null { if (v === null || v === undefined || v === '') return null; const x = typeof v === 'number' ? v : Number(v); return Number.isFinite(x) ? x : null; } /** Keep only what normalize() needs; drops bidder numbers, suggested bids, placement, thumbnails. */ export function trimItem(i: ApiItem): Item { const bids = Array.isArray(i.bids) ? (i.bids as Array<{ amount?: unknown; timestamp?: unknown }>) : []; const amounts = bids.map((b) => n(b.amount) ?? 0); const top = amounts.length ? Math.max(...amounts) : null; const topBid = bids.find((b) => n(b.amount) === top); const images = Array.isArray(i.images) ? (i.images as Array<{ w640?: string; hd?: string }>).map((im) => im.w640 ?? im.hd ?? '').filter(Boolean).slice(0, 3) : []; return ItemSchema.parse({ id: Number(i.id), catalogNumber: i.catalog_number !== null && i.catalog_number !== undefined ? String(i.catalog_number) : null, auctionId: n(i.auction_id), currency: String(i.currency ?? ''), estimate: n(i.estimate), upperEstimate: n(i.upper_estimate), reserveMet: typeof i.reserve_met === 'boolean' ? i.reserve_met : null, state: String(i.state ?? ''), hammered: typeof i.hammered === 'boolean' ? i.hammered : null, title: String(i.title ?? '').trim(), description: stripHtml(typeof i.description === 'string' ? i.description : null, 600), condition: stripHtml(typeof i.condition === 'string' ? i.condition : null, 300), companyId: n(i.company_id), categoryId: n(i.category_id), endsAt: n(i.ends_at), publishedAt: n(i.published_at), type: typeof i.type === 'string' ? i.type : null, location: typeof i.location === 'string' && i.location ? i.location : null, house: typeof i.house === 'string' && i.house ? i.house : null, url: String(i.url ?? ''), images, winningBid: top && top > 0 ? top : null, winningBidAt: topBid ? n(topBid.timestamp) : null, bidCount: bids.length, }); } export function parseItemsPage(json: unknown, url: string, page: number, slice: { categoryId: number | null; countryCode: string | null }): PagePayload | null { const j = json as { items?: ApiItem[]; pagination?: { total_entries?: number; total_pages?: number } } | null; if (!j || !Array.isArray(j.items)) return null; return { kind: 'items_page', url, page, totalEntries: n(j.pagination?.total_entries), slice, items: j.items.filter((i) => i && typeof i === 'object' && i.id !== undefined).map(trimItem) }; } /** * Auctionet category id → default taxonomy slug + department hint for title refinement. * `null` slug = out of taxonomy / regulated (licence weapons, firearms, vehicle parts…) → item skipped. * Ids captured live from /api/v2/categories.json (2026-09-08); unknown ids fall back to keyword sweep. */ export const CATEGORY_MAP: Record = { 25: { slug: 'art', hint: 'art', path: 'Art' }, 119: { slug: 'art', hint: 'art', path: 'Art > Drawings' }, 27: { slug: 'art', hint: 'prints', path: 'Art > Engravings & Prints' }, 30: { slug: 'art', hint: 'art', path: 'Art > Other' }, 28: { slug: 'art', hint: 'art', path: 'Art > Paintings' }, 26: { slug: 'photography', hint: 'photographs', path: 'Art > Photography' }, 29: { slug: 'art', hint: 'art', path: 'Art > Sculptures & Bronzes' }, 117: { slug: 'antiques', hint: 'asian', path: 'Asiatica' }, 319: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Bronzes' }, 325: { slug: 'porcelain', hint: 'ceramics', path: 'Asiatica > Ceramics & Porcelain' }, 323: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Cloisonné' }, 320: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Jade' }, 355: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Other' }, 321: { slug: 'art', hint: 'asian', path: 'Asiatica > Scrolls' }, 322: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Textiles' }, 324: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Wood' }, 50: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts' }, 206: { slug: 'historical_documents', hint: 'books', path: 'Books, Maps & Manuscripts > Autographs & Manuscripts' }, 204: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts > Books' }, 205: { slug: 'maps', hint: 'books', path: 'Books, Maps & Manuscripts > Maps' }, 207: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts > Other' }, 35: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles' }, 36: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Carpets' }, 285: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > European' }, 287: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Oriental' }, 286: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Persian' }, 37: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Textiles' }, 9: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain' }, 10: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > European' }, 11: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Oriental' }, 12: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Rest of the world' }, 210: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Tableware' }, 31: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches' }, 258: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Carriage & Miniature Clocks' }, 32: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Longcase clocks' }, 33: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Mantel clocks' }, 34: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Other clocks' }, 110: { slug: 'other_watches', hint: 'watches', path: 'Clocks & Watches > Pocket & Stop Watches' }, 127: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Wall Clocks' }, 15: { slug: 'other_watches', hint: 'watches', path: 'Clocks & Watches > Wristwatches' }, 46: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps' }, 128: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps > Coins & Banknotes' }, 135: { slug: 'medals', hint: 'militaria', path: 'Coins, Medals & Stamps > Orders & Medals' }, 131: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps > Other' }, 136: { slug: 'stamps', hint: 'stamps', path: 'Coins, Medals & Stamps > Stamps' }, 261: { slug: 'antiques', hint: 'popular_culture', path: 'Collectables' }, 262: { slug: 'advertising', hint: 'popular_culture', path: 'Collectables > Advertising & Signs' }, 269: { slug: 'music', hint: 'music', path: 'Collectables > Audio, Vinyl & Hi-Fi' }, 268: { slug: 'trading_cards', hint: 'cards', path: 'Collectables > Collectible trading cards' }, 54: { slug: 'sports_memorabilia', hint: 'sports', path: 'Collectables > Fishing equipment' }, 265: { slug: 'movie_memorabilia', hint: 'movies', path: 'Collectables > Movie memorabilia' }, 266: { slug: 'music_memorabilia', hint: 'music', path: 'Collectables > Music memorabilia' }, 51: { slug: 'musical_instruments', hint: 'music', path: 'Collectables > Musical instruments' }, 267: { slug: 'antiques', hint: 'popular_culture', path: 'Collectables > Other collectables' }, 263: { slug: 'pens', hint: 'pens', path: 'Collectables > Pens' }, 264: { slug: 'sports_memorabilia', hint: 'sports', path: 'Collectables > Sports memorabilia' }, 45: { slug: 'scientific_instruments', hint: 'science', path: 'Collectables > Technica & Nautica' }, 134: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica' }, 283: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > African tribal art' }, 284: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > Other' }, 282: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > Sami Arts & Crafts' }, 16: { slug: 'antiques', hint: 'furniture', path: 'Furniture' }, 18: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Armchairs & Chairs' }, 24: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Chests of drawers' }, 280: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Coffee Tables' }, 23: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Cupboards, Cabinets & Shelves' }, 279: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Desks' }, 22: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Dining room furniture' }, 281: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Dining tables' }, 17: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Other' }, 20: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Sofas & Seatings' }, 19: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Tables' }, 270: { slug: null, hint: 'unknown', path: 'Garden & Architectural' }, 272: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Architectural details' }, 21: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Garden' }, 271: { slug: 'antiques', hint: 'furniture', path: 'Garden & Architectural > Garden Sculptures & Urns' }, 273: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Other' }, 6: { slug: 'glass_crystal', hint: 'glass', path: 'Glass' }, 208: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Art glass' }, 8: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Other' }, 7: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Tableware' }, 209: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Utility glass' }, 13: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones' }, 106: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Bracelets' }, 107: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Brooches & Pendants' }, 259: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Costume Jewellery' }, 111: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Cufflinks & Tie Pins' }, 115: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Earrings' }, 48: { slug: 'gemstones', hint: 'jewelry', path: 'Jewellery & Gemstones > Gemstones' }, 14: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Jewellery' }, 109: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Jewellery Suites' }, 104: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Necklace' }, 118: { slug: 'antiques', hint: 'silver', path: 'Jewellery & Gemstones > Objet de vertu & Miscellaneous' }, 112: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Rings' }, 108: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Tiara' }, 59: { slug: null, hint: 'unknown', path: 'Licence weapons' }, 64: { slug: null, hint: 'unknown', path: 'Licence weapons > Airguns' }, 63: { slug: null, hint: 'unknown', path: 'Licence weapons > Combi/Combo' }, 60: { slug: null, hint: 'unknown', path: 'Licence weapons > Double express rifles' }, 70: { slug: null, hint: 'unknown', path: 'Licence weapons > Drilling' }, 65: { slug: null, hint: 'unknown', path: 'Licence weapons > Military weapons' }, 69: { slug: null, hint: 'unknown', path: 'Licence weapons > Other weapons' }, 67: { slug: null, hint: 'unknown', path: 'Licence weapons > Pistols' }, 68: { slug: null, hint: 'unknown', path: 'Licence weapons > Revolvers' }, 61: { slug: null, hint: 'unknown', path: 'Licence weapons > Rifles' }, 62: { slug: null, hint: 'unknown', path: 'Licence weapons > Shotguns' }, 1: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps' }, 4: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Candlesticks' }, 3: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Ceiling lights' }, 203: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Chandeliers' }, 2: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Floor lights' }, 5: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Other lighting' }, 125: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Table Lamps' }, 124: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Wall Lights' }, 42: { slug: 'antiques', hint: 'decorative', path: 'Mirrors' }, 43: { slug: 'antiques', hint: 'unknown', path: 'Miscellaneous' }, 47: { slug: 'antiques', hint: 'unknown', path: 'Miscellaneous > Miscellaneous' }, 133: { slug: null, hint: 'unknown', path: 'Miscellaneous > Modern Tools' }, 52: { slug: null, hint: 'unknown', path: 'Miscellaneous > Modern consumer electronics' }, 57: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses' }, 71: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses > Cameras & accessories' }, 66: { slug: 'scientific_instruments', hint: 'science', path: 'Photo, Cameras & Lenses > Optics' }, 72: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses > Other' }, 38: { slug: 'silver', hint: 'silver', path: 'Silver & Metals' }, 40: { slug: 'antiques', hint: 'decorative', path: 'Silver & Metals > Other metals' }, 41: { slug: 'antiques', hint: 'decorative', path: 'Silver & Metals > Pewter, Brass & Copper' }, 39: { slug: 'silver', hint: 'silver', path: 'Silver & Metals > Silver' }, 213: { slug: 'silver', hint: 'silver', path: 'Silver & Metals > Silver plated' }, 58: { slug: 'antiques', hint: 'furniture', path: 'Swedish Folk Art' }, 121: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Bowls & Boxes' }, 122: { slug: 'antiques', hint: 'furniture', path: 'Swedish Folk Art > Furniture' }, 123: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Other' }, 120: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Tools & Gears' }, 44: { slug: 'vintage_toys', hint: 'toys', path: 'Toys' }, 276: { slug: 'action_figures', hint: 'toys', path: 'Toys > Action figures & Sci-Fi' }, 211: { slug: 'independent_comics', hint: 'comics', path: 'Toys > Comics' }, 274: { slug: 'dolls', hint: 'toys', path: 'Toys > Dolls & Teddybears' }, 275: { slug: 'model_cars', hint: 'toys', path: 'Toys > Model cars' }, 278: { slug: 'model_trains', hint: 'toys', path: 'Toys > Model railways' }, 277: { slug: 'vintage_toys', hint: 'toys', path: 'Toys > Other toys' }, 212: { slug: 'vintage_toys', hint: 'toys', path: 'Toys > Toys' }, 249: { slug: null, hint: 'cars', path: 'Vehicles, Boats & Parts' }, 255: { slug: 'automotive_memorabilia', hint: 'automobilia', path: 'Vehicles, Boats & Parts > Automobilia & Transport' }, 132: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Bicycles' }, 250: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Boats & Accessories' }, 253: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Car parts' }, 215: { slug: 'automobiles', hint: 'cars', path: 'Vehicles, Boats & Parts > Cars' }, 254: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Moped parts' }, 216: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Mopeds' }, 252: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Motorcycle parts' }, 251: { slug: 'motorcycles', hint: 'motorcycles', path: 'Vehicles, Boats & Parts > Motorcycles' }, 256: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Other' }, 49: { slug: 'fashion_streetwear', hint: 'fashion', path: 'Vintage & Designer Fashion' }, 137: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria' }, 257: { slug: null, hint: 'unknown', path: 'Weapons & Militaria > Airguns' }, 138: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Armour & Uniform' }, 130: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Edged weapons' }, 129: { slug: null, hint: 'unknown', path: 'Weapons & Militaria > Guns & Rifles' }, 214: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Other' }, 170: { slug: 'wine', hint: 'wine', path: 'Wine, Port & Spirits' }, }; /** Multilingual title cues that are more reliable than the (English-only) auction-lib sweep. */ const LOCAL_CUES: Array<[RegExp, string]> = [ [/\b(armbandsur|armbanduhr|wristwatch|herrur|damur|reloj de pulsera|rannekello)\b/i, 'other_watches'], [/\b(fickur|taschenuhr|pocket watch|reloj de bolsillo|taskukello)\b/i, 'other_watches'], [/\b(sedlar|sedel|banknote|geldschein|billete)\b/i, 'banknotes'], [/\b(mynt|münze|münzen|coin|moneda|kolikko|ducat|dukat|riksdaler|daler|thaler|taler)\b/i, 'coins'], [/\b(frimärken|frimärke|briefmarke|stamp|sello|postimerkki)\b/i, 'stamps'], [/\b(orden|medalj|medaille|medal|orden och medaljer)\b/i, 'medals'], [/\b(serietidning|serietidningar|comic|comics|tegneserie|sarjakuva)\b/i, 'independent_comics'], [/\b(lp|vinyl|skivor|schallplatte)\b/i, 'music'], [/\b(kamera|camera|leica|hasselblad|rolleiflex)\b/i, 'cameras'], [/\b(whisky|whiskey|bourbon|cognac|armagnac|rom|rhum|rum)\b/i, 'whisky'], [/\b(handväska|handtasche|handbag|bolso|hermès|hermes|chanel|louis vuitton)\b/i, 'luxury_handbags'], [/\b(sneakers|air jordan|nike|yeezy)\b/i, 'sneakers'], [/\blego\b/i, 'lego_sets'], [/\b(pokémon|pokemon|magic: the gathering|yu-gi-oh)\b/i, 'trading_cards'], ]; /** Resolve the taxonomy slug for an item; null → skip (regulated/out-of-taxonomy category). */ export function categoryFor(categoryId: number | null, title: string): { slug: string | null; hint: DeptHint; path: string | null } { const c = categoryId !== null ? CATEGORY_MAP[categoryId] : undefined; if (c && c.slug === null) return { slug: null, hint: c.hint, path: c.path }; const hint: DeptHint = c?.hint ?? 'unknown'; let slug: string | null = null; if (hint === 'watches') slug = slugFromTitle(title, 'watches'); else if (hint === 'wine') slug = slugFromTitle(title, 'wine'); else if (hint === 'coins') slug = LOCAL_CUES.find(([re, s]) => re.test(title) && ['banknotes', 'coins', 'medals'].includes(s))?.[1] ?? c?.slug ?? 'coins'; else if (hint === 'toys') slug = /\blego\b/i.test(title) ? 'lego_sets' : c && c.slug !== 'vintage_toys' ? c.slug : slugFromTitle(title, 'toys') ?? 'vintage_toys'; else if (hint === 'cards' || hint === 'comics' || hint === 'popular_culture' || hint === 'fashion' || hint === 'cars') slug = slugFromTitle(title, hint) ?? c?.slug ?? null; else if (hint === 'furniture' || hint === 'decorative') slug = slugFromTitle(title, 'furniture') ?? c?.slug ?? null; else if (hint === 'unknown') slug = LOCAL_CUES.find(([re]) => re.test(title))?.[1] ?? slugFromTitle(title) ?? c?.slug ?? null; else slug = c?.slug ?? null; // multilingual refinements for lots filed under broad categories if (slug && ['antiques', 'art', 'vintage_toys', 'fashion_streetwear'].includes(slug)) { const cue = LOCAL_CUES.find(([re]) => re.test(title))?.[1]; if (cue && !(slug === 'art' && cue === 'music')) slug = cue; } if (slug === 'trading_cards') slug = slugFromTitle(title, 'cards') ?? 'other_tcg'; return { slug, hint, path: c?.path ?? null }; } function withParams(base: string, params: Record): string { return adapters.withParams(base, params); } type Cursor = { sinceEndsAt?: number; inProgress?: { newest: number; page: number } | null; sliceIndex?: number; page?: number; itemsProcessed?: number; done?: boolean }; export class AuctionetConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; private get apiBase(): string { return String(this.meta.config.apiBase ?? 'https://auctionet.com/api/v2'); } private get perPage(): number { return Math.min(500, Math.max(10, Number(this.meta.config.perPage ?? 100))); } private get maxPage(): number { return Math.floor(Number(this.meta.config.apiItemCap ?? 10_000) / this.perPage); } private async fetchPage(ctx: CrawlContext, page: number, slice: { categoryId: number | null; countryCode: string | null }): Promise<{ payload: PagePayload | null; url: string; res: Awaited> }> { const url = withParams(`${this.apiBase}/items.json`, { is: 'ended', per_page: this.perPage, page, category_id: slice.categoryId, country_code: slice.countryCode }); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', timeoutMs: 60_000, expect: ['title', 'price', 'currency', 'date'], parse: (r) => { const p = parseItemsPage(r.json, url, page, slice); const sold = p?.items.find((i) => i.winningBid); return p ? { title: p.items[0]?.title, price: sold?.winningBid ?? null, currency: sold?.currency ?? null, date: sold?.endsAt ?? null } : null; }, }); const payload = res.success ? parseItemsPage(res.json, url, page, slice) : null; return { payload, url, res }; } async *crawl(ctx: CrawlContext): AsyncIterable { if (ctx.options.mode === 'backfill') { yield* this.backfill(ctx); return; } const cursor = (ctx.options.cursor ?? {}) as Cursor; const since = cursor.sinceEndsAt ?? 0; const maxPages = Math.min(Number(this.meta.config.incrementalMaxPages ?? 60), this.maxPage); let newest = cursor.inProgress?.newest ?? 0; let page = cursor.inProgress?.page ?? 1; let count = 0; const slice = { categoryId: null, countryCode: null }; for (; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const { payload, url, res } = await this.fetchPage(ctx, page, slice); if (!payload) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } if (payload.items.length === 0) break; const pageNewest = Math.max(...payload.items.map((i) => i.endsAt ?? 0)); newest = Math.max(newest, pageNewest); const fresh = payload.items.filter((i) => (i.endsAt ?? 0) > since); const reachedCheckpoint = fresh.length < payload.items.length; if (fresh.length) { count++; yield { url, externalId: `ended:p${page}:${fresh[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { ...payload, items: fresh }, fetchedAt: res.fetchedAt }; } if (reachedCheckpoint) break; await ctx.setCursor({ sinceEndsAt: since, inProgress: { newest, page: page + 1 } }); } if (ctx.options.mode !== 'probe') await ctx.setCursor({ sinceEndsAt: Math.max(since, newest), inProgress: null }); } /** Backfill: category × country slices (each capped at apiItemCap items by the API), resumable. */ private async *backfill(ctx: CrawlContext): AsyncIterable { const cursor = (ctx.options.cursor ?? {}) as Cursor; if (cursor.done) return; const slices = this.slices(ctx.options.categories); let sliceIndex = cursor.sliceIndex ?? 0; let page = cursor.page ?? 1; let itemsProcessed = cursor.itemsProcessed ?? 0; let count = 0; let reachedDate: Date | null = null; const maxPagesRun = this.policy.backfillMaxPages; let fetched = 0; for (; sliceIndex < slices.length; sliceIndex++, page = 1) { const slice = slices[sliceIndex]!; let totalPages: number | null = null; for (; page <= this.maxPage; page++) { if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= maxPagesRun) { await ctx.setCursor({ sliceIndex, page, itemsProcessed }); return; } const { payload, url, res } = await this.fetchPage(ctx, page, slice); fetched++; if (!payload) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } if (payload.totalEntries !== null) totalPages = Math.min(this.maxPage, Math.ceil(payload.totalEntries / this.perPage)); if (payload.items.length === 0) break; count++; itemsProcessed += payload.items.length; const oldest = Math.min(...payload.items.map((i) => i.endsAt ?? Number.MAX_SAFE_INTEGER)); if (Number.isFinite(oldest) && oldest < Number.MAX_SAFE_INTEGER) reachedDate = fromUnix(oldest); yield { url, externalId: `backfill:c${slice.categoryId ?? 'all'}:${slice.countryCode ?? 'all'}:p${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ sliceIndex, page: page + 1, itemsProcessed }); await ctx.progress({ page: sliceIndex * this.maxPage + page, totalPages: slices.length * this.maxPage, itemsProcessed, reachedDate, cursor: { sliceIndex, page: page + 1 } }); if (totalPages !== null && page >= totalPages) break; } } await ctx.setCursor({ done: true, itemsProcessed }); await ctx.progress({ page: slices.length * this.maxPage, totalPages: slices.length * this.maxPage, itemsProcessed, reachedDate, cursor: { done: true } }); } /** Leaf categories kept in taxonomy (optionally restricted to requested slugs) × configured countries. */ slices(categorySlugs?: string[]): Array<{ categoryId: number | null; countryCode: string | null }> { const configured = (this.meta.config.backfillCategoryIds as number[] | undefined) ?? []; let ids = configured.length ? configured : leafCategoryIds(); if (categorySlugs?.length) { const want = new Set(categorySlugs); ids = ids.filter((id) => want.has(CATEGORY_MAP[id]!.slug!) || want.has(familyOf(CATEGORY_MAP[id]!.slug!))); } const countries = (this.meta.config.countryCodes as string[] | undefined) ?? [null]; const out: Array<{ categoryId: number | null; countryCode: string | null }> = []; for (const id of ids) for (const cc of countries.length ? countries : [null]) out.push({ categoryId: id, countryCode: cc }); return out; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const item of p.items) { const rec = normalizeItem(this.meta, item, raw.fetchedAt, p.slice); if (rec) out.push(rec); } return out; } } /** Leaf categories kept in taxonomy: a node is a leaf when no other node's path extends it (roots without children count). */ export function leafCategoryIds(): number[] { const paths = Object.values(CATEGORY_MAP).map((c) => c.path); return Object.entries(CATEGORY_MAP) .filter(([, c]) => c.slug !== null && !paths.some((p) => p.startsWith(`${c.path} > `))) .map(([id]) => Number(id)); } function familyOf(slug: string): string { const fam: Record = { rolex: 'watches', omega: 'watches', patek_philippe: 'watches', audemars_piguet: 'watches', other_watches: 'watches', independent_comics: 'comics', marvel_comics: 'comics', dc_comics: 'comics', manga: 'comics', lego_sets: 'lego', banknotes: 'coins', medals: 'militaria', whisky: 'wine', cognac: 'wine', rum: 'wine' }; return fam[slug] ?? slug; } export function normalizeItem(meta: ConnectorMeta, item: Item, observedAt: Date, slice?: { categoryId: number | null; countryCode: string | null }): NormalizedRecord | null { const cat = categoryFor(item.categoryId, item.title); if (!cat.slug) return null; if (!isSupportedCurrency(item.currency)) return null; const endsAt = fromUnix(item.endsAt); if (!endsAt) return null; const g = parseGradeFromTitle(item.title); const count = parenCount(item.title); const isBundle = isBundleMultilingual(item.title); const brand = brandFromSlug(cat.slug, item.title); const reference = ['rolex', 'omega', 'patek_philippe', 'audemars_piguet', 'other_watches'].includes(cat.slug) ? watchReference(item.title) : null; const attributes = AssetAttributesSchema.parse({ categorySlug: cat.slug, name: item.title, brand, reference, year: yearFromTitle(item.title), identifiers: { auctionet_item_id: String(item.id) }, metadata: { house: item.house, company_id: item.companyId, auction_id: item.auctionId, auctionet_category_id: item.categoryId, auctionet_category: cat.path, estimate: item.estimate, upper_estimate: item.upperEstimate, reserve_met: item.reserveMet, bid_count: item.bidCount, auction_type: item.type, state: item.state, price_basis: 'winning_bid_excl_buyer_fee', slice_country: slice?.countryCode ?? null, }, }); const base = { connectorId: meta.id, sourceId: meta.sourceId, sourceUrl: item.url, externalId: String(item.id), rawTitle: item.title, description: item.description, imageUrls: item.images, attributes, grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: item.condition, completeness: null }, observedAt, confidence: cat.hint === 'unknown' ? 0.75 : 0.85, parserVersion: PARSER_VERSION, }; const currency = item.currency; if (item.state === 'sold' && item.winningBid) { return NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate: endsAt, price: item.winningBid, currency, buyerPremiumIncluded: false, quantity: count && !isBundle ? count : 1, isBundle, location: item.location, auctionHouse: item.house ?? 'Auctionet', lotNumber: item.catalogNumber }); } const status = item.state === 'published' ? (endsAt.getTime() > observedAt.getTime() ? 'live' : 'ended') : 'ended'; return NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: item.house ?? 'Auctionet', auctionName: null, lotNumber: item.catalogNumber, startsAt: fromUnix(item.publishedAt), endsAt, estimateLow: item.estimate, estimateHigh: item.upperEstimate, currentBid: item.winningBid, currency, status, location: item.location }); } export default function createConnector(meta: ConnectorMeta) { return new AuctionetConnector(meta); }