TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, adapters, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseGradeFromTitle } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';5import { brandFromSlug, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js';6import { fromUnix, isBundleMultilingual, isSupportedCurrency, parenCount, stripHtml, yearFromTitle } from '../_g8-auctions-eu-apac-lib/index.js';78/**9 * Auctionet — public JSON API (https://auctionet.com/api/v2/items.json). One raw record per API page10 * (trimmed items); normalise → `sale` (winning bid = hammer, native SEK/EUR/DKK/GBP) for sold lots and11 * `auction_lot` (status ended) for unsold ones. Titles are in the consigning house's language12 * (sv/de/da/fi/es/en); taxonomy comes from Auctionet's category tree + title keywords.13 */1415const PARSER_VERSION = '1.0.0';1617export const ItemSchema = z.object({18 id: z.number(),19 catalogNumber: z.string().nullable(),20 auctionId: z.number().nullable(),21 currency: z.string(),22 estimate: z.number().nullable(),23 upperEstimate: z.number().nullable(),24 reserveMet: z.boolean().nullable(),25 state: z.string(),26 hammered: z.boolean().nullable(),27 title: z.string(),28 description: z.string().nullable(),29 condition: z.string().nullable(),30 companyId: z.number().nullable(),31 categoryId: z.number().nullable(),32 endsAt: z.number().nullable(),33 publishedAt: z.number().nullable(),34 type: z.string().nullable(),35 location: z.string().nullable(),36 house: z.string().nullable(),37 url: z.string(),38 images: z.array(z.string()),39 winningBid: z.number().nullable(),40 winningBidAt: z.number().nullable(),41 bidCount: z.number(),42});43export type Item = z.infer<typeof ItemSchema>;4445export const PagePayloadSchema = z.object({46 kind: z.literal('items_page'),47 url: z.string(),48 page: z.number(),49 totalEntries: z.number().nullable(),50 slice: z.object({ categoryId: z.number().nullable(), countryCode: z.string().nullable() }),51 items: z.array(ItemSchema),52});53export type PagePayload = z.infer<typeof PagePayloadSchema>;5455type ApiItem = Record<string, unknown>;5657function n(v: unknown): number | null {58 if (v === null || v === undefined || v === '') return null;59 const x = typeof v === 'number' ? v : Number(v);60 return Number.isFinite(x) ? x : null;61}6263/** Keep only what normalize() needs; drops bidder numbers, suggested bids, placement, thumbnails. */64export function trimItem(i: ApiItem): Item {65 const bids = Array.isArray(i.bids) ? (i.bids as Array<{ amount?: unknown; timestamp?: unknown }>) : [];66 const amounts = bids.map((b) => n(b.amount) ?? 0);67 const top = amounts.length ? Math.max(...amounts) : null;68 const topBid = bids.find((b) => n(b.amount) === top);69 const images = Array.isArray(i.images) ? (i.images as Array<{ w640?: string; hd?: string }>).map((im) => im.w640 ?? im.hd ?? '').filter(Boolean).slice(0, 3) : [];70 return ItemSchema.parse({71 id: Number(i.id),72 catalogNumber: i.catalog_number !== null && i.catalog_number !== undefined ? String(i.catalog_number) : null,73 auctionId: n(i.auction_id),74 currency: String(i.currency ?? ''),75 estimate: n(i.estimate),76 upperEstimate: n(i.upper_estimate),77 reserveMet: typeof i.reserve_met === 'boolean' ? i.reserve_met : null,78 state: String(i.state ?? ''),79 hammered: typeof i.hammered === 'boolean' ? i.hammered : null,80 title: String(i.title ?? '').trim(),81 description: stripHtml(typeof i.description === 'string' ? i.description : null, 600),82 condition: stripHtml(typeof i.condition === 'string' ? i.condition : null, 300),83 companyId: n(i.company_id),84 categoryId: n(i.category_id),85 endsAt: n(i.ends_at),86 publishedAt: n(i.published_at),87 type: typeof i.type === 'string' ? i.type : null,88 location: typeof i.location === 'string' && i.location ? i.location : null,89 house: typeof i.house === 'string' && i.house ? i.house : null,90 url: String(i.url ?? ''),91 images,92 winningBid: top && top > 0 ? top : null,93 winningBidAt: topBid ? n(topBid.timestamp) : null,94 bidCount: bids.length,95 });96}9798export function parseItemsPage(json: unknown, url: string, page: number, slice: { categoryId: number | null; countryCode: string | null }): PagePayload | null {99 const j = json as { items?: ApiItem[]; pagination?: { total_entries?: number; total_pages?: number } } | null;100 if (!j || !Array.isArray(j.items)) return null;101 return { kind: 'items_page', url, page, totalEntries: n(j.pagination?.total_entries), slice, items: j.items.filter((i) => i && typeof i === 'object' && i.id !== undefined).map(trimItem) };102}103104/**105 * Auctionet category id → default taxonomy slug + department hint for title refinement.106 * `null` slug = out of taxonomy / regulated (licence weapons, firearms, vehicle parts…) → item skipped.107 * Ids captured live from /api/v2/categories.json (2026-09-08); unknown ids fall back to keyword sweep.108 */109export const CATEGORY_MAP: Record<number, { slug: string | null; hint: DeptHint; path: string }> = {110 25: { slug: 'art', hint: 'art', path: 'Art' }, 119: { slug: 'art', hint: 'art', path: 'Art > Drawings' }, 27: { slug: 'art', hint: 'prints', path: 'Art > Engravings & Prints' }, 30: { slug: 'art', hint: 'art', path: 'Art > Other' }, 28: { slug: 'art', hint: 'art', path: 'Art > Paintings' }, 26: { slug: 'photography', hint: 'photographs', path: 'Art > Photography' }, 29: { slug: 'art', hint: 'art', path: 'Art > Sculptures & Bronzes' },111 117: { slug: 'antiques', hint: 'asian', path: 'Asiatica' }, 319: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Bronzes' }, 325: { slug: 'porcelain', hint: 'ceramics', path: 'Asiatica > Ceramics & Porcelain' }, 323: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Cloisonné' }, 320: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Jade' }, 355: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Other' }, 321: { slug: 'art', hint: 'asian', path: 'Asiatica > Scrolls' }, 322: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Textiles' }, 324: { slug: 'antiques', hint: 'asian', path: 'Asiatica > Wood' },112 50: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts' }, 206: { slug: 'historical_documents', hint: 'books', path: 'Books, Maps & Manuscripts > Autographs & Manuscripts' }, 204: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts > Books' }, 205: { slug: 'maps', hint: 'books', path: 'Books, Maps & Manuscripts > Maps' }, 207: { slug: 'books', hint: 'books', path: 'Books, Maps & Manuscripts > Other' },113 35: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles' }, 36: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Carpets' }, 285: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > European' }, 287: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Oriental' }, 286: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Persian' }, 37: { slug: 'antiques', hint: 'furniture', path: 'Carpets & Textiles > Textiles' },114 9: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain' }, 10: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > European' }, 11: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Oriental' }, 12: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Rest of the world' }, 210: { slug: 'porcelain', hint: 'ceramics', path: 'Ceramics & Porcelain > Tableware' },115 31: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches' }, 258: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Carriage & Miniature Clocks' }, 32: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Longcase clocks' }, 33: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Mantel clocks' }, 34: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Other clocks' }, 110: { slug: 'other_watches', hint: 'watches', path: 'Clocks & Watches > Pocket & Stop Watches' }, 127: { slug: 'clocks', hint: 'clocks', path: 'Clocks & Watches > Wall Clocks' }, 15: { slug: 'other_watches', hint: 'watches', path: 'Clocks & Watches > Wristwatches' },116 46: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps' }, 128: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps > Coins & Banknotes' }, 135: { slug: 'medals', hint: 'militaria', path: 'Coins, Medals & Stamps > Orders & Medals' }, 131: { slug: 'coins', hint: 'coins', path: 'Coins, Medals & Stamps > Other' }, 136: { slug: 'stamps', hint: 'stamps', path: 'Coins, Medals & Stamps > Stamps' },117 261: { slug: 'antiques', hint: 'popular_culture', path: 'Collectables' }, 262: { slug: 'advertising', hint: 'popular_culture', path: 'Collectables > Advertising & Signs' }, 269: { slug: 'music', hint: 'music', path: 'Collectables > Audio, Vinyl & Hi-Fi' }, 268: { slug: 'trading_cards', hint: 'cards', path: 'Collectables > Collectible trading cards' }, 54: { slug: 'sports_memorabilia', hint: 'sports', path: 'Collectables > Fishing equipment' }, 265: { slug: 'movie_memorabilia', hint: 'movies', path: 'Collectables > Movie memorabilia' }, 266: { slug: 'music_memorabilia', hint: 'music', path: 'Collectables > Music memorabilia' }, 51: { slug: 'musical_instruments', hint: 'music', path: 'Collectables > Musical instruments' }, 267: { slug: 'antiques', hint: 'popular_culture', path: 'Collectables > Other collectables' }, 263: { slug: 'pens', hint: 'pens', path: 'Collectables > Pens' }, 264: { slug: 'sports_memorabilia', hint: 'sports', path: 'Collectables > Sports memorabilia' }, 45: { slug: 'scientific_instruments', hint: 'science', path: 'Collectables > Technica & Nautica' },118 134: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica' }, 283: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > African tribal art' }, 284: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > Other' }, 282: { slug: 'antiques', hint: 'tribal', path: 'Ethnographica > Sami Arts & Crafts' },119 16: { slug: 'antiques', hint: 'furniture', path: 'Furniture' }, 18: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Armchairs & Chairs' }, 24: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Chests of drawers' }, 280: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Coffee Tables' }, 23: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Cupboards, Cabinets & Shelves' }, 279: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Desks' }, 22: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Dining room furniture' }, 281: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Dining tables' }, 17: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Other' }, 20: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Sofas & Seatings' }, 19: { slug: 'antiques', hint: 'furniture', path: 'Furniture > Tables' },120 270: { slug: null, hint: 'unknown', path: 'Garden & Architectural' }, 272: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Architectural details' }, 21: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Garden' }, 271: { slug: 'antiques', hint: 'furniture', path: 'Garden & Architectural > Garden Sculptures & Urns' }, 273: { slug: null, hint: 'unknown', path: 'Garden & Architectural > Other' },121 6: { slug: 'glass_crystal', hint: 'glass', path: 'Glass' }, 208: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Art glass' }, 8: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Other' }, 7: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Tableware' }, 209: { slug: 'glass_crystal', hint: 'glass', path: 'Glass > Utility glass' },122 13: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones' }, 106: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Bracelets' }, 107: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Brooches & Pendants' }, 259: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Costume Jewellery' }, 111: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Cufflinks & Tie Pins' }, 115: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Earrings' }, 48: { slug: 'gemstones', hint: 'jewelry', path: 'Jewellery & Gemstones > Gemstones' }, 14: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Jewellery' }, 109: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Jewellery Suites' }, 104: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Necklace' }, 118: { slug: 'antiques', hint: 'silver', path: 'Jewellery & Gemstones > Objet de vertu & Miscellaneous' }, 112: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Rings' }, 108: { slug: 'jewelry', hint: 'jewelry', path: 'Jewellery & Gemstones > Tiara' },123 59: { slug: null, hint: 'unknown', path: 'Licence weapons' }, 64: { slug: null, hint: 'unknown', path: 'Licence weapons > Airguns' }, 63: { slug: null, hint: 'unknown', path: 'Licence weapons > Combi/Combo' }, 60: { slug: null, hint: 'unknown', path: 'Licence weapons > Double express rifles' }, 70: { slug: null, hint: 'unknown', path: 'Licence weapons > Drilling' }, 65: { slug: null, hint: 'unknown', path: 'Licence weapons > Military weapons' }, 69: { slug: null, hint: 'unknown', path: 'Licence weapons > Other weapons' }, 67: { slug: null, hint: 'unknown', path: 'Licence weapons > Pistols' }, 68: { slug: null, hint: 'unknown', path: 'Licence weapons > Revolvers' }, 61: { slug: null, hint: 'unknown', path: 'Licence weapons > Rifles' }, 62: { slug: null, hint: 'unknown', path: 'Licence weapons > Shotguns' },124 1: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps' }, 4: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Candlesticks' }, 3: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Ceiling lights' }, 203: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Chandeliers' }, 2: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Floor lights' }, 5: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Other lighting' }, 125: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Table Lamps' }, 124: { slug: 'antiques', hint: 'decorative', path: 'Lighting & Lamps > Wall Lights' },125 42: { slug: 'antiques', hint: 'decorative', path: 'Mirrors' },126 43: { slug: 'antiques', hint: 'unknown', path: 'Miscellaneous' }, 47: { slug: 'antiques', hint: 'unknown', path: 'Miscellaneous > Miscellaneous' }, 133: { slug: null, hint: 'unknown', path: 'Miscellaneous > Modern Tools' }, 52: { slug: null, hint: 'unknown', path: 'Miscellaneous > Modern consumer electronics' },127 57: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses' }, 71: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses > Cameras & accessories' }, 66: { slug: 'scientific_instruments', hint: 'science', path: 'Photo, Cameras & Lenses > Optics' }, 72: { slug: 'cameras', hint: 'cameras', path: 'Photo, Cameras & Lenses > Other' },128 38: { slug: 'silver', hint: 'silver', path: 'Silver & Metals' }, 40: { slug: 'antiques', hint: 'decorative', path: 'Silver & Metals > Other metals' }, 41: { slug: 'antiques', hint: 'decorative', path: 'Silver & Metals > Pewter, Brass & Copper' }, 39: { slug: 'silver', hint: 'silver', path: 'Silver & Metals > Silver' }, 213: { slug: 'silver', hint: 'silver', path: 'Silver & Metals > Silver plated' },129 58: { slug: 'antiques', hint: 'furniture', path: 'Swedish Folk Art' }, 121: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Bowls & Boxes' }, 122: { slug: 'antiques', hint: 'furniture', path: 'Swedish Folk Art > Furniture' }, 123: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Other' }, 120: { slug: 'antiques', hint: 'decorative', path: 'Swedish Folk Art > Tools & Gears' },130 44: { slug: 'vintage_toys', hint: 'toys', path: 'Toys' }, 276: { slug: 'action_figures', hint: 'toys', path: 'Toys > Action figures & Sci-Fi' }, 211: { slug: 'independent_comics', hint: 'comics', path: 'Toys > Comics' }, 274: { slug: 'dolls', hint: 'toys', path: 'Toys > Dolls & Teddybears' }, 275: { slug: 'model_cars', hint: 'toys', path: 'Toys > Model cars' }, 278: { slug: 'model_trains', hint: 'toys', path: 'Toys > Model railways' }, 277: { slug: 'vintage_toys', hint: 'toys', path: 'Toys > Other toys' }, 212: { slug: 'vintage_toys', hint: 'toys', path: 'Toys > Toys' },131 249: { slug: null, hint: 'cars', path: 'Vehicles, Boats & Parts' }, 255: { slug: 'automotive_memorabilia', hint: 'automobilia', path: 'Vehicles, Boats & Parts > Automobilia & Transport' }, 132: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Bicycles' }, 250: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Boats & Accessories' }, 253: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Car parts' }, 215: { slug: 'automobiles', hint: 'cars', path: 'Vehicles, Boats & Parts > Cars' }, 254: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Moped parts' }, 216: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Mopeds' }, 252: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Motorcycle parts' }, 251: { slug: 'motorcycles', hint: 'motorcycles', path: 'Vehicles, Boats & Parts > Motorcycles' }, 256: { slug: null, hint: 'unknown', path: 'Vehicles, Boats & Parts > Other' },132 49: { slug: 'fashion_streetwear', hint: 'fashion', path: 'Vintage & Designer Fashion' },133 137: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria' }, 257: { slug: null, hint: 'unknown', path: 'Weapons & Militaria > Airguns' }, 138: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Armour & Uniform' }, 130: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Edged weapons' }, 129: { slug: null, hint: 'unknown', path: 'Weapons & Militaria > Guns & Rifles' }, 214: { slug: 'militaria', hint: 'militaria', path: 'Weapons & Militaria > Other' },134 170: { slug: 'wine', hint: 'wine', path: 'Wine, Port & Spirits' },135};136137/** Multilingual title cues that are more reliable than the (English-only) auction-lib sweep. */138const LOCAL_CUES: Array<[RegExp, string]> = [139 [/\b(armbandsur|armbanduhr|wristwatch|herrur|damur|reloj de pulsera|rannekello)\b/i, 'other_watches'],140 [/\b(fickur|taschenuhr|pocket watch|reloj de bolsillo|taskukello)\b/i, 'other_watches'],141 [/\b(sedlar|sedel|banknote|geldschein|billete)\b/i, 'banknotes'],142 [/\b(mynt|münze|münzen|coin|moneda|kolikko|ducat|dukat|riksdaler|daler|thaler|taler)\b/i, 'coins'],143 [/\b(frimärken|frimärke|briefmarke|stamp|sello|postimerkki)\b/i, 'stamps'],144 [/\b(orden|medalj|medaille|medal|orden och medaljer)\b/i, 'medals'],145 [/\b(serietidning|serietidningar|comic|comics|tegneserie|sarjakuva)\b/i, 'independent_comics'],146 [/\b(lp|vinyl|skivor|schallplatte)\b/i, 'music'],147 [/\b(kamera|camera|leica|hasselblad|rolleiflex)\b/i, 'cameras'],148 [/\b(whisky|whiskey|bourbon|cognac|armagnac|rom|rhum|rum)\b/i, 'whisky'],149 [/\b(handväska|handtasche|handbag|bolso|hermès|hermes|chanel|louis vuitton)\b/i, 'luxury_handbags'],150 [/\b(sneakers|air jordan|nike|yeezy)\b/i, 'sneakers'],151 [/\blego\b/i, 'lego_sets'],152 [/\b(pokémon|pokemon|magic: the gathering|yu-gi-oh)\b/i, 'trading_cards'],153];154155/** Resolve the taxonomy slug for an item; null → skip (regulated/out-of-taxonomy category). */156export function categoryFor(categoryId: number | null, title: string): { slug: string | null; hint: DeptHint; path: string | null } {157 const c = categoryId !== null ? CATEGORY_MAP[categoryId] : undefined;158 if (c && c.slug === null) return { slug: null, hint: c.hint, path: c.path };159 const hint: DeptHint = c?.hint ?? 'unknown';160 let slug: string | null = null;161 if (hint === 'watches') slug = slugFromTitle(title, 'watches');162 else if (hint === 'wine') slug = slugFromTitle(title, 'wine');163 else if (hint === 'coins') slug = LOCAL_CUES.find(([re, s]) => re.test(title) && ['banknotes', 'coins', 'medals'].includes(s))?.[1] ?? c?.slug ?? 'coins';164 else if (hint === 'toys') slug = /\blego\b/i.test(title) ? 'lego_sets' : c && c.slug !== 'vintage_toys' ? c.slug : slugFromTitle(title, 'toys') ?? 'vintage_toys';165 else if (hint === 'cards' || hint === 'comics' || hint === 'popular_culture' || hint === 'fashion' || hint === 'cars') slug = slugFromTitle(title, hint) ?? c?.slug ?? null;166 else if (hint === 'furniture' || hint === 'decorative') slug = slugFromTitle(title, 'furniture') ?? c?.slug ?? null;167 else if (hint === 'unknown') slug = LOCAL_CUES.find(([re]) => re.test(title))?.[1] ?? slugFromTitle(title) ?? c?.slug ?? null;168 else slug = c?.slug ?? null;169 // multilingual refinements for lots filed under broad categories170 if (slug && ['antiques', 'art', 'vintage_toys', 'fashion_streetwear'].includes(slug)) {171 const cue = LOCAL_CUES.find(([re]) => re.test(title))?.[1];172 if (cue && !(slug === 'art' && cue === 'music')) slug = cue;173 }174 if (slug === 'trading_cards') slug = slugFromTitle(title, 'cards') ?? 'other_tcg';175 return { slug, hint, path: c?.path ?? null };176}177178function withParams(base: string, params: Record<string, string | number | null | undefined>): string {179 return adapters.withParams(base, params);180}181182type Cursor = { sinceEndsAt?: number; inProgress?: { newest: number; page: number } | null; sliceIndex?: number; page?: number; itemsProcessed?: number; done?: boolean };183184export class AuctionetConnector extends BaseConnector {185 readonly version = '1.0.0';186 readonly parserVersion = PARSER_VERSION;187 protected override minIntervalMs = 1500;188189 private get apiBase(): string {190 return String(this.meta.config.apiBase ?? 'https://auctionet.com/api/v2');191 }192 private get perPage(): number {193 return Math.min(500, Math.max(10, Number(this.meta.config.perPage ?? 100)));194 }195 private get maxPage(): number {196 return Math.floor(Number(this.meta.config.apiItemCap ?? 10_000) / this.perPage);197 }198199 private async fetchPage(ctx: CrawlContext, page: number, slice: { categoryId: number | null; countryCode: string | null }): Promise<{ payload: PagePayload | null; url: string; res: Awaited<ReturnType<CrawlContext['fetch']>> }> {200 const url = withParams(`${this.apiBase}/items.json`, { is: 'ended', per_page: this.perPage, page, category_id: slice.categoryId, country_code: slice.countryCode });201 await this.throttle(url);202 const res = await ctx.fetch(url, {203 engines: ['api'],204 responseType: 'json',205 timeoutMs: 60_000,206 expect: ['title', 'price', 'currency', 'date'],207 parse: (r) => {208 const p = parseItemsPage(r.json, url, page, slice);209 const sold = p?.items.find((i) => i.winningBid);210 return p ? { title: p.items[0]?.title, price: sold?.winningBid ?? null, currency: sold?.currency ?? null, date: sold?.endsAt ?? null } : null;211 },212 });213 const payload = res.success ? parseItemsPage(res.json, url, page, slice) : null;214 return { payload, url, res };215 }216217 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {218 if (ctx.options.mode === 'backfill') {219 yield* this.backfill(ctx);220 return;221 }222 const cursor = (ctx.options.cursor ?? {}) as Cursor;223 const since = cursor.sinceEndsAt ?? 0;224 const maxPages = Math.min(Number(this.meta.config.incrementalMaxPages ?? 60), this.maxPage);225 let newest = cursor.inProgress?.newest ?? 0;226 let page = cursor.inProgress?.page ?? 1;227 let count = 0;228 const slice = { categoryId: null, countryCode: null };229 for (; page <= maxPages; page++) {230 if (ctx.signal?.aborted || this.reached(ctx, count)) break;231 const { payload, url, res } = await this.fetchPage(ctx, page, slice);232 if (!payload) {233 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);234 break;235 }236 if (payload.items.length === 0) break;237 const pageNewest = Math.max(...payload.items.map((i) => i.endsAt ?? 0));238 newest = Math.max(newest, pageNewest);239 const fresh = payload.items.filter((i) => (i.endsAt ?? 0) > since);240 const reachedCheckpoint = fresh.length < payload.items.length;241 if (fresh.length) {242 count++;243 yield { url, externalId: `ended:p${page}:${fresh[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { ...payload, items: fresh }, fetchedAt: res.fetchedAt };244 }245 if (reachedCheckpoint) break;246 await ctx.setCursor({ sinceEndsAt: since, inProgress: { newest, page: page + 1 } });247 }248 if (ctx.options.mode !== 'probe') await ctx.setCursor({ sinceEndsAt: Math.max(since, newest), inProgress: null });249 }250251 /** Backfill: category × country slices (each capped at apiItemCap items by the API), resumable. */252 private async *backfill(ctx: CrawlContext): AsyncIterable<RawRecordInput> {253 const cursor = (ctx.options.cursor ?? {}) as Cursor;254 if (cursor.done) return;255 const slices = this.slices(ctx.options.categories);256 let sliceIndex = cursor.sliceIndex ?? 0;257 let page = cursor.page ?? 1;258 let itemsProcessed = cursor.itemsProcessed ?? 0;259 let count = 0;260 let reachedDate: Date | null = null;261 const maxPagesRun = this.policy.backfillMaxPages;262 let fetched = 0;263 for (; sliceIndex < slices.length; sliceIndex++, page = 1) {264 const slice = slices[sliceIndex]!;265 let totalPages: number | null = null;266 for (; page <= this.maxPage; page++) {267 if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= maxPagesRun) {268 await ctx.setCursor({ sliceIndex, page, itemsProcessed });269 return;270 }271 const { payload, url, res } = await this.fetchPage(ctx, page, slice);272 fetched++;273 if (!payload) {274 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);275 break;276 }277 if (payload.totalEntries !== null) totalPages = Math.min(this.maxPage, Math.ceil(payload.totalEntries / this.perPage));278 if (payload.items.length === 0) break;279 count++;280 itemsProcessed += payload.items.length;281 const oldest = Math.min(...payload.items.map((i) => i.endsAt ?? Number.MAX_SAFE_INTEGER));282 if (Number.isFinite(oldest) && oldest < Number.MAX_SAFE_INTEGER) reachedDate = fromUnix(oldest);283 yield { url, externalId: `backfill:c${slice.categoryId ?? 'all'}:${slice.countryCode ?? 'all'}:p${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };284 await ctx.setCursor({ sliceIndex, page: page + 1, itemsProcessed });285 await ctx.progress({ page: sliceIndex * this.maxPage + page, totalPages: slices.length * this.maxPage, itemsProcessed, reachedDate, cursor: { sliceIndex, page: page + 1 } });286 if (totalPages !== null && page >= totalPages) break;287 }288 }289 await ctx.setCursor({ done: true, itemsProcessed });290 await ctx.progress({ page: slices.length * this.maxPage, totalPages: slices.length * this.maxPage, itemsProcessed, reachedDate, cursor: { done: true } });291 }292293 /** Leaf categories kept in taxonomy (optionally restricted to requested slugs) × configured countries. */294 slices(categorySlugs?: string[]): Array<{ categoryId: number | null; countryCode: string | null }> {295 const configured = (this.meta.config.backfillCategoryIds as number[] | undefined) ?? [];296 let ids = configured.length ? configured : leafCategoryIds();297 if (categorySlugs?.length) {298 const want = new Set(categorySlugs);299 ids = ids.filter((id) => want.has(CATEGORY_MAP[id]!.slug!) || want.has(familyOf(CATEGORY_MAP[id]!.slug!)));300 }301 const countries = (this.meta.config.countryCodes as string[] | undefined) ?? [null];302 const out: Array<{ categoryId: number | null; countryCode: string | null }> = [];303 for (const id of ids) for (const cc of countries.length ? countries : [null]) out.push({ categoryId: id, countryCode: cc });304 return out;305 }306307 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {308 const p = PagePayloadSchema.parse(raw.payload);309 const out: NormalizedRecord[] = [];310 for (const item of p.items) {311 const rec = normalizeItem(this.meta, item, raw.fetchedAt, p.slice);312 if (rec) out.push(rec);313 }314 return out;315 }316}317318/** Leaf categories kept in taxonomy: a node is a leaf when no other node's path extends it (roots without children count). */319export function leafCategoryIds(): number[] {320 const paths = Object.values(CATEGORY_MAP).map((c) => c.path);321 return Object.entries(CATEGORY_MAP)322 .filter(([, c]) => c.slug !== null && !paths.some((p) => p.startsWith(`${c.path} > `)))323 .map(([id]) => Number(id));324}325326function familyOf(slug: string): string {327 const fam: Record<string, string> = { rolex: 'watches', omega: 'watches', patek_philippe: 'watches', audemars_piguet: 'watches', other_watches: 'watches', independent_comics: 'comics', marvel_comics: 'comics', dc_comics: 'comics', manga: 'comics', lego_sets: 'lego', banknotes: 'coins', medals: 'militaria', whisky: 'wine', cognac: 'wine', rum: 'wine' };328 return fam[slug] ?? slug;329}330331export function normalizeItem(meta: ConnectorMeta, item: Item, observedAt: Date, slice?: { categoryId: number | null; countryCode: string | null }): NormalizedRecord | null {332 const cat = categoryFor(item.categoryId, item.title);333 if (!cat.slug) return null;334 if (!isSupportedCurrency(item.currency)) return null;335 const endsAt = fromUnix(item.endsAt);336 if (!endsAt) return null;337 const g = parseGradeFromTitle(item.title);338 const count = parenCount(item.title);339 const isBundle = isBundleMultilingual(item.title);340 const brand = brandFromSlug(cat.slug, item.title);341 const reference = ['rolex', 'omega', 'patek_philippe', 'audemars_piguet', 'other_watches'].includes(cat.slug) ? watchReference(item.title) : null;342 const attributes = AssetAttributesSchema.parse({343 categorySlug: cat.slug,344 name: item.title,345 brand,346 reference,347 year: yearFromTitle(item.title),348 identifiers: { auctionet_item_id: String(item.id) },349 metadata: {350 house: item.house,351 company_id: item.companyId,352 auction_id: item.auctionId,353 auctionet_category_id: item.categoryId,354 auctionet_category: cat.path,355 estimate: item.estimate,356 upper_estimate: item.upperEstimate,357 reserve_met: item.reserveMet,358 bid_count: item.bidCount,359 auction_type: item.type,360 state: item.state,361 price_basis: 'winning_bid_excl_buyer_fee',362 slice_country: slice?.countryCode ?? null,363 },364 });365 const base = {366 connectorId: meta.id,367 sourceId: meta.sourceId,368 sourceUrl: item.url,369 externalId: String(item.id),370 rawTitle: item.title,371 description: item.description,372 imageUrls: item.images,373 attributes,374 grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, qualifier: null, certificationNumber: null },375 condition: { condition: null, conditionRaw: item.condition, completeness: null },376 observedAt,377 confidence: cat.hint === 'unknown' ? 0.75 : 0.85,378 parserVersion: PARSER_VERSION,379 };380 const currency = item.currency;381 if (item.state === 'sold' && item.winningBid) {382 return NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate: endsAt, price: item.winningBid, currency, buyerPremiumIncluded: false, quantity: count && !isBundle ? count : 1, isBundle, location: item.location, auctionHouse: item.house ?? 'Auctionet', lotNumber: item.catalogNumber });383 }384 const status = item.state === 'published' ? (endsAt.getTime() > observedAt.getTime() ? 'live' : 'ended') : 'ended';385 return NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: item.house ?? 'Auctionet', auctionName: null, lotNumber: item.catalogNumber, startsAt: fromUnix(item.publishedAt), endsAt, estimateLow: item.estimate, estimateHigh: item.upperEstimate, currentBid: item.winningBid, currency, status, location: item.location });386}387388export default function createConnector(meta: ConnectorMeta) {389 return new AuctionetConnector(meta);390}391