TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { adapters, BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../../firecrawl/_carlib/index.js';5import { clean, coinYear, isNumisBundle, numisAttributes, numisCategory, parseAuctionDate, parseCoinGrade, realized } from '../../firecrawl/_g5-numismatics-lib/index.js';67/**8 * Fritz Rudolf Künker (Osnabrück) — Europe's largest coin auction house. Every past lot keeps a static9 * Shopware product page (≈ 725 000 URLs in the product sitemaps) with auction name, lot number, sale10 * date, estimate and "Hammer price" in EUR, plus a properties table (Nominal/Year, Mint, Condition with11 * PCGS/NGC slab data, references). Plain HTTPS; the JS lot listings (/widgets/) are robots-disallowed and12 * never used — discovery goes through the sitemaps.13 */1415const SITE = 'https://www.kuenker.de';16const PARSER_VERSION = '1.0.0';1718export const PayloadSchema = z.object({19 kind: z.literal('lot_page'),20 url: z.string(),21 productId: z.string(),22 title: z.string(),23 section1: z.string().nullable(),24 section2: z.string().nullable(),25 auctionName: z.string().nullable(),26 lotNumber: z.string().nullable(),27 dateText: z.string().nullable(),28 statusText: z.string().nullable(),29 estimateText: z.string().nullable(),30 hammerText: z.string().nullable(),31 properties: z.record(z.string(), z.string()),32 description: z.string().nullable(),33 images: z.array(z.string()),34});35export type Payload = z.infer<typeof PayloadSchema>;3637export function parseLotPage(htmlText: string, url: string): Payload | null {38 const $ = H.load(htmlText);39 const productId = url.match(/\/(\d+)\/?(?:[?#].*)?$/)?.[1] ?? null;40 const h1 = $('h1.product-detail-name').first();41 const section1 = H.text(h1.find('.section-1').first());42 const section2 = H.text(h1.find('.section-2').first());43 const ld = H.jsonLd(htmlText, 'Product')[0] as { name?: string; sku?: string; image?: string[] | string; description?: string } | undefined;44 // the JSON-LD name carries the full issuer line ("SCHWEDEN KÖNIGREICH Oskar II. …"); the mobile h1 may drop the country45 const title = (ld?.name ? clean(ld.name) : '') || clean(`${section1 ?? ''} ${section2 ?? ''}`);46 if (!productId || !title) return null;47 const info = $('.auction-lot-info').first();48 const auctionName = H.text(info.find('b').first());49 const lotNumber = (H.text(info.find('span').first()) ?? '').match(/(\d+[A-Za-z]?)/)?.[1] ?? null;50 const dateBox = $('.date-status-container').first();51 const dateText = H.text(dateBox.find('span').first());52 const statusText = clean(dateBox.clone().children('span').remove().end().text()) || null;53 const estimateText = ($('.estimated-price').first().text().match(/:\s*([^\n]+)/)?.[1] ?? '').trim() || null;54 const hammerBlock = $('.bid-status-finished').first();55 const hammerText = /hammer price|zuschlag/i.test(hammerBlock.text()) ? H.text(hammerBlock.find('.price').first()) : null;56 const properties: Record<string, string> = {};57 $('table.product-detail-properties-table tr.properties-row').each((_, tr) => {58 const k = H.text($(tr).find('th').first());59 const v = H.text($(tr).find('td').first());60 if (k && v) properties[k] = v;61 });62 const images: string[] = [];63 const ldImages = Array.isArray(ld?.image) ? ld!.image : ld?.image ? [ld.image] : [];64 for (const i of ldImages) if (typeof i === 'string') images.push(i.replace(/\\\//g, '/'));65 const og = $('meta[property="og:image"]').attr('content');66 if (og && !images.includes(og)) images.push(og);67 const description = ld?.description ? clean(ld.description.replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"')) : null;68 return { kind: 'lot_page', url, productId, title, section1, section2, auctionName, lotNumber, dateText, statusText, estimateText, hammerText, properties, description, images: images.slice(0, 3) };69}7071/** "Fast Stempelglanz / In US-Plastikholder der PCGS mit der Bewertung MS 64 PL (812624.64/38930174)." → slab ids. */72export function slabIds(condition: string | null | undefined): { pcgsNumber: string | null; cert: string | null } {73 const m = condition?.match(/\((\d{4,7})\.(\d{2})\/(\d{7,10})\)/);74 if (m) return { pcgsNumber: m[1]!, cert: m[3]! };75 const c = condition?.match(/\b(?:Zertifikat|cert(?:ificate)?|No\.?|Nr\.?)\s*#?\s*(\d{7,10})\b/i);76 return { pcgsNumber: null, cert: c?.[1] ?? null };77}7879interface Cursor {80 sitemapIndex?: number;81 urlIndex?: number;82 done?: boolean;83 updatedAt?: string;84}8586export class KuenkerConnector extends BaseConnector {87 readonly version = '1.0.0';88 readonly parserVersion = PARSER_VERSION;89 protected override minIntervalMs = 1500;90 override readonly urlPatterns = [/kuenker\.de\/[^/]+\/\d+\/?$/i];9192 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {93 const lotsPerRun = Number(this.meta.config.lotsPerRun ?? 100);94 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };95 const seeds = ctx.options.seeds?.filter((s) => /kuenker\.de\//.test(s)) ?? [];96 let urls: string[] = seeds;97 if (!urls.length) {98 await this.throttle();99 const index = await ctx.fetch(`${SITE}/sitemap.xml`, { engines: ['api'], responseType: 'text', minQuality: 0, force: true });100 const children = index.success && index.html ? adapters.parseSitemapIndex(index.html).filter((c) => /sitemap-products-/.test(c.loc)) : [];101 if (!children.length) {102 ctx.anomaly('sitemap_fetch_failed', `sitemap.xml: ${index.error ?? index.httpStatus ?? 'no product sitemaps'}`);103 return;104 }105 children.sort((a, b) => Number(a.loc.match(/products-(\d+)/)?.[1] ?? 0) - Number(b.loc.match(/products-(\d+)/)?.[1] ?? 0));106 const si = Math.min(cursor.sitemapIndex ?? 0, children.length - 1);107 await this.throttle();108 const sm = await ctx.fetch(children[si]!.loc, { engines: ['api'], responseType: 'text', minQuality: 0, force: true });109 const entries = sm.success && sm.html ? adapters.parseUrlset(sm.html) : [];110 if (!entries.length) {111 ctx.anomaly('sitemap_fetch_failed', `${children[si]!.loc}: ${sm.error ?? sm.httpStatus}`);112 return;113 }114 const start = cursor.urlIndex ?? 0;115 urls = entries.slice(start, start + lotsPerRun).map((e) => e.loc);116 const nextIndex = start + urls.length;117 const exhausted = nextIndex >= entries.length;118 cursor.sitemapIndex = exhausted ? (si + 1 >= children.length ? 0 : si + 1) : si;119 cursor.urlIndex = exhausted ? 0 : nextIndex;120 cursor.done = ctx.options.mode === 'backfill' && exhausted && si + 1 >= children.length;121 if (ctx.options.mode === 'backfill') await ctx.progress({ page: si * 25_000 + nextIndex, totalPages: children.length * 25_000, itemsProcessed: 0 });122 }123 let count = 0;124 for (const url of urls) {125 if (ctx.signal?.aborted || this.reached(ctx, count)) break;126 if (ctx.options.mode !== 'backfill' && !(await ctx.shouldFetch(url))) continue;127 await this.throttle();128 const res = await ctx.fetch(url, {129 engines: ['api', 'firecrawl'],130 responseType: 'text',131 expect: ['title', 'price', 'date'],132 parse: (r) => {133 const p = r.html ? parseLotPage(r.html, url) : null;134 return p ? { title: p.title, price: p.hammerText ?? p.estimateText, date: p.dateText } : null;135 },136 });137 const payload = res.success && res.html ? parseLotPage(res.html, url) : null;138 if (!payload) {139 ctx.anomaly(res.success ? 'parse_failure_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);140 continue;141 }142 count++;143 yield { url, externalId: payload.productId, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };144 }145 await ctx.setCursor({ ...cursor, updatedAt: new Date().toISOString() });146 }147148 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {149 const p = PayloadSchema.parse(raw.payload);150 const price = realized(p.hammerText, 'EUR');151 const saleDate = parseAuctionDate(p.dateText);152 if (!price || !saleDate) return []; // unsold, withdrawn or still running → no sale153 const condition = p.properties.Condition ?? p.properties.Erhaltung ?? null;154 const text = `${p.title} ${condition ?? ''}`;155 const g = parseCoinGrade(text);156 const slab = slabIds(condition);157 const categorySlug = numisCategory(p.title, `${p.auctionName ?? ''} ${p.section1 ?? ''}`, /medaille|medal|orden/i.test(p.title) && !/münze|coin|taler|dukat|mark|pfennig|kreuzer/i.test(p.title) ? 'medals' : 'coins');158 const identifiers: Record<string, string> = { kuenker_product_id: p.productId };159 if (slab.pcgsNumber) identifiers.pcgs_number = slab.pcgsNumber;160 if (slab.cert && g.grader) identifiers[`${g.grader}_cert`] = slab.cert;161 const estimate = realized(p.estimateText, 'EUR');162 const attributes = numisAttributes({163 categorySlug,164 title: p.title,165 section: p.section1,166 materialHint: p.properties.Weight ?? p.properties.Gewicht ?? null,167 year: coinYear(p.properties['Nominal/Year'] ?? p.properties['Nominal/Jahr'] ?? '').year,168 identifiers,169 metadata: {170 auction: p.auctionName,171 status: p.statusText,172 estimate: estimate?.amount ?? null,173 estimate_currency: estimate?.currency ?? null,174 hammer_price: price.amount,175 buyer_premium: "excluded — page label 'Hammer price' (Zuschlag); Künker's premium is stated in the auction terms",176 nominal_year: p.properties['Nominal/Year'] ?? p.properties['Nominal/Jahr'] ?? null,177 mint: p.properties.Mint ?? p.properties.Prägestätte ?? null,178 rarity: p.properties.Rarity ?? p.properties.Seltenheit ?? null,179 weight: p.properties.Weight ?? p.properties.Gewicht ?? null,180 references: p.properties.Quotes ?? p.properties.Zitate ?? null,181 condition_text: condition,182 },183 });184 if (p.properties.Mint && !attributes.variant) attributes.variant = p.properties.Mint.replace(/\.$/, '');185 const sale = makeSale({186 meta: this.meta,187 sourceUrl: p.url,188 externalId: p.productId,189 rawTitle: p.title,190 description: p.description,191 attributes,192 price: price.amount,193 currency: price.currency,194 saleDate,195 buyerPremiumIncluded: false,196 auctionHouse: 'Fritz Rudolf Künker',197 lotNumber: p.lotNumber,198 imageUrls: p.images,199 observedAt: raw.fetchedAt,200 parserVersion: PARSER_VERSION,201 confidence: g.grader ? 0.9 : 0.82,202 isBundle: isNumisBundle(p.title) || /\bLot\s+von\b|\bLots?\b.*\bStück\b/i.test(p.title),203 conditionRaw: condition?.split('/')[0]?.trim() ?? g.conditionRaw,204 location: 'DE',205 });206 sale.grade = { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: slab.cert ?? g.certificationNumber };207 return [sale];208 }209}210211export default (meta: ConnectorMeta) => new KuenkerConnector(meta);212