TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../../firecrawl/_carlib/index.js';5import { clean, isNumisBundle, numisAttributes, parseAuctionDate, realized } from '../../firecrawl/_g5-numismatics-lib/index.js';67/**8 * Corinphila Auctions (Zürich) — Switzerland's oldest stamp auction house. The archive on corinphila.ch9 * (c4ms platform) is static HTML: auction overview (catalogue parts with lot counts, c4msEnv.auctionData10 * JSON with currency and dates) → lot list pages of 100 lots with country, description, starting bid and11 * "Hammer price : 260.00 CHF" (or "not sold"). Live catalogues (auction.corinphila.ch) are JS/ajax and12 * are not used. robots: Crawl-Delay 10 → 10 s between requests.13 */1415const SITE = 'https://corinphila.ch';16const PARSER_VERSION = '1.0.0';1718export const AuctionSchema = z.object({ id: z.string(), name: z.string(), currency: z.string().nullable(), startDate: z.string().nullable(), endDate: z.string().nullable(), status: z.string().nullable() });19export const PartSchema = z.object({ catalogPart: z.string(), title: z.string().nullable(), lotCount: z.number().int().nullable() });20export const LotSchema = z.object({21 lotNo: z.string(),22 country: z.string().nullable(),23 description: z.string(),24 startText: z.string().nullable(),25 hammerText: z.string().nullable(),26 conditionCodes: z.array(z.string()).default([]),27 images: z.array(z.string()).default([]),28});29export const PayloadSchema = z.object({ kind: z.literal('lots_page'), auction: AuctionSchema, part: PartSchema, url: z.string(), page: z.number().int(), totalPages: z.number().int().nullable(), lots: z.array(LotSchema) });30export type Payload = z.infer<typeof PayloadSchema>;3132/** Archive page (…&action=show&id=211) → auction ids with printed names/dates. */33export function parseArchive(htmlText: string): Array<{ id: string; name: string; dateText: string | null }> {34 const $ = H.load(htmlText);35 const out: Array<{ id: string; name: string; dateText: string | null }> = [];36 $('.auctionBox').each((_, box) => {37 const e = $(box);38 const id = e.find('a[href*="showAuctionOverview"]').first().attr('href')?.match(/auctionID=(\d+)/)?.[1];39 if (!id || out.some((a) => a.id === id)) return;40 const name = clean(e.find('h4').first().text());41 const dateText = clean(e.find('.date').first().text()) || null;42 if (name) out.push({ id, name, dateText });43 });44 return out;45}4647/** Auction overview → auctionData JSON + catalogue parts ("Show all lots" links carry the lot count). */48export function parseOverview(htmlText: string, id: string): { auction: z.infer<typeof AuctionSchema>; parts: z.infer<typeof PartSchema>[] } | null {49 const m = htmlText.match(/c4msEnv\.auctionData\s*=\s*(\{[\s\S]*?\});/);50 let data: { currency?: string; startDate?: string; endDate?: string; status?: string; name?: string } = {};51 if (m) {52 try {53 data = JSON.parse(m[1]!) as typeof data;54 } catch {55 data = {};56 }57 }58 const $ = H.load(htmlText);59 const name = clean(data.name ?? '') || clean($('h1').first().text());60 if (!name) return null;61 const parts: z.infer<typeof PartSchema>[] = [];62 $('.bookmark_left_red').each((_, hdr) => {63 const title = clean($(hdr).find('h3').first().text()) || null;64 const box = $(hdr).nextAll('.countryBox').first();65 const link = box.find(`a[href*="action=showLots"][href*="auctionID=${id}"][href*="show_all_lots=1"]`).first();66 const cp = link.attr('href')?.match(/catalogPart=(\d+)/)?.[1];67 if (!cp || parts.some((p) => p.catalogPart === cp)) return;68 const count = Number(clean(box.find('a.lotCounter').first().text()).replace(/\D/g, ''));69 parts.push({ catalogPart: cp, title, lotCount: Number.isFinite(count) && count > 0 ? count : null });70 });71 return { auction: { id, name, currency: data.currency ?? null, startDate: data.startDate ?? null, endDate: data.endDate ?? null, status: data.status ?? null }, parts };72}7374export function lotsUrl(auctionId: string, catalogPart: string, page: number): string {75 return `${SITE}/en/_auctions/&action=showLots&auctionID=${auctionId}&catalogPart=${catalogPart}&show_all_lots=1&page=${page}`;76}7778/** Lot list page → lots + page count. */79export function parseLotsPage(htmlText: string, auction: z.infer<typeof AuctionSchema>, part: z.infer<typeof PartSchema>, url: string): Payload {80 const $ = H.load(htmlText);81 const page = Number(url.match(/[?&]page=(\d+)/)?.[1] ?? 1);82 const totalPagesText = clean($('.pageCounterLabel').first().text());83 const lots: z.infer<typeof LotSchema>[] = [];84 $('div.lot[data-lotno], div.lot[data-lotNo]').each((_, el) => {85 const e = $(el);86 const lotNo = e.attr('data-lotno') ?? e.attr('data-lotNo') ?? clean(e.find('.lotno').first().text());87 const description = clean(e.find('.lotDesc .text').first().text());88 if (!lotNo || !description) return;89 const start = clean(e.find('.prices .start .value').first().text()) || null;90 const hammer = clean(e.find('.prices .bid .value').first().text()) || null;91 const images = e.find('.picContainer img[data-src]').map((__, img) => $(img).attr('data-src') ?? '').get().filter(Boolean);92 const conditionCodes = e.find('.lot-cond').map((__, c) => ($(c).attr('class') ?? '').match(/lot-cond-(\d+)/)?.[1] ?? '').get().filter(Boolean);93 lots.push({ lotNo, country: clean(e.find('.lotCountry').first().text()) || null, description, startText: start, hammerText: hammer, conditionCodes, images: images.slice(0, 3) });94 });95 return { kind: 'lots_page', auction, part, url, page, totalPages: totalPagesText ? Number(totalPagesText) : null, lots };96}9798interface Cursor {99 doneAuctions?: string[];100 inProgress?: { auction: z.infer<typeof AuctionSchema>; parts: z.infer<typeof PartSchema>[]; partIndex: number; page: number } | null;101 done?: boolean;102 updatedAt?: string;103}104105export class CorinphilaConnector extends BaseConnector {106 readonly version = '1.0.0';107 readonly parserVersion = PARSER_VERSION;108 protected override minIntervalMs = 10_000;109110 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {111 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 20);112 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 1);113 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };114 const done = new Set(cursor.doneAuctions ?? []);115 let pages = 0;116 let yielded = 0;117 let finished = 0;118 const text = (url: string, expect: Parameters<CrawlContext['fetch']>[1] = {}) => ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0, ...expect });119120 await this.throttle();121 const archive = await text(`${SITE}/en/_pages/&action=show&id=211`);122 pages++;123 const auctions = archive.success && archive.html ? parseArchive(archive.html) : [];124 if (!auctions.length) {125 ctx.anomaly(archive.success ? 'selector_missing' : 'page_fetch_failed', `archive: ${archive.error ?? archive.httpStatus ?? 'no auction boxes'}`);126 return;127 }128 auctions.sort((a, b) => Number(b.id) - Number(a.id));129 const queue = [...(cursor.inProgress ? [cursor.inProgress.auction.id] : []), ...auctions.map((a) => a.id).filter((id) => !done.has(id) && id !== cursor.inProgress?.auction.id)];130 for (const id of queue) {131 if (ctx.signal?.aborted || finished >= auctionsPerRun || pages >= pagesPerRun || this.reached(ctx, yielded)) break;132 let state = cursor.inProgress?.auction.id === id ? cursor.inProgress : null;133 if (!state) {134 await this.throttle();135 const ov = await text(`${SITE}/en/_auctions/&action=showAuctionOverview&auctionID=${id}`);136 pages++;137 const parsed = ov.success && ov.html ? parseOverview(ov.html, id) : null;138 if (!parsed || !parsed.parts.length) {139 ctx.anomaly(ov.success ? 'parse_failure_page' : 'page_fetch_failed', `overview ${id}: ${ov.error ?? ov.httpStatus ?? 'no catalogue parts'}`);140 if (ov.success) done.add(id);141 continue;142 }143 if (parsed.auction.status && parsed.auction.status !== 'closed') continue; // running sale144 state = { auction: parsed.auction, parts: parsed.parts, partIndex: 0, page: 1 };145 }146 while (state.partIndex < state.parts.length && pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, yielded)) {147 const part = state.parts[state.partIndex]!;148 const url = lotsUrl(id, part.catalogPart, state.page);149 await this.throttle();150 const res = await text(url, {151 expect: ['title', 'price'],152 parse: (r) => {153 const p = r.html ? parseLotsPage(r.html, state!.auction, part, url) : null;154 return p?.lots.length ? { title: p.lots[0]!.description, price: p.lots.find((l) => l.hammerText && !/not sold/i.test(l.hammerText))?.hammerText ?? null } : null;155 },156 });157 pages++;158 if (!res.success || !res.html) {159 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);160 break;161 }162 const payload = parseLotsPage(res.html, state.auction, part, url);163 if (!payload.lots.length && state.page === 1) ctx.anomaly('parse_failure_page', `${url}: no lot blocks`);164 if (payload.lots.some((l) => l.hammerText && !/not sold/i.test(l.hammerText))) {165 yielded++;166 yield { url, externalId: `auction:${id}:part:${part.catalogPart}:p${payload.page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };167 }168 const more = payload.lots.length > 0 && payload.totalPages !== null && payload.page < payload.totalPages;169 if (more) state.page++;170 else {171 state.partIndex++;172 state.page = 1;173 }174 cursor.inProgress = state;175 await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-100), updatedAt: new Date().toISOString() });176 }177 if (state.partIndex >= state.parts.length) {178 done.add(id);179 finished++;180 cursor.inProgress = null;181 if (ctx.options.mode === 'backfill') await ctx.progress({ page: auctions.findIndex((a) => a.id === id) + 1, totalPages: auctions.length, itemsProcessed: yielded, reachedDate: parseAuctionDate(state.auction.startDate) });182 }183 await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-100), updatedAt: new Date().toISOString() });184 }185 if (ctx.options.mode === 'backfill' && auctions.every((a) => done.has(a.id))) await ctx.setCursor({ ...cursor, done: true, doneAuctions: [...done].slice(-100), updatedAt: new Date().toISOString() });186 }187188 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {189 const p = PayloadSchema.parse(raw.payload);190 const saleDate = parseAuctionDate(p.auction.startDate);191 if (!saleDate) return [];192 const out: NormalizedSale[] = [];193 for (const lot of p.lots) {194 if (!lot.hammerText || /not sold|withdrawn/i.test(lot.hammerText)) continue;195 const price = realized(lot.hammerText, (p.auction.currency as 'CHF' | null) ?? 'CHF');196 if (!price) continue;197 const start = realized(lot.startText, price.currency);198 const title = lot.country ? `${lot.country}: ${lot.description}` : lot.description;199 const attributes = numisAttributes({200 categorySlug: 'stamps',201 title,202 section: p.part.title,203 identifiers: { corinphila_lot: `${p.auction.id}/${lot.lotNo}` },204 metadata: { auction_id: p.auction.id, auction_name: p.auction.name, catalogue_part: p.part.title, country_label: lot.country, starting_bid: start?.amount ?? null, hammer_price: price.amount, buyer_premium: "excluded — page label 'Hammer price'; Corinphila's premium is stated in the conditions of sale", condition_codes: lot.conditionCodes, michel_or_sg: lot.description.match(/\b(?:SG|Mi\.?|Michel|Zumstein|Yv\.?|Scott)\s*\d+[a-z]?/i)?.[0] ?? null },205 });206 const sale = makeSale({207 meta: this.meta,208 sourceUrl: `${SITE}/en/_auctions/&action=showLot&auctionID=${p.auction.id}&lotno=${lot.lotNo}`,209 externalId: `${p.auction.id}-${lot.lotNo}`,210 rawTitle: title.length > 240 ? `${title.slice(0, 239)}…` : title,211 description: lot.description,212 attributes,213 price: price.amount,214 currency: price.currency,215 saleDate,216 buyerPremiumIncluded: false,217 auctionHouse: 'Corinphila Auctions',218 lotNumber: lot.lotNo,219 imageUrls: lot.images,220 observedAt: raw.fetchedAt,221 parserVersion: PARSER_VERSION,222 confidence: 0.85,223 isBundle: isNumisBundle(lot.description) || /\b(collection|accumulation|lot of|group of|balance)\b/i.test(lot.description),224 location: 'CH',225 });226 out.push(sale);227 }228 return out;229 }230}231232export default (meta: ConnectorMeta) => new CorinphilaConnector(meta);233