import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedAuctionLotSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; import { normalizeCondition, parseGradeFromTitle } from '@rareindex/taxonomy'; import { brandFromSlug, hintFromLabel, isBundleTitle, legoSetNumber, safeYear, slugFromTitle, watchBrand, watchReference, type DeptHint } from '../../api/_auction-lib/categories.js'; /** * Catawiki — closed-lot results (hammer = final bid, EUR) and live lots, two-phase crawl. * Engine: Scrapfly. See meta.json accessNotes for the access rationale and cost model. */ const SITE = 'https://www.catawiki.com'; const SeedSchema = z.object({ id: z.number().int(), path: z.string(), hint: z.string() }); type Seed = z.infer; export const AuctionInfoSchema = z.object({ id: z.number().int(), title: z.string(), url: z.string(), status: z.string().nullable(), startAt: z.string().nullable(), closeAt: z.string().nullable(), closedAt: z.string().nullable(), categories: z.array(z.string()), lotCount: z.number().nullable(), }); export type AuctionInfo = z.infer; export const LiveLotSchema = z.object({ id: z.number().int(), title: z.string(), subtitle: z.string().nullable(), url: z.string(), imageUrl: z.string().nullable(), reservePriceSet: z.boolean().nullable(), biddingStartTime: z.string().nullable(), }); export const AuctionLotsPayloadSchema = z.object({ kind: z.literal('auction_lots'), seedHint: z.string(), auction: AuctionInfoSchema, lots: z.array(LiveLotSchema), }); export type AuctionLotsPayload = z.infer; export const LotPayloadSchema = z.object({ kind: z.literal('lot'), seedHint: z.string(), id: z.number().int(), url: z.string(), title: z.string(), subtitle: z.string().nullable(), description: z.string().nullable(), images: z.array(z.string()), categoryId: z.number().nullable(), categoryUrl: z.string().nullable(), auction: AuctionInfoSchema.nullable(), specs: z.array(z.object({ name: z.string(), value: z.string() })), estimateMinEur: z.number().nullable(), estimateMaxEur: z.number().nullable(), sellerCountry: z.string().nullable(), sellerName: z.string().nullable(), sellerIsPro: z.boolean().nullable(), bidding: z.object({ closed: z.boolean().nullable(), sold: z.boolean().nullable(), finalBidEur: z.number().nullable(), biddingStartTime: z.number().nullable(), biddingEndTime: z.number().nullable(), bidCount: z.number().nullable(), reservePriceMet: z.boolean().nullable(), }), }); export type LotPayload = z.infer; const ConfigSchema = z.object({ seeds: z.array(SeedSchema).default([]), maxNewAuctionsPerRun: z.number().int().default(30), maxLotFetchesPerRun: z.number().int().default(400), pendingCap: z.number().int().default(6000), harvestDelayMinutes: z.number().int().default(20), }); interface Pending { id: number; url: string; auctionId: number; closeAt: string; hint: string; } interface Cursor { auctions?: Record; pending?: Pending[]; } export function nextData(html: string): Record | null { const m = html.match(/