TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedAuctionLotSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';4import { fetchFirecrawl } from '../../api/_wlib/index.js';56/**7 * e-Rocks — weekly online mineral auctions run by dealers. Auction pages (rendered by Firecrawl)8 * list every lot with locality, size class, current bid in EUR (+ USD conversion) and bid count,9 * plus the auction end time. Ended auction pages hide the final price (only a 'Sold' marker), so10 * lots are recorded as auction lots (current bid while live), never as sales.11 */12const BASE = 'https://e-rocks.com';13const PARSER_VERSION = '1.0.0';1415export const LotSchema = z.object({16 id: z.string(),17 url: z.string(),18 name: z.string(),19 locality: z.string().nullable(),20 sizeClass: z.string().nullable(),21 priceText: z.string().nullable(),22 bids: z.number().nullable(),23 seller: z.string().nullable(),24 image: z.string().nullable(),25 sold: z.boolean().default(false),26});27export type Lot = z.infer<typeof LotSchema>;28export const PagePayloadSchema = z.object({ kind: z.literal('auction_page'), url: z.string(), auctionId: z.string(), auctionName: z.string(), startsAt: z.string().nullable(), endsAt: z.string().nullable(), lots: z.array(LotSchema) });29export type PagePayload = z.infer<typeof PagePayloadSchema>;3031/** "07/09/2026 22:10 BST" → ISO (BST = UTC+1, GMT = UTC). */32export function parseErocksDate(s: string | null | undefined): string | null {33 const m = s?.match(/(\d{2})\/(\d{2})\/(\d{4})\s+(\d{2}):(\d{2})\s*(BST|GMT|UTC)?/);34 if (!m) return null;35 const offset = m[6] === 'BST' ? 1 : 0;36 const d = new Date(Date.UTC(Number(m[3]), Number(m[2]) - 1, Number(m[1]), Number(m[4]) - offset, Number(m[5])));37 return Number.isNaN(d.getTime()) ? null : d.toISOString();38}3940export function parseAuctionMarkdown(md: string, url: string): PagePayload {41 const auctionId = url.match(/\/items\/auction\/(\d+)/)?.[1] ?? url;42 const auctionName = md.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? auctionId;43 const startsAt = parseErocksDate(md.match(/Start:\s*([^|\n]+?)\s{2,}End:/)?.[1] ?? md.match(/Start:\s*([\d/]+\s+[\d:]+\s*\w*)/)?.[1]);44 const endsAt = parseErocksDate(md.match(/End:\s*([\d/]+\s+[\d:]+\s*\w*)/)?.[1]);45 const lots: Lot[] = [];46 const seen = new Set<string>();47 // Each lot starts with an image link to /item/<id>/<slug>; following cells hold locality, size, price, bids, seller.48 const re = /!\[([^\]]*)\]\(([^)\s]+)\)\]\((https?:\/\/e-rocks\.com\/item\/([a-z0-9]+)\/[^)\s]*)\)([\s\S]*?)(?=!\[[^\]]*\]\([^)\s]+\)\]\(https?:\/\/e-rocks\.com\/item\/|$)/g;49 for (const m of md.matchAll(re)) {50 const id = m[4]!.toUpperCase();51 if (seen.has(id)) continue;52 seen.add(id);53 const block = m[5]!;54 // live pages render lots as table cells ("|"), ended pages as line breaks — accept both.55 const cells = block56 .split(/\n|\|/)57 .map((c) => c.replace(/\\/g, '').trim())58 .filter((c) => c && !/^\[?(Bid|Watch)\]?/.test(c) && !/^(You are|You have|Proxy bid|Delayed)/i.test(c) && !/^€$/.test(c) && !/^\(reserve/.test(c) && !/^!\[/.test(c) && !/^- /.test(c));59 const name = (m[1] ?? '').trim() || cells.find((c) => /^\[[^\]]+\]\(/.test(c))?.replace(/^\[([^\]]+)\].*/, '$1') || id;60 const plain = cells.filter((c) => !/^\[/.test(c));61 const priceCell = plain.find((c) => /^[€$£]\s?\d/.test(c)) ?? null;62 const bidsCell = plain.find((c) => /\d+\s*bids?/.test(c));63 const sizeCell = plain.find((c) => /\(\d+(?:\.\d+)?\s*-\s*\d+(?:\.\d+)?\s*(?:cm|mm)\)|^(Thumbnail|Miniature|Small miniature|Small cabinet|Cabinet|Large cabinet|Museum)/i.test(c)) ?? null;64 const locality = plain.find((c) => c !== name && /,/.test(c) && !/€|bids?|\(\d/.test(c)) ?? null;65 const sold = plain.some((c) => /^Sold$/i.test(c));66 const seller = plain.filter((c) => /^[A-Z0-9 .&'-]{3,}$/.test(c) && c !== id && !/^(SOLD|BIDS?)$/.test(c)).pop() ?? null;67 lots.push({ id, url: m[3]!.split('?')[0]!, name, locality, sizeClass: sizeCell, priceText: priceCell, bids: bidsCell ? Number(bidsCell.match(/(\d+)\s*bids?/)![1]) : null, seller, image: m[2] ?? null, sold });68 }69 return { kind: 'auction_page', url, auctionId, auctionName, startsAt, endsAt, lots };70}7172export class ERocksConnector extends BaseConnector {73 readonly version = '1.0.0';74 readonly parserVersion = PARSER_VERSION;75 protected override minIntervalMs = 10_000; // robots.txt Crawl-delay: 107677 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {78 const maxAuctions = Number(this.meta.config.auctionsPerRun ?? 4);79 let count = 0;80 await this.throttle();81 const index = await fetchFirecrawl(ctx, `${BASE}/auctions`, { timeoutMs: 90_000 });82 const links = [...new Set((index.markdown ?? '').match(/https:\/\/e-rocks\.com\/items\/auction\/\d+\/[a-z0-9-]+/g) ?? [])].sort((a, b) => Number(b.match(/auction\/(\d+)/)![1]) - Number(a.match(/auction\/(\d+)/)![1]));83 if (!links.length) {84 ctx.anomaly('page_fetch_failed', `auction index: ${index.error ?? index.httpStatus ?? 'no auction links'}`);85 return;86 }87 for (const url of links.slice(0, maxAuctions)) {88 if (ctx.signal?.aborted || this.reached(ctx, count)) return;89 await this.throttle();90 const res = await fetchFirecrawl(ctx, url, { timeoutMs: 90_000, parse: (r) => (r.markdown ? parseAuctionMarkdown(r.markdown, url).lots.length : 0) });91 const payload = res.success && res.markdown ? parseAuctionMarkdown(res.markdown, url) : null;92 if (!payload?.lots.length) {93 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no lots'}`);94 continue;95 }96 count++;97 yield { url, externalId: `auction:${payload.auctionId}`, kind: 'auction_lot', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };98 }99 }100101 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {102 const p = PagePayloadSchema.parse(raw.payload);103 const out: NormalizedRecord[] = [];104 const ends = p.endsAt ? new Date(p.endsAt) : null;105 const status = ends && ends.getTime() < raw.fetchedAt.getTime() ? 'ended' : 'live';106 for (const l of p.lots) {107 const price = parsePrice(l.priceText ?? null, 'EUR');108 const country = l.locality?.split(',').pop()?.trim() ?? null;109 const attributes = AssetAttributesSchema.parse({110 categorySlug: 'minerals',111 name: l.name,112 country,113 size: l.sizeClass,114 identifiers: { erocks_item: l.id },115 metadata: { locality: l.locality, seller: l.seller, auction: p.auctionName, unique_specimen: true, sold_flag: status === 'ended' ? l.sold : false },116 });117 out.push(118 NormalizedAuctionLotSchema.parse({119 kind: 'auction_lot',120 connectorId: this.meta.id,121 sourceId: this.meta.sourceId,122 sourceUrl: l.url,123 externalId: l.id,124 rawTitle: l.locality ? `${l.name} — ${l.locality}` : l.name,125 imageUrls: l.image ? [l.image] : [],126 attributes,127 observedAt: raw.fetchedAt,128 confidence: 0.7,129 parserVersion: PARSER_VERSION,130 auctionHouse: 'e-Rocks',131 auctionName: p.auctionName,132 lotNumber: l.id,133 startsAt: p.startsAt ? new Date(p.startsAt) : null,134 endsAt: ends,135 currentBid: price?.amount ?? null,136 currency: price?.currency ?? 'EUR',137 status,138 location: country,139 }),140 );141 }142 return out;143 }144}145146export default function createConnector(meta: ConnectorMeta) {147 return new ERocksConnector(meta);148}149