TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';45const BASE = 'https://www.fossilera.com';6const PARSER_VERSION = '1.0.0';78export const SpecimenSchema = z.object({9 item: z.string(),10 url: z.string(),11 title: z.string(),12 price: z.number().nullable(),13 oldPrice: z.number().nullable(),14 sold: z.boolean(),15 image: z.string().nullable(),16 species: z.string().nullable().default(null),17 age: z.string().nullable().default(null),18 location: z.string().nullable().default(null),19 formation: z.string().nullable().default(null),20 size: z.string().nullable().default(null),21 category: z.string().nullable().default(null),22 subCategory: z.string().nullable().default(null),23});24export type Specimen = z.infer<typeof SpecimenSchema>;25export const PagePayloadSchema = z.object({ kind: z.enum(['category_page', 'specimen_page']), url: z.string(), categorySlug: z.string(), items: z.array(SpecimenSchema) });26export type PagePayload = z.infer<typeof PagePayloadSchema>;2728const abs = (u: string | null | undefined) => (u ? (u.startsWith('//') ? `https:${u}` : u.startsWith('http') ? u : BASE + u) : null);2930export function parseCategoryPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {31 const $ = H.load(htmlText);32 const items: Specimen[] = [];33 $('a[href^="/fossils/"], a[href^="/minerals/"], a[href^="/meteorites/"]').each((_, el) => {34 const a = $(el);35 if (!a.find('.info').length) return;36 const url = BASE + a.attr('href')!;37 const alt = a.find('img').attr('alt') ?? '';38 const item = alt.match(/#(\d+)\s*$/)?.[1] ?? url.split('/').pop()!;39 const info = a.find('.info').clone();40 const priceEl = info.find('.price');41 const oldPrice = parsePrice(H.text(priceEl.find('.old-price')), 'USD')?.amount ?? null;42 priceEl.find('.old-price').remove();43 const priceText = H.text(priceEl) ?? '';44 const sold = /sold/i.test(priceText);45 const price = sold ? null : (parsePrice(priceText, 'USD')?.amount ?? null);46 priceEl.remove();47 const title = H.text(info) ?? alt.replace(/\s*#\d+\s*$/, '');48 if (!title) return;49 items.push({ item, url, title, price, oldPrice, sold, image: abs(a.find('img').attr('src')), species: null, age: null, location: null, formation: null, size: null, category: null, subCategory: null });50 });51 return { kind: 'category_page', url: pageUrl, categorySlug, items };52}5354export function parseSpecimenPage(htmlText: string, url: string, categorySlug: string): PagePayload | null {55 const $ = H.load(htmlText);56 const title = H.text($('h1').first());57 if (!title) return null;58 const detail = (label: string) => {59 let v: string | null = null;60 $('[class*="detail"]').each((_, el) => {61 const t = $(el).text().replace(/\s+/g, ' ').trim();62 const m = t.match(new RegExp(`^${label}\\s+(.+)$`, 'i'));63 if (m && !v) v = m[1]!.trim();64 });65 return v;66 };67 const body = $('body').text().replace(/\s+/g, ' ');68 const item = body.match(/ITEM\s*#\s*(\d+)/i)?.[1] ?? url.split('/').pop()!;69 const priceBox = $('.price').first();70 const oldPrice = parsePrice(H.text(priceBox.find('.old-price')), 'USD')?.amount ?? null;71 const priceText = priceBox.clone().find('.old-price').remove().end().text();72 const sold = /this (specimen|item) (has been|was) sold|sold out/i.test(body);73 const price = sold ? null : (parsePrice(priceText, 'USD')?.amount ?? null);74 return {75 kind: 'specimen_page',76 url,77 categorySlug,78 items: [{ item, url, title, price, oldPrice, sold, image: abs($('meta[property="og:image"]').attr('content')), species: detail('SPECIES'), age: detail('AGE'), location: detail('LOCATION'), formation: detail('FORMATION'), size: detail('SIZE'), category: detail('CATEGORY'), subCategory: detail('SUB CATEGORY') }],79 };80}8182export class FossilEraConnector extends BaseConnector {83 readonly version = '1.0.0';84 readonly parserVersion = PARSER_VERSION;85 protected override minIntervalMs = 1500;86 override readonly urlPatterns = [/^https?:\/\/(www\.)?fossilera\.com\/(fossils|minerals|meteorites)\/[a-z0-9-]+/i];8788 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {89 const seeds = (this.meta.config.seeds as Array<{ path: string; categorySlug: string }> | undefined) ?? [];90 const pages = Number(this.meta.config.pagesPerSeed ?? 1);91 const cap = ctx.options.limit;92 let count = 0;93 const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;94 for (let i = start; i < seeds.length; i++) {95 const seed = seeds[i]!;96 for (let page = 1; page <= pages; page++) {97 if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;98 const url = `${BASE}${seed.path}${page > 1 ? `?page=${page}` : ''}`;99 await this.throttle();100 const res = await ctx.fetch(url, {101 engines: ['api', 'firecrawl'],102 responseType: 'text',103 expect: ['title', 'price'],104 parse: (r) => {105 const p = r.html ? parseCategoryPage(r.html, url, seed.categorySlug) : null;106 return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;107 },108 });109 const payload = res.success && res.html ? parseCategoryPage(res.html, url, seed.categorySlug) : null;110 if (!payload?.items.length) {111 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no specimen cards'}`);112 break;113 }114 count++;115 yield { url, externalId: `${seed.path}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };116 }117 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });118 }119 }120121 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {122 const m = url.match(this.urlPatterns[0]!);123 if (!m) return [];124 const slug = m[2]!.toLowerCase() === 'minerals' ? 'minerals' : m[2]!.toLowerCase() === 'meteorites' ? 'meteorites' : 'fossils';125 const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0 });126 const payload = res.success && res.html ? parseSpecimenPage(res.html, url, slug) : null;127 if (!payload) return [];128 return [{ url, externalId: payload.items[0]!.item, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];129 }130131 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {132 const p = PagePayloadSchema.parse(raw.payload);133 const out: NormalizedRecord[] = [];134 for (const s of p.items) {135 const loc = s.location ?? s.title.match(/ - ([A-Z][A-Za-z .]+)$/)?.[1] ?? null;136 const attributes = AssetAttributesSchema.parse({137 categorySlug: p.categorySlug,138 name: s.title,139 size: s.size ?? s.title.match(/(\d+(?:\.\d+)?")/)?.[1] ?? null,140 country: loc,141 identifiers: { fossilera_item: s.item },142 metadata: { species: s.species, geological_age: s.age, formation: s.formation, locality: s.location, category: s.category, sub_category: s.subCategory, previous_price_usd: s.oldPrice, dealer_guarantee: 'FossilEra authenticity guarantee (dealer statement)', unique_specimen: true },143 });144 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: s.url, rawTitle: s.title, imageUrls: s.image ? [s.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };145 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: s.item, confidence: 0.85, listingType: 'fixed_price', price: s.price, currency: 'USD', seller: 'FossilEra', location: 'US', availability: s.sold ? 'sold' : s.price ? 'available' : 'unknown' }));146 }147 return out;148 }149}150151export default function createConnector(meta: ConnectorMeta) {152 return new FossilEraConnector(meta);153}154