TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, NormalizedSaleSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';4import { attrs, priceObservation } from '../../api/_lib/shared.js';5import { BOT_HEADERS, cardCategorySlug, dayOf, fetchMaybeGzip, money, parseMonthDay, parseSitemap } from '../../api/_lib/wave4.js';67/** Alt item pages → Alt Value observations, recent transactions (sales) and Alt list-price listings. */8const PARSER_VERSION = '1.0.0';9const SITE = 'https://alt.xyz';1011const TxSchema = z.object({ type: z.string(), date: z.string(), price: z.number(), url: z.string().nullable() });12const RawPayloadSchema = z.object({13 id: z.string(),14 title: z.string(),15 category: z.string().nullable(),16 serial: z.string().nullable(),17 grader: z.string().nullable(),18 grade: z.string().nullable(),19 pop: z.number().nullable(),20 listPrice: z.number().nullable(),21 listingKind: z.enum(['fixed_price', 'auction']).nullable().default(null),22 altValue: z.number().nullable(),23 altLow: z.number().nullable(),24 altHigh: z.number().nullable(),25 transactions: z.array(TxSchema),26 images: z.array(z.string()),27});28export type AltPayload = z.infer<typeof RawPayloadSchema>;2930/** Parse the Firecrawl markdown of an item page. Exported for tests. */31export function parseItemMarkdown(md: string, id: string): AltPayload | null {32 const text = md.replace(/\\\n/g, '\n').replace(/\\([#$])/g, '$1');33 const title = text.match(/^#\s+(.+)$/m)?.[1]?.trim();34 if (!title) return null;35 const afterTitle = text.slice(text.indexOf(title) + title.length);36 const category = afterTitle.match(/^##\s+([A-Za-z' \-]+Cards?)\s*$/m)?.[1]?.trim() ?? afterTitle.match(/^##\s+(Pok[eé]mon|Magic[^\n]*|Yu-Gi-Oh!?)\s*$/m)?.[1]?.trim() ?? null;37 const serial = afterTitle.match(/Serial\s*(\d+\s*\/\s*\d+)/)?.[1]?.replace(/\s/g, '') ?? null;38 const gm = afterTitle.match(/^(PSA|BGS|SGC|CGC|CSG|TAG|HGA)\s?(\d{1,2}(?:\.\d)?)\s*$/m);39 const pop = afterTitle.match(/^Pop\s*([\d,]+)\s*$/m)?.[1];40 const fixed = afterTitle.match(/##\s+List price\s*\n+\s*###\s+\$([\d,]+(?:\.\d+)?)/)?.[1] ?? null;41 const bid = afterTitle.match(/(?:Starting bid|Current bid|High bid)\s*\$([\d,]+(?:\.\d+)?)/)?.[1] ?? null;42 const listPrice = fixed ?? bid;43 const listingKind: 'fixed_price' | 'auction' | null = fixed ? 'fixed_price' : bid ? 'auction' : null;44 const av = afterTitle.match(/##\s+(?:LT|Alt) Value\s*\n+\s*\$([\d,]+(?:\.\d+)?)\s*\n+\s*###\s+\$([\d,]+(?:\.\d+)?)\s*-\s*\$([\d,]+(?:\.\d+)?)/);45 const txStart = afterTitle.search(/Recent transactions/i);46 const txEnd = afterTitle.search(/\n##?\s+(Listings|Similar listings)|\nListings\s*\n/);47 const txBlock = txStart >= 0 ? afterTitle.slice(txStart, txEnd > txStart ? txEnd : undefined) : '';48 const transactions: z.infer<typeof TxSchema>[] = [];49 const re = /(Auction|Best offer|Fixed price|Buy now|Sale)\s*([A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})\s*\n+\s*\$([\d,]+(?:\.\d+)?)\]\((https?:[^)\s]+)\)/g;50 let m: RegExpExecArray | null;51 while ((m = re.exec(txBlock))) {52 const price = money(m[3]);53 if (!price) continue;54 transactions.push({ type: m[1]!, date: m[2]!, price, url: m[4] ?? null });55 }56 const images = [...text.matchAll(/!\[[^\]]*\]\((https:\/\/alt-images\.b-cdn\.net\/public\/[^)\s]+width=324[^)\s]*)\)/g)].map((x) => x[1]!);57 return {58 id,59 title,60 category,61 serial,62 grader: gm ? gm[1]!.toLowerCase() : null,63 grade: gm ? gm[2]! : null,64 pop: pop ? Number(pop.replace(/,/g, '')) : null,65 listPrice: money(listPrice),66 listingKind,67 altValue: av ? money(av[1]) : null,68 altLow: av ? money(av[2]) : null,69 altHigh: av ? money(av[3]) : null,70 transactions,71 images: [...new Set(images)],72 };73}7475async function fetchGzText(url: string, signal?: AbortSignal): Promise<string> {76 return (await fetchMaybeGzip(url, signal)).toString('utf8');77}7879export class AltConnector extends BaseConnector {80 readonly version = '1.0.0';81 readonly parserVersion = PARSER_VERSION;82 override readonly urlPatterns = [/^https?:\/\/(www\.)?alt\.xyz\/itm\/[0-9a-f-]{36}/i];83 protected override minIntervalMs = 1500;8485 private async fetchItem(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {86 await this.throttle();87 const res = await ctx.fetch(url, { engines: ['firecrawl'], waitForMs: 9000, timeoutMs: 90_000, expect: ['title', 'price'], parse: (r) => ({ title: r.markdown?.match(/^#\s+.+$/m) ? 'ok' : null, price: r.markdown && /\$\d/.test(r.markdown) ? 1 : null }) });88 if (!res.success || !res.markdown) {89 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);90 return null;91 }92 const id = url.match(/itm\/([0-9a-f-]{36})/i)?.[1] ?? url;93 const payload = parseItemMarkdown(res.markdown, id);94 if (!payload) {95 ctx.anomaly('parse_failure_item', url);96 return null;97 }98 return { url: `${SITE}/itm/${id}`, externalId: id, kind: 'sale', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };99 }100101 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {102 const max = ctx.options.mode === 'backfill' ? Number(this.meta.config.maxItemsPerRun ?? 100) * 5 : Number(this.meta.config.maxItemsPerRun ?? 100);103 let urls: Array<{ loc: string; lastmod: string }> = [];104 if (ctx.options.seeds?.length) urls = ctx.options.seeds.map((s) => ({ loc: s.startsWith('http') ? s : `${SITE}/itm/${s}`, lastmod: '' }));105 else {106 const idx = await ctx.fetch(`${SITE}/sitemap.xml`, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'application/xml' }, responseType: 'text' });107 if (!idx.success || !idx.html) throw new Error(`alt sitemap index failed: ${idx.error ?? idx.httpStatus}`);108 const wanted = (this.meta.config.sitemaps as string[] | undefined) ?? ['fixed-price', 'auctions'];109 const children = [...idx.html.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((x) => x[1]!).filter((u) => wanted.some((w) => u.includes(`/${w}-`)));110 for (const child of children.slice(0, 6)) {111 try {112 const xml = await fetchGzText(child, ctx.signal);113 for (const e of parseSitemap(xml)) urls.push({ loc: e.loc, lastmod: e.lastmod ?? '' });114 } catch (err) {115 ctx.anomaly('page_fetch_failed', `${child}: ${err instanceof Error ? err.message : String(err)}`);116 }117 }118 const since = String(ctx.options.cursor?.since ?? '');119 urls = urls.filter((u) => !since || u.lastmod > since).sort((a, b) => b.lastmod.localeCompare(a.lastmod));120 }121 let count = 0;122 let newest = String(ctx.options.cursor?.since ?? '');123 for (const u of urls.slice(0, max)) {124 if (ctx.signal?.aborted) return;125 if (this.reached(ctx, count)) break;126 const rec = await this.fetchItem(ctx, u.loc);127 if (!rec) continue;128 count++;129 if (u.lastmod > newest) newest = u.lastmod;130 yield rec;131 if (count % 20 === 0) await ctx.setCursor({ since: newest, at: new Date().toISOString() });132 }133 await ctx.setCursor({ since: newest, completedAt: new Date().toISOString() });134 }135136 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {137 const rec = await this.fetchItem(ctx, url);138 return rec ? [rec] : [];139 }140141 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {142 const p = RawPayloadSchema.parse(raw.payload);143 const categorySlug = cardCategorySlug(p.category) ?? cardCategorySlug(p.title);144 if (!categorySlug) return [];145 const year = extractYear(p.title);146 const number = p.title.match(/#\s?([A-Za-z0-9-]+)/)?.[1] ?? null;147 const name = p.title.replace(/^\d{4}(?:-\d{2})?\s+/, '').replace(/\s*#\S+\s*$/, '').trim() || p.title;148 const a = attrs({ categorySlug, name, year, number, identifiers: { alt_item_id: p.id }, metadata: { serial: p.serial, pop: p.pop } });149 const grade = { grader: p.grader, grade: p.grade, qualifier: null, certificationNumber: null };150 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, rawTitle: p.title, imageUrls: p.images, attributes: a, grade, condition: {}, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };151 const out: NormalizedRecord[] = [];152 if (p.altValue) {153 out.push(priceObservation({ kind: 'price_observation', ...base, externalId: `${p.id}:altvalue:${dayOf(raw.fetchedAt).toISOString().slice(0, 10)}`, confidence: 0.6, priceKind: 'guide_value', price: p.altValue, currency: 'USD', observationDate: dayOf(raw.fetchedAt), sampleSize: p.transactions.length || null, attributes: { ...a, metadata: { ...a.metadata, low: p.altLow, high: p.altHigh, model: 'Alt Value' } } }));154 }155 for (const t of p.transactions) {156 const saleDate = parseMonthDay(t.date);157 if (!saleDate) continue;158 const saleType = /auction/i.test(t.type) ? 'auction' : /best offer/i.test(t.type) ? 'best_offer' : 'fixed_price';159 out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, externalId: `${p.id}:${saleDate.toISOString().slice(0, 10)}:${t.price}`, confidence: 0.65, saleType, saleDate, price: t.price, currency: 'USD', buyerPremiumIncluded: null, quantity: 1, isBundle: false, location: null, auctionHouse: t.url?.includes('ebay.') ? 'eBay (via Alt)' : null, lotNumber: null, attributes: { ...a, metadata: { ...a.metadata, external_url: t.url, via: 'alt.xyz' } } }));160 }161 if (p.listPrice) {162 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: p.id, confidence: 0.75, listingType: p.listingKind ?? 'fixed_price', price: p.listPrice, currency: 'USD', seller: 'Alt marketplace', availability: 'available' }));163 }164 return out;165 }166}167168export default (meta: ConnectorMeta) => new AltConnector(meta);169