TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { epochSeconds, isBundleTitle, plainText, safeYear } from '../_g10-lib/index.js';67/**8 * Etsy Open API v3 (official; x-api-key header = the app keystring). findAllListingsActive —9 * GET /v3/application/listings/active?keywords=&taxonomy_id=&limit=100&offset=&sort_on=created&sort_order=desc —10 * returns active listings with Money prices {amount, divisor, currency_code} in the shop's currency.11 * Images/shop names are not part of that response; they are attached with one12 * GET /v3/application/listings/batch?listing_ids=…&includes=Images,Shop call per page (≤ 100 ids).13 * Taxonomy ids are resolved at run time from /v3/application/seller-taxonomy/nodes by node-name path.14 * Gated: requires ETSY_API_KEY. Listings only (Etsy exposes no sold prices publicly).15 */16const API = 'https://openapi.etsy.com/v3/application';17const PARSER_VERSION = '1.0.0';18const MAX_LIMIT = 100;1920export const QuerySchema = z.object({21 keywords: z.string().optional(),22 taxonomyId: z.number().int().optional(),23 /** node names from the seller taxonomy root, e.g. ["Art & Collectibles", "Collectibles"] */24 taxonomyPath: z.array(z.string()).optional(),25 categorySlug: z.string(),26 minPrice: z.number().optional(),27 maxPrice: z.number().optional(),28});29export type Query = z.infer<typeof QuerySchema>;3031const ConfigSchema = z.object({32 queries: z.array(QuerySchema).min(1),33 limit: z.number().int().min(1).max(MAX_LIMIT).default(MAX_LIMIT),34 pagesPerQuery: z.number().int().min(1).default(1),35 backfillPages: z.number().int().min(1).default(5),36 queriesPerRun: z.number().int().min(1).default(8),37 fetchImages: z.boolean().default(true),38 sortOn: z.enum(['created', 'price', 'updated', 'score']).default('created'),39 sortOrder: z.enum(['asc', 'desc']).default('desc'),40});4142export const MoneySchema = z.object({ amount: z.number(), divisor: z.number().positive(), currency_code: z.string() });43export const ListingSchema = z.object({44 listing_id: z.number().int(),45 shop_id: z.number().int().nullable().optional(),46 title: z.string(),47 description: z.string().nullable().optional(),48 state: z.string().nullable().optional(),49 url: z.string(),50 quantity: z.number().int().nullable().optional(),51 price: MoneySchema,52 taxonomy_id: z.number().int().nullable().optional(),53 tags: z.array(z.string()).default([]),54 materials: z.array(z.string()).default([]),55 when_made: z.string().nullable().optional(),56 who_made: z.string().nullable().optional(),57 is_supply: z.boolean().nullable().optional(),58 is_customizable: z.boolean().nullable().optional(),59 has_variations: z.boolean().nullable().optional(),60 listing_type: z.string().nullable().optional(),61 language: z.string().nullable().optional(),62 creation_timestamp: z.number().nullable().optional(),63 original_creation_timestamp: z.number().nullable().optional(),64 ending_timestamp: z.number().nullable().optional(),65 last_modified_timestamp: z.number().nullable().optional(),66 num_favorers: z.number().int().nullable().optional(),67 views: z.number().int().nullable().optional(),68 /** attached from /listings/batch?includes=Images,Shop */69 images: z.array(z.object({ url_570xN: z.string().optional(), url_fullxfull: z.string().optional(), listing_image_id: z.number().optional() })).default([]),70 shop: z.object({ shop_name: z.string().optional(), url: z.string().optional(), shop_location_country_iso: z.string().nullable().optional(), review_average: z.number().nullable().optional(), review_count: z.number().nullable().optional() }).nullable().optional(),71});72export type Listing = z.infer<typeof ListingSchema>;7374export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), query: QuerySchema, offset: z.number().int(), limit: z.number().int(), count: z.number().int().nullable(), listings: z.array(ListingSchema) });75export type PagePayload = z.infer<typeof PagePayloadSchema>;7677/** Trim one API listing to the persisted shape (schema-drift tolerant: unknown → dropped). */78export function trimListing(raw: unknown): Listing | null {79 if (!raw || typeof raw !== 'object') return null;80 const r = raw as Record<string, unknown>;81 const parsed = ListingSchema.safeParse({ ...r, description: typeof r.description === 'string' ? plainText(r.description, 800) : null, tags: Array.isArray(r.tags) ? r.tags.slice(0, 20) : [], materials: Array.isArray(r.materials) ? r.materials.slice(0, 10) : [] });82 return parsed.success ? parsed.data : null;83}8485export function parseSearchResponse(json: unknown): { count: number | null; listings: Listing[]; rejected: number } | null {86 const j = json as { count?: number; results?: unknown[] } | null;87 if (!j || !Array.isArray(j.results)) return null;88 const listings: Listing[] = [];89 let rejected = 0;90 for (const r of j.results) {91 const t = trimListing(r);92 if (t) listings.push(t);93 else rejected++;94 }95 return { count: typeof j.count === 'number' ? j.count : null, listings, rejected };96}9798export function searchUrl(q: Query, opts: { limit: number; offset: number; sortOn: string; sortOrder: string }, taxonomyId: number | null): string {99 const u = new URL(`${API}/listings/active`);100 if (q.keywords) u.searchParams.set('keywords', q.keywords);101 if (taxonomyId) u.searchParams.set('taxonomy_id', String(taxonomyId));102 if (q.minPrice !== undefined) u.searchParams.set('min_price', String(q.minPrice));103 if (q.maxPrice !== undefined) u.searchParams.set('max_price', String(q.maxPrice));104 u.searchParams.set('sort_on', opts.sortOn);105 u.searchParams.set('sort_order', opts.sortOrder);106 u.searchParams.set('limit', String(opts.limit));107 u.searchParams.set('offset', String(opts.offset));108 return u.toString();109}110111/** Walk the seller taxonomy tree by node names (case-insensitive). */112export function resolveTaxonomyPath(nodes: unknown, path: string[]): number | null {113 let level = (nodes as { results?: unknown[] } | null)?.results ?? (Array.isArray(nodes) ? nodes : null);114 let id: number | null = null;115 for (const name of path) {116 if (!Array.isArray(level)) return null;117 const hit = (level as Array<{ id?: number; name?: string; children?: unknown[] }>).find((n) => (n.name ?? '').toLowerCase() === name.toLowerCase());118 if (!hit || typeof hit.id !== 'number') return null;119 id = hit.id;120 level = hit.children ?? [];121 }122 return id;123}124125export function priceOf(m: z.infer<typeof MoneySchema>): number | null {126 const v = m.amount / m.divisor;127 return Number.isFinite(v) && v > 0 ? v : null;128}129130export class EtsyConnector extends BaseConnector {131 readonly version = '1.0.0';132 readonly parserVersion = PARSER_VERSION;133 protected override minIntervalMs = 400;134 override readonly urlPatterns = [/^https?:\/\/(www\.)?etsy\.com\/(?:[a-z]{2}\/)?listing\/(\d+)/i];135 private readonly cfg: z.infer<typeof ConfigSchema>;136137 constructor(meta: ConnectorMeta) {138 super(meta);139 this.cfg = ConfigSchema.parse(meta.config);140 }141142 private apiKey(ctx: CrawlContext): string | null {143 const key = process.env.ETSY_API_KEY?.trim();144 if (!key) ctx.anomaly('missing_credentials', 'ETSY_API_KEY not set — connector is gated (DISABLED)');145 return key ?? null;146 }147148 private headers(key: string): Record<string, string> {149 return { 'x-api-key': key, accept: 'application/json' };150 }151152 private async taxonomyId(ctx: CrawlContext, key: string, q: Query, cache: Record<string, number>): Promise<number | null> {153 if (q.taxonomyId) return q.taxonomyId;154 if (!q.taxonomyPath?.length) return null;155 const k = q.taxonomyPath.join(' > ');156 if (cache[k]) return cache[k]!;157 await this.throttle();158 const res = await ctx.fetch(`${API}/seller-taxonomy/nodes`, { engines: ['api'], responseType: 'json', headers: this.headers(key), minQuality: 0, force: true });159 const id = res.success ? resolveTaxonomyPath(res.json, q.taxonomyPath) : null;160 if (!id) ctx.anomaly('selector_missing', `Etsy taxonomy path not found: ${k}`);161 else cache[k] = id;162 return id;163 }164165 private async attachImages(ctx: CrawlContext, key: string, listings: Listing[]): Promise<void> {166 if (!this.cfg.fetchImages || !listings.length) return;167 const ids = listings.map((l) => l.listing_id);168 const url = `${API}/listings/batch?listing_ids=${ids.join(',')}&includes=Images,Shop`;169 await this.throttle(url);170 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', headers: this.headers(key), minQuality: 0 });171 const results = (res.json as { results?: Array<Record<string, unknown>> } | null)?.results;172 if (!res.success || !Array.isArray(results)) {173 ctx.anomaly('page_fetch_failed', `listings/batch: ${res.error ?? res.httpStatus}`);174 return;175 }176 const byId = new Map(results.map((r) => [Number(r.listing_id), r] as const));177 for (const l of listings) {178 const full = byId.get(l.listing_id);179 if (!full) continue;180 const imgs = Array.isArray(full.images) ? (full.images as Array<Record<string, unknown>>).slice(0, 6).map((i) => ({ url_570xN: typeof i.url_570xN === 'string' ? i.url_570xN : undefined, url_fullxfull: typeof i.url_fullxfull === 'string' ? i.url_fullxfull : undefined, listing_image_id: typeof i.listing_image_id === 'number' ? i.listing_image_id : undefined })) : [];181 const shop = full.shop && typeof full.shop === 'object' ? (full.shop as Record<string, unknown>) : null;182 l.images = imgs;183 l.shop = shop ? { shop_name: typeof shop.shop_name === 'string' ? shop.shop_name : undefined, url: typeof shop.url === 'string' ? shop.url : undefined, shop_location_country_iso: typeof shop.shop_location_country_iso === 'string' ? shop.shop_location_country_iso : null, review_average: typeof shop.review_average === 'number' ? shop.review_average : null, review_count: typeof shop.review_count === 'number' ? shop.review_count : null } : null;184 }185 }186187 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {188 const key = this.apiKey(ctx);189 if (!key) return;190 const backfill = ctx.options.mode === 'backfill';191 const pages = backfill ? this.cfg.backfillPages : this.cfg.pagesPerQuery;192 const queries = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => QuerySchema.parse({ keywords: s, categorySlug: 'trading_cards' })) : this.cfg.queries;193 const cursor = (ctx.options.cursor ?? {}) as { queryIndex?: number; taxonomy?: Record<string, number> };194 const taxonomyCache: Record<string, number> = { ...(cursor.taxonomy ?? {}) };195 const startQ = backfill ? 0 : Math.min(cursor.queryIndex ?? 0, queries.length - 1);196 const perRun = backfill ? queries.length : Math.min(queries.length, this.cfg.queriesPerRun);197 let count = 0;198 let items = 0;199 for (let k = 0; k < perRun; k++) {200 const qi = (startQ + k) % queries.length;201 const q = queries[qi]!;202 const taxId = await this.taxonomyId(ctx, key, q, taxonomyCache);203 if (q.taxonomyPath?.length && !taxId) continue;204 for (let page = 0; page < pages; page++) {205 if (ctx.signal?.aborted || this.reached(ctx, count)) return;206 const offset = page * this.cfg.limit;207 const url = searchUrl(q, { limit: this.cfg.limit, offset, sortOn: this.cfg.sortOn, sortOrder: this.cfg.sortOrder }, taxId);208 await this.throttle(url);209 const res = await ctx.fetch(url, {210 engines: ['api'],211 responseType: 'json',212 headers: this.headers(key),213 expect: ['title', 'price', 'currency'],214 parse: (r) => {215 const p = parseSearchResponse(r.json);216 return p ? { title: p.listings[0]?.title ?? (p.count === 0 ? 'empty' : null), price: p.listings[0]?.price.amount ?? null, currency: p.listings[0]?.price.currency_code ?? null } : null;217 },218 minQuality: 0.2,219 });220 const parsed = res.success ? parseSearchResponse(res.json) : null;221 if (!parsed) {222 const err = (res.json as { error?: string } | null)?.error;223 ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${q.keywords ?? q.taxonomyPath?.join('>')} offset ${offset}: ${res.httpStatus} ${err ?? res.error ?? ''}`);224 break;225 }226 if (parsed.rejected) ctx.anomaly('schema_drift', `${parsed.rejected} listings rejected by schema`);227 if (!parsed.listings.length) break;228 await this.attachImages(ctx, key, parsed.listings);229 count++;230 items += parsed.listings.length;231 const payload: PagePayload = { kind: 'search_page', url, query: q, offset, limit: this.cfg.limit, count: parsed.count, listings: parsed.listings };232 yield { url, externalId: `${q.keywords ?? ''}:${taxId ?? ''}:${offset}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };233 await ctx.progress({ page: page + 1, totalPages: parsed.count ? Math.ceil(parsed.count / this.cfg.limit) : null, itemsProcessed: items });234 if (parsed.listings.length < this.cfg.limit || (parsed.count !== null && offset + this.cfg.limit >= parsed.count)) break;235 }236 await ctx.setCursor({ queryIndex: (qi + 1) % queries.length, taxonomy: taxonomyCache, at: new Date().toISOString() });237 }238 if (backfill) await ctx.setCursor({ queryIndex: 0, taxonomy: taxonomyCache, done: true, at: new Date().toISOString() });239 }240241 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {242 const id = url.match(this.urlPatterns[0]!)?.[2];243 const key = id ? this.apiKey(ctx) : null;244 if (!id || !key) return [];245 const apiUrl = `${API}/listings/${id}?includes=Images,Shop`;246 await this.throttle(apiUrl);247 const res = await ctx.fetch(apiUrl, { engines: ['api'], responseType: 'json', headers: this.headers(key), minQuality: 0 });248 const l = res.success ? trimListing(res.json) : null;249 if (!l) return [];250 const payload: PagePayload = { kind: 'search_page', url: apiUrl, query: { categorySlug: 'trading_cards' }, offset: 0, limit: 1, count: 1, listings: [l] };251 return [{ url: apiUrl, externalId: `listing:${id}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];252 }253254 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {255 const p = PagePayloadSchema.parse(raw.payload);256 const out: NormalizedRecord[] = [];257 for (const l of p.listings) {258 if (l.is_supply || l.when_made === 'made_to_order' || (l.state && l.state !== 'active')) continue;259 const price = priceOf(l.price);260 if (!price) continue;261 const grade = parseGradeFromTitle(l.title);262 const decade = l.when_made?.match(/^(1[6-9]\d0|20[0-2]0)s$/)?.[0] ?? null;263 const attributes = AssetAttributesSchema.parse({264 categorySlug: p.query.categorySlug,265 name: l.title,266 year: safeYear(l.title),267 material: l.materials[0] ?? null,268 language: l.language ?? null,269 identifiers: { etsy_listing_id: String(l.listing_id) },270 metadata: { when_made: l.when_made ?? null, decade, who_made: l.who_made ?? null, tags: l.tags, materials: l.materials, taxonomy_id: l.taxonomy_id ?? null, num_favorers: l.num_favorers ?? null, has_variations: l.has_variations ?? null, shop_id: l.shop_id ?? null, shop_country: l.shop?.shop_location_country_iso ?? null, keywords: p.query.keywords ?? null, is_bundle_title: isBundleTitle(l.title) },271 });272 out.push(273 NormalizedListingSchema.parse({274 kind: 'listing',275 connectorId: this.meta.id,276 sourceId: this.meta.sourceId,277 sourceUrl: l.url,278 externalId: String(l.listing_id),279 rawTitle: l.title,280 description: l.description ?? null,281 imageUrls: l.images.map((i) => i.url_fullxfull ?? i.url_570xN).filter((x): x is string => Boolean(x)),282 attributes,283 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null },284 condition: { condition: null, conditionRaw: null, completeness: null },285 observedAt: raw.fetchedAt,286 confidence: 0.7,287 parserVersion: PARSER_VERSION,288 listingType: 'fixed_price',289 price,290 currency: /^[A-Z]{3}$/.test(l.price.currency_code) ? l.price.currency_code : null,291 seller: l.shop?.shop_name ?? null,292 sellerReputation: l.shop?.review_average != null && l.shop.review_count != null ? `${l.shop.review_average.toFixed(1)}★ (${l.shop.review_count} reviews)` : null,293 location: l.shop?.shop_location_country_iso ?? null,294 quantity: l.quantity ?? null,295 listedAt: epochSeconds(l.original_creation_timestamp ?? l.creation_timestamp),296 endsAt: epochSeconds(l.ending_timestamp),297 availability: 'available',298 }),299 );300 }301 return out;302 }303}304305export default function createConnector(meta: ConnectorMeta): EtsyConnector {306 return new EtsyConnector(meta);307}308