TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { cachedToken, isBundleTitle, isoDate, safeYear, slugFromTitle } from '../_g10-lib/index.js';67/**8 * eBay Browse API (official, OAuth2 client-credentials, scope https://api.ebay.com/oauth/api_scope).9 * GET /buy/browse/v1/item_summary/search?q=&category_ids=&filter=&limit=200&offset=N with the10 * X-EBAY-C-MARKETPLACE-ID header selects the marketplace (EBAY_US, EBAY_CA, EBAY_GB, EBAY_DE, EBAY_FR,11 * EBAY_AU…). The Browse API returns LIVE listings only — no sold/completed items (Marketplace Insights12 * is a limited-release API, see meta.json). Gated: requires EBAY_CLIENT_ID + EBAY_CLIENT_SECRET.13 */14const PARSER_VERSION = '1.0.0';15const MAX_LIMIT = 200;16const MAX_RESULTS = 10_000; // Browse caps offset + limit at 10,000 per query1718export function apiBase(env: string | undefined): { api: string; token: string } {19 const sandbox = (env ?? '').toLowerCase() === 'sandbox';20 return sandbox ? { api: 'https://api.sandbox.ebay.com', token: 'https://api.sandbox.ebay.com/identity/v1/oauth2/token' } : { api: 'https://api.ebay.com', token: 'https://api.ebay.com/identity/v1/oauth2/token' };21}2223export const MarketplaceSchema = z.object({ id: z.string().regex(/^EBAY_[A-Z]{2,4}$/), currency: z.string().length(3), country: z.string().length(2), locale: z.string().default('en-US') });24export type Marketplace = z.infer<typeof MarketplaceSchema>;25export const QuerySchema = z.object({ q: z.string().optional(), categoryIds: z.array(z.string()).default([]), categorySlug: z.string().nullable().default(null), filter: z.string().optional(), marketplaces: z.array(z.string()).optional() });26export type Query = z.infer<typeof QuerySchema>;2728const ConfigSchema = z.object({29 marketplaces: z.array(MarketplaceSchema).min(1),30 queries: z.array(QuerySchema).min(1),31 limit: z.number().int().min(1).max(MAX_LIMIT).default(MAX_LIMIT),32 pagesPerQuery: z.number().int().min(1).default(1),33 backfillPages: z.number().int().min(1).default(10),34 /** default eBay field filter appended to every query (buying options, conditions, price range…) */35 defaultFilter: z.string().default('buyingOptions:{FIXED_PRICE|AUCTION|BEST_OFFER}'),36 sort: z.string().default('newlyListed'),37 queriesPerRun: z.number().int().min(1).default(6),38});3940const Amount = z.object({ value: z.string(), currency: z.string(), convertedFromValue: z.string().optional(), convertedFromCurrency: z.string().optional() });41export const ItemSummarySchema = z.object({42 itemId: z.string(),43 legacyItemId: z.string().optional(),44 title: z.string(),45 shortDescription: z.string().optional(),46 price: Amount.optional(),47 currentBidPrice: Amount.optional(),48 bidCount: z.number().int().optional(),49 buyingOptions: z.array(z.string()).default([]),50 condition: z.string().optional(),51 conditionId: z.string().optional(),52 itemWebUrl: z.string(),53 itemAffiliateWebUrl: z.string().optional(),54 image: z.object({ imageUrl: z.string() }).optional(),55 thumbnailImages: z.array(z.object({ imageUrl: z.string() })).default([]),56 additionalImages: z.array(z.object({ imageUrl: z.string() })).default([]),57 seller: z.object({ username: z.string().optional(), feedbackPercentage: z.string().optional(), feedbackScore: z.number().int().optional(), sellerAccountType: z.string().optional() }).optional(),58 shippingOptions: z.array(z.object({ shippingCost: Amount.optional(), shippingCostType: z.string().optional(), guaranteedDelivery: z.boolean().optional() })).default([]),59 itemLocation: z.object({ city: z.string().optional(), stateOrProvince: z.string().optional(), postalCode: z.string().optional(), country: z.string().optional() }).optional(),60 epid: z.string().optional(),61 categories: z.array(z.object({ categoryId: z.string(), categoryName: z.string().optional() })).default([]),62 leafCategoryIds: z.array(z.string()).default([]),63 itemCreationDate: z.string().optional(),64 itemEndDate: z.string().optional(),65 itemGroupHref: z.string().optional(),66 itemGroupType: z.string().optional(),67 listingMarketplaceId: z.string().optional(),68 adultOnly: z.boolean().optional(),69 topRatedBuyingExperience: z.boolean().optional(),70 watchCount: z.number().int().optional(),71});72export type ItemSummary = z.infer<typeof ItemSummarySchema>;7374export const SearchPayloadSchema = z.object({75 kind: z.literal('search_page'),76 url: z.string(),77 marketplace: MarketplaceSchema,78 query: QuerySchema,79 offset: z.number().int(),80 limit: z.number().int(),81 total: z.number().int().nullable(),82 next: z.string().nullable(),83 items: z.array(ItemSummarySchema),84 warnings: z.array(z.unknown()).default([]),85});86export type SearchPayload = z.infer<typeof SearchPayloadSchema>;8788/** Trim an API item summary to the fields we persist (drops marketing/compat noise; never invents). */89export function trimItem(raw: unknown): ItemSummary | null {90 const r = ItemSummarySchema.safeParse(raw);91 return r.success ? r.data : null;92}9394export function parseSearchResponse(json: unknown): { items: ItemSummary[]; total: number | null; next: string | null; warnings: unknown[]; rejected: number } | null {95 const j = json as { itemSummaries?: unknown[]; total?: number; next?: string; warnings?: unknown[] } | null;96 if (!j || typeof j !== 'object' || (!Array.isArray(j.itemSummaries) && typeof j.total !== 'number')) return null;97 const items: ItemSummary[] = [];98 let rejected = 0;99 for (const it of j.itemSummaries ?? []) {100 const t = trimItem(it);101 if (t) items.push(t);102 else rejected++;103 }104 return { items, total: typeof j.total === 'number' ? j.total : null, next: typeof j.next === 'string' ? j.next : null, warnings: j.warnings ?? [], rejected };105}106107export function searchUrl(base: string, q: Query, opts: { limit: number; offset: number; filter: string; sort: string }): string {108 const u = new URL(`${base}/buy/browse/v1/item_summary/search`);109 if (q.q) u.searchParams.set('q', q.q);110 if (q.categoryIds.length) u.searchParams.set('category_ids', q.categoryIds.join(','));111 const filters = [opts.filter, q.filter].filter(Boolean).join(',');112 if (filters) u.searchParams.set('filter', filters);113 if (opts.sort) u.searchParams.set('sort', opts.sort);114 u.searchParams.set('limit', String(opts.limit));115 u.searchParams.set('offset', String(opts.offset));116 return u.toString();117}118119/** Client-credentials token (cached process-wide until 60 s before expiry). Direct fetch: the token endpoint needs a form body + Basic auth. */120export async function getAppToken(clientId: string, clientSecret: string, tokenUrl: string, fetchImpl: typeof fetch = fetch): Promise<string> {121 return cachedToken(`ebay:${tokenUrl}:${clientId}`, async () => {122 const res = await fetchImpl(tokenUrl, {123 method: 'POST',124 headers: { authorization: `Basic ${Buffer.from(`${clientId}:${clientSecret}`).toString('base64')}`, 'content-type': 'application/x-www-form-urlencoded' },125 body: new URLSearchParams({ grant_type: 'client_credentials', scope: 'https://api.ebay.com/oauth/api_scope' }).toString(),126 });127 const text = await res.text();128 if (!res.ok) throw new Error(`eBay token HTTP ${res.status}: ${text.slice(0, 200)}`);129 const j = JSON.parse(text) as { access_token?: string; expires_in?: number; token_type?: string };130 if (!j.access_token) throw new Error('eBay token response without access_token');131 return { token: j.access_token, expiresInSeconds: Number(j.expires_in ?? 7200) };132 });133}134135const num = (a: { value: string } | undefined): number | null => {136 if (!a) return null;137 const n = Number.parseFloat(a.value);138 return Number.isFinite(n) && n >= 0 ? n : null;139};140141export class EbayBrowseConnector extends BaseConnector {142 readonly version = '1.0.0';143 readonly parserVersion = PARSER_VERSION;144 protected override minIntervalMs = 600;145 override readonly urlPatterns = [/^https?:\/\/(www\.)?ebay\.(com|ca|co\.uk|de|fr|com\.au|it|es)\/itm\/(?:[^/]+\/)?(\d+)/i];146 private readonly cfg: z.infer<typeof ConfigSchema>;147148 constructor(meta: ConnectorMeta) {149 super(meta);150 this.cfg = ConfigSchema.parse(meta.config);151 }152153 private credentials(ctx: CrawlContext): { id: string; secret: string; base: ReturnType<typeof apiBase> } | null {154 const id = process.env.EBAY_CLIENT_ID?.trim();155 const secret = process.env.EBAY_CLIENT_SECRET?.trim();156 if (!id || !secret) {157 ctx.anomaly('missing_credentials', 'EBAY_CLIENT_ID / EBAY_CLIENT_SECRET not set — connector is gated (DISABLED)');158 return null;159 }160 return { id, secret, base: apiBase(process.env.EBAY_ENV) };161 }162163 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {164 const cred = this.credentials(ctx);165 if (!cred) return;166 let token: string;167 try {168 token = await getAppToken(cred.id, cred.secret, cred.base.token);169 } catch (err) {170 ctx.anomaly('auth_failed', err instanceof Error ? err.message : String(err));171 return;172 }173 const backfill = ctx.options.mode === 'backfill';174 const pages = backfill ? this.cfg.backfillPages : this.cfg.pagesPerQuery;175 const queries = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => QuerySchema.parse({ q: s })) : this.cfg.queries;176 const cursor = (ctx.options.cursor ?? {}) as { queryIndex?: number };177 const startQ = backfill ? 0 : Math.min(cursor.queryIndex ?? 0, queries.length - 1);178 const perRun = backfill ? queries.length : Math.min(queries.length, this.cfg.queriesPerRun);179 let count = 0;180 let items = 0;181 for (let k = 0; k < perRun; k++) {182 const qi = (startQ + k) % queries.length;183 const q = queries[qi]!;184 const markets = this.cfg.marketplaces.filter((m) => !q.marketplaces || q.marketplaces.includes(m.id));185 for (const market of markets) {186 for (let page = 0; page < pages; page++) {187 if (ctx.signal?.aborted || this.reached(ctx, count)) return;188 const offset = page * this.cfg.limit;189 if (offset + this.cfg.limit > MAX_RESULTS) break;190 const url = searchUrl(cred.base.api, q, { limit: this.cfg.limit, offset, filter: this.cfg.defaultFilter, sort: this.cfg.sort });191 await this.throttle(url);192 const res = await ctx.fetch(url, {193 engines: ['api'],194 responseType: 'json',195 headers: { authorization: `Bearer ${token}`, 'x-ebay-c-marketplace-id': market.id, 'accept-language': market.locale, accept: 'application/json' },196 expect: ['title', 'price', 'currency'],197 parse: (r) => {198 const p = parseSearchResponse(r.json);199 return p ? { title: p.items[0]?.title ?? (p.total === 0 ? 'empty' : null), price: p.items[0]?.price?.value ?? null, currency: p.items[0]?.price?.currency ?? null } : null;200 },201 minQuality: 0.2,202 });203 const parsed = res.success ? parseSearchResponse(res.json) : null;204 if (!parsed) {205 const errs = (res.json as { errors?: Array<{ errorId?: number; message?: string }> } | null)?.errors;206 ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${market.id} ${q.q ?? q.categoryIds.join(',')} offset ${offset}: ${res.httpStatus} ${errs?.map((e) => `${e.errorId} ${e.message}`).join('; ') ?? res.error ?? ''}`);207 break;208 }209 if (parsed.rejected) ctx.anomaly('schema_drift', `${parsed.rejected} item summaries rejected by schema`);210 if (!parsed.items.length) break;211 count++;212 items += parsed.items.length;213 const payload: SearchPayload = { kind: 'search_page', url, marketplace: market, query: q, offset, limit: this.cfg.limit, total: parsed.total, next: parsed.next, items: parsed.items, warnings: parsed.warnings };214 yield { url, externalId: `${market.id}:${q.q ?? ''}:${q.categoryIds.join('+')}:${offset}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };215 await ctx.progress({ page: page + 1, totalPages: parsed.total ? Math.ceil(Math.min(parsed.total, MAX_RESULTS) / this.cfg.limit) : null, itemsProcessed: items });216 if (!parsed.next || parsed.items.length < this.cfg.limit) break;217 }218 }219 await ctx.setCursor({ queryIndex: (qi + 1) % queries.length, at: new Date().toISOString() });220 }221 if (backfill) await ctx.setCursor({ queryIndex: 0, done: true, at: new Date().toISOString() });222 }223224 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {225 const legacy = url.match(this.urlPatterns[0]!)?.[3];226 const cred = legacy ? this.credentials(ctx) : null;227 if (!legacy || !cred) return [];228 const token = await getAppToken(cred.id, cred.secret, cred.base.token);229 const tld = url.match(/ebay\.([a-z.]+)\//i)?.[1] ?? 'com';230 const market = this.cfg.marketplaces.find((m) => ({ com: 'EBAY_US', ca: 'EBAY_CA', 'co.uk': 'EBAY_GB', de: 'EBAY_DE', fr: 'EBAY_FR', 'com.au': 'EBAY_AU', it: 'EBAY_IT', es: 'EBAY_ES' })[tld] === m.id) ?? this.cfg.marketplaces[0]!;231 const apiUrl = `${cred.base.api}/buy/browse/v1/item/get_item_by_legacy_id?legacy_item_id=${encodeURIComponent(legacy)}`;232 await this.throttle(apiUrl);233 const res = await ctx.fetch(apiUrl, { engines: ['api'], responseType: 'json', headers: { authorization: `Bearer ${token}`, 'x-ebay-c-marketplace-id': market.id }, minQuality: 0 });234 const item = res.success ? trimItem({ ...(res.json as object), buyingOptions: (res.json as { buyingOptions?: string[] })?.buyingOptions ?? [] }) : null;235 if (!item) return [];236 const payload: SearchPayload = { kind: 'search_page', url: apiUrl, marketplace: market, query: { categoryIds: [], categorySlug: null }, offset: 0, limit: 1, total: 1, next: null, items: [item], warnings: [] };237 return [{ url: apiUrl, externalId: `item:${item.itemId}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];238 }239240 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {241 const p = SearchPayloadSchema.parse(raw.payload);242 const out: NormalizedRecord[] = [];243 for (const it of p.items) {244 if (it.adultOnly) continue;245 const categorySlug = p.query.categorySlug ?? slugFromTitle(it.title);246 if (!categorySlug) continue;247 const priceAmt = it.price ?? it.currentBidPrice;248 const price = num(priceAmt);249 const currency = priceAmt?.currency ?? p.marketplace.currency;250 const converted = Boolean(priceAmt?.convertedFromCurrency && priceAmt.convertedFromCurrency !== priceAmt.currency);251 const opts = it.buyingOptions;252 const listingType = opts.includes('AUCTION') ? 'auction' : opts.includes('BEST_OFFER') ? 'best_offer' : opts.includes('FIXED_PRICE') ? 'fixed_price' : 'unknown';253 const ship = it.shippingOptions.find((s) => s.shippingCost && s.shippingCost.currency === currency);254 const grade = parseGradeFromTitle(it.title);255 const images = [...new Set([it.image?.imageUrl, ...it.additionalImages.map((i) => i.imageUrl), ...it.thumbnailImages.map((i) => i.imageUrl)].filter((x): x is string => Boolean(x)))].slice(0, 8);256 const attributes = AssetAttributesSchema.parse({257 categorySlug,258 name: it.title,259 year: safeYear(it.title),260 identifiers: { ebay_item_id: it.itemId, ...(it.legacyItemId ? { ebay_legacy_item_id: it.legacyItemId } : {}), ...(it.epid ? { ebay_epid: it.epid } : {}) },261 metadata: {262 marketplace: it.listingMarketplaceId ?? p.marketplace.id,263 buying_options: opts,264 condition_id: it.conditionId ?? null,265 categories: it.categories,266 leaf_category_ids: it.leafCategoryIds,267 item_group_type: it.itemGroupType ?? null,268 price_converted_by_ebay: converted,269 original_price: converted ? { value: priceAmt?.convertedFromValue, currency: priceAmt?.convertedFromCurrency } : null,270 shipping_cost_type: ship?.shippingCostType ?? null,271 watch_count: it.watchCount ?? null,272 top_rated_buying_experience: it.topRatedBuyingExperience ?? null,273 query: p.query.q ?? null,274 is_bundle_title: isBundleTitle(it.title),275 },276 });277 out.push(278 NormalizedListingSchema.parse({279 kind: 'listing',280 connectorId: this.meta.id,281 sourceId: this.meta.sourceId,282 sourceUrl: it.itemWebUrl,283 externalId: it.itemId,284 rawTitle: it.title,285 description: it.shortDescription ?? null,286 imageUrls: images,287 attributes,288 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null },289 condition: { condition: null, conditionRaw: it.condition ?? null, completeness: null },290 observedAt: raw.fetchedAt,291 confidence: it.epid ? 0.85 : p.query.categorySlug ? 0.78 : 0.65,292 parserVersion: PARSER_VERSION,293 listingType,294 price,295 currency: currency && /^[A-Z]{3}$/.test(currency) ? currency : null,296 seller: it.seller?.username ?? null,297 sellerReputation: it.seller ? [it.seller.feedbackPercentage ? `${it.seller.feedbackPercentage}%` : null, typeof it.seller.feedbackScore === 'number' ? `${it.seller.feedbackScore} feedback` : null].filter(Boolean).join(' · ') || null : null,298 location: [it.itemLocation?.city, it.itemLocation?.stateOrProvince, it.itemLocation?.country].filter(Boolean).join(', ') || null,299 shippingCost: num(ship?.shippingCost),300 listedAt: isoDate(it.itemCreationDate),301 endsAt: isoDate(it.itemEndDate),302 availability: 'available',303 bidCount: it.bidCount ?? null,304 }),305 );306 }307 return out;308 }309}310311export default function createConnector(meta: ConnectorMeta): EbayBrowseConnector {312 return new EbayBrowseConnector(meta);313}314