import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; import { dotNetDate, hintFromLabel, isBundleTitle, safeYear, slugFromTitle } from '../_g10-lib/index.js'; import type { DeptHint } from '../_auction-lib/categories.js'; /** * Trade Me (New Zealand) — official API v1. GET /v1/Search/General.json?category=&rows=&page= * signed with OAuth 1.0a PLAINTEXT using only the application's consumer key/secret (public methods * accept application-only authorisation: Authorization: OAuth oauth_consumer_key="…", * oauth_signature_method="PLAINTEXT", oauth_signature="&"). NZD listings from the * Antiques & collectables, Toys & models (vintage/die-cast/Lego), Gaming (trading cards), Music (vinyl), * Jewellery & watches, Pottery & glass and Rare books categories. Gated: TRADEME_CONSUMER_KEY/SECRET. */ const PARSER_VERSION = '1.0.0'; export function apiBase(env: string | undefined): string { return (env ?? '').toLowerCase() === 'sandbox' ? 'https://api.tmsandbox.co.nz/v1' : 'https://api.trademe.co.nz/v1'; } export const SeedSchema = z.object({ category: z.string().regex(/^\d{4}(-\d{4})*-?$/), categorySlug: z.string(), hint: z.string().optional(), searchString: z.string().optional() }); export type Seed = z.infer; const ConfigSchema = z.object({ categories: z.array(SeedSchema).min(1), rows: z.number().int().min(1).max(500).default(25), pagesPerCategory: z.number().int().min(1).default(1), backfillPages: z.number().int().min(1).default(8), categoriesPerRun: z.number().int().min(1).default(8), sortOrder: z.string().default('Default'), condition: z.enum(['All', 'New', 'Used']).default('All'), photoSize: z.enum(['Thumbnail', 'List', 'Medium', 'Gallery', 'Large', 'FullSize']).default('Large'), }); export const ListingSchema = z.object({ ListingId: z.number().int(), Title: z.string(), Subtitle: z.string().nullable().optional(), Category: z.string().nullable().optional(), CategoryPath: z.string().nullable().optional(), StartPrice: z.number().nullable().optional(), BuyNowPrice: z.number().nullable().optional(), MaxBidAmount: z.number().nullable().optional(), PriceDisplay: z.string().nullable().optional(), StartDate: z.string().nullable().optional(), EndDate: z.string().nullable().optional(), PictureHref: z.string().nullable().optional(), Region: z.string().nullable().optional(), Suburb: z.string().nullable().optional(), BidCount: z.number().int().nullable().optional(), HasBuyNow: z.boolean().nullable().optional(), HasReserve: z.boolean().nullable().optional(), IsReserveMet: z.boolean().nullable().optional(), ReserveState: z.number().int().nullable().optional(), IsClassified: z.boolean().nullable().optional(), IsNew: z.boolean().nullable().optional(), Attributes: z.array(z.object({ Name: z.string().nullable().optional(), DisplayName: z.string().nullable().optional(), Value: z.string().nullable().optional() })).default([]), }); export type Listing = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), totalCount: z.number().int().nullable(), pageSize: z.number().int().nullable(), listings: z.array(ListingSchema) }); export type PagePayload = z.infer; export function parseSearchResponse(json: unknown): { totalCount: number | null; pageSize: number | null; listings: Listing[]; rejected: number } | null { const j = json as { TotalCount?: number; PageSize?: number; List?: unknown[] } | null; if (!j || typeof j !== 'object' || !Array.isArray(j.List)) return null; const listings: Listing[] = []; let rejected = 0; for (const l of j.List) { const r = ListingSchema.safeParse(l); if (r.success) listings.push(r.data); else rejected++; } return { totalCount: typeof j.TotalCount === 'number' ? j.TotalCount : null, pageSize: typeof j.PageSize === 'number' ? j.PageSize : null, listings, rejected }; } export function searchUrl(base: string, seed: Seed, opts: { rows: number; page: number; sortOrder: string; condition: string; photoSize: string }): string { const u = new URL(`${base}/Search/General.json`); u.searchParams.set('category', seed.category); if (seed.searchString) u.searchParams.set('search_string', seed.searchString); u.searchParams.set('rows', String(opts.rows)); u.searchParams.set('page', String(opts.page)); u.searchParams.set('sort_order', opts.sortOrder); if (opts.condition !== 'All') u.searchParams.set('condition', opts.condition); u.searchParams.set('photo_size', opts.photoSize); return u.toString(); } /** Application-only OAuth 1.0a PLAINTEXT header (Trade Me accepts it for public/unauthenticated methods). */ export function authHeader(consumerKey: string, consumerSecret: string): string { const enc = (s: string) => encodeURIComponent(s).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`); return `OAuth oauth_consumer_key="${enc(consumerKey)}", oauth_signature_method="PLAINTEXT", oauth_signature="${enc(consumerSecret)}%26"`; } /** Current asking price: Buy Now when offered, else leading bid, else start price. */ export function listingPrice(l: Listing): { price: number | null; listingType: 'fixed_price' | 'auction' | 'ask' | 'unknown' } { const bids = l.BidCount ?? 0; if (l.IsClassified) return { price: l.StartPrice && l.StartPrice > 0 ? l.StartPrice : null, listingType: 'ask' }; if (bids > 0 && l.MaxBidAmount && l.MaxBidAmount > 0) return { price: l.MaxBidAmount, listingType: 'auction' }; if (l.HasBuyNow && l.BuyNowPrice && l.BuyNowPrice > 0) return { price: l.BuyNowPrice, listingType: 'fixed_price' }; if (l.StartPrice && l.StartPrice > 0) return { price: l.StartPrice, listingType: 'auction' }; return { price: null, listingType: 'unknown' }; } export class TradeMeConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/(www\.)?trademe\.co\.nz\/(?:a\/)?.*?\/listing\/(\d+)/i]; private readonly cfg: z.infer; constructor(meta: ConnectorMeta) { super(meta); this.cfg = ConfigSchema.parse(meta.config); } private credentials(ctx: CrawlContext): { header: string; base: string } | null { const key = process.env.TRADEME_CONSUMER_KEY?.trim(); const secret = process.env.TRADEME_CONSUMER_SECRET?.trim(); if (!key || !secret) { ctx.anomaly('missing_credentials', 'TRADEME_CONSUMER_KEY / TRADEME_CONSUMER_SECRET not set — connector is gated (DISABLED)'); return null; } return { header: authHeader(key, secret), base: apiBase(process.env.TRADEME_ENV) }; } async *crawl(ctx: CrawlContext): AsyncIterable { const cred = this.credentials(ctx); if (!cred) return; const backfill = ctx.options.mode === 'backfill'; const pages = backfill ? this.cfg.backfillPages : this.cfg.pagesPerCategory; const seeds = this.cfg.categories; const cursor = (ctx.options.cursor ?? {}) as { seedIndex?: number }; const start = backfill ? 0 : Math.min(cursor.seedIndex ?? 0, seeds.length - 1); const perRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.categoriesPerRun); let count = 0; let items = 0; for (let k = 0; k < perRun; k++) { const si = (start + k) % seeds.length; const seed = seeds[si]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = searchUrl(cred.base, seed, { rows: this.cfg.rows, page, sortOrder: this.cfg.sortOrder, condition: this.cfg.condition, photoSize: this.cfg.photoSize }); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', headers: { authorization: cred.header, accept: 'application/json' }, expect: ['title', 'price', 'currency'], parse: (r) => { const p = parseSearchResponse(r.json); return p ? { title: p.listings[0]?.Title ?? (p.totalCount === 0 ? 'empty' : null), price: p.listings[0]?.StartPrice ?? null, currency: p.listings.length ? 'NZD' : null } : null; }, minQuality: 0.2, }); const parsed = res.success ? parseSearchResponse(res.json) : null; if (!parsed) { const err = (res.json as { ErrorDescription?: string } | null)?.ErrorDescription; ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${seed.category} p${page}: ${res.httpStatus} ${err ?? res.error ?? ''}`); break; } if (parsed.rejected) ctx.anomaly('schema_drift', `${parsed.rejected} listings rejected by schema`); if (!parsed.listings.length) break; count++; items += parsed.listings.length; const payload: PagePayload = { kind: 'search_page', url, seed, page, totalCount: parsed.totalCount, pageSize: parsed.pageSize, listings: parsed.listings }; yield { url, externalId: `${seed.category}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.progress({ page, totalPages: parsed.totalCount ? Math.ceil(parsed.totalCount / this.cfg.rows) : null, itemsProcessed: items }); if (parsed.listings.length < this.cfg.rows || (parsed.totalCount !== null && page * this.cfg.rows >= parsed.totalCount)) break; } await ctx.setCursor({ seedIndex: (si + 1) % seeds.length, at: new Date().toISOString() }); } if (backfill) await ctx.setCursor({ seedIndex: 0, done: true, at: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const l of p.listings) { const { price, listingType } = listingPrice(l); const hint = (p.seed.hint as DeptHint | undefined) ?? hintFromLabel(l.CategoryPath ?? null); const categorySlug = p.seed.hint ? (slugFromTitle(l.Title, hint) ?? p.seed.categorySlug) : p.seed.categorySlug; const grade = parseGradeFromTitle(l.Title); const attrs = Object.fromEntries(l.Attributes.filter((a) => a.Name && a.Value).map((a) => [a.Name!, a.Value!])); const attributes = AssetAttributesSchema.parse({ categorySlug, name: l.Title, year: safeYear(`${l.Title} ${l.Subtitle ?? ''}`), identifiers: { trademe_listing_id: String(l.ListingId) }, metadata: { category_number: l.Category ?? null, category_path: l.CategoryPath ?? null, start_price: l.StartPrice ?? null, buy_now_price: l.BuyNowPrice ?? null, max_bid: l.MaxBidAmount ?? null, price_display: l.PriceDisplay ?? null, has_reserve: l.HasReserve ?? null, reserve_state: l.ReserveState ?? null, is_classified: l.IsClassified ?? null, is_new: l.IsNew ?? null, attributes: attrs, is_bundle_title: isBundleTitle(l.Title) }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `https://www.trademe.co.nz/a/listing/${l.ListingId}`, externalId: String(l.ListingId), rawTitle: l.Subtitle ? `${l.Title} — ${l.Subtitle}` : l.Title, imageUrls: l.PictureHref ? [l.PictureHref] : [], attributes, grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null }, condition: { condition: null, conditionRaw: l.IsNew === true ? 'New' : l.IsNew === false ? 'Used' : null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.7, parserVersion: PARSER_VERSION, listingType, price, currency: price !== null ? 'NZD' : null, location: [l.Suburb, l.Region].filter(Boolean).join(', ') || null, listedAt: dotNetDate(l.StartDate), endsAt: dotNetDate(l.EndDate), availability: 'available', bidCount: l.BidCount ?? null, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): TradeMeConnector { return new TradeMeConnector(meta); }