TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { dotNetDate, hintFromLabel, isBundleTitle, safeYear, slugFromTitle } from '../_g10-lib/index.js';6import type { DeptHint } from '../_auction-lib/categories.js';78/**9 * Trade Me (New Zealand) — official API v1. GET /v1/Search/General.json?category=<id>&rows=&page=10 * signed with OAuth 1.0a PLAINTEXT using only the application's consumer key/secret (public methods11 * accept application-only authorisation: Authorization: OAuth oauth_consumer_key="…",12 * oauth_signature_method="PLAINTEXT", oauth_signature="<consumer_secret>&"). NZD listings from the13 * Antiques & collectables, Toys & models (vintage/die-cast/Lego), Gaming (trading cards), Music (vinyl),14 * Jewellery & watches, Pottery & glass and Rare books categories. Gated: TRADEME_CONSUMER_KEY/SECRET.15 */16const PARSER_VERSION = '1.0.0';1718export function apiBase(env: string | undefined): string {19 return (env ?? '').toLowerCase() === 'sandbox' ? 'https://api.tmsandbox.co.nz/v1' : 'https://api.trademe.co.nz/v1';20}2122export const SeedSchema = z.object({ category: z.string().regex(/^\d{4}(-\d{4})*-?$/), categorySlug: z.string(), hint: z.string().optional(), searchString: z.string().optional() });23export type Seed = z.infer<typeof SeedSchema>;2425const ConfigSchema = z.object({26 categories: z.array(SeedSchema).min(1),27 rows: z.number().int().min(1).max(500).default(25),28 pagesPerCategory: z.number().int().min(1).default(1),29 backfillPages: z.number().int().min(1).default(8),30 categoriesPerRun: z.number().int().min(1).default(8),31 sortOrder: z.string().default('Default'),32 condition: z.enum(['All', 'New', 'Used']).default('All'),33 photoSize: z.enum(['Thumbnail', 'List', 'Medium', 'Gallery', 'Large', 'FullSize']).default('Large'),34});3536export const ListingSchema = z.object({37 ListingId: z.number().int(),38 Title: z.string(),39 Subtitle: z.string().nullable().optional(),40 Category: z.string().nullable().optional(),41 CategoryPath: z.string().nullable().optional(),42 StartPrice: z.number().nullable().optional(),43 BuyNowPrice: z.number().nullable().optional(),44 MaxBidAmount: z.number().nullable().optional(),45 PriceDisplay: z.string().nullable().optional(),46 StartDate: z.string().nullable().optional(),47 EndDate: z.string().nullable().optional(),48 PictureHref: z.string().nullable().optional(),49 Region: z.string().nullable().optional(),50 Suburb: z.string().nullable().optional(),51 BidCount: z.number().int().nullable().optional(),52 HasBuyNow: z.boolean().nullable().optional(),53 HasReserve: z.boolean().nullable().optional(),54 IsReserveMet: z.boolean().nullable().optional(),55 ReserveState: z.number().int().nullable().optional(),56 IsClassified: z.boolean().nullable().optional(),57 IsNew: z.boolean().nullable().optional(),58 Attributes: z.array(z.object({ Name: z.string().nullable().optional(), DisplayName: z.string().nullable().optional(), Value: z.string().nullable().optional() })).default([]),59});60export type Listing = z.infer<typeof ListingSchema>;6162export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), totalCount: z.number().int().nullable(), pageSize: z.number().int().nullable(), listings: z.array(ListingSchema) });63export type PagePayload = z.infer<typeof PagePayloadSchema>;6465export function parseSearchResponse(json: unknown): { totalCount: number | null; pageSize: number | null; listings: Listing[]; rejected: number } | null {66 const j = json as { TotalCount?: number; PageSize?: number; List?: unknown[] } | null;67 if (!j || typeof j !== 'object' || !Array.isArray(j.List)) return null;68 const listings: Listing[] = [];69 let rejected = 0;70 for (const l of j.List) {71 const r = ListingSchema.safeParse(l);72 if (r.success) listings.push(r.data);73 else rejected++;74 }75 return { totalCount: typeof j.TotalCount === 'number' ? j.TotalCount : null, pageSize: typeof j.PageSize === 'number' ? j.PageSize : null, listings, rejected };76}7778export function searchUrl(base: string, seed: Seed, opts: { rows: number; page: number; sortOrder: string; condition: string; photoSize: string }): string {79 const u = new URL(`${base}/Search/General.json`);80 u.searchParams.set('category', seed.category);81 if (seed.searchString) u.searchParams.set('search_string', seed.searchString);82 u.searchParams.set('rows', String(opts.rows));83 u.searchParams.set('page', String(opts.page));84 u.searchParams.set('sort_order', opts.sortOrder);85 if (opts.condition !== 'All') u.searchParams.set('condition', opts.condition);86 u.searchParams.set('photo_size', opts.photoSize);87 return u.toString();88}8990/** Application-only OAuth 1.0a PLAINTEXT header (Trade Me accepts it for public/unauthenticated methods). */91export function authHeader(consumerKey: string, consumerSecret: string): string {92 const enc = (s: string) => encodeURIComponent(s).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);93 return `OAuth oauth_consumer_key="${enc(consumerKey)}", oauth_signature_method="PLAINTEXT", oauth_signature="${enc(consumerSecret)}%26"`;94}9596/** Current asking price: Buy Now when offered, else leading bid, else start price. */97export function listingPrice(l: Listing): { price: number | null; listingType: 'fixed_price' | 'auction' | 'ask' | 'unknown' } {98 const bids = l.BidCount ?? 0;99 if (l.IsClassified) return { price: l.StartPrice && l.StartPrice > 0 ? l.StartPrice : null, listingType: 'ask' };100 if (bids > 0 && l.MaxBidAmount && l.MaxBidAmount > 0) return { price: l.MaxBidAmount, listingType: 'auction' };101 if (l.HasBuyNow && l.BuyNowPrice && l.BuyNowPrice > 0) return { price: l.BuyNowPrice, listingType: 'fixed_price' };102 if (l.StartPrice && l.StartPrice > 0) return { price: l.StartPrice, listingType: 'auction' };103 return { price: null, listingType: 'unknown' };104}105106export class TradeMeConnector extends BaseConnector {107 readonly version = '1.0.0';108 readonly parserVersion = PARSER_VERSION;109 protected override minIntervalMs = 1500;110 override readonly urlPatterns = [/^https?:\/\/(www\.)?trademe\.co\.nz\/(?:a\/)?.*?\/listing\/(\d+)/i];111 private readonly cfg: z.infer<typeof ConfigSchema>;112113 constructor(meta: ConnectorMeta) {114 super(meta);115 this.cfg = ConfigSchema.parse(meta.config);116 }117118 private credentials(ctx: CrawlContext): { header: string; base: string } | null {119 const key = process.env.TRADEME_CONSUMER_KEY?.trim();120 const secret = process.env.TRADEME_CONSUMER_SECRET?.trim();121 if (!key || !secret) {122 ctx.anomaly('missing_credentials', 'TRADEME_CONSUMER_KEY / TRADEME_CONSUMER_SECRET not set — connector is gated (DISABLED)');123 return null;124 }125 return { header: authHeader(key, secret), base: apiBase(process.env.TRADEME_ENV) };126 }127128 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {129 const cred = this.credentials(ctx);130 if (!cred) return;131 const backfill = ctx.options.mode === 'backfill';132 const pages = backfill ? this.cfg.backfillPages : this.cfg.pagesPerCategory;133 const seeds = this.cfg.categories;134 const cursor = (ctx.options.cursor ?? {}) as { seedIndex?: number };135 const start = backfill ? 0 : Math.min(cursor.seedIndex ?? 0, seeds.length - 1);136 const perRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.categoriesPerRun);137 let count = 0;138 let items = 0;139 for (let k = 0; k < perRun; k++) {140 const si = (start + k) % seeds.length;141 const seed = seeds[si]!;142 for (let page = 1; page <= pages; page++) {143 if (ctx.signal?.aborted || this.reached(ctx, count)) return;144 const url = searchUrl(cred.base, seed, { rows: this.cfg.rows, page, sortOrder: this.cfg.sortOrder, condition: this.cfg.condition, photoSize: this.cfg.photoSize });145 await this.throttle(url);146 const res = await ctx.fetch(url, {147 engines: ['api'],148 responseType: 'json',149 headers: { authorization: cred.header, accept: 'application/json' },150 expect: ['title', 'price', 'currency'],151 parse: (r) => {152 const p = parseSearchResponse(r.json);153 return p ? { title: p.listings[0]?.Title ?? (p.totalCount === 0 ? 'empty' : null), price: p.listings[0]?.StartPrice ?? null, currency: p.listings.length ? 'NZD' : null } : null;154 },155 minQuality: 0.2,156 });157 const parsed = res.success ? parseSearchResponse(res.json) : null;158 if (!parsed) {159 const err = (res.json as { ErrorDescription?: string } | null)?.ErrorDescription;160 ctx.anomaly(res.httpStatus === 429 ? 'rate_limited' : 'page_fetch_failed', `${seed.category} p${page}: ${res.httpStatus} ${err ?? res.error ?? ''}`);161 break;162 }163 if (parsed.rejected) ctx.anomaly('schema_drift', `${parsed.rejected} listings rejected by schema`);164 if (!parsed.listings.length) break;165 count++;166 items += parsed.listings.length;167 const payload: PagePayload = { kind: 'search_page', url, seed, page, totalCount: parsed.totalCount, pageSize: parsed.pageSize, listings: parsed.listings };168 yield { url, externalId: `${seed.category}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };169 await ctx.progress({ page, totalPages: parsed.totalCount ? Math.ceil(parsed.totalCount / this.cfg.rows) : null, itemsProcessed: items });170 if (parsed.listings.length < this.cfg.rows || (parsed.totalCount !== null && page * this.cfg.rows >= parsed.totalCount)) break;171 }172 await ctx.setCursor({ seedIndex: (si + 1) % seeds.length, at: new Date().toISOString() });173 }174 if (backfill) await ctx.setCursor({ seedIndex: 0, done: true, at: new Date().toISOString() });175 }176177 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {178 const p = PagePayloadSchema.parse(raw.payload);179 const out: NormalizedRecord[] = [];180 for (const l of p.listings) {181 const { price, listingType } = listingPrice(l);182 const hint = (p.seed.hint as DeptHint | undefined) ?? hintFromLabel(l.CategoryPath ?? null);183 const categorySlug = p.seed.hint ? (slugFromTitle(l.Title, hint) ?? p.seed.categorySlug) : p.seed.categorySlug;184 const grade = parseGradeFromTitle(l.Title);185 const attrs = Object.fromEntries(l.Attributes.filter((a) => a.Name && a.Value).map((a) => [a.Name!, a.Value!]));186 const attributes = AssetAttributesSchema.parse({187 categorySlug,188 name: l.Title,189 year: safeYear(`${l.Title} ${l.Subtitle ?? ''}`),190 identifiers: { trademe_listing_id: String(l.ListingId) },191 metadata: { category_number: l.Category ?? null, category_path: l.CategoryPath ?? null, start_price: l.StartPrice ?? null, buy_now_price: l.BuyNowPrice ?? null, max_bid: l.MaxBidAmount ?? null, price_display: l.PriceDisplay ?? null, has_reserve: l.HasReserve ?? null, reserve_state: l.ReserveState ?? null, is_classified: l.IsClassified ?? null, is_new: l.IsNew ?? null, attributes: attrs, is_bundle_title: isBundleTitle(l.Title) },192 });193 out.push(194 NormalizedListingSchema.parse({195 kind: 'listing',196 connectorId: this.meta.id,197 sourceId: this.meta.sourceId,198 sourceUrl: `https://www.trademe.co.nz/a/listing/${l.ListingId}`,199 externalId: String(l.ListingId),200 rawTitle: l.Subtitle ? `${l.Title} — ${l.Subtitle}` : l.Title,201 imageUrls: l.PictureHref ? [l.PictureHref] : [],202 attributes,203 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null },204 condition: { condition: null, conditionRaw: l.IsNew === true ? 'New' : l.IsNew === false ? 'Used' : null, completeness: null },205 observedAt: raw.fetchedAt,206 confidence: 0.7,207 parserVersion: PARSER_VERSION,208 listingType,209 price,210 currency: price !== null ? 'NZD' : null,211 location: [l.Suburb, l.Region].filter(Boolean).join(', ') || null,212 listedAt: dotNetDate(l.StartDate),213 endsAt: dotNetDate(l.EndDate),214 availability: 'available',215 bidCount: l.BidCount ?? null,216 }),217 );218 }219 return out;220 }221}222223export default function createConnector(meta: ConnectorMeta): TradeMeConnector {224 return new TradeMeConnector(meta);225}226