import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared'; /** * aucfree — closed Yahoo! Auctions Japan lots (JPY hammer prices) for Japanese collectibles. * One raw record per search-result page; normalise → one sale per row. */ const BASE = 'https://aucfree.com'; const PARSER_VERSION = '1.0.0'; const SeedSchema = z.object({ q: z.string(), category: z.string(), language: z.string().nullable().default(null) }); type Seed = z.infer; export const RowSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), priceJpy: z.number(), bids: z.number().nullable(), endedOn: z.string(), image: z.string().nullable() }); export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), seed: SeedSchema, page: z.number(), rows: z.array(RowSchema) }); export type PagePayload = z.infer; export function parseSearchPage(htmlText: string, url: string, seed: Seed, page: number): PagePayload { const $ = H.load(htmlText); const rows: z.infer[] = []; $('tr.results_bid').each((_, tr) => { const $tr = $(tr); const a = $tr.find('a.item_title').first(); const href = a.attr('href') ?? ''; const id = href.match(/\/items\/([a-z0-9]+)/i)?.[1]; const title = H.text(a); const priceTxt = H.text($tr.find('.item_price').first()) ?? ''; const price = Number(priceTxt.replace(/[^\d]/g, '')); const bidsTxt = H.text($tr.find('td.results-bid').first()); const bids = bidsTxt ? Number(bidsTxt.replace(/[^\d]/g, '')) : null; const endedOn = H.text($tr.find('td.results-limit').first()) ?? ''; const img = $tr.find('.results_bid-image img').attr('data-src') ?? $tr.find('.results_bid-image img').attr('src') ?? null; if (!id || !title || !price || !endedOn) return; rows.push({ id, url: `${BASE}/items/${id}`, title, priceJpy: price, bids: Number.isFinite(bids as number) ? bids : null, endedOn, image: img }); }); return { kind: 'search_page', url, seed, page, rows }; } /** "2026年9月6日" → UTC date */ export function parseJapaneseDate(s: string): Date | null { const m = s.match(/(\d{4})年\s*(\d{1,2})月\s*(\d{1,2})日/); if (!m) return null; return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]))); } /** Normalise "PSA10" / "PSA10" / "BGS9.5" so the shared grade parser can read them. */ export function normaliseGradeText(title: string): string { return title .normalize('NFKC') .replace(/\b(PSA|BGS|CGC|SGC|ARS|ACE)\s*(\d{1,2}(?:\.\d)?)/gi, '$1 $2') .replace(/【|】|\[|\]/g, ' '); } const BUNDLE_RE = /まとめ|セット売り|大量|\d+\s*枚セット|\d+\s*点セット|おまとめ|引退品/; export class AucfreeConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []); const pages = Number(this.meta.config.pagesPerSeed ?? 2); const filter = ctx.options.categories; let count = 0; for (const seed of seeds) { if (filter?.length && !filter.includes(seed.category)) continue; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${BASE}/search?o=t2&q=${encodeURIComponent(seed.q)}${page > 1 ? `&p=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], expect: ['title', 'price', 'date', 'status'], parse: (r) => { const p = r.html ? parseSearchPage(r.html, url, seed, page) : null; const row = p?.rows[0]; return row ? { title: row.title, price: row.priceJpy, date: row.endedOn, status: 'sold' } : null; }, }); const payload = res.success && res.html ? parseSearchPage(res.html, url, seed, page) : null; if (!payload || payload.rows.length === 0) { ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } count++; yield { url, externalId: `search:${seed.q}:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const r of p.rows) { const saleDate = parseJapaneseDate(r.endedOn); if (!saleDate) continue; const cleaned = normaliseGradeText(r.title); const g = parseGradeFromTitle(cleaned); const isBundle = BUNDLE_RE.test(r.title); const attributes = AssetAttributesSchema.parse({ categorySlug: p.seed.category, name: r.title.normalize('NFKC').replace(/\s+/g, ' ').trim(), language: p.seed.language, country: 'JP', identifiers: { yahoo_auction_id: r.id }, metadata: { seed_query: p.seed.q, bids: r.bids }, }); out.push( NormalizedSaleSchema.parse({ kind: 'sale', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: r.url, externalId: r.id, rawTitle: r.title, imageUrls: r.image ? [r.image] : [], attributes, grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.7, parserVersion: PARSER_VERSION, saleType: 'auction', saleDate, price: r.priceJpy, currency: 'JPY', buyerPremiumIncluded: false, quantity: 1, isBundle, location: 'Japan', auctionHouse: 'Yahoo! Auctions Japan', lotNumber: r.id, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new AucfreeConnector(meta); }