TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseGradeFromTitle } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';56/**7 * aucfree — closed Yahoo! Auctions Japan lots (JPY hammer prices) for Japanese collectibles.8 * One raw record per search-result page; normalise → one sale per row.9 */1011const BASE = 'https://aucfree.com';12const PARSER_VERSION = '1.0.0';1314const SeedSchema = z.object({ q: z.string(), category: z.string(), language: z.string().nullable().default(null) });15type Seed = z.infer<typeof SeedSchema>;1617export const RowSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), priceJpy: z.number(), bids: z.number().nullable(), endedOn: z.string(), image: z.string().nullable() });18export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), seed: SeedSchema, page: z.number(), rows: z.array(RowSchema) });19export type PagePayload = z.infer<typeof PagePayloadSchema>;2021export function parseSearchPage(htmlText: string, url: string, seed: Seed, page: number): PagePayload {22 const $ = H.load(htmlText);23 const rows: z.infer<typeof RowSchema>[] = [];24 $('tr.results_bid').each((_, tr) => {25 const $tr = $(tr);26 const a = $tr.find('a.item_title').first();27 const href = a.attr('href') ?? '';28 const id = href.match(/\/items\/([a-z0-9]+)/i)?.[1];29 const title = H.text(a);30 const priceTxt = H.text($tr.find('.item_price').first()) ?? '';31 const price = Number(priceTxt.replace(/[^\d]/g, ''));32 const bidsTxt = H.text($tr.find('td.results-bid').first());33 const bids = bidsTxt ? Number(bidsTxt.replace(/[^\d]/g, '')) : null;34 const endedOn = H.text($tr.find('td.results-limit').first()) ?? '';35 const img = $tr.find('.results_bid-image img').attr('data-src') ?? $tr.find('.results_bid-image img').attr('src') ?? null;36 if (!id || !title || !price || !endedOn) return;37 rows.push({ id, url: `${BASE}/items/${id}`, title, priceJpy: price, bids: Number.isFinite(bids as number) ? bids : null, endedOn, image: img });38 });39 return { kind: 'search_page', url, seed, page, rows };40}4142/** "2026年9月6日" → UTC date */43export function parseJapaneseDate(s: string): Date | null {44 const m = s.match(/(\d{4})年\s*(\d{1,2})月\s*(\d{1,2})日/);45 if (!m) return null;46 return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])));47}4849/** Normalise "PSA10" / "ＰＳＡ１０" / "BGS9.5" so the shared grade parser can read them. */50export function normaliseGradeText(title: string): string {51 return title52 .normalize('NFKC')53 .replace(/\b(PSA|BGS|CGC|SGC|ARS|ACE)\s*(\d{1,2}(?:\.\d)?)/gi, '$1 $2')54 .replace(/【|】|\[|\]/g, ' ');55}5657const BUNDLE_RE = /まとめ|セット売り|大量|\d+\s*枚セット|\d+\s*点セット|おまとめ|引退品/;5859export class AucfreeConnector extends BaseConnector {60 readonly version = '1.0.0';61 readonly parserVersion = PARSER_VERSION;62 protected override minIntervalMs = 2000;6364 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {65 const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []);66 const pages = Number(this.meta.config.pagesPerSeed ?? 2);67 const filter = ctx.options.categories;68 let count = 0;69 for (const seed of seeds) {70 if (filter?.length && !filter.includes(seed.category)) continue;71 for (let page = 1; page <= pages; page++) {72 if (ctx.signal?.aborted || this.reached(ctx, count)) return;73 const url = `${BASE}/search?o=t2&q=${encodeURIComponent(seed.q)}${page > 1 ? `&p=${page}` : ''}`;74 await this.throttle();75 const res = await ctx.fetch(url, {76 engines: ['firecrawl', 'scrapfly'],77 expect: ['title', 'price', 'date', 'status'],78 parse: (r) => {79 const p = r.html ? parseSearchPage(r.html, url, seed, page) : null;80 const row = p?.rows[0];81 return row ? { title: row.title, price: row.priceJpy, date: row.endedOn, status: 'sold' } : null;82 },83 });84 const payload = res.success && res.html ? parseSearchPage(res.html, url, seed, page) : null;85 if (!payload || payload.rows.length === 0) {86 ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);87 break;88 }89 count++;90 yield { url, externalId: `search:${seed.q}:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };91 }92 }93 }9495 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {96 const p = PagePayloadSchema.parse(raw.payload);97 const out: NormalizedRecord[] = [];98 for (const r of p.rows) {99 const saleDate = parseJapaneseDate(r.endedOn);100 if (!saleDate) continue;101 const cleaned = normaliseGradeText(r.title);102 const g = parseGradeFromTitle(cleaned);103 const isBundle = BUNDLE_RE.test(r.title);104 const attributes = AssetAttributesSchema.parse({105 categorySlug: p.seed.category,106 name: r.title.normalize('NFKC').replace(/\s+/g, ' ').trim(),107 language: p.seed.language,108 country: 'JP',109 identifiers: { yahoo_auction_id: r.id },110 metadata: { seed_query: p.seed.q, bids: r.bids },111 });112 out.push(113 NormalizedSaleSchema.parse({114 kind: 'sale',115 connectorId: this.meta.id,116 sourceId: this.meta.sourceId,117 sourceUrl: r.url,118 externalId: r.id,119 rawTitle: r.title,120 imageUrls: r.image ? [r.image] : [],121 attributes,122 grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier, certificationNumber: null },123 condition: { condition: null, conditionRaw: null, completeness: null },124 observedAt: raw.fetchedAt,125 confidence: 0.7,126 parserVersion: PARSER_VERSION,127 saleType: 'auction',128 saleDate,129 price: r.priceJpy,130 currency: 'JPY',131 buyerPremiumIncluded: false,132 quantity: 1,133 isBundle,134 location: 'Japan',135 auctionHouse: 'Yahoo! Auctions Japan',136 lotNumber: r.id,137 }),138 );139 }140 return out;141 }142}143144export default function createConnector(meta: ConnectorMeta) {145 return new AucfreeConnector(meta);146}147