TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs, makeTitle } from '../_lib/shared.js';5import { HTML_HEADERS, JSON_HEADERS, cleanText, yen } from '../_g1-cards-eu-jp-lib/index.js';67/**8 * Hareruya (晴れる屋) — Japan's largest MTG shop. The public search page loads its results from a9 * JSON endpoint (unisearch_api); we read that endpoint per card set. One raw record per API page;10 * normalize emits one JPY listing per offer (product × language × condition × foil).11 */12const SITE = 'https://www.hareruyamtg.com';13const API = `${SITE}/en/products/search/unisearch_api`;14const PARSER_VERSION = '1.0.0';1516export const LANGUAGES: Record<string, string> = { '1': 'Japanese', '2': 'English', '3': 'Chinese (Simplified)', '4': 'Chinese (Traditional)', '5': 'French', '6': 'German', '7': 'Italian', '8': 'Korean', '9': 'Russian', '10': 'Spanish', '11': 'Portuguese' };17const LANG_CODE: Record<string, string> = { '1': 'JP', '2': 'EN', '3': 'CS', '4': 'CT', '5': 'FR', '6': 'DE', '7': 'IT', '8': 'KO', '9': 'RU', '10': 'ES', '11': 'PT' };1819const str = z.union([z.string(), z.number()]).transform((v) => String(v));20export const DocSchema = z.object({21 product: str,22 product_name: z.string().nullable().optional(),23 product_name_en: z.string().nullable().optional(),24 card_name: z.string().nullable().optional(),25 language: str.nullable().optional(),26 price: str.nullable().optional(),27 image_url: z.string().nullable().optional(),28 foil_flg: str.nullable().optional(),29 stock: str.nullable().optional(),30 weekly_sales: str.nullable().optional(),31 product_class: str.nullable().optional(),32 card_condition: str.nullable().optional(),33 sale_flg: str.nullable().optional(),34 high_price_code: str.nullable().optional(),35});36export type HareruyaDoc = z.infer<typeof DocSchema>;37const RawPayloadSchema = z.object({ cardset: z.object({ id: z.string(), name: z.string() }), page: z.number().int(), numFound: z.number().int().nullable(), docs: z.array(DocSchema) });38export type HareruyaPayload = z.infer<typeof RawPayloadSchema>;39const ApiResponseSchema = z.object({ response: z.object({ numFound: z.number().int().optional(), page: z.union([z.number(), z.string()]).optional(), docs: z.array(z.unknown()).default([]) }) });4041/** Card-set options of the public search form. */42export function parseCardsets(doc: string): Array<{ id: string; name: string }> {43 const $ = html.load(doc);44 const out: Array<{ id: string; name: string }> = [];45 $('select[name="cardset"] option').each((_, el) => {46 const id = $(el).attr('value')?.trim();47 const name = cleanText($(el).text());48 if (id && name) out.push({ id, name });49 });50 return out;51}5253/** "(001)《Ugin, Eye of the Storms》[TDM]" → { number: "001", name: "Ugin, Eye of the Storms", setCode: "TDM" }; "《Black Lotus》[2ED] 茶R" keeps only those parts. */54export function parseProductName(s: string | null | undefined): { name: string | null; setCode: string | null; number: string | null } {55 if (!s) return { name: null, setCode: null, number: null };56 const name = s.match(/《([^》]+)》/)?.[1]?.trim() ?? null;57 const setCode = s.match(/\[([A-Za-z0-9_+-]+)\]/)?.[1]?.trim() ?? null;58 const number = s.match(/^\s*\(([A-Za-z0-9]{1,4}[-/]?[A-Za-z0-9]{0,4})\)\s*《/)?.[1] ?? null;59 return { name, setCode, number };60}6162export function apiUrl(cardsetId: string, page: number, rows: number): string {63 return `${API}?fq.category_id=1&fq.cardset=${encodeURIComponent(cardsetId)}&fq.price=1%7E%2A&rows=${rows}&page=${page}`;64}6566export class HareruyaConnector extends BaseConnector {67 readonly version = '1.0.0';68 readonly parserVersion = PARSER_VERSION;69 protected override minIntervalMs = 4000;7071 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {72 const rows = Number(this.meta.config.rows ?? 60);73 const backfill = ctx.options.mode === 'backfill';74 const setsPerRun = backfill ? Infinity : Number(this.meta.config.setsPerRun ?? 12);75 const maxPages = Math.min(Number(this.meta.config.maxPagesPerSet ?? 67), Math.ceil(4000 / rows));76 let cardsets = (this.meta.config.cardsets as Array<{ id: string; name: string }> | undefined) ?? [];77 if (!cardsets.length) {78 const formUrl = `${SITE}/en/products/search`;79 await this.throttle(formUrl);80 const form = await ctx.fetch(formUrl, { engines: ['api'], headers: HTML_HEADERS, responseType: 'text', minQuality: 0 });81 if (!form.success || !form.html) throw new Error(`hareruya search form failed: ${form.error ?? form.httpStatus}`);82 cardsets = parseCardsets(form.html);83 if (!cardsets.length) throw new Error('hareruya: no cardset options found (selector_missing)');84 }85 if (ctx.options.seeds?.length) cardsets = cardsets.filter((c) => ctx.options.seeds!.includes(c.id) || ctx.options.seeds!.includes(c.name));86 cardsets = cardsets.slice(0, Number.isFinite(setsPerRun) ? setsPerRun : cardsets.length);87 let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);88 let page = Number(ctx.options.cursor?.page ?? 1);89 let count = 0;90 for (; setIdx < cardsets.length; setIdx++, page = 1) {91 const cardset = cardsets[setIdx]!;92 for (; page <= maxPages; page++) {93 if (ctx.signal?.aborted) return;94 if (this.reached(ctx, count)) {95 await ctx.setCursor({ setIdx, page });96 return;97 }98 const url = apiUrl(cardset.id, page, rows);99 await this.throttle(url);100 const res = await ctx.fetch(url, { engines: ['api'], headers: { ...JSON_HEADERS, referer: `${SITE}/en/products/search?cardset=${cardset.id}` }, minQuality: 0 });101 const parsed = ApiResponseSchema.safeParse(res.json);102 if (!res.success || !parsed.success) {103 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? 'unexpected response shape'}`);104 break;105 }106 const docs: HareruyaDoc[] = [];107 let bad = 0;108 for (const d of parsed.data.response.docs) {109 const p = DocSchema.safeParse(d);110 if (p.success) docs.push(p.data);111 else bad++;112 }113 if (bad) ctx.anomaly('schema_drift', `${url}: ${bad} docs failed DocSchema`);114 const numFound = parsed.data.response.numFound ?? null;115 if (!docs.length) {116 if (page === 1 && (numFound ?? 0) > 0) ctx.anomaly('parse_failure_page', url);117 break;118 }119 count++;120 const payload: HareruyaPayload = { cardset, page, numFound, docs };121 yield { url, externalId: `${cardset.id}:p${page}`, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };122 await ctx.setCursor({ setIdx, page: page + 1 });123 const total = Math.min(numFound ?? 0, 4000);124 await ctx.progress({ page, totalPages: total ? Math.ceil(total / rows) : null, itemsProcessed: count });125 if (numFound !== null && page * rows >= total) break;126 }127 }128 await ctx.setCursor({ setIdx: 0, page: 1, completedAt: new Date().toISOString(), done: true });129 }130131 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {132 const p = RawPayloadSchema.parse(raw.payload);133 const out: NormalizedRecord[] = [];134 const observedAt = raw.fetchedAt;135 const seen = new Set<string>();136 for (const d of p.docs) {137 const price = yen(d.price);138 if (price === null) continue;139 const pn = parseProductName(d.product_name_en ?? d.product_name);140 const name = d.card_name?.trim() || pn.name;141 if (!name) continue;142 const langCode = d.language ?? null;143 const language = langCode ? (LANGUAGES[langCode] ?? null) : null;144 const foil = d.foil_flg === '1';145 const externalId = `${d.product}:${d.product_class ?? `${langCode ?? 'x'}-${d.card_condition ?? 'x'}-${foil ? 'f' : 'n'}`}`;146 if (seen.has(externalId)) continue;147 seen.add(externalId);148 const stock = d.stock !== null && d.stock !== undefined && d.stock !== '' ? Number(d.stock) : null;149 const url = `${SITE}/en/products/detail/${d.product}?lang=${LANG_CODE[langCode ?? ''] ?? 'EN'}${d.high_price_code && d.high_price_code !== '0' && d.product_class ? `&class=${d.product_class}` : ''}`;150 const a = attrs({151 categorySlug: 'magic_the_gathering',152 franchise: 'Magic: The Gathering',153 brand: 'Wizards of the Coast',154 set: p.cardset.name,155 setCode: pn.setCode,156 name,157 number: pn.number,158 variant: foil ? 'Foil' : null,159 language,160 identifiers: { hareruya_product_id: d.product, ...(d.product_class ? { hareruya_product_class: d.product_class } : {}) },161 metadata: { hareruya_cardset_id: p.cardset.id, product_name_ja: d.product_name ?? null, condition_code: d.card_condition ?? null, weekly_sales: d.weekly_sales !== undefined && d.weekly_sales !== null ? Number(d.weekly_sales) : null, on_sale: d.sale_flg === '1', high_price_code: d.high_price_code && d.high_price_code !== '0' ? d.high_price_code : null },162 });163 const rawTitle = `${makeTitle({ name, set: p.cardset.name, number: pn.number, variant: foil ? 'Foil' : null })}${language ? ` (${language})` : ''}${d.card_condition ? ` · cond ${d.card_condition}` : ''}`;164 out.push(165 NormalizedListingSchema.parse({166 kind: 'listing',167 connectorId: this.meta.id,168 sourceId: this.meta.sourceId,169 sourceUrl: url,170 externalId,171 rawTitle,172 imageUrls: d.image_url ? [d.image_url] : [],173 attributes: a,174 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },175 condition: { condition: null, conditionRaw: null, completeness: null },176 observedAt,177 confidence: 0.8,178 parserVersion: PARSER_VERSION,179 listingType: 'fixed_price',180 price,181 currency: 'JPY',182 seller: 'Hareruya',183 location: 'JP',184 quantity: Number.isFinite(stock) ? stock : null,185 availability: stock === 0 ? 'ended' : 'available',186 }),187 );188 }189 return out;190 }191}192193export default (meta: ConnectorMeta) => new HareruyaConnector(meta);194