import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; /** * AmiAmi โ€” Japanese figure/hobby retailer. Uses the storefront's public JSON API via Scrapfly * (Cloudflare blocks plain data-centre requests). One raw record per result page. */ const API = 'https://api.amiami.com/api/v1.0'; const SITE = 'https://www.amiami.com'; const IMG = 'https://img.amiami.com'; const PARSER_VERSION = '1.0.0'; const ItemSchema = z.object({ gcode: z.string(), gname: z.string(), thumb_url: z.string().nullable().optional(), min_price: z.number().nullable().optional(), max_price: z.number().nullable().optional(), c_price_taxed: z.number().nullable().optional(), maker_name: z.string().nullable().optional(), condition_flg: z.number().nullable().optional(), instock_flg: z.number().nullable().optional(), order_closed_flg: z.number().nullable().optional(), releasedate: z.string().nullable().optional(), jancode: z.string().nullable().optional(), preowned_sale_flg: z.number().nullable().optional(), resale_flg: z.number().nullable().optional(), saleitem: z.number().nullable().optional(), }); export type AmiAmiItem = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), params: z.string(), categorySlug: z.string(), page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema), }); export type PagePayload = z.infer; const KEEP: Array = ['gcode', 'gname', 'thumb_url', 'min_price', 'max_price', 'c_price_taxed', 'maker_name', 'condition_flg', 'instock_flg', 'order_closed_flg', 'releasedate', 'jancode', 'preowned_sale_flg', 'resale_flg', 'saleitem']; export function trimItem(raw: Record): AmiAmiItem | null { const o: Record = {}; for (const k of KEEP) if (k in raw) o[k] = raw[k]; const p = ItemSchema.safeParse(o); return p.success ? p.data : null; } /** "[AmiAmi Exclusive Bonus] Azur Lane Glorious Chinese New Year Ver. 1/7 Complete Figure" โ†’ clean name + flags */ export function cleanName(gname: string): { name: string; bonus: boolean; scale: string | null } { let name = gname.replace(/\[[^\]]*(Bonus|Exclusive|Limited)[^\]]*\]\s*/gi, '').trim(); const bonus = name !== gname.trim(); const scale = name.match(/\b(1\/\d{1,3})\b/)?.[1] ?? null; name = name.replace(/\s+/g, ' ').trim(); return { name, bonus, scale }; } /** "(Pre-owned ITEM:A/BOX:B)" grades โ†’ normalised condition for the boxed_toys scale. */ export function preownedCondition(item: AmiAmiItem): { condition: string | null; raw: string | null } { if (item.condition_flg !== 1) return { condition: 'mint_in_box', raw: 'New' }; const m = item.gname.match(/ITEM:([A-Z][+-]?)\/BOX:([A-Z][+-]?)/i); if (!m) return { condition: 'boxed', raw: 'Pre-owned' }; const itemGrade = m[1]!.toUpperCase(); const cond = itemGrade.startsWith('A') ? 'near_mint_box' : itemGrade.startsWith('B') ? 'boxed' : 'loose_complete'; return { condition: cond, raw: `Pre-owned ITEM:${m[1]}/BOX:${m[2]}` }; } export function releaseYear(s: string | null | undefined): number | null { const m = s?.match(/(\d{4})/); return m ? Number(m[1]) : null; } export class AmiAmiConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/(www\.)?amiami\.com\/(eng|jp)\/detail\/?\?.*gcode=([A-Z0-9-]+)/i]; private apiOpts() { return { engines: ['scrapfly' as const], renderJs: false, country: 'jp', headers: { 'X-User-Key': 'amiami_dev' }, timeoutMs: 60_000 }; } async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.seeds as Array<{ params: string; categorySlug: string }> | undefined) ?? []; const pages = Number(this.meta.config.pagesPerSeed ?? 2); const pageSize = Number(this.meta.config.pageSize ?? 50); const cap = ctx.options.limit; let count = 0; const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; for (let i = startSeed; i < seeds.length; i++) { const seed = seeds[i]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return; const url = `${API}/items?pagemax=${pageSize}&lang=eng&${seed.params}&pagecnt=${page}`; await this.throttle(); const res = await ctx.fetch(url, { ...this.apiOpts(), expect: ['title', 'price'], parse: (r) => { const j = r.json as { items?: unknown[] } | null; return j?.items?.length ? { title: 'ok', price: 1 } : null; }, }); const j = res.json as { items?: Record[]; search_result?: { total_results?: number } } | null; if (!res.success || !j?.items) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const items = j.items.map(trimItem).filter((x): x is AmiAmiItem => Boolean(x)); const payload: PagePayload = { kind: 'search_page', params: seed.params, categorySlug: seed.categorySlug, page, total: j.search_result?.total_results ?? null, items }; count++; yield { url, externalId: `${seed.params}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (items.length < pageSize) break; } await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async lookup(url: string, ctx: CrawlContext): Promise { const m = url.match(this.urlPatterns[0]!); if (!m) return []; const gcode = m[3]!; const api = `${API}/item?gcode=${encodeURIComponent(gcode)}&lang=eng`; const res = await ctx.fetch(api, { ...this.apiOpts(), minQuality: 0 }); const j = res.json as { item?: Record } | null; if (!res.success || !j?.item) return []; const it = { ...j.item, min_price: j.item.price ?? j.item.min_price, max_price: j.item.price ?? j.item.max_price, c_price_taxed: j.item.list_price ?? j.item.c_price_taxed }; const item = trimItem(it as Record); if (!item) return []; const payload: PagePayload = { kind: 'search_page', params: `gcode=${gcode}`, categorySlug: 'action_figures', page: 1, total: 1, items: [item] }; return [{ url: api, externalId: gcode, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const it of p.items) { const { name, bonus, scale } = cleanName(it.gname); const year = releaseYear(it.releasedate); const cond = preownedCondition(it); const sourceUrl = `${SITE}/eng/detail/?gcode=${encodeURIComponent(it.gcode)}`; const image = it.thumb_url ? `${IMG}${it.thumb_url}` : null; const baseCode = it.gcode.replace(/-R$/, ''); // "-R" = pre-owned listing of the same product const attributes = AssetAttributesSchema.parse({ categorySlug: p.categorySlug, brand: it.maker_name ?? null, name, year, size: scale, originalMsrp: it.c_price_taxed ?? null, originalMsrpCurrency: it.c_price_taxed ? 'JPY' : null, identifiers: { amiami_gcode: baseCode, ...(it.jancode ? { jan: it.jancode } : {}) }, metadata: { bonus_edition: bonus, release: it.releasedate ?? null, resale: it.resale_flg === 1 }, }); const rawTitle = `${name}${it.maker_name ? ` ยท ${it.maker_name}` : ''}${year ? ` (${year})` : ''}`; const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle, imageUrls: image ? [image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: baseCode, confidence: 0.85, releaseDate: null })); const price = it.min_price ?? it.max_price ?? null; if (price && price > 0) { const available = it.instock_flg === 1 && it.order_closed_flg !== 1; out.push( NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: it.gcode, confidence: 0.85, listingType: 'fixed_price', price, currency: 'JPY', seller: 'AmiAmi', location: 'Japan', condition: { condition: cond.condition, conditionRaw: cond.raw, completeness: it.condition_flg === 1 ? 'boxed' : 'sealed' }, availability: available ? 'available' : 'sold', quantity: null, }), ); } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new AmiAmiConnector(meta); }