TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';45/**6 * AmiAmi — Japanese figure/hobby retailer. Uses the storefront's public JSON API via Scrapfly7 * (Cloudflare blocks plain data-centre requests). One raw record per result page.8 */9const API = 'https://api.amiami.com/api/v1.0';10const SITE = 'https://www.amiami.com';11const IMG = 'https://img.amiami.com';12const PARSER_VERSION = '1.0.0';1314const ItemSchema = z.object({15 gcode: z.string(),16 gname: z.string(),17 thumb_url: z.string().nullable().optional(),18 min_price: z.number().nullable().optional(),19 max_price: z.number().nullable().optional(),20 c_price_taxed: z.number().nullable().optional(),21 maker_name: z.string().nullable().optional(),22 condition_flg: z.number().nullable().optional(),23 instock_flg: z.number().nullable().optional(),24 order_closed_flg: z.number().nullable().optional(),25 releasedate: z.string().nullable().optional(),26 jancode: z.string().nullable().optional(),27 preowned_sale_flg: z.number().nullable().optional(),28 resale_flg: z.number().nullable().optional(),29 saleitem: z.number().nullable().optional(),30});31export type AmiAmiItem = z.infer<typeof ItemSchema>;3233export const PagePayloadSchema = z.object({34 kind: z.literal('search_page'),35 params: z.string(),36 categorySlug: z.string(),37 page: z.number(),38 total: z.number().nullable(),39 items: z.array(ItemSchema),40});41export type PagePayload = z.infer<typeof PagePayloadSchema>;4243const KEEP: Array<keyof AmiAmiItem> = ['gcode', 'gname', 'thumb_url', 'min_price', 'max_price', 'c_price_taxed', 'maker_name', 'condition_flg', 'instock_flg', 'order_closed_flg', 'releasedate', 'jancode', 'preowned_sale_flg', 'resale_flg', 'saleitem'];4445export function trimItem(raw: Record<string, unknown>): AmiAmiItem | null {46 const o: Record<string, unknown> = {};47 for (const k of KEEP) if (k in raw) o[k] = raw[k];48 const p = ItemSchema.safeParse(o);49 return p.success ? p.data : null;50}5152/** "[AmiAmi Exclusive Bonus] Azur Lane Glorious Chinese New Year Ver. 1/7 Complete Figure" → clean name + flags */53export function cleanName(gname: string): { name: string; bonus: boolean; scale: string | null } {54 let name = gname.replace(/\[[^\]]*(Bonus|Exclusive|Limited)[^\]]*\]\s*/gi, '').trim();55 const bonus = name !== gname.trim();56 const scale = name.match(/\b(1\/\d{1,3})\b/)?.[1] ?? null;57 name = name.replace(/\s+/g, ' ').trim();58 return { name, bonus, scale };59}6061/** "(Pre-owned ITEM:A/BOX:B)" grades → normalised condition for the boxed_toys scale. */62export function preownedCondition(item: AmiAmiItem): { condition: string | null; raw: string | null } {63 if (item.condition_flg !== 1) return { condition: 'mint_in_box', raw: 'New' };64 const m = item.gname.match(/ITEM:([A-Z][+-]?)\/BOX:([A-Z][+-]?)/i);65 if (!m) return { condition: 'boxed', raw: 'Pre-owned' };66 const itemGrade = m[1]!.toUpperCase();67 const cond = itemGrade.startsWith('A') ? 'near_mint_box' : itemGrade.startsWith('B') ? 'boxed' : 'loose_complete';68 return { condition: cond, raw: `Pre-owned ITEM:${m[1]}/BOX:${m[2]}` };69}7071export function releaseYear(s: string | null | undefined): number | null {72 const m = s?.match(/(\d{4})/);73 return m ? Number(m[1]) : null;74}7576export class AmiAmiConnector extends BaseConnector {77 readonly version = '1.0.0';78 readonly parserVersion = PARSER_VERSION;79 protected override minIntervalMs = 1500;80 override readonly urlPatterns = [/^https?:\/\/(www\.)?amiami\.com\/(eng|jp)\/detail\/?\?.*gcode=([A-Z0-9-]+)/i];8182 private apiOpts() {83 return { engines: ['scrapfly' as const], renderJs: false, country: 'jp', headers: { 'X-User-Key': 'amiami_dev' }, timeoutMs: 60_000 };84 }8586 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {87 const seeds = (this.meta.config.seeds as Array<{ params: string; categorySlug: string }> | undefined) ?? [];88 const pages = Number(this.meta.config.pagesPerSeed ?? 2);89 const pageSize = Number(this.meta.config.pageSize ?? 50);90 const cap = ctx.options.limit;91 let count = 0;92 const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;93 for (let i = startSeed; i < seeds.length; i++) {94 const seed = seeds[i]!;95 for (let page = 1; page <= pages; page++) {96 if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;97 const url = `${API}/items?pagemax=${pageSize}&lang=eng&${seed.params}&pagecnt=${page}`;98 await this.throttle();99 const res = await ctx.fetch(url, {100 ...this.apiOpts(),101 expect: ['title', 'price'],102 parse: (r) => {103 const j = r.json as { items?: unknown[] } | null;104 return j?.items?.length ? { title: 'ok', price: 1 } : null;105 },106 });107 const j = res.json as { items?: Record<string, unknown>[]; search_result?: { total_results?: number } } | null;108 if (!res.success || !j?.items) {109 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);110 break;111 }112 const items = j.items.map(trimItem).filter((x): x is AmiAmiItem => Boolean(x));113 const payload: PagePayload = { kind: 'search_page', params: seed.params, categorySlug: seed.categorySlug, page, total: j.search_result?.total_results ?? null, items };114 count++;115 yield { url, externalId: `${seed.params}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };116 if (items.length < pageSize) break;117 }118 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });119 }120 }121122 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {123 const m = url.match(this.urlPatterns[0]!);124 if (!m) return [];125 const gcode = m[3]!;126 const api = `${API}/item?gcode=${encodeURIComponent(gcode)}&lang=eng`;127 const res = await ctx.fetch(api, { ...this.apiOpts(), minQuality: 0 });128 const j = res.json as { item?: Record<string, unknown> } | null;129 if (!res.success || !j?.item) return [];130 const it = { ...j.item, min_price: j.item.price ?? j.item.min_price, max_price: j.item.price ?? j.item.max_price, c_price_taxed: j.item.list_price ?? j.item.c_price_taxed };131 const item = trimItem(it as Record<string, unknown>);132 if (!item) return [];133 const payload: PagePayload = { kind: 'search_page', params: `gcode=${gcode}`, categorySlug: 'action_figures', page: 1, total: 1, items: [item] };134 return [{ url: api, externalId: gcode, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];135 }136137 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {138 const p = PagePayloadSchema.parse(raw.payload);139 const out: NormalizedRecord[] = [];140 for (const it of p.items) {141 const { name, bonus, scale } = cleanName(it.gname);142 const year = releaseYear(it.releasedate);143 const cond = preownedCondition(it);144 const sourceUrl = `${SITE}/eng/detail/?gcode=${encodeURIComponent(it.gcode)}`;145 const image = it.thumb_url ? `${IMG}${it.thumb_url}` : null;146 const baseCode = it.gcode.replace(/-R$/, ''); // "-R" = pre-owned listing of the same product147 const attributes = AssetAttributesSchema.parse({148 categorySlug: p.categorySlug,149 brand: it.maker_name ?? null,150 name,151 year,152 size: scale,153 originalMsrp: it.c_price_taxed ?? null,154 originalMsrpCurrency: it.c_price_taxed ? 'JPY' : null,155 identifiers: { amiami_gcode: baseCode, ...(it.jancode ? { jan: it.jancode } : {}) },156 metadata: { bonus_edition: bonus, release: it.releasedate ?? null, resale: it.resale_flg === 1 },157 });158 const rawTitle = `${name}${it.maker_name ? ` · ${it.maker_name}` : ''}${year ? ` (${year})` : ''}`;159 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle, imageUrls: image ? [image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };160 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: baseCode, confidence: 0.85, releaseDate: null }));161 const price = it.min_price ?? it.max_price ?? null;162 if (price && price > 0) {163 const available = it.instock_flg === 1 && it.order_closed_flg !== 1;164 out.push(165 NormalizedListingSchema.parse({166 kind: 'listing',167 ...base,168 externalId: it.gcode,169 confidence: 0.85,170 listingType: 'fixed_price',171 price,172 currency: 'JPY',173 seller: 'AmiAmi',174 location: 'Japan',175 condition: { condition: cond.condition, conditionRaw: cond.raw, completeness: it.condition_flg === 1 ? 'boxed' : 'sealed' },176 availability: available ? 'available' : 'sold',177 quantity: null,178 }),179 );180 }181 }182 return out;183 }184}185186export default function createConnector(meta: ConnectorMeta) {187 return new AmiAmiConnector(meta);188}189