TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../_lib/shared.js';5import { HTML_HEADERS, JP_EXCLUDE_RE, cleanText, isJpBundle, jpCondition, jpVariant, jsonLdObjects, parseJpCardTitle, yen } from '../_g1-cards-eu-jp-lib/index.js';67/**8 * magi (magi.camp) — Japanese card marketplace. Series browse pages → item cards with title, JPY9 * price and SOLD marker; item pages carry a schema.org Product JSON-LD (lookup). Listings only.10 */11const SITE = 'https://magi.camp';12const PARSER_VERSION = '1.0.0';1314const SeedSchema = z.object({ series: z.string(), categorySlug: z.string(), name: z.string().optional(), franchise: z.string().nullable().optional() });15export type MagiSeed = z.infer<typeof SeedSchema>;1617export const ItemSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), priceJpy: z.number().nullable(), sold: z.boolean(), image: z.string().nullable(), certifiedSeller: z.boolean(), favorites: z.number().int().nullable() });18export type MagiItem = z.infer<typeof ItemSchema>;19const ListPayloadSchema = z.object({ type: z.literal('list'), seed: SeedSchema, page: z.number().int(), items: z.array(ItemSchema) });20const ItemPayloadSchema = z.object({ type: z.literal('item'), seed: SeedSchema.nullable(), id: z.string(), url: z.string(), title: z.string(), description: z.string().nullable(), image: z.string().nullable(), priceJpy: z.number().nullable(), availability: z.string().nullable(), condition: z.string().nullable() });21const RawPayloadSchema = z.union([ListPayloadSchema, ItemPayloadSchema]);22export type MagiPayload = z.infer<typeof RawPayloadSchema>;2324/** Parse a series browse page (/series/<id>/items?page=n). */25export function parseSeriesPage(doc: string): { items: MagiItem[]; hasNext: boolean; heading: string | null } {26 const $ = html.load(doc);27 const items: MagiItem[] = [];28 $('.item-list__box').each((_, el) => {29 const $el = $(el);30 const a = $el.find('a.item-list__link').first();31 const href = a.attr('href');32 const id = href?.match(/\/items\/(\d+)/)?.[1];33 const title = cleanText($el.find('.item-list__item-name').first().text());34 if (!href || !id || !title) return;35 const priceJpy = yen(cleanText($el.find('.item-list__price-box--price').first().text()));36 const sold = $el.find('.item-list__sold-icon').length > 0;37 const img = $el.find('img').first();38 const image = img.attr('data-src') ?? (img.attr('src')?.includes('lazy-dummy') ? null : img.attr('src')) ?? null;39 const certifiedSeller = $el.find('.item-list__badge img[alt*="認定"]').length > 0;40 const fav = cleanText($el.find('.item-list__price-box--favorite-number').first().text());41 items.push({ id, url: href.startsWith('http') ? href : `${SITE}${href}`, title, priceJpy, sold, image, certifiedSeller, favorites: fav ? Number(fav.replace(/\D/g, '')) || null : null });42 });43 const hasNext = $('a[href*="page="]').toArray().some((a) => /[?&]page=\d+/.test($(a).attr('href') ?? '') && /次|next|›|»/i.test($(a).text() + ($(a).attr('rel') ?? ''))) || $('a[rel="next"]').length > 0;44 const headings = $('h1').toArray().map((h) => cleanText($(h).text())).filter((t): t is string => Boolean(t));45 const heading = headings.find((t) => /出品/.test(t)) ?? headings[0] ?? null;46 return { items, hasNext: hasNext || items.length >= 96, heading };47}4849/** Item page: schema.org Product JSON-LD + title. */50export function parseItemPage(doc: string, url: string): Omit<z.infer<typeof ItemPayloadSchema>, 'type' | 'seed'> | null {51 const ld = jsonLdObjects(doc, 'Product')[0] as { sku?: string; name?: string; description?: string; image?: string; offers?: { price?: string | number; availability?: string; itemCondition?: string } } | undefined;52 const id = url.match(/\/items\/(\d+)/)?.[1] ?? ld?.sku;53 if (!id) return null;54 const $ = html.load(doc);55 const title = cleanText(ld?.name) ?? cleanText($('.item-detail-title h2, h2[aria-label="商品名"]').first().text()) ?? cleanText($('title').first().text()?.split('|')[0]?.replace(/の通販.*$/, ''));56 if (!title) return null;57 const priceJpy = yen(ld?.offers?.price ?? cleanText($('.item-detail__price').first().text()));58 const soldOut = $('.item-sold-out, .item-badge--sold').length > 0;59 return { id, url, title, description: cleanText(ld?.description) ?? null, image: ld?.image ?? null, priceJpy, availability: ld?.offers?.availability ?? (soldOut ? 'https://schema.org/OutOfStock' : null), condition: ld?.offers?.itemCondition ?? null };60}6162export class MagiConnector extends BaseConnector {63 readonly version = '1.0.0';64 readonly parserVersion = PARSER_VERSION;65 protected override minIntervalMs = 4000;66 override readonly urlPatterns = [/^https?:\/\/(www\.)?magi\.camp\/items\/\d+/];6768 private seeds(): MagiSeed[] {69 return z.array(SeedSchema).parse(this.meta.config.seeds ?? []);70 }7172 private async page(ctx: CrawlContext, url: string) {73 await this.throttle(url);74 return ctx.fetch(url, { engines: ['api'], headers: HTML_HEADERS, responseType: 'text', minQuality: 0 });75 }7677 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {78 let seeds = this.seeds();79 if (ctx.options.seeds?.length) seeds = seeds.filter((s) => ctx.options.seeds!.includes(s.series) || ctx.options.seeds!.includes(s.categorySlug));80 const backfill = ctx.options.mode === 'backfill';81 const maxPages = backfill ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 3);82 let seedIdx = Number(ctx.options.cursor?.seedIdx ?? 0);83 let page = Number(ctx.options.cursor?.page ?? 1);84 let count = 0;85 for (; seedIdx < seeds.length; seedIdx++, page = 1) {86 const seed = seeds[seedIdx]!;87 for (; page <= maxPages; page++) {88 if (ctx.signal?.aborted) return;89 if (this.reached(ctx, count)) {90 await ctx.setCursor({ seedIdx, page });91 return;92 }93 const url = `${SITE}/series/${seed.series}/items${page > 1 ? `?page=${page}` : ''}`;94 const res = await this.page(ctx, url);95 if (!res.success || !res.html) {96 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);97 break;98 }99 const { items, hasNext } = parseSeriesPage(res.html);100 if (!items.length) {101 if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no .item-list__box`);102 break;103 }104 count++;105 const payload: MagiPayload = { type: 'list', seed, page, items };106 yield { url, externalId: `series:${seed.series}:p${page}`, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };107 await ctx.setCursor({ seedIdx, page: page + 1 });108 await ctx.progress({ page, totalPages: null, itemsProcessed: count });109 if (!hasNext) break;110 }111 }112 await ctx.setCursor({ seedIdx: 0, page: 1, completedAt: new Date().toISOString(), done: true });113 }114115 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {116 if (!this.urlPatterns.some((re) => re.test(url))) return [];117 const res = await this.page(ctx, url);118 if (!res.success || !res.html) return [];119 const item = parseItemPage(res.html, url);120 if (!item) return [];121 const $ = html.load(res.html);122 const seriesId = $('a[href^="/series/"]').first().attr('href')?.match(/\/series\/(\d+)/)?.[1];123 const seed = this.seeds().find((s) => s.series === seriesId) ?? null;124 const payload: MagiPayload = { type: 'item', seed, ...item };125 return [{ url, externalId: item.id, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];126 }127128 private listing(seed: MagiSeed | null, it: { id: string; url: string; title: string; priceJpy: number | null; sold: boolean; image: string | null; certifiedSeller: boolean | null; favorites: number | null; description?: string | null }, observedAt: Date): NormalizedRecord | null {129 if (it.priceJpy === null) return null;130 if (JP_EXCLUDE_RE.test(it.title)) return null;131 const t = parseJpCardTitle(it.title);132 const categorySlug = seed?.categorySlug ?? 'trading_cards';133 const language = t.language ?? (t.notes.some((n) => /英語/.test(n)) ? 'English' : 'Japanese');134 const isBundle = isJpBundle(it.title, t.quantity);135 const a = attrs({136 categorySlug,137 franchise: seed?.franchise ?? null,138 name: t.name,139 number: t.number,140 variant: jpVariant(t.notes, t.name),141 language,142 rarity: t.rarity,143 identifiers: { magi_item_id: it.id },144 metadata: { total: t.total, notes: t.notes, quantity: t.quantity, certified_seller: it.certifiedSeller, favorites: it.favorites, series: seed?.series ?? null, sealed: t.sealed },145 });146 return NormalizedListingSchema.parse({147 kind: 'listing',148 connectorId: this.meta.id,149 sourceId: this.meta.sourceId,150 sourceUrl: it.url,151 externalId: it.id,152 rawTitle: it.title,153 description: it.description ?? null,154 imageUrls: it.image ? [it.image] : [],155 attributes: a,156 grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier, certificationNumber: null },157 condition: { condition: jpCondition(t.conditionRaw), conditionRaw: t.conditionRaw, completeness: t.sealed ? 'sealed' : null },158 observedAt,159 confidence: t.grade ? 0.75 : 0.65,160 parserVersion: PARSER_VERSION,161 listingType: 'fixed_price',162 price: it.priceJpy,163 currency: 'JPY',164 seller: null,165 location: 'JP',166 quantity: isBundle ? t.quantity : 1,167 availability: it.sold ? 'sold' : 'available',168 });169 }170171 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {172 const p = RawPayloadSchema.parse(raw.payload);173 const out: NormalizedRecord[] = [];174 if (p.type === 'list') {175 const seen = new Set<string>();176 for (const it of p.items) {177 if (seen.has(it.id)) continue;178 seen.add(it.id);179 const l = this.listing(p.seed, it, raw.fetchedAt);180 if (l) out.push(l);181 }182 return out;183 }184 const sold = /OutOfStock|SoldOut/i.test(p.availability ?? '');185 const l = this.listing(p.seed, { id: p.id, url: p.url, title: p.title, priceJpy: p.priceJpy, sold, image: p.image, certifiedSeller: null, favorites: null, description: p.description }, raw.fetchedAt);186 if (l) out.push(l);187 return out;188 }189}190191export default (meta: ConnectorMeta) => new MagiConnector(meta);192