SPB Git forge
spboucher.ai ↗

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%

Connectors: JP market, official LEGO store, coin dealers, FossilEra (7 sources, agent N)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 17 days ago (Sep 7, 2026) parent 0962122

35 changed files +2,624 −0

added connectors/api/fossilera/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('fossilera', 'dinosaur-teeth-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['listing'], requiredFields: ['attributes.identifiers.fossilera_item'] }, note: 'Captured live from fossilera.com dinosaur teeth category, trimmed to 8 specimens' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/api/fossilera/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseCategoryPage, parseSpecimenPage } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('fossilera', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits one USD listing per unique specimen with the FossilEra item number', async () => {
13 + const fx = loadFixture('fossilera', 'dinosaur-teeth-p1');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThanOrEqual(8);
16 + for (const l of out) {
17 + if (l.kind !== 'listing') throw new Error('expected listing');
18 + expect(l.attributes.categorySlug).toBe('fossils');
19 + expect(l.attributes.identifiers.fossilera_item).toMatch(/^\d+$/);
20 + expect(l.currency).toBe('USD');
21 + expect(l.seller).toBe('FossilEra');
22 + expect(l.sourceUrl).toMatch(/fossilera\.com\/fossils\//);
23 + expect(l.attributes.metadata.unique_specimen).toBe(true);
24 + }
25 + const reduced = out.find((l) => l.kind === 'listing' && l.attributes.metadata.previous_price_usd);
26 + expect(reduced).toBeTruthy();
27 + });
28 +
29 + it('parsers handle cards, reduced prices and specimen details', () => {
30 + const html = `<a href="/fossils/x-tooth"><div class="image"><img alt="X Tooth #123" src="//assets0.fossilera.com/x.jpg"></div><div class="info"> 3.1" X Tooth - Montana <div class="price"><span class="old-price">$300</span> $250</div></div></a>`;
31 + const p = parseCategoryPage(html, 'u', 'fossils');
32 + expect(p.items).toHaveLength(1);
33 + expect(p.items[0]).toMatchObject({ item: '123', price: 250, oldPrice: 300, sold: false, title: '3.1" X Tooth - Montana', image: 'https://assets0.fossilera.com/x.jpg' });
34 + const detail = `<html><body><h1>Big Tooth</h1><div class="price">$1,000</div><div class="detail">SPECIES Tyrannosaurus rex</div><div class="detail">AGE Late Cretaceous</div><div class="detail">LOCATION Garfield County, Montana</div><div class="detail">FORMATION Hell Creek Formation</div><div class="detail">SIZE 3.95" tooth</div>ITEM #355472</body></html>`;
35 + const d = parseSpecimenPage(detail, 'https://www.fossilera.com/fossils/big-tooth', 'fossils');
36 + expect(d?.items[0]).toMatchObject({ item: '355472', price: 1000, species: 'Tyrannosaurus rex', formation: 'Hell Creek Formation', location: 'Garfield County, Montana', size: '3.95" tooth' });
37 + });
38 +});
added connectors/api/fossilera/index.ts +153 −0
@@ -0,0 +1,153 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const BASE = 'https://www.fossilera.com';
6 +const PARSER_VERSION = '1.0.0';
7 +
8 +export const SpecimenSchema = z.object({
9 + item: z.string(),
10 + url: z.string(),
11 + title: z.string(),
12 + price: z.number().nullable(),
13 + oldPrice: z.number().nullable(),
14 + sold: z.boolean(),
15 + image: z.string().nullable(),
16 + species: z.string().nullable().default(null),
17 + age: z.string().nullable().default(null),
18 + location: z.string().nullable().default(null),
19 + formation: z.string().nullable().default(null),
20 + size: z.string().nullable().default(null),
21 + category: z.string().nullable().default(null),
22 + subCategory: z.string().nullable().default(null),
23 +});
24 +export type Specimen = z.infer<typeof SpecimenSchema>;
25 +export const PagePayloadSchema = z.object({ kind: z.enum(['category_page', 'specimen_page']), url: z.string(), categorySlug: z.string(), items: z.array(SpecimenSchema) });
26 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
27 +
28 +const abs = (u: string | null | undefined) => (u ? (u.startsWith('//') ? `https:${u}` : u.startsWith('http') ? u : BASE + u) : null);
29 +
30 +export function parseCategoryPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
31 + const $ = H.load(htmlText);
32 + const items: Specimen[] = [];
33 + $('a[href^="/fossils/"], a[href^="/minerals/"], a[href^="/meteorites/"]').each((_, el) => {
34 + const a = $(el);
35 + if (!a.find('.info').length) return;
36 + const url = BASE + a.attr('href')!;
37 + const alt = a.find('img').attr('alt') ?? '';
38 + const item = alt.match(/#(\d+)\s*$/)?.[1] ?? url.split('/').pop()!;
39 + const info = a.find('.info').clone();
40 + const priceEl = info.find('.price');
41 + const oldPrice = parsePrice(H.text(priceEl.find('.old-price')), 'USD')?.amount ?? null;
42 + priceEl.find('.old-price').remove();
43 + const priceText = H.text(priceEl) ?? '';
44 + const sold = /sold/i.test(priceText);
45 + const price = sold ? null : (parsePrice(priceText, 'USD')?.amount ?? null);
46 + priceEl.remove();
47 + const title = H.text(info) ?? alt.replace(/\s*#\d+\s*$/, '');
48 + if (!title) return;
49 + items.push({ item, url, title, price, oldPrice, sold, image: abs(a.find('img').attr('src')), species: null, age: null, location: null, formation: null, size: null, category: null, subCategory: null });
50 + });
51 + return { kind: 'category_page', url: pageUrl, categorySlug, items };
52 +}
53 +
54 +export function parseSpecimenPage(htmlText: string, url: string, categorySlug: string): PagePayload | null {
55 + const $ = H.load(htmlText);
56 + const title = H.text($('h1').first());
57 + if (!title) return null;
58 + const detail = (label: string) => {
59 + let v: string | null = null;
60 + $('[class*="detail"]').each((_, el) => {
61 + const t = $(el).text().replace(/\s+/g, ' ').trim();
62 + const m = t.match(new RegExp(`^${label}\\s+(.+)$`, 'i'));
63 + if (m && !v) v = m[1]!.trim();
64 + });
65 + return v;
66 + };
67 + const body = $('body').text().replace(/\s+/g, ' ');
68 + const item = body.match(/ITEM\s*#\s*(\d+)/i)?.[1] ?? url.split('/').pop()!;
69 + const priceBox = $('.price').first();
70 + const oldPrice = parsePrice(H.text(priceBox.find('.old-price')), 'USD')?.amount ?? null;
71 + const priceText = priceBox.clone().find('.old-price').remove().end().text();
72 + const sold = /this (specimen|item) (has been|was) sold|sold out/i.test(body);
73 + const price = sold ? null : (parsePrice(priceText, 'USD')?.amount ?? null);
74 + return {
75 + kind: 'specimen_page',
76 + url,
77 + categorySlug,
78 + items: [{ item, url, title, price, oldPrice, sold, image: abs($('meta[property="og:image"]').attr('content')), species: detail('SPECIES'), age: detail('AGE'), location: detail('LOCATION'), formation: detail('FORMATION'), size: detail('SIZE'), category: detail('CATEGORY'), subCategory: detail('SUB CATEGORY') }],
79 + };
80 +}
81 +
82 +export class FossilEraConnector extends BaseConnector {
83 + readonly version = '1.0.0';
84 + readonly parserVersion = PARSER_VERSION;
85 + protected override minIntervalMs = 1500;
86 + override readonly urlPatterns = [/^https?:\/\/(www\.)?fossilera\.com\/(fossils|minerals|meteorites)\/[a-z0-9-]+/i];
87 +
88 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
89 + const seeds = (this.meta.config.seeds as Array<{ path: string; categorySlug: string }> | undefined) ?? [];
90 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
91 + const cap = ctx.options.limit;
92 + let count = 0;
93 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
94 + for (let i = start; i < seeds.length; i++) {
95 + const seed = seeds[i]!;
96 + for (let page = 1; page <= pages; page++) {
97 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
98 + const url = `${BASE}${seed.path}${page > 1 ? `?page=${page}` : ''}`;
99 + await this.throttle();
100 + const res = await ctx.fetch(url, {
101 + engines: ['api', 'firecrawl'],
102 + responseType: 'text',
103 + expect: ['title', 'price'],
104 + parse: (r) => {
105 + const p = r.html ? parseCategoryPage(r.html, url, seed.categorySlug) : null;
106 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
107 + },
108 + });
109 + const payload = res.success && res.html ? parseCategoryPage(res.html, url, seed.categorySlug) : null;
110 + if (!payload?.items.length) {
111 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no specimen cards'}`);
112 + break;
113 + }
114 + count++;
115 + yield { url, externalId: `${seed.path}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
116 + }
117 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
118 + }
119 + }
120 +
121 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
122 + const m = url.match(this.urlPatterns[0]!);
123 + if (!m) return [];
124 + const slug = m[2]!.toLowerCase() === 'minerals' ? 'minerals' : m[2]!.toLowerCase() === 'meteorites' ? 'meteorites' : 'fossils';
125 + const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0 });
126 + const payload = res.success && res.html ? parseSpecimenPage(res.html, url, slug) : null;
127 + if (!payload) return [];
128 + return [{ url, externalId: payload.items[0]!.item, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
129 + }
130 +
131 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
132 + const p = PagePayloadSchema.parse(raw.payload);
133 + const out: NormalizedRecord[] = [];
134 + for (const s of p.items) {
135 + const loc = s.location ?? s.title.match(/ - ([A-Z][A-Za-z .]+)$/)?.[1] ?? null;
136 + const attributes = AssetAttributesSchema.parse({
137 + categorySlug: p.categorySlug,
138 + name: s.title,
139 + size: s.size ?? s.title.match(/(\d+(?:\.\d+)?")/)?.[1] ?? null,
140 + country: loc,
141 + identifiers: { fossilera_item: s.item },
142 + metadata: { species: s.species, geological_age: s.age, formation: s.formation, locality: s.location, category: s.category, sub_category: s.subCategory, previous_price_usd: s.oldPrice, dealer_guarantee: 'FossilEra authenticity guarantee (dealer statement)', unique_specimen: true },
143 + });
144 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: s.url, rawTitle: s.title, imageUrls: s.image ? [s.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
145 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: s.item, confidence: 0.85, listingType: 'fixed_price', price: s.price, currency: 'USD', seller: 'FossilEra', location: 'US', availability: s.sold ? 'sold' : s.price ? 'available' : 'unknown' }));
146 + }
147 + return out;
148 + }
149 +}
150 +
151 +export default function createConnector(meta: ConnectorMeta) {
152 + return new FossilEraConnector(meta);
153 +}
added connectors/api/fossilera/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "fossilera",
3 + "displayName": "FossilEra (fossils, minerals & meteorites dealer, USD)",
4 + "sourceId": "fossilera",
5 + "sourceName": "FossilEra",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.fossilera.com",
8 + "module": "api/fossilera",
9 + "enginePriority": ["api", "firecrawl"],
10 + "categories": ["fossils", "minerals", "meteorites"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.fossilera.com/pages/terms-of-service",
26 + "accessNotes": "Public category pages (fossilera.com/fossils-for-sale/<category>?page=N) and specimen pages fetched over plain HTTPS with the RareIndex user agent (robots.txt only disallows gift-card/checkout helpers). Each card gives the specimen title, price (and previous price when reduced), image and the FossilEra item number; specimen pages add species, geological age, location, formation, size and category. Every specimen is unique, so each becomes its own asset with one fixed-price listing (available or, when the page states it, sold — no sale date is published, so sold items are listings with availability=sold, never transactions). Compliance: fossilera states all items are legally collected; locality is kept in attributes.country/region so jurisdiction flags (taxonomy compliance 'jurisdiction_check') can be applied downstream.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "path": "/fossils-for-sale/dinosaur-teeth", "categorySlug": "fossils" },
32 + { "path": "/fossils-for-sale/megalodon-teeth", "categorySlug": "fossils" },
33 + { "path": "/fossils-for-sale/ammonites", "categorySlug": "fossils" },
34 + { "path": "/fossils-for-sale/trilobites", "categorySlug": "fossils" },
35 + { "path": "/minerals-for-sale", "categorySlug": "minerals" },
36 + { "path": "/meteorites-for-sale", "categorySlug": "meteorites" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/api/ma-shops/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('ma-shops', 'denarius-search-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['listing'], requiredFields: ['price', 'currency'] }, note: 'Captured live from ma-shops.com search.php?keywords=denarius gallery, trimmed to 8 cells' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/api/ma-shops/index.test.ts +34 −0
@@ -0,0 +1,34 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseMaPrice } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('ma-shops', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits dealer asks with the displayed currency and parsed grades', async () => {
13 + const fx = loadFixture('ma-shops', 'denarius-search-p1');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThanOrEqual(8);
16 + for (const l of out) {
17 + if (l.kind !== 'listing') throw new Error('expected listing');
18 + expect(l.attributes.categorySlug).toBe('coins');
19 + expect(l.attributes.identifiers.ma_shops_item).toMatch(/^[a-z0-9_-]+\/\d+$/i);
20 + expect(['CAD', 'USD', 'EUR', 'GBP', 'CHF']).toContain(l.currency);
21 + expect(l.price).toBeGreaterThan(0);
22 + expect(l.seller).toBeTruthy();
23 + expect(l.sourceUrl).toMatch(/ma-shops\.com\/[a-z0-9_-]+\/item\.php\?id=\d+/i);
24 + }
25 + expect(out.some((l) => l.kind === 'listing' && l.grade.grader === 'pcgs')).toBe(true);
26 + });
27 +
28 + it('parses displayed prices', () => {
29 + expect(parseMaPrice('642.98 CAN$')).toEqual({ price: 642.98, currency: 'CAD' });
30 + expect(parseMaPrice('1,250.00 US$')).toEqual({ price: 1250, currency: 'USD' });
31 + expect(parseMaPrice('89.00 EUR')).toEqual({ price: 89, currency: 'EUR' });
32 + expect(parseMaPrice(null)).toEqual({ price: null, currency: null });
33 + });
34 +});
added connectors/api/ma-shops/index.ts +117 −0
@@ -0,0 +1,117 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedListingSchema, extractYear, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared';
4 +import { parseGradeFromTitle } from '@rareindex/taxonomy';
5 +
6 +const BASE = 'https://www.ma-shops.com';
7 +const PARSER_VERSION = '1.0.0';
8 +
9 +export const CellSchema = z.object({
10 + shop: z.string(),
11 + id: z.string(),
12 + url: z.string(),
13 + title: z.string(),
14 + price: z.number().nullable(),
15 + currency: z.string().nullable(),
16 + seller: z.string().nullable(),
17 + image: z.string().nullable(),
18 +});
19 +export type Cell = z.infer<typeof CellSchema>;
20 +export const PagePayloadSchema = z.object({ kind: z.literal('gallery_page'), url: z.string(), categorySlug: z.string(), items: z.array(CellSchema) });
21 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
22 +
23 +/** "642.98 CAN$" | "1,250.00 US$" | "89.00 EUR" → amount + ISO currency. */
24 +export function parseMaPrice(text: string | null | undefined): { price: number | null; currency: CurrencyCode | null } {
25 + if (!text) return { price: null, currency: null };
26 + const m = text.replace(/\s+/g, ' ').match(/([\d.,]+)\s*(CAN\$|US\$|EUR|€|GBP|£|CHF)/i);
27 + if (!m) return { price: null, currency: null };
28 + const num = Number(m[1]!.replace(/,/g, ''));
29 + const cur = /CAN/i.test(m[2]!) ? 'CAD' : /US/i.test(m[2]!) || m[2] === '$' ? 'USD' : /EUR|€/i.test(m[2]!) ? 'EUR' : /GBP|£/i.test(m[2]!) ? 'GBP' : 'CHF';
30 + return { price: Number.isFinite(num) && num > 0 ? num : null, currency: cur };
31 +}
32 +
33 +export function parseGalleryPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
34 + const $ = H.load(htmlText);
35 + const items: Cell[] = [];
36 + const seen = new Set<string>();
37 + $('.galleryCell').each((_, el) => {
38 + const e = $(el);
39 + const a = e.find('.galleryItemTitle a, a[href*="item.php?id="]').first();
40 + const href = a.attr('href');
41 + const m = href?.match(/^\/?([a-z0-9_-]+)\/item\.php\?id=(\d+)/i);
42 + if (!href || !m) return;
43 + const key = `${m[1]}/${m[2]}`;
44 + if (seen.has(key)) return;
45 + seen.add(key);
46 + // gallery titles are truncated with an ellipsis; the thumbnail alt/title carries the full title
47 + const title = ((e.find('img.thumb').attr('title') || e.find('img.thumb').attr('alt') || H.text(e.find('.galleryItemTitle').first())) ?? '').replace(/\s+/g, ' ').trim();
48 + if (!title) return;
49 + const { price, currency } = parseMaPrice(H.text(e.find('.itemPrice').first()));
50 + const seller = H.text(e.find('.gallerySellerName').first());
51 + const image = e.find('img.thumb').attr('src') ?? null;
52 + items.push({ shop: m[1]!, id: m[2]!, url: `${BASE}/${m[1]}/item.php?id=${m[2]}`, title, price, currency, seller, image: image ? (image.startsWith('http') ? image : BASE + image) : null });
53 + });
54 + return { kind: 'gallery_page', url: pageUrl, categorySlug, items };
55 +}
56 +
57 +export class MaShopsConnector extends BaseConnector {
58 + readonly version = '1.0.0';
59 + readonly parserVersion = PARSER_VERSION;
60 + protected override minIntervalMs = 2000;
61 +
62 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
63 + const seeds = (this.meta.config.seeds as Array<{ path: string; categorySlug: string }> | undefined) ?? [];
64 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
65 + const cap = ctx.options.limit;
66 + let count = 0;
67 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
68 + for (let i = start; i < seeds.length; i++) {
69 + const seed = seeds[i]!;
70 + for (let page = 1; page <= pages; page++) {
71 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
72 + const sep = seed.path.includes('?') ? '&' : '?';
73 + const url = `${BASE}${seed.path}${page > 1 ? `${sep}page=${page}` : ''}`;
74 + await this.throttle();
75 + const res = await ctx.fetch(url, {
76 + engines: ['api', 'firecrawl'],
77 + responseType: 'text',
78 + expect: ['title', 'price'],
79 + parse: (r) => {
80 + const p = r.html ? parseGalleryPage(r.html, url, seed.categorySlug) : null;
81 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
82 + },
83 + });
84 + const payload = res.success && res.html ? parseGalleryPage(res.html, url, seed.categorySlug) : null;
85 + if (!payload?.items.length) {
86 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no gallery cells'}`);
87 + break;
88 + }
89 + count++;
90 + yield { url, externalId: `${seed.path}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
91 + }
92 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
93 + }
94 + }
95 +
96 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
97 + const p = PagePayloadSchema.parse(raw.payload);
98 + const out: NormalizedRecord[] = [];
99 + for (const c of p.items) {
100 + const g = parseGradeFromTitle(c.title);
101 + const attributes = AssetAttributesSchema.parse({
102 + categorySlug: p.categorySlug,
103 + name: c.title,
104 + year: extractYear(c.title),
105 + identifiers: { ma_shops_item: `${c.shop}/${c.id}` },
106 + metadata: { dealer: c.seller, shop_slug: c.shop, unique_item: true },
107 + });
108 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
109 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `${c.shop}/${c.id}`, confidence: 0.8, listingType: 'fixed_price', price: c.price, currency: c.currency, seller: c.seller ?? c.shop, location: 'EU', availability: c.price ? 'available' : 'unknown' }));
110 + }
111 + return out;
112 + }
113 +}
114 +
115 +export default function createConnector(meta: ConnectorMeta) {
116 + return new MaShopsConnector(meta);
117 +}
added connectors/api/ma-shops/meta.json +87 −0
@@ -0,0 +1,87 @@
1 +{
2 + "id": "ma-shops",
3 + "displayName": "MA-Shops (European coin, banknote & militaria dealer network)",
4 + "sourceId": "ma-shops",
5 + "sourceName": "MA-Shops",
6 + "sourceType": "marketplace",
7 + "sourceUrl": "https://www.ma-shops.com",
8 + "module": "api/ma-shops",
9 + "enginePriority": [
10 + "api",
11 + "firecrawl"
12 + ],
13 + "categories": [
14 + "coins",
15 + "banknotes",
16 + "medals",
17 + "militaria"
18 + ],
19 + "regions": [
20 + "DE",
21 + "EU"
22 + ],
23 + "languages": [
24 + "en",
25 + "de"
26 + ],
27 + "currency": [
28 + "EUR",
29 + "USD",
30 + "CAD"
31 + ],
32 + "supportsListings": true,
33 + "supportsSold": false,
34 + "supportsAuctions": false,
35 + "supportsImages": true,
36 + "supportsCatalog": false,
37 + "supportsPopulation": false,
38 + "supportsLookup": false,
39 + "refreshFrequencyMinutes": 1440,
40 + "priority": "low",
41 + "trustScore": 0.75,
42 + "attributionRequired": true,
43 + "termsUrl": "https://www.ma-shops.com/shops/help.php?id=1",
44 + "accessNotes": "Public search gallery pages (ma-shops.com/search.php?keywords=…, 45 items per page; category landing pages only show a handful of featured items) fetched over plain HTTPS with the RareIndex user agent; robots.txt is fully permissive. Each gallery cell gives the dealer's item URL, title, price with the displayed currency (the site picks a currency per visitor — CAN$, US$ or EUR — which is parsed and stored as-is), the dealer name and the primary image. Emits dealer asks (fixed-price listings). Militaria items carry the taxonomy compliance flags (jurisdiction_check, restricted_symbols_review) and are presented neutrally.",
45 + "enabled": true,
46 + "schemaVersion": "1.0",
47 + "config": {
48 + "seeds": [
49 + {
50 + "path": "/search.php?keywords=denarius",
51 + "categorySlug": "coins",
52 + "label": "Roman denarius"
53 + },
54 + {
55 + "path": "/search.php?keywords=thaler",
56 + "categorySlug": "coins",
57 + "label": "Thaler"
58 + },
59 + {
60 + "path": "/search.php?keywords=tetradrachm",
61 + "categorySlug": "coins",
62 + "label": "Greek tetradrachm"
63 + },
64 + {
65 + "path": "/search.php?keywords=20+mark+gold",
66 + "categorySlug": "coins",
67 + "label": "German 20 Mark gold"
68 + },
69 + {
70 + "path": "/search.php?keywords=sovereign",
71 + "categorySlug": "coins",
72 + "label": "Sovereign"
73 + },
74 + {
75 + "path": "/search.php?keywords=banknote",
76 + "categorySlug": "banknotes",
77 + "label": "Banknotes"
78 + },
79 + {
80 + "path": "/search.php?keywords=medaille",
81 + "categorySlug": "medals",
82 + "label": "Medals"
83 + }
84 + ],
85 + "pagesPerSeed": 1
86 + }
87 +}
added connectors/firecrawl/apmex/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('apmex', 'liberty-eagle-category-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from apmex.com $10 Liberty Eagle category, trimmed to 8 cards' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/apmex/index.test.ts +39 −0
@@ -0,0 +1,39 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseCoinTitle } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('apmex', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits coin catalog items + USD dealer asks with parsed grades', async () => {
13 + const fx = loadFixture('apmex', 'liberty-eagle-category-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + const lst = out.filter((r) => r.kind === 'listing');
17 + expect(cats.length).toBeGreaterThan(0);
18 + expect(lst.length).toBeGreaterThan(0);
19 + for (const c of cats) {
20 + if (c.kind !== 'catalog_item') continue;
21 + expect(c.attributes.categorySlug).toBe('coins');
22 + expect(c.attributes.series).toBe('$10 Liberty Gold Eagle');
23 + expect(c.attributes.identifiers.apmex_sku).toMatch(/^\d+$/);
24 + }
25 + for (const l of lst) {
26 + if (l.kind !== 'listing') continue;
27 + expect(l.currency).toBe('USD');
28 + expect(l.price).toBeGreaterThan(100);
29 + expect(l.seller).toBe('APMEX');
30 + }
31 + });
32 +
33 + it('parses coin titles', () => {
34 + expect(parseCoinTitle('1887-S $10 Liberty Gold Eagle MS-63 NGC')).toMatchObject({ year: 1887, mintMark: 'S', grader: 'ngc', grade: 'MS63' });
35 + expect(parseCoinTitle('1900 $10 Liberty Gold Eagle MS-64+ PCGS CAC')).toMatchObject({ year: 1900, mintMark: null, grader: 'pcgs', grade: 'MS64+', designation: 'CAC' });
36 + expect(parseCoinTitle('$10 Liberty Gold Eagle XF (Random Year)')).toMatchObject({ year: null, grader: null, random: true });
37 + expect(parseCoinTitle('$10 Liberty Gold Eagle (Cleaned)').cleaned).toBe(true);
38 + });
39 +});
added connectors/firecrawl/apmex/index.ts +126 −0
@@ -0,0 +1,126 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const PARSER_VERSION = '1.0.0';
6 +
7 +export const CardSchema = z.object({
8 + id: z.string(),
9 + url: z.string(),
10 + title: z.string(),
11 + price: z.number().nullable(),
12 + message: z.string().nullable(),
13 + image: z.string().nullable(),
14 + badge: z.string().nullable(),
15 +});
16 +export type Card = z.infer<typeof CardSchema>;
17 +export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), series: z.string().nullable(), country: z.string().nullable(), items: z.array(CardSchema) });
18 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
19 +
20 +export function parseCategoryPage(htmlText: string, pageUrl: string, series: string | null, country: string | null): PagePayload {
21 + const $ = H.load(htmlText);
22 + const items: Card[] = [];
23 + const seen = new Set<string>();
24 + $('.mod-product-card').each((_, el) => {
25 + const e = $(el);
26 + const a = e.find('a.item-link').first();
27 + const url = a.attr('href');
28 + const id = a.attr('data-product-id') ?? url?.match(/\/product\/(\d+)\//)?.[1];
29 + if (!url || !id || seen.has(id)) return;
30 + seen.add(id);
31 + const title = (a.attr('data-product-name') ?? H.text(e.find('.mod-product-title').first()) ?? '').replace(/\s+/g, ' ').trim();
32 + if (!title) return;
33 + const price = parsePrice(H.text(e.find('.mod-product-pricing .price').first()), 'USD')?.amount ?? null;
34 + const message = H.text(e.find('.mod-product-message').first());
35 + const image = e.find('.mod-product-img img').attr('src') ?? null;
36 + const badge = H.text(e.find('.product-badge').first());
37 + items.push({ id, url, title, price, message, image, badge });
38 + });
39 + return { kind: 'category_page', url: pageUrl, series, country, items };
40 +}
41 +
42 +/** "1887-S $10 Liberty Gold Eagle MS-63 NGC" → parts. Unknown fields stay null. */
43 +export function parseCoinTitle(title: string): { year: number | null; mintMark: string | null; grader: string | null; grade: string | null; designation: string | null; cleaned: boolean; random: boolean } {
44 + const y = title.match(/\b(1[6-9]\d{2}|20\d{2})(?:-([A-Z]{1,2}))?\b/);
45 + const g = title.match(/\b(MS|AU|XF|EF|VF|F|VG|G|AG|PR|PF|PL|SP|BU)-?(\d{1,2})(\+?)(?!\d)/i);
46 + const grader = title.match(/\b(PCGS|NGC|ANACS|ICG|CAC)\b/i)?.[1]?.toLowerCase() ?? null;
47 + const designation = title.match(/\b(CAC|DMPL|PL|FBL|FB|RD|RB|BN|Cameo|Ultra Cameo|DCAM|First Strike|Early Releases)\b/i)?.[1] ?? null;
48 + return {
49 + year: y ? Number(y[1]) : null,
50 + mintMark: y?.[2] ?? null,
51 + grader,
52 + grade: g ? `${g[1]!.toUpperCase()}${g[2]}${g[3] ?? ''}` : null,
53 + designation,
54 + cleaned: /\(cleaned\)|cleaned/i.test(title),
55 + random: /random/i.test(title),
56 + };
57 +}
58 +
59 +export class ApmexConnector extends BaseConnector {
60 + readonly version = '1.0.0';
61 + readonly parserVersion = PARSER_VERSION;
62 + protected override minIntervalMs = 2500;
63 +
64 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
65 + const seeds = (this.meta.config.seeds as Array<{ url: string; series?: string; country?: string }> | undefined) ?? [];
66 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
67 + const cap = ctx.options.limit;
68 + let count = 0;
69 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
70 + for (let i = start; i < seeds.length; i++) {
71 + const seed = seeds[i]!;
72 + for (let page = 1; page <= pages; page++) {
73 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
74 + const url = page > 1 ? `${seed.url}?page=${page}` : seed.url;
75 + await this.throttle();
76 + const res = await ctx.fetch(url, {
77 + engines: ['firecrawl', 'scrapfly'],
78 + expect: ['title', 'price'],
79 + parse: (r) => {
80 + const p = r.html ? parseCategoryPage(r.html, url, seed.series ?? null, seed.country ?? null) : null;
81 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
82 + },
83 + });
84 + const payload = res.success && res.html ? parseCategoryPage(res.html, url, seed.series ?? null, seed.country ?? null) : null;
85 + if (!payload?.items.length) {
86 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product cards'}`);
87 + break;
88 + }
89 + count++;
90 + yield { url, externalId: `${seed.url}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
91 + }
92 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
93 + }
94 + }
95 +
96 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
97 + const p = PagePayloadSchema.parse(raw.payload);
98 + const out: NormalizedRecord[] = [];
99 + for (const c of p.items) {
100 + const t = parseCoinTitle(c.title);
101 + const name = c.title.replace(/\s*\(cleaned\)/i, '').replace(/\b(PCGS|NGC|ANACS|ICG)\b/gi, '').replace(/\b(MS|AU|XF|EF|VF|F|VG|G|PR|PF|BU)-?\d{1,2}\+?\b/gi, '').replace(/\s+/g, ' ').trim();
102 + const attributes = AssetAttributesSchema.parse({
103 + categorySlug: 'coins',
104 + series: p.series,
105 + name,
106 + year: t.year,
107 + variant: [t.mintMark ? `${t.mintMark} mint` : null, t.designation, t.cleaned ? 'Cleaned' : null].filter(Boolean).join(' · ') || null,
108 + country: p.country,
109 + identifiers: { apmex_sku: c.id },
110 + metadata: { mint_mark: t.mintMark, designation: t.designation, random_year: t.random, badge: c.badge },
111 + });
112 + const grade = t.grader && t.grade ? { grader: t.grader, grade: t.grade, qualifier: t.designation, certificationNumber: null } : { grader: null, grade: t.grade, qualifier: null, certificationNumber: null };
113 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
114 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.85, releaseDate: null }));
115 + if (c.price) {
116 + const availability = /out of stock|sold out|notify/i.test(c.message ?? '') ? 'sold' : 'available';
117 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'USD', seller: 'APMEX', location: 'US', condition: { condition: t.grade ? null : /\bBU\b|brilliant uncirculated/i.test(c.title) ? 'mint_state' : null, conditionRaw: c.title.match(/\b(BU|Brilliant Uncirculated|Cull|Cleaned|Uncirculated|Proof)\b/i)?.[1] ?? null, completeness: null }, availability }));
118 + }
119 + }
120 + return out;
121 + }
122 +}
123 +
124 +export default function createConnector(meta: ConnectorMeta) {
125 + return new ApmexConnector(meta);
126 +}
added connectors/firecrawl/apmex/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "apmex",
3 + "displayName": "APMEX (coins & bullion dealer, USD asks)",
4 + "sourceId": "apmex",
5 + "sourceName": "APMEX",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.apmex.com",
8 + "module": "firecrawl/apmex",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["coins"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.apmex.com/terms-and-conditions",
26 + "accessNotes": "Public category grids (apmex.com/category/<id>/<slug>) of one of the largest US coin dealers; robots.txt disallows /catalog/, /cart/, /account/, /spotprice/ etc., none of which are used. Plain HTTPS gets an Akamai 403 for non-browser clients, so pages are fetched with Firecrawl (1 credit per ~80-product page). Parsed per product card: APMEX product id, title, 'As Low As' price (USD, cash/wire tier), stock message, image. Year, mint mark, grade and grading service (PCGS/NGC/ANACS/CAC) are parsed from the title; graded coins become grader/grade variants, raw/BU coins stay raw. Emits catalog items + dealer fixed-price listings (asks, not transactions).",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "url": "https://www.apmex.com/category/11903/10-liberty-eagle-coins-1795-1907", "series": "$10 Liberty Gold Eagle", "country": "US" },
32 + { "url": "https://www.apmex.com/category/11904/20-liberty-double-eagle-coins-1849-1907", "series": "$20 Liberty Double Eagle", "country": "US" },
33 + { "url": "https://www.apmex.com/category/11905/20-saint-gaudens-double-eagle-coins-1907-1933", "series": "$20 Saint-Gaudens Double Eagle", "country": "US" },
34 + { "url": "https://www.apmex.com/category/25310/morgan-dollars-1878-1921", "series": "Morgan Dollar", "country": "US" },
35 + { "url": "https://www.apmex.com/category/25320/peace-dollars-1921-1935", "series": "Peace Dollar", "country": "US" },
36 + { "url": "https://www.apmex.com/category/11440/american-gold-eagles", "series": "American Gold Eagle", "country": "US" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/firecrawl/hobbysearch/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('hobbysearch', 'gundam-mg-search-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from 1999.co.jp search (Gundam MG), trimmed to 8 cards' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/hobbysearch/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { jpy } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('hobbysearch', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits catalog items with MSRP and JPY retailer listings', async () => {
13 + const fx = loadFixture('hobbysearch', 'gundam-mg-search-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + const lst = out.filter((r) => r.kind === 'listing');
17 + expect(cats.length).toBeGreaterThan(0);
18 + expect(lst.length).toBeGreaterThan(0);
19 + for (const c of cats) {
20 + if (c.kind !== 'catalog_item') continue;
21 + expect(['gundam', 'action_figures']).toContain(c.attributes.categorySlug);
22 + expect(c.attributes.identifiers.hobbysearch_id).toMatch(/^\d+$/);
23 + expect(c.sourceUrl).toMatch(/1999\.co\.jp\/eng\/\d+$/);
24 + }
25 + for (const l of lst) {
26 + if (l.kind !== 'listing') continue;
27 + expect(l.currency).toBe('JPY');
28 + expect(l.price).toBeGreaterThan(0);
29 + expect(l.seller).toBe('HobbySearch');
30 + }
31 + });
32 +
33 + it('parses JPY strings', () => {
34 + expect(jpy('7,200 JPY')).toBe(7200);
35 + expect(jpy('110')).toBe(110);
36 + expect(jpy(null)).toBeNull();
37 + });
38 +});
added connectors/firecrawl/hobbysearch/index.ts +130 −0
@@ -0,0 +1,130 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const BASE = 'https://www.1999.co.jp';
6 +const PARSER_VERSION = '1.0.0';
7 +
8 +export const CardSchema = z.object({
9 + id: z.string(),
10 + url: z.string(),
11 + title: z.string(),
12 + image: z.string().nullable(),
13 + price: z.number().nullable(),
14 + listPrice: z.number().nullable(),
15 + stock: z.string().nullable(),
16 + released: z.string().nullable(),
17 +});
18 +export type Card = z.infer<typeof CardSchema>;
19 +export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) });
20 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
21 +
22 +export function jpy(s: string | null | undefined): number | null {
23 + const m = s?.replace(/,/g, '').match(/(\d+)\s*JPY/i) ?? s?.replace(/,/g, '').match(/(\d+)/);
24 + return m ? Number(m[1]) : null;
25 +}
26 +
27 +export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
28 + const $ = H.load(htmlText);
29 + const items: Card[] = [];
30 + $('.c-product-list__item').each((_, el) => {
31 + const e = $(el);
32 + const a = e.find('a[href*="1999.co.jp/eng/"]').filter((__, x) => /\/eng\/\d{5,9}$/.test($(x).attr('href') ?? '')).first();
33 + const url = a.attr('href');
34 + const id = url?.match(/\/eng\/(\d+)$/)?.[1];
35 + if (!url || !id) return;
36 + const img = e.find('img').first();
37 + const title = (img.attr('alt') || img.attr('title') || H.text(a) || '').replace(/\s+/g, ' ').trim();
38 + if (!title) return;
39 + const image = img.attr('src') ?? img.attr('data-src') ?? null;
40 + const price = jpy(H.text(e.find('.c-card__price-element').first()));
41 + const listPrice = jpy(H.text(e.find('.c-card__price-proper').first()));
42 + const text = e.text().replace(/\s+/g, ' ');
43 + const stock = text.match(/(In Stock|Sold Out|Pre-Order|Back-?order|Order Stop|Reservation)/i)?.[1] ?? null;
44 + const released = text.match(/((?:Early|Mid|Late)\s+[A-Z][a-z]{2}\.?,?\s+\d{4}|[A-Z][a-z]{2}\.?,?\s+\d{4})\s+Released/)?.[1] ?? null;
45 + items.push({ id, url, title, image: image ? (image.startsWith('http') ? image : BASE + image) : null, price, listPrice, stock, released });
46 + });
47 + return { kind: 'search_page', url: pageUrl, categorySlug, items };
48 +}
49 +
50 +const GUNDAM_RE = /gundam|gunpla|\bHG(UC|CE|AC|BF|IBO|GTO|BD)?\b|\bMG\b|\bRG\b|\bPG\b|\bMGEX\b|\bSDCS\b|zaku|zeta|char's|unicorn|barbatos|strike freedom|nu\b/i;
51 +
52 +export class HobbySearchConnector extends BaseConnector {
53 + readonly version = '1.0.0';
54 + readonly parserVersion = PARSER_VERSION;
55 + protected override minIntervalMs = 2000;
56 + override readonly urlPatterns = [/^https?:\/\/(www\.)?1999\.co\.jp\/eng\/(\d{5,9})/i];
57 +
58 + private searchUrl(key: string, page: number): string {
59 + return `${BASE}/eng/search?typ1_c=100&cat=&state=&sold=0&sortid=7&searchkey=${encodeURIComponent(key)}${page > 1 ? `&page=${page}` : ''}`;
60 + }
61 +
62 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
63 + const seeds = (this.meta.config.seeds as Array<{ key: string; categorySlug: string }> | undefined) ?? [];
64 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
65 + const cap = ctx.options.limit;
66 + let count = 0;
67 + const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
68 + for (let i = startSeed; i < seeds.length; i++) {
69 + const seed = seeds[i]!;
70 + for (let page = 1; page <= pages; page++) {
71 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
72 + const url = this.searchUrl(seed.key, page);
73 + await this.throttle();
74 + const res = await ctx.fetch(url, {
75 + engines: ['firecrawl', 'scrapfly'],
76 + expect: ['title', 'price'],
77 + parse: (r) => {
78 + const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null;
79 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
80 + },
81 + });
82 + const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null;
83 + if (!payload?.items.length) {
84 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`);
85 + break;
86 + }
87 + count++;
88 + yield { url, externalId: `${seed.key}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
89 + }
90 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
91 + }
92 + }
93 +
94 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
95 + const p = PagePayloadSchema.parse(raw.payload);
96 + const out: NormalizedRecord[] = [];
97 + for (const c of p.items) {
98 + const isGundam = p.categorySlug === 'gundam' && GUNDAM_RE.test(c.title);
99 + const categorySlug = p.categorySlug === 'gundam' ? (isGundam ? 'gundam' : 'action_figures') : p.categorySlug;
100 + const scale = c.title.match(/\b(1\/\d{1,3})\b/)?.[1] ?? null;
101 + const year = c.released?.match(/(\d{4})/)?.[1] ? Number(c.released.match(/(\d{4})/)![1]) : null;
102 + const name = c.title.replace(/\s*\((Plastic model|Completed|Figure|Action Figure|PVC Figure)\)\s*$/i, '').trim();
103 + const attributes = AssetAttributesSchema.parse({
104 + categorySlug,
105 + brand: isGundam ? 'Bandai' : null,
106 + franchise: isGundam ? 'Gundam' : null,
107 + name,
108 + year,
109 + size: scale,
110 + region: 'JP',
111 + originalMsrp: c.listPrice ?? c.price,
112 + originalMsrpCurrency: c.listPrice ?? c.price ? 'JPY' : null,
113 + identifiers: { hobbysearch_id: c.id },
114 + metadata: { product_type: c.title.match(/\(([^)]+)\)\s*$/)?.[1] ?? null, released: c.released, stock: c.stock },
115 + });
116 + const rawTitle = `${name}${scale ? ` ${scale}` : ''}${year ? ` (${year})` : ''}`;
117 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
118 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: null }));
119 + if (c.price) {
120 + const availability = /sold out|order stop/i.test(c.stock ?? '') ? 'sold' : 'available';
121 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'JPY', seller: 'HobbySearch', location: 'Japan', condition: { condition: 'mint_in_box', conditionRaw: 'New', completeness: 'sealed' }, availability }));
122 + }
123 + }
124 + return out;
125 + }
126 +}
127 +
128 +export default function createConnector(meta: ConnectorMeta) {
129 + return new HobbySearchConnector(meta);
130 +}
added connectors/firecrawl/hobbysearch/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "hobbysearch",
3 + "displayName": "HobbySearch 1999.co.jp (Gunpla & model kits, JPY)",
4 + "sourceId": "hobbysearch",
5 + "sourceName": "HobbySearch",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.1999.co.jp",
8 + "module": "firecrawl/hobbysearch",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["gundam", "action_figures", "model_cars"],
11 + "regions": ["JP"],
12 + "languages": ["en", "ja"],
13 + "currency": ["JPY"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.8,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.1999.co.jp/eng/guide/",
26 + "accessNotes": "Public English search/list pages (1999.co.jp/eng/search?...; robots.txt allows all for generic agents) and item pages. Direct HTTPS returns 403 to non-browser clients, so pages go through Firecrawl (1 credit per page of 60 items). Parsed per card: HobbySearch item id, title, street price and list price (JPY), discount, stock state (In Stock / Sold Out / Pre-Order / Back-order), image. Emits catalog items (list price = manufacturer MSRP) and the retailer's fixed-price listing. Gundam categorisation is keyword-based on the seed + title; other kits fall back to the seed's category.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "key": "Gundam MG", "categorySlug": "gundam" },
32 + { "key": "Gundam RG", "categorySlug": "gundam" },
33 + { "key": "Gundam PG", "categorySlug": "gundam" },
34 + { "key": "HGUC", "categorySlug": "gundam" },
35 + { "key": "Nendoroid", "categorySlug": "action_figures" },
36 + { "key": "figma", "categorySlug": "action_figures" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/firecrawl/lego-shop/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.products = p.products.slice(0, 8);
19 + saveFixture('lego-shop', 'star-wars-theme-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'], requiredFields: ['attributes.identifiers.lego_set_number'] }, note: 'Captured live from lego.com/en-us/themes/star-wars, trimmed to 8 products' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/lego-shop/index.test.ts +35 −0
@@ -0,0 +1,35 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('lego-shop', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits official MSRP catalog items and store listings keyed by set number', async () => {
13 + const fx = loadFixture('lego-shop', 'star-wars-theme-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + const lst = out.filter((r) => r.kind === 'listing');
17 + expect(cats.length).toBeGreaterThanOrEqual(4);
18 + expect(lst.length).toBeGreaterThanOrEqual(4);
19 + for (const c of cats) {
20 + if (c.kind !== 'catalog_item') continue;
21 + expect(c.attributes.categorySlug).toBe('lego_sets');
22 + expect(c.attributes.identifiers.lego_set_number).toMatch(/^\d{4,6}$/);
23 + expect(c.attributes.number).toBe(c.attributes.identifiers.lego_set_number);
24 + expect(c.attributes.brand).toBe('LEGO');
25 + expect(c.attributes.franchise).toBe('Star Wars');
26 + expect(c.attributes.originalMsrpCurrency).toBe('USD');
27 + expect(c.attributes.originalMsrp).toBeGreaterThan(0);
28 + expect(c.sourceUrl).toMatch(/lego\.com\/en-us\/product\//);
29 + }
30 + const l = lst[0]!;
31 + if (l.kind !== 'listing') throw new Error();
32 + expect(l.seller).toBe('LEGO Shop (official)');
33 + expect(l.currency).toBe('USD');
34 + });
35 +});
added connectors/firecrawl/lego-shop/index.ts +149 −0
@@ -0,0 +1,149 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const BASE = 'https://www.lego.com';
6 +const PARSER_VERSION = '1.0.0';
7 +
8 +export const LeafSchema = z.object({
9 + code: z.string(),
10 + url: z.string(),
11 + name: z.string(),
12 + price: z.number().nullable(),
13 + badges: z.array(z.string()),
14 + image: z.string().nullable(),
15 + availability: z.string().nullable(),
16 + pieces: z.number().nullable(),
17 + ages: z.string().nullable(),
18 +});
19 +export type Leaf = z.infer<typeof LeafSchema>;
20 +export const PagePayloadSchema = z.object({ kind: z.literal('theme_page'), url: z.string(), theme: z.string(), products: z.array(LeafSchema) });
21 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
22 +
23 +const BADGE_RE = /^(New|Retiring soon|Coming Soon|Exclusive|Hard to find|Sold out|Out of stock|Backorders accepted|Pre-order|Limited edition|Insiders)$/i;
24 +
25 +export const THEME_NAMES: Record<string, string> = {
26 + 'star-wars': 'Star Wars', icons: 'Icons', technic: 'Technic', ideas: 'Ideas', 'harry-potter': 'Harry Potter', marvel: 'Marvel', architecture: 'Architecture', ninjago: 'Ninjago', 'creator-expert': 'Creator Expert', 'botanical-collection': 'Botanical Collection', 'lord-of-the-rings': 'The Lord of the Rings', 'dc': 'DC', 'speed-champions': 'Speed Champions', 'super-mario': 'Super Mario',
27 +};
28 +
29 +export function parseThemePage(htmlText: string, pageUrl: string, theme: string): PagePayload {
30 + const $ = H.load(htmlText);
31 + const products: Leaf[] = [];
32 + const seen = new Set<string>();
33 + $('[data-test="product-leaf"]').each((_, el) => {
34 + const e = $(el);
35 + const a = e.find('a[href*="/product/"]').first();
36 + const url = a.attr('href');
37 + const code = url?.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1];
38 + if (!url || !code || seen.has(code)) return;
39 + seen.add(code);
40 + const name = (a.attr('aria-label') ?? H.text(e.find('[data-test="product-leaf-title"]').first()) ?? '').replace(/\s+/g, ' ').trim();
41 + if (!name) return;
42 + const priceText = H.text(e.find('[data-test="product-leaf-price"]').first()) ?? H.text(e.find('[data-test="product-leaf-price-row"]').first());
43 + const price = parsePrice(priceText, 'USD')?.amount ?? null;
44 + const badges: string[] = [];
45 + e.find('span, div').each((__, b) => {
46 + const t = $(b).children().length ? '' : $(b).text().trim();
47 + if (t && BADGE_RE.test(t) && !badges.includes(t)) badges.push(t);
48 + });
49 + const image = e.find('img[data-test="product-leaf-image-1"]').attr('src') ?? e.find('img').first().attr('src') ?? null;
50 + products.push({ code, url: url.startsWith('http') ? url : BASE + url, name, price, badges, image, availability: null, pieces: null, ages: null });
51 + });
52 + return { kind: 'theme_page', url: pageUrl, theme, products };
53 +}
54 +
55 +export function parseProductPage(htmlText: string, url: string, theme: string): PagePayload | null {
56 + const $ = H.load(htmlText);
57 + const code = url.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1];
58 + const name = H.text($('[data-test="product-overview-name"]').first());
59 + if (!code || !name) return null;
60 + const price = parsePrice(H.text($('[data-test="product-price-display-price"]').first()), 'USD')?.amount ?? null;
61 + const availability = H.text($('[data-test="product-overview-availability"]').first());
62 + const image = $('meta[property="og:image"]').attr('content') ?? null;
63 + const body = $('body').text().replace(/\s+/g, ' ');
64 + const pieces = body.match(/(\d[\d,]*)\s*Pieces/)?.[1];
65 + const ages = body.match(/(\d{1,2}\+)\s*Ages/)?.[1] ?? null;
66 + return { kind: 'theme_page', url, theme, products: [{ code, url, name, price, badges: [], image, availability, pieces: pieces ? Number(pieces.replace(/,/g, '')) : null, ages }] };
67 +}
68 +
69 +export class LegoShopConnector extends BaseConnector {
70 + readonly version = '1.0.0';
71 + readonly parserVersion = PARSER_VERSION;
72 + protected override minIntervalMs = 2000;
73 + override readonly urlPatterns = [/^https?:\/\/(www\.)?lego\.com\/[a-z]{2}-[a-z]{2}\/product\/[a-z0-9-]+-\d{4,6}/i];
74 +
75 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
76 + const themes = (this.meta.config.themes as string[] | undefined) ?? [];
77 + const pages = Number(this.meta.config.pagesPerTheme ?? 1);
78 + const cap = ctx.options.limit;
79 + let count = 0;
80 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.themeIndex ?? 0) : 0;
81 + for (let i = start; i < themes.length; i++) {
82 + const theme = themes[i]!;
83 + for (let page = 1; page <= pages; page++) {
84 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
85 + const url = `${BASE}/en-us/themes/${theme}${page > 1 ? `?page=${page}` : ''}`;
86 + await this.throttle();
87 + const res = await ctx.fetch(url, {
88 + engines: ['firecrawl', 'scrapfly'],
89 + waitForMs: 3000,
90 + expect: ['title', 'price'],
91 + parse: (r) => {
92 + const p = r.html ? parseThemePage(r.html, url, theme) : null;
93 + return p?.products.length ? { title: 'ok', price: p.products.some((x) => x.price) ? 1 : null } : null;
94 + },
95 + });
96 + const payload = res.success && res.html ? parseThemePage(res.html, url, theme) : null;
97 + if (!payload?.products.length) {
98 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product leaves'}`);
99 + break;
100 + }
101 + count++;
102 + yield { url, externalId: `${theme}|p${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
103 + }
104 + await ctx.setCursor({ themeIndex: i + 1 >= themes.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
105 + }
106 + }
107 +
108 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
109 + if (!this.urlPatterns[0]!.test(url)) return [];
110 + const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], waitForMs: 3000, minQuality: 0 });
111 + const payload = res.success && res.html ? parseProductPage(res.html, url, 'product') : null;
112 + if (!payload) return [];
113 + return [{ url, externalId: payload.products[0]!.code, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
114 + }
115 +
116 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
117 + const p = PagePayloadSchema.parse(raw.payload);
118 + const themeName = THEME_NAMES[p.theme] ?? (p.theme === 'product' ? null : p.theme.replace(/-/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()));
119 + const out: NormalizedRecord[] = [];
120 + for (const l of p.products) {
121 + const name = l.name.replace(/[™®]/g, '').replace(/^LEGO\s+/i, '').trim();
122 + const attributes = AssetAttributesSchema.parse({
123 + categorySlug: 'lego_sets',
124 + brand: 'LEGO',
125 + franchise: themeName,
126 + set: themeName,
127 + name,
128 + number: l.code,
129 + originalMsrp: l.price,
130 + originalMsrpCurrency: l.price ? 'USD' : null,
131 + identifiers: { lego_set_number: l.code },
132 + metadata: { badges: l.badges, pieces: l.pieces, ages: l.ages, availability: l.availability, retiring_soon: l.badges.some((b) => /retiring/i.test(b)), official_store: true },
133 + });
134 + const rawTitle = `LEGO ${l.code} ${name}${themeName ? ` · ${themeName}` : ''}`;
135 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: l.url, rawTitle, imageUrls: l.image ? [l.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
136 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: l.code, confidence: 0.95, releaseDate: null }));
137 + if (l.price) {
138 + const soldOut = l.badges.some((b) => /sold out|out of stock/i.test(b)) || /out of stock|sold out|retired/i.test(l.availability ?? '');
139 + const coming = l.badges.some((b) => /coming soon|pre-order/i.test(b)) || /coming soon|pre-?order/i.test(l.availability ?? '');
140 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: l.code, confidence: 0.95, listingType: 'fixed_price', price: l.price, currency: 'USD', seller: 'LEGO Shop (official)', location: 'US', condition: { condition: 'sealed', conditionRaw: 'New (official retail)', completeness: 'sealed' }, availability: soldOut ? 'sold' : coming ? 'unknown' : 'available' }));
141 + }
142 + }
143 + return out;
144 + }
145 +}
146 +
147 +export default function createConnector(meta: ConnectorMeta) {
148 + return new LegoShopConnector(meta);
149 +}
added connectors/firecrawl/lego-shop/meta.json +33 −0
@@ -0,0 +1,33 @@
1 +{
2 + "id": "lego-shop",
3 + "displayName": "LEGO.com Shop (official retail price & availability)",
4 + "sourceId": "lego-shop",
5 + "sourceName": "LEGO Shop (official)",
6 + "sourceType": "manufacturer",
7 + "sourceUrl": "https://www.lego.com",
8 + "module": "firecrawl/lego-shop",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["lego_sets", "lego"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.95,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.lego.com/en-us/legal/notices-and-policies/terms-of-use",
26 + "accessNotes": "Official manufacturer store. Public theme pages (lego.com/en-us/themes/<theme>?page=N) and product pages are rendered client-side and return 403 to plain non-browser clients, so they are fetched with Firecrawl (1 credit per page; ~22 products per theme page). Parsed per product leaf: set number (from the product URL), name, US retail price, badges (New / Retiring soon / Coming Soon / Exclusive / Hard to find / Sold out), image; product pages add availability, age and piece count. Emits catalog items (MSRP USD, identifiers.lego_set_number shared with PriceCharting/Brickset/BrickEconomy) and the official-store fixed-price listing. robots.txt only disallows account/checkout/identity paths.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "themes": ["star-wars", "icons", "technic", "ideas", "harry-potter", "marvel", "architecture", "ninjago", "creator-expert", "botanical-collection"],
31 + "pagesPerTheme": 1
32 + }
33 +}
added connectors/firecrawl/surugaya/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('surugaya', 'pokemon-psa-search-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from suruga-ya.jp search (Pokémon PSA), trimmed to 8 cards' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/surugaya/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseJpCardTitle, yen } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('surugaya', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits JPY listings with PSA grades parsed from Japanese titles', async () => {
13 + const fx = loadFixture('surugaya', 'pokemon-psa-search-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const listings = out.filter((r) => r.kind === 'listing');
16 + expect(listings.length).toBeGreaterThan(0);
17 + const graded = out.find((r) => r.kind === 'catalog_item' && r.grade.grader === 'psa');
18 + expect(graded).toBeTruthy();
19 + if (graded?.kind !== 'catalog_item') throw new Error();
20 + expect(graded.attributes.categorySlug).toBe('pokemon');
21 + expect(graded.attributes.number).toMatch(/^\d{3}\/\d{3}$/);
22 + expect(graded.attributes.language).toBe('Japanese');
23 + expect(graded.attributes.identifiers.surugaya_id).toBeTruthy();
24 + for (const l of listings) {
25 + if (l.kind !== 'listing') continue;
26 + expect(l.currency).toBe('JPY');
27 + expect(l.sourceUrl).toMatch(/suruga-ya\.jp\/product\//);
28 + }
29 + });
30 +
31 + it('helpers', () => {
32 + expect(yen('中古：￥12,800')).toBe(12800);
33 + expect(yen('品切れ')).toBeNull();
34 + expect(parseJpCardTitle('114/083[SAR]：【PSA/GEM MT 10】(キラ)メガゲッコウガex')).toEqual({ number: '114/083', rarity: 'SAR', name: 'メガゲッコウガex', grader: 'psa', grade: '10' });
35 + expect(parseJpCardTitle('096/071[SAR]：【PSA/MINT 9】(キラ)ナンジャモ').grade).toBe('9');
36 + expect(parseJpCardTitle('ポケモンカードゲーム デッキシールド').grader).toBeNull();
37 + });
38 +});
added connectors/firecrawl/surugaya/index.ts +168 −0
@@ -0,0 +1,168 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';
4 +import { parseGradeFromTitle } from '@rareindex/taxonomy';
5 +
6 +const BASE = 'https://www.suruga-ya.jp';
7 +const PARSER_VERSION = '1.0.0';
8 +
9 +export const CardSchema = z.object({
10 + id: z.string(),
11 + url: z.string(),
12 + title: z.string(),
13 + image: z.string().nullable(),
14 + typeLabel: z.string().nullable(),
15 + releaseDate: z.string().nullable(),
16 + brand: z.string().nullable(),
17 + price: z.number().nullable(),
18 + soldOut: z.boolean(),
19 + listPrice: z.number().nullable(),
20 + marketplacePrice: z.number().nullable(),
21 + used: z.boolean(),
22 +});
23 +export type Card = z.infer<typeof CardSchema>;
24 +export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) });
25 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
26 +
27 +export function yen(s: string | null | undefined): number | null {
28 + const m = s?.replace(/,/g, '').match(/[￥¥]\s*(\d+)/);
29 + return m ? Number(m[1]) : null;
30 +}
31 +
32 +/** Parse a search results page into compact cards. */
33 +export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
34 + const $ = H.load(htmlText);
35 + const items: Card[] = [];
36 + $('.item').each((_, el) => {
37 + const e = $(el);
38 + const a = e.find('a[href*="/product/detail/"], a[href*="/product/other/"]').first();
39 + const href = a.attr('href');
40 + if (!href) return;
41 + const url = href.split('?')[0]!;
42 + const id = url.match(/\/product\/(?:detail|other)\/([A-Za-z0-9]+)/)?.[1];
43 + const title = H.text(e.find('h3.product-name').first()) ?? H.text(a);
44 + if (!id || !title) return;
45 + const image = e.find('img').first().attr('src') ?? null;
46 + const typeLabel = H.text(e.find('.condition').filter((__, c) => Boolean($(c).text().trim())).first());
47 + const releaseRaw = H.text(e.find('.release_date').first());
48 + const releaseDate = releaseRaw?.match(/(\d{4}\/\d{2}\/\d{2})/)?.[1] ?? null;
49 + const brand = H.text(e.find('.brand').first())?.replace(/^\[|\]$/g, '').trim() || null;
50 + const priceText = H.text(e.find('.item_price .price').first()) ?? '';
51 + const soldOut = /品切れ/.test(priceText);
52 + const price = soldOut ? null : yen(priceText);
53 + const listPrice = yen(H.text(e.find('.price_teika').first()));
54 + const marketplacePrice = yen(e.find('.highlight-box strong').map((__, x) => $(x).text()).get().find((t) => /[￥¥]/.test(t)) ?? null);
55 + const used = /中古/.test(e.text()) || /中古/.test(priceText);
56 + items.push({ id, url: url.startsWith('http') ? url : BASE + url, title, image: image && image.startsWith('http') ? image : image ? BASE + image : null, typeLabel: typeLabel ?? null, releaseDate, brand, price, soldOut, listPrice, marketplacePrice, used });
57 + });
58 + return { kind: 'search_page', url: pageUrl, categorySlug, items };
59 +}
60 +
61 +/**
62 + * Japanese card titles look like "114/083[SAR]：【PSA/GEM MT 10】(キラ)メガゲッコウガex".
63 + * Extract card number, rarity code, grade and the bare name; everything unknown stays null.
64 + */
65 +export function parseJpCardTitle(title: string): { number: string | null; rarity: string | null; name: string; grader: string | null; grade: string | null } {
66 + const number = title.match(/^(\d{1,3}\/\d{1,3})/)?.[1] ?? null;
67 + const rarity = title.match(/\[([A-Z]{1,4})\]/)?.[1] ?? null;
68 + const g = parseGradeFromTitle(title.replace(/【([A-Z]{2,4})\/?/g, '【$1 ').replace(/[【】]/g, ' '));
69 + let name = title.replace(/^\d{1,3}\/\d{1,3}\s*/, '').replace(/\[[A-Z]{1,4}\]\s*[：:]?\s*/, '').replace(/【[^】]*】\s*/g, '').replace(/\((キラ|ミラー|ノーマル)\)\s*/g, '').trim();
70 + if (!name) name = title;
71 + return { number, rarity, name, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade };
72 +}
73 +
74 +export class SurugayaConnector extends BaseConnector {
75 + readonly version = '1.0.0';
76 + readonly parserVersion = PARSER_VERSION;
77 + protected override minIntervalMs = 2000;
78 +
79 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
80 + const seeds = (this.meta.config.seeds as Array<{ query: string; categorySlug: string }> | undefined) ?? [];
81 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
82 + const cap = ctx.options.limit;
83 + let count = 0;
84 + const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
85 + for (let i = startSeed; i < seeds.length; i++) {
86 + const seed = seeds[i]!;
87 + for (let page = 1; page <= pages; page++) {
88 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
89 + const url = `${BASE}/search?${seed.query}${page > 1 ? `&page=${page}` : ''}`;
90 + await this.throttle();
91 + const res = await ctx.fetch(url, {
92 + // Firecrawl only: Scrapfly's rendered fallback costs ~40 credits/page here and adds nothing.
93 + engines: ['firecrawl'],
94 + minQuality: 0.3, // pages where every card is sold out (no price) are still valid catalog data
95 + expect: ['title', 'price'],
96 + parse: (r) => {
97 + const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null;
98 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price || x.marketplacePrice) ? 1 : null } : null;
99 + },
100 + });
101 + const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null;
102 + if (!payload?.items.length) {
103 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`);
104 + break;
105 + }
106 + count++;
107 + yield { url, externalId: `${seed.query}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
108 + }
109 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
110 + }
111 + }
112 +
113 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
114 + const p = PagePayloadSchema.parse(raw.payload);
115 + const out: NormalizedRecord[] = [];
116 + for (const c of p.items) {
117 + const release = c.releaseDate ? parseSourceDate(c.releaseDate) : null;
118 + const year = release ? release.getUTCFullYear() : null;
119 + const isCard = ['pokemon', 'yugioh', 'one_piece_card_game'].includes(p.categorySlug);
120 + const card = isCard ? parseJpCardTitle(c.title) : null;
121 + const attributes = AssetAttributesSchema.parse({
122 + categorySlug: p.categorySlug,
123 + brand: c.brand,
124 + franchise: c.brand,
125 + name: card?.name ?? c.title,
126 + number: card?.number ?? null,
127 + rarity: card?.rarity ?? null,
128 + year,
129 + language: 'Japanese',
130 + region: 'JP',
131 + originalMsrp: c.listPrice,
132 + originalMsrpCurrency: c.listPrice ? 'JPY' : null,
133 + identifiers: { surugaya_id: c.id },
134 + metadata: { type_label: c.typeLabel, release: c.releaseDate },
135 + });
136 + const rawTitle = `${c.title}${c.brand ? ` [${c.brand}]` : ''}${year ? ` (${year})` : ''}`;
137 + const grade = { grader: card?.grader ?? null, grade: card?.grade ?? null, qualifier: null, certificationNumber: null };
138 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
139 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: release }));
140 + const price = c.price ?? c.marketplacePrice;
141 + if (price) {
142 + const viaMarketplace = c.price === null && c.marketplacePrice !== null;
143 + out.push(
144 + NormalizedListingSchema.parse({
145 + kind: 'listing',
146 + ...base,
147 + externalId: viaMarketplace ? `${c.id}:mp` : c.id,
148 + confidence: 0.8,
149 + listingType: 'fixed_price',
150 + price,
151 + currency: 'JPY',
152 + seller: viaMarketplace ? 'Suruga-ya marketplace seller' : 'Suruga-ya',
153 + location: 'Japan',
154 + condition: { condition: c.used || viaMarketplace ? null : 'mint_in_box', conditionRaw: viaMarketplace ? '中古 (marketplace)' : c.used ? '中古' : '新品', completeness: null },
155 + availability: 'available',
156 + }),
157 + );
158 + } else if (c.soldOut) {
159 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.7, listingType: 'fixed_price', price: null, currency: 'JPY', seller: 'Suruga-ya', location: 'Japan', availability: 'sold' }));
160 + }
161 + }
162 + return out;
163 + }
164 +}
165 +
166 +export default function createConnector(meta: ConnectorMeta) {
167 + return new SurugayaConnector(meta);
168 +}
added connectors/firecrawl/surugaya/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "surugaya",
3 + "displayName": "Suruga-ya (Japanese second-hand hobby retailer, JPY)",
4 + "sourceId": "surugaya",
5 + "sourceName": "Suruga-ya",
6 + "sourceType": "marketplace",
7 + "sourceUrl": "https://www.suruga-ya.jp",
8 + "module": "firecrawl/surugaya",
9 + "enginePriority": ["firecrawl"],
10 + "categories": ["pokemon", "yugioh", "one_piece_card_game", "gundam", "action_figures", "nintendo_games", "sega_games", "playstation_games", "designer_toys"],
11 + "regions": ["JP"],
12 + "languages": ["ja"],
13 + "currency": ["JPY"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.75,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.suruga-ya.jp/man/index.html",
26 + "accessNotes": "Public search result pages (suruga-ya.jp/search?...; robots.txt: 'Allow: /' for generic agents with Content-Signal search=yes, ai-train=no, use=reference — we only index prices and never train on content). Plain HTTPS returns 403 from data-centre IPs, so pages are fetched through Firecrawl (1 credit per page, ~24 items). Parsed per card: product id (shinaban), title, condition/type label, release date, brand, list price (定価), Suruga-ya price or 品切れ (sold out), and the marketplace (マケプレ) lowest price when Suruga-ya itself is out of stock. JPY. Emits catalog items + fixed-price listings; no sold history is available publicly.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "query": "category=5&search_word=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA", "categorySlug": "pokemon", "label": "Pokémon cards PSA" },
32 + { "query": "category=5&search_word=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20SAR", "categorySlug": "pokemon", "label": "Pokémon SAR" },
33 + { "query": "category=5&search_word=%E9%81%8A%E6%88%AF%E7%8E%8B%20%E3%83%97%E3%83%AA%E3%82%BA%E3%83%9E", "categorySlug": "yugioh", "label": "Yu-Gi-Oh! prismatic" },
34 + { "query": "category=5&search_word=%E3%82%AC%E3%83%B3%E3%83%97%E3%83%A9%20MG", "categorySlug": "gundam", "label": "Gunpla MG" },
35 + { "query": "category=5&search_word=figma", "categorySlug": "action_figures", "label": "figma" },
36 + { "query": "category=5&search_word=%E3%82%B9%E3%83%BC%E3%83%91%E3%83%BC%E3%83%95%E3%82%A1%E3%83%9F%E3%82%B3%E3%83%B3%20%E3%82%BD%E3%83%95%E3%83%88", "categorySlug": "nintendo_games", "label": "Super Famicom software" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/scrapfly/amiami/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as { items: unknown[] };
18 + p.items = p.items.slice(0, 6);
19 + saveFixture('amiami', 'figures-preowned-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 6, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from api.amiami.com (pre-owned figures, page 1, trimmed to 6 items)' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/scrapfly/amiami/index.test.ts +37 −0
@@ -0,0 +1,37 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { cleanName, preownedCondition, releaseYear } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('amiami', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits catalog + JPY listings with JAN and maker', async () => {
13 + const fx = loadFixture('amiami', 'figures-preowned-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + const lst = out.filter((r) => r.kind === 'listing');
17 + expect(cats.length).toBeGreaterThan(0);
18 + expect(lst.length).toBeGreaterThan(0);
19 + const l = lst[0]!;
20 + if (l.kind !== 'listing') throw new Error();
21 + expect(l.currency).toBe('JPY');
22 + expect(l.price).toBeGreaterThan(0);
23 + expect(l.attributes.categorySlug).toBe('action_figures');
24 + expect(l.attributes.identifiers.amiami_gcode).toBeTruthy();
25 + expect(l.sourceUrl).toContain('amiami.com/eng/detail/?gcode=');
26 + expect(cats.some((c) => c.kind === 'catalog_item' && c.attributes.identifiers.jan)).toBe(true);
27 + });
28 +
29 + it('helpers', () => {
30 + expect(cleanName('[AmiAmi Exclusive Bonus] Azur Lane Glorious 1/7 Complete Figure')).toEqual({ name: 'Azur Lane Glorious 1/7 Complete Figure', bonus: true, scale: '1/7' });
31 + expect(preownedCondition({ gcode: 'X', gname: '(Pre-owned ITEM:A/BOX:B) Foo', condition_flg: 1 })).toEqual({ condition: 'near_mint_box', raw: 'Pre-owned ITEM:A/BOX:B' });
32 + expect(preownedCondition({ gcode: 'X', gname: 'Foo', condition_flg: 0 }).condition).toBe('mint_in_box');
33 + expect(releaseYear('2026-04-30 00:00:00')).toBe(2026);
34 + expect(releaseYear('Apr-2026')).toBe(2026);
35 + expect(releaseYear(null)).toBeNull();
36 + });
37 +});
added connectors/scrapfly/amiami/index.ts +188 −0
@@ -0,0 +1,188 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +/**
6 + * AmiAmi — Japanese figure/hobby retailer. Uses the storefront's public JSON API via Scrapfly
7 + * (Cloudflare blocks plain data-centre requests). One raw record per result page.
8 + */
9 +const API = 'https://api.amiami.com/api/v1.0';
10 +const SITE = 'https://www.amiami.com';
11 +const IMG = 'https://img.amiami.com';
12 +const PARSER_VERSION = '1.0.0';
13 +
14 +const ItemSchema = z.object({
15 + gcode: z.string(),
16 + gname: z.string(),
17 + thumb_url: z.string().nullable().optional(),
18 + min_price: z.number().nullable().optional(),
19 + max_price: z.number().nullable().optional(),
20 + c_price_taxed: z.number().nullable().optional(),
21 + maker_name: z.string().nullable().optional(),
22 + condition_flg: z.number().nullable().optional(),
23 + instock_flg: z.number().nullable().optional(),
24 + order_closed_flg: z.number().nullable().optional(),
25 + releasedate: z.string().nullable().optional(),
26 + jancode: z.string().nullable().optional(),
27 + preowned_sale_flg: z.number().nullable().optional(),
28 + resale_flg: z.number().nullable().optional(),
29 + saleitem: z.number().nullable().optional(),
30 +});
31 +export type AmiAmiItem = z.infer<typeof ItemSchema>;
32 +
33 +export const PagePayloadSchema = z.object({
34 + kind: z.literal('search_page'),
35 + params: z.string(),
36 + categorySlug: z.string(),
37 + page: z.number(),
38 + total: z.number().nullable(),
39 + items: z.array(ItemSchema),
40 +});
41 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
42 +
43 +const KEEP: Array<keyof AmiAmiItem> = ['gcode', 'gname', 'thumb_url', 'min_price', 'max_price', 'c_price_taxed', 'maker_name', 'condition_flg', 'instock_flg', 'order_closed_flg', 'releasedate', 'jancode', 'preowned_sale_flg', 'resale_flg', 'saleitem'];
44 +
45 +export function trimItem(raw: Record<string, unknown>): AmiAmiItem | null {
46 + const o: Record<string, unknown> = {};
47 + for (const k of KEEP) if (k in raw) o[k] = raw[k];
48 + const p = ItemSchema.safeParse(o);
49 + return p.success ? p.data : null;
50 +}
51 +
52 +/** "[AmiAmi Exclusive Bonus] Azur Lane Glorious Chinese New Year Ver. 1/7 Complete Figure" → clean name + flags */
53 +export function cleanName(gname: string): { name: string; bonus: boolean; scale: string | null } {
54 + let name = gname.replace(/\[[^\]]*(Bonus|Exclusive|Limited)[^\]]*\]\s*/gi, '').trim();
55 + const bonus = name !== gname.trim();
56 + const scale = name.match(/\b(1\/\d{1,3})\b/)?.[1] ?? null;
57 + name = name.replace(/\s+/g, ' ').trim();
58 + return { name, bonus, scale };
59 +}
60 +
61 +/** "(Pre-owned ITEM:A/BOX:B)" grades → normalised condition for the boxed_toys scale. */
62 +export function preownedCondition(item: AmiAmiItem): { condition: string | null; raw: string | null } {
63 + if (item.condition_flg !== 1) return { condition: 'mint_in_box', raw: 'New' };
64 + const m = item.gname.match(/ITEM:([A-Z][+-]?)\/BOX:([A-Z][+-]?)/i);
65 + if (!m) return { condition: 'boxed', raw: 'Pre-owned' };
66 + const itemGrade = m[1]!.toUpperCase();
67 + const cond = itemGrade.startsWith('A') ? 'near_mint_box' : itemGrade.startsWith('B') ? 'boxed' : 'loose_complete';
68 + return { condition: cond, raw: `Pre-owned ITEM:${m[1]}/BOX:${m[2]}` };
69 +}
70 +
71 +export function releaseYear(s: string | null | undefined): number | null {
72 + const m = s?.match(/(\d{4})/);
73 + return m ? Number(m[1]) : null;
74 +}
75 +
76 +export class AmiAmiConnector extends BaseConnector {
77 + readonly version = '1.0.0';
78 + readonly parserVersion = PARSER_VERSION;
79 + protected override minIntervalMs = 1500;
80 + override readonly urlPatterns = [/^https?:\/\/(www\.)?amiami\.com\/(eng|jp)\/detail\/?\?.*gcode=([A-Z0-9-]+)/i];
81 +
82 + private apiOpts() {
83 + return { engines: ['scrapfly' as const], renderJs: false, country: 'jp', headers: { 'X-User-Key': 'amiami_dev' }, timeoutMs: 60_000 };
84 + }
85 +
86 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
87 + const seeds = (this.meta.config.seeds as Array<{ params: string; categorySlug: string }> | undefined) ?? [];
88 + const pages = Number(this.meta.config.pagesPerSeed ?? 2);
89 + const pageSize = Number(this.meta.config.pageSize ?? 50);
90 + const cap = ctx.options.limit;
91 + let count = 0;
92 + const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
93 + for (let i = startSeed; i < seeds.length; i++) {
94 + const seed = seeds[i]!;
95 + for (let page = 1; page <= pages; page++) {
96 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
97 + const url = `${API}/items?pagemax=${pageSize}&lang=eng&${seed.params}&pagecnt=${page}`;
98 + await this.throttle();
99 + const res = await ctx.fetch(url, {
100 + ...this.apiOpts(),
101 + expect: ['title', 'price'],
102 + parse: (r) => {
103 + const j = r.json as { items?: unknown[] } | null;
104 + return j?.items?.length ? { title: 'ok', price: 1 } : null;
105 + },
106 + });
107 + const j = res.json as { items?: Record<string, unknown>[]; search_result?: { total_results?: number } } | null;
108 + if (!res.success || !j?.items) {
109 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
110 + break;
111 + }
112 + const items = j.items.map(trimItem).filter((x): x is AmiAmiItem => Boolean(x));
113 + const payload: PagePayload = { kind: 'search_page', params: seed.params, categorySlug: seed.categorySlug, page, total: j.search_result?.total_results ?? null, items };
114 + count++;
115 + yield { url, externalId: `${seed.params}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
116 + if (items.length < pageSize) break;
117 + }
118 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
119 + }
120 + }
121 +
122 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
123 + const m = url.match(this.urlPatterns[0]!);
124 + if (!m) return [];
125 + const gcode = m[3]!;
126 + const api = `${API}/item?gcode=${encodeURIComponent(gcode)}&lang=eng`;
127 + const res = await ctx.fetch(api, { ...this.apiOpts(), minQuality: 0 });
128 + const j = res.json as { item?: Record<string, unknown> } | null;
129 + if (!res.success || !j?.item) return [];
130 + const it = { ...j.item, min_price: j.item.price ?? j.item.min_price, max_price: j.item.price ?? j.item.max_price, c_price_taxed: j.item.list_price ?? j.item.c_price_taxed };
131 + const item = trimItem(it as Record<string, unknown>);
132 + if (!item) return [];
133 + const payload: PagePayload = { kind: 'search_page', params: `gcode=${gcode}`, categorySlug: 'action_figures', page: 1, total: 1, items: [item] };
134 + return [{ url: api, externalId: gcode, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
135 + }
136 +
137 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
138 + const p = PagePayloadSchema.parse(raw.payload);
139 + const out: NormalizedRecord[] = [];
140 + for (const it of p.items) {
141 + const { name, bonus, scale } = cleanName(it.gname);
142 + const year = releaseYear(it.releasedate);
143 + const cond = preownedCondition(it);
144 + const sourceUrl = `${SITE}/eng/detail/?gcode=${encodeURIComponent(it.gcode)}`;
145 + const image = it.thumb_url ? `${IMG}${it.thumb_url}` : null;
146 + const baseCode = it.gcode.replace(/-R$/, ''); // "-R" = pre-owned listing of the same product
147 + const attributes = AssetAttributesSchema.parse({
148 + categorySlug: p.categorySlug,
149 + brand: it.maker_name ?? null,
150 + name,
151 + year,
152 + size: scale,
153 + originalMsrp: it.c_price_taxed ?? null,
154 + originalMsrpCurrency: it.c_price_taxed ? 'JPY' : null,
155 + identifiers: { amiami_gcode: baseCode, ...(it.jancode ? { jan: it.jancode } : {}) },
156 + metadata: { bonus_edition: bonus, release: it.releasedate ?? null, resale: it.resale_flg === 1 },
157 + });
158 + const rawTitle = `${name}${it.maker_name ? ` · ${it.maker_name}` : ''}${year ? ` (${year})` : ''}`;
159 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle, imageUrls: image ? [image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
160 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: baseCode, confidence: 0.85, releaseDate: null }));
161 + const price = it.min_price ?? it.max_price ?? null;
162 + if (price && price > 0) {
163 + const available = it.instock_flg === 1 && it.order_closed_flg !== 1;
164 + out.push(
165 + NormalizedListingSchema.parse({
166 + kind: 'listing',
167 + ...base,
168 + externalId: it.gcode,
169 + confidence: 0.85,
170 + listingType: 'fixed_price',
171 + price,
172 + currency: 'JPY',
173 + seller: 'AmiAmi',
174 + location: 'Japan',
175 + condition: { condition: cond.condition, conditionRaw: cond.raw, completeness: it.condition_flg === 1 ? 'boxed' : 'sealed' },
176 + availability: available ? 'available' : 'sold',
177 + quantity: null,
178 + }),
179 + );
180 + }
181 + }
182 + return out;
183 + }
184 +}
185 +
186 +export default function createConnector(meta: ConnectorMeta) {
187 + return new AmiAmiConnector(meta);
188 +}
added connectors/scrapfly/amiami/meta.json +39 −0
@@ -0,0 +1,39 @@
1 +{
2 + "id": "amiami",
3 + "displayName": "AmiAmi (figures & hobby — new + pre-owned, JPY)",
4 + "sourceId": "amiami",
5 + "sourceName": "AmiAmi",
6 + "sourceType": "marketplace",
7 + "sourceUrl": "https://www.amiami.com",
8 + "module": "scrapfly/amiami",
9 + "enginePriority": ["scrapfly"],
10 + "categories": ["action_figures", "gundam", "plush", "designer_toys"],
11 + "regions": ["JP"],
12 + "languages": ["en", "ja"],
13 + "currency": ["JPY"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.8,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.amiami.com/eng/guide/",
26 + "accessNotes": "Reads the public JSON API that AmiAmi's own storefront calls (api.amiami.com/api/v1.0/items with the storefront's X-User-Key header, no account or login). Plain requests from data-centre IPs get a Cloudflare 'Attention Required' page, so the request is routed through Scrapfly (asp, no JS, country jp; 1 credit per 50-item page). Data: item code, JAN, maker, release date, list price (tax-in), current sell price (pre-owned range min/max or new price), stock and pre-owned flags — all JPY. Emits catalog items + fixed-price listings; pre-owned condition comes from the storefront's ITEM/BOX grades when present in the title. No purchase, cart or account endpoints are touched.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "params": "s_cate_tag=14&s_st_condition_flg=1", "categorySlug": "action_figures", "label": "Figures — pre-owned" },
32 + { "params": "s_cate_tag=14&s_st_condition_flg=0", "categorySlug": "action_figures", "label": "Figures — new" },
33 + { "params": "s_keywords=Gundam&s_st_condition_flg=1", "categorySlug": "gundam", "label": "Gundam — pre-owned" },
34 + { "params": "s_keywords=Nendoroid", "categorySlug": "action_figures", "label": "Nendoroid" }
35 + ],
36 + "pagesPerSeed": 2,
37 + "pageSize": 50
38 + }
39 +}
added data/fixtures/amiami/figures-preowned-p1.json +129 −0
@@ -0,0 +1,129 @@
1 +{
2 + "raw": {
3 + "url": "https://api.amiami.com/api/v1.0/items?pagemax=50&lang=eng&s_cate_tag=14&s_st_condition_flg=1&pagecnt=1",
4 + "externalId": "s_cate_tag=14&s_st_condition_flg=1|p1",
5 + "kind": "listing",
6 + "engine": "scrapfly",
7 + "fetchedAt": "2026-09-07T06:25:56.894Z",
8 + "payload": {
9 + "kind": "search_page",
10 + "params": "s_cate_tag=14&s_st_condition_flg=1",
11 + "categorySlug": "action_figures",
12 + "page": 1,
13 + "total": 7244,
14 + "items": [
15 + {
16 + "gcode": "FIGURE-180313-R",
17 + "gname": "[AmiAmi Exclusive Bonus] Azur Lane Glorious Chinese New Year Ver. 1/7 Complete Figure",
18 + "thumb_url": "/images/product/main/244/FIGURE-180313.jpg",
19 + "min_price": 20680,
20 + "max_price": 22980,
21 + "c_price_taxed": 30580,
22 + "maker_name": "Alter",
23 + "condition_flg": 1,
24 + "instock_flg": 1,
25 + "order_closed_flg": 0,
26 + "releasedate": "2026-04-30 00:00:00",
27 + "jancode": "4560228207415",
28 + "preowned_sale_flg": 0,
29 + "resale_flg": 0,
30 + "saleitem": 0
31 + },
32 + {
33 + "gcode": "FIG-MOE-9206-R",
34 + "gname": "Super Sonico SoniComi Package ver. 1/5 Complete Figure",
35 + "thumb_url": "/images/product/main/132/FIG-MOE-9206.jpg",
36 + "min_price": 32380,
37 + "max_price": 35980,
38 + "c_price_taxed": 12571,
39 + "maker_name": "OrchidSeed",
40 + "condition_flg": 1,
41 + "instock_flg": 1,
42 + "order_closed_flg": 0,
43 + "releasedate": "2013-12-28 00:00:00",
44 + "jancode": "4582292601265",
45 + "preowned_sale_flg": 0,
46 + "resale_flg": 0,
47 + "saleitem": 0
48 + },
49 + {
50 + "gcode": "FIGURE-191723-R",
51 + "gname": "Super Sonico Nurse Bikini ver. Complete Figure",
52 + "thumb_url": "/images/product/main/253/FIGURE-191723.jpg",
53 + "min_price": 39580,
54 + "max_price": 43980,
55 + "c_price_taxed": 25300,
56 + "maker_name": "Union Creative",
57 + "condition_flg": 1,
58 + "instock_flg": 1,
59 + "order_closed_flg": 0,
60 + "releasedate": "2026-05-31 00:00:00",
61 + "jancode": "4589642716778",
62 + "preowned_sale_flg": 0,
63 + "resale_flg": 0,
64 + "saleitem": 0
65 + },
66 + {
67 + "gcode": "FIGURE-201904-R",
68 + "gname": "Puella Magi Madoka Magica the Movie -Walpurgisnacht: Rising- Madoka Kaname Figure (Game-prize)",
69 + "thumb_url": "/images/product/main/262/FIGURE-201904.jpg",
70 + "min_price": 3280,
71 + "max_price": 3280,
72 + "c_price_taxed": 0,
73 + "maker_name": "BANDAI SPIRITS",
74 + "condition_flg": 1,
75 + "instock_flg": 1,
76 + "order_closed_flg": 0,
77 + "releasedate": "2026-05-31 00:00:00",
78 + "jancode": "4573102705037",
79 + "preowned_sale_flg": 0,
80 + "resale_flg": 0,
81 + "saleitem": 0
82 + },
83 + {
84 + "gcode": "FIG-MOE-4497-R",
85 + "gname": "Mikumo #01 Original Collection - Koi-Iro Byoutou (Love-Colored Hospital Ward) Complete Figure",
86 + "thumb_url": "/images/product/main/114/FIG-MOE-4497.jpg",
87 + "min_price": 110380,
88 + "max_price": 119980,
89 + "c_price_taxed": 4126,
90 + "maker_name": "Hobby Stock",
91 + "condition_flg": 1,
92 + "instock_flg": 1,
93 + "order_closed_flg": 0,
94 + "releasedate": "2012-01-28 00:00:00",
95 + "jancode": "4582225000110",
96 + "preowned_sale_flg": 0,
97 + "resale_flg": 0,
98 + "saleitem": 0
99 + },
100 + {
101 + "gcode": "FIGURE-184067-R",
102 + "gname": "DISH Original Priestess Shibuna 1/7 Complete Figure",
103 + "thumb_url": "/images/product/main/251/FIGURE-184067.jpg",
104 + "min_price": 34380,
105 + "max_price": 42980,
106 + "c_price_taxed": 26180,
107 + "maker_name": "ques Q",
108 + "condition_flg": 1,
109 + "instock_flg": 1,
110 + "order_closed_flg": 0,
111 + "releasedate": "2026-05-31 00:00:00",
112 + "jancode": "4560393843104",
113 + "preowned_sale_flg": 0,
114 + "resale_flg": 0,
115 + "saleitem": 0
116 + }
117 + ]
118 + }
119 + },
120 + "expect": {
121 + "minCount": 6,
122 + "kinds": [
123 + "catalog_item",
124 + "listing"
125 + ]
126 + },
127 + "note": "Captured live from api.amiami.com (pre-owned figures, page 1, trimmed to 6 items)",
128 + "capturedAt": "2026-09-07T06:25:56.899Z"
129 +}
\ No newline at end of file
added data/fixtures/apmex/liberty-eagle-category-p1.json +98 −0
@@ -0,0 +1,98 @@
1 +{
2 + "raw": {
3 + "url": "https://www.apmex.com/category/11903/10-liberty-eagle-coins-1795-1907",
4 + "externalId": "https://www.apmex.com/category/11903/10-liberty-eagle-coins-1795-1907|p1",
5 + "kind": "listing",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T06:29:19.655Z",
8 + "payload": {
9 + "kind": "category_page",
10 + "url": "https://www.apmex.com/category/11903/10-liberty-eagle-coins-1795-1907",
11 + "series": "$10 Liberty Gold Eagle",
12 + "country": "US",
13 + "items": [
14 + {
15 + "id": "9121",
16 + "url": "https://www.apmex.com/product/9121/10-liberty-gold-eagle-cleaned",
17 + "title": "$10 Liberty Gold Eagle (Cleaned)",
18 + "price": 2159.66,
19 + "message": "As Low As",
20 + "image": "https://www.images-apmex.com/images/products/10-liberty-gold-eagle-cleaned_9121_Slab.jpg?v=20191107213552?v=20250325121239&width=130&height=130",
21 + "badge": "Top Pick"
22 + },
23 + {
24 + "id": "118",
25 + "url": "https://www.apmex.com/product/118/10-liberty-gold-eagle-xf-random-year",
26 + "title": "$10 Liberty Gold Eagle XF (Random Year)",
27 + "price": 2167.66,
28 + "message": "As Low As",
29 + "image": "https://www.images-apmex.com/images/products/10-liberty-gold-eagle-xf-random-year_118_Slab.jpg?v=20191107213552?v=20260831030714&width=130&height=130",
30 + "badge": "Top Pick"
31 + },
32 + {
33 + "id": "1123",
34 + "url": "https://www.apmex.com/product/1123/10-liberty-gold-eagle-au-random-year",
35 + "title": "$10 Liberty Gold Eagle AU (Random Year)",
36 + "price": 2184.66,
37 + "message": "Any Quantity",
38 + "image": "https://www.images-apmex.com/images/products/10-liberty-gold-eagle-au-random-year_1123_slab.jpg?v=20191107093552&width=130&height=130",
39 + "badge": "Sale"
40 + },
41 + {
42 + "id": "98270",
43 + "url": "https://www.apmex.com/product/98270/10-liberty-gold-eagle-bu-random-year",
44 + "title": "$10 Liberty Gold Eagle BU (Random Year)",
45 + "price": 2204.66,
46 + "message": "As Low As",
47 + "image": "https://www.images-apmex.com/images/products/10-liberty-gold-eagle-bu-random-year_98270_Obv.jpg?v=20191107213552?v=20260616101018&width=130&height=130",
48 + "badge": "Sale"
49 + },
50 + {
51 + "id": "90236",
52 + "url": "https://www.apmex.com/product/90236/1838-1866-10-liberty-gold-eagle-no-motto-cleaned",
53 + "title": "1838-1866 $10 Liberty Gold Eagle No Motto (Cleaned)",
54 + "price": 2194.66,
55 + "message": "As Low As",
56 + "image": "https://www.images-apmex.com/images/products/1838-1866-10-liberty-gold-eagle-no-motto-cleaned_90236_Obv.jpg?v=20191025091623&v=20260318163529&v=?v=20260507121536&width=130&height=130",
57 + "badge": null
58 + },
59 + {
60 + "id": "230455",
61 + "url": "https://www.apmex.com/product/230455/10-liberty-gold-eagle-new-orleans-mint-cleaned",
62 + "title": "$10 Liberty Gold Eagle New Orleans Mint (Cleaned)",
63 + "price": 2219.66,
64 + "message": "Any Quantity",
65 + "image": "https://www.images-apmex.com/images/products/10-liberty-gold-eagle-new-orleans-mint-cleaned_230455_obv.jpg?v=20210324090408?v=20260630105820&width=130&height=130",
66 + "badge": null
67 + },
68 + {
69 + "id": "23197",
70 + "url": "https://www.apmex.com/product/23197/10-liberty-gold-eagle-ms-61-ngc-random",
71 + "title": "$10 Liberty Gold Eagle MS-61 NGC (Random)",
72 + "price": 2234.66,
73 + "message": "Any Quantity",
74 + "image": "https://www.images-apmex.com/images/products/10-liberty-gold-eagle-ms-61-ngc-random_23197_Slab.jpg?v=20191025091623?v=20250314114952&width=130&height=130",
75 + "badge": null
76 + },
77 + {
78 + "id": "1125",
79 + "url": "https://www.apmex.com/product/1125/10-liberty-gold-eagle-ms-62-ngc-random",
80 + "title": "$10 Liberty Gold Eagle MS-62 NGC (Random)",
81 + "price": 2244.66,
82 + "message": "Any Quantity",
83 + "image": "https://www.images-apmex.com/images/products/10-liberty-gold-eagle-ms-62-ngc-random_1125_slab.jpg?v=20191025091623&width=130&height=130",
84 + "badge": "Sale"
85 + }
86 + ]
87 + }
88 + },
89 + "expect": {
90 + "minCount": 8,
91 + "kinds": [
92 + "catalog_item",
93 + "listing"
94 + ]
95 + },
96 + "note": "Captured live from apmex.com $10 Liberty Eagle category, trimmed to 8 cards",
97 + "capturedAt": "2026-09-07T06:29:19.743Z"
98 +}
\ No newline at end of file
added data/fixtures/fossilera/dinosaur-teeth-p1.json +155 −0
@@ -0,0 +1,155 @@
1 +{
2 + "raw": {
3 + "url": "https://www.fossilera.com/fossils-for-sale/dinosaur-teeth",
4 + "externalId": "/fossils-for-sale/dinosaur-teeth|p1",
5 + "kind": "listing",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T06:29:21.057Z",
8 + "payload": {
9 + "kind": "category_page",
10 + "url": "https://www.fossilera.com/fossils-for-sale/dinosaur-teeth",
11 + "categorySlug": "fossils",
12 + "items": [
13 + {
14 + "item": "355472",
15 + "url": "https://www.fossilera.com/fossils/beastly-3-95-serrated-tyrannosaurus-t-rex-tooth-montana",
16 + "title": "Beastly 3.95\" Serrated Tyrannosaurus (T. rex) Tooth - Montana",
17 + "price": 24500,
18 + "oldPrice": 29500,
19 + "sold": false,
20 + "image": "https://assets0.fossilera.com/sp/845790/t-rex-teeth/325x215/tyrannosaurus-rex.jpg",
21 + "species": null,
22 + "age": null,
23 + "location": null,
24 + "formation": null,
25 + "size": null,
26 + "category": null,
27 + "subCategory": null
28 + },
29 + {
30 + "item": "359350",
31 + "url": "https://www.fossilera.com/fossils/4-57-serrated-carcharodontosaurus-tooth-giant-dinosaur-tooth",
32 + "title": "4.57\" Serrated Carcharodontosaurus Tooth - Giant Dinosaur Tooth",
33 + "price": 1495,
34 + "oldPrice": null,
35 + "sold": false,
36 + "image": "https://assets0.fossilera.com/sp/849367/carcharodontosaurus-teeth/325x215/carcharodontosaurus-sp.jpg",
37 + "species": null,
38 + "age": null,
39 + "location": null,
40 + "formation": null,
41 + "size": null,
42 + "category": null,
43 + "subCategory": null
44 + },
45 + {
46 + "item": "352102",
47 + "url": "https://www.fossilera.com/fossils/1-00-fossil-iguanodontid-mantellisaurus-tooth-england",
48 + "title": "1.00\" Fossil Iguanodontid (I. bernissartensis) Tooth - England",
49 + "price": 1250,
50 + "oldPrice": null,
51 + "sold": false,
52 + "image": "https://assets3.fossilera.com/sp/835821/dinosaur-teeth/325x215/iguanodon-bernissartensis.jpg",
53 + "species": null,
54 + "age": null,
55 + "location": null,
56 + "formation": null,
57 + "size": null,
58 + "category": null,
59 + "subCategory": null
60 + },
61 + {
62 + "item": "356969",
63 + "url": "https://www.fossilera.com/fossils/2-55-camarasaurus-tooth-with-partial-root-in-situ-bone-cabin-quarry",
64 + "title": "2.55\" Camarasaurus Tooth with Partial Root in Situ - Bone Cabin Quarry",
65 + "price": 1150,
66 + "oldPrice": null,
67 + "sold": false,
68 + "image": "https://assets2.fossilera.com/sp/844696/sauropod/325x215/camarasaurus-sp.jpg",
69 + "species": null,
70 + "age": null,
71 + "location": null,
72 + "formation": null,
73 + "size": null,
74 + "category": null,
75 + "subCategory": null
76 + },
77 + {
78 + "item": "364766",
79 + "url": "https://www.fossilera.com/fossils/serrated-3-33-carcharodontosaurus-tooth-real-dinosaur-tooth",
80 + "title": "Serrated, 3.33\" Carcharodontosaurus Tooth - Real Dinosaur Tooth",
81 + "price": 795,
82 + "oldPrice": null,
83 + "sold": false,
84 + "image": "https://assets3.fossilera.com/sp/858717/carcharodontosaurus-teeth/325x215/carcharodontosaurus-sp.jpg",
85 + "species": null,
86 + "age": null,
87 + "location": null,
88 + "formation": null,
89 + "size": null,
90 + "category": null,
91 + "subCategory": null
92 + },
93 + {
94 + "item": "374003",
95 + "url": "https://www.fossilera.com/fossils/4-30-real-spinosaurus-tooth-robust-dinosaur-tooth",
96 + "title": "4.30\" Real Spinosaurus Tooth - Robust Dinosaur Tooth",
97 + "price": 395,
98 + "oldPrice": null,
99 + "sold": false,
100 + "image": "https://assets1.fossilera.com/sp/875464/spinosaurus-teeth/325x215/spinosaurus-sp.jpg",
101 + "species": null,
102 + "age": null,
103 + "location": null,
104 + "formation": null,
105 + "size": null,
106 + "category": null,
107 + "subCategory": null
108 + },
109 + {
110 + "item": "351686",
111 + "url": "https://www.fossilera.com/fossils/serrated-2-89-carcharodontosaurus-tooth-feeding-worn-tip",
112 + "title": "Serrated, 2.89\" Carcharodontosaurus Tooth - Feeding Worn Tip",
113 + "price": 395,
114 + "oldPrice": null,
115 + "sold": false,
116 + "image": "https://assets3.fossilera.com/sp/835034/carcharodontosaurus-teeth/325x215/carcharodontosaurus-sp.jpg",
117 + "species": null,
118 + "age": null,
119 + "location": null,
120 + "formation": null,
121 + "size": null,
122 + "category": null,
123 + "subCategory": null
124 + },
125 + {
126 + "item": "316988",
127 + "url": "https://www.fossilera.com/fossils/rare-92-fossil-theropod-neovenator-tooth-england",
128 + "title": "Rare .92\" Fossil Theropod (Neovenator) Tooth - England",
129 + "price": 395,
130 + "oldPrice": null,
131 + "sold": false,
132 + "image": "https://assets2.fossilera.com/sp/771351/dinosaur-teeth/325x215/neovenator-cf-salerii.jpg",
133 + "species": null,
134 + "age": null,
135 + "location": null,
136 + "formation": null,
137 + "size": null,
138 + "category": null,
139 + "subCategory": null
140 + }
141 + ]
142 + }
143 + },
144 + "expect": {
145 + "minCount": 8,
146 + "kinds": [
147 + "listing"
148 + ],
149 + "requiredFields": [
150 + "attributes.identifiers.fossilera_item"
151 + ]
152 + },
153 + "note": "Captured live from fossilera.com dinosaur teeth category, trimmed to 8 specimens",
154 + "capturedAt": "2026-09-07T06:29:21.077Z"
155 +}
\ No newline at end of file
added data/fixtures/hobbysearch/gundam-mg-search-p1.json +105 −0
@@ -0,0 +1,105 @@
1 +{
2 + "raw": {
3 + "url": "https://www.1999.co.jp/eng/search?typ1_c=100&cat=&state=&sold=0&sortid=7&searchkey=Gundam%20MG",
4 + "externalId": "Gundam MG|p1",
5 + "kind": "listing",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T06:29:10.807Z",
8 + "payload": {
9 + "kind": "search_page",
10 + "url": "https://www.1999.co.jp/eng/search?typ1_c=100&cat=&state=&sold=0&sortid=7&searchkey=Gundam%20MG",
11 + "categorySlug": "gundam",
12 + "items": [
13 + {
14 + "id": "11200296",
15 + "url": "https://www.1999.co.jp/eng/11200296",
16 + "title": "1/12 Little Armory [LA-MD02] Megami Device Equipment Set / SMG (Plastic model)",
17 + "image": "https://www.1999.co.jp/itbig120/11200296.jpg",
18 + "price": 3060,
19 + "listPrice": 3740,
20 + "stock": "In Stock",
21 + "released": "Late Sep., 2025"
22 + },
23 + {
24 + "id": "11116127",
25 + "url": "https://www.1999.co.jp/eng/11116127",
26 + "title": "*Bargain Item* Goddess of Victory: Nikke Gungirl Weapons Seventh Dwarf (Anime Toy)",
27 + "image": "https://www.1999.co.jp/itbig111/11116127.jpg",
28 + "price": 1320,
29 + "listPrice": 2640,
30 + "stock": "In Stock",
31 + "released": "Late Nov., 2024"
32 + },
33 + {
34 + "id": "11116128",
35 + "url": "https://www.1999.co.jp/eng/11116128",
36 + "title": "*Bargain Item* Goddess of Victory: Nikke Gungirl Weapons Lonely Freedom (Anime Toy)",
37 + "image": "https://www.1999.co.jp/itbig111/11116128.jpg",
38 + "price": 1200,
39 + "listPrice": 2640,
40 + "stock": "In Stock",
41 + "released": "Late Nov., 2024"
42 + },
43 + {
44 + "id": "11116131",
45 + "url": "https://www.1999.co.jp/eng/11116131",
46 + "title": "*Bargain Item* Goddess of Victory: Nikke Gungirl Weapons Vitamin Bomb (Anime Toy)",
47 + "image": "https://www.1999.co.jp/itbig111/11116131.jpg",
48 + "price": 840,
49 + "listPrice": 2640,
50 + "stock": "In Stock",
51 + "released": "Late Nov., 2024"
52 + },
53 + {
54 + "id": "11116126",
55 + "url": "https://www.1999.co.jp/eng/11116126",
56 + "title": "*Bargain Item* Goddess of Victory: Nikke Gungirl Weapons Wolfsbane (Anime Toy)",
57 + "image": "https://www.1999.co.jp/itbig111/11116126.jpg",
58 + "price": 1080,
59 + "listPrice": 2640,
60 + "stock": "In Stock",
61 + "released": "Late Nov., 2024"
62 + },
63 + {
64 + "id": "11116130",
65 + "url": "https://www.1999.co.jp/eng/11116130",
66 + "title": "*Bargain Item* Goddess of Victory: Nikke Gungirl Weapons Amazing Wonderland (Anime Toy)",
67 + "image": "https://www.1999.co.jp/itbig111/11116130.jpg",
68 + "price": 1080,
69 + "listPrice": 2640,
70 + "stock": "In Stock",
71 + "released": "Late Nov., 2024"
72 + },
73 + {
74 + "id": "11116129",
75 + "url": "https://www.1999.co.jp/eng/11116129",
76 + "title": "*Bargain Item* Goddess of Victory: Nikke Gungirl Weapons Crown of Warriors (Anime Toy)",
77 + "image": "https://www.1999.co.jp/itbig111/11116129.jpg",
78 + "price": 1416,
79 + "listPrice": 2640,
80 + "stock": "In Stock",
81 + "released": "Late Nov., 2024"
82 + },
83 + {
84 + "id": "11241743",
85 + "url": "https://www.1999.co.jp/eng/11241743",
86 + "title": "*Bargain Item* GGG Series Mobile Suit Z Gundam Four Murasame -The Spirit of Z Revived- (Figure)",
87 + "image": "https://www.1999.co.jp/itbig124/11241743.jpg",
88 + "price": 15400,
89 + "listPrice": 24200,
90 + "stock": "In Stock",
91 + "released": "Late Feb., 2026"
92 + }
93 + ]
94 + }
95 + },
96 + "expect": {
97 + "minCount": 8,
98 + "kinds": [
99 + "catalog_item",
100 + "listing"
101 + ]
102 + },
103 + "note": "Captured live from 1999.co.jp search (Gundam MG), trimmed to 8 cards",
104 + "capturedAt": "2026-09-07T06:29:10.899Z"
105 +}
\ No newline at end of file
added data/fixtures/lego-shop/star-wars-theme-p1.json +122 −0
@@ -0,0 +1,122 @@
1 +{
2 + "raw": {
3 + "url": "https://www.lego.com/en-us/themes/star-wars",
4 + "externalId": "star-wars|p1",
5 + "kind": "catalog_item",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T06:29:18.358Z",
8 + "payload": {
9 + "kind": "theme_page",
10 + "url": "https://www.lego.com/en-us/themes/star-wars",
11 + "theme": "star-wars",
12 + "products": [
13 + {
14 + "code": "75456",
15 + "url": "https://www.lego.com/en-us/product/advent-calendar-2026-75456",
16 + "name": "LEGO® Star Wars™ Advent Calendar 2026",
17 + "price": 44.99,
18 + "badges": [
19 + "New"
20 + ],
21 + "image": "https://www.lego.com/cdn/cs/set/assets/bltfefed03b11b7bc39/blt691e598d84db744d-75456_Box1_v29.png?fit=crop&quality=80&width=400&height=400&dpr=1",
22 + "availability": null,
23 + "pieces": null,
24 + "ages": null
25 + },
26 + {
27 + "code": "75451",
28 + "url": "https://www.lego.com/en-us/product/hutt-palace-sentry-droid-showdown-75451",
29 + "name": "Hutt Palace Sentry Droid Showdown",
30 + "price": 49.99,
31 + "badges": [
32 + "New"
33 + ],
34 + "image": "https://www.lego.com/cdn/cs/set/assets/bltcbdf08565d92df57/75451_Prod.png?fit=crop&quality=80&width=400&height=400&dpr=1",
35 + "availability": null,
36 + "pieces": null,
37 + "ages": null
38 + },
39 + {
40 + "code": "75455",
41 + "url": "https://www.lego.com/en-us/product/boba-fett-75455",
42 + "name": "Boba Fett™",
43 + "price": 169.99,
44 + "badges": [
45 + "New"
46 + ],
47 + "image": "https://www.lego.com/cdn/cs/set/assets/bltb124c16e2313b0f1/blt0fed9d785e5abb17-75455_Prod.png?fit=crop&quality=80&width=400&height=400&dpr=1",
48 + "availability": null,
49 + "pieces": null,
50 + "ages": null
51 + },
52 + {
53 + "code": "75419",
54 + "url": "https://www.lego.com/en-us/product/death-star-75419",
55 + "name": "Death Star™",
56 + "price": 999.99,
57 + "badges": [],
58 + "image": "https://www.lego.com/cdn/cs/set/assets/blt597e64c3c0f4786c/75419_Prod.png?fit=crop&quality=80&width=400&height=400&dpr=1",
59 + "availability": null,
60 + "pieces": null,
61 + "ages": null
62 + },
63 + {
64 + "code": "40806",
65 + "url": "https://www.lego.com/en-us/product/gingerbread-at-at-walker-40806",
66 + "name": "Gingerbread AT-AT™ Walker",
67 + "price": 59.99,
68 + "badges": [],
69 + "image": "https://www.lego.com/cdn/cs/set/assets/blt6d4c85ec676b3157/40806_Prod.png?fit=crop&quality=80&width=400&height=400&dpr=1",
70 + "availability": null,
71 + "pieces": null,
72 + "ages": null
73 + },
74 + {
75 + "code": "75397",
76 + "url": "https://www.lego.com/en-us/product/jabbas-sail-barge-75397",
77 + "name": "Jabba's Sail Barge™",
78 + "price": 499.99,
79 + "badges": [],
80 + "image": "https://www.lego.com/cdn/cs/set/assets/bltad48b7c771f86707/75397_Prod.png?fit=crop&quality=80&width=400&height=400&dpr=1",
81 + "availability": null,
82 + "pieces": null,
83 + "ages": null
84 + },
85 + {
86 + "code": "75367",
87 + "url": "https://www.lego.com/en-us/product/venator-class-republic-attack-cruiser-75367",
88 + "name": "Venator-Class Republic Attack Cruiser™",
89 + "price": 649.99,
90 + "badges": [],
91 + "image": "https://www.lego.com/cdn/cs/set/assets/blt06c6593d8e8d1c13/75367.png?fit=crop&quality=80&width=400&height=400&dpr=1",
92 + "availability": null,
93 + "pieces": null,
94 + "ages": null
95 + },
96 + {
97 + "code": "75192",
98 + "url": "https://www.lego.com/en-us/product/millennium-falcon-75192",
99 + "name": "Millennium Falcon™",
100 + "price": 849.99,
101 + "badges": [],
102 + "image": "https://www.lego.com/cdn/cs/set/assets/blt3349f56c6f192e18/75192_Prod.png?fit=crop&quality=80&width=400&height=400&dpr=1",
103 + "availability": null,
104 + "pieces": null,
105 + "ages": null
106 + }
107 + ]
108 + }
109 + },
110 + "expect": {
111 + "minCount": 8,
112 + "kinds": [
113 + "catalog_item",
114 + "listing"
115 + ],
116 + "requiredFields": [
117 + "attributes.identifiers.lego_set_number"
118 + ]
119 + },
120 + "note": "Captured live from lego.com/en-us/themes/star-wars, trimmed to 8 products",
121 + "capturedAt": "2026-09-07T06:29:18.427Z"
122 +}
\ No newline at end of file
added data/fixtures/ma-shops/denarius-search-p1.json +108 −0
@@ -0,0 +1,108 @@
1 +{
2 + "raw": {
3 + "url": "https://www.ma-shops.com/search.php?keywords=denarius",
4 + "externalId": "/search.php?keywords=denarius|p1",
5 + "kind": "listing",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T06:29:58.910Z",
8 + "payload": {
9 + "kind": "gallery_page",
10 + "url": "https://www.ma-shops.com/search.php?keywords=denarius",
11 + "categorySlug": "coins",
12 + "items": [
13 + {
14 + "shop": "krueger",
15 + "id": "3889",
16 + "url": "https://www.ma-shops.com/krueger/item.php?id=3889",
17 + "title": "Königreich Lydien: Alyattes bis Kroisos Trite (1/3 Stater) ca. 610-550 v. Chr. Sardeis, Löwenkopf /",
18 + "price": 2148.75,
19 + "currency": "USD",
20 + "seller": "Divus",
21 + "image": "https://img.ma-shops.com/krueger/pic/500px/3889_100496k00.jpg"
22 + },
23 + {
24 + "shop": "henzen",
25 + "id": "81428",
26 + "url": "https://www.ma-shops.com/henzen/item.php?id=81428",
27 + "title": "30 stuivers 1521-1524 ZUIDELIJKE NEDERLANDEN (SOUTHERN NETHERLANDS) - GRAAFSCHAP VLAANDEREN - KAREL",
28 + "price": 2084.87,
29 + "currency": "USD",
30 + "seller": "Henzen",
31 + "image": "https://img.ma-shops.com/henzen/pic/500px/81428.jpg"
32 + },
33 + {
34 + "shop": "bodde",
35 + "id": "260422010",
36 + "url": "https://www.ma-shops.com/bodde/item.php?id=260422010",
37 + "title": "Sachsen-Coburg-Gotha 5 Mark 1895 A Alfred 1893-1900. PCGS MS64",
38 + "price": 8420.79,
39 + "currency": "USD",
40 + "seller": "Bodde",
41 + "image": "https://img.ma-shops.com/bodde/pic/500px/jk3qnwntp0ykdprkxzurfq.jpg"
42 + },
43 + {
44 + "shop": "mayer",
45 + "id": "14164",
46 + "url": "https://www.ma-shops.com/mayer/item.php?id=14164",
47 + "title": "Österreich, Haus Habsburg 1 Dukat 1915 Franz Josef I. 1848-1916 fstgl",
48 + "price": 609.78,
49 + "currency": "USD",
50 + "seller": "Mayer",
51 + "image": "https://img.ma-shops.com/mayer/pic/500px/14164_16180k.jpg"
52 + },
53 + {
54 + "shop": "koelnermuenzkabinett",
55 + "id": "110570",
56 + "url": "https://www.ma-shops.com/koelnermuenzkabinett/item.php?id=110570",
57 + "title": "USA 20 Dollars 1878 S Liberty Head - Double Eagle, With motto - Kursmünze (1877-1907) VF",
58 + "price": 5337.04,
59 + "currency": "USD",
60 + "seller": "Kölner Münzkabinett",
61 + "image": "https://img.ma-shops.com/koelnermuenzkabinett/pic/500px/artid110570_combined.jpg"
62 + },
63 + {
64 + "shop": "coinsnb",
65 + "id": "131397",
66 + "url": "https://www.ma-shops.com/coinsnb/item.php?id=131397",
67 + "title": "India, Portuguese 1881 Calcutta Mint India Portuguese colony 1881 ½ / Meia Rupia - Luíz I (Calcutta",
68 + "price": 108.6,
69 + "currency": "USD",
70 + "seller": "Coins NB",
71 + "image": "https://img.ma-shops.com/coinsnb/pic/500px/131397.jpg"
72 + },
73 + {
74 + "shop": "hanke",
75 + "id": "6285",
76 + "url": "https://www.ma-shops.com/hanke/item.php?id=6285",
77 + "title": "Deutschland, 20 Mark, 1906, Kaiserreich, Hessen, Großherzogtum, Ernst Ludwig , 1892-1918, AU,",
78 + "price": 1614.47,
79 + "currency": "USD",
80 + "seller": "Hanke",
81 + "image": "https://img.ma-shops.com/hanke/pic/500px/artid6285_combined.jpg"
82 + },
83 + {
84 + "shop": "dylla",
85 + "id": "43100",
86 + "url": "https://www.ma-shops.com/dylla/item.php?id=43100",
87 + "title": "Somalia 1000 Schilling 2020 African Wildlife,RAR! Auflage 100 Stück Proof (PP) Proof",
88 + "price": 5749.37,
89 + "currency": "USD",
90 + "seller": "Dylla, Gerhard",
91 + "image": "https://img.ma-shops.com/dylla/pic/500px/43100_p1210911.jpg"
92 + }
93 + ]
94 + }
95 + },
96 + "expect": {
97 + "minCount": 8,
98 + "kinds": [
99 + "listing"
100 + ],
101 + "requiredFields": [
102 + "price",
103 + "currency"
104 + ]
105 + },
106 + "note": "Captured live from ma-shops.com search.php?keywords=denarius gallery, trimmed to 8 cells",
107 + "capturedAt": "2026-09-07T06:29:58.940Z"
108 +}
\ No newline at end of file
added data/fixtures/surugaya/pokemon-psa-search-p1.json +137 −0
@@ -0,0 +1,137 @@
1 +{
2 + "raw": {
3 + "url": "https://www.suruga-ya.jp/search?category=5&search_word=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA",
4 + "externalId": "category=5&search_word=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA|p1",
5 + "kind": "listing",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T06:31:09.262Z",
8 + "payload": {
9 + "kind": "search_page",
10 + "url": "https://www.suruga-ya.jp/search?category=5&search_word=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA",
11 + "categorySlug": "pokemon",
12 + "items": [
13 + {
14 + "id": "GU823224",
15 + "url": "https://www.suruga-ya.jp/product/detail/GU823224",
16 + "title": "114/083[SAR]：【PSA/GEM MT 10】(キラ)メガゲッコウガex",
17 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GU823224&size=m",
18 + "typeLabel": "新入荷",
19 + "releaseDate": "2026/03/13",
20 + "brand": "ポケモン",
21 + "price": null,
22 + "soldOut": true,
23 + "listPrice": null,
24 + "marketplacePrice": 61200,
25 + "used": true
26 + },
27 + {
28 + "id": "GL885959",
29 + "url": "https://www.suruga-ya.jp/product/detail/GL885959",
30 + "title": "096/071[SAR]：【PSA/GEM MT 10】(キラ)ナンジャモ",
31 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GL885959&size=m",
32 + "typeLabel": "新入荷",
33 + "releaseDate": "2023/04/14",
34 + "brand": "ポケモン",
35 + "price": null,
36 + "soldOut": true,
37 + "listPrice": null,
38 + "marketplacePrice": 95760,
39 + "used": true
40 + },
41 + {
42 + "id": "GL900718",
43 + "url": "https://www.suruga-ya.jp/product/detail/GL900718",
44 + "title": "096/071[SAR]：【PSA/MINT 9】(キラ)ナンジャモ",
45 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GL900718&size=m",
46 + "typeLabel": "新入荷",
47 + "releaseDate": "2023/04/14",
48 + "brand": "ポケモン",
49 + "price": null,
50 + "soldOut": true,
51 + "listPrice": null,
52 + "marketplacePrice": 42800,
53 + "used": true
54 + },
55 + {
56 + "id": "GU692147",
57 + "url": "https://www.suruga-ya.jp/product/detail/GU692147",
58 + "title": "236/193[SAR]：【PSA/GEM MT 10】(キラ)ナンジャモのハラバリーex",
59 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GU692147&size=m",
60 + "typeLabel": "新入荷",
61 + "releaseDate": "2025/11/28",
62 + "brand": "ポケモン",
63 + "price": null,
64 + "soldOut": true,
65 + "listPrice": null,
66 + "marketplacePrice": 23800,
67 + "used": true
68 + },
69 + {
70 + "id": "GN100474",
71 + "url": "https://www.suruga-ya.jp/product/detail/GN100474",
72 + "title": "080/073[AR]：【PSA/GEM MT 10】(キラ)コイキング",
73 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GN100474&size=m",
74 + "typeLabel": "新入荷",
75 + "releaseDate": "2023/03/10",
76 + "brand": "ポケモン",
77 + "price": null,
78 + "soldOut": true,
79 + "listPrice": null,
80 + "marketplacePrice": 59760,
81 + "used": true
82 + },
83 + {
84 + "id": "GU718227",
85 + "url": "https://www.suruga-ya.jp/product/detail/GU718227",
86 + "title": "116/080[MUR]：【PSA/GEM MT 10】メガリザードンXex",
87 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GU718227&size=m",
88 + "typeLabel": "新入荷",
89 + "releaseDate": "2025/09/26",
90 + "brand": "ポケモン",
91 + "price": null,
92 + "soldOut": true,
93 + "listPrice": null,
94 + "marketplacePrice": 660000,
95 + "used": true
96 + },
97 + {
98 + "id": "GL885960",
99 + "url": "https://www.suruga-ya.jp/product/detail/GL885960",
100 + "title": "091/071[SR]：【PSA/GEM MT 10】(キラ)ナンジャモ",
101 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GL885960&size=m",
102 + "typeLabel": "新入荷",
103 + "releaseDate": "2023/04/14",
104 + "brand": "ポケモン",
105 + "price": null,
106 + "soldOut": true,
107 + "listPrice": null,
108 + "marketplacePrice": 17800,
109 + "used": true
110 + },
111 + {
112 + "id": "GN521133",
113 + "url": "https://www.suruga-ya.jp/product/detail/GN521133",
114 + "title": "192/172[AR]：【PSA/GEM MT 10】(キラ)フォクスライ",
115 + "image": "https://www.suruga-ya.jp/database/photo.php?shinaban=GN521133&size=m",
116 + "typeLabel": "新入荷",
117 + "releaseDate": "2022/12/02",
118 + "brand": "ポケモン",
119 + "price": null,
120 + "soldOut": true,
121 + "listPrice": null,
122 + "marketplacePrice": 7680,
123 + "used": true
124 + }
125 + ]
126 + }
127 + },
128 + "expect": {
129 + "minCount": 8,
130 + "kinds": [
131 + "catalog_item",
132 + "listing"
133 + ]
134 + },
135 + "note": "Captured live from suruga-ya.jp search (Pokémon PSA), trimmed to 8 cards",
136 + "capturedAt": "2026-09-07T06:31:09.300Z"
137 +}
\ No newline at end of file
138