SPB Git
spboucher.ai

spb/rareindex

Public

Connectors: JP market, official LEGO store, coin dealers, FossilEra (7 sources, agent N)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 7, 2026) parent 0962122 Browse files

35 changed files +2,624 −0

35 changed files
Load all 35 files Unified Split
added connectors/api/fossilera/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('fossilera', 'dinosaur-teeth-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['listing'], requiredFields: ['attributes.identifiers.fossilera_item'] }, note: 'Captured live from fossilera.com dinosaur teeth category, trimmed to 8 specimens' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/api/fossilera/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseCategoryPage, parseSpecimenPage } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('fossilera', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits one USD listing per unique specimen with the FossilEra item number', async () => {
13 + const fx = loadFixture('fossilera', 'dinosaur-teeth-p1');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThanOrEqual(8);
16 + for (const l of out) {
17 + if (l.kind !== 'listing') throw new Error('expected listing');
18 + expect(l.attributes.categorySlug).toBe('fossils');
19 + expect(l.attributes.identifiers.fossilera_item).toMatch(/^\d+$/);
20 + expect(l.currency).toBe('USD');
21 + expect(l.seller).toBe('FossilEra');
22 + expect(l.sourceUrl).toMatch(/fossilera\.com\/fossils\//);
23 + expect(l.attributes.metadata.unique_specimen).toBe(true);
24 + }
25 + const reduced = out.find((l) => l.kind === 'listing' && l.attributes.metadata.previous_price_usd);
26 + expect(reduced).toBeTruthy();
27 + });
28 +
29 + it('parsers handle cards, reduced prices and specimen details', () => {
30 + const html = `<a href="/fossils/x-tooth"><div class="image"><img alt="X Tooth #123" src="//assets0.fossilera.com/x.jpg"></div><div class="info"> 3.1" X Tooth - Montana <div class="price"><span class="old-price">$300</span> $250</div></div></a>`;
31 + const p = parseCategoryPage(html, 'u', 'fossils');
32 + expect(p.items).toHaveLength(1);
33 + expect(p.items[0]).toMatchObject({ item: '123', price: 250, oldPrice: 300, sold: false, title: '3.1" X Tooth - Montana', image: 'https://assets0.fossilera.com/x.jpg' });
34 + const detail = `<html><body><h1>Big Tooth</h1><div class="price">$1,000</div><div class="detail">SPECIES Tyrannosaurus rex</div><div class="detail">AGE Late Cretaceous</div><div class="detail">LOCATION Garfield County, Montana</div><div class="detail">FORMATION Hell Creek Formation</div><div class="detail">SIZE 3.95" tooth</div>ITEM #355472</body></html>`;
35 + const d = parseSpecimenPage(detail, 'https://www.fossilera.com/fossils/big-tooth', 'fossils');
36 + expect(d?.items[0]).toMatchObject({ item: '355472', price: 1000, species: 'Tyrannosaurus rex', formation: 'Hell Creek Formation', location: 'Garfield County, Montana', size: '3.95" tooth' });
37 + });
38 +});
added connectors/api/fossilera/index.ts +153 −0
@@ -0,0 +1,153 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const BASE = 'https://www.fossilera.com';
6 +const PARSER_VERSION = '1.0.0';
7 +
8 +export const SpecimenSchema = z.object({
9 + item: z.string(),
10 + url: z.string(),
11 + title: z.string(),
12 + price: z.number().nullable(),
13 + oldPrice: z.number().nullable(),
14 + sold: z.boolean(),
15 + image: z.string().nullable(),
16 + species: z.string().nullable().default(null),
17 + age: z.string().nullable().default(null),
18 + location: z.string().nullable().default(null),
19 + formation: z.string().nullable().default(null),
20 + size: z.string().nullable().default(null),
21 + category: z.string().nullable().default(null),
22 + subCategory: z.string().nullable().default(null),
23 +});
24 +export type Specimen = z.infer<typeof SpecimenSchema>;
25 +export const PagePayloadSchema = z.object({ kind: z.enum(['category_page', 'specimen_page']), url: z.string(), categorySlug: z.string(), items: z.array(SpecimenSchema) });
26 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
27 +
28 +const abs = (u: string | null | undefined) => (u ? (u.startsWith('//') ? `https:${u}` : u.startsWith('http') ? u : BASE + u) : null);
29 +
30 +export function parseCategoryPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
31 + const $ = H.load(htmlText);
32 + const items: Specimen[] = [];
33 + $('a[href^="/fossils/"], a[href^="/minerals/"], a[href^="/meteorites/"]').each((_, el) => {
34 + const a = $(el);
35 + if (!a.find('.info').length) return;
36 + const url = BASE + a.attr('href')!;
37 + const alt = a.find('img').attr('alt') ?? '';
38 + const item = alt.match(/#(\d+)\s*$/)?.[1] ?? url.split('/').pop()!;
39 + const info = a.find('.info').clone();
40 + const priceEl = info.find('.price');
41 + const oldPrice = parsePrice(H.text(priceEl.find('.old-price')), 'USD')?.amount ?? null;
42 + priceEl.find('.old-price').remove();
43 + const priceText = H.text(priceEl) ?? '';
44 + const sold = /sold/i.test(priceText);
45 + const price = sold ? null : (parsePrice(priceText, 'USD')?.amount ?? null);
46 + priceEl.remove();
47 + const title = H.text(info) ?? alt.replace(/\s*#\d+\s*$/, '');
48 + if (!title) return;
49 + items.push({ item, url, title, price, oldPrice, sold, image: abs(a.find('img').attr('src')), species: null, age: null, location: null, formation: null, size: null, category: null, subCategory: null });
50 + });
51 + return { kind: 'category_page', url: pageUrl, categorySlug, items };
52 +}
53 +
54 +export function parseSpecimenPage(htmlText: string, url: string, categorySlug: string): PagePayload | null {
55 + const $ = H.load(htmlText);
56 + const title = H.text($('h1').first());
57 + if (!title) return null;
58 + const detail = (label: string) => {
59 + let v: string | null = null;
60 + $('[class*="detail"]').each((_, el) => {
61 + const t = $(el).text().replace(/\s+/g, ' ').trim();
62 + const m = t.match(new RegExp(`^${label}\\s+(.+)$`, 'i'));
63 + if (m && !v) v = m[1]!.trim();
64 + });
65 + return v;
66 + };
67 + const body = $('body').text().replace(/\s+/g, ' ');
68 + const item = body.match(/ITEM\s*#\s*(\d+)/i)?.[1] ?? url.split('/').pop()!;
69 + const priceBox = $('.price').first();
70 + const oldPrice = parsePrice(H.text(priceBox.find('.old-price')), 'USD')?.amount ?? null;
71 + const priceText = priceBox.clone().find('.old-price').remove().end().text();
72 + const sold = /this (specimen|item) (has been|was) sold|sold out/i.test(body);
73 + const price = sold ? null : (parsePrice(priceText, 'USD')?.amount ?? null);
74 + return {
75 + kind: 'specimen_page',
76 + url,
77 + categorySlug,
78 + items: [{ item, url, title, price, oldPrice, sold, image: abs($('meta[property="og:image"]').attr('content')), species: detail('SPECIES'), age: detail('AGE'), location: detail('LOCATION'), formation: detail('FORMATION'), size: detail('SIZE'), category: detail('CATEGORY'), subCategory: detail('SUB CATEGORY') }],
79 + };
80 +}
81 +
82 +export class FossilEraConnector extends BaseConnector {
83 + readonly version = '1.0.0';
84 + readonly parserVersion = PARSER_VERSION;
85 + protected override minIntervalMs = 1500;
86 + override readonly urlPatterns = [/^https?:\/\/(www\.)?fossilera\.com\/(fossils|minerals|meteorites)\/[a-z0-9-]+/i];
87 +
88 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
89 + const seeds = (this.meta.config.seeds as Array<{ path: string; categorySlug: string }> | undefined) ?? [];
90 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
91 + const cap = ctx.options.limit;
92 + let count = 0;
93 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
94 + for (let i = start; i < seeds.length; i++) {
95 + const seed = seeds[i]!;
96 + for (let page = 1; page <= pages; page++) {
97 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
98 + const url = `${BASE}${seed.path}${page > 1 ? `?page=${page}` : ''}`;
99 + await this.throttle();
100 + const res = await ctx.fetch(url, {
101 + engines: ['api', 'firecrawl'],
102 + responseType: 'text',
103 + expect: ['title', 'price'],
104 + parse: (r) => {
105 + const p = r.html ? parseCategoryPage(r.html, url, seed.categorySlug) : null;
106 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
107 + },
108 + });
109 + const payload = res.success && res.html ? parseCategoryPage(res.html, url, seed.categorySlug) : null;
110 + if (!payload?.items.length) {
111 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no specimen cards'}`);
112 + break;
113 + }
114 + count++;
115 + yield { url, externalId: `${seed.path}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
116 + }
117 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
118 + }
119 + }
120 +
121 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
122 + const m = url.match(this.urlPatterns[0]!);
123 + if (!m) return [];
124 + const slug = m[2]!.toLowerCase() === 'minerals' ? 'minerals' : m[2]!.toLowerCase() === 'meteorites' ? 'meteorites' : 'fossils';
125 + const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0 });
126 + const payload = res.success && res.html ? parseSpecimenPage(res.html, url, slug) : null;
127 + if (!payload) return [];
128 + return [{ url, externalId: payload.items[0]!.item, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
129 + }
130 +
131 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
132 + const p = PagePayloadSchema.parse(raw.payload);
133 + const out: NormalizedRecord[] = [];
134 + for (const s of p.items) {
135 + const loc = s.location ?? s.title.match(/ - ([A-Z][A-Za-z .]+)$/)?.[1] ?? null;
136 + const attributes = AssetAttributesSchema.parse({
137 + categorySlug: p.categorySlug,
138 + name: s.title,
139 + size: s.size ?? s.title.match(/(\d+(?:\.\d+)?")/)?.[1] ?? null,
140 + country: loc,
141 + identifiers: { fossilera_item: s.item },
142 + metadata: { species: s.species, geological_age: s.age, formation: s.formation, locality: s.location, category: s.category, sub_category: s.subCategory, previous_price_usd: s.oldPrice, dealer_guarantee: 'FossilEra authenticity guarantee (dealer statement)', unique_specimen: true },
143 + });
144 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: s.url, rawTitle: s.title, imageUrls: s.image ? [s.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
145 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: s.item, confidence: 0.85, listingType: 'fixed_price', price: s.price, currency: 'USD', seller: 'FossilEra', location: 'US', availability: s.sold ? 'sold' : s.price ? 'available' : 'unknown' }));
146 + }
147 + return out;
148 + }
149 +}
150 +
151 +export default function createConnector(meta: ConnectorMeta) {
152 + return new FossilEraConnector(meta);
153 +}
added connectors/api/fossilera/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "fossilera",
3 + "displayName": "FossilEra (fossils, minerals & meteorites dealer, USD)",
4 + "sourceId": "fossilera",
5 + "sourceName": "FossilEra",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.fossilera.com",
8 + "module": "api/fossilera",
9 + "enginePriority": ["api", "firecrawl"],
10 + "categories": ["fossils", "minerals", "meteorites"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.fossilera.com/pages/terms-of-service",
26 + "accessNotes": "Public category pages (fossilera.com/fossils-for-sale/<category>?page=N) and specimen pages fetched over plain HTTPS with the RareIndex user agent (robots.txt only disallows gift-card/checkout helpers). Each card gives the specimen title, price (and previous price when reduced), image and the FossilEra item number; specimen pages add species, geological age, location, formation, size and category. Every specimen is unique, so each becomes its own asset with one fixed-price listing (available or, when the page states it, sold — no sale date is published, so sold items are listings with availability=sold, never transactions). Compliance: fossilera states all items are legally collected; locality is kept in attributes.country/region so jurisdiction flags (taxonomy compliance 'jurisdiction_check') can be applied downstream.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "path": "/fossils-for-sale/dinosaur-teeth", "categorySlug": "fossils" },
32 + { "path": "/fossils-for-sale/megalodon-teeth", "categorySlug": "fossils" },
33 + { "path": "/fossils-for-sale/ammonites", "categorySlug": "fossils" },
34 + { "path": "/fossils-for-sale/trilobites", "categorySlug": "fossils" },
35 + { "path": "/minerals-for-sale", "categorySlug": "minerals" },
36 + { "path": "/meteorites-for-sale", "categorySlug": "meteorites" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/api/ma-shops/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('ma-shops', 'denarius-search-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['listing'], requiredFields: ['price', 'currency'] }, note: 'Captured live from ma-shops.com search.php?keywords=denarius gallery, trimmed to 8 cells' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/api/ma-shops/index.test.ts +34 −0
@@ -0,0 +1,34 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseMaPrice } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('ma-shops', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits dealer asks with the displayed currency and parsed grades', async () => {
13 + const fx = loadFixture('ma-shops', 'denarius-search-p1');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThanOrEqual(8);
16 + for (const l of out) {
17 + if (l.kind !== 'listing') throw new Error('expected listing');
18 + expect(l.attributes.categorySlug).toBe('coins');
19 + expect(l.attributes.identifiers.ma_shops_item).toMatch(/^[a-z0-9_-]+\/\d+$/i);
20 + expect(['CAD', 'USD', 'EUR', 'GBP', 'CHF']).toContain(l.currency);
21 + expect(l.price).toBeGreaterThan(0);
22 + expect(l.seller).toBeTruthy();
23 + expect(l.sourceUrl).toMatch(/ma-shops\.com\/[a-z0-9_-]+\/item\.php\?id=\d+/i);
24 + }
25 + expect(out.some((l) => l.kind === 'listing' && l.grade.grader === 'pcgs')).toBe(true);
26 + });
27 +
28 + it('parses displayed prices', () => {
29 + expect(parseMaPrice('642.98 CAN$')).toEqual({ price: 642.98, currency: 'CAD' });
30 + expect(parseMaPrice('1,250.00 US$')).toEqual({ price: 1250, currency: 'USD' });
31 + expect(parseMaPrice('89.00 EUR')).toEqual({ price: 89, currency: 'EUR' });
32 + expect(parseMaPrice(null)).toEqual({ price: null, currency: null });
33 + });
34 +});
added connectors/api/ma-shops/index.ts +117 −0
@@ -0,0 +1,117 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedListingSchema, extractYear, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared';
4 +import { parseGradeFromTitle } from '@rareindex/taxonomy';
5 +
6 +const BASE = 'https://www.ma-shops.com';
7 +const PARSER_VERSION = '1.0.0';
8 +
9 +export const CellSchema = z.object({
10 + shop: z.string(),
11 + id: z.string(),
12 + url: z.string(),
13 + title: z.string(),
14 + price: z.number().nullable(),
15 + currency: z.string().nullable(),
16 + seller: z.string().nullable(),
17 + image: z.string().nullable(),
18 +});
19 +export type Cell = z.infer<typeof CellSchema>;
20 +export const PagePayloadSchema = z.object({ kind: z.literal('gallery_page'), url: z.string(), categorySlug: z.string(), items: z.array(CellSchema) });
21 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
22 +
23 +/** "642.98 CAN$" | "1,250.00 US$" | "89.00 EUR" → amount + ISO currency. */
24 +export function parseMaPrice(text: string | null | undefined): { price: number | null; currency: CurrencyCode | null } {
25 + if (!text) return { price: null, currency: null };
26 + const m = text.replace(/\s+/g, ' ').match(/([\d.,]+)\s*(CAN\$|US\$|EUR|€|GBP|£|CHF)/i);
27 + if (!m) return { price: null, currency: null };
28 + const num = Number(m[1]!.replace(/,/g, ''));
29 + const cur = /CAN/i.test(m[2]!) ? 'CAD' : /US/i.test(m[2]!) || m[2] === '$' ? 'USD' : /EUR|€/i.test(m[2]!) ? 'EUR' : /GBP|£/i.test(m[2]!) ? 'GBP' : 'CHF';
30 + return { price: Number.isFinite(num) && num > 0 ? num : null, currency: cur };
31 +}
32 +
33 +export function parseGalleryPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
34 + const $ = H.load(htmlText);
35 + const items: Cell[] = [];
36 + const seen = new Set<string>();
37 + $('.galleryCell').each((_, el) => {
38 + const e = $(el);
39 + const a = e.find('.galleryItemTitle a, a[href*="item.php?id="]').first();
40 + const href = a.attr('href');
41 + const m = href?.match(/^\/?([a-z0-9_-]+)\/item\.php\?id=(\d+)/i);
42 + if (!href || !m) return;
43 + const key = `${m[1]}/${m[2]}`;
44 + if (seen.has(key)) return;
45 + seen.add(key);
46 + // gallery titles are truncated with an ellipsis; the thumbnail alt/title carries the full title
47 + const title = ((e.find('img.thumb').attr('title') || e.find('img.thumb').attr('alt') || H.text(e.find('.galleryItemTitle').first())) ?? '').replace(/\s+/g, ' ').trim();
48 + if (!title) return;
49 + const { price, currency } = parseMaPrice(H.text(e.find('.itemPrice').first()));
50 + const seller = H.text(e.find('.gallerySellerName').first());
51 + const image = e.find('img.thumb').attr('src') ?? null;
52 + items.push({ shop: m[1]!, id: m[2]!, url: `${BASE}/${m[1]}/item.php?id=${m[2]}`, title, price, currency, seller, image: image ? (image.startsWith('http') ? image : BASE + image) : null });
53 + });
54 + return { kind: 'gallery_page', url: pageUrl, categorySlug, items };
55 +}
56 +
57 +export class MaShopsConnector extends BaseConnector {
58 + readonly version = '1.0.0';
59 + readonly parserVersion = PARSER_VERSION;
60 + protected override minIntervalMs = 2000;
61 +
62 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
63 + const seeds = (this.meta.config.seeds as Array<{ path: string; categorySlug: string }> | undefined) ?? [];
64 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
65 + const cap = ctx.options.limit;
66 + let count = 0;
67 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
68 + for (let i = start; i < seeds.length; i++) {
69 + const seed = seeds[i]!;
70 + for (let page = 1; page <= pages; page++) {
71 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
72 + const sep = seed.path.includes('?') ? '&' : '?';
73 + const url = `${BASE}${seed.path}${page > 1 ? `${sep}page=${page}` : ''}`;
74 + await this.throttle();
75 + const res = await ctx.fetch(url, {
76 + engines: ['api', 'firecrawl'],
77 + responseType: 'text',
78 + expect: ['title', 'price'],
79 + parse: (r) => {
80 + const p = r.html ? parseGalleryPage(r.html, url, seed.categorySlug) : null;
81 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
82 + },
83 + });
84 + const payload = res.success && res.html ? parseGalleryPage(res.html, url, seed.categorySlug) : null;
85 + if (!payload?.items.length) {
86 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no gallery cells'}`);
87 + break;
88 + }
89 + count++;
90 + yield { url, externalId: `${seed.path}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
91 + }
92 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
93 + }
94 + }
95 +
96 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
97 + const p = PagePayloadSchema.parse(raw.payload);
98 + const out: NormalizedRecord[] = [];
99 + for (const c of p.items) {
100 + const g = parseGradeFromTitle(c.title);
101 + const attributes = AssetAttributesSchema.parse({
102 + categorySlug: p.categorySlug,
103 + name: c.title,
104 + year: extractYear(c.title),
105 + identifiers: { ma_shops_item: `${c.shop}/${c.id}` },
106 + metadata: { dealer: c.seller, shop_slug: c.shop, unique_item: true },
107 + });
108 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
109 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `${c.shop}/${c.id}`, confidence: 0.8, listingType: 'fixed_price', price: c.price, currency: c.currency, seller: c.seller ?? c.shop, location: 'EU', availability: c.price ? 'available' : 'unknown' }));
110 + }
111 + return out;
112 + }
113 +}
114 +
115 +export default function createConnector(meta: ConnectorMeta) {
116 + return new MaShopsConnector(meta);
117 +}
added connectors/api/ma-shops/meta.json +87 −0
@@ -0,0 +1,87 @@
1 +{
2 + "id": "ma-shops",
3 + "displayName": "MA-Shops (European coin, banknote & militaria dealer network)",
4 + "sourceId": "ma-shops",
5 + "sourceName": "MA-Shops",
6 + "sourceType": "marketplace",
7 + "sourceUrl": "https://www.ma-shops.com",
8 + "module": "api/ma-shops",
9 + "enginePriority": [
10 + "api",
11 + "firecrawl"
12 + ],
13 + "categories": [
14 + "coins",
15 + "banknotes",
16 + "medals",
17 + "militaria"
18 + ],
19 + "regions": [
20 + "DE",
21 + "EU"
22 + ],
23 + "languages": [
24 + "en",
25 + "de"
26 + ],
27 + "currency": [
28 + "EUR",
29 + "USD",
30 + "CAD"
31 + ],
32 + "supportsListings": true,
33 + "supportsSold": false,
34 + "supportsAuctions": false,
35 + "supportsImages": true,
36 + "supportsCatalog": false,
37 + "supportsPopulation": false,
38 + "supportsLookup": false,
39 + "refreshFrequencyMinutes": 1440,
40 + "priority": "low",
41 + "trustScore": 0.75,
42 + "attributionRequired": true,
43 + "termsUrl": "https://www.ma-shops.com/shops/help.php?id=1",
44 + "accessNotes": "Public search gallery pages (ma-shops.com/search.php?keywords=…, 45 items per page; category landing pages only show a handful of featured items) fetched over plain HTTPS with the RareIndex user agent; robots.txt is fully permissive. Each gallery cell gives the dealer's item URL, title, price with the displayed currency (the site picks a currency per visitor — CAN$, US$ or EUR — which is parsed and stored as-is), the dealer name and the primary image. Emits dealer asks (fixed-price listings). Militaria items carry the taxonomy compliance flags (jurisdiction_check, restricted_symbols_review) and are presented neutrally.",
45 + "enabled": true,
46 + "schemaVersion": "1.0",
47 + "config": {
48 + "seeds": [
49 + {
50 + "path": "/search.php?keywords=denarius",
51 + "categorySlug": "coins",
52 + "label": "Roman denarius"
53 + },
54 + {
55 + "path": "/search.php?keywords=thaler",
56 + "categorySlug": "coins",
57 + "label": "Thaler"
58 + },
59 + {
60 + "path": "/search.php?keywords=tetradrachm",
61 + "categorySlug": "coins",
62 + "label": "Greek tetradrachm"
63 + },
64 + {
65 + "path": "/search.php?keywords=20+mark+gold",
66 + "categorySlug": "coins",
67 + "label": "German 20 Mark gold"
68 + },
69 + {
70 + "path": "/search.php?keywords=sovereign",
71 + "categorySlug": "coins",
72 + "label": "Sovereign"
73 + },
74 + {
75 + "path": "/search.php?keywords=banknote",
76 + "categorySlug": "banknotes",
77 + "label": "Banknotes"
78 + },
79 + {
80 + "path": "/search.php?keywords=medaille",
81 + "categorySlug": "medals",
82 + "label": "Medals"
83 + }
84 + ],
85 + "pagesPerSeed": 1
86 + }
87 +}
added connectors/firecrawl/apmex/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('apmex', 'liberty-eagle-category-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from apmex.com $10 Liberty Eagle category, trimmed to 8 cards' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/apmex/index.test.ts +39 −0
@@ -0,0 +1,39 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseCoinTitle } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('apmex', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits coin catalog items + USD dealer asks with parsed grades', async () => {
13 + const fx = loadFixture('apmex', 'liberty-eagle-category-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + const lst = out.filter((r) => r.kind === 'listing');
17 + expect(cats.length).toBeGreaterThan(0);
18 + expect(lst.length).toBeGreaterThan(0);
19 + for (const c of cats) {
20 + if (c.kind !== 'catalog_item') continue;
21 + expect(c.attributes.categorySlug).toBe('coins');
22 + expect(c.attributes.series).toBe('$10 Liberty Gold Eagle');
23 + expect(c.attributes.identifiers.apmex_sku).toMatch(/^\d+$/);
24 + }
25 + for (const l of lst) {
26 + if (l.kind !== 'listing') continue;
27 + expect(l.currency).toBe('USD');
28 + expect(l.price).toBeGreaterThan(100);
29 + expect(l.seller).toBe('APMEX');
30 + }
31 + });
32 +
33 + it('parses coin titles', () => {
34 + expect(parseCoinTitle('1887-S $10 Liberty Gold Eagle MS-63 NGC')).toMatchObject({ year: 1887, mintMark: 'S', grader: 'ngc', grade: 'MS63' });
35 + expect(parseCoinTitle('1900 $10 Liberty Gold Eagle MS-64+ PCGS CAC')).toMatchObject({ year: 1900, mintMark: null, grader: 'pcgs', grade: 'MS64+', designation: 'CAC' });
36 + expect(parseCoinTitle('$10 Liberty Gold Eagle XF (Random Year)')).toMatchObject({ year: null, grader: null, random: true });
37 + expect(parseCoinTitle('$10 Liberty Gold Eagle (Cleaned)').cleaned).toBe(true);
38 + });
39 +});
added connectors/firecrawl/apmex/index.ts +126 −0
@@ -0,0 +1,126 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const PARSER_VERSION = '1.0.0';
6 +
7 +export const CardSchema = z.object({
8 + id: z.string(),
9 + url: z.string(),
10 + title: z.string(),
11 + price: z.number().nullable(),
12 + message: z.string().nullable(),
13 + image: z.string().nullable(),
14 + badge: z.string().nullable(),
15 +});
16 +export type Card = z.infer<typeof CardSchema>;
17 +export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), series: z.string().nullable(), country: z.string().nullable(), items: z.array(CardSchema) });
18 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
19 +
20 +export function parseCategoryPage(htmlText: string, pageUrl: string, series: string | null, country: string | null): PagePayload {
21 + const $ = H.load(htmlText);
22 + const items: Card[] = [];
23 + const seen = new Set<string>();
24 + $('.mod-product-card').each((_, el) => {
25 + const e = $(el);
26 + const a = e.find('a.item-link').first();
27 + const url = a.attr('href');
28 + const id = a.attr('data-product-id') ?? url?.match(/\/product\/(\d+)\//)?.[1];
29 + if (!url || !id || seen.has(id)) return;
30 + seen.add(id);
31 + const title = (a.attr('data-product-name') ?? H.text(e.find('.mod-product-title').first()) ?? '').replace(/\s+/g, ' ').trim();
32 + if (!title) return;
33 + const price = parsePrice(H.text(e.find('.mod-product-pricing .price').first()), 'USD')?.amount ?? null;
34 + const message = H.text(e.find('.mod-product-message').first());
35 + const image = e.find('.mod-product-img img').attr('src') ?? null;
36 + const badge = H.text(e.find('.product-badge').first());
37 + items.push({ id, url, title, price, message, image, badge });
38 + });
39 + return { kind: 'category_page', url: pageUrl, series, country, items };
40 +}
41 +
42 +/** "1887-S $10 Liberty Gold Eagle MS-63 NGC" → parts. Unknown fields stay null. */
43 +export function parseCoinTitle(title: string): { year: number | null; mintMark: string | null; grader: string | null; grade: string | null; designation: string | null; cleaned: boolean; random: boolean } {
44 + const y = title.match(/\b(1[6-9]\d{2}|20\d{2})(?:-([A-Z]{1,2}))?\b/);
45 + const g = title.match(/\b(MS|AU|XF|EF|VF|F|VG|G|AG|PR|PF|PL|SP|BU)-?(\d{1,2})(\+?)(?!\d)/i);
46 + const grader = title.match(/\b(PCGS|NGC|ANACS|ICG|CAC)\b/i)?.[1]?.toLowerCase() ?? null;
47 + const designation = title.match(/\b(CAC|DMPL|PL|FBL|FB|RD|RB|BN|Cameo|Ultra Cameo|DCAM|First Strike|Early Releases)\b/i)?.[1] ?? null;
48 + return {
49 + year: y ? Number(y[1]) : null,
50 + mintMark: y?.[2] ?? null,
51 + grader,
52 + grade: g ? `${g[1]!.toUpperCase()}${g[2]}${g[3] ?? ''}` : null,
53 + designation,
54 + cleaned: /\(cleaned\)|cleaned/i.test(title),
55 + random: /random/i.test(title),
56 + };
57 +}
58 +
59 +export class ApmexConnector extends BaseConnector {
60 + readonly version = '1.0.0';
61 + readonly parserVersion = PARSER_VERSION;
62 + protected override minIntervalMs = 2500;
63 +
64 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
65 + const seeds = (this.meta.config.seeds as Array<{ url: string; series?: string; country?: string }> | undefined) ?? [];
66 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
67 + const cap = ctx.options.limit;
68 + let count = 0;
69 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
70 + for (let i = start; i < seeds.length; i++) {
71 + const seed = seeds[i]!;
72 + for (let page = 1; page <= pages; page++) {
73 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
74 + const url = page > 1 ? `${seed.url}?page=${page}` : seed.url;
75 + await this.throttle();
76 + const res = await ctx.fetch(url, {
77 + engines: ['firecrawl', 'scrapfly'],
78 + expect: ['title', 'price'],
79 + parse: (r) => {
80 + const p = r.html ? parseCategoryPage(r.html, url, seed.series ?? null, seed.country ?? null) : null;
81 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
82 + },
83 + });
84 + const payload = res.success && res.html ? parseCategoryPage(res.html, url, seed.series ?? null, seed.country ?? null) : null;
85 + if (!payload?.items.length) {
86 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product cards'}`);
87 + break;
88 + }
89 + count++;
90 + yield { url, externalId: `${seed.url}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
91 + }
92 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
93 + }
94 + }
95 +
96 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
97 + const p = PagePayloadSchema.parse(raw.payload);
98 + const out: NormalizedRecord[] = [];
99 + for (const c of p.items) {
100 + const t = parseCoinTitle(c.title);
101 + const name = c.title.replace(/\s*\(cleaned\)/i, '').replace(/\b(PCGS|NGC|ANACS|ICG)\b/gi, '').replace(/\b(MS|AU|XF|EF|VF|F|VG|G|PR|PF|BU)-?\d{1,2}\+?\b/gi, '').replace(/\s+/g, ' ').trim();
102 + const attributes = AssetAttributesSchema.parse({
103 + categorySlug: 'coins',
104 + series: p.series,
105 + name,
106 + year: t.year,
107 + variant: [t.mintMark ? `${t.mintMark} mint` : null, t.designation, t.cleaned ? 'Cleaned' : null].filter(Boolean).join(' · ') || null,
108 + country: p.country,
109 + identifiers: { apmex_sku: c.id },
110 + metadata: { mint_mark: t.mintMark, designation: t.designation, random_year: t.random, badge: c.badge },
111 + });
112 + const grade = t.grader && t.grade ? { grader: t.grader, grade: t.grade, qualifier: t.designation, certificationNumber: null } : { grader: null, grade: t.grade, qualifier: null, certificationNumber: null };
113 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
114 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.85, releaseDate: null }));
115 + if (c.price) {
116 + const availability = /out of stock|sold out|notify/i.test(c.message ?? '') ? 'sold' : 'available';
117 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'USD', seller: 'APMEX', location: 'US', condition: { condition: t.grade ? null : /\bBU\b|brilliant uncirculated/i.test(c.title) ? 'mint_state' : null, conditionRaw: c.title.match(/\b(BU|Brilliant Uncirculated|Cull|Cleaned|Uncirculated|Proof)\b/i)?.[1] ?? null, completeness: null }, availability }));
118 + }
119 + }
120 + return out;
121 + }
122 +}
123 +
124 +export default function createConnector(meta: ConnectorMeta) {
125 + return new ApmexConnector(meta);
126 +}
added connectors/firecrawl/apmex/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "apmex",
3 + "displayName": "APMEX (coins & bullion dealer, USD asks)",
4 + "sourceId": "apmex",
5 + "sourceName": "APMEX",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.apmex.com",
8 + "module": "firecrawl/apmex",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["coins"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.apmex.com/terms-and-conditions",
26 + "accessNotes": "Public category grids (apmex.com/category/<id>/<slug>) of one of the largest US coin dealers; robots.txt disallows /catalog/, /cart/, /account/, /spotprice/ etc., none of which are used. Plain HTTPS gets an Akamai 403 for non-browser clients, so pages are fetched with Firecrawl (1 credit per ~80-product page). Parsed per product card: APMEX product id, title, 'As Low As' price (USD, cash/wire tier), stock message, image. Year, mint mark, grade and grading service (PCGS/NGC/ANACS/CAC) are parsed from the title; graded coins become grader/grade variants, raw/BU coins stay raw. Emits catalog items + dealer fixed-price listings (asks, not transactions).",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "url": "https://www.apmex.com/category/11903/10-liberty-eagle-coins-1795-1907", "series": "$10 Liberty Gold Eagle", "country": "US" },
32 + { "url": "https://www.apmex.com/category/11904/20-liberty-double-eagle-coins-1849-1907", "series": "$20 Liberty Double Eagle", "country": "US" },
33 + { "url": "https://www.apmex.com/category/11905/20-saint-gaudens-double-eagle-coins-1907-1933", "series": "$20 Saint-Gaudens Double Eagle", "country": "US" },
34 + { "url": "https://www.apmex.com/category/25310/morgan-dollars-1878-1921", "series": "Morgan Dollar", "country": "US" },
35 + { "url": "https://www.apmex.com/category/25320/peace-dollars-1921-1935", "series": "Peace Dollar", "country": "US" },
36 + { "url": "https://www.apmex.com/category/11440/american-gold-eagles", "series": "American Gold Eagle", "country": "US" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/firecrawl/hobbysearch/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('hobbysearch', 'gundam-mg-search-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from 1999.co.jp search (Gundam MG), trimmed to 8 cards' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/hobbysearch/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { jpy } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('hobbysearch', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits catalog items with MSRP and JPY retailer listings', async () => {
13 + const fx = loadFixture('hobbysearch', 'gundam-mg-search-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + const lst = out.filter((r) => r.kind === 'listing');
17 + expect(cats.length).toBeGreaterThan(0);
18 + expect(lst.length).toBeGreaterThan(0);
19 + for (const c of cats) {
20 + if (c.kind !== 'catalog_item') continue;
21 + expect(['gundam', 'action_figures']).toContain(c.attributes.categorySlug);
22 + expect(c.attributes.identifiers.hobbysearch_id).toMatch(/^\d+$/);
23 + expect(c.sourceUrl).toMatch(/1999\.co\.jp\/eng\/\d+$/);
24 + }
25 + for (const l of lst) {
26 + if (l.kind !== 'listing') continue;
27 + expect(l.currency).toBe('JPY');
28 + expect(l.price).toBeGreaterThan(0);
29 + expect(l.seller).toBe('HobbySearch');
30 + }
31 + });
32 +
33 + it('parses JPY strings', () => {
34 + expect(jpy('7,200 JPY')).toBe(7200);
35 + expect(jpy('110')).toBe(110);
36 + expect(jpy(null)).toBeNull();
37 + });
38 +});
added connectors/firecrawl/hobbysearch/index.ts +130 −0
@@ -0,0 +1,130 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const BASE = 'https://www.1999.co.jp';
6 +const PARSER_VERSION = '1.0.0';
7 +
8 +export const CardSchema = z.object({
9 + id: z.string(),
10 + url: z.string(),
11 + title: z.string(),
12 + image: z.string().nullable(),
13 + price: z.number().nullable(),
14 + listPrice: z.number().nullable(),
15 + stock: z.string().nullable(),
16 + released: z.string().nullable(),
17 +});
18 +export type Card = z.infer<typeof CardSchema>;
19 +export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) });
20 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
21 +
22 +export function jpy(s: string | null | undefined): number | null {
23 + const m = s?.replace(/,/g, '').match(/(\d+)\s*JPY/i) ?? s?.replace(/,/g, '').match(/(\d+)/);
24 + return m ? Number(m[1]) : null;
25 +}
26 +
27 +export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
28 + const $ = H.load(htmlText);
29 + const items: Card[] = [];
30 + $('.c-product-list__item').each((_, el) => {
31 + const e = $(el);
32 + const a = e.find('a[href*="1999.co.jp/eng/"]').filter((__, x) => /\/eng\/\d{5,9}$/.test($(x).attr('href') ?? '')).first();
33 + const url = a.attr('href');
34 + const id = url?.match(/\/eng\/(\d+)$/)?.[1];
35 + if (!url || !id) return;
36 + const img = e.find('img').first();
37 + const title = (img.attr('alt') || img.attr('title') || H.text(a) || '').replace(/\s+/g, ' ').trim();
38 + if (!title) return;
39 + const image = img.attr('src') ?? img.attr('data-src') ?? null;
40 + const price = jpy(H.text(e.find('.c-card__price-element').first()));
41 + const listPrice = jpy(H.text(e.find('.c-card__price-proper').first()));
42 + const text = e.text().replace(/\s+/g, ' ');
43 + const stock = text.match(/(In Stock|Sold Out|Pre-Order|Back-?order|Order Stop|Reservation)/i)?.[1] ?? null;
44 + const released = text.match(/((?:Early|Mid|Late)\s+[A-Z][a-z]{2}\.?,?\s+\d{4}|[A-Z][a-z]{2}\.?,?\s+\d{4})\s+Released/)?.[1] ?? null;
45 + items.push({ id, url, title, image: image ? (image.startsWith('http') ? image : BASE + image) : null, price, listPrice, stock, released });
46 + });
47 + return { kind: 'search_page', url: pageUrl, categorySlug, items };
48 +}
49 +
50 +const GUNDAM_RE = /gundam|gunpla|\bHG(UC|CE|AC|BF|IBO|GTO|BD)?\b|\bMG\b|\bRG\b|\bPG\b|\bMGEX\b|\bSDCS\b|zaku|zeta|char's|unicorn|barbatos|strike freedom|nu\b/i;
51 +
52 +export class HobbySearchConnector extends BaseConnector {
53 + readonly version = '1.0.0';
54 + readonly parserVersion = PARSER_VERSION;
55 + protected override minIntervalMs = 2000;
56 + override readonly urlPatterns = [/^https?:\/\/(www\.)?1999\.co\.jp\/eng\/(\d{5,9})/i];
57 +
58 + private searchUrl(key: string, page: number): string {
59 + return `${BASE}/eng/search?typ1_c=100&cat=&state=&sold=0&sortid=7&searchkey=${encodeURIComponent(key)}${page > 1 ? `&page=${page}` : ''}`;
60 + }
61 +
62 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
63 + const seeds = (this.meta.config.seeds as Array<{ key: string; categorySlug: string }> | undefined) ?? [];
64 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
65 + const cap = ctx.options.limit;
66 + let count = 0;
67 + const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
68 + for (let i = startSeed; i < seeds.length; i++) {
69 + const seed = seeds[i]!;
70 + for (let page = 1; page <= pages; page++) {
71 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
72 + const url = this.searchUrl(seed.key, page);
73 + await this.throttle();
74 + const res = await ctx.fetch(url, {
75 + engines: ['firecrawl', 'scrapfly'],
76 + expect: ['title', 'price'],
77 + parse: (r) => {
78 + const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null;
79 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;
80 + },
81 + });
82 + const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null;
83 + if (!payload?.items.length) {
84 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`);
85 + break;
86 + }
87 + count++;
88 + yield { url, externalId: `${seed.key}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
89 + }
90 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
91 + }
92 + }
93 +
94 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
95 + const p = PagePayloadSchema.parse(raw.payload);
96 + const out: NormalizedRecord[] = [];
97 + for (const c of p.items) {
98 + const isGundam = p.categorySlug === 'gundam' && GUNDAM_RE.test(c.title);
99 + const categorySlug = p.categorySlug === 'gundam' ? (isGundam ? 'gundam' : 'action_figures') : p.categorySlug;
100 + const scale = c.title.match(/\b(1\/\d{1,3})\b/)?.[1] ?? null;
101 + const year = c.released?.match(/(\d{4})/)?.[1] ? Number(c.released.match(/(\d{4})/)![1]) : null;
102 + const name = c.title.replace(/\s*\((Plastic model|Completed|Figure|Action Figure|PVC Figure)\)\s*$/i, '').trim();
103 + const attributes = AssetAttributesSchema.parse({
104 + categorySlug,
105 + brand: isGundam ? 'Bandai' : null,
106 + franchise: isGundam ? 'Gundam' : null,
107 + name,
108 + year,
109 + size: scale,
110 + region: 'JP',
111 + originalMsrp: c.listPrice ?? c.price,
112 + originalMsrpCurrency: c.listPrice ?? c.price ? 'JPY' : null,
113 + identifiers: { hobbysearch_id: c.id },
114 + metadata: { product_type: c.title.match(/\(([^)]+)\)\s*$/)?.[1] ?? null, released: c.released, stock: c.stock },
115 + });
116 + const rawTitle = `${name}${scale ? ` ${scale}` : ''}${year ? ` (${year})` : ''}`;
117 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
118 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: null }));
119 + if (c.price) {
120 + const availability = /sold out|order stop/i.test(c.stock ?? '') ? 'sold' : 'available';
121 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'JPY', seller: 'HobbySearch', location: 'Japan', condition: { condition: 'mint_in_box', conditionRaw: 'New', completeness: 'sealed' }, availability }));
122 + }
123 + }
124 + return out;
125 + }
126 +}
127 +
128 +export default function createConnector(meta: ConnectorMeta) {
129 + return new HobbySearchConnector(meta);
130 +}
added connectors/firecrawl/hobbysearch/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "hobbysearch",
3 + "displayName": "HobbySearch 1999.co.jp (Gunpla & model kits, JPY)",
4 + "sourceId": "hobbysearch",
5 + "sourceName": "HobbySearch",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.1999.co.jp",
8 + "module": "firecrawl/hobbysearch",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["gundam", "action_figures", "model_cars"],
11 + "regions": ["JP"],
12 + "languages": ["en", "ja"],
13 + "currency": ["JPY"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.8,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.1999.co.jp/eng/guide/",
26 + "accessNotes": "Public English search/list pages (1999.co.jp/eng/search?...; robots.txt allows all for generic agents) and item pages. Direct HTTPS returns 403 to non-browser clients, so pages go through Firecrawl (1 credit per page of 60 items). Parsed per card: HobbySearch item id, title, street price and list price (JPY), discount, stock state (In Stock / Sold Out / Pre-Order / Back-order), image. Emits catalog items (list price = manufacturer MSRP) and the retailer's fixed-price listing. Gundam categorisation is keyword-based on the seed + title; other kits fall back to the seed's category.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "key": "Gundam MG", "categorySlug": "gundam" },
32 + { "key": "Gundam RG", "categorySlug": "gundam" },
33 + { "key": "Gundam PG", "categorySlug": "gundam" },
34 + { "key": "HGUC", "categorySlug": "gundam" },
35 + { "key": "Nendoroid", "categorySlug": "action_figures" },
36 + { "key": "figma", "categorySlug": "action_figures" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/firecrawl/lego-shop/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.products = p.products.slice(0, 8);
19 + saveFixture('lego-shop', 'star-wars-theme-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'], requiredFields: ['attributes.identifiers.lego_set_number'] }, note: 'Captured live from lego.com/en-us/themes/star-wars, trimmed to 8 products' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/lego-shop/index.test.ts +35 −0
@@ -0,0 +1,35 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('lego-shop', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits official MSRP catalog items and store listings keyed by set number', async () => {
13 + const fx = loadFixture('lego-shop', 'star-wars-theme-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const cats = out.filter((r) => r.kind === 'catalog_item');
16 + const lst = out.filter((r) => r.kind === 'listing');
17 + expect(cats.length).toBeGreaterThanOrEqual(4);
18 + expect(lst.length).toBeGreaterThanOrEqual(4);
19 + for (const c of cats) {
20 + if (c.kind !== 'catalog_item') continue;
21 + expect(c.attributes.categorySlug).toBe('lego_sets');
22 + expect(c.attributes.identifiers.lego_set_number).toMatch(/^\d{4,6}$/);
23 + expect(c.attributes.number).toBe(c.attributes.identifiers.lego_set_number);
24 + expect(c.attributes.brand).toBe('LEGO');
25 + expect(c.attributes.franchise).toBe('Star Wars');
26 + expect(c.attributes.originalMsrpCurrency).toBe('USD');
27 + expect(c.attributes.originalMsrp).toBeGreaterThan(0);
28 + expect(c.sourceUrl).toMatch(/lego\.com\/en-us\/product\//);
29 + }
30 + const l = lst[0]!;
31 + if (l.kind !== 'listing') throw new Error();
32 + expect(l.seller).toBe('LEGO Shop (official)');
33 + expect(l.currency).toBe('USD');
34 + });
35 +});
added connectors/firecrawl/lego-shop/index.ts +149 −0
@@ -0,0 +1,149 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';
4 +
5 +const BASE = 'https://www.lego.com';
6 +const PARSER_VERSION = '1.0.0';
7 +
8 +export const LeafSchema = z.object({
9 + code: z.string(),
10 + url: z.string(),
11 + name: z.string(),
12 + price: z.number().nullable(),
13 + badges: z.array(z.string()),
14 + image: z.string().nullable(),
15 + availability: z.string().nullable(),
16 + pieces: z.number().nullable(),
17 + ages: z.string().nullable(),
18 +});
19 +export type Leaf = z.infer<typeof LeafSchema>;
20 +export const PagePayloadSchema = z.object({ kind: z.literal('theme_page'), url: z.string(), theme: z.string(), products: z.array(LeafSchema) });
21 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
22 +
23 +const BADGE_RE = /^(New|Retiring soon|Coming Soon|Exclusive|Hard to find|Sold out|Out of stock|Backorders accepted|Pre-order|Limited edition|Insiders)$/i;
24 +
25 +export const THEME_NAMES: Record<string, string> = {
26 + 'star-wars': 'Star Wars', icons: 'Icons', technic: 'Technic', ideas: 'Ideas', 'harry-potter': 'Harry Potter', marvel: 'Marvel', architecture: 'Architecture', ninjago: 'Ninjago', 'creator-expert': 'Creator Expert', 'botanical-collection': 'Botanical Collection', 'lord-of-the-rings': 'The Lord of the Rings', 'dc': 'DC', 'speed-champions': 'Speed Champions', 'super-mario': 'Super Mario',
27 +};
28 +
29 +export function parseThemePage(htmlText: string, pageUrl: string, theme: string): PagePayload {
30 + const $ = H.load(htmlText);
31 + const products: Leaf[] = [];
32 + const seen = new Set<string>();
33 + $('[data-test="product-leaf"]').each((_, el) => {
34 + const e = $(el);
35 + const a = e.find('a[href*="/product/"]').first();
36 + const url = a.attr('href');
37 + const code = url?.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1];
38 + if (!url || !code || seen.has(code)) return;
39 + seen.add(code);
40 + const name = (a.attr('aria-label') ?? H.text(e.find('[data-test="product-leaf-title"]').first()) ?? '').replace(/\s+/g, ' ').trim();
41 + if (!name) return;
42 + const priceText = H.text(e.find('[data-test="product-leaf-price"]').first()) ?? H.text(e.find('[data-test="product-leaf-price-row"]').first());
43 + const price = parsePrice(priceText, 'USD')?.amount ?? null;
44 + const badges: string[] = [];
45 + e.find('span, div').each((__, b) => {
46 + const t = $(b).children().length ? '' : $(b).text().trim();
47 + if (t && BADGE_RE.test(t) && !badges.includes(t)) badges.push(t);
48 + });
49 + const image = e.find('img[data-test="product-leaf-image-1"]').attr('src') ?? e.find('img').first().attr('src') ?? null;
50 + products.push({ code, url: url.startsWith('http') ? url : BASE + url, name, price, badges, image, availability: null, pieces: null, ages: null });
51 + });
52 + return { kind: 'theme_page', url: pageUrl, theme, products };
53 +}
54 +
55 +export function parseProductPage(htmlText: string, url: string, theme: string): PagePayload | null {
56 + const $ = H.load(htmlText);
57 + const code = url.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1];
58 + const name = H.text($('[data-test="product-overview-name"]').first());
59 + if (!code || !name) return null;
60 + const price = parsePrice(H.text($('[data-test="product-price-display-price"]').first()), 'USD')?.amount ?? null;
61 + const availability = H.text($('[data-test="product-overview-availability"]').first());
62 + const image = $('meta[property="og:image"]').attr('content') ?? null;
63 + const body = $('body').text().replace(/\s+/g, ' ');
64 + const pieces = body.match(/(\d[\d,]*)\s*Pieces/)?.[1];
65 + const ages = body.match(/(\d{1,2}\+)\s*Ages/)?.[1] ?? null;
66 + return { kind: 'theme_page', url, theme, products: [{ code, url, name, price, badges: [], image, availability, pieces: pieces ? Number(pieces.replace(/,/g, '')) : null, ages }] };
67 +}
68 +
69 +export class LegoShopConnector extends BaseConnector {
70 + readonly version = '1.0.0';
71 + readonly parserVersion = PARSER_VERSION;
72 + protected override minIntervalMs = 2000;
73 + override readonly urlPatterns = [/^https?:\/\/(www\.)?lego\.com\/[a-z]{2}-[a-z]{2}\/product\/[a-z0-9-]+-\d{4,6}/i];
74 +
75 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
76 + const themes = (this.meta.config.themes as string[] | undefined) ?? [];
77 + const pages = Number(this.meta.config.pagesPerTheme ?? 1);
78 + const cap = ctx.options.limit;
79 + let count = 0;
80 + const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.themeIndex ?? 0) : 0;
81 + for (let i = start; i < themes.length; i++) {
82 + const theme = themes[i]!;
83 + for (let page = 1; page <= pages; page++) {
84 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
85 + const url = `${BASE}/en-us/themes/${theme}${page > 1 ? `?page=${page}` : ''}`;
86 + await this.throttle();
87 + const res = await ctx.fetch(url, {
88 + engines: ['firecrawl', 'scrapfly'],
89 + waitForMs: 3000,
90 + expect: ['title', 'price'],
91 + parse: (r) => {
92 + const p = r.html ? parseThemePage(r.html, url, theme) : null;
93 + return p?.products.length ? { title: 'ok', price: p.products.some((x) => x.price) ? 1 : null } : null;
94 + },
95 + });
96 + const payload = res.success && res.html ? parseThemePage(res.html, url, theme) : null;
97 + if (!payload?.products.length) {
98 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product leaves'}`);
99 + break;
100 + }
101 + count++;
102 + yield { url, externalId: `${theme}|p${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
103 + }
104 + await ctx.setCursor({ themeIndex: i + 1 >= themes.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
105 + }
106 + }
107 +
108 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
109 + if (!this.urlPatterns[0]!.test(url)) return [];
110 + const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], waitForMs: 3000, minQuality: 0 });
111 + const payload = res.success && res.html ? parseProductPage(res.html, url, 'product') : null;
112 + if (!payload) return [];
113 + return [{ url, externalId: payload.products[0]!.code, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
114 + }
115 +
116 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
117 + const p = PagePayloadSchema.parse(raw.payload);
118 + const themeName = THEME_NAMES[p.theme] ?? (p.theme === 'product' ? null : p.theme.replace(/-/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()));
119 + const out: NormalizedRecord[] = [];
120 + for (const l of p.products) {
121 + const name = l.name.replace(/[™®]/g, '').replace(/^LEGO\s+/i, '').trim();
122 + const attributes = AssetAttributesSchema.parse({
123 + categorySlug: 'lego_sets',
124 + brand: 'LEGO',
125 + franchise: themeName,
126 + set: themeName,
127 + name,
128 + number: l.code,
129 + originalMsrp: l.price,
130 + originalMsrpCurrency: l.price ? 'USD' : null,
131 + identifiers: { lego_set_number: l.code },
132 + metadata: { badges: l.badges, pieces: l.pieces, ages: l.ages, availability: l.availability, retiring_soon: l.badges.some((b) => /retiring/i.test(b)), official_store: true },
133 + });
134 + const rawTitle = `LEGO ${l.code} ${name}${themeName ? ` · ${themeName}` : ''}`;
135 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: l.url, rawTitle, imageUrls: l.image ? [l.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
136 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: l.code, confidence: 0.95, releaseDate: null }));
137 + if (l.price) {
138 + const soldOut = l.badges.some((b) => /sold out|out of stock/i.test(b)) || /out of stock|sold out|retired/i.test(l.availability ?? '');
139 + const coming = l.badges.some((b) => /coming soon|pre-order/i.test(b)) || /coming soon|pre-?order/i.test(l.availability ?? '');
140 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: l.code, confidence: 0.95, listingType: 'fixed_price', price: l.price, currency: 'USD', seller: 'LEGO Shop (official)', location: 'US', condition: { condition: 'sealed', conditionRaw: 'New (official retail)', completeness: 'sealed' }, availability: soldOut ? 'sold' : coming ? 'unknown' : 'available' }));
141 + }
142 + }
143 + return out;
144 + }
145 +}
146 +
147 +export default function createConnector(meta: ConnectorMeta) {
148 + return new LegoShopConnector(meta);
149 +}
added connectors/firecrawl/lego-shop/meta.json +33 −0
@@ -0,0 +1,33 @@
1 +{
2 + "id": "lego-shop",
3 + "displayName": "LEGO.com Shop (official retail price & availability)",
4 + "sourceId": "lego-shop",
5 + "sourceName": "LEGO Shop (official)",
6 + "sourceType": "manufacturer",
7 + "sourceUrl": "https://www.lego.com",
8 + "module": "firecrawl/lego-shop",
9 + "enginePriority": ["firecrawl", "scrapfly"],
10 + "categories": ["lego_sets", "lego"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.95,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.lego.com/en-us/legal/notices-and-policies/terms-of-use",
26 + "accessNotes": "Official manufacturer store. Public theme pages (lego.com/en-us/themes/<theme>?page=N) and product pages are rendered client-side and return 403 to plain non-browser clients, so they are fetched with Firecrawl (1 credit per page; ~22 products per theme page). Parsed per product leaf: set number (from the product URL), name, US retail price, badges (New / Retiring soon / Coming Soon / Exclusive / Hard to find / Sold out), image; product pages add availability, age and piece count. Emits catalog items (MSRP USD, identifiers.lego_set_number shared with PriceCharting/Brickset/BrickEconomy) and the official-store fixed-price listing. robots.txt only disallows account/checkout/identity paths.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "themes": ["star-wars", "icons", "technic", "ideas", "harry-potter", "marvel", "architecture", "ninjago", "creator-expert", "botanical-collection"],
31 + "pagesPerTheme": 1
32 + }
33 +}
added connectors/firecrawl/surugaya/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as any;
18 + p.items = p.items.slice(0, 8);
19 + saveFixture('surugaya', 'pokemon-psa-search-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from suruga-ya.jp search (Pokémon PSA), trimmed to 8 cards' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/firecrawl/surugaya/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import metaJson from './meta.json' with { type: 'json' };
4 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5 +import createConnector, { parseJpCardTitle, yen } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('surugaya', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('emits JPY listings with PSA grades parsed from Japanese titles', async () => {
13 + const fx = loadFixture('surugaya', 'pokemon-psa-search-p1');
14 + const out = await connector.normalize(fx.raw);
15 + const listings = out.filter((r) => r.kind === 'listing');
16 + expect(listings.length).toBeGreaterThan(0);
17 + const graded = out.find((r) => r.kind === 'catalog_item' && r.grade.grader === 'psa');
18 + expect(graded).toBeTruthy();
19 + if (graded?.kind !== 'catalog_item') throw new Error();
20 + expect(graded.attributes.categorySlug).toBe('pokemon');
21 + expect(graded.attributes.number).toMatch(/^\d{3}\/\d{3}$/);
22 + expect(graded.attributes.language).toBe('Japanese');
23 + expect(graded.attributes.identifiers.surugaya_id).toBeTruthy();
24 + for (const l of listings) {
25 + if (l.kind !== 'listing') continue;
26 + expect(l.currency).toBe('JPY');
27 + expect(l.sourceUrl).toMatch(/suruga-ya\.jp\/product\//);
28 + }
29 + });
30 +
31 + it('helpers', () => {
32 + expect(yen('中古：￥12,800')).toBe(12800);
33 + expect(yen('品切れ')).toBeNull();
34 + expect(parseJpCardTitle('114/083[SAR]：【PSA/GEM MT 10】(キラ)メガゲッコウガex')).toEqual({ number: '114/083', rarity: 'SAR', name: 'メガゲッコウガex', grader: 'psa', grade: '10' });
35 + expect(parseJpCardTitle('096/071[SAR]：【PSA/MINT 9】(キラ)ナンジャモ').grade).toBe('9');
36 + expect(parseJpCardTitle('ポケモンカードゲーム デッキシールド').grader).toBeNull();
37 + });
38 +});
added connectors/firecrawl/surugaya/index.ts +168 −0
@@ -0,0 +1,168 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';
4 +import { parseGradeFromTitle } from '@rareindex/taxonomy';
5 +
6 +const BASE = 'https://www.suruga-ya.jp';
7 +const PARSER_VERSION = '1.0.0';
8 +
9 +export const CardSchema = z.object({
10 + id: z.string(),
11 + url: z.string(),
12 + title: z.string(),
13 + image: z.string().nullable(),
14 + typeLabel: z.string().nullable(),
15 + releaseDate: z.string().nullable(),
16 + brand: z.string().nullable(),
17 + price: z.number().nullable(),
18 + soldOut: z.boolean(),
19 + listPrice: z.number().nullable(),
20 + marketplacePrice: z.number().nullable(),
21 + used: z.boolean(),
22 +});
23 +export type Card = z.infer<typeof CardSchema>;
24 +export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), categorySlug: z.string(), items: z.array(CardSchema) });
25 +export type PagePayload = z.infer<typeof PagePayloadSchema>;
26 +
27 +export function yen(s: string | null | undefined): number | null {
28 + const m = s?.replace(/,/g, '').match(/[￥¥]\s*(\d+)/);
29 + return m ? Number(m[1]) : null;
30 +}
31 +
32 +/** Parse a search results page into compact cards. */
33 +export function parseSearchPage(htmlText: string, pageUrl: string, categorySlug: string): PagePayload {
34 + const $ = H.load(htmlText);
35 + const items: Card[] = [];
36 + $('.item').each((_, el) => {
37 + const e = $(el);
38 + const a = e.find('a[href*="/product/detail/"], a[href*="/product/other/"]').first();
39 + const href = a.attr('href');
40 + if (!href) return;
41 + const url = href.split('?')[0]!;
42 + const id = url.match(/\/product\/(?:detail|other)\/([A-Za-z0-9]+)/)?.[1];
43 + const title = H.text(e.find('h3.product-name').first()) ?? H.text(a);
44 + if (!id || !title) return;
45 + const image = e.find('img').first().attr('src') ?? null;
46 + const typeLabel = H.text(e.find('.condition').filter((__, c) => Boolean($(c).text().trim())).first());
47 + const releaseRaw = H.text(e.find('.release_date').first());
48 + const releaseDate = releaseRaw?.match(/(\d{4}\/\d{2}\/\d{2})/)?.[1] ?? null;
49 + const brand = H.text(e.find('.brand').first())?.replace(/^\[|\]$/g, '').trim() || null;
50 + const priceText = H.text(e.find('.item_price .price').first()) ?? '';
51 + const soldOut = /品切れ/.test(priceText);
52 + const price = soldOut ? null : yen(priceText);
53 + const listPrice = yen(H.text(e.find('.price_teika').first()));
54 + const marketplacePrice = yen(e.find('.highlight-box strong').map((__, x) => $(x).text()).get().find((t) => /[￥¥]/.test(t)) ?? null);
55 + const used = /中古/.test(e.text()) || /中古/.test(priceText);
56 + items.push({ id, url: url.startsWith('http') ? url : BASE + url, title, image: image && image.startsWith('http') ? image : image ? BASE + image : null, typeLabel: typeLabel ?? null, releaseDate, brand, price, soldOut, listPrice, marketplacePrice, used });
57 + });
58 + return { kind: 'search_page', url: pageUrl, categorySlug, items };
59 +}
60 +
61 +/**
62 + * Japanese card titles look like "114/083[SAR]：【PSA/GEM MT 10】(キラ)メガゲッコウガex".
63 + * Extract card number, rarity code, grade and the bare name; everything unknown stays null.
64 + */
65 +export function parseJpCardTitle(title: string): { number: string | null; rarity: string | null; name: string; grader: string | null; grade: string | null } {
66 + const number = title.match(/^(\d{1,3}\/\d{1,3})/)?.[1] ?? null;
67 + const rarity = title.match(/\[([A-Z]{1,4})\]/)?.[1] ?? null;
68 + const g = parseGradeFromTitle(title.replace(/【([A-Z]{2,4})\/?/g, '【$1 ').replace(/[【】]/g, ' '));
69 + let name = title.replace(/^\d{1,3}\/\d{1,3}\s*/, '').replace(/\[[A-Z]{1,4}\]\s*[：:]?\s*/, '').replace(/【[^】]*】\s*/g, '').replace(/\((キラ|ミラー|ノーマル)\)\s*/g, '').trim();
70 + if (!name) name = title;
71 + return { number, rarity, name, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grade };
72 +}
73 +
74 +export class SurugayaConnector extends BaseConnector {
75 + readonly version = '1.0.0';
76 + readonly parserVersion = PARSER_VERSION;
77 + protected override minIntervalMs = 2000;
78 +
79 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
80 + const seeds = (this.meta.config.seeds as Array<{ query: string; categorySlug: string }> | undefined) ?? [];
81 + const pages = Number(this.meta.config.pagesPerSeed ?? 1);
82 + const cap = ctx.options.limit;
83 + let count = 0;
84 + const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;
85 + for (let i = startSeed; i < seeds.length; i++) {
86 + const seed = seeds[i]!;
87 + for (let page = 1; page <= pages; page++) {
88 + if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;
89 + const url = `${BASE}/search?${seed.query}${page > 1 ? `&page=${page}` : ''}`;
90 + await this.throttle();
91 + const res = await ctx.fetch(url, {
92 + // Firecrawl only: Scrapfly's rendered fallback costs ~40 credits/page here and adds nothing.
93 + engines: ['firecrawl'],
94 + minQuality: 0.3, // pages where every card is sold out (no price) are still valid catalog data
95 + expect: ['title', 'price'],
96 + parse: (r) => {
97 + const p = r.html ? parseSearchPage(r.html, url, seed.categorySlug) : null;
98 + return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price || x.marketplacePrice) ? 1 : null } : null;
99 + },
100 + });
101 + const payload = res.success && res.html ? parseSearchPage(res.html, url, seed.categorySlug) : null;
102 + if (!payload?.items.length) {
103 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no cards'}`);
104 + break;
105 + }
106 + count++;
107 + yield { url, externalId: `${seed.query}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
108 + }
109 + await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });
110 + }
111 + }
112 +
113 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
114 + const p = PagePayloadSchema.parse(raw.payload);
115 + const out: NormalizedRecord[] = [];
116 + for (const c of p.items) {
117 + const release = c.releaseDate ? parseSourceDate(c.releaseDate) : null;
118 + const year = release ? release.getUTCFullYear() : null;
119 + const isCard = ['pokemon', 'yugioh', 'one_piece_card_game'].includes(p.categorySlug);
120 + const card = isCard ? parseJpCardTitle(c.title) : null;
121 + const attributes = AssetAttributesSchema.parse({
122 + categorySlug: p.categorySlug,
123 + brand: c.brand,
124 + franchise: c.brand,
125 + name: card?.name ?? c.title,
126 + number: card?.number ?? null,
127 + rarity: card?.rarity ?? null,
128 + year,
129 + language: 'Japanese',
130 + region: 'JP',
131 + originalMsrp: c.listPrice,
132 + originalMsrpCurrency: c.listPrice ? 'JPY' : null,
133 + identifiers: { surugaya_id: c.id },
134 + metadata: { type_label: c.typeLabel, release: c.releaseDate },
135 + });
136 + const rawTitle = `${c.title}${c.brand ? ` [${c.brand}]` : ''}${year ? ` (${year})` : ''}`;
137 + const grade = { grader: card?.grader ?? null, grade: card?.grade ?? null, qualifier: null, certificationNumber: null };
138 + const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
139 + out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.8, releaseDate: release }));
140 + const price = c.price ?? c.marketplacePrice;
141 + if (price) {
142 + const viaMarketplace = c.price === null && c.marketplacePrice !== null;
143 + out.push(
144 + NormalizedListingSchema.parse({
145 + kind: 'listing',
146 + ...base,
147 + externalId: viaMarketplace ? `${c.id}:mp` : c.id,
148 + confidence: 0.8,
149 + listingType: 'fixed_price',
150 + price,
151 + currency: 'JPY',
152 + seller: viaMarketplace ? 'Suruga-ya marketplace seller' : 'Suruga-ya',
153 + location: 'Japan',
154 + condition: { condition: c.used || viaMarketplace ? null : 'mint_in_box', conditionRaw: viaMarketplace ? '中古 (marketplace)' : c.used ? '中古' : '新品', completeness: null },
155 + availability: 'available',
156 + }),
157 + );
158 + } else if (c.soldOut) {
159 + out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.7, listingType: 'fixed_price', price: null, currency: 'JPY', seller: 'Suruga-ya', location: 'Japan', availability: 'sold' }));
160 + }
161 + }
162 + return out;
163 + }
164 +}
165 +
166 +export default function createConnector(meta: ConnectorMeta) {
167 + return new SurugayaConnector(meta);
168 +}
added connectors/firecrawl/surugaya/meta.json +40 −0
@@ -0,0 +1,40 @@
1 +{
2 + "id": "surugaya",
3 + "displayName": "Suruga-ya (Japanese second-hand hobby retailer, JPY)",
4 + "sourceId": "surugaya",
5 + "sourceName": "Suruga-ya",
6 + "sourceType": "marketplace",
7 + "sourceUrl": "https://www.suruga-ya.jp",
8 + "module": "firecrawl/surugaya",
9 + "enginePriority": ["firecrawl"],
10 + "categories": ["pokemon", "yugioh", "one_piece_card_game", "gundam", "action_figures", "nintendo_games", "sega_games", "playstation_games", "designer_toys"],
11 + "regions": ["JP"],
12 + "languages": ["ja"],
13 + "currency": ["JPY"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.75,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.suruga-ya.jp/man/index.html",
26 + "accessNotes": "Public search result pages (suruga-ya.jp/search?...; robots.txt: 'Allow: /' for generic agents with Content-Signal search=yes, ai-train=no, use=reference — we only index prices and never train on content). Plain HTTPS returns 403 from data-centre IPs, so pages are fetched through Firecrawl (1 credit per page, ~24 items). Parsed per card: product id (shinaban), title, condition/type label, release date, brand, list price (定価), Suruga-ya price or 品切れ (sold out), and the marketplace (マケプレ) lowest price when Suruga-ya itself is out of stock. JPY. Emits catalog items + fixed-price listings; no sold history is available publicly.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + { "query": "category=5&search_word=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20PSA", "categorySlug": "pokemon", "label": "Pokémon cards PSA" },
32 + { "query": "category=5&search_word=%E3%83%9D%E3%82%B1%E3%83%A2%E3%83%B3%E3%82%AB%E3%83%BC%E3%83%89%20SAR", "categorySlug": "pokemon", "label": "Pokémon SAR" },
33 + { "query": "category=5&search_word=%E9%81%8A%E6%88%AF%E7%8E%8B%20%E3%83%97%E3%83%AA%E3%82%BA%E3%83%9E", "categorySlug": "yugioh", "label": "Yu-Gi-Oh! prismatic" },
34 + { "query": "category=5&search_word=%E3%82%AC%E3%83%B3%E3%83%97%E3%83%A9%20MG", "categorySlug": "gundam", "label": "Gunpla MG" },
35 + { "query": "category=5&search_word=figma", "categorySlug": "action_figures", "label": "figma" },
36 + { "query": "category=5&search_word=%E3%82%B9%E3%83%BC%E3%83%91%E3%83%BC%E3%83%95%E3%82%A1%E3%83%9F%E3%82%B3%E3%83%B3%20%E3%82%BD%E3%83%95%E3%83%88", "categorySlug": "nintendo_games", "label": "Super Famicom software" }
37 + ],
38 + "pagesPerSeed": 1
39 + }
40 +}
added connectors/scrapfly/amiami/_smoke.ts +23 −0
@@ -0,0 +1,23 @@
1 +import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
2 +import metaJson from './meta.json' with { type: 'json' };
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import createConnector from './index.js';
5 +
6 +const save = process.argv.includes('--save');
7 +const meta = ConnectorMetaSchema.parse(metaJson);
8 +const connector = createConnector(meta);
9 +const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });
10 +const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });
11 +let i = 0;
12 +for await (const raw of connector.crawl(ctx)) {
13 + const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
14 + console.log(`raw ${raw.externalId} → ${out.length} records`);
15 + for (const r of out.slice(0, 4)) console.log(' ', JSON.stringify(r).slice(0, 320));
16 + if (save && i === 0) {
17 + const p = raw.payload as { items: unknown[] };
18 + p.items = p.items.slice(0, 6);
19 + saveFixture('amiami', 'figures-preowned-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 6, kinds: ['catalog_item', 'listing'] }, note: 'Captured live from api.amiami.com (pre-owned figures, page 1, trimmed to 6 items)' });
20 + }
21 + i++;
22 +}
23 +console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);
added connectors/scrapfly/amiami/index.test.ts +37 −0
Large diff (37 lines) is not loaded by default to keep the page fast.Load diff
added connectors/scrapfly/amiami/index.ts +188 −0
Large diff (188 lines) is not loaded by default to keep the page fast.Load diff
added connectors/scrapfly/amiami/meta.json +39 −0
Large diff (39 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/amiami/figures-preowned-p1.json +129 −0
Large diff (130 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/apmex/liberty-eagle-category-p1.json +98 −0
Large diff (99 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/fossilera/dinosaur-teeth-p1.json +155 −0
Large diff (156 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/hobbysearch/gundam-mg-search-p1.json +105 −0
Large diff (106 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/lego-shop/star-wars-theme-p1.json +122 −0
Large diff (123 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/ma-shops/denarius-search-p1.json +108 −0
Large diff (109 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/surugaya/pokemon-psa-search-p1.json +137 −0
Large diff (139 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere