import { describe, expect, it } from 'vitest';
import { ConnectorMetaSchema } from '../types.js';
import { HostGates, CircuitOpenError } from '../circuit.js';
import { policyFor, setDomains } from '../domains.js';
import { isChallengePage } from '../quality.js';
import { parseRobots, robotsAllows, parseSitemapIndex, parseUrlset, isSitemapIndex } from './sitemap.js';
import { parseFeed } from './rss.js';
import { productsFromHtml } from './schemaorg.js';
import { parsePricesRealised } from './pdf.js';
import { identifiersFromBarcode, mapProduct, StorefrontConfigSchema } from './storefront.js';
import { ShopifyStoreConnector } from './shopify.js';
import { WooCommerceStoreConnector } from './woocommerce.js';
describe('domain policy', () => {
it('merges defaults → parent domain → host', () => {
setDomains({ version: '1', defaults: { minIntervalMs: 1000, concurrency: 2 }, domains: { 'example.com': { minIntervalMs: 3000 }, 'shop.example.com': { concurrency: 1 } } });
const p = policyFor('https://www.shop.example.com/products.json');
expect(p.minIntervalMs).toBe(3000);
expect(p.concurrency).toBe(1);
expect(policyFor('other.org').minIntervalMs).toBe(1000);
});
});
describe('circuit breaker', () => {
it('opens after consecutive failures and half-opens after cooldown', async () => {
const gates = new HostGates(() => ({ concurrency: 2, minIntervalMs: 0, circuitFailures: 2, circuitCooldownMs: 50 }));
(await gates.acquire('h'))();
gates.report('h', false);
(await gates.acquire('h'))();
gates.report('h', false);
await expect(gates.acquire('h')).rejects.toBeInstanceOf(CircuitOpenError);
await new Promise((r) => setTimeout(r, 60));
const rel = await gates.acquire('h');
rel();
gates.report('h', true);
expect(gates.snapshot('h').state).toBe('closed');
});
});
describe('challenge detection', () => {
it('flags short cloudflare interstitials, not long pages mentioning cloudflare', () => {
expect(isChallengePage({ html: 'Just a moment...Enable JavaScript and cookies to continue
', httpStatus: 403 })).toBe(true);
expect(isChallengePage({ html: `${'x'.repeat(30_000)} powered by cloudflare`, httpStatus: 200 })).toBe(false);
expect(isChallengePage({ json: { ok: true } })).toBe(false);
});
});
describe('sitemap + robots', () => {
it('parses indexes, urlsets and robots rules', () => {
const idx = 'https://a.com/s1.xml.gz2026-01-01';
expect(isSitemapIndex(idx)).toBe(true);
expect(parseSitemapIndex(idx)[0]?.loc).toBe('https://a.com/s1.xml.gz');
const us = 'https://a.com/p/1?x=1&y=22026-02-020.8';
expect(parseUrlset(us)[0]).toMatchObject({ loc: 'https://a.com/p/1?x=1&y=2', lastmod: '2026-02-02', priority: 0.8 });
const r = parseRobots('User-agent: *\nDisallow: /search/\nAllow: /search/public\nCrawl-delay: 5\nSitemap: https://a.com/sitemap.xml');
expect(r.sitemaps).toEqual(['https://a.com/sitemap.xml']);
expect(r.crawlDelay).toBe(5);
expect(robotsAllows(r, '/search/x')).toBe(false);
expect(robotsAllows(r, '/search/public/1')).toBe(true);
expect(robotsAllows(r, '/items/1')).toBe(true);
});
});
describe('rss + schema.org + pdf rows', () => {
it('parses RSS items', () => {
const f = parseFeed('T- Lot 1https://a.com/1Mon, 01 Sep 2026 10:00:00 GMTg1
');
expect(f.title).toBe('T');
expect(f.items[0]).toMatchObject({ id: 'g1', link: 'https://a.com/1' });
expect(f.items[0]?.publishedAt?.toISOString()).toBe('2026-09-01T10:00:00.000Z');
});
it('extracts JSON-LD products with offers', () => {
const html = ``;
const [p] = productsFromHtml(html);
expect(p).toMatchObject({ name: 'Charizard', sku: 'PK-4', gtin: '0820650803000', brand: 'Pokemon' });
expect(p?.offers[0]).toMatchObject({ price: 1299, currency: 'USD', availability: 'available' });
});
it('parses prices-realised rows', () => {
const rows = parsePricesRealised(['Lot 12: $1,250', '13 ........ 1.250,00', 'Header line', '14A 900']);
expect(rows.map((r) => r.lot)).toEqual(['12', '13', '14A']);
expect(rows[0]?.priceText).toBe('$1,250');
});
});
const meta = ConnectorMetaSchema.parse({
id: 'demo-shop',
displayName: 'Demo',
sourceId: 'demo-shop',
sourceName: 'Demo',
sourceType: 'dealer',
sourceUrl: 'https://demo.example',
module: 'api/demo-shop',
enginePriority: ['api'],
categories: ['pokemon'],
currency: ['CAD'],
config: { currency: 'CAD', seller: 'Demo', collections: [{ handle: 'pokemon-singles', categorySlug: 'pokemon', franchise: 'Pokémon' }], rules: [{ match: 'magic|mtg', categorySlug: 'magic_the_gathering' }], defaultCategory: null },
});
describe('storefront mapping', () => {
it('maps by collection then rule; excludes supplies; extracts GTINs', () => {
const cfg = StorefrontConfigSchema.parse(meta.config);
const base = { id: '1', title: 'Charizard', url: 'u', description: null, vendor: null, productType: null, tags: [], collection: 'pokemon-singles', images: [], publishedAt: null, updatedAt: null, variants: [] };
expect(mapProduct(cfg, base)?.categorySlug).toBe('pokemon');
expect(mapProduct(cfg, { ...base, title: 'MTG Black Lotus', collection: null })?.categorySlug).toBe('magic_the_gathering');
expect(mapProduct(cfg, { ...base, title: 'Dragon Shield Sleeves' })).toBeNull();
expect(mapProduct(cfg, { ...base, title: 'Random', collection: null })).toBeNull();
expect(identifiersFromBarcode('820650803000')).toEqual({ upc: '820650803000' });
expect(identifiersFromBarcode('4521329281940')).toEqual({ jan: '4521329281940', ean: '4521329281940' });
expect(identifiersFromBarcode('9780306406157')).toEqual({ isbn: '9780306406157', ean: '9780306406157' });
});
});
describe('shopify adapter', () => {
it('normalises a products.json product into per-variant CAD listings with grade + sku', async () => {
const c = new ShopifyStoreConnector(meta);
const payload = {
collection: 'pokemon-singles',
product: { id: 71, title: 'Charizard - Base Set 4/102 [PSA 9]', handle: 'charizard-base-4', body_html: 'Holo
', published_at: '2026-08-01T00:00:00Z', vendor: 'Pokemon', product_type: 'Single', tags: ['Base Set'], variants: [{ id: 1, title: 'Near Mint', sku: 'PK-4-NM', price: '1999.99', available: true }, { id: 2, title: 'Lightly Played', sku: 'PK-4-LP', price: '1500.00', available: false }], images: [{ src: 'https://cdn/1.jpg' }] },
};
const out = await c.normalize({ url: 'https://demo.example/products/charizard-base-4', kind: 'listing', engine: 'api', payload, fetchedAt: new Date('2026-09-08T00:00:00Z') });
expect(out).toHaveLength(2);
const [a, b] = out;
expect(a?.kind).toBe('listing');
if (a?.kind === 'listing' && b?.kind === 'listing') {
expect(a.currency).toBe('CAD');
expect(a.price).toBe(1999.99);
expect(a.externalId).toBe('71:1');
expect(a.attributes.identifiers.sku).toBe('PK-4-NM');
expect(a.grade.grader).toBe('psa');
expect(a.grade.grade).toBe('9');
expect(a.condition.conditionRaw).toBe('Near Mint');
expect(b.availability).toBe('ended');
expect(a.attributes.franchise).toBe('Pokémon');
}
});
});
describe('woocommerce adapter', () => {
it('normalises Store API minor-unit prices with the payload currency', async () => {
const c = new WooCommerceStoreConnector({ ...meta, config: { currency: 'USD', collections: [{ handle: 'sealed', categorySlug: 'pokemon' }] } });
const out = await c.normalize({ url: 'https://demo.example/product/x', kind: 'listing', engine: 'api', fetchedAt: new Date(), payload: { category: 'sealed', product: { id: 5, name: 'Booster Box Evolving Skies', permalink: 'https://demo.example/product/x', sku: 'BB-ES', prices: { price: '24999', regular_price: '29999', currency_code: 'GBP', currency_minor_unit: 2 }, images: [], categories: [{ name: 'Sealed', slug: 'sealed' }], tags: [], is_in_stock: true } } });
expect(out).toHaveLength(1);
if (out[0]?.kind === 'listing') {
expect(out[0].price).toBe(249.99);
expect(out[0].currency).toBe('GBP');
expect(out[0].condition.completeness).toBeNull();
}
});
});