TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { describe, expect, it } from 'vitest';2import { ConnectorMetaSchema } from '../types.js';3import { HostGates, CircuitOpenError } from '../circuit.js';4import { policyFor, setDomains } from '../domains.js';5import { isChallengePage } from '../quality.js';6import { parseRobots, robotsAllows, parseSitemapIndex, parseUrlset, isSitemapIndex } from './sitemap.js';7import { parseFeed } from './rss.js';8import { productsFromHtml } from './schemaorg.js';9import { parsePricesRealised } from './pdf.js';10import { identifiersFromBarcode, mapProduct, StorefrontConfigSchema } from './storefront.js';11import { ShopifyStoreConnector } from './shopify.js';12import { WooCommerceStoreConnector } from './woocommerce.js';1314describe('domain policy', () => {15 it('merges defaults → parent domain → host', () => {16 setDomains({ version: '1', defaults: { minIntervalMs: 1000, concurrency: 2 }, domains: { 'example.com': { minIntervalMs: 3000 }, 'shop.example.com': { concurrency: 1 } } });17 const p = policyFor('https://www.shop.example.com/products.json');18 expect(p.minIntervalMs).toBe(3000);19 expect(p.concurrency).toBe(1);20 expect(policyFor('other.org').minIntervalMs).toBe(1000);21 });22});2324describe('circuit breaker', () => {25 it('opens after consecutive failures and half-opens after cooldown', async () => {26 const gates = new HostGates(() => ({ concurrency: 2, minIntervalMs: 0, circuitFailures: 2, circuitCooldownMs: 50 }));27 (await gates.acquire('h'))();28 gates.report('h', false);29 (await gates.acquire('h'))();30 gates.report('h', false);31 await expect(gates.acquire('h')).rejects.toBeInstanceOf(CircuitOpenError);32 await new Promise((r) => setTimeout(r, 60));33 const rel = await gates.acquire('h');34 rel();35 gates.report('h', true);36 expect(gates.snapshot('h').state).toBe('closed');37 });38});3940describe('challenge detection', () => {41 it('flags short cloudflare interstitials, not long pages mentioning cloudflare', () => {42 expect(isChallengePage({ html: '<html><title>Just a moment...</title><div id="cf-challenge">Enable JavaScript and cookies to continue</div></html>', httpStatus: 403 })).toBe(true);43 expect(isChallengePage({ html: `<html>${'x'.repeat(30_000)} powered by cloudflare</html>`, httpStatus: 200 })).toBe(false);44 expect(isChallengePage({ json: { ok: true } })).toBe(false);45 });46});4748describe('sitemap + robots', () => {49 it('parses indexes, urlsets and robots rules', () => {50 const idx = '<sitemapindex><sitemap><loc>https://a.com/s1.xml.gz</loc><lastmod>2026-01-01</lastmod></sitemap></sitemapindex>';51 expect(isSitemapIndex(idx)).toBe(true);52 expect(parseSitemapIndex(idx)[0]?.loc).toBe('https://a.com/s1.xml.gz');53 const us = '<urlset><url><loc>https://a.com/p/1?x=1&y=2</loc><lastmod>2026-02-02</lastmod><priority>0.8</priority></url></urlset>';54 expect(parseUrlset(us)[0]).toMatchObject({ loc: 'https://a.com/p/1?x=1&y=2', lastmod: '2026-02-02', priority: 0.8 });55 const r = parseRobots('User-agent: *\nDisallow: /search/\nAllow: /search/public\nCrawl-delay: 5\nSitemap: https://a.com/sitemap.xml');56 expect(r.sitemaps).toEqual(['https://a.com/sitemap.xml']);57 expect(r.crawlDelay).toBe(5);58 expect(robotsAllows(r, '/search/x')).toBe(false);59 expect(robotsAllows(r, '/search/public/1')).toBe(true);60 expect(robotsAllows(r, '/items/1')).toBe(true);61 });62});6364describe('rss + schema.org + pdf rows', () => {65 it('parses RSS items', () => {66 const f = parseFeed('<rss><channel><title>T</title><item><title>Lot 1</title><link>https://a.com/1</link><pubDate>Mon, 01 Sep 2026 10:00:00 GMT</pubDate><guid>g1</guid></item></channel></rss>');67 expect(f.title).toBe('T');68 expect(f.items[0]).toMatchObject({ id: 'g1', link: 'https://a.com/1' });69 expect(f.items[0]?.publishedAt?.toISOString()).toBe('2026-09-01T10:00:00.000Z');70 });71 it('extracts JSON-LD products with offers', () => {72 const html = `<script type="application/ld+json">{"@context":"https://schema.org","@type":"Product","name":"Charizard","sku":"PK-4","gtin13":"0820650803000","brand":{"@type":"Brand","name":"Pokemon"},"image":["https://i/1.jpg"],"offers":{"@type":"Offer","price":"1299.00","priceCurrency":"USD","availability":"https://schema.org/InStock"}}</script>`;73 const [p] = productsFromHtml(html);74 expect(p).toMatchObject({ name: 'Charizard', sku: 'PK-4', gtin: '0820650803000', brand: 'Pokemon' });75 expect(p?.offers[0]).toMatchObject({ price: 1299, currency: 'USD', availability: 'available' });76 });77 it('parses prices-realised rows', () => {78 const rows = parsePricesRealised(['Lot 12: $1,250', '13 ........ 1.250,00', 'Header line', '14A 900']);79 expect(rows.map((r) => r.lot)).toEqual(['12', '13', '14A']);80 expect(rows[0]?.priceText).toBe('$1,250');81 });82});8384const meta = ConnectorMetaSchema.parse({85 id: 'demo-shop',86 displayName: 'Demo',87 sourceId: 'demo-shop',88 sourceName: 'Demo',89 sourceType: 'dealer',90 sourceUrl: 'https://demo.example',91 module: 'api/demo-shop',92 enginePriority: ['api'],93 categories: ['pokemon'],94 currency: ['CAD'],95 config: { currency: 'CAD', seller: 'Demo', collections: [{ handle: 'pokemon-singles', categorySlug: 'pokemon', franchise: 'Pokémon' }], rules: [{ match: 'magic|mtg', categorySlug: 'magic_the_gathering' }], defaultCategory: null },96});9798describe('storefront mapping', () => {99 it('maps by collection then rule; excludes supplies; extracts GTINs', () => {100 const cfg = StorefrontConfigSchema.parse(meta.config);101 const base = { id: '1', title: 'Charizard', url: 'u', description: null, vendor: null, productType: null, tags: [], collection: 'pokemon-singles', images: [], publishedAt: null, updatedAt: null, variants: [] };102 expect(mapProduct(cfg, base)?.categorySlug).toBe('pokemon');103 expect(mapProduct(cfg, { ...base, title: 'MTG Black Lotus', collection: null })?.categorySlug).toBe('magic_the_gathering');104 expect(mapProduct(cfg, { ...base, title: 'Dragon Shield Sleeves' })).toBeNull();105 expect(mapProduct(cfg, { ...base, title: 'Random', collection: null })).toBeNull();106 expect(identifiersFromBarcode('820650803000')).toEqual({ upc: '820650803000' });107 expect(identifiersFromBarcode('4521329281940')).toEqual({ jan: '4521329281940', ean: '4521329281940' });108 expect(identifiersFromBarcode('9780306406157')).toEqual({ isbn: '9780306406157', ean: '9780306406157' });109 });110});111112describe('shopify adapter', () => {113 it('normalises a products.json product into per-variant CAD listings with grade + sku', async () => {114 const c = new ShopifyStoreConnector(meta);115 const payload = {116 collection: 'pokemon-singles',117 product: { id: 71, title: 'Charizard - Base Set 4/102 [PSA 9]', handle: 'charizard-base-4', body_html: '<p>Holo</p>', published_at: '2026-08-01T00:00:00Z', vendor: 'Pokemon', product_type: 'Single', tags: ['Base Set'], variants: [{ id: 1, title: 'Near Mint', sku: 'PK-4-NM', price: '1999.99', available: true }, { id: 2, title: 'Lightly Played', sku: 'PK-4-LP', price: '1500.00', available: false }], images: [{ src: 'https://cdn/1.jpg' }] },118 };119 const out = await c.normalize({ url: 'https://demo.example/products/charizard-base-4', kind: 'listing', engine: 'api', payload, fetchedAt: new Date('2026-09-08T00:00:00Z') });120 expect(out).toHaveLength(2);121 const [a, b] = out;122 expect(a?.kind).toBe('listing');123 if (a?.kind === 'listing' && b?.kind === 'listing') {124 expect(a.currency).toBe('CAD');125 expect(a.price).toBe(1999.99);126 expect(a.externalId).toBe('71:1');127 expect(a.attributes.identifiers.sku).toBe('PK-4-NM');128 expect(a.grade.grader).toBe('psa');129 expect(a.grade.grade).toBe('9');130 expect(a.condition.conditionRaw).toBe('Near Mint');131 expect(b.availability).toBe('ended');132 expect(a.attributes.franchise).toBe('Pokémon');133 }134 });135});136137describe('woocommerce adapter', () => {138 it('normalises Store API minor-unit prices with the payload currency', async () => {139 const c = new WooCommerceStoreConnector({ ...meta, config: { currency: 'USD', collections: [{ handle: 'sealed', categorySlug: 'pokemon' }] } });140 const out = await c.normalize({ url: 'https://demo.example/product/x', kind: 'listing', engine: 'api', fetchedAt: new Date(), payload: { category: 'sealed', product: { id: 5, name: 'Booster Box Evolving Skies', permalink: 'https://demo.example/product/x', sku: 'BB-ES', prices: { price: '24999', regular_price: '29999', currency_code: 'GBP', currency_minor_unit: 2 }, images: [], categories: [{ name: 'Sealed', slug: 'sealed' }], tags: [], is_in_stock: true } } });141 expect(out).toHaveLength(1);142 if (out[0]?.kind === 'listing') {143 expect(out[0].price).toBe(249.99);144 expect(out[0].currency).toBe('GBP');145 expect(out[0].condition.completeness).toBeNull();146 }147 });148});149