TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1{2 "$comment": "Central per-domain acquisition policy (SPEC §16). Resolution: exact host → parent domain → defaults. Connectors read it via policyFor(url); never hard-code rate limits or timeouts in connector code. Keys are hosts without www.",3 "version": "1.0",4 "defaults": {5 "minIntervalMs": 1000,6 "concurrency": 2,7 "timeoutMs": 45000,8 "maxRetries": 2,9 "crawlDepth": 3,10 "backfillMaxPages": 200,11 "userAgent": "bot",12 "circuitFailures": 6,13 "circuitCooldownMs": 60000014 },15 "domains": {16 "api.scryfall.com": { "minIntervalMs": 120, "concurrency": 2, "notes": "Scryfall asks for 50–100 ms between requests; bulk files preferred." },17 "api.pokemontcg.io": { "minIntervalMs": 1500, "concurrency": 1, "maxRetries": 4, "timeoutMs": 60000, "notes": "Flaky; 500s frequent; GitHub mirror fallback." },18 "db.ygoprodeck.com": { "minIntervalMs": 100, "concurrency": 2, "notes": "20 req/s documented limit." },19 "tcgcsv.com": { "minIntervalMs": 300, "concurrency": 3 },20 "mtgjson.com": { "minIntervalMs": 2000, "concurrency": 1, "timeoutMs": 180000, "notes": "large gzip bulk files" },21 "pricecharting.com": { "minIntervalMs": 1500, "concurrency": 2, "crawlDepth": 4, "backfillMaxPages": 400 },22 "sportscardspro.com": { "minIntervalMs": 1500, "concurrency": 2 },23 "comicconnect.com": { "minIntervalMs": 2000, "concurrency": 1, "backfillMaxPages": 500 },24 "brickeconomy.com": { "minIntervalMs": 3000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },25 "chrono24.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["scrapfly"], "scrapfly": { "renderJs": true, "asp": true, "maxCreditsPerRun": 2000 } },26 "goldin.co": { "minIntervalMs": 5000, "concurrency": 1, "engines": ["scrapfly"], "scrapfly": { "renderJs": true, "asp": true } },27 "catawiki.com": { "minIntervalMs": 3000, "concurrency": 1, "engines": ["scrapfly", "firecrawl"] },28 "amiami.com": { "minIntervalMs": 3000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja" },29 "suruga-ya.jp": { "minIntervalMs": 3000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja" },30 "aucfree.com": { "minIntervalMs": 3000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja" },31 "auctions.yahoo.co.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["scrapfly"], "scrapfly": { "country": "jp", "renderJs": false } },32 "downloads.s3.cardmarket.com": { "minIntervalMs": 2000, "concurrency": 1, "timeoutMs": 180000, "currency": "EUR", "notes": "Cardmarket public price-guide / product bulk files (daily)." },33 "cardmarket.com": { "minIntervalMs": 5000, "concurrency": 1, "currency": "EUR", "engines": ["api"], "notes": "Only the public bulk files are read; storefront pages are Cloudflare-protected and not crawled." },34 "myshopify.com": { "minIntervalMs": 1500, "concurrency": 1, "notes": "Shopify storefront JSON; polite 1 req / 1.5 s per shop." },35 "api.ebay.com": { "minIntervalMs": 300, "concurrency": 3, "notes": "Browse API (OAuth client credentials); 5 000 calls/day default quota." },36 "openapi.etsy.com": { "minIntervalMs": 300, "concurrency": 2, "notes": "10 req/s, 10 000/day per API key." },37 "api.trademe.co.nz": { "minIntervalMs": 1000, "concurrency": 1, "country": "nz", "currency": "NZD" },38 "app.rakuten.co.jp": { "minIntervalMs": 1100, "concurrency": 1, "country": "jp", "currency": "JPY", "notes": "1 req/s per app id." },39 "shopping.yahooapis.jp": { "minIntervalMs": 1100, "concurrency": 1, "country": "jp", "currency": "JPY" },40 "psacard.com": { "minIntervalMs": 6000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"], "notes": "Only public /cert/<n> verification pages; APR and population require a Collectors login and are never fetched." },41 "cgccomics.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },42 "cgccards.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },43 "beckett.com": { "minIntervalMs": 5000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },44 "gosgc.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["api", "firecrawl", "scrapfly"] },45 "pcgs.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },46 "ngccoin.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },47 "pmgnotes.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },48 "watagames.com": { "minIntervalMs": 4000, "concurrency": 1 },49 "my.taggrading.com": { "minIntervalMs": 4000, "concurrency": 1, "engines": ["firecrawl"], "firecrawl": { "waitForMs": 8000 } },50 "numisbids.com": { "minIntervalMs": 3000, "concurrency": 1, "backfillMaxPages": 1000, "notes": "Deep historical coin-auction archive; archive pages are refreshed at most monthly." },51 "sixbid.com": { "minIntervalMs": 3000, "concurrency": 1 },52 "auctionet.com": { "minIntervalMs": 1000, "concurrency": 2, "currency": "SEK", "notes": "Public JSON API." },53 "hibid.com": { "minIntervalMs": 3000, "concurrency": 1, "engines": ["api", "scrapfly"] },54 "comiclink.com": { "minIntervalMs": 4000, "concurrency": 1, "backfillMaxPages": 600 },55 "mycomicshop.com": { "minIntervalMs": 4000, "concurrency": 1 },56 "watchcharts.com": { "minIntervalMs": 5000, "concurrency": 1, "engines": ["firecrawl", "scrapfly"] },57 "collectiblend.com": { "minIntervalMs": 3000, "concurrency": 1 },58 "buyee.jp": { "minIntervalMs": 4000, "concurrency": 1, "currency": "JPY" },59 "magi.camp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja" },60 "yuyu-tei.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja" },61 "hareruyamtg.com": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY" },62 "cardrush-pokemon.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja" },63 "ruten.com.tw": { "minIntervalMs": 3000, "concurrency": 1, "country": "tw", "currency": "TWD", "locale": "zh" },64 "api.firecrawl.dev": { "minIntervalMs": 0, "concurrency": 8, "timeoutMs": 120000, "maxRetries": 1, "circuitFailures": 20 },65 "api.scrapfly.io": { "minIntervalMs": 0, "concurrency": 6, "timeoutMs": 150000, "maxRetries": 1, "circuitFailures": 20 }66 }67}68