TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1{2 "domains": {3 "downloads.s3.cardmarket.com": { "minIntervalMs": 2000, "concurrency": 1, "timeoutMs": 240000, "maxRetries": 3, "currency": "EUR", "engines": ["api"], "notes": "Cardmarket public bulk files (price_guide_<idGame>.json 1–26 MB, products_singles/nonsingles). No robots.txt on the bucket (S3 AccessDenied). Only these files are read; the storefront is never crawled." },4 "cardmarket.com": { "minIntervalMs": 5000, "concurrency": 1, "currency": "EUR", "engines": ["api"], "notes": "Cloudflare-protected storefront: NOT crawled (blocked). Covered by the bulk files on downloads.s3.cardmarket.com." },5 "limitlesstcg.com": { "minIntervalMs": 1500, "concurrency": 1, "currency": "USD", "engines": ["api"], "notes": "robots.txt fully permissive. Set index, ?display=list set tables and the page's own /api/cards/<id>/prices XHR. No decklist/tournament pages." },6 "yuyu-tei.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["api"], "notes": "robots.txt: only Crawl-delay for Slurp/bingbot/AhrefsBot; sell pages are public, server-rendered. Buy-list and member pages never fetched." },7 "hareruyamtg.com": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "engines": ["api"], "notes": "robots.txt disallows /en/products/search?*page=*|sort=|order=* (HTML result pages) — never requested; we only call the JSON endpoint /en/products/search/unisearch_api the page itself loads." },8 "magi.camp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["api"], "backfillMaxPages": 60, "notes": "robots.txt allows everything for generic agents; /items/search and /products/search are disallowed for AI crawlers and costly for the site — never called. Series browse pages + item JSON-LD only." },9 "cardrush-mtg.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["api"], "notes": "robots.txt disallows only GPTBot/Bytespider/TikTokSpider/meta-externalagent. Plain HTTP product-list pages." },10 "cardrush-op.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["api"], "notes": "Cardrush One Piece store; same platform/robots as cardrush-mtg.jp." },11 "cardrush.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["api"], "notes": "Cardrush Yu-Gi-Oh! store; same platform/robots as cardrush-mtg.jp." },12 "cardrush-dm.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["api"], "notes": "Cardrush Duel Masters store." },13 "cardrush-digimon.jp": { "minIntervalMs": 4000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["api"], "notes": "Cardrush Digimon store." },14 "cardrush-pokemon.jp": { "minIntervalMs": 8000, "concurrency": 1, "country": "jp", "currency": "JPY", "locale": "ja", "engines": ["scrapfly"], "scrapfly": { "renderJs": false, "asp": true, "country": "jp", "maxCreditsPerRun": 400 }, "backfillMaxPages": 3, "notes": "Cloudflare managed challenge answers 403 to every non-browser client (robots.txt allows * with Content-Signal search=yes, ai-train=no). Public listing pages fetched through Scrapfly ASP, ~25 credits per page — capped to 1 page per category per run." },15 "api.cardtrader.com": { "minIntervalMs": 1000, "concurrency": 1, "currency": "EUR", "engines": ["api"], "notes": "Official REST API v2, Bearer token required (CARDTRADER_API_TOKEN). Docs: 200 req/10 s overall, 10 req/s on /marketplace/products." },16 "cardtrader.com": { "minIntervalMs": 5000, "concurrency": 1, "currency": "EUR", "engines": ["api"], "notes": "Website is Cloudflare-fronted and not crawled; only the API host is used." },17 "pokellector.com": { "minIntervalMs": 60000, "concurrency": 1, "engines": [], "notes": "BLOCKED: Terms of Use forbid automated access to their data ('You agree not to use automated means to interact with the data we store… retrieving prices, images'). Never crawled." },18 "tcdb.com": { "minIntervalMs": 60000, "concurrency": 1, "engines": [], "notes": "BLOCKED: Cloudflare managed challenge on every path including /robots.txt (noindex challenge page); terms cannot even be read by a bot. Never crawled." }19 }20}21