TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1{2 "domains": {3 "comics.org": {4 "minIntervalMs": 2500,5 "concurrency": 1,6 "maxRetries": 0,7 "engines": [8 "api"9 ],10 "country": "US",11 "currency": "USD",12 "backfillMaxPages": 40,13 "refreshClass": "normal",14 "notes": "GCD public JSON API (no key). robots: Allow / (Content-Signal search=yes, ai-train=no). Throttled server-side: ~60 requests/hour per client — HTTP 429 with Retry-After up to ~1,741 s (verified twice 2026-09-08) → maxRetries 0 so the engine never sleeps on Retry-After; connector stops at its checkpoint. HTML pages and /download/ are Cloudflare-challenged + account-gated: not fetched."15 },16 "comiclink.com": {17 "minIntervalMs": 3000,18 "concurrency": 1,19 "engines": [20 "scrapfly"21 ],22 "scrapfly": {23 "renderJs": false,24 "asp": true,25 "country": "us",26 "maxCreditsPerRun": 6027 },28 "country": "US",29 "currency": "USD",30 "backfillMaxPages": 40,31 "refreshClass": "archive",32 "notes": "robots: Allow / (Cloudflare managed content signals search=yes, ai-train=no). Every page served with a Cloudflare managed JS challenge to non-browser clients → Scrapfly without rendering (~1 credit/page). Only /auctions/auctionschedule.asp and /results/generate_highlights.asp are read; item pages hide prices behind login and are never fetched."33 },34 "mycomicshop.com": {35 "minIntervalMs": 3000,36 "concurrency": 1,37 "timeoutMs": 120000,38 "engines": [39 "scrapfly"40 ],41 "scrapfly": {42 "renderJs": false,43 "asp": true,44 "country": "us",45 "maxCreditsPerRun": 4046 },47 "country": "US",48 "currency": "USD",49 "backfillMaxPages": 60,50 "refreshClass": "normal",51 "notes": "robots: /search allowed; /auctions, /newlistings, /restocked, /account, /cart, /wantlist disallowed (never fetched). Imperva Incapsula interstitial for all non-browser clients → Scrapfly (~1 credit per title page, pages are 1–3 MB)."52 },53 "entertainmentearth.com": {54 "minIntervalMs": 3000,55 "concurrency": 1,56 "engines": [57 "scrapfly"58 ],59 "scrapfly": {60 "renderJs": false,61 "asp": true,62 "country": "us",63 "maxCreditsPerRun": 6064 },65 "country": "US",66 "currency": "USD",67 "backfillMaxPages": 80,68 "refreshClass": "normal",69 "notes": "robots: only /news/* admin paths disallowed; /s/ listing pages allowed; sitemap index at /sitemap/sitemap_index.xml. Cloudflare managed challenge for non-browser clients → Scrapfly without rendering (~1 credit/page). Product pages not fetched."70 },71 "miniaturemarket.com": {72 "minIntervalMs": 10000,73 "concurrency": 1,74 "engines": [75 "api"76 ],77 "country": "US",78 "currency": "USD",79 "backfillMaxPages": 60,80 "refreshClass": "normal",81 "notes": "Shopware 6 storefront, plain HTTPS 200 for the bot UA. robots: Crawl-delay 10 (honoured: 10 s interval), Allow ?p= pagination, Disallow ?dir=/?mode=/?limit=/?SID=, /search/, /checkout/, /account/, /wishlist/ (never used)."82 },83 "videogametrader.com": {84 "minIntervalMs": 1500,85 "concurrency": 2,86 "engines": [87 "api"88 ],89 "country": "US",90 "currency": "USD",91 "refreshClass": "normal",92 "notes": "Shopify storefront; public /collections/<handle>/products.json. robots (Shopify standard) allows products/collections; checkout/cart never touched."93 },94 "creations.mattel.com": {95 "minIntervalMs": 1500,96 "concurrency": 2,97 "engines": [98 "api"99 ],100 "country": "US",101 "currency": "USD",102 "refreshClass": "normal",103 "notes": "Shopify storefront (Mattel Creations); public /collections/<handle>/products.json. robots (Shopify standard) allows products/collections; checkout/cart never touched."104 },105 "estarland.com": {106 "minIntervalMs": 3000,107 "concurrency": 1,108 "engines": [109 "scrapfly"110 ],111 "scrapfly": {112 "renderJs": false,113 "asp": true,114 "country": "us",115 "maxCreditsPerRun": 40116 },117 "country": "US",118 "currency": "USD",119 "backfillMaxPages": 60,120 "refreshClass": "normal",121 "notes": "robots: Allow / except /login, /handler*, /checkout, /shopcart, /forgot-password*, /sandbox; sitemap declared. Cloudflare managed challenge for non-browser clients → Scrapfly without rendering (~1 credit/page). Product pages not fetched."122 }123 }124}125