SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%
6.7 KB · 205 lines json
Raw Blame History
1{2  "domains": {3    "totalcards.net": {4      "minIntervalMs": 1500,5      "concurrency": 1,6      "country": "gb",7      "currency": "GBP",8      "locale": "en",9      "crawlDepth": 3,10      "backfillMaxPages": 120,11      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency GBP. Polite 1 req / 1.5 s."12    },13    "thecardvault.co.uk": {14      "minIntervalMs": 1500,15      "concurrency": 1,16      "country": "gb",17      "currency": "GBP",18      "locale": "en",19      "crawlDepth": 3,20      "backfillMaxPages": 120,21      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency GBP. Polite 1 req / 1.5 s."22    },23    "travellingman.com": {24      "minIntervalMs": 1500,25      "concurrency": 1,26      "country": "gb",27      "currency": "GBP",28      "locale": "en",29      "crawlDepth": 3,30      "backfillMaxPages": 120,31      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency GBP. Polite 1 req / 1.5 s."32    },33    "kickgame.co.uk": {34      "minIntervalMs": 1500,35      "concurrency": 1,36      "country": "gb",37      "currency": "GBP",38      "locale": "en",39      "crawlDepth": 3,40      "backfillMaxPages": 120,41      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency GBP. Polite 1 req / 1.5 s."42    },43    "crepslocker.com": {44      "minIntervalMs": 1500,45      "concurrency": 1,46      "country": "gb",47      "currency": "GBP",48      "locale": "en",49      "crawlDepth": 3,50      "backfillMaxPages": 120,51      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency GBP. Polite 1 req / 1.5 s."52    },53    "footdistrict.com": {54      "minIntervalMs": 1500,55      "concurrency": 1,56      "country": "es",57      "currency": "EUR",58      "locale": "es",59      "crawlDepth": 3,60      "backfillMaxPages": 120,61      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency EUR. Polite 1 req / 1.5 s."62    },63    "overkillshop.com": {64      "minIntervalMs": 1500,65      "concurrency": 1,66      "country": "de",67      "currency": "EUR",68      "locale": "en",69      "crawlDepth": 3,70      "backfillMaxPages": 120,71      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency EUR. Polite 1 req / 1.5 s."72    },73    "cherrycollectables.com.au": {74      "minIntervalMs": 1500,75      "concurrency": 1,76      "country": "au",77      "currency": "AUD",78      "locale": "en",79      "crawlDepth": 3,80      "backfillMaxPages": 120,81      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency AUD. Polite 1 req / 1.5 s."82    },83    "goodgames.com.au": {84      "minIntervalMs": 1500,85      "concurrency": 1,86      "country": "au",87      "currency": "AUD",88      "locale": "en",89      "crawlDepth": 3,90      "backfillMaxPages": 120,91      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency AUD. Polite 1 req / 1.5 s."92    },93    "gameology.com.au": {94      "minIntervalMs": 1500,95      "concurrency": 1,96      "country": "au",97      "currency": "AUD",98      "locale": "en",99      "crawlDepth": 3,100      "backfillMaxPages": 120,101      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency AUD. Polite 1 req / 1.5 s."102    },103    "gamesportal.com.au": {104      "minIntervalMs": 1500,105      "concurrency": 1,106      "country": "au",107      "currency": "AUD",108      "locale": "en",109      "crawlDepth": 3,110      "backfillMaxPages": 120,111      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency AUD. Polite 1 req / 1.5 s."112    },113    "kingscomics.com": {114      "minIntervalMs": 1500,115      "concurrency": 1,116      "country": "au",117      "currency": "AUD",118      "locale": "en",119      "crawlDepth": 3,120      "backfillMaxPages": 120,121      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency AUD. Polite 1 req / 1.5 s."122    },123    "cardmerchant.co.nz": {124      "minIntervalMs": 1500,125      "concurrency": 1,126      "country": "nz",127      "currency": "NZD",128      "locale": "en",129      "crawlDepth": 3,130      "backfillMaxPages": 120,131      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency NZD. Polite 1 req / 1.5 s."132    },133    "goblingames.co.nz": {134      "minIntervalMs": 1500,135      "concurrency": 1,136      "country": "nz",137      "currency": "NZD",138      "locale": "en",139      "crawlDepth": 3,140      "backfillMaxPages": 120,141      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency NZD. Polite 1 req / 1.5 s."142    },143    "ninoma.com": {144      "minIntervalMs": 1500,145      "concurrency": 1,146      "country": "jp",147      "currency": "JPY",148      "locale": "en",149      "crawlDepth": 3,150      "backfillMaxPages": 120,151      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency JPY. Polite 1 req / 1.5 s."152    },153    "solarisjapan.com": {154      "minIntervalMs": 1500,155      "concurrency": 1,156      "country": "us",157      "currency": "USD",158      "locale": "en",159      "crawlDepth": 3,160      "backfillMaxPages": 120,161      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency USD. Polite 1 req / 1.5 s."162    },163    "toyzeroplus.com": {164      "minIntervalMs": 1500,165      "concurrency": 1,166      "country": "hk",167      "currency": "HKD",168      "locale": "en",169      "crawlDepth": 3,170      "backfillMaxPages": 120,171      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency HKD. Polite 1 req / 1.5 s."172    },173    "flagshipgames.sg": {174      "minIntervalMs": 1500,175      "concurrency": 1,176      "country": "sg",177      "currency": "SGD",178      "locale": "en",179      "crawlDepth": 3,180      "backfillMaxPages": 120,181      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency SGD. Polite 1 req / 1.5 s."182    },183    "shop.hodinkee.com": {184      "minIntervalMs": 1500,185      "concurrency": 1,186      "country": "us",187      "currency": "USD",188      "locale": "en",189      "crawlDepth": 3,190      "backfillMaxPages": 120,191      "notes": "Shopify storefront JSON; standard Shopify robots (products.json allowed); base currency USD. Polite 1 req / 1.5 s."192    },193    "kapowtoys.co.uk": {194      "minIntervalMs": 1500,195      "concurrency": 1,196      "country": "gb",197      "currency": "GBP",198      "locale": "en",199      "crawlDepth": 3,200      "backfillMaxPages": 120,201      "notes": "WooCommerce Store API; robots allows /wp-json; base currency GBP. Polite 1 req / 1.5 s."202    }203  }204}205