SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%
8.3 KB · 237 lines json
Raw Blame History
1{2  "id": "toy-tokyo",3  "displayName": "Toy Tokyo (US collector-toy store, USD)",4  "sourceId": "toy-tokyo",5  "sourceName": "Toy Tokyo",6  "sourceType": "dealer",7  "sourceUrl": "https://www.toytokyo.com",8  "module": "api/toy-tokyo",9  "enginePriority": [10    "api"11  ],12  "categories": [13    "funko",14    "designer_toys",15    "action_figures"16  ],17  "regions": [18    "US"19  ],20  "languages": [21    "en"22  ],23  "currency": [24    "USD"25  ],26  "supportsListings": true,27  "supportsSold": false,28  "supportsAuctions": false,29  "supportsImages": true,30  "supportsCatalog": false,31  "supportsPopulation": false,32  "supportsLookup": true,33  "refreshFrequencyMinutes": 720,34  "priority": "medium",35  "trustScore": 0.75,36  "attributionRequired": true,37  "termsUrl": "https://www.toytokyo.com/policies/terms-of-service",38  "accessNotes": "Toy Tokyo (toytokyo.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 31 configured collections (funko, pop-animation, pop-anime, pop-disney, pop-television, pop-marvel, pop-games, pop-dc … (+23 more, see config.collections)) and /products/<handle>.json for URL lookups (~860 Funko Pop! (incl. Toy Tokyo exclusives), 455 Medicom / 160 BE@RBRICK, 390 Banpresto, KAWS, sofubi). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=US cookie — the shop's own home market, exactly what a US visitor gets — so prices are always the declared USD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in USD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — accessories/apparel/enamel pins/keychains/plush, gift and holiday umbrella collections, NTWRK drops. No personal data is collected; seller = the store itself.",39  "enabled": true,40  "schemaVersion": "1.0",41  "acquisitionMethod": "Shopify storefront products.json",42  "historicalDepth": "none",43  "requires": [],44  "config": {45    "currency": "USD",46    "market": "US",47    "seller": "Toy Tokyo",48    "location": "New York, NY, USA",49    "collections": [50      {51        "handle": "funko",52        "categorySlug": "funko",53        "brand": "Funko"54      },55      {56        "handle": "pop-animation",57        "categorySlug": "funko",58        "brand": "Funko"59      },60      {61        "handle": "pop-anime",62        "categorySlug": "funko",63        "brand": "Funko"64      },65      {66        "handle": "pop-disney",67        "categorySlug": "funko",68        "brand": "Funko"69      },70      {71        "handle": "pop-television",72        "categorySlug": "funko",73        "brand": "Funko"74      },75      {76        "handle": "pop-marvel",77        "categorySlug": "funko",78        "brand": "Funko"79      },80      {81        "handle": "pop-games",82        "categorySlug": "funko",83        "brand": "Funko"84      },85      {86        "handle": "pop-dc",87        "categorySlug": "funko",88        "brand": "Funko"89      },90      {91        "handle": "pop-star-wars",92        "categorySlug": "funko",93        "brand": "Funko"94      },95      {96        "handle": "pop-movies",97        "categorySlug": "funko",98        "brand": "Funko"99      },100      {101        "handle": "pop-heroes",102        "categorySlug": "funko",103        "brand": "Funko"104      },105      {106        "handle": "pop-pokemon-1",107        "categorySlug": "funko",108        "brand": "Funko"109      },110      {111        "handle": "pop-icons",112        "categorySlug": "funko",113        "brand": "Funko"114      },115      {116        "handle": "pop-toy-tokyo",117        "categorySlug": "funko",118        "brand": "Funko"119      },120      {121        "handle": "pop-soda",122        "categorySlug": "funko",123        "brand": "Funko"124      },125      {126        "handle": "pop-rocks-1",127        "categorySlug": "funko",128        "brand": "Funko"129      },130      {131        "handle": "medicom-toy",132        "categorySlug": "designer_toys",133        "brand": "Medicom Toy"134      },135      {136        "handle": "be-rbricks",137        "categorySlug": "designer_toys",138        "brand": "Medicom Toy",139        "series": "BE@RBRICK"140      },141      {142        "handle": "1000-be-rbrick",143        "categorySlug": "designer_toys",144        "brand": "Medicom Toy",145        "series": "BE@RBRICK"146      },147      {148        "handle": "100-be-rbrick",149        "categorySlug": "designer_toys",150        "brand": "Medicom Toy",151        "series": "BE@RBRICK"152      },153      {154        "handle": "vcd",155        "categorySlug": "designer_toys",156        "brand": "Medicom Toy",157        "series": "VCD"158      },159      {160        "handle": "kaws",161        "categorySlug": "designer_toys",162        "brand": "KAWS"163      },164      {165        "handle": "sofubi",166        "categorySlug": "designer_toys"167      },168      {169        "handle": "vinyl",170        "categorySlug": "designer_toys"171      },172      {173        "handle": "action-figures",174        "categorySlug": "action_figures"175      },176      {177        "handle": "banpresto",178        "categorySlug": "action_figures",179        "brand": "Banpresto"180      },181      {182        "handle": "bandai",183        "categorySlug": "action_figures",184        "brand": "Bandai"185      },186      {187        "handle": "neca",188        "categorySlug": "action_figures",189        "brand": "NECA"190      },191      {192        "handle": "mezco",193        "categorySlug": "action_figures",194        "brand": "Mezco Toyz"195      },196      {197        "handle": "star-wars",198        "categorySlug": "action_figures",199        "franchise": "Star Wars"200      },201      {202        "handle": "godzilla-1",203        "categorySlug": "action_figures",204        "franchise": "Godzilla"205      }206    ],207    "rules": [208      {209        "match": "\\bfunko\\b|\\bpop!",210        "categorySlug": "funko",211        "brand": "Funko"212      },213      {214        "match": "be@rbrick|bearbrick|be-rbrick",215        "categorySlug": "designer_toys",216        "brand": "Medicom Toy",217        "series": "BE@RBRICK"218      },219      {220        "match": "\\bkaws\\b",221        "categorySlug": "designer_toys",222        "brand": "KAWS"223      },224      {225        "match": "\\bsofubi\\b|\\bsoft vinyl\\b",226        "categorySlug": "designer_toys"227      }228    ],229    "defaultCategory": null,230    "exclude": "gift card|mystery|\\| (Apparel|T-Shirts?|Hats?|Accessories|Pins?|Enamel Pins?|Patches?|Socks|Hoodies?|Bags?|Stationery|Stickers?|Prints?|Posters?|Merch|Keychains?|Lanyards?|Pins? Lanyard Set|Books?|Toy Displays?|Home Goods|Drinkware) \\||t-shirt|\\btee\\b|hoodie|sweatshirt|\\bhat\\b|\\bcap\\b|beanie|\\bsocks?\\b|enamel pin|\\bpins? set\\b|\\bpatch\\b|sticker|\\bkeychain\\b|lanyard|\\bmug\\b|\\btote\\b|tumbler|display case|acrylic case|pop protector only|art print|\\bposter\\b|\\bzine\\b|\\bbook\\b|\\bgame\\b|puzzle|blanket|towel|\\bwallet\\b|backpack|\\bshirt\\b",231    "keepOutOfStock": true,232    "fetchBarcodes": false,233    "wholeShop": false,234    "pageSize": 250235  }236}237