SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%
7.5 KB · 184 lines json
Raw Blame History
1{2  "id": "toy-snowman",3  "displayName": "Toy Snowman (Canadian TCG & collector-toy store, CAD)",4  "sourceId": "toy-snowman",5  "sourceName": "Toy Snowman",6  "sourceType": "dealer",7  "sourceUrl": "https://www.toysnowman.ca",8  "module": "api/toy-snowman",9  "enginePriority": [10    "api"11  ],12  "categories": [13    "magic_the_gathering",14    "pokemon",15    "one_piece_card_game",16    "other_tcg",17    "action_figures",18    "gundam",19    "lego_sets",20    "funko",21    "warhammer"22  ],23  "regions": [24    "CA"25  ],26  "languages": [27    "en"28  ],29  "currency": [30    "CAD"31  ],32  "supportsListings": true,33  "supportsSold": false,34  "supportsAuctions": false,35  "supportsImages": true,36  "supportsCatalog": false,37  "supportsPopulation": false,38  "supportsLookup": true,39  "refreshFrequencyMinutes": 720,40  "priority": "medium",41  "trustScore": 0.75,42  "attributionRequired": true,43  "termsUrl": "https://www.toysnowman.ca/policies/terms-of-service",44  "accessNotes": "Toy Snowman (toysnowman.ca) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 22 configured collections (magic-the-gathering-singles, pokemon-tcg-singles, pokemon-tcg, one-piece-tcg, riftbound-tcg-ccg, gundam-card-game-gundam-tcg, star-wars-the-black-series, marvel-legends … (+14 more, see config.collections)) and /products/<handle>.json for URL lookups (~9.5k products; 6k MTG singles, 700 Pokémon singles, ~2k collector figures). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — paints/hobby supplies, Black Library books/codexes, sale umbrella collections, accessories & supplies for toys. No personal data is collected; seller = the store itself.",45  "enabled": true,46  "schemaVersion": "1.0",47  "acquisitionMethod": "Shopify storefront products.json",48  "historicalDepth": "none",49  "requires": [],50  "config": {51    "currency": "CAD",52    "market": "CA",53    "seller": "Toy Snowman",54    "location": "Canada (online)",55    "collections": [56      {57        "handle": "magic-the-gathering-singles",58        "categorySlug": "magic_the_gathering",59        "franchise": "Magic: The Gathering"60      },61      {62        "handle": "pokemon-tcg-singles",63        "categorySlug": "pokemon",64        "franchise": "Pokémon"65      },66      {67        "handle": "pokemon-tcg",68        "categorySlug": "pokemon",69        "franchise": "Pokémon"70      },71      {72        "handle": "one-piece-tcg",73        "categorySlug": "one_piece_card_game",74        "franchise": "One Piece"75      },76      {77        "handle": "riftbound-tcg-ccg",78        "categorySlug": "other_tcg",79        "franchise": "Riftbound"80      },81      {82        "handle": "gundam-card-game-gundam-tcg",83        "categorySlug": "other_tcg",84        "franchise": "Gundam Card Game"85      },86      {87        "handle": "star-wars-the-black-series",88        "categorySlug": "action_figures",89        "franchise": "Star Wars"90      },91      {92        "handle": "marvel-legends",93        "categorySlug": "action_figures",94        "brand": "Hasbro",95        "franchise": "Marvel",96        "series": "Marvel Legends"97      },98      {99        "handle": "sh-monsterarts-sh-figuarts",100        "categorySlug": "action_figures",101        "brand": "Bandai",102        "series": "S.H.Figuarts / S.H.MonsterArts"103      },104      {105        "handle": "four-horsemen-mythic-legion",106        "categorySlug": "action_figures",107        "brand": "Four Horsemen Studios",108        "series": "Mythic Legions"109      },110      {111        "handle": "mythic-legions",112        "categorySlug": "action_figures",113        "brand": "Four Horsemen Studios",114        "series": "Mythic Legions"115      },116      {117        "handle": "neca",118        "categorySlug": "action_figures",119        "brand": "NECA"120      },121      {122        "handle": "transformers",123        "categorySlug": "action_figures",124        "franchise": "Transformers"125      },126      {127        "handle": "godzilla",128        "categorySlug": "action_figures",129        "franchise": "Godzilla"130      },131      {132        "handle": "mcfarlane-toys-1",133        "categorySlug": "action_figures",134        "brand": "McFarlane Toys"135      },136      {137        "handle": "threezero",138        "categorySlug": "action_figures",139        "brand": "threezero"140      },141      {142        "handle": "hiya-toys",143        "categorySlug": "action_figures",144        "brand": "Hiya Toys"145      },146      {147        "handle": "gundam",148        "categorySlug": "gundam",149        "brand": "Bandai",150        "franchise": "Gundam"151      },152      {153        "handle": "lego",154        "categorySlug": "lego_sets",155        "brand": "LEGO"156      },157      {158        "handle": "funko",159        "categorySlug": "funko",160        "brand": "Funko"161      },162      {163        "handle": "warhammer-40k",164        "categorySlug": "warhammer",165        "brand": "Games Workshop",166        "franchise": "Warhammer 40,000"167      },168      {169        "handle": "age-of-sigmar",170        "categorySlug": "warhammer",171        "brand": "Games Workshop",172        "franchise": "Warhammer Age of Sigmar"173      }174    ],175    "rules": [],176    "defaultCategory": null,177    "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live|\\bglue\\b|\\btools?\\b|display case|acrylic case|\\bstand\\b|\\bcodex\\b|\\bbattletome\\b|rulebook|\\bnovel\\b|paperback|hardcover",178    "keepOutOfStock": true,179    "fetchBarcodes": false,180    "wholeShop": false,181    "pageSize": 250182  }183}184