SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%
5.9 KB · 126 lines json
Raw Blame History
1{2  "id": "hobbiesville",3  "displayName": "Hobbiesville (Canadian TCG & collector-toy store, CAD)",4  "sourceId": "hobbiesville",5  "sourceName": "Hobbiesville",6  "sourceType": "dealer",7  "sourceUrl": "https://www.hobbiesville.com",8  "module": "api/hobbiesville",9  "enginePriority": [10    "api"11  ],12  "categories": [13    "magic_the_gathering",14    "flesh_and_blood",15    "disney_lorcana",16    "other_tcg",17    "pokemon",18    "yugioh",19    "funko",20    "lego_sets",21    "action_figures",22    "lego_minifigures"23  ],24  "regions": [25    "CA"26  ],27  "languages": [28    "en"29  ],30  "currency": [31    "CAD"32  ],33  "supportsListings": true,34  "supportsSold": false,35  "supportsAuctions": false,36  "supportsImages": true,37  "supportsCatalog": false,38  "supportsPopulation": false,39  "supportsLookup": true,40  "refreshFrequencyMinutes": 720,41  "priority": "medium",42  "trustScore": 0.75,43  "attributionRequired": true,44  "termsUrl": "https://www.hobbiesville.com/policies/terms-of-service",45  "accessNotes": "Hobbiesville (hobbiesville.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 10 configured collections (magic-singles, fab-singles, disney-lorcana-singles, gundam-card-game-singles, japanese-pokemon, bf-yugioh, funko-pop, lego … (+2 more, see config.collections)) and /products/<handle>.json for URL lookups (~200k products; 21k singles, 8.4k MTG singles). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed; extra merchant rules only target Nutch (Disallow: /) and Ahrefs/MJ12 (Crawl-delay: 10), not User-agent *. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — gift-guide/sale umbrella collections, board games, blind boxes, supplies. No personal data is collected; seller = the store itself.",46  "enabled": true,47  "schemaVersion": "1.0",48  "acquisitionMethod": "Shopify storefront products.json",49  "historicalDepth": "none",50  "requires": [],51  "config": {52    "currency": "CAD",53    "market": "CA",54    "seller": "Hobbiesville",55    "location": "Toronto, ON, Canada",56    "collections": [57      {58        "handle": "magic-singles",59        "categorySlug": "magic_the_gathering",60        "franchise": "Magic: The Gathering"61      },62      {63        "handle": "fab-singles",64        "categorySlug": "flesh_and_blood",65        "franchise": "Flesh and Blood"66      },67      {68        "handle": "disney-lorcana-singles",69        "categorySlug": "disney_lorcana",70        "franchise": "Disney Lorcana"71      },72      {73        "handle": "gundam-card-game-singles",74        "categorySlug": "other_tcg",75        "franchise": "Gundam Card Game"76      },77      {78        "handle": "japanese-pokemon",79        "categorySlug": "pokemon",80        "franchise": "Pokémon"81      },82      {83        "handle": "bf-yugioh",84        "categorySlug": "yugioh",85        "franchise": "Yu-Gi-Oh!"86      },87      {88        "handle": "funko-pop",89        "categorySlug": "funko",90        "brand": "Funko"91      },92      {93        "handle": "lego",94        "categorySlug": "lego_sets",95        "brand": "LEGO"96      },97      {98        "handle": "lego-star-wars",99        "categorySlug": "lego_sets",100        "brand": "LEGO",101        "franchise": "Star Wars"102      },103      {104        "handle": "banpresto-one-piece",105        "categorySlug": "action_figures",106        "brand": "Banpresto",107        "franchise": "One Piece"108      }109    ],110    "rules": [111      {112        "match": "\\blego\\b[\\s\\S]*minifig|minifig[\\s\\S]*\\blego\\b",113        "categorySlug": "lego_minifigures",114        "brand": "LEGO"115      }116    ],117    "defaultCategory": null,118    "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|buylist|submission|event ticket|\\bentry\\b|tournament|store credit|pre-?release event|supplies|card savers?|\\brepack|bulk lot|\\blot of\\b|\\bpaints?\\b|primer|brushes|online (pack|code)|digital code|code card|unused digital|ptcgo|ptcg live",119    "keepOutOfStock": true,120    "fetchBarcodes": false,121    "wholeShop": false,122    "pageSize": 250,123    "titlePattern": "^(?<name>.+?) \\((?<set>[A-Z0-9]{2,6})-(?<number>[A-Za-z0-9]+)\\)"124  }125}126