SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%
7.0 KB · 162 lines json
Raw Blame History
1{2  "id": "big-b-comics",3  "displayName": "Big B Comics (Canadian comic & sports-card & TCG store, CAD)",4  "sourceId": "big-b-comics",5  "sourceName": "Big B Comics",6  "sourceType": "dealer",7  "sourceUrl": "https://www.bigbcomics.com",8  "module": "api/big-b-comics",9  "enginePriority": [10    "api"11  ],12  "categories": [13    "comics",14    "marvel_comics",15    "dc_comics",16    "independent_comics",17    "non_sport_cards",18    "pokemon",19    "funko",20    "action_figures"21  ],22  "regions": [23    "CA"24  ],25  "languages": [26    "en"27  ],28  "currency": [29    "CAD"30  ],31  "supportsListings": true,32  "supportsSold": false,33  "supportsAuctions": false,34  "supportsImages": true,35  "supportsCatalog": false,36  "supportsPopulation": false,37  "supportsLookup": true,38  "refreshFrequencyMinutes": 720,39  "priority": "medium",40  "trustScore": 0.75,41  "attributionRequired": true,42  "termsUrl": "https://www.bigbcomics.com/policies/terms-of-service",43  "accessNotes": "Big B Comics (bigbcomics.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 18 configured collections (cgc-graded-comics, vintage-comics, comics, ratio-variants, marvel-comics, vintage-spider-man, vintage-x-men, dc-comics … (+10 more, see config.collections)) and /products/<handle>.json for URL lookups (~12.7k products; 3.1k vintage comics, 300 CGC slabs, 400 trading cards). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=CA cookie — the shop's own home market, exactly what a CA visitor gets — so prices are always the declared CAD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in CAD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Third-party grades in titles (PSA/BGS/CGC/ICCS/PMG…) are parsed by parseGradeFromTitle; cert numbers are not extracted. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — graphic novels/manga/TPB collections (books, not collectibles), pre-orders, supplies. No personal data is collected; seller = the store itself.",44  "enabled": true,45  "schemaVersion": "1.0",46  "acquisitionMethod": "Shopify storefront products.json",47  "historicalDepth": "none",48  "requires": [],49  "config": {50    "currency": "CAD",51    "market": "CA",52    "seller": "Big B Comics",53    "location": "Hamilton / Barrie / Niagara Falls, ON, Canada",54    "collections": [55      {56        "handle": "cgc-graded-comics",57        "categorySlug": "comics"58      },59      {60        "handle": "vintage-comics",61        "categorySlug": "comics"62      },63      {64        "handle": "comics",65        "categorySlug": "comics"66      },67      {68        "handle": "ratio-variants",69        "categorySlug": "comics"70      },71      {72        "handle": "marvel-comics",73        "categorySlug": "marvel_comics",74        "franchise": "Marvel"75      },76      {77        "handle": "vintage-spider-man",78        "categorySlug": "marvel_comics",79        "franchise": "Marvel"80      },81      {82        "handle": "vintage-x-men",83        "categorySlug": "marvel_comics",84        "franchise": "Marvel"85      },86      {87        "handle": "dc-comics",88        "categorySlug": "dc_comics",89        "franchise": "DC"90      },91      {92        "handle": "vintage-batman",93        "categorySlug": "dc_comics",94        "franchise": "DC"95      },96      {97        "handle": "indie-comics",98        "categorySlug": "independent_comics"99      },100      {101        "handle": "vintage-star-wars",102        "categorySlug": "comics",103        "franchise": "Star Wars"104      },105      {106        "handle": "trading-cards",107        "categorySlug": "non_sport_cards"108      },109      {110        "handle": "pokemon-cards",111        "categorySlug": "pokemon",112        "franchise": "Pokémon"113      },114      {115        "handle": "funko-pops",116        "categorySlug": "funko",117        "brand": "Funko"118      },119      {120        "handle": "mcfarlane-toys",121        "categorySlug": "action_figures",122        "brand": "McFarlane Toys"123      },124      {125        "handle": "super7-reaction-figures",126        "categorySlug": "action_figures",127        "brand": "Super7",128        "series": "ReAction"129      },130      {131        "handle": "hasbro-toy-group",132        "categorySlug": "action_figures",133        "brand": "Hasbro"134      },135      {136        "handle": "banpresto-toys",137        "categorySlug": "action_figures",138        "brand": "Banpresto"139      }140    ],141    "rules": [142      {143        "match": "^(?=[\\s\\S]*\\b(marvel|spider-?man|x-men|avengers|wolverine|hulk|deadpool|iron man|captain america|fantastic four|daredevil|venom|punisher)\\b)[\\s\\S]*\\| (comics|vintage-comics|cgc-graded-comics|ratio-variants)$",144        "categorySlug": "marvel_comics",145        "franchise": "Marvel"146      },147      {148        "match": "^(?=[\\s\\S]*\\b(dc comics|batman|superman|wonder woman|justice league|the flash|green lantern|aquaman|detective comics|action comics|harley quinn|joker|nightwing|swamp thing)\\b)[\\s\\S]*\\| (comics|vintage-comics|cgc-graded-comics|ratio-variants)$",149        "categorySlug": "dc_comics",150        "franchise": "DC"151      }152    ],153    "defaultCategory": null,154    "exclude": "gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery|bags? (and|&) boards|\\bmylar|comic bags|backing boards|supplies|graphic novel|\\btpb\\b|trade paperback|hardcover|omnibus|\\bmanga\\b",155    "keepOutOfStock": true,156    "fetchBarcodes": false,157    "wholeShop": false,158    "pageSize": 250,159    "titlePattern": "^(?<name>[^#]+?) (?:\\d{4} )?#(?<number>\\d+[A-Za-z]?)"160  }161}162