SPB Git forge

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%
5.8 KB · 121 lines json
Raw Blame History
1{2  "id": "kidrobot",3  "displayName": "Kidrobot (US collector-toy store, USD)",4  "sourceId": "kidrobot",5  "sourceName": "Kidrobot",6  "sourceType": "dealer",7  "sourceUrl": "https://www.kidrobot.com",8  "module": "api/kidrobot",9  "enginePriority": [10    "api"11  ],12  "categories": [13    "designer_toys",14    "sanrio",15    "disney_collectibles",16    "plush"17  ],18  "regions": [19    "US"20  ],21  "languages": [22    "en"23  ],24  "currency": [25    "USD"26  ],27  "supportsListings": true,28  "supportsSold": false,29  "supportsAuctions": false,30  "supportsImages": true,31  "supportsCatalog": false,32  "supportsPopulation": false,33  "supportsLookup": true,34  "refreshFrequencyMinutes": 720,35  "priority": "medium",36  "trustScore": 0.75,37  "attributionRequired": true,38  "termsUrl": "https://www.kidrobot.com/policies/terms-of-service",39  "accessNotes": "Kidrobot (kidrobot.com) runs on Shopify. The connector reads ONLY the public, unauthenticated storefront JSON that every Shopify shop serves to any visitor: /collections/<handle>/products.json?limit=250&page=N for the 10 configured collections (dunny, labbit, art-toys, art-toys-pop-culture, blind-boxes-mini-figures, limited-edition, kidrobot-com-exclusives, hello-kitty-and-friends … (+2 more, see config.collections)) and /products/<handle>.json for URL lookups (~1.9k live products; Dunny 430, limited editions 690, blind boxes 230). robots.txt (verified 2026-09-08, User-agent *): standard Shopify rules — Disallow /admin, /cart, /checkout(s), /orders, /account, /services, /sf_*, /cart.js, /recommendations/products, /cdn/wpm/*.js and the filter/sort crawl traps (/collections/*sort_by*, /collections/*+*, /collections/*filter*&*filter*); no Crawl-delay for *. The /collections/<handle>/products.json and /products/<handle>.json paths we read carry none of those patterns and are allowed. Sitemap declared at /sitemap.xml (not needed). Access behaviour: plain HTTP 200 JSON with the honest RareIndexBot UA, no anti-bot challenge, no login, no key. Market pinning: Shopify Markets localises products.json prices by the requester IP as soon as an Accept-Language header is present (observed on Markets-enabled shops: a Canadian crawler receives CAD-converted prices with content-language en-CA while the JSON carries no currency field). Every request therefore sends the public localization=US cookie — the shop's own home market, exactly what a US visitor gets — so prices are always the declared USD (verified 2026-09-08: 30.00 vs 43.00 on a Markets shop). Politeness: 1 request per 1.5 s, concurrency 1 per host (connectors/domains.d/g3-shops-na.json), crawlDepth pages per collection per incremental run, backfill bounded by backfillMaxPages. Data: asking prices in USD (the shop's declared currency) — these are LISTINGS, never sales (§111); one listing per variant (condition/size/edition), variant SKU kept as identifier, out-of-stock variants kept as 'ended' listings, 0.00-priced placeholders dropped. Dates: published_at only; no fetch time is ever used as a sale date. Deliberately NOT fetched: cart/checkout/account/customer endpoints, per-product .js barcode enrichment (fetchBarcodes=false: one extra request per product is not worth it for listings), the whole-shop /products.json, and unmapped collections — gift-guide/holiday umbrella collections, apparel, accessories, enamel pins, keychains, home goods. No personal data is collected; seller = the store itself.",40  "enabled": true,41  "schemaVersion": "1.0",42  "acquisitionMethod": "Shopify storefront products.json",43  "historicalDepth": "none",44  "requires": [],45  "config": {46    "currency": "USD",47    "market": "US",48    "seller": "Kidrobot",49    "location": "Boulder, CO, USA",50    "collections": [51      {52        "handle": "dunny",53        "categorySlug": "designer_toys",54        "brand": "Kidrobot",55        "series": "Dunny"56      },57      {58        "handle": "labbit",59        "categorySlug": "designer_toys",60        "brand": "Kidrobot",61        "series": "Labbit"62      },63      {64        "handle": "art-toys",65        "categorySlug": "designer_toys",66        "brand": "Kidrobot"67      },68      {69        "handle": "art-toys-pop-culture",70        "categorySlug": "designer_toys",71        "brand": "Kidrobot"72      },73      {74        "handle": "blind-boxes-mini-figures",75        "categorySlug": "designer_toys",76        "brand": "Kidrobot"77      },78      {79        "handle": "limited-edition",80        "categorySlug": "designer_toys",81        "brand": "Kidrobot"82      },83      {84        "handle": "kidrobot-com-exclusives",85        "categorySlug": "designer_toys",86        "brand": "Kidrobot"87      },88      {89        "handle": "hello-kitty-and-friends",90        "categorySlug": "sanrio",91        "brand": "Kidrobot",92        "franchise": "Hello Kitty"93      },94      {95        "handle": "disney",96        "categorySlug": "disney_collectibles",97        "brand": "Kidrobot",98        "franchise": "Disney"99      },100      {101        "handle": "hello-kitty-plush-toys-collectibles",102        "categorySlug": "plush",103        "brand": "Kidrobot",104        "franchise": "Hello Kitty"105      }106    ],107    "rules": [108      {109        "match": "\\| Plush \\||\\bplush\\b",110        "categorySlug": "plush"111      }112    ],113    "defaultCategory": null,114    "exclude": "gift card|mystery|\\| (Apparel|T-Shirts?|Hats?|Accessories|Pins?|Enamel Pins?|Patches?|Socks|Hoodies?|Bags?|Stationery|Stickers?|Prints?|Posters?|Merch|Keychains?|Lanyards?|Pins? Lanyard Set|Books?|Toy Displays?|Home Goods|Drinkware) \\||t-shirt|\\btee\\b|hoodie|sweatshirt|\\bhat\\b|\\bcap\\b|beanie|\\bsocks?\\b|enamel pin|\\bpins? set\\b|\\bpatch\\b|sticker|\\bkeychain\\b|lanyard|\\bmug\\b|\\btote\\b|tumbler|display case|acrylic case|pop protector only|art print|\\bposter\\b|\\bzine\\b|\\bbook\\b|\\bgame\\b|puzzle|blanket|towel|\\bwallet\\b|backpack|\\bshirt\\b",115    "keepOutOfStock": true,116    "fetchBarcodes": false,117    "wholeShop": false,118    "pageSize": 250119  }120}121