TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';45/**6 * Novelship — sneaker catalog with last sale / lowest ask read from the server-rendered7 * (React Server Components) payload of public browse pages. One raw record per browse page.8 */910const BASE = 'https://novelship.com';11const PARSER_VERSION = '1.0.0';1213export const ProductSchema = z.object({14 id: z.number(),15 name: z.string(),16 nameSlug: z.string(),17 sku: z.string().nullable(),18 mainBrand: z.string().nullable(),19 subBrand: z.string().nullable(),20 colorway: z.string().nullable(),21 category: z.string().nullable(),22 gender: z.string().nullable(),23 dropDate: z.string().nullable(),24 costRetail: z.number().nullable(),25 lastSalePrice: z.number().nullable(),26 lowestListingPrice: z.number().nullable(),27 salesCount180: z.number().nullable(),28 image: z.string().nullable(),29});30export type Product = z.infer<typeof ProductSchema>;31export const PagePayloadSchema = z.object({ kind: z.literal('browse_page'), url: z.string(), seed: z.string(), page: z.number(), products: z.array(ProductSchema) });32export type PagePayload = z.infer<typeof PagePayloadSchema>;3334const VALUE_RE = '("(?:[^"\\\\]|\\\\.)*"|-?[0-9.]+|null|true|false)';35function decode(v: string | undefined): string | null {36 if (v === undefined || v === 'null') return null;37 if (v.startsWith('"')) return JSON.parse(v.replace(/\\u0026/g, '&')) as string;38 return v;39}40/** Last occurrence of `"key":value` in `seg` (used for keys serialised before name_slug). */41function pickLast(seg: string, key: string): string | null {42 const re = new RegExp(`"${key}":${VALUE_RE}`, 'g');43 let last: string | undefined;44 for (const m of seg.matchAll(re)) last = m[1];45 return decode(last);46}47/** First occurrence of `"key":value` in `seg` (used for keys serialised after name_slug). */48function pickFirst(seg: string, key: string): string | null {49 const m = seg.match(new RegExp(`"${key}":${VALUE_RE}`));50 return decode(m?.[1]);51}52function numOf(v: string | null): number | null {53 if (v === null) return null;54 const n = Number(v);55 return Number.isFinite(n) && n > 0 ? n : null;56}5758/**59 * Extract product objects from the RSC payload embedded in the HTML. Browse pages serialise60 * product keys alphabetically (keys < "name_slug" precede it, keys > follow it), product pages61 * keep insertion order; both are covered by looking on the matching side first.62 */63export function parseBrowsePage(htmlText: string, url: string, seed: string, page: number): PagePayload {64 const s = htmlText.replace(/\\"/g, '"').replace(/\\\\/g, '\\');65 const products = new Map<string, Product>();66 const hits = [...s.matchAll(/"name_slug":"([a-z0-9-]+)"/g)];67 for (let i = 0; i < hits.length; i++) {68 const h = hits[i]!;69 const slug = h[1]!;70 const at = h.index!;71 const prevEnd = i > 0 ? hits[i - 1]!.index! + hits[i - 1]![0].length : Math.max(0, at - 12000);72 const nextStart = i + 1 < hits.length ? hits[i + 1]!.index! : Math.min(s.length, at + 12000);73 const before = s.slice(Math.max(prevEnd, at - 12000), at);74 const after = s.slice(at + h[0].length, Math.min(nextStart, at + 12000));75 const get = (key: string) => (key < 'name_slug' ? (pickLast(before, key) ?? pickFirst(after, key)) : (pickFirst(after, key) ?? pickLast(before, key)));76 const id = Number(get('id'));77 const name = get('name');78 if (!Number.isFinite(id) || !name || products.has(slug)) continue;79 const sales = get('sales_count_180');80 const p: Product = {81 id,82 name,83 nameSlug: slug,84 sku: get('sku'),85 mainBrand: get('main_brand'),86 subBrand: get('sub_brand') || null,87 colorway: get('colorway'),88 category: get('category'),89 gender: get('gender'),90 dropDate: get('drop_date'),91 costRetail: numOf(get('cost_retail')),92 lastSalePrice: numOf(get('last_sale_price')),93 lowestListingPrice: numOf(get('lowest_listing_price')),94 salesCount180: sales === null ? null : Number(sales),95 image: get('image'),96 };97 if (p.sku || p.lastSalePrice || p.lowestListingPrice) products.set(slug, p);98 }99 return { kind: 'browse_page', url, seed, page, products: [...products.values()] };100}101102export function brandCategory(brand: string | null, name: string): string {103 const b = `${brand ?? ''} ${name}`.toLowerCase();104 if (/jordan|nike/.test(b)) return 'nike_jordan';105 if (/adidas|yeezy/.test(b)) return 'adidas_yeezy';106 return 'new_balance_asics_other';107}108109export class NovelshipConnector extends BaseConnector {110 readonly version = '1.0.0';111 readonly parserVersion = PARSER_VERSION;112 protected override minIntervalMs = 1500;113 override readonly urlPatterns = [/^https?:\/\/(www\.)?novelship\.com\/([a-z0-9-]+)$/i];114115 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {116 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];117 const pages = Number(this.meta.config.pagesPerSeed ?? 3);118 let count = 0;119 for (const seed of seeds) {120 for (let page = 1; page <= pages; page++) {121 if (ctx.signal?.aborted || this.reached(ctx, count)) return;122 const url = `${BASE}/sneakers/${seed}${page > 1 ? `?page=${page}` : ''}`;123 await this.throttle();124 const res = await ctx.fetch(url, {125 responseType: 'text',126 expect: ['title', 'price', 'identifiers'],127 parse: (r) => {128 const p = r.html ? parseBrowsePage(r.html, url, seed, page) : null;129 const first = p?.products[0];130 return first ? { title: first.name, price: first.lastSalePrice ?? first.lowestListingPrice, identifiers: first.sku ? { sku: first.sku } : null } : null;131 },132 });133 const payload = res.success && res.html ? parseBrowsePage(res.html, url, seed, page) : null;134 if (!payload || payload.products.length === 0) {135 ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);136 break;137 }138 count++;139 yield { url, externalId: `browse:${seed}:${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };140 }141 }142 }143144 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {145 const slug = url.match(this.urlPatterns[0]!)?.[2];146 if (!slug || ['sneakers', 'apparel', 'collectibles', 'browse'].includes(slug)) return [];147 await this.throttle();148 const res = await ctx.fetch(`${BASE}/${slug}`, { responseType: 'text', minQuality: 0.2 });149 if (!res.success || !res.html) return [];150 const payload = parseBrowsePage(res.html, `${BASE}/${slug}`, `product:${slug}`, 1);151 payload.products = payload.products.filter((p) => p.nameSlug === slug);152 if (!payload.products.length) return [];153 return [{ url: `${BASE}/${slug}`, externalId: `product:${slug}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];154 }155156 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {157 const p = PagePayloadSchema.parse(raw.payload);158 const out: NormalizedRecord[] = [];159 for (const pr of p.products) {160 const year = pr.dropDate?.match(/^(\d{4})/)?.[1];161 const attributes = AssetAttributesSchema.parse({162 categorySlug: brandCategory(pr.mainBrand, pr.name),163 brand: pr.mainBrand,164 series: pr.subBrand,165 name: pr.name.replace(/\s+[A-Z0-9]{2,}-?[A-Z0-9]{2,}$/i, (m0) => (pr.sku && m0.trim() === pr.sku ? '' : m0)).trim(),166 color: pr.colorway,167 year: year ? Number(year) : null,168 originalMsrp: pr.costRetail,169 originalMsrpCurrency: pr.costRetail ? 'USD' : null,170 identifiers: { ...(pr.sku ? { style_code: pr.sku } : {}), novelship_id: String(pr.id) },171 metadata: { gender: pr.gender, category: pr.category, drop_date: pr.dropDate, sales_count_180: pr.salesCount180 },172 });173 const sourceUrl = `${BASE}/${pr.nameSlug}`;174 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle: pr.name, imageUrls: pr.image ? [pr.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };175 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${pr.id}`, confidence: 0.9, releaseDate: pr.dropDate && /^\d{4}-\d{2}-\d{2}/.test(pr.dropDate) ? new Date(`${pr.dropDate.slice(0, 10)}T00:00:00Z`) : null }));176 const cond = { condition: 'new', conditionRaw: 'Brand new (marketplace standard)', completeness: 'with_box' };177 if (pr.lastSalePrice) {178 out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `product:${pr.id}:last_sale`, confidence: 0.7, condition: cond, priceKind: 'last_sale_reported', price: pr.lastSalePrice, currency: 'USD', observationDate: raw.fetchedAt, sampleSize: pr.salesCount180 }));179 }180 if (pr.lowestListingPrice) {181 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `product:${pr.id}:lowest_ask`, confidence: 0.7, condition: cond, listingType: 'ask', price: pr.lowestListingPrice, currency: 'USD', seller: null, availability: 'available', listedAt: null }));182 }183 }184 return out;185 }186}187188export default function createConnector(meta: ConnectorMeta) {189 return new NovelshipConnector(meta);190}191