TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { conditionFromParens, moneyNum, normalizeConditionWord, readSeedCursor, yearOrDecade } from '../_g10-lib/index.js';56/**7 * Peyton Street Pens (Santa Cruz, CA) — vintage & modern fountain-pen dealer on BigCommerce. Category8 * pages (/<category>/?page=N, 24 cards) are server-rendered: each <li class="product"> card carries the9 * product URL, title, SKU, brand, price ($, USD) and CDN image. Dealer asking prices → listings.10 */11const BASE = 'https://www.peytonstreetpens.com';12const PARSER_VERSION = '1.0.0';13const PAGE_SIZE = 24;1415export const SeedSchema = z.object({ path: z.string(), brand: z.string().nullable().optional() });16export type Seed = z.infer<typeof SeedSchema>;1718export const CardSchema = z.object({ productId: z.string().nullable(), sku: z.string().nullable(), url: z.string(), title: z.string(), brand: z.string().nullable(), price: z.number().nullable(), rrp: z.number().nullable(), image: z.string().nullable(), soldOut: z.boolean().default(false) });19export type Card = z.infer<typeof CardSchema>;2021export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: SeedSchema, page: z.number().int(), totalPages: z.number().int().nullable(), cards: z.array(CardSchema), snapshot: z.string().optional() });22export type PagePayload = z.infer<typeof PagePayloadSchema>;2324const ConfigSchema = z.object({25 seeds: z.array(SeedSchema).min(1),26 pagesPerSeed: z.number().int().min(1).default(2),27 seedsPerRun: z.number().int().min(1).default(4),28 /** titles matching this are accessories/consumables, not pens */29 exclude: z.string().default('\\b(ink|inks|inkwell|ink well|converter|refills?|leads|notebook|paper|pen case|pen wrap|pouch|box only|parts|sac|o-ring|gift certificate|book|poster|t-shirt|stand|display)\\b'),30});3132/** BigCommerce "brand" on this shop is often the product type (PEN SET, BALLPOINT…) rather than a maker. */33const GENERIC_BRANDS = /^(pen set|pen sets|ballpoint|rollerball|fountain pen|mechanical pencil|pencil|pens?|desk set|dip pen|other|misc|unknown|various|n\/a|none|vintage|new|used)$/i;34const PEN_TYPE = /\b(fountain pen|ballpoint pen|ballpoint|rollerball|roller ball|mechanical pencil|pencil|pen & pencil set|pen and pencil set|pen set|desk set|dip pen|felt tip|fineliner|pen\b)/i;3536export function parseCategoryHtml(htmlText: string): { cards: Card[]; totalPages: number | null } {37 const $ = H.load(htmlText);38 const cards: Card[] = [];39 $('li.product').each((_, el) => {40 const e = $(el);41 const a = e.find('.card-title a').first();42 const url = a.attr('href');43 const title = H.text(a);44 if (!url || !title) return;45 const priceText = H.text(e.find('.price--withoutTax').filter((_, p) => !$(p).hasClass('price--rrp')).first());46 const rrpText = H.text(e.find('.price--rrp').first());47 const img = e.find('img.card-image').first();48 const image = img.attr('data-src') ?? (img.attr('src')?.includes('/products/') ? img.attr('src')! : null);49 cards.push({50 productId: e.find('[data-product-id]').first().attr('data-product-id') ?? null,51 sku: H.text(e.find('.sku-value').first()),52 url: url.startsWith('http') ? url : `${BASE}${url}`,53 title,54 brand: H.text(e.find('.card-text.brand a').first()),55 price: moneyNum(priceText),56 rrp: moneyNum(rrpText),57 image: image ? image.replace(/\/stencil\/\d+x\d+\//, '/stencil/1280x1280/') : null,58 soldOut: /\b(sold out|sold)\b/i.test(e.find('.card-figcaption, .sale-flag, .card-badge, .card-text').text()) || /\bSOLD\b/.test(title),59 });60 });61 const pages = $('a[href*="?page="]').map((_, a) => Number(($(a).attr('href') ?? '').match(/[?&]page=(\d+)/)?.[1] ?? 0)).get().filter((n) => n > 0);62 return { cards, totalPages: pages.length ? Math.max(...pages) : cards.length ? 1 : null };63}6465/** First `n` product cards + pagination anchors (fixture snapshot). */66export function trimCategoryHtml(htmlText: string, n = 3): string {67 const $ = H.load(htmlText);68 const cards = $('li.product').toArray().slice(0, n).map((el) => $.html(el));69 const pages = [...new Set($('a[href*="?page="]').map((_, a) => $(a).attr('href') ?? '').get())].map((h) => `<a href="${h}">p</a>`).join('');70 return `<!doctype html><html><body><ul class="productGrid">${cards.join('\n')}</ul><nav class="pagination">${pages}</nav></body></html>`;71}7273export function pageUrl(seedPath: string, page: number): string {74 const p = seedPath.endsWith('/') ? seedPath : `${seedPath}/`;75 return `${BASE}${p}${page > 1 ? `?page=${page}` : ''}`;76}7778/** "Sheaffer Craftsman Fountain Pen & Pencil Set (1950s) - Burgundy w/GT, …" → brand/model/type. */79export function parsePenTitle(title: string, cardBrand: string | null): { brand: string | null; model: string | null; penType: string | null; variant: string | null } {80 const head = title.split(/\s+-\s+|\s*\(/)[0]?.trim() ?? title;81 const typeM = head.match(PEN_TYPE);82 const penType = typeM ? typeM[0].replace(/\s+/g, ' ').toLowerCase() : null;83 const beforeType = typeM ? head.slice(0, typeM.index).trim() : head;84 let brand = cardBrand && !GENERIC_BRANDS.test(cardBrand) ? cardBrand.replace(/\s+/g, ' ').trim() : null;85 if (brand && /^[A-Z0-9 &'.-]+$/.test(brand) && brand.length > 3) brand = brand.split(' ').map((w) => (w.length > 2 ? w[0] + w.slice(1).toLowerCase() : w)).join(' ');86 const words = beforeType.split(/\s+/).filter(Boolean);87 if (!brand && words.length) brand = /^(montblanc|mont|parker|sheaffer|waterman|wahl|eversharp|pelikan|pilot|namiki|sailor|platinum|nakaya|aurora|omas|visconti|montegrappa|conklin|esterbrook|conway|lamy|kaweco|cross|dunhill|cartier|tiffany|delta|stipula|ranga|lotus|leonardo|scribo|faber-castell|graf|caran|twsbi|danitrio|s\.t\.|st\.)$/i.test(words[0]!) ? (words[0] === 'Conway' || words[0] === 'Wahl' || /^(mont|caran|graf|s\.t\.|st\.)$/i.test(words[0]!) ? words.slice(0, 2).join(' ') : words[0]!) : words[0]!;88 let model: string | null = null;89 if (brand) {90 const rest = beforeType.replace(new RegExp(`^${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*`, 'i'), '').trim();91 model = rest && rest.length <= 40 ? rest : null;92 }93 const variant = title.match(/\s+-\s+([^()]+?)(?:\s*\(|$)/)?.[1]?.trim() ?? null;94 return { brand, model: model || null, penType, variant: variant && variant.length <= 80 ? variant : null };95}9697export class PeytonStreetPensConnector extends BaseConnector {98 readonly version = '1.0.0';99 readonly parserVersion = PARSER_VERSION;100 protected override minIntervalMs = 3000;101 private readonly cfg: z.infer<typeof ConfigSchema>;102103 constructor(meta: ConnectorMeta) {104 super(meta);105 this.cfg = ConfigSchema.parse(meta.config);106 }107108 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {109 const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => SeedSchema.parse({ path: s.startsWith('/') ? s : new URL(s).pathname })) : this.cfg.seeds;110 const backfill = ctx.options.mode === 'backfill';111 const maxPages = backfill ? this.policy.backfillMaxPages : this.cfg.pagesPerSeed;112 const start = readSeedCursor(ctx.options.cursor, seeds.length);113 const seedsThisRun = backfill ? seeds.length : Math.min(seeds.length, this.cfg.seedsPerRun);114 let count = 0;115 let items = 0;116 for (let k = 0; k < seedsThisRun; k++) {117 const seedIndex = (start.seedIndex + k) % seeds.length;118 const seed = seeds[seedIndex]!;119 let page = k === 0 ? start.page : 1;120 for (; page <= maxPages; page++) {121 if (ctx.signal?.aborted || this.reached(ctx, count)) return;122 const url = pageUrl(seed.path, page);123 await this.throttle(url);124 const res = await ctx.fetch(url, {125 engines: ['api'],126 responseType: 'text',127 failOnHttpError: false,128 expect: ['title', 'price', 'currency'],129 parse: (r) => {130 const p = r.html ? parseCategoryHtml(r.html) : null;131 const priced = p?.cards.find((c) => c.price);132 return p?.cards.length ? { title: p.cards[0]!.title, price: priced?.price ?? null, currency: priced ? 'USD' : null } : null;133 },134 minQuality: 0.3,135 });136 if (res.httpStatus === 404) {137 ctx.anomaly('selector_missing', `${url}: category not found (404) — update config.seeds`);138 break;139 }140 const parsed = res.success && res.html ? parseCategoryHtml(res.html) : null;141 if (!parsed) {142 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);143 break;144 }145 if (!parsed.cards.length) {146 if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no product cards`);147 break;148 }149 count++;150 items += parsed.cards.length;151 const payload: PagePayload = { kind: 'listing_page', url, seed, page, totalPages: parsed.totalPages, cards: parsed.cards };152 yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };153 await ctx.setCursor({ seedIndex, page: page + 1, at: new Date().toISOString() });154 await ctx.progress({ page, totalPages: parsed.totalPages, itemsProcessed: items });155 if (parsed.cards.length < PAGE_SIZE || (parsed.totalPages !== null && page >= parsed.totalPages)) break;156 }157 const nextSeed = (seedIndex + 1) % seeds.length;158 await ctx.setCursor({ seedIndex: nextSeed, page: 1, at: new Date().toISOString(), ...(backfill && nextSeed === 0 ? { done: true } : {}) });159 }160 }161162 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {163 const p = PagePayloadSchema.parse(raw.payload);164 const exclude = new RegExp(this.cfg.exclude, 'i');165 const out: NormalizedRecord[] = [];166 for (const c of p.cards) {167 if (!c.price) continue;168 const isPen = PEN_TYPE.test(c.title) || /\bpen\b/i.test(c.title);169 const isLighter = /\blighter\b/i.test(c.title);170 if (!isLighter && (!isPen || (exclude.test(c.title) && !/\b(fountain pen|ballpoint|rollerball|pencil set|pen set)\b/i.test(c.title)))) continue;171 const t = parsePenTitle(c.title, c.brand && !GENERIC_BRANDS.test(c.brand.trim()) ? c.brand : (p.seed.brand ?? null));172 const condRaw = conditionFromParens(c.title);173 const { year, decade } = yearOrDecade(c.title);174 const nib = c.title.match(/\b(extra[- ]fine|fine|medium|broad|stub|italic|flex(?:ible)?|oblique)\b[^,()-]*\bnib\b/i)?.[0] ?? null;175 const attributes = AssetAttributesSchema.parse({176 categorySlug: isLighter ? 'lighters' : 'pens',177 brand: t.brand,178 model: t.model,179 name: t.brand && t.model ? `${t.brand} ${t.model}${t.penType ? ` ${t.penType}` : ''}` : c.title.split(/\s+-\s+|\s*\(/)[0]!.trim(),180 variant: t.variant,181 year,182 identifiers: { ...(c.sku ? { peyton_sku: c.sku } : {}), ...(c.productId ? { peyton_product_id: c.productId } : {}) },183 metadata: { pen_type: t.penType, decade, nib, list_price: c.rrp, seed: p.seed.path, restored: /\brestored\b/i.test(c.title), new_old_stock: /\b(new old stock|nos)\b/i.test(c.title) },184 });185 out.push(186 NormalizedListingSchema.parse({187 kind: 'listing',188 connectorId: this.meta.id,189 sourceId: this.meta.sourceId,190 sourceUrl: c.url,191 externalId: c.sku ?? c.productId ?? c.url,192 rawTitle: c.title,193 imageUrls: c.image ? [c.image] : [],194 attributes,195 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },196 condition: { condition: normalizeConditionWord(condRaw), conditionRaw: condRaw, completeness: /\b(new in box|in box|with box)\b/i.test(c.title) ? 'box_papers' : null },197 observedAt: raw.fetchedAt,198 confidence: 0.78,199 parserVersion: PARSER_VERSION,200 listingType: 'fixed_price',201 price: c.price,202 currency: 'USD',203 seller: 'Peyton Street Pens',204 location: 'Santa Cruz, California, United States',205 availability: c.soldOut ? 'sold' : 'available',206 }),207 );208 }209 return out;210 }211}212213export default function createConnector(meta: ConnectorMeta): PeytonStreetPensConnector {214 return new PeytonStreetPensConnector(meta);215}216