TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';5import { ShopifyProductSchema, caseSize, fetchShopifyPage, moneyNumber, trimShopifyProduct, watchCategory, watchCompleteness, watchConditionRaw, watchMaterial, watchReferenceFromText, yearFrom } from '../_luxury-lib/index.js';67/** Analog:Shift — vintage & pre-owned watch dealer (Shopify storefront, USD). One product = one watch. */8const BASE = 'https://analogshift.com';9const PARSER_VERSION = '1.0.0';1011export const PagePayloadSchema = z.object({ kind: z.literal('shopify_page'), url: z.string(), seed: z.string(), page: z.number(), products: z.array(ShopifyProductSchema) });12export type PagePayload = z.infer<typeof PagePayloadSchema>;1314/** Spec lines Analog:Shift writes in the description: "Reference: 1675", "Year: 1968", "Case Size: 40mm". */15export function specs(body: string | null | undefined): Record<string, string> {16 const out: Record<string, string> = {};17 if (!body) return out;18 for (const m of body.matchAll(/\b(Reference|Ref\.?|Year|Case Size|Case Diameter|Movement|Caliber|Dial|Bracelet|Strap|Box|Papers|Condition|Material|Case Material)\s*:\s*([^:]{1,60}?)(?=\s+[A-Z][a-z]+(?: [A-Z][a-z]+)?\s*:|$)/g)) {19 const KEYS: Record<string, string> = { ref: 'reference', reference: 'reference', 'case diameter': 'case size', 'case material': 'material' };20 const rawKey = m[1]!.toLowerCase().replace(/\.$/, '');21 const k = KEYS[rawKey] ?? rawKey;22 if (!(k in out)) out[k] = m[2]!.trim();23 }24 return out;25}2627export class AnalogShiftConnector extends BaseConnector {28 readonly version = '1.0.0';29 readonly parserVersion = PARSER_VERSION;30 protected override minIntervalMs = 1500;31 override readonly urlPatterns = [/^https?:\/\/(www\.)?analogshift\.com\/products\/([a-z0-9-]+)/i];3233 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {34 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? ['all'];35 const pages = ctx.options.mode === 'backfill' ? 20 : Number(this.meta.config.pagesPerSeed ?? 3);36 let count = 0;37 for (const seed of seeds) {38 for (let page = 1; page <= pages; page++) {39 if (ctx.signal?.aborted || this.reached(ctx, count)) return;40 await this.throttle();41 const path = seed === 'all' ? '/products.json' : `/collections/${seed}/products.json`;42 const { products, res } = await fetchShopifyPage(ctx, BASE, path, page);43 if (!res.success) {44 ctx.anomaly('page_fetch_failed', `${seed} p${page}: ${res.error ?? res.httpStatus}`);45 break;46 }47 if (products.length === 0) break;48 count++;49 const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}${path}?page=${page}`, seed, page, products: products.map(trimShopifyProduct) };50 yield { url: payload.url, externalId: `collection:${seed}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };51 if (products.length < 250) break;52 }53 }54 }5556 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {57 const handle = url.match(this.urlPatterns[0]!)?.[2];58 if (!handle) return [];59 await this.throttle();60 const res = await ctx.fetch(`${BASE}/products/${handle}.json`, { engines: ['api'], responseType: 'json', minQuality: 0.2 });61 const parsed = z.object({ product: ShopifyProductSchema }).safeParse(res.json);62 if (!res.success || !parsed.success) return [];63 const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}/products/${handle}`, seed: `product:${handle}`, page: 1, products: [trimShopifyProduct(parsed.data.product)] };64 return [{ url: payload.url, externalId: `product:${handle}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];65 }6667 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {68 const p = PagePayloadSchema.parse(raw.payload);69 const out: NormalizedRecord[] = [];70 for (const pr of p.products) {71 const v = pr.variants[0];72 if (!v) continue;73 const price = moneyNumber(v.price);74 const title = pr.title.replace(/\s+/g, ' ').trim();75 const body = pr.body_html ?? '';76 const sp = specs(body);77 const brand = pr.vendor?.trim() || null;78 const categorySlug = watchCategory(brand);79 if (categorySlug === 'other_watches' && /gift card|strap|book|accessor/i.test(pr.product_type ?? '')) continue;80 const ref = sp.reference ?? watchReferenceFromText(title) ?? watchReferenceFromText(body);81 const year = sp.year ? yearFrom(sp.year) : yearFrom(title) ?? yearFrom(body);82 const condRaw = sp.condition ?? watchConditionRaw(body);83 const attributes = AssetAttributesSchema.parse({84 categorySlug,85 brand,86 name: brand && !title.toLowerCase().startsWith(brand.toLowerCase()) ? `${brand} ${title}` : title,87 model: pr.product_type && !/watch/i.test(pr.product_type) ? pr.product_type : null,88 reference: ref,89 year,90 material: sp.material ?? watchMaterial(title) ?? watchMaterial(body),91 size: sp['case size'] ?? caseSize(title) ?? caseSize(body),92 identifiers: { analogshift_sku: v.sku?.trim() || String(pr.id), ...(ref ? { reference: ref } : {}) },93 metadata: { product_type: pr.product_type, tags: pr.tags?.slice(0, 12) ?? [], movement: sp.movement ?? sp.caliber ?? null, dial: sp.dial ?? null, bracelet: sp.bracelet ?? sp.strap ?? null },94 });95 const listedAt = pr.published_at ? new Date(pr.published_at) : null;96 out.push(97 NormalizedListingSchema.parse({98 kind: 'listing',99 connectorId: this.meta.id,100 sourceId: this.meta.sourceId,101 sourceUrl: `${BASE}/products/${pr.handle}`,102 externalId: String(pr.id),103 rawTitle: title,104 description: body || null,105 imageUrls: (pr.images ?? []).map((i) => i.src),106 attributes,107 condition: { condition: normalizeCondition(categorySlug, condRaw), conditionRaw: condRaw, completeness: watchCompleteness(body) },108 observedAt: raw.fetchedAt,109 confidence: 0.85,110 parserVersion: PARSER_VERSION,111 listingType: 'fixed_price',112 price,113 currency: price ? 'USD' : null,114 seller: 'Analog:Shift',115 location: 'US',116 quantity: 1,117 listedAt: listedAt && !Number.isNaN(listedAt.getTime()) ? listedAt : null,118 availability: pr.published_at ? (v.available ? 'available' : 'sold') : 'removed',119 }),120 );121 }122 return out;123 }124}125126export default function createConnector(meta: ConnectorMeta) {127 return new AnalogShiftConnector(meta);128}129