import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { normalizeCondition } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { ShopifyProductSchema, caseSize, fetchShopifyPage, moneyNumber, trimShopifyProduct, watchCategory, watchCompleteness, watchConditionRaw, watchMaterial, watchReferenceFromText, yearFrom } from '../_luxury-lib/index.js'; /** Analog:Shift — vintage & pre-owned watch dealer (Shopify storefront, USD). One product = one watch. */ const BASE = 'https://analogshift.com'; const PARSER_VERSION = '1.0.0'; export const PagePayloadSchema = z.object({ kind: z.literal('shopify_page'), url: z.string(), seed: z.string(), page: z.number(), products: z.array(ShopifyProductSchema) }); export type PagePayload = z.infer; /** Spec lines Analog:Shift writes in the description: "Reference: 1675", "Year: 1968", "Case Size: 40mm". */ export function specs(body: string | null | undefined): Record { const out: Record = {}; if (!body) return out; for (const m of body.matchAll(/\b(Reference|Ref\.?|Year|Case Size|Case Diameter|Movement|Caliber|Dial|Bracelet|Strap|Box|Papers|Condition|Material|Case Material)\s*:\s*([^:]{1,60}?)(?=\s+[A-Z][a-z]+(?: [A-Z][a-z]+)?\s*:|$)/g)) { const KEYS: Record = { ref: 'reference', reference: 'reference', 'case diameter': 'case size', 'case material': 'material' }; const rawKey = m[1]!.toLowerCase().replace(/\.$/, ''); const k = KEYS[rawKey] ?? rawKey; if (!(k in out)) out[k] = m[2]!.trim(); } return out; } export class AnalogShiftConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/(www\.)?analogshift\.com\/products\/([a-z0-9-]+)/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? ['all']; const pages = ctx.options.mode === 'backfill' ? 20 : Number(this.meta.config.pagesPerSeed ?? 3); let count = 0; for (const seed of seeds) { for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; await this.throttle(); const path = seed === 'all' ? '/products.json' : `/collections/${seed}/products.json`; const { products, res } = await fetchShopifyPage(ctx, BASE, path, page); if (!res.success) { ctx.anomaly('page_fetch_failed', `${seed} p${page}: ${res.error ?? res.httpStatus}`); break; } if (products.length === 0) break; count++; const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}${path}?page=${page}`, seed, page, products: products.map(trimShopifyProduct) }; yield { url: payload.url, externalId: `collection:${seed}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (products.length < 250) break; } } } async lookup(url: string, ctx: CrawlContext): Promise { const handle = url.match(this.urlPatterns[0]!)?.[2]; if (!handle) return []; await this.throttle(); const res = await ctx.fetch(`${BASE}/products/${handle}.json`, { engines: ['api'], responseType: 'json', minQuality: 0.2 }); const parsed = z.object({ product: ShopifyProductSchema }).safeParse(res.json); if (!res.success || !parsed.success) return []; const payload: PagePayload = { kind: 'shopify_page', url: `${BASE}/products/${handle}`, seed: `product:${handle}`, page: 1, products: [trimShopifyProduct(parsed.data.product)] }; return [{ url: payload.url, externalId: `product:${handle}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const pr of p.products) { const v = pr.variants[0]; if (!v) continue; const price = moneyNumber(v.price); const title = pr.title.replace(/\s+/g, ' ').trim(); const body = pr.body_html ?? ''; const sp = specs(body); const brand = pr.vendor?.trim() || null; const categorySlug = watchCategory(brand); if (categorySlug === 'other_watches' && /gift card|strap|book|accessor/i.test(pr.product_type ?? '')) continue; const ref = sp.reference ?? watchReferenceFromText(title) ?? watchReferenceFromText(body); const year = sp.year ? yearFrom(sp.year) : yearFrom(title) ?? yearFrom(body); const condRaw = sp.condition ?? watchConditionRaw(body); const attributes = AssetAttributesSchema.parse({ categorySlug, brand, name: brand && !title.toLowerCase().startsWith(brand.toLowerCase()) ? `${brand} ${title}` : title, model: pr.product_type && !/watch/i.test(pr.product_type) ? pr.product_type : null, reference: ref, year, material: sp.material ?? watchMaterial(title) ?? watchMaterial(body), size: sp['case size'] ?? caseSize(title) ?? caseSize(body), identifiers: { analogshift_sku: v.sku?.trim() || String(pr.id), ...(ref ? { reference: ref } : {}) }, metadata: { product_type: pr.product_type, tags: pr.tags?.slice(0, 12) ?? [], movement: sp.movement ?? sp.caliber ?? null, dial: sp.dial ?? null, bracelet: sp.bracelet ?? sp.strap ?? null }, }); const listedAt = pr.published_at ? new Date(pr.published_at) : null; out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${BASE}/products/${pr.handle}`, externalId: String(pr.id), rawTitle: title, description: body || null, imageUrls: (pr.images ?? []).map((i) => i.src), attributes, condition: { condition: normalizeCondition(categorySlug, condRaw), conditionRaw: condRaw, completeness: watchCompleteness(body) }, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price, currency: price ? 'USD' : null, seller: 'Analog:Shift', location: 'US', quantity: 1, listedAt: listedAt && !Number.isNaN(listedAt.getTime()) ? listedAt : null, availability: pr.published_at ? (v.available ? 'available' : 'sold') : 'removed', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new AnalogShiftConnector(meta); }