TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';45/**6 * Saatchi Art — original artworks listed for sale. One raw record per browse page (25 works);7 * normalise → one listing per artwork (USD list price; sold works → availability 'sold').8 */910const BASE = 'https://www.saatchiart.com';11const PARSER_VERSION = '1.0.0';1213const SeedSchema = z.object({ path: z.string(), category: z.string() });14type Seed = z.infer<typeof SeedSchema>;1516export const WorkSchema = z.object({17 artworkId: z.string(),18 url: z.string(),19 title: z.string(),20 artistFirstName: z.string().nullable(),21 artistLastName: z.string().nullable(),22 artistId: z.number().nullable(),23 category: z.string().nullable(),24 mediums: z.array(z.string()),25 materials: z.array(z.string()),26 styles: z.array(z.string()),27 subject: z.string().nullable(),28 widthCm: z.number().nullable(),29 heightCm: z.number().nullable(),30 depthCm: z.number().nullable(),31 listPriceCents: z.number().nullable(),32 status: z.string().nullable(),33 sku: z.string().nullable(),34 image: z.string().nullable(),35 countryName: z.string().nullable(),36 uploadedAt: z.number().nullable(),37});38export const PagePayloadSchema = z.object({ kind: z.literal('browse_page'), url: z.string(), seed: SeedSchema, page: z.number(), totalResults: z.number().nullable(), works: z.array(WorkSchema) });39export type PagePayload = z.infer<typeof PagePayloadSchema>;4041export function parseBrowsePage(htmlText: string, url: string, seed: Seed, page: number): PagePayload | null {42 const data = H.nextData(htmlText) as { props?: { pageProps?: { initialState?: { searchProvider?: { serverSideData?: { results?: Array<Record<string, unknown>>; totalResults?: number } } } } } } | null;43 const ssd = data?.props?.pageProps?.initialState?.searchProvider?.serverSideData;44 if (!ssd?.results) return null;45 const works = ssd.results46 .map((r) => {47 const dims = (r.dimensions as { widthInCentimeters?: number; heightInCentimeters?: number; depthInCentimeters?: number } | undefined) ?? {};48 const path = String(r.artworkOriginalUrl ?? '');49 return {50 artworkId: String(r.artworkId ?? ''),51 url: path.startsWith('http') ? path : BASE + path,52 title: String(r.artworkTitle ?? '').trim(),53 artistFirstName: r.artistFirstName ? String(r.artistFirstName) : null,54 artistLastName: r.artistLastName ? String(r.artistLastName) : null,55 artistId: typeof r.artistId === 'number' ? r.artistId : null,56 category: r.category ? String(r.category) : null,57 mediums: Array.isArray(r.mediums) ? (r.mediums as unknown[]).map(String) : [],58 materials: Array.isArray(r.materials) ? (r.materials as unknown[]).map(String) : [],59 styles: Array.isArray(r.styles) ? (r.styles as unknown[]).map(String) : [],60 subject: r.subject ? String(r.subject) : null,61 widthCm: typeof dims.widthInCentimeters === 'number' ? dims.widthInCentimeters : null,62 heightCm: typeof dims.heightInCentimeters === 'number' ? dims.heightInCentimeters : null,63 depthCm: typeof dims.depthInCentimeters === 'number' ? dims.depthInCentimeters : null,64 listPriceCents: typeof r.listPriceInCents === 'number' ? r.listPriceInCents : null,65 status: r.originalArtworkStatus ? String(r.originalArtworkStatus) : null,66 sku: r.sku ? String(r.sku) : null,67 image: r.imageUrl ? String(r.imageUrl) : null,68 countryName: r.countryName ? String(r.countryName) : null,69 uploadedAt: typeof r.artworkUploadedDate === 'number' ? r.artworkUploadedDate : null,70 };71 })72 .filter((w) => w.artworkId && w.title);73 return { kind: 'browse_page', url, seed, page, totalResults: typeof ssd.totalResults === 'number' ? ssd.totalResults : null, works };74}7576function titleCase(s: string): string {77 return s.replace(/\b\w/g, (c) => c.toUpperCase());78}7980export class SaatchiArtConnector extends BaseConnector {81 readonly version = '1.0.0';82 readonly parserVersion = PARSER_VERSION;83 override readonly urlPatterns = [/saatchiart\.com\/(?:en-[a-z]+\/)?art\/[^/]+\/\d+\/\d+\/view/];84 protected override minIntervalMs = 2000;8586 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {87 const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []);88 const pages = Number(this.meta.config.pagesPerSeed ?? 4);89 let count = 0;90 for (const seed of seeds) {91 if (ctx.options.categories?.length && !ctx.options.categories.includes(seed.category)) continue;92 for (let page = 1; page <= pages; page++) {93 if (ctx.signal?.aborted || this.reached(ctx, count)) return;94 const url = `${BASE}${seed.path}${page > 1 ? `?page=${page}` : ''}`;95 await this.throttle();96 const res = await ctx.fetch(url, {97 engines: ['api'],98 responseType: 'text',99 expect: ['title', 'price', 'currency', 'images'],100 parse: (r) => {101 const p = r.html ? parseBrowsePage(r.html, url, seed, page) : null;102 const w = p?.works[0];103 return w ? { title: w.title, price: w.listPriceCents, currency: 'USD', images: w.image ? [w.image] : [] } : null;104 },105 });106 const payload = res.success && res.html ? parseBrowsePage(res.html, url, seed, page) : null;107 if (!payload || payload.works.length === 0) {108 ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);109 break;110 }111 count++;112 yield { url, externalId: `browse:${seed.path}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };113 }114 }115 }116117 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {118 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 });119 if (!res.success || !res.html) return [];120 const data = H.nextData(res.html) as { props?: { pageProps?: { initialState?: { page?: { data?: { pdpArtwork?: Record<string, unknown>; artwork?: Record<string, unknown> } } } } } } | null;121 const a = data?.props?.pageProps?.initialState?.page?.data?.pdpArtwork ?? data?.props?.pageProps?.initialState?.page?.data?.artwork;122 if (!a) return [];123 const dims = (a.dimensions as { widthInCentimeters?: number; heightInCentimeters?: number; depthInCentimeters?: number; width?: number; height?: number; depth?: number } | undefined) ?? {};124 const artist = (a.artist as { firstName?: string; lastName?: string; id?: number } | undefined) ?? {};125 const work: z.infer<typeof WorkSchema> = {126 artworkId: String(a.artworkId ?? a.id ?? url.match(/\/(\d+)\/view/)?.[1] ?? ''),127 url,128 title: String(a.title ?? a.artworkTitle ?? ''),129 artistFirstName: artist.firstName ?? (a.artistFirstName ? String(a.artistFirstName) : null),130 artistLastName: artist.lastName ?? (a.artistLastName ? String(a.artistLastName) : null),131 artistId: artist.id ?? null,132 category: a.category ? String(a.category) : null,133 mediums: Array.isArray(a.mediums) ? (a.mediums as unknown[]).map(String) : [],134 materials: Array.isArray(a.materials) ? (a.materials as unknown[]).map(String) : [],135 styles: Array.isArray(a.styles) ? (a.styles as unknown[]).map(String) : [],136 subject: a.subject ? String(a.subject) : null,137 widthCm: dims.widthInCentimeters ?? dims.width ?? null,138 heightCm: dims.heightInCentimeters ?? dims.height ?? null,139 depthCm: dims.depthInCentimeters ?? dims.depth ?? null,140 listPriceCents: typeof a.listPriceInCents === 'number' ? a.listPriceInCents : typeof a.price === 'number' ? Math.round(a.price * 100) : null,141 status: a.originalArtworkStatus ? String(a.originalArtworkStatus) : null,142 sku: a.sku ? String(a.sku) : null,143 image: a.imageUrl ? String(a.imageUrl) : null,144 countryName: null,145 uploadedAt: null,146 };147 if (!work.artworkId || !work.title) return [];148 const payload: PagePayload = { kind: 'browse_page', url, seed: { path: 'lookup', category: work.category?.toLowerCase() === 'photography' ? 'photography' : 'contemporary_art' }, page: 0, totalResults: null, works: [work] };149 return [{ url, externalId: `artwork:${work.artworkId}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];150 }151152 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {153 const p = PagePayloadSchema.parse(raw.payload);154 const out: NormalizedRecord[] = [];155 for (const w of p.works) {156 const artist = [w.artistFirstName, w.artistLastName].filter(Boolean).map((s) => titleCase(String(s))).join(' ') || null;157 const size = w.widthCm && w.heightCm ? `${w.widthCm} × ${w.heightCm}${w.depthCm ? ` × ${w.depthCm}` : ''} cm` : null;158 const categorySlug = w.category?.toLowerCase() === 'photography' ? 'photography' : p.seed.category;159 const attributes = AssetAttributesSchema.parse({160 categorySlug,161 brand: artist,162 name: w.title,163 material: [...w.mediums, ...w.materials].join(', ') || null,164 size,165 country: w.countryName ? titleCase(w.countryName) : null,166 identifiers: { saatchi_artwork_id: w.artworkId, ...(w.sku ? { saatchi_sku: w.sku } : {}) },167 metadata: { category: w.category, styles: w.styles, subject: w.subject, artist_id: w.artistId, uploaded_at: w.uploadedAt ? new Date(w.uploadedAt * 1000).toISOString() : null, unique_original: true },168 });169 out.push(170 NormalizedListingSchema.parse({171 kind: 'listing',172 connectorId: this.meta.id,173 sourceId: this.meta.sourceId,174 sourceUrl: w.url,175 externalId: w.artworkId,176 rawTitle: artist ? `${artist} — ${w.title}` : w.title,177 description: null,178 imageUrls: w.image ? [w.image] : [],179 attributes,180 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },181 condition: { condition: null, conditionRaw: null, completeness: null },182 observedAt: raw.fetchedAt,183 confidence: 0.85,184 parserVersion: PARSER_VERSION,185 listingType: 'fixed_price',186 price: w.listPriceCents !== null ? w.listPriceCents / 100 : null,187 currency: w.listPriceCents !== null ? 'USD' : null,188 seller: artist,189 location: w.countryName ? titleCase(w.countryName) : null,190 quantity: 1,191 listedAt: w.uploadedAt ? new Date(w.uploadedAt * 1000) : null,192 availability: w.status === 'sold' ? 'sold' : w.status === 'avail' ? 'available' : 'unknown',193 }),194 );195 }196 return out;197 }198}199200export default function createConnector(meta: ConnectorMeta) {201 return new SaatchiArtConnector(meta);202}203