import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type CurrencyCode, type NormalizedRecord, SUPPORTED_CURRENCIES } from '@rareindex/shared'; /** * Discogs — music catalog (pressings of iconic masters) + marketplace "lowest ask" observations. * Raw records: `master_versions` (one page of versions for a master) and `release_stats` * (one release + its marketplace stats). Everything comes from the official public API. */ const API = 'https://api.discogs.com'; const WEB = 'https://www.discogs.com'; const PARSER_VERSION = '1.0.0'; const VersionSchema = z.object({ id: z.number(), label: z.string().nullable().optional(), country: z.string().nullable().optional(), title: z.string(), major_formats: z.array(z.string()).default([]), format: z.string().nullable().optional(), catno: z.string().nullable().optional(), released: z.string().nullable().optional(), thumb: z.string().nullable().optional(), stats: z.object({ community: z.object({ in_wantlist: z.number().optional(), in_collection: z.number().optional() }).optional() }).optional(), }); export type Version = z.infer; const MasterSchema = z.object({ id: z.number(), title: z.string(), year: z.union([z.number(), z.string()]).nullable().optional(), genre: z.array(z.string()).optional(), style: z.array(z.string()).optional(), cover_image: z.string().nullable().optional() }); export const VersionsPayloadSchema = z.object({ kind: z.literal('master_versions'), master: MasterSchema, versions: z.array(VersionSchema), page: z.number() }); export const StatsPayloadSchema = z.object({ kind: z.literal('release_stats'), master: MasterSchema.nullable(), version: VersionSchema, stats: z.object({ num_for_sale: z.number().nullable().optional(), lowest_price: z.object({ value: z.number(), currency: z.string() }).nullable().optional(), blocked_from_sale: z.boolean().optional() }), fetchedAt: z.string(), }); export type VersionsPayload = z.infer; export type StatsPayload = z.infer; /** "Nirvana - Nevermind" → { artist: 'Nirvana', title: 'Nevermind' } */ export function splitMasterTitle(title: string): { artist: string | null; title: string } { const m = title.match(/^(.*?)\s+[-–]\s+(.*)$/); return m ? { artist: m[1]!.trim(), title: m[2]!.trim() } : { artist: null, title: title.trim() }; } /** "Album, Reissue, 180 Gram" → variant string without the plain "Album"/"LP" noise; null when nothing notable. */ export function variantFromFormat(format: string | null | undefined): string | null { if (!format) return null; const parts = format.split(',').map((s) => s.trim()).filter((s) => s && !/^(album|lp|12"|7"|10"|single|ep|stereo|mono)$/i.test(s)); return parts.length ? parts.join(', ') : null; } export function yearFromReleased(released: string | null | undefined): number | null { const m = released?.match(/\b(19|20)\d{2}\b/); return m ? Number(m[0]) : null; } export class DiscogsConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2600; // 25 req/min unauthenticated override readonly urlPatterns = [/^https?:\/\/(www\.)?discogs\.com\/(?:[a-z]{2}\/)?release\/(\d+)/i]; private headers() { return { accept: 'application/vnd.discogs.v2.discogs+json' }; } async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? []; const formats = ((this.meta.config.formats as string[] | undefined) ?? ['Vinyl']).map((f) => f.toLowerCase()); const perMaster = Number(this.meta.config.versionsPerMaster ?? 100); const statsPer = Number(this.meta.config.statsPerMaster ?? 6); const startIdx = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; let count = 0; for (let i = startIdx; i < seeds.length; i++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const q = seeds[i]!; await this.throttle(); const search = await ctx.fetch(`${API}/database/search?q=${encodeURIComponent(q)}&type=master&per_page=3`, { engines: ['api'], headers: this.headers() }); const results = (search.json as { results?: Array<{ id: number; title: string; year?: string; genre?: string[]; style?: string[]; cover_image?: string }> } | null)?.results ?? []; if (!search.success || results.length === 0) { ctx.anomaly('search_failed', `${q}: ${search.error ?? 'no master'}`); continue; } const master = results[0]!; await this.throttle(); const vres = await ctx.fetch(`${API}/masters/${master.id}/versions?per_page=${Math.min(perMaster, 100)}&sort=released&sort_order=asc`, { engines: ['api'], headers: this.headers() }); const all = (vres.json as { versions?: unknown[] } | null)?.versions ?? []; const versions = all.map((v) => VersionSchema.safeParse(v)).filter((r) => r.success).map((r) => r.data).filter((v) => v.major_formats.some((f) => formats.includes(f.toLowerCase()))); if (!vres.success || versions.length === 0) { ctx.anomaly('versions_failed', `${master.title}: ${vres.error ?? 'no versions'}`); continue; } const m = { id: master.id, title: master.title, year: master.year ?? null, genre: master.genre, style: master.style, cover_image: master.cover_image ?? null }; const payload: VersionsPayload = { kind: 'master_versions', master: m, versions, page: 1 }; count++; yield { url: `${WEB}/master/${master.id}`, externalId: `master:${master.id}:p1`, kind: 'catalog_item', engine: 'api', httpStatus: vres.httpStatus, payload, fetchedAt: vres.fetchedAt }; // marketplace stats for the most collected pressings const top = [...versions].sort((a, b) => (b.stats?.community?.in_collection ?? 0) - (a.stats?.community?.in_collection ?? 0)).slice(0, statsPer); for (const v of top) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; await this.throttle(); const sres = await ctx.fetch(`${API}/marketplace/stats/${v.id}`, { engines: ['api'], headers: this.headers(), failOnHttpError: false }); const stats = sres.json as StatsPayload['stats'] | null; if (!sres.success || !stats) { ctx.anomaly('stats_failed', `${v.id}: ${sres.error ?? sres.httpStatus}`); continue; } count++; const sp: StatsPayload = { kind: 'release_stats', master: m, version: v, stats, fetchedAt: sres.fetchedAt.toISOString() }; yield { url: `${WEB}/release/${v.id}`, externalId: `release:${v.id}:stats`, kind: 'price_observation', engine: 'api', httpStatus: sres.httpStatus, payload: sp, fetchedAt: sres.fetchedAt }; } await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async lookup(url: string, ctx: CrawlContext): Promise { const id = url.match(this.urlPatterns[0]!)?.[2]; if (!id) return []; await this.throttle(); const rres = await ctx.fetch(`${API}/releases/${id}`, { engines: ['api'], headers: this.headers() }); const rel = rres.json as { id: number; title: string; artists?: Array<{ name: string }>; labels?: Array<{ name: string; catno?: string }>; country?: string; year?: number; released?: string; formats?: Array<{ name: string; descriptions?: string[] }>; master_id?: number; thumb?: string; images?: Array<{ uri: string }>; community?: { have?: number; want?: number } } | null; if (!rres.success || !rel) return []; await this.throttle(); const sres = await ctx.fetch(`${API}/marketplace/stats/${id}`, { engines: ['api'], headers: this.headers(), failOnHttpError: false }); const artist = rel.artists?.map((a) => a.name.replace(/\s\(\d+\)$/, '')).join(', ') ?? ''; const version: Version = { id: rel.id, label: rel.labels?.[0]?.name ?? null, country: rel.country ?? null, title: rel.title, major_formats: rel.formats?.map((f) => f.name) ?? [], format: rel.formats?.[0]?.descriptions?.join(', ') ?? null, catno: rel.labels?.[0]?.catno ?? null, released: rel.released ?? (rel.year ? String(rel.year) : null), thumb: rel.images?.[0]?.uri ?? rel.thumb ?? null, stats: { community: { in_collection: rel.community?.have, in_wantlist: rel.community?.want } }, }; const master = { id: rel.master_id ?? rel.id, title: `${artist} - ${rel.title}`, year: rel.year ?? null, cover_image: version.thumb }; const payload: StatsPayload = { kind: 'release_stats', master, version, stats: (sres.json as StatsPayload['stats'] | null) ?? {}, fetchedAt: rres.fetchedAt.toISOString() }; return [{ url: `${WEB}/release/${id}`, externalId: `release:${id}:stats`, kind: 'price_observation', engine: 'api', httpStatus: rres.httpStatus, payload, fetchedAt: rres.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = raw.payload as { kind?: string }; if (p?.kind === 'master_versions') { const v = VersionsPayloadSchema.parse(raw.payload); return v.versions.map((ver) => this.catalogFor(v.master, ver, raw.fetchedAt)); } if (p?.kind === 'release_stats') { const s = StatsPayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = [this.catalogFor(s.master ?? { id: s.version.id, title: s.version.title }, s.version, raw.fetchedAt)]; const low = s.stats.lowest_price; const cur = low?.currency?.toUpperCase(); if (low && low.value > 0 && cur && (SUPPORTED_CURRENCIES as readonly string[]).includes(cur)) { const cat = out[0] as Extract; out.push( NormalizedPriceObservationSchema.parse({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: cat.sourceUrl, externalId: `release:${s.version.id}:low`, rawTitle: cat.rawTitle, imageUrls: cat.imageUrls, attributes: cat.attributes, condition: { condition: null, conditionRaw: 'marketplace lowest ask (any condition)', completeness: null }, observedAt: raw.fetchedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'low', price: low.value, currency: cur as CurrencyCode, observationDate: new Date(s.fetchedAt), sampleSize: s.stats.num_for_sale ?? null, }), ); } return out; } throw new Error(`discogs: unknown payload kind ${String(p?.kind)}`); } private catalogFor(master: { id: number; title: string; year?: number | string | null; genre?: string[]; style?: string[]; cover_image?: string | null }, v: Version, fetchedAt: Date) { const { artist, title } = splitMasterTitle(master.title); const year = yearFromReleased(v.released) ?? (master.year ? Number(master.year) || null : null); const variant = variantFromFormat(v.format); const fmt = v.major_formats[0] ?? null; const name = artist ? `${artist} – ${title}` : title; const rawTitle = `${name} · ${[v.label, v.catno].filter(Boolean).join(' ')}${fmt ? ` ${fmt}` : ''}${variant ? ` ${variant}` : ''}${year ? ` (${year}${v.country ? `, ${v.country}` : ''})` : v.country ? ` (${v.country})` : ''}`; const attributes = AssetAttributesSchema.parse({ categorySlug: 'music', brand: v.label ?? null, series: fmt, set: v.label ?? null, name, number: v.catno ?? null, year, variant, country: v.country ?? null, identifiers: { discogs_release_id: String(v.id), discogs_master_id: String(master.id), ...(v.catno ? { catalog_number: v.catno } : {}) }, metadata: { artist, album: title, format: v.format ?? null, major_formats: v.major_formats, genres: master.genre ?? [], styles: master.style ?? [], community_have: v.stats?.community?.in_collection ?? null, community_want: v.stats?.community?.in_wantlist ?? null, }, }); return NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${WEB}/release/${v.id}`, externalId: `release:${v.id}`, rawTitle, imageUrls: [v.thumb, master.cover_image].filter((x): x is string => Boolean(x)), attributes, observedAt: fetchedAt, confidence: 0.95, parserVersion: PARSER_VERSION, releaseDate: v.released && /^\d{4}-\d{2}-\d{2}$/.test(v.released) ? new Date(`${v.released}T00:00:00Z`) : null, }); } } export default function createConnector(meta: ConnectorMeta) { return new DiscogsConnector(meta); }