TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type CurrencyCode, type NormalizedRecord, SUPPORTED_CURRENCIES } from '@rareindex/shared';45/**6 * Discogs — music catalog (pressings of iconic masters) + marketplace "lowest ask" observations.7 * Raw records: `master_versions` (one page of versions for a master) and `release_stats`8 * (one release + its marketplace stats). Everything comes from the official public API.9 */1011const API = 'https://api.discogs.com';12const WEB = 'https://www.discogs.com';13const PARSER_VERSION = '1.0.0';1415const VersionSchema = z.object({16 id: z.number(),17 label: z.string().nullable().optional(),18 country: z.string().nullable().optional(),19 title: z.string(),20 major_formats: z.array(z.string()).default([]),21 format: z.string().nullable().optional(),22 catno: z.string().nullable().optional(),23 released: z.string().nullable().optional(),24 thumb: z.string().nullable().optional(),25 stats: z.object({ community: z.object({ in_wantlist: z.number().optional(), in_collection: z.number().optional() }).optional() }).optional(),26});27export type Version = z.infer<typeof VersionSchema>;2829const MasterSchema = z.object({ id: z.number(), title: z.string(), year: z.union([z.number(), z.string()]).nullable().optional(), genre: z.array(z.string()).optional(), style: z.array(z.string()).optional(), cover_image: z.string().nullable().optional() });3031export const VersionsPayloadSchema = z.object({ kind: z.literal('master_versions'), master: MasterSchema, versions: z.array(VersionSchema), page: z.number() });32export const StatsPayloadSchema = z.object({33 kind: z.literal('release_stats'),34 master: MasterSchema.nullable(),35 version: VersionSchema,36 stats: z.object({ num_for_sale: z.number().nullable().optional(), lowest_price: z.object({ value: z.number(), currency: z.string() }).nullable().optional(), blocked_from_sale: z.boolean().optional() }),37 fetchedAt: z.string(),38});39export type VersionsPayload = z.infer<typeof VersionsPayloadSchema>;40export type StatsPayload = z.infer<typeof StatsPayloadSchema>;4142/** "Nirvana - Nevermind" → { artist: 'Nirvana', title: 'Nevermind' } */43export function splitMasterTitle(title: string): { artist: string | null; title: string } {44 const m = title.match(/^(.*?)\s+[-–]\s+(.*)$/);45 return m ? { artist: m[1]!.trim(), title: m[2]!.trim() } : { artist: null, title: title.trim() };46}4748/** "Album, Reissue, 180 Gram" → variant string without the plain "Album"/"LP" noise; null when nothing notable. */49export function variantFromFormat(format: string | null | undefined): string | null {50 if (!format) return null;51 const parts = format.split(',').map((s) => s.trim()).filter((s) => s && !/^(album|lp|12"|7"|10"|single|ep|stereo|mono)$/i.test(s));52 return parts.length ? parts.join(', ') : null;53}5455export function yearFromReleased(released: string | null | undefined): number | null {56 const m = released?.match(/\b(19|20)\d{2}\b/);57 return m ? Number(m[0]) : null;58}5960export class DiscogsConnector extends BaseConnector {61 readonly version = '1.0.0';62 readonly parserVersion = PARSER_VERSION;63 protected override minIntervalMs = 2600; // 25 req/min unauthenticated64 override readonly urlPatterns = [/^https?:\/\/(www\.)?discogs\.com\/(?:[a-z]{2}\/)?release\/(\d+)/i];6566 private headers() {67 return { accept: 'application/vnd.discogs.v2.discogs+json' };68 }6970 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {71 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];72 const formats = ((this.meta.config.formats as string[] | undefined) ?? ['Vinyl']).map((f) => f.toLowerCase());73 const perMaster = Number(this.meta.config.versionsPerMaster ?? 100);74 const statsPer = Number(this.meta.config.statsPerMaster ?? 6);75 const startIdx = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;76 let count = 0;77 for (let i = startIdx; i < seeds.length; i++) {78 if (ctx.signal?.aborted || this.reached(ctx, count)) break;79 const q = seeds[i]!;80 await this.throttle();81 const search = await ctx.fetch(`${API}/database/search?q=${encodeURIComponent(q)}&type=master&per_page=3`, { engines: ['api'], headers: this.headers() });82 const results = (search.json as { results?: Array<{ id: number; title: string; year?: string; genre?: string[]; style?: string[]; cover_image?: string }> } | null)?.results ?? [];83 if (!search.success || results.length === 0) {84 ctx.anomaly('search_failed', `${q}: ${search.error ?? 'no master'}`);85 continue;86 }87 const master = results[0]!;88 await this.throttle();89 const vres = await ctx.fetch(`${API}/masters/${master.id}/versions?per_page=${Math.min(perMaster, 100)}&sort=released&sort_order=asc`, { engines: ['api'], headers: this.headers() });90 const all = (vres.json as { versions?: unknown[] } | null)?.versions ?? [];91 const versions = all.map((v) => VersionSchema.safeParse(v)).filter((r) => r.success).map((r) => r.data).filter((v) => v.major_formats.some((f) => formats.includes(f.toLowerCase())));92 if (!vres.success || versions.length === 0) {93 ctx.anomaly('versions_failed', `${master.title}: ${vres.error ?? 'no versions'}`);94 continue;95 }96 const m = { id: master.id, title: master.title, year: master.year ?? null, genre: master.genre, style: master.style, cover_image: master.cover_image ?? null };97 const payload: VersionsPayload = { kind: 'master_versions', master: m, versions, page: 1 };98 count++;99 yield { url: `${WEB}/master/${master.id}`, externalId: `master:${master.id}:p1`, kind: 'catalog_item', engine: 'api', httpStatus: vres.httpStatus, payload, fetchedAt: vres.fetchedAt };100 // marketplace stats for the most collected pressings101 const top = [...versions].sort((a, b) => (b.stats?.community?.in_collection ?? 0) - (a.stats?.community?.in_collection ?? 0)).slice(0, statsPer);102 for (const v of top) {103 if (ctx.signal?.aborted || this.reached(ctx, count)) break;104 await this.throttle();105 const sres = await ctx.fetch(`${API}/marketplace/stats/${v.id}`, { engines: ['api'], headers: this.headers(), failOnHttpError: false });106 const stats = sres.json as StatsPayload['stats'] | null;107 if (!sres.success || !stats) {108 ctx.anomaly('stats_failed', `${v.id}: ${sres.error ?? sres.httpStatus}`);109 continue;110 }111 count++;112 const sp: StatsPayload = { kind: 'release_stats', master: m, version: v, stats, fetchedAt: sres.fetchedAt.toISOString() };113 yield { url: `${WEB}/release/${v.id}`, externalId: `release:${v.id}:stats`, kind: 'price_observation', engine: 'api', httpStatus: sres.httpStatus, payload: sp, fetchedAt: sres.fetchedAt };114 }115 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });116 }117 }118119 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {120 const id = url.match(this.urlPatterns[0]!)?.[2];121 if (!id) return [];122 await this.throttle();123 const rres = await ctx.fetch(`${API}/releases/${id}`, { engines: ['api'], headers: this.headers() });124 const rel = rres.json as { id: number; title: string; artists?: Array<{ name: string }>; labels?: Array<{ name: string; catno?: string }>; country?: string; year?: number; released?: string; formats?: Array<{ name: string; descriptions?: string[] }>; master_id?: number; thumb?: string; images?: Array<{ uri: string }>; community?: { have?: number; want?: number } } | null;125 if (!rres.success || !rel) return [];126 await this.throttle();127 const sres = await ctx.fetch(`${API}/marketplace/stats/${id}`, { engines: ['api'], headers: this.headers(), failOnHttpError: false });128 const artist = rel.artists?.map((a) => a.name.replace(/\s\(\d+\)$/, '')).join(', ') ?? '';129 const version: Version = {130 id: rel.id,131 label: rel.labels?.[0]?.name ?? null,132 country: rel.country ?? null,133 title: rel.title,134 major_formats: rel.formats?.map((f) => f.name) ?? [],135 format: rel.formats?.[0]?.descriptions?.join(', ') ?? null,136 catno: rel.labels?.[0]?.catno ?? null,137 released: rel.released ?? (rel.year ? String(rel.year) : null),138 thumb: rel.images?.[0]?.uri ?? rel.thumb ?? null,139 stats: { community: { in_collection: rel.community?.have, in_wantlist: rel.community?.want } },140 };141 const master = { id: rel.master_id ?? rel.id, title: `${artist} - ${rel.title}`, year: rel.year ?? null, cover_image: version.thumb };142 const payload: StatsPayload = { kind: 'release_stats', master, version, stats: (sres.json as StatsPayload['stats'] | null) ?? {}, fetchedAt: rres.fetchedAt.toISOString() };143 return [{ url: `${WEB}/release/${id}`, externalId: `release:${id}:stats`, kind: 'price_observation', engine: 'api', httpStatus: rres.httpStatus, payload, fetchedAt: rres.fetchedAt }];144 }145146 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {147 const p = raw.payload as { kind?: string };148 if (p?.kind === 'master_versions') {149 const v = VersionsPayloadSchema.parse(raw.payload);150 return v.versions.map((ver) => this.catalogFor(v.master, ver, raw.fetchedAt));151 }152 if (p?.kind === 'release_stats') {153 const s = StatsPayloadSchema.parse(raw.payload);154 const out: NormalizedRecord[] = [this.catalogFor(s.master ?? { id: s.version.id, title: s.version.title }, s.version, raw.fetchedAt)];155 const low = s.stats.lowest_price;156 const cur = low?.currency?.toUpperCase();157 if (low && low.value > 0 && cur && (SUPPORTED_CURRENCIES as readonly string[]).includes(cur)) {158 const cat = out[0] as Extract<NormalizedRecord, { kind: 'catalog_item' }>;159 out.push(160 NormalizedPriceObservationSchema.parse({161 kind: 'price_observation',162 connectorId: this.meta.id,163 sourceId: this.meta.sourceId,164 sourceUrl: cat.sourceUrl,165 externalId: `release:${s.version.id}:low`,166 rawTitle: cat.rawTitle,167 imageUrls: cat.imageUrls,168 attributes: cat.attributes,169 condition: { condition: null, conditionRaw: 'marketplace lowest ask (any condition)', completeness: null },170 observedAt: raw.fetchedAt,171 confidence: 0.7,172 parserVersion: PARSER_VERSION,173 priceKind: 'low',174 price: low.value,175 currency: cur as CurrencyCode,176 observationDate: new Date(s.fetchedAt),177 sampleSize: s.stats.num_for_sale ?? null,178 }),179 );180 }181 return out;182 }183 throw new Error(`discogs: unknown payload kind ${String(p?.kind)}`);184 }185186 private catalogFor(master: { id: number; title: string; year?: number | string | null; genre?: string[]; style?: string[]; cover_image?: string | null }, v: Version, fetchedAt: Date) {187 const { artist, title } = splitMasterTitle(master.title);188 const year = yearFromReleased(v.released) ?? (master.year ? Number(master.year) || null : null);189 const variant = variantFromFormat(v.format);190 const fmt = v.major_formats[0] ?? null;191 const name = artist ? `${artist} – ${title}` : title;192 const rawTitle = `${name} · ${[v.label, v.catno].filter(Boolean).join(' ')}${fmt ? ` ${fmt}` : ''}${variant ? ` ${variant}` : ''}${year ? ` (${year}${v.country ? `, ${v.country}` : ''})` : v.country ? ` (${v.country})` : ''}`;193 const attributes = AssetAttributesSchema.parse({194 categorySlug: 'music',195 brand: v.label ?? null,196 series: fmt,197 set: v.label ?? null,198 name,199 number: v.catno ?? null,200 year,201 variant,202 country: v.country ?? null,203 identifiers: { discogs_release_id: String(v.id), discogs_master_id: String(master.id), ...(v.catno ? { catalog_number: v.catno } : {}) },204 metadata: {205 artist,206 album: title,207 format: v.format ?? null,208 major_formats: v.major_formats,209 genres: master.genre ?? [],210 styles: master.style ?? [],211 community_have: v.stats?.community?.in_collection ?? null,212 community_want: v.stats?.community?.in_wantlist ?? null,213 },214 });215 return NormalizedCatalogItemSchema.parse({216 kind: 'catalog_item',217 connectorId: this.meta.id,218 sourceId: this.meta.sourceId,219 sourceUrl: `${WEB}/release/${v.id}`,220 externalId: `release:${v.id}`,221 rawTitle,222 imageUrls: [v.thumb, master.cover_image].filter((x): x is string => Boolean(x)),223 attributes,224 observedAt: fetchedAt,225 confidence: 0.95,226 parserVersion: PARSER_VERSION,227 releaseDate: v.released && /^\d{4}-\d{2}-\d{2}$/.test(v.released) ? new Date(`${v.released}T00:00:00Z`) : null,228 });229 }230}231232export default function createConnector(meta: ConnectorMeta) {233 return new DiscogsConnector(meta);234}235