spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';45/**6 * Open Library (Internet Archive) — free bibliographic API. Seeded with ISBNs of collectible modern7 * first editions; each seed resolves edition → work → author through the JSON endpoints that8 * robots.txt permits (/isbn, /books, /works, /authors — not /search). Catalog enrichment only.9 */10const BASE = 'https://openlibrary.org';11const PARSER_VERSION = '1.0.0';12const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (market data research; data@rareindex.io)', accept: 'application/json' };1314export const SeedSchema = z.object({ isbn: z.string(), note: z.string().optional() });15export const EditionPayloadSchema = z.object({16  kind: z.literal('edition'),17  isbn: z.string(),18  note: z.string().nullable(),19  edition: z.object({20    key: z.string(),21    title: z.string(),22    subtitle: z.string().nullable(),23    publishers: z.array(z.string()),24    publishDate: z.string().nullable(),25    publishPlaces: z.array(z.string()),26    isbn10: z.array(z.string()),27    isbn13: z.array(z.string()),28    pages: z.number().nullable(),29    editionName: z.string().nullable(),30    covers: z.array(z.number()),31    physicalFormat: z.string().nullable(),32    workKey: z.string().nullable(),33  }),34  work: z.object({ key: z.string(), title: z.string().nullable(), firstPublishDate: z.string().nullable(), subjects: z.array(z.string()) }).nullable(),35  authors: z.array(z.string()),36});37export type EditionPayload = z.infer<typeof EditionPayloadSchema>;3839type J = Record<string, any>;40const strList = (v: unknown): string[] => (Array.isArray(v) ? v.map(String) : []);4142export function trimEdition(e: J) {43  return {44    key: String(e.key ?? ''),45    title: String(e.title ?? ''),46    subtitle: e.subtitle ? String(e.subtitle) : null,47    publishers: strList(e.publishers),48    publishDate: e.publish_date ? String(e.publish_date) : null,49    publishPlaces: strList(e.publish_places),50    isbn10: strList(e.isbn_10),51    isbn13: strList(e.isbn_13),52    pages: typeof e.number_of_pages === 'number' ? e.number_of_pages : null,53    editionName: e.edition_name ? String(e.edition_name) : null,54    covers: Array.isArray(e.covers) ? e.covers.filter((c: unknown) => typeof c === 'number' && c > 0) : [],55    physicalFormat: e.physical_format ? String(e.physical_format) : null,56    workKey: Array.isArray(e.works) && e.works[0]?.key ? String(e.works[0].key) : null,57  };58}5960export class OpenLibraryConnector extends BaseConnector {61  readonly version = '1.0.0';62  readonly parserVersion = PARSER_VERSION;63  protected override minIntervalMs = 1000;6465  private async getJson(ctx: CrawlContext, path: string) {66    await this.throttle();67    return ctx.fetch(`${BASE}${path}`, { engines: ['api'], responseType: 'json', headers: HEADERS, minQuality: 0.2 });68  }6970  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {71    const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []);72    let count = 0;73    const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;74    for (let i = start; i < seeds.length; i++) {75      const seed = seeds[i]!;76      if (ctx.signal?.aborted || this.reached(ctx, count)) return;77      const url = `${BASE}/isbn/${seed.isbn}.json`;78      if (!(await ctx.shouldFetch(url))) continue;79      const ed = await this.getJson(ctx, `/isbn/${seed.isbn}.json`);80      if (!ed.success || !ed.json || typeof ed.json !== 'object') {81        ctx.anomaly('page_fetch_failed', `${seed.isbn}: ${ed.error ?? ed.httpStatus}`);82        continue;83      }84      const edition = trimEdition(ed.json as J);85      let work: EditionPayload['work'] = null;86      const authorKeys: string[] = [];87      if (edition.workKey) {88        const w = await this.getJson(ctx, `${edition.workKey}.json`);89        if (w.success && w.json && typeof w.json === 'object') {90          const wj = w.json as J;91          work = { key: edition.workKey, title: wj.title ? String(wj.title) : null, firstPublishDate: wj.first_publish_date ? String(wj.first_publish_date) : null, subjects: strList(wj.subjects).slice(0, 12) };92          for (const a of Array.isArray(wj.authors) ? wj.authors : []) if (a?.author?.key) authorKeys.push(String(a.author.key));93        }94      }95      for (const a of Array.isArray((ed.json as J).authors) ? (ed.json as J).authors : []) if (a?.key && !authorKeys.includes(String(a.key))) authorKeys.push(String(a.key));96      const authors: string[] = [];97      for (const key of authorKeys.slice(0, 2)) {98        const a = await this.getJson(ctx, `${key}.json`);99        const name = (a.json as J | null)?.name;100        if (a.success && name) authors.push(String(name));101      }102      const payload: EditionPayload = { kind: 'edition', isbn: seed.isbn, note: seed.note ?? null, edition, work, authors };103      count++;104      yield { url, externalId: `isbn:${seed.isbn}`, kind: 'catalog_item', engine: ed.engine, httpStatus: ed.httpStatus, payload, fetchedAt: ed.fetchedAt };105      await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });106    }107  }108109  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {110    const p = EditionPayloadSchema.parse(raw.payload);111    const e = p.edition;112    if (!e.title) return [];113    const author = p.authors[0] ?? null;114    const year = e.publishDate ? extractYear(e.publishDate) : null;115    const isbn = e.isbn13[0] ?? e.isbn10[0] ?? p.isbn;116    const olid = e.key.replace('/books/', '');117    const variant = e.editionName ?? (p.note && /first/i.test(p.note) ? 'First edition' : null);118    const attributes = AssetAttributesSchema.parse({119      categorySlug: 'books',120      brand: author,121      set: e.publishers[0] ?? null,122      name: e.subtitle ? `${e.title}: ${e.subtitle}` : e.title,123      year,124      edition: variant,125      variant,126      country: e.publishPlaces[0] ?? null,127      language: 'English',128      identifiers: { isbn, openlibrary_id: olid, ...(e.workKey ? { openlibrary_work: e.workKey.replace('/works/', '') } : {}) },129      metadata: { authors: p.authors, pages: e.pages, physical_format: e.physicalFormat, first_publish_date: p.work?.firstPublishDate ?? null, subjects: p.work?.subjects ?? [], isbn_10: e.isbn10, isbn_13: e.isbn13, seed_note: p.note },130    });131    return [132      NormalizedCatalogItemSchema.parse({133        kind: 'catalog_item',134        connectorId: this.meta.id,135        sourceId: this.meta.sourceId,136        sourceUrl: `${BASE}${e.key}`,137        externalId: olid,138        rawTitle: author ? `${author} — ${e.title}${year ? ` (${year})` : ''}` : e.title,139        imageUrls: e.covers.slice(0, 1).map((c) => `https://covers.openlibrary.org/b/id/${c}-L.jpg`),140        attributes,141        observedAt: raw.fetchedAt,142        confidence: 0.9,143        parserVersion: PARSER_VERSION,144        releaseDate: year ? new Date(Date.UTC(year, 0, 1)) : null,145      }),146    ];147  }148}149150export default function createConnector(meta: ConnectorMeta) {151  return new OpenLibraryConnector(meta);152}153