TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedCatalogItemSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../_lib/shared.js';5import { publisherCategory } from '../_g6-comics-toys-games-lib/comics.js';67/**8 * Grand Comics Database (comics.org) — public, unauthenticated JSON REST API9 * (https://www.comics.org/api/ — Django REST framework): /api/series/?page=N (50 per page, 232k series),10 * /api/series/<id>/, /api/issue/<id>/, /api/publisher/<id>/. Data is CC-BY-SA (attribution kept on every record).11 * One raw record per series (with up to `issuesPerSeries` issue details); normalize → one catalog_item per12 * issue keyed by `comics_org_id`.13 */14const API = 'https://www.comics.org/api';15const SITE = 'https://www.comics.org';16const PARSER_VERSION = '1.0.0';17const HEADERS = { accept: 'application/json' };1819export const IssueSchema = z.object({20 id: z.string(),21 number: z.string().nullable().default(null),22 volume: z.string().nullable().default(null),23 variant_name: z.string().nullable().default(null),24 variant_of: z.string().nullable().default(null),25 title: z.string().nullable().default(null),26 publication_date: z.string().nullable().default(null),27 key_date: z.string().nullable().default(null),28 on_sale_date: z.string().nullable().default(null),29 price: z.string().nullable().default(null),30 page_count: z.string().nullable().default(null),31 isbn: z.string().nullable().default(null),32 barcode: z.string().nullable().default(null),33 indicia_publisher: z.string().nullable().default(null),34 brand_emblem: z.string().nullable().default(null),35 cover: z.string().nullable().default(null),36 cover_pencils: z.string().nullable().default(null),37 cover_title: z.string().nullable().default(null),38 story_count: z.number().int().nullable().default(null),39 notes: z.string().nullable().default(null),40});41export type Issue = z.infer<typeof IssueSchema>;42export const SeriesSchema = z.object({43 id: z.string(),44 name: z.string(),45 country: z.string().nullable().default(null),46 language: z.string().nullable().default(null),47 year_began: z.number().int().nullable().default(null),48 year_ended: z.number().int().nullable().default(null),49 binding: z.string().nullable().default(null),50 publishing_format: z.string().nullable().default(null),51 color: z.string().nullable().default(null),52 issue_count: z.number().int().nullable().default(null),53});54export type Series = z.infer<typeof SeriesSchema>;55export const PublisherSchema = z.object({ id: z.string().nullable(), name: z.string().nullable(), country: z.string().nullable().default(null) });56export const PayloadSchema = z.object({ kind: z.literal('series_issues'), series: SeriesSchema, publisher: PublisherSchema, issues: z.array(IssueSchema) });57export type Payload = z.infer<typeof PayloadSchema>;5859type Json = Record<string, any>;6061export function idFromApiUrl(url: string | null | undefined): string | null {62 return url?.match(/\/api\/(?:issue|series|publisher)\/(\d+)\//)?.[1] ?? null;63}6465const str = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null);6667export function trimIssue(j: Json): Issue {68 const stories: Json[] = Array.isArray(j.story_set) ? j.story_set : [];69 const cover = stories.find((s) => s.type === 'cover');70 return IssueSchema.parse({71 id: idFromApiUrl(j.api_url) ?? String(j.id ?? ''),72 number: str(j.number),73 volume: str(j.volume),74 variant_name: str(j.variant_name),75 variant_of: idFromApiUrl(j.variant_of),76 title: str(j.title),77 publication_date: str(j.publication_date),78 key_date: str(j.key_date),79 on_sale_date: str(j.on_sale_date),80 price: str(j.price),81 page_count: str(j.page_count),82 isbn: str(j.isbn),83 barcode: str(j.barcode),84 indicia_publisher: str(j.indicia_publisher),85 brand_emblem: str(j.brand_emblem),86 cover: str(j.cover),87 cover_pencils: str(cover?.pencils),88 cover_title: str(cover?.title),89 story_count: stories.length,90 notes: str(j.notes)?.slice(0, 400) ?? null,91 });92}9394export function trimSeries(j: Json): Series {95 return SeriesSchema.parse({96 id: idFromApiUrl(j.api_url) ?? String(j.id ?? ''),97 name: String(j.name ?? ''),98 country: str(j.country),99 language: str(j.language),100 year_began: typeof j.year_began === 'number' ? j.year_began : null,101 year_ended: typeof j.year_ended === 'number' ? j.year_ended : null,102 binding: str(j.binding),103 publishing_format: str(j.publishing_format),104 color: str(j.color),105 issue_count: Array.isArray(j.active_issues) ? j.active_issues.length : null,106 });107}108109/** "1988-05-00" → { year: 1988, date: null } · "1988-02-23" → full UTC date. GCD uses 00 for unknown month/day. */110export function parseKeyDate(s: string | null | undefined): { year: number | null; date: Date | null } {111 if (!s) return { year: null, date: null };112 const m = s.match(/^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?/);113 if (!m) return { year: null, date: null };114 const year = Number(m[1]);115 if (!year || year < 1500) return { year: null, date: null };116 const mo = m[2] ? Number(m[2]) : 0;117 const d = m[3] ? Number(m[3]) : 0;118 if (mo >= 1 && mo <= 12 && d >= 1 && d <= 31) {119 const date = new Date(Date.UTC(year, mo - 1, d));120 return { year, date: Number.isNaN(date.getTime()) ? null : date };121 }122 return { year, date: null };123}124125/** "0.12 USD" · "$0.12 USD; 0.15 CAD" → first amount with an ISO currency we support. */126export function parseCoverPrice(s: string | null | undefined): { amount: number; currency: 'USD' | 'CAD' | 'GBP' | 'EUR' | 'JPY' } | null {127 if (!s) return null;128 const m = s.match(/(\d+(?:\.\d+)?)\s*(USD|CAD|GBP|EUR|JPY)\b/);129 if (!m) return null;130 const amount = Number(m[1]);131 return Number.isFinite(amount) && amount > 0 ? { amount, currency: m[2] as 'USD' } : null;132}133134export function barcodeIds(barcode: string | null | undefined): Record<string, string> {135 if (!barcode) return {};136 const d = barcode.replace(/[^0-9]/g, '');137 if (d.length === 12) return { upc: d };138 if (d.length === 13) return /^97[89]/.test(d) ? { isbn: d, ean: d } : { ean: d };139 if (d.length >= 17 && d.length <= 18) return { upc: d.slice(0, 12), upc_supplement: d.slice(12) };140 return {};141}142143export class GcdConnector extends BaseConnector {144 readonly version = '1.0.0';145 readonly parserVersion = PARSER_VERSION;146 protected override minIntervalMs = 2500;147 override readonly urlPatterns = [/^https?:\/\/(?:www\.)?comics\.org\/(issue|series)\/(\d+)\/?/i];148 private publishers = new Map<string, z.infer<typeof PublisherSchema>>();149 /** Set when the API answers 429 (GCD throttles at roughly 50 requests per ~30 min); the run then stops at its last checkpoint. */150 private rateLimited = false;151 private requests = 0;152153 private async getJson(ctx: CrawlContext, url: string): Promise<Json | null> {154 if (this.rateLimited) return null;155 await this.throttle(url);156 this.requests++;157 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', headers: HEADERS, minQuality: 0 });158 if (res.httpStatus === 429) {159 this.rateLimited = true;160 ctx.anomaly('rate_limited', `${url}: HTTP 429 after ${this.requests} requests this run — stopping at the last checkpoint`);161 return null;162 }163 if (!res.success || !res.json || typeof res.json !== 'object') {164 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);165 return null;166 }167 return res.json as Json;168 }169170 private budgetLeft(): boolean {171 return !this.rateLimited && this.requests < Number(this.meta.config.maxRequestsPerRun ?? 40);172 }173174 private async publisher(ctx: CrawlContext, apiUrl: string | null): Promise<z.infer<typeof PublisherSchema>> {175 const id = idFromApiUrl(apiUrl);176 if (!id) return { id: null, name: null, country: null };177 const hit = this.publishers.get(id);178 if (hit) return hit;179 const j = await this.getJson(ctx, `${API}/publisher/${id}/?format=json`);180 const p = { id, name: str(j?.name), country: str(j?.country) };181 this.publishers.set(id, p);182 return p;183 }184185 /** Fetch a series JSON (already loaded list entry or by id) plus up to `max` of its issues. */186 private async seriesRecord(ctx: CrawlContext, seriesJson: Json, max: number, onlyIssueIds?: string[]): Promise<RawRecordInput | null> {187 const series = trimSeries(seriesJson);188 if (!series.id) return null;189 const publisher = await this.publisher(ctx, seriesJson.publisher);190 const issueUrls: string[] = Array.isArray(seriesJson.active_issues) ? seriesJson.active_issues : [];191 const wanted = onlyIssueIds ? issueUrls.filter((u) => onlyIssueIds.includes(idFromApiUrl(u) ?? '')) : issueUrls.slice(0, max);192 const issues: Issue[] = [];193 for (const u of wanted) {194 if (ctx.signal?.aborted || !this.budgetLeft()) break;195 const j = await this.getJson(ctx, u.includes('format=') ? u : `${u}${u.includes('?') ? '&' : '?'}format=json`);196 if (!j) continue;197 try {198 issues.push(trimIssue(j));199 } catch (err) {200 ctx.anomaly('schema_drift', `${u}: ${err instanceof Error ? err.message : String(err)}`);201 }202 }203 if (!issues.length) return null;204 const payload: Payload = { kind: 'series_issues', series, publisher, issues };205 return { url: `${SITE}/series/${series.id}/`, externalId: `series:${series.id}`, kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };206 }207208 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {209 const backfill = ctx.options.mode === 'backfill';210 const issuesPerSeries = Number(this.meta.config.issuesPerSeries ?? 12);211 const seriesPerRun = ctx.options.mode === 'probe' ? Math.max(1, ctx.options.limit ?? 3) : Number(backfill ? this.meta.config.backfillSeriesPerRun ?? 200 : this.meta.config.seriesPerRun ?? 25);212 let count = 0;213214 // Manual seeds: "series:<id>", "issue:<id>" or comics.org URLs.215 if (ctx.options.seeds?.length) {216 for (const seed of ctx.options.seeds) {217 if (ctx.signal?.aborted || this.reached(ctx, count)) return;218 const m = seed.match(/(series|issue)[:/](\d+)/i);219 if (!m) continue;220 const recs = await this.lookup(`${SITE}/${m[1]!.toLowerCase()}/${m[2]}/`, ctx);221 for (const r of recs) {222 count++;223 yield r;224 }225 }226 return;227 }228229 const cursor = (ctx.options.cursor ?? {}) as { page?: number; index?: number; done?: boolean };230 let page = Math.max(1, Number(cursor.page ?? 1));231 let index = Math.max(0, Number(cursor.index ?? 0));232 let processed = 0;233 this.rateLimited = false;234 this.requests = 0;235 while (processed < seriesPerRun) {236 if (ctx.signal?.aborted || this.reached(ctx, count) || !this.budgetLeft()) return;237 const listUrl = `${API}/series/?format=json&page=${page}`;238 const list = await this.getJson(ctx, listUrl);239 const results: Json[] = Array.isArray(list?.results) ? list!.results : [];240 if (!list || !results.length) {241 if (list && list.next === null) {242 await ctx.setCursor({ done: true, page: 1, index: 0, updatedAt: new Date().toISOString() });243 return;244 }245 ctx.anomaly('pagination_failure', listUrl);246 return;247 }248 const totalPages = typeof list.count === 'number' ? Math.ceil(list.count / 50) : null;249 for (; index < results.length && processed < seriesPerRun; index++) {250 if (ctx.signal?.aborted || this.reached(ctx, count) || !this.budgetLeft()) break;251 const s = results[index]!;252 if (!Array.isArray(s.active_issues) || !s.active_issues.length) {253 processed++;254 continue;255 }256 const rec = await this.seriesRecord(ctx, s, issuesPerSeries);257 processed++;258 if (rec) {259 count++;260 yield rec;261 }262 await ctx.setCursor({ page, index: index + 1, updatedAt: new Date().toISOString() });263 }264 if (!this.budgetLeft()) return;265 if (index >= results.length) {266 if (list.next === null) {267 await ctx.setCursor({ done: true, page: 1, index: 0, updatedAt: new Date().toISOString() });268 return;269 }270 page++;271 index = 0;272 await ctx.setCursor({ page, index: 0, updatedAt: new Date().toISOString() });273 if (backfill) await ctx.progress({ page, totalPages, itemsProcessed: count });274 }275 }276 }277278 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {279 const m = url.match(this.urlPatterns[0]!);280 if (!m) return [];281 const [, kind, id] = m;282 if (kind!.toLowerCase() === 'issue') {283 const issue = await this.getJson(ctx, `${API}/issue/${id}/?format=json`);284 const seriesUrl = issue?.series;285 const sid = idFromApiUrl(seriesUrl);286 if (!issue || !sid) return [];287 const series = await this.getJson(ctx, `${API}/series/${sid}/?format=json`);288 if (!series) return [];289 const rec = await this.seriesRecord(ctx, series, 1, [id!]);290 return rec ? [rec] : [];291 }292 const series = await this.getJson(ctx, `${API}/series/${id}/?format=json`);293 if (!series) return [];294 const rec = await this.seriesRecord(ctx, series, Number(this.meta.config.issuesPerSeries ?? 12));295 return rec ? [rec] : [];296 }297298 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {299 const p = PayloadSchema.parse(raw.payload);300 const categorySlug = publisherCategory(p.publisher.name, p.series.language);301 const out: NormalizedRecord[] = [];302 for (const it of p.issues) {303 const kd = parseKeyDate(it.key_date);304 const onSale = parseKeyDate(it.on_sale_date);305 const year = kd.year ?? onSale.year ?? (p.series.year_began && p.series.year_began === p.series.year_ended ? p.series.year_began : null);306 const price = parseCoverPrice(it.price);307 const number = it.number && it.number !== '[nn]' ? it.number : null;308 const seriesLabel = p.series.year_began ? `${p.series.name} (${p.series.year_began}${p.series.year_ended && p.series.year_ended !== p.series.year_began ? `-${p.series.year_ended}` : ''} series)` : p.series.name;309 const name = number ? `${p.series.name} #${number}${it.variant_name ? ` (${it.variant_name})` : ''}` : it.title || p.series.name;310 const identifiers: Record<string, string> = { comics_org_id: it.id, gcd_series_id: p.series.id, ...barcodeIds(it.barcode) };311 if (it.isbn) identifiers.isbn = it.isbn.replace(/[^0-9Xx]/g, '');312 if (p.publisher.id) identifiers.gcd_publisher_id = p.publisher.id;313 const attributes = attrs({314 categorySlug,315 brand: p.publisher.name,316 series: seriesLabel,317 set: p.series.name,318 name,319 number,320 year,321 variant: it.variant_name,322 language: p.series.language,323 country: p.series.country ? p.series.country.toUpperCase() : null,324 originalMsrp: price?.amount ?? null,325 originalMsrpCurrency: price?.currency ?? null,326 identifiers,327 metadata: {328 key_date: it.key_date,329 publication_date: it.publication_date,330 on_sale_date: it.on_sale_date,331 cover_price_raw: it.price,332 page_count: it.page_count,333 indicia_publisher: it.indicia_publisher,334 brand_emblem: it.brand_emblem,335 variant_of: it.variant_of,336 cover_pencils: it.cover_pencils,337 story_count: it.story_count,338 binding: p.series.binding,339 publishing_format: p.series.publishing_format,340 series_year_began: p.series.year_began,341 series_year_ended: p.series.year_ended,342 license: 'CC-BY-SA (Grand Comics Database)',343 },344 });345 out.push(346 NormalizedCatalogItemSchema.parse({347 kind: 'catalog_item',348 connectorId: this.meta.id,349 sourceId: this.meta.sourceId,350 sourceUrl: `${SITE}/issue/${it.id}/`,351 externalId: it.id,352 rawTitle: `${p.series.name} #${number ?? it.title ?? ''} (${it.publication_date ?? p.series.year_began ?? '?'})`.trim(),353 description: it.notes,354 imageUrls: it.cover && /^https?:\/\//.test(it.cover) ? [it.cover] : [],355 attributes,356 grade: {},357 condition: {},358 observedAt: raw.fetchedAt,359 confidence: 0.92,360 parserVersion: PARSER_VERSION,361 releaseDate: onSale.date ?? kd.date,362 }),363 );364 }365 return out;366 }367}368369export default (meta: ConnectorMeta) => new GcdConnector(meta);370