TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedAuctionLotSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { normalizeCondition } from '@rareindex/taxonomy';5import { attrs } from '../../api/_lib/shared.js';6import { monthIndex, parseComicGrade, parseComicTitle, parseLabelNumber, publisherCategory, usd } from '../../api/_g6-comics-toys-games-lib/comics.js';78/**9 * MyComicShop (Lone Star Comics) — public title pages /search?TID=<id>&mingr=0 list every issue of a10 * series with its in-stock copies as schema.org/Product microdata (name incl. grade, sku = ItemID, price,11 * availability) plus CGC/CBCS label numbers, paper quality, consignment/premium notes. Auction copies12 * appear on the same page ("Auction Item: CGC 8.5", current bid, time left) → auction_lot records.13 * The site is fronted by Imperva Incapsula → Scrapfly (no JS rendering, ~1 credit per title page).14 */15const SITE = 'https://www.mycomicshop.com';16const PARSER_VERSION = '1.0.0';1718export const StockItemSchema = z.object({19 itemId: z.string(),20 name: z.string(),21 url: z.string(),22 /** microdata price (for consignments this already includes the 3% buyer's premium) */23 priceMicro: z.number().nullable(),24 /** price as displayed to shoppers (seller's asking price) */25 priceShown: z.number().nullable(),26 availability: z.enum(['available', 'sold', 'ended', 'unknown']),27 gradeText: z.string().nullable(),28 label: z.string().nullable(),29 notes: z.array(z.string()),30 bestOffer: z.boolean(),31 consignment: z.boolean(),32 consignor: z.string().nullable(),33 auction: z.object({ currentBid: z.number().nullable(), bids: z.number().nullable(), timeLeft: z.string().nullable(), opens: z.string().nullable() }).nullable(),34});35export type StockItem = z.infer<typeof StockItemSchema>;36export const IssueBlockSchema = z.object({37 ivid: z.string(),38 seriesTitle: z.string(),39 tid: z.string().nullable(),40 issue: z.string().nullable(),41 published: z.string().nullable(),42 publisher: z.string().nullable(),43 image: z.string().nullable(),44 tags: z.array(z.string()),45 items: z.array(StockItemSchema),46});47export type IssueBlock = z.infer<typeof IssueBlockSchema>;48export const PagePayloadSchema = z.object({ kind: z.literal('title_page'), url: z.string(), tid: z.string(), title: z.string().nullable(), issues: z.array(IssueBlockSchema), snapshot: z.string().optional() });49export type PagePayload = z.infer<typeof PagePayloadSchema>;5051function availabilityOf(v: string | undefined): StockItem['availability'] {52 const s = (v ?? '').toLowerCase();53 if (/instock|preorder|limited/.test(s)) return 'available';54 if (/soldout/.test(s)) return 'sold';55 if (/outofstock|discontinued/.test(s)) return 'ended';56 return 'unknown';57}5859/** Parse a title page into issue blocks with their stock rows. Exported for tests. */60export function parseTitlePage(htmlText: string, url: string, tid: string): PagePayload {61 const $ = H.load(htmlText);62 const title = H.text($('title')) ?? null;63 const issues: IssueBlock[] = [];64 $('li.issue').each((_, li) => {65 const $li = $(li);66 const ivid = $li.find('a[name]').first().attr('name') ?? $li.find('.fancyboxthis').first().attr('id') ?? '';67 if (!ivid) return;68 const titleA = $li.find('.othercolleft .title a').first();69 const seriesTitle = H.text(titleA) ?? '';70 const tidM = (titleA.attr('href') ?? '').match(/TID=(\d+)/);71 const issue = (H.text($li.find('.othercolleft .title .issuenum')) ?? '').replace(/^#/, '') || null;72 const right = H.text($li.find('.othercolright')) ?? '';73 const published = right.match(/Published\s+(.+?)(?:\s+by\s|$)/)?.[1]?.trim() ?? null;74 const publisher = H.text($li.find('.othercolright a[href*="pl="]')) ?? null;75 const image = $li.find('.imgcol img').first().attr('src') ?? null;76 const tags = $li.find('.indentrow a').map((__, a) => H.text($(a)) ?? '').get().filter(Boolean);77 const items: StockItem[] = [];78 $li.find('td[itemtype="http://schema.org/Product"]').each((__, td) => {79 const $td = $(td);80 const meta = (prop: string) => $td.find(`meta[itemprop="${prop}"]`).first().attr('content');81 const auctionA = $td.find('a[title="View Auction"]').first();82 const isAuction = auctionA.length > 0;83 const itemId = meta('sku') ?? (auctionA.attr('href') ?? $td.find('a[href*="ItemID="]').first().attr('href') ?? '').match(/ItemID=(\d+)/)?.[1] ?? '';84 if (!itemId) return;85 const notes = $td.find('ul li').map((___, l) => (H.text($(l)) ?? '').replace(/\s+/g, ' ').trim()).get().filter(Boolean);86 const cartText = H.text($td.find('.addcart a')) ?? '';87 const gradeText = isAuction ? (H.text(auctionA) ?? '').replace(/^Auction Item:\s*/i, '') || null : cartText.replace(/^Add to cart\s*/i, '').trim() || null;88 const shownM = ($td.find('.hasscan').first().text() ?? '').match(/\$\s?[\d,]+(?:\.\d{2})?/);89 const groupText = H.text($td) ?? '';90 const currentBid = usd(groupText.match(/Current bid:\s*(\$[\d,.]+)/i)?.[1]);91 const bids = groupText.match(/(\d+)\s+bids?\b/i)?.[1];92 const timeLeft = groupText.match(/Time left:\s*([^\n<]+?)(?:\s{2,}|$)/i)?.[1]?.trim() ?? null;93 const opens = groupText.match(/Auction opens\s+([A-Za-z]+\s+\d{1,2})/i)?.[1] ?? null;94 const consignmentNote = notes.find((n) => /consignment/i.test(n)) ?? null;95 const consignor = consignmentNote?.match(/consigned by\s+(.+)$/i)?.[1]?.trim() ?? null;96 items.push({97 itemId,98 name: meta('name') ?? `${seriesTitle} ${issue ?? ''} ${gradeText ?? ''}`.trim(),99 url: meta('url') ?? `${SITE}/search?ItemID=${itemId}`,100 priceMicro: usd(meta('price')),101 priceShown: shownM ? usd(shownM[0]) : null,102 availability: isAuction ? 'unknown' : availabilityOf(meta('availability')),103 gradeText,104 label: parseLabelNumber(notes.join(' | ')),105 notes: notes.filter((n) => !/^Label\s*#/i.test(n)),106 bestOffer: /Best Offer/i.test(groupText),107 consignment: Boolean(consignmentNote),108 consignor,109 auction: isAuction ? { currentBid, bids: bids ? Number(bids) : null, timeLeft, opens } : null,110 });111 });112 issues.push({ ivid, seriesTitle, tid: tidM?.[1] ?? null, issue, published, publisher, image, tags, items });113 });114 return { kind: 'title_page', url, tid, title, issues };115}116117/** "May 1988" → 1988; "Aug 2026" → 2026. */118export function yearOfPublished(s: string | null | undefined): number | null {119 const m = s?.match(/\b(1[89]\d{2}|20\d{2})\b/);120 return m ? Number(m[1]) : null;121}122123/** "Auction opens September 14" (no year on the page) → null (never invent a year). */124export function auctionStatus(a: NonNullable<StockItem['auction']>): 'upcoming' | 'live' | 'unknown' {125 if (a.opens) return 'upcoming';126 if (a.timeLeft || a.currentBid !== null) return 'live';127 return 'unknown';128}129130export class MyComicShopConnector extends BaseConnector {131 readonly version = '1.0.0';132 readonly parserVersion = PARSER_VERSION;133 protected override minIntervalMs = 3000;134 override readonly urlPatterns = [/^https?:\/\/(?:www\.)?mycomicshop\.com\/search\?(?:.*&)?TID=(\d+)/i];135136 private async fetchTitle(ctx: CrawlContext, tid: string): Promise<RawRecordInput | null> {137 const url = `${SITE}/search?TID=${tid}&mingr=0`;138 await this.throttle(url);139 const res = await ctx.fetch(url, {140 engines: ['scrapfly'],141 renderJs: false,142 country: 'us',143 timeoutMs: 120_000,144 expect: ['title', 'price', 'identifiers'],145 parse: (r) => {146 if (!r.html) return null;147 const p = parseTitlePage(r.html, url, tid);148 const first = p.issues.flatMap((i) => i.items)[0];149 return { title: p.issues[0]?.seriesTitle ?? null, price: first?.priceMicro ?? first?.priceShown ?? null, identifiers: first?.itemId ?? null };150 },151 });152 if (!res.success || !res.html) {153 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);154 return null;155 }156 const payload = parseTitlePage(res.html, url, tid);157 if (!payload.issues.length) {158 ctx.anomaly('parse_failure_page', url);159 return null;160 }161 return { url, externalId: `tid:${tid}`, kind: 'listing', engine: 'scrapfly', httpStatus: res.httpStatus, payload, snapshot: res.html, fetchedAt: res.fetchedAt };162 }163164 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {165 const titles = ((this.meta.config.titles as Array<{ tid: string; name: string }> | undefined) ?? []).map((t) => String(t.tid));166 const seeds = ctx.options.seeds?.length ? ctx.options.seeds.map((s) => s.match(/TID=(\d+)/i)?.[1] ?? s.replace(/\D/g, '')).filter(Boolean) : titles;167 if (!seeds.length) {168 ctx.anomaly('config_missing', 'no titles configured');169 return;170 }171 const backfill = ctx.options.mode === 'backfill';172 const perRun = ctx.options.mode === 'probe' ? Math.max(1, ctx.options.limit ?? 1) : backfill ? seeds.length : Number(this.meta.config.titlesPerRun ?? 6);173 const cursor = (ctx.options.cursor ?? {}) as { index?: number };174 let index = ctx.options.seeds?.length ? 0 : Math.max(0, Number(cursor.index ?? 0)) % seeds.length;175 let count = 0;176 for (let n = 0; n < perRun && n < seeds.length; n++) {177 if (ctx.signal?.aborted || this.reached(ctx, count)) return;178 const tid = seeds[index]!;179 const rec = await this.fetchTitle(ctx, tid);180 index = (index + 1) % seeds.length;181 if (rec) {182 count++;183 yield rec;184 }185 await ctx.setCursor({ index, updatedAt: new Date().toISOString() });186 if (backfill) await ctx.progress({ page: n + 1, totalPages: seeds.length, itemsProcessed: count });187 }188 if (backfill) await ctx.setCursor({ done: true, index: 0, updatedAt: new Date().toISOString() });189 }190191 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {192 const tid = url.match(this.urlPatterns[0]!)?.[1];193 if (!tid) return [];194 const rec = await this.fetchTitle(ctx, tid);195 return rec ? [rec] : [];196 }197198 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {199 const p = PagePayloadSchema.parse(raw.payload);200 const out: NormalizedRecord[] = [];201 for (const blk of p.issues) {202 // seriesTitle = "Amazing Spider-Man (1963 1st Series)"; the issue number comes from the page, never re-parsed.203 const t = parseComicTitle(blk.seriesTitle);204 const categorySlug = publisherCategory(blk.publisher);205 const year = yearOfPublished(blk.published) ?? t.year;206 const series = blk.seriesTitle;207 for (const it of blk.items) {208 const g = parseComicGrade(it.gradeText ?? it.name);209 const graded = g.grader && g.grader !== 'raw';210 const qualifier = it.notes.some((n) => /Signature Series/i.test(n)) ? 'Signature Series' : g.qualifier;211 const attributes = attrs({212 categorySlug,213 brand: blk.publisher,214 series,215 set: t.series,216 name: `${t.series}${blk.issue ? ` #${blk.issue}` : ''}`,217 number: blk.issue,218 year,219 variant: t.variant,220 language: 'English',221 country: 'US',222 identifiers: { mycomicshop_item_id: it.itemId, mycomicshop_ivid: blk.ivid, ...(blk.tid ? { mycomicshop_tid: blk.tid } : {}) },223 metadata: { published: blk.published, tags: blk.tags.slice(0, 10), notes: it.notes.slice(0, 8), grade_label: g.label, consignment: it.consignment, consignor: it.consignor, price_incl_buyer_premium: it.consignment ? it.priceMicro : null, buyer_premium_pct: it.consignment ? 3 : null, best_offer: it.bestOffer },224 });225 const grade = { grader: g.grader, grade: graded ? g.grade : null, qualifier, certificationNumber: graded ? it.label : null };226 const condition = { condition: !graded ? normalizeCondition(categorySlug, g.label) : null, conditionRaw: !graded && g.label ? `${g.label}${g.grade ? ` ${g.grade}` : ''}` : null, completeness: null };227 const base = {228 connectorId: this.meta.id,229 sourceId: this.meta.sourceId,230 sourceUrl: it.url,231 externalId: it.itemId,232 rawTitle: it.name,233 description: it.notes.length ? it.notes.join('; ') : null,234 imageUrls: blk.image ? [blk.image.replace('/n_iv/120/', '/n_iv/600/')] : [],235 attributes,236 grade,237 condition,238 observedAt: raw.fetchedAt,239 parserVersion: PARSER_VERSION,240 };241 if (it.auction) {242 out.push(243 NormalizedAuctionLotSchema.parse({244 ...base,245 kind: 'auction_lot',246 confidence: 0.8,247 auctionHouse: 'MyComicShop',248 auctionName: null,249 lotNumber: it.itemId,250 currentBid: it.auction.currentBid,251 currency: 'USD',252 status: auctionStatus(it.auction),253 }),254 );255 continue;256 }257 const price = it.priceShown ?? it.priceMicro;258 if (price === null) continue;259 out.push(260 NormalizedListingSchema.parse({261 ...base,262 kind: 'listing',263 confidence: graded && it.label ? 0.9 : 0.82,264 listingType: it.bestOffer ? 'best_offer' : 'fixed_price',265 price,266 currency: 'USD',267 seller: it.consignment ? (it.consignor ? `Consignment (${it.consignor})` : 'Consignment via MyComicShop') : 'MyComicShop',268 location: 'US',269 availability: it.availability,270 }),271 );272 }273 }274 return out;275 }276}277278export default (meta: ConnectorMeta) => new MyComicShopConnector(meta);279