spb/rareindex

1import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors';2import type { ExtractionResult } from '@rareindex/shared';3import { stripHtml } from '../_g8-auctions-eu-apac-lib/index.js';4import { SaleResultsConnector, pick, type HouseConfig, type ParsedLot, type ParsedSalePage, type SaleRef } from '../_g8-auctions-eu-apac-lib/sale-results.js';56/**7 * Woolley & Wallis (Salisbury, UK) — Umbraco site with a public lot-search JSON API. robots.txt DISALLOWS /umbraco/8 * (where the past-sales JSON lives) so we do not touch it: past sales are enumerated from the public /sitemap-xml9 * (/departments/<dept>/<sale-code>/ pages), the sale page gives the title + date and its first lot page10 * (/view-lot/1/) exposes `sale-id="<id>"`; lots then come from GET /api/lots/search?saleId=<id>&pageIndex=N&pageSize=10011 * (HammerPrice, estimates, LotNumber, Title, FullDescription, image, ViewUrl) — explicitly HAMMER prices, GBP.12 */13const BASE = 'https://www.woolleyandwallis.co.uk';14const PAGE_SIZE = 100;1516type ApiLot = { Id?: number; LotNumber?: string; Title?: string; FullDescription?: string; HammerPrice?: number | null; HammerPriceFormatted?: string; LowerEstimate?: number | null; UpperEstimate?: number | null; MainImageUrl?: string | null; ViewUrl?: string | null; SaleNodeId?: number; ConditionReport?: string | null; Categories?: string[]; DroitDeSuite?: boolean; NoBuyersPremium?: boolean; AlternateBuyersPremium?: boolean; LastModified?: string | null };1718/** sitemap → sale pages under /departments/<dept>/<code>/ (excluding department roots and view-lot pages). */19export function parseSitemapSales(xml: string): SaleRef[] {20  const out: SaleRef[] = [];21  const seen = new Set<string>();22  for (const m of xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)) {23    const url = m[1]!.replace(/&amp;/g, '&');24    const p = url.match(/^https:\/\/www\.woolleyandwallis\.co\.uk\/departments\/([a-z0-9-]+)\/([a-z0-9-]+)\/?$/i);25    if (!p || seen.has(p[2]!)) continue;26    seen.add(p[2]!);27    out.push({ id: p[2]!, title: p[2]!.toUpperCase(), url: url.endsWith('/') ? url : `${url}/`, date: null, location: 'Salisbury, United Kingdom', extra: { department: p[1]!.replace(/-/g, ' ') } });28  }29  return out;30}3132/** Sale page: <title> "Sale title | 20th August 2026 | Woolley and Wallis" or "20th August 2026. Starts at 10:00am". */33export function parseSalePageHeader(htmlText: string): { title: string | null; date: string | null } {34  const t = pick(htmlText, /<title>([\s\S]*?)<\/title>/);35  const h1 = pick(htmlText, /<h1[^>]*>([\s\S]*?)<\/h1>/);36  const dm = htmlText.match(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Z][a-z]{2,8})\s+(\d{4})/);37  let date: string | null = null;38  if (dm) {39    const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];40    const mo = months.indexOf(dm[2]!.slice(0, 3).toLowerCase());41    if (mo >= 0) date = new Date(Date.UTC(Number(dm[3]), mo, Number(dm[1]))).toISOString();42  }43  const title = h1 ?? t?.split('|')[0]?.trim() ?? null;44  return { title, date };45}4647export function parseSaleId(htmlText: string): string | null {48  return htmlText.match(/sale-id="(\d+)"/)?.[1] ?? htmlText.match(/saleId["']?\s*[:=]\s*["']?(\d+)/)?.[1] ?? null;49}5051export function parseLotsJson(json: unknown, sale: SaleRef, pageIndex: number): ParsedSalePage | null {52  const j = json as { TotalRecords?: number; TotalPages?: number; Results?: ApiLot[] } | null;53  if (!j || !Array.isArray(j.Results)) return null;54  const lots: ParsedLot[] = [];55  for (const l of j.Results) {56    const lotNo = String(l.LotNumber ?? '').trim();57    const title = stripHtml(l.Title ?? '', 300);58    if (!lotNo || !title) continue;59    const price = typeof l.HammerPrice === 'number' && l.HammerPrice > 0 ? l.HammerPrice : null;60    lots.push({61      lotNo,62      title,63      subtitle: null,64      description: stripHtml(l.FullDescription ?? null, 500),65      url: l.ViewUrl ? `${BASE}${l.ViewUrl}` : `${sale.url}view-lot/${lotNo}/`,66      image: l.MainImageUrl ?? null,67      price,68      currency: 'GBP',69      premiumIncluded: false,70      estimateLow: typeof l.LowerEstimate === 'number' && l.LowerEstimate > 0 ? l.LowerEstimate : null,71      estimateHigh: typeof l.UpperEstimate === 'number' && l.UpperEstimate > 0 ? l.UpperEstimate : null,72      date: null,73      sold: price !== null,74      extra: { ww_lot_id: l.Id ?? null, categories: l.Categories ?? null, condition_report: stripHtml(l.ConditionReport ?? null, 300), droit_de_suite: l.DroitDeSuite ?? null, no_buyers_premium: l.NoBuyersPremium ?? null, hammer_formatted: l.HammerPriceFormatted ?? null },75    });76  }77  const totalPages = typeof j.TotalPages === 'number' ? j.TotalPages : null;78  return { lots, hasMore: totalPages !== null ? pageIndex + 1 < totalPages : lots.length >= PAGE_SIZE, totalLots: typeof j.TotalRecords === 'number' ? j.TotalRecords : null };79}8081export class WoolleyWallisConnector extends SaleResultsConnector {82  readonly version = '1.0.0';83  readonly house: HouseConfig = { houseName: 'Woolley & Wallis', defaultCurrency: 'GBP', location: 'Salisbury, United Kingdom', idKey: 'woolley_wallis_lot', premiumIncluded: false, fallbackSlug: 'antiques', responseType: 'json', minIntervalMs: 2500, maxPagesPerSale: 20 };84  protected override minIntervalMs = 2500;8586  async listSales(ctx: CrawlContext): Promise<SaleRef[]> {87    const url = String(this.meta.config.sitemapUrl ?? `${BASE}/sitemap-xml`);88    await this.throttle(url);89    const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });90    if (!res.success || !res.html) {91      ctx.anomaly('past_list_failed', `${url}: ${res.error ?? res.httpStatus}`);92      return [];93    }94    // sitemap order = site order (newest sales last within a department) → reverse so recent sales come first95    return parseSitemapSales(res.html).reverse();96  }9798  salePageUrl(sale: SaleRef, page: number): string {99    return `${BASE}/api/lots/search?pageIndex=${page - 1}&filter=&saleId=${String(sale.extra.sale_id ?? '')}&pageSize=${PAGE_SIZE}`;100  }101102  parseSalePage(res: ExtractionResult, sale: SaleRef, page: number): ParsedSalePage | null {103    return parseLotsJson(res.json, sale, page - 1);104  }105106  /** Page 1 first resolves the numeric sale id (sale page → view-lot/1/), then calls the lot-search API. */107  protected override async fetchSalePage(ctx: CrawlContext, sale: SaleRef, page: number): Promise<{ url: string; res: ExtractionResult; parsed: ParsedSalePage | null }> {108    if (!sale.extra.sale_id) {109      await this.throttle(sale.url);110      const sp = await ctx.fetch(sale.url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });111      if (!sp.success || !sp.html) return { url: sale.url, res: sp, parsed: null };112      const header = parseSalePageHeader(sp.html);113      let saleId = parseSaleId(sp.html);114      if (!saleId) {115        const lotUrl = `${sale.url}view-lot/1/`;116        await this.throttle(lotUrl);117        const lp = await ctx.fetch(lotUrl, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000, failOnHttpError: false });118        saleId = lp.html ? parseSaleId(lp.html) : null;119      }120      if (!saleId) {121        ctx.anomaly('selector_missing', `${sale.id}: sale-id not found`);122        return { url: sale.url, res: sp, parsed: null };123      }124      Object.assign(sale, { title: header.title ?? sale.title, date: header.date ?? sale.date, extra: { ...sale.extra, sale_id: saleId } });125    }126    return super.fetchSalePage(ctx, sale, page);127  }128}129130export default function createConnector(meta: ConnectorMeta) {131  return new WoolleyWallisConnector(meta);132}133