SPB Git

Repository Public

images.ts 4.2 KB 69 lines typescript

1// Image search behind one interface with two authorised sources:2//  - Tavily (`include_images`): pictures found on the pages matching the query, attributed to those pages.3//  - Wikimedia Commons (public MediaWiki API): freely licensed media, with real pixel dimensions, author and licence.4// Nothing is scraped from another search engine. A Commons failure never fails the search; a Tavily failure does.5import type { Config } from "../config.ts";6import { fetchJSON } from "../util.ts";7import { dedupeKey, domainOf, resultId } from "./normalize.ts";8import { tavilyImageBatch } from "./tavily.ts";9import type { Batch, ImageResult, SearchRequest } from "./types.ts";1011const COMMONS_PAGE = 30;1213function plain(html: unknown, max: number): string | null {14  if (typeof html !== "string") return null;15  const text = html.replace(/<[^>]*>/g, " ").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&#0?39;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();16  return text ? text.slice(0, max) : null;17}1819export async function commonsImages(request: SearchRequest, batchIndex: number): Promise<ImageResult[]> {20  const base = process.env.COMMONS_API ?? "https://commons.wikimedia.org/w/api.php";21  const params = new URLSearchParams({22    action: "query", format: "json", generator: "search", gsrnamespace: "6", gsrlimit: String(COMMONS_PAGE), gsroffset: String(batchIndex * COMMONS_PAGE),23    gsrsearch: `${request.query} filetype:bitmap`, prop: "imageinfo", iiprop: "url|size|mime|extmetadata", iiurlwidth: "640",24    iiextmetadatafilter: "ImageDescription|Artist|LicenseShortName",25  });26  const { status, body } = await fetchJSON(`${base}?${params}`, { timeoutMs: 6000, headers: { "user-agent": "SPBrowserSearch/1.0 (https://browser.spboucher.ai)" } });27  if (status !== 200 || !body?.query?.pages) return [];28  const pages = (Object.values(body.query.pages) as any[]).sort((a, b) => (a.index ?? 0) - (b.index ?? 0));29  const out: ImageResult[] = [];30  for (const page of pages) {31    const info = page?.imageinfo?.[0];32    if (!info || typeof info.url !== "string" || !/^image\/(jpeg|png|webp|gif)$/.test(info.mime ?? "")) continue;33    if ((info.width ?? 0) < 200 || (info.height ?? 0) < 200) continue;34    const meta = info.extmetadata ?? {};35    const title = String(page.title ?? "").replace(/^File:/, "").replace(/\.[a-z0-9]+$/i, "").replace(/_/g, " ");36    out.push({37      id: resultId(dedupeKey(info.url)), imageURL: info.url, imageDomain: domainOf(info.url),38      description: plain(meta.ImageDescription?.value, 300) ?? title,39      sourceURL: typeof info.descriptionurl === "string" ? info.descriptionurl : null, sourceTitle: title, sourceDomain: "commons.wikimedia.org",40      thumbnailURL: typeof info.thumburl === "string" ? info.thumburl.split("?")[0] : null,41      width: info.width ?? null, height: info.height ?? null,42      author: plain(meta.Artist?.value, 80), license: plain(meta.LicenseShortName?.value, 40), provider: "wikimedia-commons",43    });44  }45  return out;46}4748export async function imageBatch(config: Config, request: SearchRequest, excludeDomains: string[], batchIndex: number): Promise<Batch<ImageResult>> {49  const [web, commons] = await Promise.allSettled([tavilyImageBatch(config, request, excludeDomains), commonsImages(request, batchIndex)]);50  if (web.status === "rejected") throw web.reason;51  const batch = web.value;52  const extra = commons.status === "fulfilled" ? commons.value : [];53  const seen = new Set(batch.items.map((i) => i.id));54  // Web pictures lead (they match the query in context); Commons media is woven in after the first rows.55  const merged: ImageResult[] = [];56  let e = 0;57  batch.items.forEach((item, i) => {58    merged.push(item);59    if (i >= 5 && i % 3 === 2 && e < extra.length) { const x = extra[e++]; if (!seen.has(x.id)) { seen.add(x.id); merged.push(x); } }60  });61  for (; e < extra.length; e++) if (!seen.has(extra[e].id)) { seen.add(extra[e].id); merged.push(extra[e]); }62  return {63    ...batch, items: merged,64    provider: extra.length ? "tavily+wikimedia-commons" : "tavily",65    notices: commons.status === "rejected" ? [...batch.notices, "wikimedia-commons-unavailable"] : batch.notices,66    exhausted: batch.exhausted && extra.length < COMMONS_PAGE,67  };68}69
Files main
Menu

Theme