Repository Public
1// Image search behind one interface with two authorised sources:2// - Tavily (`include_images`): pictures found on the pages matching the query, attributed to those pages.3// - Wikimedia Commons (public MediaWiki API): freely licensed media, with real pixel dimensions, author and licence.4// Nothing is scraped from another search engine. A Commons failure never fails the search; a Tavily failure does.5import type { Config } from "../config.ts";6import { fetchJSON } from "../util.ts";7import { dedupeKey, domainOf, resultId } from "./normalize.ts";8import { tavilyImageBatch } from "./tavily.ts";9import type { Batch, ImageResult, SearchRequest } from "./types.ts";1011const COMMONS_PAGE = 30;1213function plain(html: unknown, max: number): string | null {14 if (typeof html !== "string") return null;15 const text = html.replace(/<[^>]*>/g, " ").replace(/&/g, "&").replace(/"/g, '"').replace(/�?39;/g, "'").replace(/</g, "<").replace(/>/g, ">").replace(/ /g, " ").replace(/\s+/g, " ").trim();16 return text ? text.slice(0, max) : null;17}1819export async function commonsImages(request: SearchRequest, batchIndex: number): Promise<ImageResult[]> {20 const base = process.env.COMMONS_API ?? "https://commons.wikimedia.org/w/api.php";21 const params = new URLSearchParams({22 action: "query", format: "json", generator: "search", gsrnamespace: "6", gsrlimit: String(COMMONS_PAGE), gsroffset: String(batchIndex * COMMONS_PAGE),23 gsrsearch: `${request.query} filetype:bitmap`, prop: "imageinfo", iiprop: "url|size|mime|extmetadata", iiurlwidth: "640",24 iiextmetadatafilter: "ImageDescription|Artist|LicenseShortName",25 });26 const { status, body } = await fetchJSON(`${base}?${params}`, { timeoutMs: 6000, headers: { "user-agent": "SPBrowserSearch/1.0 (https://browser.spboucher.ai)" } });27 if (status !== 200 || !body?.query?.pages) return [];28 const pages = (Object.values(body.query.pages) as any[]).sort((a, b) => (a.index ?? 0) - (b.index ?? 0));29 const out: ImageResult[] = [];30 for (const page of pages) {31 const info = page?.imageinfo?.[0];32 if (!info || typeof info.url !== "string" || !/^image\/(jpeg|png|webp|gif)$/.test(info.mime ?? "")) continue;33 if ((info.width ?? 0) < 200 || (info.height ?? 0) < 200) continue;34 const meta = info.extmetadata ?? {};35 const title = String(page.title ?? "").replace(/^File:/, "").replace(/\.[a-z0-9]+$/i, "").replace(/_/g, " ");36 out.push({37 id: resultId(dedupeKey(info.url)), imageURL: info.url, imageDomain: domainOf(info.url),38 description: plain(meta.ImageDescription?.value, 300) ?? title,39 sourceURL: typeof info.descriptionurl === "string" ? info.descriptionurl : null, sourceTitle: title, sourceDomain: "commons.wikimedia.org",40 thumbnailURL: typeof info.thumburl === "string" ? info.thumburl.split("?")[0] : null,41 width: info.width ?? null, height: info.height ?? null,42 author: plain(meta.Artist?.value, 80), license: plain(meta.LicenseShortName?.value, 40), provider: "wikimedia-commons",43 });44 }45 return out;46}4748export async function imageBatch(config: Config, request: SearchRequest, excludeDomains: string[], batchIndex: number): Promise<Batch<ImageResult>> {49 const [web, commons] = await Promise.allSettled([tavilyImageBatch(config, request, excludeDomains), commonsImages(request, batchIndex)]);50 if (web.status === "rejected") throw web.reason;51 const batch = web.value;52 const extra = commons.status === "fulfilled" ? commons.value : [];53 const seen = new Set(batch.items.map((i) => i.id));54 // Web pictures lead (they match the query in context); Commons media is woven in after the first rows.55 const merged: ImageResult[] = [];56 let e = 0;57 batch.items.forEach((item, i) => {58 merged.push(item);59 if (i >= 5 && i % 3 === 2 && e < extra.length) { const x = extra[e++]; if (!seen.has(x.id)) { seen.add(x.id); merged.push(x); } }60 });61 for (; e < extra.length; e++) if (!seen.has(extra[e].id)) { seen.add(extra[e].id); merged.push(extra[e]); }62 return {63 ...batch, items: merged,64 provider: extra.length ? "tavily+wikimedia-commons" : "tavily",65 notices: commons.status === "rejected" ? [...batch.notices, "wikimedia-commons-unavailable"] : batch.notices,66 exhausted: batch.exhausted && extra.length < COMMONS_PAGE,67 };68}69