Repository Public
1import { createHash } from "node:crypto";2import type { SearchRequest, SearchType, TimeRange } from "./types.ts";3import { HttpError } from "../util.ts";45const TYPES: SearchType[] = ["web", "images", "videos"];6const RANGES: TimeRange[] = ["day", "week", "month", "year"];7const TRACKING = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref_src|_hsenc|_hsmi|yclid)$/i;89/** ISO 3166-1 alpha-2 → the lowercase country names Tavily's `country` parameter expects. */10export const COUNTRY_NAMES: Record<string, string> = {11 ar: "argentina", au: "australia", at: "austria", be: "belgium", br: "brazil", ca: "canada", cl: "chile", cn: "china",12 co: "colombia", cz: "czech republic", dk: "denmark", eg: "egypt", fi: "finland", fr: "france", de: "germany", gr: "greece",13 hk: "hong kong", hu: "hungary", in: "india", id: "indonesia", ie: "ireland", il: "israel", it: "italy", jp: "japan",14 my: "malaysia", mx: "mexico", ma: "morocco", nl: "netherlands", nz: "new zealand", ng: "nigeria", no: "norway", pk: "pakistan",15 pe: "peru", ph: "philippines", pl: "poland", pt: "portugal", ro: "romania", sa: "saudi arabia", sg: "singapore",16 za: "south africa", kr: "south korea", es: "spain", se: "sweden", ch: "switzerland", tw: "taiwan", th: "thailand",17 tr: "turkey", ua: "ukraine", ae: "united arab emirates", gb: "united kingdom", us: "united states", vn: "vietnam",18};1920export function normalizeQuery(raw: unknown): string {21 if (typeof raw !== "string") throw new HttpError(400, "invalid_query", "query must be a string");22 // Control characters out, whitespace collapsed, NFC so equivalent spellings share a cache entry.23 const text = raw.normalize("NFC").replace(/[\u0000-\u001f\u007f]/g, " ").replace(/\s+/g, " ").trim();24 if (!text) throw new HttpError(400, "empty_query", "query is empty");25 if (text.length > 400) throw new HttpError(400, "query_too_long", "query is longer than 400 characters");26 return text;27}2829export function parseSearchRequest(body: any): SearchRequest {30 if (!body || typeof body !== "object") throw new HttpError(400, "invalid_body", "JSON body expected");31 const type = (body.type ?? "web") as SearchType;32 if (!TYPES.includes(type)) throw new HttpError(400, "invalid_type", "type must be web, images or videos");33 const page = body.page === undefined ? 1 : Number(body.page);34 if (!Number.isInteger(page) || page < 1 || page > 10) throw new HttpError(400, "invalid_page", "page must be an integer between 1 and 10");35 let language: string | undefined;36 if (body.language !== undefined && body.language !== null && body.language !== "") {37 if (typeof body.language !== "string" || !/^[a-z]{2}(-[a-z]{2})?$/i.test(body.language)) throw new HttpError(400, "invalid_language", "language must be an ISO 639-1 code");38 language = body.language.toLowerCase();39 }40 let region: string | undefined;41 if (body.region !== undefined && body.region !== null && body.region !== "") {42 if (typeof body.region !== "string" || !/^[a-z]{2}$/i.test(body.region)) throw new HttpError(400, "invalid_region", "region must be an ISO 3166-1 alpha-2 code");43 region = body.region.toLowerCase();44 }45 let timeRange: TimeRange | undefined;46 if (body.timeRange) {47 if (!RANGES.includes(body.timeRange)) throw new HttpError(400, "invalid_time_range", "timeRange must be day, week, month or year");48 timeRange = body.timeRange;49 }50 let source: SearchRequest["source"];51 if (body.source) {52 if (!["youtube", "vimeo", "dailymotion"].includes(body.source)) throw new HttpError(400, "invalid_source", "unknown video source");53 source = body.source;54 }55 return {56 query: normalizeQuery(body.query), type, page, language, region, timeRange, source,57 safeSearch: body.safeSearch !== false,58 summary: body.summary === true,59 };60}6162/** Canonical form used to deduplicate results: no fragment, no tracking parameters, lowercase host, no trailing slash. */63export function canonicalURL(raw: string): string | null {64 let url: URL;65 try { url = new URL(raw); } catch { return null; }66 if (url.protocol !== "http:" && url.protocol !== "https:") return null;67 url.hash = "";68 url.hostname = url.hostname.toLowerCase();69 for (const key of [...url.searchParams.keys()]) if (TRACKING.test(key)) url.searchParams.delete(key);70 let text = url.toString();71 if (url.pathname !== "/" && text.endsWith("/")) text = text.slice(0, -1);72 return text;73}7475export function dedupeKey(raw: string): string {76 const canonical = canonicalURL(raw) ?? raw;77 return canonical.replace(/^https?:\/\/(www\.)?/, "").replace(/\/$/, "");78}7980export function domainOf(raw: string): string {81 try { return new URL(raw).hostname.toLowerCase().replace(/^www\./, ""); } catch { return ""; }82}8384export function resultId(value: string): string {85 return createHash("sha256").update(value).digest("base64url").slice(0, 16);86}8788/** Cache identity of a request batch. The query only ever lives in memory, hashed here. */89export function cacheKey(request: SearchRequest, batch: number): string {90 return createHash("sha256").update(JSON.stringify([91 request.query.toLowerCase(), request.type, request.language ?? "", request.region ?? "", request.timeRange ?? "",92 request.safeSearch, request.type === "web" ? request.summary : false, request.source ?? "", batch,93 ])).digest("hex");94}95