SPB Git

Repository Public

normalize.ts 5.2 KB 95 lines typescript

1import { createHash } from "node:crypto";2import type { SearchRequest, SearchType, TimeRange } from "./types.ts";3import { HttpError } from "../util.ts";45const TYPES: SearchType[] = ["web", "images", "videos"];6const RANGES: TimeRange[] = ["day", "week", "month", "year"];7const TRACKING = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref_src|_hsenc|_hsmi|yclid)$/i;89/** ISO 3166-1 alpha-2 → the lowercase country names Tavily's `country` parameter expects. */10export const COUNTRY_NAMES: Record<string, string> = {11  ar: "argentina", au: "australia", at: "austria", be: "belgium", br: "brazil", ca: "canada", cl: "chile", cn: "china",12  co: "colombia", cz: "czech republic", dk: "denmark", eg: "egypt", fi: "finland", fr: "france", de: "germany", gr: "greece",13  hk: "hong kong", hu: "hungary", in: "india", id: "indonesia", ie: "ireland", il: "israel", it: "italy", jp: "japan",14  my: "malaysia", mx: "mexico", ma: "morocco", nl: "netherlands", nz: "new zealand", ng: "nigeria", no: "norway", pk: "pakistan",15  pe: "peru", ph: "philippines", pl: "poland", pt: "portugal", ro: "romania", sa: "saudi arabia", sg: "singapore",16  za: "south africa", kr: "south korea", es: "spain", se: "sweden", ch: "switzerland", tw: "taiwan", th: "thailand",17  tr: "turkey", ua: "ukraine", ae: "united arab emirates", gb: "united kingdom", us: "united states", vn: "vietnam",18};1920export function normalizeQuery(raw: unknown): string {21  if (typeof raw !== "string") throw new HttpError(400, "invalid_query", "query must be a string");22  // Control characters out, whitespace collapsed, NFC so equivalent spellings share a cache entry.23  const text = raw.normalize("NFC").replace(/[\u0000-\u001f\u007f]/g, " ").replace(/\s+/g, " ").trim();24  if (!text) throw new HttpError(400, "empty_query", "query is empty");25  if (text.length > 400) throw new HttpError(400, "query_too_long", "query is longer than 400 characters");26  return text;27}2829export function parseSearchRequest(body: any): SearchRequest {30  if (!body || typeof body !== "object") throw new HttpError(400, "invalid_body", "JSON body expected");31  const type = (body.type ?? "web") as SearchType;32  if (!TYPES.includes(type)) throw new HttpError(400, "invalid_type", "type must be web, images or videos");33  const page = body.page === undefined ? 1 : Number(body.page);34  if (!Number.isInteger(page) || page < 1 || page > 10) throw new HttpError(400, "invalid_page", "page must be an integer between 1 and 10");35  let language: string | undefined;36  if (body.language !== undefined && body.language !== null && body.language !== "") {37    if (typeof body.language !== "string" || !/^[a-z]{2}(-[a-z]{2})?$/i.test(body.language)) throw new HttpError(400, "invalid_language", "language must be an ISO 639-1 code");38    language = body.language.toLowerCase();39  }40  let region: string | undefined;41  if (body.region !== undefined && body.region !== null && body.region !== "") {42    if (typeof body.region !== "string" || !/^[a-z]{2}$/i.test(body.region)) throw new HttpError(400, "invalid_region", "region must be an ISO 3166-1 alpha-2 code");43    region = body.region.toLowerCase();44  }45  let timeRange: TimeRange | undefined;46  if (body.timeRange) {47    if (!RANGES.includes(body.timeRange)) throw new HttpError(400, "invalid_time_range", "timeRange must be day, week, month or year");48    timeRange = body.timeRange;49  }50  let source: SearchRequest["source"];51  if (body.source) {52    if (!["youtube", "vimeo", "dailymotion"].includes(body.source)) throw new HttpError(400, "invalid_source", "unknown video source");53    source = body.source;54  }55  return {56    query: normalizeQuery(body.query), type, page, language, region, timeRange, source,57    safeSearch: body.safeSearch !== false,58    summary: body.summary === true,59  };60}6162/** Canonical form used to deduplicate results: no fragment, no tracking parameters, lowercase host, no trailing slash. */63export function canonicalURL(raw: string): string | null {64  let url: URL;65  try { url = new URL(raw); } catch { return null; }66  if (url.protocol !== "http:" && url.protocol !== "https:") return null;67  url.hash = "";68  url.hostname = url.hostname.toLowerCase();69  for (const key of [...url.searchParams.keys()]) if (TRACKING.test(key)) url.searchParams.delete(key);70  let text = url.toString();71  if (url.pathname !== "/" && text.endsWith("/")) text = text.slice(0, -1);72  return text;73}7475export function dedupeKey(raw: string): string {76  const canonical = canonicalURL(raw) ?? raw;77  return canonical.replace(/^https?:\/\/(www\.)?/, "").replace(/\/$/, "");78}7980export function domainOf(raw: string): string {81  try { return new URL(raw).hostname.toLowerCase().replace(/^www\./, ""); } catch { return ""; }82}8384export function resultId(value: string): string {85  return createHash("sha256").update(value).digest("base64url").slice(0, 16);86}8788/** Cache identity of a request batch. The query only ever lives in memory, hashed here. */89export function cacheKey(request: SearchRequest, batch: number): string {90  return createHash("sha256").update(JSON.stringify([91    request.query.toLowerCase(), request.type, request.language ?? "", request.region ?? "", request.timeRange ?? "",92    request.safeSearch, request.type === "web" ? request.summary : false, request.source ?? "", batch,93  ])).digest("hex");94}95
Files main
Menu

Theme