SPB Git

Repository Public

search.test.ts 15.8 KB 234 lines typescript

1import test from "node:test";2import assert from "node:assert/strict";3import { canonicalURL, dedupeKey, normalizeQuery, parseSearchRequest } from "../src/search/normalize.ts";4import { parseISODuration, parseVideoURL } from "../src/search/video.ts";5import { Client, fakeProvider, makeApp } from "./helpers.ts";67const result = (n: number, extra: any = {}) => ({ title: `Title ${n}`, url: `https://site${n}.example/page`, content: `Snippet  ${n}\n text`, score: 1 - n / 100, favicon: `https://site${n}.example/f.ico`, ...extra });89test("query normalization and request validation", () => {10  assert.equal(normalizeQuery("  apple\u0000   silicon\n ai "), "apple silicon ai");11  assert.throws(() => normalizeQuery("   "), /empty/);12  assert.throws(() => normalizeQuery("x".repeat(401)), /longer/);13  assert.throws(() => parseSearchRequest({ query: "a", type: "news" }), /type/);14  assert.throws(() => parseSearchRequest({ query: "a", page: 0 }), /page/);15  assert.throws(() => parseSearchRequest({ query: "a", language: "english" }), /language/);16  const parsed = parseSearchRequest({ query: " Swift ", type: "images", language: "FR", region: "CA", timeRange: "week" });17  assert.deepEqual({ ...parsed }, { query: "Swift", type: "images", page: 1, language: "fr", region: "ca", timeRange: "week", source: undefined, safeSearch: true, summary: false });18});1920test("URL canonicalization removes tracking and fragments", () => {21  assert.equal(canonicalURL("https://WWW.Example.com/a/?utm_source=x&id=2&fbclid=9#top"), "https://www.example.com/a/?id=2");22  assert.equal(canonicalURL("javascript:alert(1)"), null);23  assert.equal(dedupeKey("http://www.example.com/a/"), dedupeKey("https://example.com/a"));24});2526test("video URL recognition accepts watch pages only", () => {27  assert.equal(parseVideoURL("https://m.youtube.com/watch?v=dQw4w9WgXcQ&xstg=abc")?.url, "https://www.youtube.com/watch?v=dQw4w9WgXcQ");28  assert.equal(parseVideoURL("https://youtu.be/dQw4w9WgXcQ?t=3")?.platform, "youtube");29  assert.equal(parseVideoURL("https://www.youtube.com/shorts/dQw4w9WgXcQ")?.id, "dQw4w9WgXcQ");30  assert.equal(parseVideoURL("https://vimeo.com/76979871")?.embedURL, "https://player.vimeo.com/video/76979871");31  assert.equal(parseVideoURL("https://www.dailymotion.com/video/x8abc12")?.platform, "dailymotion");32  for (const url of ["https://www.youtube.com/playlist?list=PL1", "https://www.youtube.com/@channel", "https://vimeo.com/channels/staffpicks", "https://example.com/watch?v=dQw4w9WgXcQ", "nope"]) {33    assert.equal(parseVideoURL(url), null, url);34  }35  assert.equal(parseISODuration("PT1H2M3S"), 3723);36  assert.equal(parseISODuration("PT45S"), 45);37});3839test("web search: provider request, normalization, dedup, cache, paging, usage accounting", async () => {40  const provider = await fakeProvider();41  const app = makeApp({ tavilyBase: provider.url });42  const client = new Client(app);43  provider.handler = (_path, body) => body.exclude_domains44    ? { body: { results: [result(30), result(31)], images: [], usage: { credits: 1 } } }45    : { body: { answer: " Summary text. ", usage: { credits: 1 }, images: [], results: [46        ...Array.from({ length: 19 }, (_, i) => result(i)),47        result(0, { url: "http://site0.example/page/?utm_campaign=x#frag" }), // duplicate of the first48        { title: 5, url: "https://bad.example" }, { title: "No URL" }, { title: "Script", url: "javascript:alert(1)" },49      ] } };50  const first = await client.call("POST", "/api/search", { query: "  Apple   Silicon ", type: "web", summary: true, language: "en", region: "ca", timeRange: "month" });51  assert.equal(first.status, 200);52  const sent = provider.calls[0];53  assert.equal(sent.auth, "Bearer tvly-test");54  assert.equal(sent.body.query, "Apple Silicon");55  assert.equal(sent.body.max_results, 20);56  assert.equal(sent.body.country, "canada");57  assert.equal(sent.body.language, "en");58  assert.equal(sent.body.time_range, "month");59  assert.equal(sent.body.include_answer, "basic");60  assert.equal(first.json.results.length, 10);61  assert.equal(first.json.answer.text, "Summary text.");62  assert.equal(first.json.results[0].domain, "site0.example");63  assert.equal(first.json.results[0].snippet, "Snippet 0 text");64  assert.equal(first.json.cached, false);65  assert.equal(first.json.hasMore, true);66  assert.ok(!JSON.stringify(first.json).includes("tvly-test"));6768  // Same query, different spacing/case → cache hit, no provider call.69  const again = await client.call("POST", "/api/search", { query: "apple silicon", type: "web", summary: true, language: "en", region: "ca", timeRange: "month" });70  assert.equal(again.json.cached, true);71  assert.equal(provider.calls.length, 1);7273  // Page 2: the first batch has 19 unique results, so a second batch (other domains) completes the page.74  const second = await client.call("POST", "/api/search", { query: "apple silicon", type: "web", summary: true, language: "en", region: "ca", timeRange: "month", page: 2 });75  assert.equal(second.json.results.length, 10);76  assert.equal(second.json.answer, null);77  assert.equal(provider.calls.length, 2);78  assert.ok(provider.calls[1].body.exclude_domains.includes("site0.example"));79  assert.equal(provider.calls[1].body.include_answer, false);80  const ids = new Set([...first.json.results, ...second.json.results].map((r: any) => r.url));81  assert.equal(ids.size, 20);82  assert.equal(second.json.hasMore, true);8384  // Page 3: the provider had only two more results; the last one is served and nothing further is fetched.85  const third = await client.call("POST", "/api/search", { query: "apple silicon", type: "web", summary: true, language: "en", region: "ca", timeRange: "month", page: 3 });86  assert.equal(provider.calls.length, 2);87  assert.deepEqual(third.json.results.map((r: any) => r.domain), ["site31.example"]);88  assert.equal(third.json.hasMore, false);8990  const usage = app.db.prepare("SELECT requests, credits, cache_hits FROM usage_daily WHERE kind = 'web'").get() as any;91  assert.equal(usage.requests, 2);92  assert.equal(usage.credits, 2);93  assert.ok(usage.cache_hits >= 3);94  await app.server.close(); await provider.close();95});9697test("provider failures map to stable errors and never leak provider text or the key", async () => {98  const provider = await fakeProvider();99  const app = makeApp({ tavilyBase: provider.url });100  const client = new Client(app);101  const cases: [number, number, string][] = [[401, 503, "provider_auth"], [429, 429, "provider_rate_limited"], [432, 503, "provider_quota"], [500, 502, "provider_error"]];102  for (const [upstream, status, code] of cases) {103    provider.handler = () => ({ status: upstream, body: { detail: { error: "secret upstream text tvly-test" } }, headers: upstream === 429 ? { "retry-after": "7" } : {} });104    const r = await client.call("POST", "/api/search", { query: "failure " + upstream });105    assert.equal(r.status, status);106    assert.equal(r.json.error.code, code);107    assert.ok(!JSON.stringify(r.json).includes("upstream text"));108    if (upstream === 429) assert.equal(r.headers["retry-after"], "7");109  }110  const errors = (app.db.prepare("SELECT SUM(errors) AS n FROM usage_daily").get() as any).n;111  assert.equal(errors, 4);112  await app.server.close(); await provider.close();113114  const unreachable = makeApp({ tavilyBase: "http://127.0.0.1:1" });115  const r = await new Client(unreachable).call("POST", "/api/search", { query: "x" });116  assert.equal(r.status, 502);117  assert.equal(r.json.error.code, "provider_unreachable");118  await unreachable.server.close();119120  const noKey = makeApp({ tavilyKey: "" });121  assert.equal((await new Client(noKey).call("POST", "/api/search", { query: "x" })).json.error.code, "provider_not_configured");122  await noKey.server.close();123});124125test("daily budget and rate limits fail closed", async () => {126  const provider = await fakeProvider();127  provider.handler = () => ({ body: { results: [result(1)], images: [], usage: { credits: 1 } } });128  const app = makeApp({ tavilyBase: provider.url, dailyCreditBudget: 2, anonSearchPerHour: 4 });129  const client = new Client(app);130  assert.equal((await client.call("POST", "/api/search", { query: "one" })).status, 200);131  assert.equal((await client.call("POST", "/api/search", { query: "two" })).status, 200);132  const blocked = await client.call("POST", "/api/search", { query: "three" });133  assert.equal(blocked.status, 503);134  assert.equal(blocked.json.error.code, "budget_exhausted");135  assert.equal((await client.call("POST", "/api/search", { query: "one" })).status, 200); // cached answers still served136  const limited = await client.call("POST", "/api/search", { query: "one" });137  assert.equal(limited.status, 429);138  assert.equal(limited.json.error.code, "rate_limited");139  await app.server.close(); await provider.close();140141  const closed = makeApp({ anonSearch: false });142  assert.equal((await new Client(closed).call("POST", "/api/search", { query: "x" })).status, 401);143  await closed.server.close();144});145146test("image search: page attribution, junk filtering, Commons merge with dimensions", async () => {147  const provider = await fakeProvider();148  process.env.COMMONS_API = provider.url + "/commons";149  provider.handler = (path) => path === "/commons"150    ? { body: { query: { pages: {151        "1": { index: 1, title: "File:Aurora over lake.jpg", imageinfo: [{ url: "https://upload.example/aurora.jpg", descriptionurl: "https://commons.example/wiki/File:Aurora", mime: "image/jpeg", width: 4000, height: 3000, thumburl: "https://upload.example/thumb/640px-aurora.jpg?utm_source=x", extmetadata: { Artist: { value: "<a href='x'>Jane &amp; Co</a>" }, LicenseShortName: { value: "CC BY 4.0" }, ImageDescription: { value: "<p>Aurora</p>" } } }] },152        "2": { index: 2, title: "File:Tiny.png", imageinfo: [{ url: "https://upload.example/tiny.png", mime: "image/png", width: 50, height: 50 }] },153        "3": { index: 3, title: "File:Doc.pdf", imageinfo: [{ url: "https://upload.example/doc.pdf", mime: "application/pdf", width: 900, height: 900 }] },154      } } } }155    : { body: { usage: { credits: 1 }, images: [{ url: "https://cdn.example/top.jpg", description: "Top image" }, "https://cdn.example/plain.jpg"],156        results: [{ title: "Gallery", url: "https://photos.example/gallery?utm_source=a", images: [157          { url: "https://photos.example/a.jpg", description: "A", score: 5 }, { url: "https://photos.example/hero.jpg", description: null, score: 10 },158          { url: "https://photos.example/site-logo.png", score: 9 }, { url: "https://photos.example/small.jpg?width=24", score: 8 }, { url: "https://photos.example/vector.svg", score: 7 },159          { url: "data:image/png;base64,AAAA", score: 6 }, { url: "https://cdn.example/top.jpg", score: 1 },160        ] }] } };161  const app = makeApp({ tavilyBase: provider.url });162  const r = await new Client(app).call("POST", "/api/search", { query: "aurora", type: "images" });163  assert.equal(r.status, 200);164  assert.equal(provider.calls.find((c) => c.path === "/search").body.include_images, true);165  const urls = r.json.results.map((i: any) => i.imageURL);166  assert.deepEqual(urls, ["https://cdn.example/top.jpg", "https://cdn.example/plain.jpg", "https://photos.example/hero.jpg", "https://photos.example/a.jpg", "https://upload.example/aurora.jpg"]);167  const hero = r.json.results[2];168  assert.equal(hero.sourceURL, "https://photos.example/gallery");169  assert.equal(hero.sourceTitle, "Gallery");170  assert.equal(hero.width, null);171  assert.equal(r.json.results[0].sourceURL, null);172  const commons = r.json.results[4];173  assert.deepEqual([commons.width, commons.height, commons.author, commons.license, commons.provider, commons.thumbnailURL],174    [4000, 3000, "Jane & Co", "CC BY 4.0", "wikimedia-commons", "https://upload.example/thumb/640px-aurora.jpg"]);175  assert.equal(r.json.provider, "tavily+wikimedia-commons");176  delete process.env.COMMONS_API;177  await app.server.close(); await provider.close();178});179180test("video search: only platform-confirmed watch pages are returned", async () => {181  const provider = await fakeProvider();182  provider.handler = (path, _body, query) => {183    if (path === "/search") return { body: { usage: { credits: 1 }, images: [], results: [184      { title: "Video A", url: "https://www.youtube.com/watch?v=AAAAAAAAAAA&xstg=1", content: "Transcript  A" },185      { title: "Video A again", url: "https://youtu.be/AAAAAAAAAAA" },186      { title: "A playlist", url: "https://www.youtube.com/playlist?list=PL1" },187      { title: "Removed video", url: "https://www.youtube.com/watch?v=BBBBBBBBBBB" },188      { title: "Vimeo", url: "https://vimeo.com/123456789" },189      { title: "Blog about video", url: "https://vimeo.com/blog/post/hello" },190    ] } };191    const url = query.get("url")!;192    if (url.includes("BBBBBBBBBBB")) return { status: 404, body: null };193    if (path === "/yt") return { body: { type: "video", title: "Real title A", author_name: "Channel A", author_url: "https://www.youtube.com/@a", thumbnail_url: "https://i.ytimg.com/vi/AAAAAAAAAAA/hqdefault.jpg" } };194    if (path === "/vimeo") return { body: { type: "video", title: "Vimeo film", author_name: "Studio", duration: 125, upload_date: "2024-05-01 10:00:00", thumbnail_url: "https://i.vimeocdn.com/x.jpg", description: "A film" } };195    return { status: 404, body: null };196  };197  const app = makeApp({ tavilyBase: provider.url, oembedBases: { youtube: provider.url + "/yt", vimeo: provider.url + "/vimeo", dailymotion: provider.url + "/dm" } });198  const r = await new Client(app).call("POST", "/api/search", { query: "apple silicon video", type: "videos" });199  assert.equal(r.status, 200);200  const search = provider.calls.filter((c) => c.path === "/search");201  assert.equal(search.length, 1); // the query already says "video": no second phrasing202  assert.deepEqual(search[0].body.include_domains, ["youtube.com", "youtu.be", "vimeo.com", "dailymotion.com"]);203  assert.equal(r.json.results.length, 2);204  const [a, v] = r.json.results;205  assert.deepEqual([a.title, a.channel, a.platform, a.durationSeconds, a.verifiedBy, a.description], ["Real title A", "Channel A", "youtube", null, "oembed", "A"]);206  assert.equal(a.thumbnail, "https://i.ytimg.com/vi/AAAAAAAAAAA/mqdefault.jpg");207  assert.deepEqual([v.title, v.durationSeconds, v.publishedDate, v.embedURL], ["Vimeo film", 125, "2024-05-01T10:00:00", "https://player.vimeo.com/video/123456789"]);208  assert.ok(r.json.notices.includes("video-duration-unavailable-without-youtube-data-api"));209  assert.equal(r.json.provider, "tavily+oembed");210211  const filtered = await new Client(app).call("POST", "/api/search", { query: "apple silicon video", type: "videos", source: "vimeo" });212  assert.deepEqual(provider.calls.filter((c) => c.path === "/search").at(-1).body.include_domains, ["vimeo.com"]);213  assert.equal(filtered.status, 200);214  await app.server.close(); await provider.close();215});216217test("video search through the YouTube Data API when a key is configured", async () => {218  const provider = await fakeProvider();219  process.env.YOUTUBE_BASE = provider.url;220  provider.handler = (path) => path === "/search"221    ? { body: { nextPageToken: "NEXT", items: [{ id: { videoId: "AAAAAAAAAAA" } }, { id: { videoId: "GONE0000000" } }] } }222    : { body: { items: [{ id: "AAAAAAAAAAA", contentDetails: { duration: "PT10M5S" }, snippet: { title: "Talk", channelTitle: "Chan", channelId: "UC1", publishedAt: "2025-01-02T00:00:00Z", description: "Desc", thumbnails: { high: { url: "https://i.ytimg.com/h.jpg" } } } }] } };223  const app = makeApp({ youtubeKey: "yt-key", tavilyBase: "http://127.0.0.1:1" });224  const r = await new Client(app).call("POST", "/api/search", { query: "talk", type: "videos", region: "ca" });225  assert.equal(r.status, 200);226  assert.equal(provider.calls[0].query.get("regionCode"), "CA");227  assert.equal(r.json.results.length, 1);228  assert.deepEqual([r.json.results[0].durationSeconds, r.json.results[0].publishedDate, r.json.results[0].verifiedBy], [605, "2025-01-02T00:00:00Z", "youtube-data-api"]);229  assert.equal(r.json.provider, "youtube");230  assert.ok(!JSON.stringify(r.json).includes("yt-key"));231  delete process.env.YOUTUBE_BASE;232  await app.server.close(); await provider.close();233});234
Files main
Menu

Theme