spb/wp-spboucher-ai

1import { describe, expect, it } from "vitest";2import { proposeMetadata, titleSimilarity } from "@/lib/pdf/extract";3import { chunkPages } from "@/lib/pdf/chunk";4import { cosine, lexvec, LEXVEC_DIM, tokenize } from "@/lib/ai/lexvec";5const page = `UQO Working Paper No. 416Version 1.0 — November 202678The Invisible Balance Sheet: Intraday Information and Accounting Deterioration910Simon-Pierre Boucher11Université du Québec en Outaouais1213Abstract14This paper asks whether one-minute bars contain information about future accounting deterioration beyond what daily data provides. We assemble a point-in-time panel of 4,568 firms and 194,817 firm-quarters and compare information sets at matched model budgets.15Keywords: high-frequency data; accounting quality; point-in-time; LightGBM16JEL: G12, G14, M41171 Introduction`;18describe("metadata proposal", () => {19  const p = proposeMetadata(page, "The Invisible Balance Sheet", "Simon-Pierre Boucher");20  it("detects identifier, version, date", () => { expect(p.wp).toEqual({ seriesCode: "WP", number: 41 }); expect(p.version).toBe("1.0"); expect(p.dateText).toBe("November 2026"); });21  it("proposes title, authors, abstract, keywords, jel", () => { expect(p.title).toContain("The Invisible Balance Sheet"); expect(p.authors).toContain("Simon-Pierre Boucher"); expect(p.abstract).toMatch(/^This paper asks/); expect(p.keywords).toEqual(["high-frequency data", "accounting quality", "point-in-time", "LightGBM"]); expect(p.jel).toEqual(["G12", "G14", "M41"]); });22  it("never fills fields it cannot find", () => { const q = proposeMetadata("Some random text without structure", null, null); expect(q.wp).toBeNull(); expect(q.abstract).toBeNull(); expect(q.dateText).toBeNull(); expect(q.confidence.abstract).toBe(0); expect(q.confidence.wp).toBe(0); });23  it("title similarity is symmetric and bounded", () => { expect(titleSimilarity("Hedonic Models for Quebec", "Quebec hedonic models")).toBeGreaterThan(0.6); expect(titleSimilarity("a", "completely different")).toBe(0); });24});25describe("chunking and lexical vectors", () => {26  it("chunks pages with page ranges and overlap", () => { const pages = [Array(40).fill("Sentence one of the paragraph is here.").join(" ") + "\n\n" + Array(40).fill("Another paragraph with content.").join(" "), Array(60).fill("Second page text.").join(" ")]; const c = chunkPages(pages, 600, 80); expect(c.length).toBeGreaterThan(2); expect(c[0].pageFrom).toBe(1); expect(c[c.length - 1].pageTo).toBe(2); expect(c.every((x) => x.text.length > 80)).toBe(true); });27  it("lexvec is deterministic, unit-norm and discriminative", () => { const a = lexvec("hedonic house prices quebec spatial regression"); const b = lexvec("hedonic house prices quebec spatial regression"); const c = lexvec("limit order book liquidity llm agents"); expect(a).toEqual(b); expect(a.length).toBe(LEXVEC_DIM); expect(Math.abs(cosine(a, a) - 1)).toBeLessThan(1e-9); expect(cosine(a, c)).toBeLessThan(0.3); expect(tokenize("The Of and Quebec 2026")).toEqual(["quebec"]); });28});29