Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeExtractionTests.swift3// Description: Text extraction on generated PDF, DOCX (stored and deflated), Markdown, text encodings, CSV and source code; error paths.4import Foundation5import Testing6@testable import LocalChatKnowledge78@Suite struct KnowledgeExtractionTests {9 let extractor = TextExtractor()1011 @Test func pdfPagesKeepTheirNumbers() throws {12 let data = KnowledgeFixtures.pdf(pages: ["Quarterly revenue grew strongly", "", "Appendix about penguins"])13 let doc = try extractor.extract(data: data, fileName: "report.pdf")14 #expect(doc.kind == .pdf)15 #expect(doc.sections.map(\.page) == [1, 3])16 #expect(doc.sections[0].text.contains("Quarterly revenue"))17 #expect(doc.sections[1].text.contains("penguins"))18 #expect(doc.warnings.contains { $0.contains("no text layer") && $0.contains("page 2") })19 }2021 @Test func scannedPDFNeedsOCRThenIsReadByVision() throws {22 let data = KnowledgeFixtures.scannedPDF(text: "INVOICE NUMBER 48213")23 #expect(throws: ExtractionError.self) { try extractor.extract(data: data, fileName: "scan.pdf") }24 let ocr = TextExtractor(options: .init(ocrPagesWithoutText: true, ocrLanguages: ["en-US"]))25 let doc = try ocr.extract(data: data, fileName: "scan.pdf")26 #expect(doc.sections.first?.page == 1)27 #expect(doc.sections.first?.text.contains("48213") == true)28 #expect(doc.warnings.contains { $0.contains("OCR") })29 }3031 @Test func corruptAndUnsupportedFilesGiveClearErrors() {32 #expect(throws: ExtractionError.self) { try extractor.extract(data: Data("not a pdf at all".utf8), fileName: "x.pdf") }33 #expect(throws: ExtractionError.self) { try extractor.extract(data: Data([0, 1, 2, 3, 0, 0, 9, 0, 0, 0, 0, 7]), fileName: "blob.bin") }34 #expect(throws: ExtractionError.empty("the file is empty.")) { try extractor.extract(data: Data(), fileName: "a.txt") }35 let small = TextExtractor(options: .init(maxBytes: 8))36 #expect(throws: ExtractionError.tooLarge(bytes: 12, limit: 8)) { try small.extract(data: Data("twelve bytes".utf8), fileName: "a.txt") }37 #expect(throws: ExtractionError.encrypted) { try extractor.extract(data: Data([0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]), fileName: "secret.docx") }38 #expect(throws: ExtractionError.self) { try extractor.extract(data: Data("PK but not really".utf8), fileName: "bad.docx") }39 }4041 @Test(arguments: [false, true]) func docxParagraphsHeadingsAndTables(deflate: Bool) throws {42 let body = KnowledgeFixtures.paragraph("Project Atlas", style: "Title")43 + KnowledgeFixtures.paragraph("Atlas is the internal name of the migration.")44 + KnowledgeFixtures.paragraph("Budget", style: "Heading2")45 + "<w:p><w:r><w:t>First line</w:t><w:br/><w:t>second</w:t><w:tab/><w:t>tabbed & escaped</w:t></w:r></w:p>"46 + "<w:tbl><w:tr><w:tc><w:p><w:r><w:t>Item</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Cost</w:t></w:r></w:p></w:tc></w:tr>"47 + "<w:tr><w:tc><w:p><w:r><w:t>Servers</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>4200</w:t></w:r></w:p></w:tc></w:tr></w:tbl>"48 + String(repeating: KnowledgeFixtures.paragraph("Filler paragraph to make deflate worthwhile."), count: 40)49 let doc = try extractor.extract(data: KnowledgeFixtures.docx(body: body, deflate: deflate), fileName: "atlas.docx")50 #expect(doc.kind == .docx)51 #expect(doc.sections.count == 2)52 #expect(doc.sections[0].heading == "Project Atlas")53 #expect(doc.sections[0].text.contains("internal name of the migration"))54 #expect(doc.sections[1].heading == "Budget")55 #expect(doc.sections[1].text.contains("First line\nsecond\ttabbed & escaped"))56 #expect(doc.sections[1].text.contains("Item | Cost\nServers | 4200"))57 }5859 @Test func docxIntegrityAndEncryptionAreDetected() {60 let xml = Data(KnowledgeFixtures.documentXML(KnowledgeFixtures.paragraph("hello")).utf8)61 let badCRC = KnowledgeFixtures.zip([("word/document.xml", xml)], deflate: true, corruptCRC: true)62 #expect(throws: ExtractionError.self) { try extractor.extract(data: badCRC, fileName: "a.docx") }63 let encrypted = KnowledgeFixtures.zip([("word/document.xml", xml)], deflate: false, flags: 1)64 #expect(throws: ExtractionError.encrypted) { try extractor.extract(data: encrypted, fileName: "a.docx") }65 let other = KnowledgeFixtures.zip([("readme.txt", Data("hi".utf8))], deflate: false)66 #expect(throws: ExtractionError.self) { try extractor.extract(data: other, fileName: "a.docx") }67 }6869 @Test func markdownSectionsFollowHeadingsButNotFences() throws {70 let md = "Intro text.\n\n# Guide\n\nWelcome.\n\n## Install\n\nRun this:\n\n```sh\n# not a heading\nbrew install x\n```\n\n## Usage ##\n\nUse it.\n\n# Other\n\nEnd."71 let doc = try extractor.extract(data: Data(md.utf8), fileName: "guide.md")72 #expect(doc.sections.map(\.heading) == [nil, "Guide", "Guide › Install", "Guide › Usage", "Other"])73 #expect(doc.sections[2].text.contains("# not a heading"))74 #expect(doc.sections[2].text.hasPrefix("## Install"))75 }7677 @Test func textEncodingsAreDetected() throws {78 let sample = "Café déjà vu — naïve façade"79 for (data, label) in [(Data(sample.utf8), "utf8"), (Data([0xEF, 0xBB, 0xBF]) + Data(sample.utf8), "bom"),80 (sample.data(using: .utf16)!, "utf16 bom"), (sample.data(using: .utf16LittleEndian)!, "utf16le"),81 (sample.data(using: .utf16BigEndian)!, "utf16be")] {82 let doc = try extractor.extract(data: data, fileName: "a.txt")83 #expect(doc.sections[0].text == sample, "\(label)")84 }85 let latin = try extractor.extract(data: "Café déjà vu".data(using: .isoLatin1)!, fileName: "a.txt")86 #expect(latin.sections[0].text == "Café déjà vu")87 #expect(latin.warnings.contains { $0.contains("Windows-1252") })88 let unknown = try extractor.extract(data: Data("plain notes".utf8), fileName: "NOTES")89 #expect(unknown.kind == .txt)90 }9192 @Test func csvQuotingDelimitersAndRendering() throws {93 let csv = "name,comment,city\r\n\"Doe, Jane\",\"She said \"\"hi\"\"\nand left\",Paris\r\nBob,,Lyon\r\n"94 let (rows, open) = DelimitedText.parse(csv, delimiter: ",")95 #expect(!open)96 #expect(rows == [["name", "comment", "city"], ["Doe, Jane", "She said \"hi\"\nand left", "Paris"], ["Bob", "", "Lyon"]])97 #expect(DelimitedText.sniffDelimiter("a;b;c\n1,5;2;3\n4;5;6\n") == ";")98 #expect(DelimitedText.sniffDelimiter("a\tb\n1\t2\n") == "\t")99 #expect(DelimitedText.sniffDelimiter(csv) == ",")100 let doc = try extractor.extract(data: Data(csv.utf8), fileName: "people.csv")101 #expect(doc.kind == .csv)102 #expect(doc.sections[0].text == "Row 1\nname: Doe, Jane\ncomment: She said \"hi\" / and left\ncity: Paris\n\nRow 2\nname: Bob\ncity: Lyon")103 let tsv = try extractor.extract(data: Data("k\tv\nalpha\t1,5\n".utf8), fileName: "t.tsv")104 #expect(tsv.sections[0].text == "Row 1\nk: alpha\nv: 1,5")105 let broken = try extractor.extract(data: Data("a,b\n1,\"never closed\n".utf8), fileName: "b.csv")106 #expect(broken.warnings.contains { $0.contains("never closed") })107 }108109 @Test func sourceCodeKeepsItsLanguage() throws {110 let doc = try extractor.extract(data: Data("func main() {\n print(\"hi\")\n}\n".utf8), fileName: "main.swift")111 #expect(doc.kind == .code && doc.language == "swift")112 #expect(doc.sections[0].text.contains(" print"))113 #expect(TextExtractor.kind(forFileName: "Dockerfile")?.language == "dockerfile")114 #expect(TextExtractor.kind(forFileName: "a.xyz") == nil)115 }116}117