SPB Git

Repository Public

KnowledgeExtractionTests.swift 7.8 KB 117 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeExtractionTests.swift3// Description: Text extraction on generated PDF, DOCX (stored and deflated), Markdown, text encodings, CSV and source code; error paths.4import Foundation5import Testing6@testable import LocalChatKnowledge78@Suite struct KnowledgeExtractionTests {9    let extractor = TextExtractor()1011    @Test func pdfPagesKeepTheirNumbers() throws {12        let data = KnowledgeFixtures.pdf(pages: ["Quarterly revenue grew strongly", "", "Appendix about penguins"])13        let doc = try extractor.extract(data: data, fileName: "report.pdf")14        #expect(doc.kind == .pdf)15        #expect(doc.sections.map(\.page) == [1, 3])16        #expect(doc.sections[0].text.contains("Quarterly revenue"))17        #expect(doc.sections[1].text.contains("penguins"))18        #expect(doc.warnings.contains { $0.contains("no text layer") && $0.contains("page 2") })19    }2021    @Test func scannedPDFNeedsOCRThenIsReadByVision() throws {22        let data = KnowledgeFixtures.scannedPDF(text: "INVOICE NUMBER 48213")23        #expect(throws: ExtractionError.self) { try extractor.extract(data: data, fileName: "scan.pdf") }24        let ocr = TextExtractor(options: .init(ocrPagesWithoutText: true, ocrLanguages: ["en-US"]))25        let doc = try ocr.extract(data: data, fileName: "scan.pdf")26        #expect(doc.sections.first?.page == 1)27        #expect(doc.sections.first?.text.contains("48213") == true)28        #expect(doc.warnings.contains { $0.contains("OCR") })29    }3031    @Test func corruptAndUnsupportedFilesGiveClearErrors() {32        #expect(throws: ExtractionError.self) { try extractor.extract(data: Data("not a pdf at all".utf8), fileName: "x.pdf") }33        #expect(throws: ExtractionError.self) { try extractor.extract(data: Data([0, 1, 2, 3, 0, 0, 9, 0, 0, 0, 0, 7]), fileName: "blob.bin") }34        #expect(throws: ExtractionError.empty("the file is empty.")) { try extractor.extract(data: Data(), fileName: "a.txt") }35        let small = TextExtractor(options: .init(maxBytes: 8))36        #expect(throws: ExtractionError.tooLarge(bytes: 12, limit: 8)) { try small.extract(data: Data("twelve bytes".utf8), fileName: "a.txt") }37        #expect(throws: ExtractionError.encrypted) { try extractor.extract(data: Data([0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]), fileName: "secret.docx") }38        #expect(throws: ExtractionError.self) { try extractor.extract(data: Data("PK but not really".utf8), fileName: "bad.docx") }39    }4041    @Test(arguments: [false, true]) func docxParagraphsHeadingsAndTables(deflate: Bool) throws {42        let body = KnowledgeFixtures.paragraph("Project Atlas", style: "Title")43            + KnowledgeFixtures.paragraph("Atlas is the internal name of the migration.")44            + KnowledgeFixtures.paragraph("Budget", style: "Heading2")45            + "<w:p><w:r><w:t>First line</w:t><w:br/><w:t>second</w:t><w:tab/><w:t>tabbed &amp; escaped</w:t></w:r></w:p>"46            + "<w:tbl><w:tr><w:tc><w:p><w:r><w:t>Item</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Cost</w:t></w:r></w:p></w:tc></w:tr>"47            + "<w:tr><w:tc><w:p><w:r><w:t>Servers</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>4200</w:t></w:r></w:p></w:tc></w:tr></w:tbl>"48            + String(repeating: KnowledgeFixtures.paragraph("Filler paragraph to make deflate worthwhile."), count: 40)49        let doc = try extractor.extract(data: KnowledgeFixtures.docx(body: body, deflate: deflate), fileName: "atlas.docx")50        #expect(doc.kind == .docx)51        #expect(doc.sections.count == 2)52        #expect(doc.sections[0].heading == "Project Atlas")53        #expect(doc.sections[0].text.contains("internal name of the migration"))54        #expect(doc.sections[1].heading == "Budget")55        #expect(doc.sections[1].text.contains("First line\nsecond\ttabbed & escaped"))56        #expect(doc.sections[1].text.contains("Item | Cost\nServers | 4200"))57    }5859    @Test func docxIntegrityAndEncryptionAreDetected() {60        let xml = Data(KnowledgeFixtures.documentXML(KnowledgeFixtures.paragraph("hello")).utf8)61        let badCRC = KnowledgeFixtures.zip([("word/document.xml", xml)], deflate: true, corruptCRC: true)62        #expect(throws: ExtractionError.self) { try extractor.extract(data: badCRC, fileName: "a.docx") }63        let encrypted = KnowledgeFixtures.zip([("word/document.xml", xml)], deflate: false, flags: 1)64        #expect(throws: ExtractionError.encrypted) { try extractor.extract(data: encrypted, fileName: "a.docx") }65        let other = KnowledgeFixtures.zip([("readme.txt", Data("hi".utf8))], deflate: false)66        #expect(throws: ExtractionError.self) { try extractor.extract(data: other, fileName: "a.docx") }67    }6869    @Test func markdownSectionsFollowHeadingsButNotFences() throws {70        let md = "Intro text.\n\n# Guide\n\nWelcome.\n\n## Install\n\nRun this:\n\n```sh\n# not a heading\nbrew install x\n```\n\n## Usage ##\n\nUse it.\n\n# Other\n\nEnd."71        let doc = try extractor.extract(data: Data(md.utf8), fileName: "guide.md")72        #expect(doc.sections.map(\.heading) == [nil, "Guide", "Guide › Install", "Guide › Usage", "Other"])73        #expect(doc.sections[2].text.contains("# not a heading"))74        #expect(doc.sections[2].text.hasPrefix("## Install"))75    }7677    @Test func textEncodingsAreDetected() throws {78        let sample = "Café déjà vu — naïve façade"79        for (data, label) in [(Data(sample.utf8), "utf8"), (Data([0xEF, 0xBB, 0xBF]) + Data(sample.utf8), "bom"),80                              (sample.data(using: .utf16)!, "utf16 bom"), (sample.data(using: .utf16LittleEndian)!, "utf16le"),81                              (sample.data(using: .utf16BigEndian)!, "utf16be")] {82            let doc = try extractor.extract(data: data, fileName: "a.txt")83            #expect(doc.sections[0].text == sample, "\(label)")84        }85        let latin = try extractor.extract(data: "Café déjà vu".data(using: .isoLatin1)!, fileName: "a.txt")86        #expect(latin.sections[0].text == "Café déjà vu")87        #expect(latin.warnings.contains { $0.contains("Windows-1252") })88        let unknown = try extractor.extract(data: Data("plain notes".utf8), fileName: "NOTES")89        #expect(unknown.kind == .txt)90    }9192    @Test func csvQuotingDelimitersAndRendering() throws {93        let csv = "name,comment,city\r\n\"Doe, Jane\",\"She said \"\"hi\"\"\nand left\",Paris\r\nBob,,Lyon\r\n"94        let (rows, open) = DelimitedText.parse(csv, delimiter: ",")95        #expect(!open)96        #expect(rows == [["name", "comment", "city"], ["Doe, Jane", "She said \"hi\"\nand left", "Paris"], ["Bob", "", "Lyon"]])97        #expect(DelimitedText.sniffDelimiter("a;b;c\n1,5;2;3\n4;5;6\n") == ";")98        #expect(DelimitedText.sniffDelimiter("a\tb\n1\t2\n") == "\t")99        #expect(DelimitedText.sniffDelimiter(csv) == ",")100        let doc = try extractor.extract(data: Data(csv.utf8), fileName: "people.csv")101        #expect(doc.kind == .csv)102        #expect(doc.sections[0].text == "Row 1\nname: Doe, Jane\ncomment: She said \"hi\" / and left\ncity: Paris\n\nRow 2\nname: Bob\ncity: Lyon")103        let tsv = try extractor.extract(data: Data("k\tv\nalpha\t1,5\n".utf8), fileName: "t.tsv")104        #expect(tsv.sections[0].text == "Row 1\nk: alpha\nv: 1,5")105        let broken = try extractor.extract(data: Data("a,b\n1,\"never closed\n".utf8), fileName: "b.csv")106        #expect(broken.warnings.contains { $0.contains("never closed") })107    }108109    @Test func sourceCodeKeepsItsLanguage() throws {110        let doc = try extractor.extract(data: Data("func main() {\n    print(\"hi\")\n}\n".utf8), fileName: "main.swift")111        #expect(doc.kind == .code && doc.language == "swift")112        #expect(doc.sections[0].text.contains("    print"))113        #expect(TextExtractor.kind(forFileName: "Dockerfile")?.language == "dockerfile")114        #expect(TextExtractor.kind(forFileName: "a.xyz") == nil)115    }116}117
Files main
Menu

Theme