SPB Git

Repository Public

VQResearchText.swift 11.0 KB 192 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Sources/VQuantKit/ResearchAgent/VQResearchText.swift3// Description: The pure helpers of the research views: the workspace as a tree, what each file is, a CSV reader for4//              table previews, and a small Python tokenizer for syntax colouring.5import Foundation67public struct VQResearchFileNode: Sendable, Hashable, Identifiable {8    public var name: String9    public var path: String10    public var isDirectory: Bool11    public var bytes: Int12    public var modifiedAt: Date?13    public var children: [VQResearchFileNode]14    /// Number of files below (1 for a file).15    public var fileCount: Int16    public var id: String { path }17    /// nil for files: what `OutlineGroup` and `List(children:)` expect.18    public var childNodes: [VQResearchFileNode]? { isDirectory ? children : nil }19}2021public enum VQResearchFileTree {22    /// Builds the tree from the flat recursive listing. Directories first, then names (numeric-aware).23    public static func build(_ files: [VQResearchFile]) -> [VQResearchFileNode] {24        final class Dir { var node: VQResearchFileNode; var dirs: [String: Dir] = [:]; var files: [VQResearchFileNode] = []; init(_ n: VQResearchFileNode) { node = n } }25        let root = Dir(.init(name: "", path: "", isDirectory: true, bytes: 0, modifiedAt: nil, children: [], fileCount: 0))26        func dir(_ path: String) -> Dir {27            if path.isEmpty { return root }28            var current = root, sofar = ""29            for part in path.split(separator: "/") {30                sofar = sofar.isEmpty ? String(part) : sofar + "/" + part31                if let d = current.dirs[String(part)] { current = d } else {32                    let d = Dir(.init(name: String(part), path: sofar, isDirectory: true, bytes: 0, modifiedAt: nil, children: [], fileCount: 0))33                    current.dirs[String(part)] = d; current = d34                }35            }36            return current37        }38        for f in files where !f.path.isEmpty {39            if f.isDirectory { dir(f.path).node.modifiedAt = f.modifiedAt; continue }40            let parent = f.path.lastIndex(of: "/").map { String(f.path[..<$0]) } ?? ""41            dir(parent).files.append(.init(name: f.name, path: f.path, isDirectory: false, bytes: f.bytes, modifiedAt: f.modifiedAt, children: [], fileCount: 1))42        }43        func finish(_ d: Dir) -> VQResearchFileNode {44            let byName: (VQResearchFileNode, VQResearchFileNode) -> Bool = { $0.name.localizedStandardCompare($1.name) == .orderedAscending }45            var n = d.node46            n.children = d.dirs.values.map(finish).sorted(by: byName) + d.files.sorted(by: byName)47            n.bytes = n.children.reduce(0) { $0 + $1.bytes }48            n.fileCount = n.children.reduce(0) { $0 + $1.fileCount }49            return n50        }51        return finish(root).children52    }5354    public enum FileKind: String, Sendable, Hashable { case code, table, text, json, image, pdf, data, log, other }5556    public static func kind(of path: String) -> FileKind {57        let name = (path.split(separator: "/").last.map(String.init) ?? path).lowercased()58        guard let dot = name.lastIndex(of: ".") else { return .other }59        switch String(name[dot...]) {60        case ".py", ".r", ".sql", ".ipynb", ".toml", ".yaml", ".yml", ".tex": return .code61        case ".csv", ".tsv": return .table62        case ".json", ".jsonl": return .json63        case ".log": return .log64        case ".txt", ".md": return .text65        case ".png", ".svg", ".jpg", ".jpeg", ".gif", ".webp": return .image66        case ".pdf": return .pdf67        case ".parquet", ".xlsx", ".xls", ".feather": return .data68        default: return .other69        }70    }71    /// Files the text route returns by lines; the others are raw bytes.72    public static func isTextual(_ path: String) -> Bool { [.code, .table, .json, .log, .text].contains(kind(of: path)) }73    /// Files the dataset inspector can profile.74    public static func isDataset(_ path: String) -> Bool {75        let k = kind(of: path)76        return k == .table || k == .data || (path.hasPrefix("data/") && (k == .json || k == .text))77    }78}7980public enum VQDelimitedText {81    public struct Table: Sendable, Equatable {82        public var header: [String]83        public var rows: [[String]]84        public var delimiter: Character85        /// More rows exist than were read.86        public var truncated: Bool87    }8889    /// RFC 4180-style reader (quotes, doubled quotes, newlines inside quotes). The delimiter is `,` `;` or tab,90    /// guessed from the first line.91    public static func parse(_ text: String, maxRows: Int = 200) -> Table {92        let firstLine = text.prefix { $0 != "\n" && $0 != "\r\n" }93        func count(_ d: Character) -> Int { firstLine.reduce(0) { $0 + ($1 == d ? 1 : 0) } }94        let delimiter: Character = count("\t") > count(",") && count("\t") >= count(";") ? "\t" : (count(";") > count(",") ? ";" : ",")95        var rows: [[String]] = [], row: [String] = [], field = ""96        var quoted = false, truncated = false97        var it = text.makeIterator()98        var pending: Character? = nil99        func endRow() { row.append(field); field = ""; if !(row.count == 1 && row[0].isEmpty) { rows.append(row) }; row = [] }100        while let c = pending ?? it.next() {101            pending = nil102            if quoted {103                if c == "\"" { if let n = it.next() { if n == "\"" { field.append("\"") } else { quoted = false; pending = n } } else { quoted = false } }104                else { field.append(c) }105            } else if c == "\"" && field.isEmpty { quoted = true }106            else if c == delimiter { row.append(field); field = "" }107            else if c == "\n" || c == "\r\n" || c == "\r" {108                endRow()109                if rows.count > maxRows { truncated = true; break }110            } else { field.append(c) }111        }112        if !truncated, !field.isEmpty || !row.isEmpty { endRow() }113        if rows.count > maxRows + 1 { rows = Array(rows.prefix(maxRows + 1)); truncated = true }114        guard let header = rows.first else { return Table(header: [], rows: [], delimiter: delimiter, truncated: false) }115        if rows.count - 1 > maxRows { truncated = true }116        return Table(header: header, rows: Array(rows.dropFirst().prefix(maxRows)), delimiter: delimiter, truncated: truncated)117    }118}119120/// A single-pass Python tokenizer for colouring: keywords, built-ins, strings (triple-quoted and prefixed included),121/// numbers, comments, decorators, definitions. Not a parser: fast, and good enough to read code.122public enum VQPythonSyntax {123    public enum Kind: String, Sendable, Hashable { case plain, keyword, builtin, string, number, comment, decorator, definition, `operator` }124    public struct Token: Sendable, Equatable {125        public var kind: Kind126        /// Range in UTF-16 units of the source (what `NSRange` and `AttributedString` indices are built from).127        public var range: Range<Int>128    }129130    static let keywords: Set<String> = ["False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "finally",131                                        "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "match", "case"]132    static let builtins: Set<String> = ["print", "len", "range", "int", "float", "str", "list", "dict", "set", "tuple", "bool", "enumerate", "zip", "map", "filter", "sorted", "sum", "min", "max",133                                        "abs", "round", "open", "isinstance", "type", "any", "all", "repr", "getattr", "setattr", "hasattr", "super", "self", "Exception", "ValueError", "KeyError"]134135    /// Tokens other than `.plain`, in order. Text between them is plain.136    public static func tokens(_ source: String) -> [Token] {137        let u = Array(source.utf16)138        var out: [Token] = []139        var i = 0140        let n = u.count141        func isIdentStart(_ c: UInt16) -> Bool { (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c == 95 || c > 127 }142        func isDigit(_ c: UInt16) -> Bool { c >= 48 && c <= 57 }143        var expectDefinition = false144        while i < n {145            let c = u[i]146            if c == 35 {                                                    // # comment147                var j = i; while j < n, u[j] != 10 { j += 1 }148                out.append(Token(kind: .comment, range: i..<j)); i = j; continue149            }150            if c == 34 || c == 39 {                                         // string151                let triple = i + 2 < n && u[i + 1] == c && u[i + 2] == c152                var j = i + (triple ? 3 : 1)153                while j < n {154                    if u[j] == 92 { j += 2; continue }155                    if triple { if j + 2 < n + 0, u[j] == c, j + 2 <= n - 1, u[j + 1] == c, u[j + 2] == c { j += 3; break } }156                    else if u[j] == c { j += 1; break } else if u[j] == 10 { break }157                    j += 1158                }159                j = min(j, n)160                out.append(Token(kind: .string, range: i..<j)); i = j; continue161            }162            if c == 64, i == 0 || u[i - 1] == 10 || u[i - 1] == 32 || u[i - 1] == 9 {     // @decorator at line start163                var k = i - 1; var lineStart = true164                while k >= 0, u[k] != 10 { if u[k] != 32 && u[k] != 9 { lineStart = false; break }; k -= 1 }165                if lineStart {166                    var j = i + 1; while j < n, isIdentStart(u[j]) || isDigit(u[j]) || u[j] == 46 { j += 1 }167                    out.append(Token(kind: .decorator, range: i..<j)); i = j; continue168                }169            }170            if isDigit(c) || (c == 46 && i + 1 < n && isDigit(u[i + 1])) {   // number171                var j = i + 1172                while j < n, isDigit(u[j]) || u[j] == 46 || u[j] == 95 || (u[j] >= 97 && u[j] <= 122) || (u[j] >= 65 && u[j] <= 90)173                        || ((u[j] == 43 || u[j] == 45) && (u[j - 1] == 101 || u[j - 1] == 69)) { j += 1 }174                out.append(Token(kind: .number, range: i..<j)); i = j; continue175            }176            if isIdentStart(c) {177                var j = i + 1; while j < n, isIdentStart(u[j]) || isDigit(u[j]) { j += 1 }178                let word = String(utf16CodeUnits: Array(u[i..<j]), count: j - i)179                // string prefixes: r"", f"", b"", rb""…180                if j < n, (u[j] == 34 || u[j] == 39), word.count <= 2, word.lowercased().allSatisfy({ "rbfu".contains($0) }) { i = j; continue }181                if expectDefinition { out.append(Token(kind: .definition, range: i..<j)); expectDefinition = false }182                else if keywords.contains(word) { out.append(Token(kind: .keyword, range: i..<j)); expectDefinition = word == "def" || word == "class" }183                else if builtins.contains(word), !(i > 0 && u[i - 1] == 46) { out.append(Token(kind: .builtin, range: i..<j)) }184                i = j; continue185            }186            if c != 32 && c != 10 && c != 9 { expectDefinition = false }187            i += 1188        }189        return out190    }191}192
Files main
Menu

Theme