Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Sources/VQuantKit/ResearchAgent/VQResearchText.swift3// Description: The pure helpers of the research views: the workspace as a tree, what each file is, a CSV reader for4// table previews, and a small Python tokenizer for syntax colouring.5import Foundation67public struct VQResearchFileNode: Sendable, Hashable, Identifiable {8 public var name: String9 public var path: String10 public var isDirectory: Bool11 public var bytes: Int12 public var modifiedAt: Date?13 public var children: [VQResearchFileNode]14 /// Number of files below (1 for a file).15 public var fileCount: Int16 public var id: String { path }17 /// nil for files: what `OutlineGroup` and `List(children:)` expect.18 public var childNodes: [VQResearchFileNode]? { isDirectory ? children : nil }19}2021public enum VQResearchFileTree {22 /// Builds the tree from the flat recursive listing. Directories first, then names (numeric-aware).23 public static func build(_ files: [VQResearchFile]) -> [VQResearchFileNode] {24 final class Dir { var node: VQResearchFileNode; var dirs: [String: Dir] = [:]; var files: [VQResearchFileNode] = []; init(_ n: VQResearchFileNode) { node = n } }25 let root = Dir(.init(name: "", path: "", isDirectory: true, bytes: 0, modifiedAt: nil, children: [], fileCount: 0))26 func dir(_ path: String) -> Dir {27 if path.isEmpty { return root }28 var current = root, sofar = ""29 for part in path.split(separator: "/") {30 sofar = sofar.isEmpty ? String(part) : sofar + "/" + part31 if let d = current.dirs[String(part)] { current = d } else {32 let d = Dir(.init(name: String(part), path: sofar, isDirectory: true, bytes: 0, modifiedAt: nil, children: [], fileCount: 0))33 current.dirs[String(part)] = d; current = d34 }35 }36 return current37 }38 for f in files where !f.path.isEmpty {39 if f.isDirectory { dir(f.path).node.modifiedAt = f.modifiedAt; continue }40 let parent = f.path.lastIndex(of: "/").map { String(f.path[..<$0]) } ?? ""41 dir(parent).files.append(.init(name: f.name, path: f.path, isDirectory: false, bytes: f.bytes, modifiedAt: f.modifiedAt, children: [], fileCount: 1))42 }43 func finish(_ d: Dir) -> VQResearchFileNode {44 let byName: (VQResearchFileNode, VQResearchFileNode) -> Bool = { $0.name.localizedStandardCompare($1.name) == .orderedAscending }45 var n = d.node46 n.children = d.dirs.values.map(finish).sorted(by: byName) + d.files.sorted(by: byName)47 n.bytes = n.children.reduce(0) { $0 + $1.bytes }48 n.fileCount = n.children.reduce(0) { $0 + $1.fileCount }49 return n50 }51 return finish(root).children52 }5354 public enum FileKind: String, Sendable, Hashable { case code, table, text, json, image, pdf, data, log, other }5556 public static func kind(of path: String) -> FileKind {57 let name = (path.split(separator: "/").last.map(String.init) ?? path).lowercased()58 guard let dot = name.lastIndex(of: ".") else { return .other }59 switch String(name[dot...]) {60 case ".py", ".r", ".sql", ".ipynb", ".toml", ".yaml", ".yml", ".tex": return .code61 case ".csv", ".tsv": return .table62 case ".json", ".jsonl": return .json63 case ".log": return .log64 case ".txt", ".md": return .text65 case ".png", ".svg", ".jpg", ".jpeg", ".gif", ".webp": return .image66 case ".pdf": return .pdf67 case ".parquet", ".xlsx", ".xls", ".feather": return .data68 default: return .other69 }70 }71 /// Files the text route returns by lines; the others are raw bytes.72 public static func isTextual(_ path: String) -> Bool { [.code, .table, .json, .log, .text].contains(kind(of: path)) }73 /// Files the dataset inspector can profile.74 public static func isDataset(_ path: String) -> Bool {75 let k = kind(of: path)76 return k == .table || k == .data || (path.hasPrefix("data/") && (k == .json || k == .text))77 }78}7980public enum VQDelimitedText {81 public struct Table: Sendable, Equatable {82 public var header: [String]83 public var rows: [[String]]84 public var delimiter: Character85 /// More rows exist than were read.86 public var truncated: Bool87 }8889 /// RFC 4180-style reader (quotes, doubled quotes, newlines inside quotes). The delimiter is `,` `;` or tab,90 /// guessed from the first line.91 public static func parse(_ text: String, maxRows: Int = 200) -> Table {92 let firstLine = text.prefix { $0 != "\n" && $0 != "\r\n" }93 func count(_ d: Character) -> Int { firstLine.reduce(0) { $0 + ($1 == d ? 1 : 0) } }94 let delimiter: Character = count("\t") > count(",") && count("\t") >= count(";") ? "\t" : (count(";") > count(",") ? ";" : ",")95 var rows: [[String]] = [], row: [String] = [], field = ""96 var quoted = false, truncated = false97 var it = text.makeIterator()98 var pending: Character? = nil99 func endRow() { row.append(field); field = ""; if !(row.count == 1 && row[0].isEmpty) { rows.append(row) }; row = [] }100 while let c = pending ?? it.next() {101 pending = nil102 if quoted {103 if c == "\"" { if let n = it.next() { if n == "\"" { field.append("\"") } else { quoted = false; pending = n } } else { quoted = false } }104 else { field.append(c) }105 } else if c == "\"" && field.isEmpty { quoted = true }106 else if c == delimiter { row.append(field); field = "" }107 else if c == "\n" || c == "\r\n" || c == "\r" {108 endRow()109 if rows.count > maxRows { truncated = true; break }110 } else { field.append(c) }111 }112 if !truncated, !field.isEmpty || !row.isEmpty { endRow() }113 if rows.count > maxRows + 1 { rows = Array(rows.prefix(maxRows + 1)); truncated = true }114 guard let header = rows.first else { return Table(header: [], rows: [], delimiter: delimiter, truncated: false) }115 if rows.count - 1 > maxRows { truncated = true }116 return Table(header: header, rows: Array(rows.dropFirst().prefix(maxRows)), delimiter: delimiter, truncated: truncated)117 }118}119120/// A single-pass Python tokenizer for colouring: keywords, built-ins, strings (triple-quoted and prefixed included),121/// numbers, comments, decorators, definitions. Not a parser: fast, and good enough to read code.122public enum VQPythonSyntax {123 public enum Kind: String, Sendable, Hashable { case plain, keyword, builtin, string, number, comment, decorator, definition, `operator` }124 public struct Token: Sendable, Equatable {125 public var kind: Kind126 /// Range in UTF-16 units of the source (what `NSRange` and `AttributedString` indices are built from).127 public var range: Range<Int>128 }129130 static let keywords: Set<String> = ["False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "finally",131 "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "match", "case"]132 static let builtins: Set<String> = ["print", "len", "range", "int", "float", "str", "list", "dict", "set", "tuple", "bool", "enumerate", "zip", "map", "filter", "sorted", "sum", "min", "max",133 "abs", "round", "open", "isinstance", "type", "any", "all", "repr", "getattr", "setattr", "hasattr", "super", "self", "Exception", "ValueError", "KeyError"]134135 /// Tokens other than `.plain`, in order. Text between them is plain.136 public static func tokens(_ source: String) -> [Token] {137 let u = Array(source.utf16)138 var out: [Token] = []139 var i = 0140 let n = u.count141 func isIdentStart(_ c: UInt16) -> Bool { (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c == 95 || c > 127 }142 func isDigit(_ c: UInt16) -> Bool { c >= 48 && c <= 57 }143 var expectDefinition = false144 while i < n {145 let c = u[i]146 if c == 35 { // # comment147 var j = i; while j < n, u[j] != 10 { j += 1 }148 out.append(Token(kind: .comment, range: i..<j)); i = j; continue149 }150 if c == 34 || c == 39 { // string151 let triple = i + 2 < n && u[i + 1] == c && u[i + 2] == c152 var j = i + (triple ? 3 : 1)153 while j < n {154 if u[j] == 92 { j += 2; continue }155 if triple { if j + 2 < n + 0, u[j] == c, j + 2 <= n - 1, u[j + 1] == c, u[j + 2] == c { j += 3; break } }156 else if u[j] == c { j += 1; break } else if u[j] == 10 { break }157 j += 1158 }159 j = min(j, n)160 out.append(Token(kind: .string, range: i..<j)); i = j; continue161 }162 if c == 64, i == 0 || u[i - 1] == 10 || u[i - 1] == 32 || u[i - 1] == 9 { // @decorator at line start163 var k = i - 1; var lineStart = true164 while k >= 0, u[k] != 10 { if u[k] != 32 && u[k] != 9 { lineStart = false; break }; k -= 1 }165 if lineStart {166 var j = i + 1; while j < n, isIdentStart(u[j]) || isDigit(u[j]) || u[j] == 46 { j += 1 }167 out.append(Token(kind: .decorator, range: i..<j)); i = j; continue168 }169 }170 if isDigit(c) || (c == 46 && i + 1 < n && isDigit(u[i + 1])) { // number171 var j = i + 1172 while j < n, isDigit(u[j]) || u[j] == 46 || u[j] == 95 || (u[j] >= 97 && u[j] <= 122) || (u[j] >= 65 && u[j] <= 90)173 || ((u[j] == 43 || u[j] == 45) && (u[j - 1] == 101 || u[j - 1] == 69)) { j += 1 }174 out.append(Token(kind: .number, range: i..<j)); i = j; continue175 }176 if isIdentStart(c) {177 var j = i + 1; while j < n, isIdentStart(u[j]) || isDigit(u[j]) { j += 1 }178 let word = String(utf16CodeUnits: Array(u[i..<j]), count: j - i)179 // string prefixes: r"", f"", b"", rb""…180 if j < n, (u[j] == 34 || u[j] == 39), word.count <= 2, word.lowercased().allSatisfy({ "rbfu".contains($0) }) { i = j; continue }181 if expectDefinition { out.append(Token(kind: .definition, range: i..<j)); expectDefinition = false }182 else if keywords.contains(word) { out.append(Token(kind: .keyword, range: i..<j)); expectDefinition = word == "def" || word == "class" }183 else if builtins.contains(word), !(i > 0 && u[i - 1] == 46) { out.append(Token(kind: .builtin, range: i..<j)) }184 i = j; continue185 }186 if c != 32 && c != 10 && c != 9 { expectDefinition = false }187 i += 1188 }189 return out190 }191}192