spb/metrika Public

1//2//  DuckDBFastReader.swift3//  Metrika4//5//  Author:  Simon-Pierre Boucher6//  Contact: contact@spboucher.ai7//  Copyright © 2026 Simon-Pierre Boucher. All rights reserved.8//910import Foundation1112// MARK: - DuckDB C API bridge13//14// duckdb-swift's per-element access scans every chunk on every read —15// O(rows × chunks), which turns a 10M-row load into minutes. The C API16// underneath is statically linked into this binary, and its POINTER-BASED17// entry points expose materialized results as contiguous column arrays.18// Only functions taking pointer arguments are declared (never by-value19// duckdb_result), so there is no struct-ABI risk.2021@_silgen_name("duckdb_open")22private func c_open(23    _ path: UnsafePointer<CChar>?, _ out: UnsafeMutablePointer<OpaquePointer?>24) -> Int322526@_silgen_name("duckdb_close")27private func c_close(_ database: UnsafeMutablePointer<OpaquePointer?>)2829@_silgen_name("duckdb_connect")30private func c_connect(31    _ database: OpaquePointer?, _ out: UnsafeMutablePointer<OpaquePointer?>32) -> Int323334@_silgen_name("duckdb_disconnect")35private func c_disconnect(_ connection: UnsafeMutablePointer<OpaquePointer?>)3637@_silgen_name("duckdb_query")38private func c_query(39    _ connection: OpaquePointer?, _ sql: UnsafePointer<CChar>?,40    _ outResult: UnsafeMutableRawPointer?41) -> Int324243@_silgen_name("duckdb_destroy_result")44private func c_destroyResult(_ result: UnsafeMutableRawPointer?)4546@_silgen_name("duckdb_result_error")47private func c_resultError(_ result: UnsafeMutableRawPointer?) -> UnsafePointer<CChar>?4849@_silgen_name("duckdb_column_count")50private func c_columnCount(_ result: UnsafeMutableRawPointer?) -> UInt645152@_silgen_name("duckdb_row_count")53private func c_rowCount(_ result: UnsafeMutableRawPointer?) -> UInt645455@_silgen_name("duckdb_column_name")56private func c_columnName(57    _ result: UnsafeMutableRawPointer?, _ column: UInt6458) -> UnsafePointer<CChar>?5960@_silgen_name("duckdb_column_data")61private func c_columnData(62    _ result: UnsafeMutableRawPointer?, _ column: UInt6463) -> UnsafeMutableRawPointer?6465@_silgen_name("duckdb_nullmask_data")66private func c_nullmaskData(67    _ result: UnsafeMutableRawPointer?, _ column: UInt6468) -> UnsafeMutablePointer<Bool>?6970@_silgen_name("duckdb_value_varchar")71private func c_valueVarchar(72    _ result: UnsafeMutableRawPointer?, _ column: UInt64, _ row: UInt6473) -> UnsafeMutablePointer<CChar>?7475@_silgen_name("duckdb_free")76private func c_free(_ pointer: UnsafeMutableRawPointer?)7778// MARK: - Fast reader7980/// Bulk column extraction through the DuckDB C API. Owns a private81/// in-memory instance (loads read files directly, so no state is shared82/// with the store's duckdb-swift database). The caller pre-casts every83/// column to DOUBLE or VARCHAR in SQL, so extraction is a memcpy for84/// numerics and one call per row for strings.85final class DuckDBFastReader {86    struct ColumnSpec {87        let name: String88        let isNumeric: Bool89    }9091    /// Opaque duckdb_result storage — the struct is ~48 bytes; 512 gives92    /// generous headroom across duckdb versions.93    private static let resultBlobSize = 5129495    private var database: OpaquePointer?96    private var connection: OpaquePointer?9798    init() throws(ZQDataError) {99        guard c_open(nil, &database) == 0 else {100            throw ZQDataError("fast reader: could not open DuckDB")101        }102        guard c_connect(database, &connection) == 0 else {103            var db = database104            c_close(&db)105            throw ZQDataError("fast reader: could not connect")106        }107    }108109    deinit {110        var conn = connection111        c_disconnect(&conn)112        var db = database113        c_close(&db)114    }115116    /// Runs `sql` (whose SELECT list must already match `columns`' order117    /// and DOUBLE/VARCHAR types) and materializes a data frame.118    func read(sql: String, columns: [ColumnSpec]) throws(ZQDataError) -> ZQDataFrame {119        let result = UnsafeMutableRawPointer.allocate(120            byteCount: Self.resultBlobSize, alignment: 16121        )122        result.initializeMemory(as: UInt8.self, repeating: 0, count: Self.resultBlobSize)123        defer {124            c_destroyResult(result)125            result.deallocate()126        }127128        guard c_query(connection, sql, result) == 0 else {129            let message = c_resultError(result).map { String(cString: $0) } ?? "unknown error"130            throw ZQDataError("query failed: \(message)")131        }132133        let rowCount = Int(c_rowCount(result))134        let columnCount = Int(c_columnCount(result))135        guard columnCount == columns.count else {136            throw ZQDataError(137                "fast reader: expected \(columns.count) columns, got \(columnCount)"138            )139        }140141        var frameColumns: [ZQColumn] = []142        frameColumns.reserveCapacity(columnCount)143        for (index, spec) in columns.enumerated() {144            let column = UInt64(index)145            if spec.isNumeric {146                var values = [Double](repeating: .nan, count: rowCount)147                var missing = [Bool](repeating: false, count: rowCount)148                if rowCount > 0 {149                    guard let data = c_columnData(result, column) else {150                        throw ZQDataError("fast reader: no data for '\(spec.name)'")151                    }152                    let doubles = data.assumingMemoryBound(to: Double.self)153                    values.withUnsafeMutableBufferPointer { buffer in154                        buffer.baseAddress!.update(from: doubles, count: rowCount)155                    }156                    if let nulls = c_nullmaskData(result, column) {157                        missing.withUnsafeMutableBufferPointer { buffer in158                            buffer.baseAddress!.update(from: nulls, count: rowCount)159                        }160                    }161                    for i in 0..<rowCount where missing[i] || values[i].isNaN {162                        missing[i] = true163                        values[i] = .nan164                    }165                }166                frameColumns.append(ZQColumn(167                    name: spec.name,168                    data: .float64(values: values, missing: missing)169                ))170            } else {171                var values = [String?](repeating: nil, count: rowCount)172                if rowCount > 0 {173                    let nulls = c_nullmaskData(result, column)174                    for row in 0..<rowCount {175                        if let nulls, nulls[row] { continue }176                        if let text = c_valueVarchar(result, column, UInt64(row)) {177                            values[row] = String(cString: text)178                            c_free(text)179                        }180                    }181                }182                frameColumns.append(ZQColumn(name: spec.name, data: .string(values)))183            }184        }185        return try ZQDataFrame(columns: frameColumns)186    }187}188