SPB Git

spb/metrika Public

Stata-class statistics, GPU-accelerated by Apple Silicon. Native Swift — no Electron, no Python runtime, no compromises.

macosswiftstatisticsgpu
Swift 92.4% HTML 3.3% R 3% Shell 1.3%

feat: bootstrap Metrika v0.1 skeleton with working ZQL vertical slice

- MetrikaKit SPM package: ZQParser, ZQPlanner, ZQEngine, ZQData, ZQStats,
  ZQGPU, ZQGraphics, ZQPlugins (Swift 6, strict concurrency)
- ZQL parser: lexer, Pratt expressions, factor variables, prefix commands,
  column-cited errors with Levenshtein verb suggestions
- ZQData: DuckDB-backed load/save (parquet, csv, json, arrow)
- ZQStats: OLS via LAPACK QR, HC0-HC3 and cluster-robust SE, summarize,
  t/F distributions accurate in the far tails
- ZQGPU: Philox4x32-10 reference RNG, counter-addressable bootstrap
- ZQEngine: session actor with use/save/gen/replace/drop/keep/summarize/
  regress/count/list/graph/bootstrap/set seed/xtset/log
- SwiftUI app (xcodegen): console with history, variables sidebar,
  Swift Charts plots; sandboxed + hardened runtime entitlements
- Tests: 44 green (parser golden, R fixtures at 1e-10, Philox KAT,
  end-to-end engine); Tests/Fixtures/generate.R; Tests/Bench harness
- scripts: check_headers.sh + pre-commit hook, make_icns.sh, release.sh

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
simon-pierre boucher committed 5 days ago (Aug 5, 2026)

Showing 69 changed files with +6,758 and −0

added .gitignore +22 −0
@@ -0,0 +1,22 @@
1 +# macOS
2 +.DS_Store
3 +
4 +# Xcode
5 +build/
6 +DerivedData/
7 +*.xcuserstate
8 +xcuserdata/
9 +*.xcscmblueprint
10 +*.xccheckout
11 +
12 +# SPM
13 +.build/
14 +.swiftpm/
15 +
16 +# Generated artifacts
17 +*.icns
18 +*.dmg
19 +
20 +# Local credentials (never commit)
21 +Metrika-local/
22 +Zyquo-local/
added Assets/AppIcon/metrika-icon.svg +131 −0
@@ -0,0 +1,131 @@
1 +<?xml version="1.0" encoding="UTF-8"?>
2 +<!--
3 + metrika-icon.svg — Metrika
4 + Author: Simon-Pierre Boucher
5 + Contact: contact@spboucher.ai
6 + Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
7 +-->
8 +<svg width="1024" height="1024" viewBox="0 0 1024 1024" xmlns="http://www.w3.org/2000/svg">
9 + <defs>
10 + <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
11 + <stop offset="0%" stop-color="#0B1026"/>
12 + <stop offset="45%" stop-color="#141B3D"/>
13 + <stop offset="100%" stop-color="#1E0B38"/>
14 + </linearGradient>
15 + <radialGradient id="glow" cx="50%" cy="52%" r="55%">
16 + <stop offset="0%" stop-color="#7C5CFF" stop-opacity="0.55"/>
17 + <stop offset="55%" stop-color="#4A3AC8" stop-opacity="0.18"/>
18 + <stop offset="100%" stop-color="#000000" stop-opacity="0"/>
19 + </radialGradient>
20 + <linearGradient id="mstroke" x1="0%" y1="0%" x2="100%" y2="0%">
21 + <stop offset="0%" stop-color="#3EF2D4"/>
22 + <stop offset="50%" stop-color="#7C5CFF"/>
23 + <stop offset="100%" stop-color="#FF4FA3"/>
24 + </linearGradient>
25 + <linearGradient id="mcore" x1="0%" y1="0%" x2="100%" y2="0%">
26 + <stop offset="0%" stop-color="#EAFFFB"/>
27 + <stop offset="50%" stop-color="#E4DBFF"/>
28 + <stop offset="100%" stop-color="#FFE1F0"/>
29 + </linearGradient>
30 + <linearGradient id="area" x1="0%" y1="0%" x2="0%" y2="100%">
31 + <stop offset="0%" stop-color="#7C5CFF" stop-opacity="0.30"/>
32 + <stop offset="100%" stop-color="#7C5CFF" stop-opacity="0"/>
33 + </linearGradient>
34 + <filter id="blurBig" x="-40%" y="-40%" width="180%" height="180%">
35 + <feGaussianBlur stdDeviation="26"/>
36 + </filter>
37 + <filter id="blurSoft" x="-40%" y="-40%" width="180%" height="180%">
38 + <feGaussianBlur stdDeviation="7"/>
39 + </filter>
40 + <linearGradient id="sheen" x1="0%" y1="0%" x2="0%" y2="100%">
41 + <stop offset="0%" stop-color="#FFFFFF" stop-opacity="0.16"/>
42 + <stop offset="35%" stop-color="#FFFFFF" stop-opacity="0.04"/>
43 + <stop offset="60%" stop-color="#FFFFFF" stop-opacity="0"/>
44 + </linearGradient>
45 + <clipPath id="squircle">
46 + <rect x="64" y="64" width="896" height="896" rx="200"/>
47 + </clipPath>
48 + </defs>
49 +
50 + <rect x="64" y="64" width="896" height="896" rx="200" fill="url(#bg)"/>
51 +
52 + <g clip-path="url(#squircle)">
53 + <!-- Plot grid -->
54 + <g stroke="#8FA3FF" stroke-opacity="0.07" stroke-width="2">
55 + <line x1="64" y1="288" x2="960" y2="288"/>
56 + <line x1="64" y1="464" x2="960" y2="464"/>
57 + <line x1="64" y1="640" x2="960" y2="640"/>
58 + <line x1="64" y1="816" x2="960" y2="816"/>
59 + <line x1="256" y1="64" x2="256" y2="960"/>
60 + <line x1="432" y1="64" x2="432" y2="960"/>
61 + <line x1="608" y1="64" x2="608" y2="960"/>
62 + <line x1="784" y1="64" x2="784" y2="960"/>
63 + </g>
64 +
65 + <rect x="64" y="64" width="896" height="896" fill="url(#glow)"/>
66 +
67 + <!-- Area fill under the M trace (chart vibe) -->
68 + <path d="M 220 800 L 220 300 L 420 640 L 512 380 L 604 640 L 804 300 L 804 800 Z"
69 + fill="url(#area)"/>
70 +
71 + <!-- Scatter cloud around the trace -->
72 + <g fill="#9DB4FF" fill-opacity="0.55">
73 + <circle cx="286" cy="418" r="9"/>
74 + <circle cx="330" cy="512" r="7"/>
75 + <circle cx="378" cy="566" r="10"/>
76 + <circle cx="452" cy="536" r="8"/>
77 + <circle cx="486" cy="446" r="7"/>
78 + <circle cx="542" cy="452" r="9"/>
79 + <circle cx="580" cy="560" r="8"/>
80 + <circle cx="646" cy="576" r="10"/>
81 + <circle cx="700" cy="470" r="7"/>
82 + <circle cx="748" cy="392" r="9"/>
83 + <circle cx="264" cy="640" r="7"/>
84 + <circle cx="760" cy="640" r="8"/>
85 + </g>
86 + <g fill="#3EF2D4" fill-opacity="0.85">
87 + <circle cx="252" cy="356" r="6"/>
88 + <circle cx="470" cy="410" r="6"/>
89 + <circle cx="676" cy="520" r="6"/>
90 + </g>
91 + <g fill="#FF4FA3" fill-opacity="0.8">
92 + <circle cx="356" cy="600" r="6"/>
93 + <circle cx="556" cy="500" r="7"/>
94 + <circle cx="772" cy="340" r="6"/>
95 + </g>
96 +
97 + <!-- The M : time-series trace -->
98 + <path d="M 220 800 L 220 300 L 420 640 L 512 380 L 604 640 L 804 300 L 804 800"
99 + fill="none" stroke="url(#mstroke)" stroke-width="96"
100 + stroke-linecap="round" stroke-linejoin="round"
101 + filter="url(#blurBig)" opacity="0.75"/>
102 + <path d="M 220 800 L 220 300 L 420 640 L 512 380 L 604 640 L 804 300 L 804 800"
103 + fill="none" stroke="url(#mstroke)" stroke-width="74"
104 + stroke-linecap="round" stroke-linejoin="round"/>
105 + <path d="M 220 800 L 220 300 L 420 640 L 512 380 L 604 640 L 804 300 L 804 800"
106 + fill="none" stroke="url(#mcore)" stroke-width="20"
107 + stroke-linecap="round" stroke-linejoin="round"
108 + filter="url(#blurSoft)" opacity="0.9"/>
109 +
110 + <!-- Data-point nodes at the vertices of the trace -->
111 + <g fill="#FFFFFF">
112 + <circle cx="220" cy="300" r="16"/>
113 + <circle cx="420" cy="640" r="16"/>
114 + <circle cx="512" cy="380" r="16"/>
115 + <circle cx="604" cy="640" r="16"/>
116 + <circle cx="804" cy="300" r="16"/>
117 + </g>
118 + <g fill="none" stroke="#FFFFFF" stroke-opacity="0.35" stroke-width="6">
119 + <circle cx="220" cy="300" r="30"/>
120 + <circle cx="420" cy="640" r="30"/>
121 + <circle cx="512" cy="380" r="30"/>
122 + <circle cx="604" cy="640" r="30"/>
123 + <circle cx="804" cy="300" r="30"/>
124 + </g>
125 +
126 + <rect x="64" y="64" width="896" height="520" fill="url(#sheen)"/>
127 + </g>
128 +
129 + <rect x="66" y="66" width="892" height="892" rx="198" fill="none"
130 + stroke="#FFFFFF" stroke-opacity="0.10" stroke-width="3"/>
131 +</svg>
added CLAUDE.md +374 −0
@@ -0,0 +1,374 @@
1 +# CLAUDE.md — Metrika
2 +
3 +> **Metrika** — A next-generation statistical & data science application for macOS.
4 +> Stata-class command syntax, GPU-accelerated by Apple Silicon (MLX/Metal), powered by DuckDB.
5 +> Native Swift + SwiftUI. No Electron. No Python runtime dependency. No compromises.
6 +
7 +---
8 +
9 +## 0. Non-Negotiable Project Rules
10 +
11 +1. **Every source file MUST begin with this header** (adapt comment style to the language):
12 +
13 +```swift
14 +//
15 +// <FileName>.swift
16 +// Metrika
17 +//
18 +// Author: Simon-Pierre Boucher
19 +// Contact: contact@spboucher.ai
20 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
21 +//
22 +```
23 +
24 +For Metal shaders (`.metal`), shell scripts, and Python tooling scripts, use the equivalent comment syntax (`//` or `#`). **No file is committed without this header.** A pre-commit hook (see §12) enforces it.
25 +
26 +2. Language: **Swift 6**, strict concurrency enabled (`-strict-concurrency=complete`).
27 +3. UI: **SwiftUI first**, AppKit bridges (`NSViewRepresentable`) only where SwiftUI is insufficient (text editor, data grid virtualization).
28 +4. Minimum target: **macOS 14 (Sonoma)**, Apple Silicon only (`arm64`). No Intel builds.
29 +5. All numerical results must match reference implementations (R / Stata) to at least 1e-10 relative tolerance for CPU paths; GPU paths documented tolerance 1e-6 (float32) or 1e-10 (float64 where MLX supports it).
30 +6. English for all code, comments, commit messages, and docs.
31 +
32 +---
33 +
34 +## 1. Product Vision
35 +
36 +Metrika is a complete data science environment:
37 +
38 +- **Command-driven**, like Stata: `reg y x1 x2, robust cluster(id)` — one line, publication-ready output.
39 +- **Scriptable**: `.zyq` do-files with full reproducibility (logs, seeds, versioned datasets).
40 +- **GPU-native**: massively parallel workloads (bootstrap, Monte Carlo, MCMC, ML training, permutation tests) run on Apple GPU via MLX with unified memory.
41 +- **Extensible**: user-defined commands ("home-made" commands) via a plugin protocol — drop a Swift package or a `.zyq` ado-style script into `~/Library/Application Support/Metrika/Commands/`.
42 +- **Big-data capable**: DuckDB engine → out-of-core, columnar, multi-threaded. 100M+ rows on a laptop.
43 +
44 +### Feature pillars (v1 scope)
45 +
46 +| Pillar | Contents |
47 +|---|---|
48 +| Data management | import/export CSV, Parquet, Arrow, Excel, Stata `.dta`, JSON; merge, append, reshape, collapse, egen-style generators |
49 +| Estimation | summarize, tabulate, correlate, `regress` (OLS), `logit`, `probit`, `poisson`, `ivregress` (2SLS), `xtreg` (FE/RE), `areg`, quantile regression |
50 +| Inference | robust (HC0–HC3), cluster-robust SE, **GPU bootstrap** (pairs, wild, cluster), permutation tests, jackknife |
51 +| Simulation | Monte Carlo engine, random number generation (Philox counter-based, reproducible on GPU) |
52 +| ML | gradient boosting, elastic net, k-means, PCA — MLX-backed |
53 +| Graphics | scatter, line, histogram, kdensity, coefplot, binscatter — Swift Charts + custom Metal renderer for >1M points |
54 +| Reproducibility | do-file editor, logging (`log using`), `set seed` honored across CPU & GPU |
55 +
56 +---
57 +
58 +## 2. App Name, Bundle & Identity
59 +
60 +- **App name:** Metrika
61 +- **Bundle ID:** `ai.spboucher.metrika`
62 +- **Version scheme:** SemVer (`MARKETING_VERSION`), build number = CI run number.
63 +- **Icon:** `Assets/AppIcon/metrika-icon.svg` is the master (see §11). Rasterize to all required sizes via `scripts/make_icns.sh`.
64 +
65 +---
66 +
67 +## 3. Architecture Overview
68 +
69 +```
70 +┌─────────────────────────────────────────────────────────┐
71 +│ MetrikaApp (SwiftUI) │
72 +│ CommandConsole · DoFileEditor · DataBrowser · Plots │
73 +└──────────────────────────┬──────────────────────────────┘
74 + │
75 +┌──────────────────────────▼──────────────────────────────┐
76 +│ MetrikaKit (SPM package) │
77 +│ │
78 +│ ZQParser command grammar → AST │
79 +│ ZQPlanner AST → ExecutionPlan (CPU/GPU/hybrid) │
80 +│ ZQEngine orchestration, sessions, logging │
81 +│ ZQData DataFrame façade over DuckDB/Arrow │
82 +│ ZQStats CPU estimators (Accelerate/LAPACK) │
83 +│ ZQGPU MLX batched estimators, RNG, kernels │
84 +│ ZQGraphics plot spec → Swift Charts / Metal │
85 +│ ZQPlugins user command discovery & sandboxing │
86 +└─────────────────────────────────────────────────────────┘
87 +```
88 +
89 +### Module rules
90 +- `MetrikaKit` is a **Swift Package** with zero UI dependencies → fully testable via `swift test`.
91 +- The app target only imports `MetrikaKit` and renders.
92 +- `ZQGPU` is the **only** module allowed to import MLX/Metal. `ZQStats` is the only one importing Accelerate. This keeps backends swappable.
93 +
94 +### Dependencies (SPM)
95 +```
96 +duckdb/duckdb-swift — data engine
97 +ml-explore/mlx-swift — GPU tensor compute
98 +apple/swift-collections — deques, ordered dicts
99 +pointfreeco/swift-parsing — command grammar
100 +apple/swift-argument-parser — metrika-cli companion tool
101 +```
102 +
103 +---
104 +
105 +## 4. The Command Language (ZQL)
106 +
107 +Stata-compatible mental model, cleaner grammar:
108 +
109 +```
110 +command [varlist] [if expr] [in range] [weight] [, options]
111 +```
112 +
113 +Examples that MUST work in v1:
114 +
115 +```
116 +use sales.parquet
117 +summarize revenue price, detail
118 +gen log_rev = ln(revenue)
119 +reg log_rev price i.region, robust
120 +bootstrap, reps(100000) seed(42): reg log_rev price
121 +xtset firm_id year
122 +xtreg log_rev price, fe cluster(firm_id)
123 +graph scatter log_rev price, by(region)
124 +```
125 +
126 +### Parser (`ZQParser`)
127 +- Built with `swift-parsing`; grammar defined in `Sources/ZQParser/Grammar/`.
128 +- Produces a typed AST: `ZQCommand { verb, varlist, condition, range, weight, options }`.
129 +- Factor variables (`i.region`, `c.age#c.age`) expand at plan time, not parse time.
130 +- Error messages must cite column position and suggest fixes (Levenshtein on known verbs).
131 +
132 +### Home-made commands
133 +Two tiers:
134 +1. **Script commands** (`.zyq` files): sequences of ZQL with `args`/`syntax` declarations — the Stata ado-file analog.
135 +2. **Native plugins**: Swift packages conforming to:
136 +
137 +```swift
138 +public protocol ZQCommandPlugin: Sendable {
139 + static var verb: String { get }
140 + static var syntax: ZQSyntaxSpec { get }
141 + func execute(_ ctx: ZQContext) async throws -> ZQResult
142 +}
143 +```
144 +
145 +Discovery path: `~/Library/Application Support/Metrika/Commands/`. Plugins run in-process but receive a **read-only DataFrame view** unless they declare `mutates: true`.
146 +
147 +---
148 +
149 +## 5. Execution Planner — CPU vs GPU Dispatch
150 +
151 +The planner decides the backend **automatically**. The user never chooses.
152 +
153 +```swift
154 +enum Backend { case cpu, gpu, hybrid }
155 +
156 +func plan(_ cmd: ZQCommand, data: ZQDataFrame) -> ExecutionPlan {
157 + // Heuristics, tuned by benchmarks in Tests/Bench:
158 + // 1. Single estimation, n < 5_000_000 → .cpu (LAPACK QR, dqrls-style)
159 + // 2. reps ≥ 500 independent replicates → .gpu (batched solve)
160 + // 3. MCMC / iterative + large n → .hybrid (GPU likelihood, CPU control)
161 +}
162 +```
163 +
164 +### CPU path (`ZQStats`)
165 +- OLS: QR via LAPACK `dgeqrf`/`dormqr` (never form X'X for conditioning reasons).
166 +- Robust/cluster SE: sandwich estimators, computed with BLAS level-3.
167 +- MLE (logit/probit/poisson): Newton–Raphson with analytic Hessians; IRLS fallback.
168 +- All CPU compute in **Float64**.
169 +
170 +### GPU path (`ZQGPU`, MLX)
171 +- **Batched OLS**: resample indices as `(reps, n)` int tensor → gather → batched Cholesky/QR solve on `(reps, k, k)` — thousands of regressions per GPU pass.
172 +- **RNG**: Philox4x32 counter-based generator → identical streams CPU/GPU given same seed; `set seed` maps to Philox key. Reproducibility is a release blocker.
173 +- **Cluster bootstrap** (ragged n per replicate): pad-to-max with weight masks; custom Metal kernel only if profiling shows >2× overhead from padding.
174 +- Precision: MLX float32 by default; accumulate cross-products in float64 emulation (Kahan/two-sum) where it matters; document tolerances per command.
175 +- **Chunk reps** to respect memory: `chunk = min(reps, budget / (n*k*4))`, stream results, report progress to UI via `AsyncStream`.
176 +
177 +### Hybrid
178 +- Example: Bayesian regression — GPU computes log-likelihood over chains in parallel, CPU runs the sampler control loop.
179 +
180 +---
181 +
182 +## 6. Data Layer (`ZQData`)
183 +
184 +- Canonical store: **DuckDB** in-process database; DataFrames are Arrow views over query results.
185 +- Zero-copy handoff Arrow → MLX tensors where dtype allows; otherwise single cast pass.
186 +- `use file.parquet` → registers a DuckDB view, lazy. `gen`/`replace` compile to SQL expressions when possible; fall back to vectorized Swift over Arrow buffers.
187 +- Missing values: Arrow validity bitmaps end-to-end; every estimator does listwise deletion by default with an explicit report line ("(12 observations dropped due to missing values)").
188 +- `.dta` reader/writer implemented natively in `ZQData/Stata/` (spec v117–v121).
189 +
190 +---
191 +
192 +## 7. UI (SwiftUI)
193 +
194 +Windows/panes:
195 +1. **Console** — command input with history (↑/↓), autocomplete on verbs & varnames, ANSI-style rich results (monospaced, aligned tables via `AttributedString`).
196 +2. **Do-file editor** — `NSTextView` bridge, syntax highlighting for ZQL, ⌘R runs selection.
197 +3. **Data browser** — virtualized grid (custom `NSTableView` bridge; SwiftUI `Table` is too slow past ~100k rows), sortable, filter bar compiles to `if` expressions.
198 +4. **Plots** — Swift Charts for standard plots; custom `MTKView` renderer for scatter >1M points (instanced point sprites).
199 +5. **Sidebar** — variables list with types, labels, missing counts.
200 +
201 +State: `@Observable` session model; every command mutation goes through `ZQEngine` so console, do-files, and plugins share one execution path. Undo = dataset snapshots via DuckDB `CHECKPOINT` + copy-on-write.
202 +
203 +---
204 +
205 +## 8. Performance Budgets (release blockers)
206 +
207 +| Operation | Data | Budget (M3 Pro) |
208 +|---|---|---|
209 +| `use` parquet | 10M rows × 20 cols | < 1.5 s |
210 +| `summarize` all vars | 10M × 20 | < 300 ms |
211 +| `reg` 5 covariates | 10M rows | < 900 ms |
212 +| `bootstrap, reps(10000): reg` | 100k rows | < 3 s |
213 +| `bootstrap, reps(100000): reg` | 100k rows | < 25 s |
214 +| Scatter render | 2M points | 60 fps pan/zoom |
215 +
216 +Benchmarks live in `Tests/Bench/` and run in CI on self-hosted Apple Silicon runner; regressions >10% fail the build.
217 +
218 +---
219 +
220 +## 9. Testing
221 +
222 +- `swift test` on MetrikaKit: parser golden tests, estimator numerical tests vs fixtures generated by R (`Tests/Fixtures/generate.R`).
223 +- Property tests: OLS invariance (scaling, permutation), bootstrap CI coverage simulation (n small, known DGP).
224 +- GPU vs CPU cross-checks: same seed ⇒ identical resample indices; estimates within documented tolerance.
225 +- UI: XCUITest smoke — launch, load sample dataset, run `reg`, assert output table.
226 +
227 +---
228 +
229 +## 10. Build, Signing, Notarization & DMG
230 +
231 +> ⚠️ **PLACEHOLDERS** — the `Zyquo-local/` folder containing the real notarization credentials was not available when this file was generated. Fill every `<...>` below from `Zyquo-local/notarization.md` (or equivalent) before the first release build. Do **not** commit real credentials; keep them in the local keychain / CI secrets.
232 +
233 +### Identifiers
234 +```
235 +DEVELOPMENT_TEAM = <TEAM_ID> # from Metrika-local
236 +CODE_SIGN_IDENTITY = "Developer ID Application: Simon-Pierre Boucher (<TEAM_ID>)"
237 +NOTARY_PROFILE = "<KEYCHAIN_PROFILE_NAME>" # created via: xcrun notarytool store-credentials
238 +BUNDLE_ID = ai.spboucher.metrika
239 +```
240 +
241 +### One-time setup
242 +```bash
243 +xcrun notarytool store-credentials "<KEYCHAIN_PROFILE_NAME>" \
244 + --apple-id "<APPLE_ID_EMAIL>" \
245 + --team-id "<TEAM_ID>" \
246 + --password "<APP_SPECIFIC_PASSWORD>"
247 +```
248 +
249 +### Release pipeline — `scripts/release.sh`
250 +```bash
251 +#!/bin/bash
252 +#
253 +# release.sh — Metrika
254 +# Author: Simon-Pierre Boucher
255 +# Contact: contact@spboucher.ai
256 +#
257 +set -euo pipefail
258 +
259 +APP="Metrika"
260 +SCHEME="Metrika"
261 +BUILD_DIR="build"
262 +DMG="${APP}.dmg"
263 +
264 +# 1. Archive
265 +xcodebuild -scheme "$SCHEME" -configuration Release -arch arm64 \
266 + -archivePath "$BUILD_DIR/$APP.xcarchive" archive
267 +
268 +# 2. Export with Developer ID
269 +xcodebuild -exportArchive \
270 + -archivePath "$BUILD_DIR/$APP.xcarchive" \
271 + -exportOptionsPlist scripts/ExportOptions.plist \
272 + -exportPath "$BUILD_DIR/export"
273 +
274 +# 3. Verify hardened runtime + entitlements
275 +codesign -dv --verbose=4 "$BUILD_DIR/export/$APP.app"
276 +codesign --verify --deep --strict "$BUILD_DIR/export/$APP.app"
277 +
278 +# 4. Build DMG (create-dmg, background art in Assets/DMG/)
279 +create-dmg \
280 + --volname "$APP" \
281 + --window-size 540 380 \
282 + --icon-size 128 \
283 + --icon "$APP.app" 130 190 \
284 + --app-drop-link 400 190 \
285 + --background "Assets/DMG/background.png" \
286 + "$DMG" "$BUILD_DIR/export/"
287 +
288 +# 5. Sign the DMG itself
289 +codesign --sign "Developer ID Application: Simon-Pierre Boucher (<TEAM_ID>)" "$DMG"
290 +
291 +# 6. Notarize & wait
292 +xcrun notarytool submit "$DMG" \
293 + --keychain-profile "<KEYCHAIN_PROFILE_NAME>" \
294 + --wait
295 +
296 +# 7. Staple
297 +xcrun stapler staple "$DMG"
298 +xcrun stapler validate "$DMG"
299 +
300 +echo "✅ $DMG notarized and stapled."
301 +```
302 +
303 +### `scripts/ExportOptions.plist`
304 +```xml
305 +<?xml version="1.0" encoding="UTF-8"?>
306 +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
307 + "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
308 +<plist version="1.0"><dict>
309 + <key>method</key><string>developer-id</string>
310 + <key>teamID</key><string><TEAM_ID></string>
311 + <key>signingStyle</key><string>automatic</string>
312 + <key>destination</key><string>export</string>
313 +</dict></plist>
314 +```
315 +
316 +### Entitlements (`Metrika.entitlements`)
317 +- Hardened Runtime: ON.
318 +- App Sandbox: ON, with `com.apple.security.files.user-selected.read-write` (open/save datasets) and bookmark entitlements for recent files.
319 +- No JIT, no unsigned executable memory (MLX does not require them).
320 +- If plugin loading of external dylibs is added later, revisit `com.apple.security.cs.disable-library-validation` — avoid if possible; prefer SPM-compiled-in plugins.
321 +
322 +---
323 +
324 +## 11. App Icon
325 +
326 +- Master: `Assets/AppIcon/metrika-icon.svg` (1024×1024 design).
327 +- Concept: a glowing **M-shaped time-series line** — a zigzag chart trace cutting through a scatter of data points on a deep space-gradient squircle — econometrics meets GPU power.
328 +- Generate `.icns`: `scripts/make_icns.sh` rasterizes the SVG (via `rsvg-convert` or `qlmanage`) to 16→1024 px and runs `iconutil -c icns`.
329 +- macOS Sonoma+ icon grid: content within the 824×824 safe zone of the 1024 canvas; system applies the squircle mask — but we bake our own rounded-rect background per Big Sur+ convention.
330 +
331 +---
332 +
333 +## 12. Repo Conventions
334 +
335 +```
336 +Metrika/
337 +├── CLAUDE.md
338 +├── Metrika.xcodeproj
339 +├── Metrika/ # app target (UI)
340 +├── MetrikaKit/ # SPM package (all logic)
341 +│ └── Sources/{ZQParser,ZQPlanner,ZQEngine,ZQData,ZQStats,ZQGPU,ZQGraphics,ZQPlugins}
342 +├── Assets/{AppIcon,DMG}
343 +├── scripts/{release.sh,make_icns.sh,check_headers.sh}
344 +└── Tests/{Unit,Bench,Fixtures}
345 +```
346 +
347 +- **Pre-commit hook** runs `scripts/check_headers.sh`: rejects any staged source file (`.swift`, `.metal`, `.sh`, `.py`, `.zyq`) missing the `Author: Simon-Pierre Boucher` / `Contact: contact@spboucher.ai` header.
348 +- Commits: Conventional Commits (`feat(parser): factor variable expansion`).
349 +- Branches: `main` (protected), `dev`, feature branches `feat/*`.
350 +- CI: GitHub Actions on self-hosted arm64 macOS runner — build, test, bench, header check.
351 +
352 +---
353 +
354 +## 13. Roadmap
355 +
356 +**v0.1 (MVP, ~8 weeks of focused work)**
357 +use/save (parquet, csv, dta) · summarize · tabulate · gen/replace · reg (robust, cluster) · logit · basic scatter/histogram · console + data browser · do-file execution.
358 +
359 +**v0.2** — GPU bootstrap + permutation tests · xtreg FE · ivregress · Swift Charts full suite · plugin protocol.
360 +
361 +**v0.3** — MCMC/Bayesian module · gradient boosting & elastic net (MLX) · Metal 2M-point renderer · margins/predict.
362 +
363 +**v1.0** — notarized DMG release, docs site, sample datasets, command reference (`help regress` in-app).
364 +
365 +---
366 +
367 +## 14. What Claude Should Do When Working in This Repo
368 +
369 +1. Read this file first; respect §0 rules absolutely (headers, Swift 6 strict concurrency).
370 +2. Before touching numerics, read the matching fixture in `Tests/Fixtures/` and keep tests green.
371 +3. Any new command: grammar entry → AST case → planner rule → CPU impl → (optional) GPU impl → golden test → in-app help entry. All six or the PR is incomplete.
372 +4. Never introduce a Python or Node runtime dependency into the app bundle.
373 +5. When unsure between CPU and GPU, implement CPU first — correctness before speed — then add the GPU path behind the planner.
374 +6. Update the performance table (§8) whenever a benchmark materially changes.
added Metrika.xcodeproj/project.pbxproj +383 −0
@@ -0,0 +1,383 @@
1 +// !$*UTF8*$!
2 +{
3 + archiveVersion = 1;
4 + classes = {
5 + };
6 + objectVersion = 77;
7 + objects = {
8 +
9 +/* Begin PBXBuildFile section */
10 + 0F7000A49C9879593F09A0D3 /* MetrikaKit in Frameworks */ = {isa = PBXBuildFile; productRef = 498C4115F45BADDF16191A57 /* MetrikaKit */; };
11 + 3084AE2A328ED6746D01F46E /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 0404FDEAF8A57414EFB3C649 /* Assets.xcassets */; };
12 + 7E936A23CCC90823D6B08BCC /* SessionModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0453FB0F256E853D2C2F481C /* SessionModel.swift */; };
13 + 8B0CDB213D64E3EE87593808 /* ConsoleView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 159695CF89D6BB25DF883F07 /* ConsoleView.swift */; };
14 + 9BE65709781719FAE0CA740A /* MetrikaApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9C5393DC050C555E4FB69FC0 /* MetrikaApp.swift */; };
15 + DDCC2DC487AF9FAC7D6B8E3E /* PlotView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AD551AA707CD1853C3D6D21 /* PlotView.swift */; };
16 +/* End PBXBuildFile section */
17 +
18 +/* Begin PBXFileReference section */
19 + 0404FDEAF8A57414EFB3C649 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
20 + 0453FB0F256E853D2C2F481C /* SessionModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SessionModel.swift; sourceTree = "<group>"; };
21 + 159695CF89D6BB25DF883F07 /* ConsoleView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConsoleView.swift; sourceTree = "<group>"; };
22 + 24D84DAB354D702D8575D587 /* MetrikaKit */ = {isa = PBXFileReference; lastKnownFileType = folder; name = MetrikaKit; path = MetrikaKit; sourceTree = SOURCE_ROOT; };
23 + 4AD551AA707CD1853C3D6D21 /* PlotView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlotView.swift; sourceTree = "<group>"; };
24 + 5A0213CC9FE05A339DEEFA28 /* Metrika.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Metrika.app; sourceTree = BUILT_PRODUCTS_DIR; };
25 + 6E94C5817E96C83B888FB579 /* Metrika.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Metrika.entitlements; sourceTree = "<group>"; };
26 + 9C5393DC050C555E4FB69FC0 /* MetrikaApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MetrikaApp.swift; sourceTree = "<group>"; };
27 +/* End PBXFileReference section */
28 +
29 +/* Begin PBXFrameworksBuildPhase section */
30 + 168283EE9449D9A4324BFE40 /* Frameworks */ = {
31 + isa = PBXFrameworksBuildPhase;
32 + buildActionMask = 2147483647;
33 + files = (
34 + 0F7000A49C9879593F09A0D3 /* MetrikaKit in Frameworks */,
35 + );
36 + runOnlyForDeploymentPostprocessing = 0;
37 + };
38 +/* End PBXFrameworksBuildPhase section */
39 +
40 +/* Begin PBXGroup section */
41 + 133558B8D49331BB5120F8F8 /* Products */ = {
42 + isa = PBXGroup;
43 + children = (
44 + 5A0213CC9FE05A339DEEFA28 /* Metrika.app */,
45 + );
46 + name = Products;
47 + sourceTree = "<group>";
48 + };
49 + 58DB3AB5F820BD1E6C82E1CB /* Sources */ = {
50 + isa = PBXGroup;
51 + children = (
52 + 159695CF89D6BB25DF883F07 /* ConsoleView.swift */,
53 + 9C5393DC050C555E4FB69FC0 /* MetrikaApp.swift */,
54 + 4AD551AA707CD1853C3D6D21 /* PlotView.swift */,
55 + 0453FB0F256E853D2C2F481C /* SessionModel.swift */,
56 + );
57 + path = Sources;
58 + sourceTree = "<group>";
59 + };
60 + 63F77F28B4EF42E0407BEC4F /* Packages */ = {
61 + isa = PBXGroup;
62 + children = (
63 + 24D84DAB354D702D8575D587 /* MetrikaKit */,
64 + );
65 + name = Packages;
66 + sourceTree = "<group>";
67 + };
68 + A905F8EA05CC983967935417 /* Metrika */ = {
69 + isa = PBXGroup;
70 + children = (
71 + 0404FDEAF8A57414EFB3C649 /* Assets.xcassets */,
72 + 6E94C5817E96C83B888FB579 /* Metrika.entitlements */,
73 + 58DB3AB5F820BD1E6C82E1CB /* Sources */,
74 + );
75 + path = Metrika;
76 + sourceTree = "<group>";
77 + };
78 + DC657A037F59093C5529DD71 = {
79 + isa = PBXGroup;
80 + children = (
81 + A905F8EA05CC983967935417 /* Metrika */,
82 + 63F77F28B4EF42E0407BEC4F /* Packages */,
83 + 133558B8D49331BB5120F8F8 /* Products */,
84 + );
85 + sourceTree = "<group>";
86 + };
87 +/* End PBXGroup section */
88 +
89 +/* Begin PBXNativeTarget section */
90 + 5E03B33ABE9DE3162C326F87 /* Metrika */ = {
91 + isa = PBXNativeTarget;
92 + buildConfigurationList = A087F0DEACCF786E9B7CC9A5 /* Build configuration list for PBXNativeTarget "Metrika" */;
93 + buildPhases = (
94 + C8918D5868AE76549F9247E4 /* Sources */,
95 + D09C5D18E3FF44946887A6C9 /* Resources */,
96 + 168283EE9449D9A4324BFE40 /* Frameworks */,
97 + );
98 + buildRules = (
99 + );
100 + dependencies = (
101 + );
102 + name = Metrika;
103 + packageProductDependencies = (
104 + 498C4115F45BADDF16191A57 /* MetrikaKit */,
105 + );
106 + productName = Metrika;
107 + productReference = 5A0213CC9FE05A339DEEFA28 /* Metrika.app */;
108 + productType = "com.apple.product-type.application";
109 + };
110 +/* End PBXNativeTarget section */
111 +
112 +/* Begin PBXProject section */
113 + 8571D0059EB0A97DE47F96F7 /* Project object */ = {
114 + isa = PBXProject;
115 + attributes = {
116 + BuildIndependentTargetsInParallel = YES;
117 + LastUpgradeCheck = 1430;
118 + TargetAttributes = {
119 + 5E03B33ABE9DE3162C326F87 = {
120 + ProvisioningStyle = Automatic;
121 + };
122 + };
123 + };
124 + buildConfigurationList = 02D661894CE6081B5C88E20C /* Build configuration list for PBXProject "Metrika" */;
125 + developmentRegion = en;
126 + hasScannedForEncodings = 0;
127 + knownRegions = (
128 + Base,
129 + en,
130 + );
131 + mainGroup = DC657A037F59093C5529DD71;
132 + minimizedProjectReferenceProxies = 1;
133 + packageReferences = (
134 + D626AF14268CCEC6F4756ABC /* XCLocalSwiftPackageReference "MetrikaKit" */,
135 + );
136 + preferredProjectObjectVersion = 77;
137 + productRefGroup = 133558B8D49331BB5120F8F8 /* Products */;
138 + projectDirPath = "";
139 + projectRoot = "";
140 + targets = (
141 + 5E03B33ABE9DE3162C326F87 /* Metrika */,
142 + );
143 + };
144 +/* End PBXProject section */
145 +
146 +/* Begin PBXResourcesBuildPhase section */
147 + D09C5D18E3FF44946887A6C9 /* Resources */ = {
148 + isa = PBXResourcesBuildPhase;
149 + buildActionMask = 2147483647;
150 + files = (
151 + 3084AE2A328ED6746D01F46E /* Assets.xcassets in Resources */,
152 + );
153 + runOnlyForDeploymentPostprocessing = 0;
154 + };
155 +/* End PBXResourcesBuildPhase section */
156 +
157 +/* Begin PBXSourcesBuildPhase section */
158 + C8918D5868AE76549F9247E4 /* Sources */ = {
159 + isa = PBXSourcesBuildPhase;
160 + buildActionMask = 2147483647;
161 + files = (
162 + 8B0CDB213D64E3EE87593808 /* ConsoleView.swift in Sources */,
163 + 9BE65709781719FAE0CA740A /* MetrikaApp.swift in Sources */,
164 + DDCC2DC487AF9FAC7D6B8E3E /* PlotView.swift in Sources */,
165 + 7E936A23CCC90823D6B08BCC /* SessionModel.swift in Sources */,
166 + );
167 + runOnlyForDeploymentPostprocessing = 0;
168 + };
169 +/* End PBXSourcesBuildPhase section */
170 +
171 +/* Begin XCBuildConfiguration section */
172 + 3CB58B068179C9DB2CD1CAC1 /* Release */ = {
173 + isa = XCBuildConfiguration;
174 + buildSettings = {
175 + ALWAYS_SEARCH_USER_PATHS = NO;
176 + CLANG_ANALYZER_NONNULL = YES;
177 + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
178 + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
179 + CLANG_CXX_LIBRARY = "libc++";
180 + CLANG_ENABLE_MODULES = YES;
181 + CLANG_ENABLE_OBJC_ARC = YES;
182 + CLANG_ENABLE_OBJC_WEAK = YES;
183 + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
184 + CLANG_WARN_BOOL_CONVERSION = YES;
185 + CLANG_WARN_COMMA = YES;
186 + CLANG_WARN_CONSTANT_CONVERSION = YES;
187 + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
188 + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
189 + CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
190 + CLANG_WARN_EMPTY_BODY = YES;
191 + CLANG_WARN_ENUM_CONVERSION = YES;
192 + CLANG_WARN_INFINITE_RECURSION = YES;
193 + CLANG_WARN_INT_CONVERSION = YES;
194 + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
195 + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
196 + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
197 + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
198 + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
199 + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
200 + CLANG_WARN_STRICT_PROTOTYPES = YES;
201 + CLANG_WARN_SUSPICIOUS_MOVE = YES;
202 + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
203 + CLANG_WARN_UNREACHABLE_CODE = YES;
204 + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
205 + COPY_PHASE_STRIP = NO;
206 + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
207 + ENABLE_NS_ASSERTIONS = NO;
208 + ENABLE_STRICT_OBJC_MSGSEND = YES;
209 + GCC_C_LANGUAGE_STANDARD = gnu11;
210 + GCC_NO_COMMON_BLOCKS = YES;
211 + GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
212 + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
213 + GCC_WARN_UNDECLARED_SELECTOR = YES;
214 + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
215 + GCC_WARN_UNUSED_FUNCTION = YES;
216 + GCC_WARN_UNUSED_VARIABLE = YES;
217 + MACOSX_DEPLOYMENT_TARGET = 14.0;
218 + MTL_ENABLE_DEBUG_INFO = NO;
219 + MTL_FAST_MATH = YES;
220 + PRODUCT_NAME = "$(TARGET_NAME)";
221 + SDKROOT = macosx;
222 + SWIFT_COMPILATION_MODE = wholemodule;
223 + SWIFT_OPTIMIZATION_LEVEL = "-O";
224 + SWIFT_VERSION = 5.0;
225 + };
226 + name = Release;
227 + };
228 + 4D5E4B6EED8197B03EAD7E51 /* Debug */ = {
229 + isa = XCBuildConfiguration;
230 + buildSettings = {
231 + ARCHS = arm64;
232 + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
233 + CODE_SIGN_ENTITLEMENTS = Metrika/Metrika.entitlements;
234 + CODE_SIGN_STYLE = Automatic;
235 + COMBINE_HIDPI_IMAGES = YES;
236 + CURRENT_PROJECT_VERSION = 1;
237 + ENABLE_APP_SANDBOX = YES;
238 + ENABLE_HARDENED_RUNTIME = YES;
239 + GENERATE_INFOPLIST_FILE = YES;
240 + INFOPLIST_KEY_NSHumanReadableCopyright = "© 2026 Simon-Pierre Boucher. All rights reserved.";
241 + INFOPLIST_KEY_NSPrincipalClass = NSApplication;
242 + LD_RUNPATH_SEARCH_PATHS = (
243 + "$(inherited)",
244 + "@executable_path/../Frameworks",
245 + );
246 + MARKETING_VERSION = 0.1.0;
247 + ONLY_ACTIVE_ARCH = YES;
248 + PRODUCT_BUNDLE_IDENTIFIER = ai.spboucher.metrika;
249 + SDKROOT = macosx;
250 + SWIFT_STRICT_CONCURRENCY = complete;
251 + SWIFT_VERSION = 6.0;
252 + };
253 + name = Debug;
254 + };
255 + C241F8724816C2E6FEFA98F2 /* Release */ = {
256 + isa = XCBuildConfiguration;
257 + buildSettings = {
258 + ARCHS = arm64;
259 + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
260 + CODE_SIGN_ENTITLEMENTS = Metrika/Metrika.entitlements;
261 + CODE_SIGN_STYLE = Automatic;
262 + COMBINE_HIDPI_IMAGES = YES;
263 + CURRENT_PROJECT_VERSION = 1;
264 + ENABLE_APP_SANDBOX = YES;
265 + ENABLE_HARDENED_RUNTIME = YES;
266 + GENERATE_INFOPLIST_FILE = YES;
267 + INFOPLIST_KEY_NSHumanReadableCopyright = "© 2026 Simon-Pierre Boucher. All rights reserved.";
268 + INFOPLIST_KEY_NSPrincipalClass = NSApplication;
269 + LD_RUNPATH_SEARCH_PATHS = (
270 + "$(inherited)",
271 + "@executable_path/../Frameworks",
272 + );
273 + MARKETING_VERSION = 0.1.0;
274 + ONLY_ACTIVE_ARCH = YES;
275 + PRODUCT_BUNDLE_IDENTIFIER = ai.spboucher.metrika;
276 + SDKROOT = macosx;
277 + SWIFT_STRICT_CONCURRENCY = complete;
278 + SWIFT_VERSION = 6.0;
279 + };
280 + name = Release;
281 + };
282 + DC41815671596F5C0AC0A0A6 /* Debug */ = {
283 + isa = XCBuildConfiguration;
284 + buildSettings = {
285 + ALWAYS_SEARCH_USER_PATHS = NO;
286 + CLANG_ANALYZER_NONNULL = YES;
287 + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
288 + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
289 + CLANG_CXX_LIBRARY = "libc++";
290 + CLANG_ENABLE_MODULES = YES;
291 + CLANG_ENABLE_OBJC_ARC = YES;
292 + CLANG_ENABLE_OBJC_WEAK = YES;
293 + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
294 + CLANG_WARN_BOOL_CONVERSION = YES;
295 + CLANG_WARN_COMMA = YES;
296 + CLANG_WARN_CONSTANT_CONVERSION = YES;
297 + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
298 + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
299 + CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
300 + CLANG_WARN_EMPTY_BODY = YES;
301 + CLANG_WARN_ENUM_CONVERSION = YES;
302 + CLANG_WARN_INFINITE_RECURSION = YES;
303 + CLANG_WARN_INT_CONVERSION = YES;
304 + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
305 + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
306 + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
307 + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
308 + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
309 + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
310 + CLANG_WARN_STRICT_PROTOTYPES = YES;
311 + CLANG_WARN_SUSPICIOUS_MOVE = YES;
312 + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
313 + CLANG_WARN_UNREACHABLE_CODE = YES;
314 + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
315 + COPY_PHASE_STRIP = NO;
316 + DEBUG_INFORMATION_FORMAT = dwarf;
317 + ENABLE_STRICT_OBJC_MSGSEND = YES;
318 + ENABLE_TESTABILITY = YES;
319 + GCC_C_LANGUAGE_STANDARD = gnu11;
320 + GCC_DYNAMIC_NO_PIC = NO;
321 + GCC_NO_COMMON_BLOCKS = YES;
322 + GCC_OPTIMIZATION_LEVEL = 0;
323 + GCC_PREPROCESSOR_DEFINITIONS = (
324 + "$(inherited)",
325 + "DEBUG=1",
326 + );
327 + GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
328 + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
329 + GCC_WARN_UNDECLARED_SELECTOR = YES;
330 + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
331 + GCC_WARN_UNUSED_FUNCTION = YES;
332 + GCC_WARN_UNUSED_VARIABLE = YES;
333 + MACOSX_DEPLOYMENT_TARGET = 14.0;
334 + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
335 + MTL_FAST_MATH = YES;
336 + ONLY_ACTIVE_ARCH = YES;
337 + PRODUCT_NAME = "$(TARGET_NAME)";
338 + SDKROOT = macosx;
339 + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
340 + SWIFT_OPTIMIZATION_LEVEL = "-Onone";
341 + SWIFT_VERSION = 5.0;
342 + };
343 + name = Debug;
344 + };
345 +/* End XCBuildConfiguration section */
346 +
347 +/* Begin XCConfigurationList section */
348 + 02D661894CE6081B5C88E20C /* Build configuration list for PBXProject "Metrika" */ = {
349 + isa = XCConfigurationList;
350 + buildConfigurations = (
351 + DC41815671596F5C0AC0A0A6 /* Debug */,
352 + 3CB58B068179C9DB2CD1CAC1 /* Release */,
353 + );
354 + defaultConfigurationIsVisible = 0;
355 + defaultConfigurationName = Debug;
356 + };
357 + A087F0DEACCF786E9B7CC9A5 /* Build configuration list for PBXNativeTarget "Metrika" */ = {
358 + isa = XCConfigurationList;
359 + buildConfigurations = (
360 + 4D5E4B6EED8197B03EAD7E51 /* Debug */,
361 + C241F8724816C2E6FEFA98F2 /* Release */,
362 + );
363 + defaultConfigurationIsVisible = 0;
364 + defaultConfigurationName = Debug;
365 + };
366 +/* End XCConfigurationList section */
367 +
368 +/* Begin XCLocalSwiftPackageReference section */
369 + D626AF14268CCEC6F4756ABC /* XCLocalSwiftPackageReference "MetrikaKit" */ = {
370 + isa = XCLocalSwiftPackageReference;
371 + relativePath = MetrikaKit;
372 + };
373 +/* End XCLocalSwiftPackageReference section */
374 +
375 +/* Begin XCSwiftPackageProductDependency section */
376 + 498C4115F45BADDF16191A57 /* MetrikaKit */ = {
377 + isa = XCSwiftPackageProductDependency;
378 + productName = MetrikaKit;
379 + };
380 +/* End XCSwiftPackageProductDependency section */
381 + };
382 + rootObject = 8571D0059EB0A97DE47F96F7 /* Project object */;
383 +}
added Metrika.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 −0
@@ -0,0 +1,7 @@
1 +<?xml version="1.0" encoding="UTF-8"?>
2 +<Workspace
3 + version = "1.0">
4 + <FileRef
5 + location = "self:">
6 + </FileRef>
7 +</Workspace>
added Metrika/Assets.xcassets/AppIcon.appiconset/Contents.json +15 −0
@@ -0,0 +1,15 @@
1 +{
2 + "images": [
3 + { "filename": "icon_16.png", "idiom": "mac", "scale": "1x", "size": "16x16" },
4 + { "filename": "icon_16@2x.png", "idiom": "mac", "scale": "2x", "size": "16x16" },
5 + { "filename": "icon_32.png", "idiom": "mac", "scale": "1x", "size": "32x32" },
6 + { "filename": "icon_32@2x.png", "idiom": "mac", "scale": "2x", "size": "32x32" },
7 + { "filename": "icon_128.png", "idiom": "mac", "scale": "1x", "size": "128x128" },
8 + { "filename": "icon_128@2x.png", "idiom": "mac", "scale": "2x", "size": "128x128" },
9 + { "filename": "icon_256.png", "idiom": "mac", "scale": "1x", "size": "256x256" },
10 + { "filename": "icon_256@2x.png", "idiom": "mac", "scale": "2x", "size": "256x256" },
11 + { "filename": "icon_512.png", "idiom": "mac", "scale": "1x", "size": "512x512" },
12 + { "filename": "icon_512@2x.png", "idiom": "mac", "scale": "2x", "size": "512x512" }
13 + ],
14 + "info": { "author": "xcode", "version": 1 }
15 +}
added Metrika/Assets.xcassets/AppIcon.appiconset/icon_128.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_128@2x.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_16.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_16@2x.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_256.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_256@2x.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_32.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_32@2x.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_512.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/AppIcon.appiconset/icon_512@2x.png +0 −0

Binary file not shown.

added Metrika/Assets.xcassets/Contents.json +1 −0
@@ -0,0 +1 @@
1 +{ "info": { "author": "xcode", "version": 1 } }
added Metrika/Metrika.entitlements +13 −0
@@ -0,0 +1,13 @@
1 +<?xml version="1.0" encoding="UTF-8"?>
2 +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
3 + "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
4 +<plist version="1.0">
5 +<dict>
6 + <key>com.apple.security.app-sandbox</key>
7 + <true/>
8 + <key>com.apple.security.files.user-selected.read-write</key>
9 + <true/>
10 + <key>com.apple.security.files.bookmarks.app-scope</key>
11 + <true/>
12 +</dict>
13 +</plist>
added Metrika/Sources/ConsoleView.swift +120 −0
@@ -0,0 +1,120 @@
1 +//
2 +// ConsoleView.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import SwiftUI
11 +
12 +/// Command console: scrollback of executed commands with their rendered
13 +/// output, plus an input line with ↑/↓ history recall (CLAUDE.md §7).
14 +struct ConsoleView: View {
15 + @Environment(SessionModel.self) private var model
16 + @State private var input = ""
17 + @State private var historyCursor: Int?
18 + @FocusState private var inputFocused: Bool
19 +
20 + var body: some View {
21 + VStack(spacing: 0) {
22 + ScrollViewReader { proxy in
23 + ScrollView {
24 + LazyVStack(alignment: .leading, spacing: 10) {
25 + ForEach(model.entries) { entry in
26 + VStack(alignment: .leading, spacing: 2) {
27 + if !entry.command.isEmpty {
28 + Text(". \(entry.command)")
29 + .foregroundStyle(.secondary)
30 + }
31 + if !entry.output.isEmpty {
32 + Text(entry.output)
33 + .foregroundStyle(
34 + entry.isError ? Color.red : Color.primary
35 + )
36 + .textSelection(.enabled)
37 + }
38 + }
39 + .id(entry.id)
40 + }
41 + }
42 + .padding(12)
43 + .frame(maxWidth: .infinity, alignment: .leading)
44 + }
45 + .onChange(of: model.entries) {
46 + if let last = model.entries.last {
47 + proxy.scrollTo(last.id, anchor: .bottom)
48 + }
49 + }
50 + }
51 +
52 + Divider()
53 +
54 + HStack(spacing: 8) {
55 + Text(".")
56 + .foregroundStyle(.secondary)
57 + TextField("Type a command — e.g. use sales.parquet", text: $input)
58 + .textFieldStyle(.plain)
59 + .focused($inputFocused)
60 + .onSubmit(submit)
61 + .onKeyPress(.upArrow) { recallHistory(step: -1) }
62 + .onKeyPress(.downArrow) { recallHistory(step: 1) }
63 + if model.isRunning {
64 + ProgressView()
65 + .controlSize(.small)
66 + }
67 + }
68 + .padding(10)
69 + }
70 + .fontDesign(.monospaced)
71 + .font(.system(size: 12))
72 + .onAppear { inputFocused = true }
73 + }
74 +
75 + private func submit() {
76 + let command = input
77 + input = ""
78 + historyCursor = nil
79 + Task { await model.run(command) }
80 + }
81 +
82 + private func recallHistory(step: Int) -> KeyPress.Result {
83 + let history = model.history
84 + guard !history.isEmpty else { return .ignored }
85 + var cursor = historyCursor ?? history.count
86 + cursor = min(max(cursor + step, 0), history.count)
87 + historyCursor = cursor
88 + input = cursor < history.count ? history[cursor] : ""
89 + return .handled
90 + }
91 +}
92 +
93 +/// Sidebar: variables of the working dataset with type and missing count
94 +/// (CLAUDE.md §7, pane 5).
95 +struct VariablesSidebar: View {
96 + @Environment(SessionModel.self) private var model
97 +
98 + var body: some View {
99 + List {
100 + Section("Variables (\(model.observationCount) obs)") {
101 + ForEach(model.variables) { variable in
102 + HStack {
103 + Text(variable.name)
104 + .fontDesign(.monospaced)
105 + Spacer()
106 + if variable.missingCount > 0 {
107 + Text("\(variable.missingCount) mi")
108 + .foregroundStyle(.orange)
109 + .font(.caption)
110 + }
111 + Text(variable.type)
112 + .foregroundStyle(.secondary)
113 + .font(.caption)
114 + }
115 + }
116 + }
117 + }
118 + .listStyle(.sidebar)
119 + }
120 +}
added Metrika/Sources/MetrikaApp.swift +44 −0
@@ -0,0 +1,44 @@
1 +//
2 +// MetrikaApp.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import SwiftUI
11 +
12 +@main
13 +struct MetrikaApp: App {
14 + @State private var model = SessionModel()
15 +
16 + var body: some Scene {
17 + WindowGroup {
18 + ContentView()
19 + .environment(model)
20 + .frame(minWidth: 900, minHeight: 560)
21 + }
22 + .windowStyle(.automatic)
23 + }
24 +}
25 +
26 +struct ContentView: View {
27 + @Environment(SessionModel.self) private var model
28 +
29 + var body: some View {
30 + NavigationSplitView {
31 + VariablesSidebar()
32 + } detail: {
33 + HSplitView {
34 + ConsoleView()
35 + .frame(minWidth: 480)
36 + if model.lastPlot != nil {
37 + PlotView()
38 + .frame(minWidth: 300)
39 + }
40 + }
41 + }
42 + .navigationTitle("Metrika")
43 + }
44 +}
added Metrika/Sources/PlotView.swift +58 −0
@@ -0,0 +1,58 @@
1 +//
2 +// PlotView.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Charts
11 +import SwiftUI
12 +import ZQGraphics
13 +
14 +/// Swift Charts rendering of the session's latest plot spec. The custom
15 +/// Metal renderer takes over past ~1M points (v0.3, CLAUDE.md §7).
16 +struct PlotView: View {
17 + @Environment(SessionModel.self) private var model
18 +
19 + var body: some View {
20 + if let plot = model.lastPlot {
21 + Chart {
22 + ForEach(Array(plot.series.enumerated()), id: \.offset) { _, series in
23 + ForEach(series.x.indices, id: \.self) { index in
24 + switch plot.kind {
25 + case .scatter:
26 + PointMark(
27 + x: .value(plot.xLabel, series.x[index]),
28 + y: .value(plot.yLabel, series.y[index])
29 + )
30 + .symbolSize(20)
31 + .foregroundStyle(by: .value("Series", series.label))
32 + case .line, .kdensity:
33 + LineMark(
34 + x: .value(plot.xLabel, series.x[index]),
35 + y: .value(plot.yLabel, series.y[index])
36 + )
37 + .foregroundStyle(by: .value("Series", series.label))
38 + case .histogram:
39 + BarMark(
40 + x: .value(plot.xLabel, series.x[index]),
41 + y: .value(plot.yLabel, series.y[index])
42 + )
43 + .foregroundStyle(by: .value("Series", series.label))
44 + }
45 + }
46 + }
47 + }
48 + .chartXAxisLabel(plot.xLabel)
49 + .chartYAxisLabel(plot.yLabel)
50 + .padding(12)
51 + } else {
52 + ContentUnavailableView(
53 + "No plot", systemImage: "chart.xyaxis.line",
54 + description: Text("Run a graph command to see it here.")
55 + )
56 + }
57 + }
58 +}
added Metrika/Sources/SessionModel.swift +84 −0
@@ -0,0 +1,84 @@
1 +//
2 +// SessionModel.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Observation
11 +import SwiftUI
12 +import ZQData
13 +import ZQEngine
14 +import ZQGraphics
15 +
16 +/// Observable session state for the UI. Every command — typed in the
17 +/// console or replayed from a do-file — goes through the one `ZQSession`
18 +/// execution path (CLAUDE.md §7).
19 +@Observable @MainActor
20 +final class SessionModel {
21 + struct ConsoleEntry: Identifiable, Equatable {
22 + let id = UUID()
23 + let command: String
24 + let output: String
25 + let isError: Bool
26 + }
27 +
28 + struct VariableInfo: Identifiable, Equatable {
29 + var id: String { name }
30 + let name: String
31 + let type: String
32 + let missingCount: Int
33 + }
34 +
35 + private(set) var entries: [ConsoleEntry] = []
36 + private(set) var variables: [VariableInfo] = []
37 + private(set) var observationCount = 0
38 + private(set) var lastPlot: ZQPlotSpec?
39 + private(set) var isRunning = false
40 + private(set) var history: [String] = []
41 +
42 + private var session: ZQSession?
43 +
44 + init() {
45 + do {
46 + session = try ZQSession()
47 + } catch {
48 + entries.append(ConsoleEntry(
49 + command: "",
50 + output: "failed to start engine: \(error)",
51 + isError: true
52 + ))
53 + }
54 + }
55 +
56 + func run(_ command: String) async {
57 + let line = command.trimmingCharacters(in: .whitespaces)
58 + guard !line.isEmpty, let session else { return }
59 + history.append(line)
60 + isRunning = true
61 + defer { isRunning = false }
62 + do {
63 + let result = try await session.execute(line)
64 + entries.append(ConsoleEntry(command: line, output: result.text, isError: false))
65 + } catch {
66 + entries.append(ConsoleEntry(command: line, output: "\(error)", isError: true))
67 + }
68 + await refreshDatasetState()
69 + }
70 +
71 + private func refreshDatasetState() async {
72 + guard let session else { return }
73 + let frame = await session.frame
74 + observationCount = frame.rowCount
75 + variables = frame.columns.map { column in
76 + VariableInfo(
77 + name: column.name,
78 + type: column.data.isNumeric ? "float64" : "string",
79 + missingCount: column.data.missingCount
80 + )
81 + }
82 + lastPlot = await session.lastPlot
83 + }
84 +}
added MetrikaKit/Package.resolved +33 −0
@@ -0,0 +1,33 @@
1 +{
2 + "originHash" : "e4ae512a67bec7ae66f2abd23f1e87240c39d28811734cdfafc7a91e4323f6db",
3 + "pins" : [
4 + {
5 + "identity" : "duckdb-swift",
6 + "kind" : "remoteSourceControl",
7 + "location" : "https://github.com/duckdb/duckdb-swift",
8 + "state" : {
9 + "revision" : "9583f44fedc20a8e0b6151d2e43f6d850016712f",
10 + "version" : "1.1.3"
11 + }
12 + },
13 + {
14 + "identity" : "swift-argument-parser",
15 + "kind" : "remoteSourceControl",
16 + "location" : "https://github.com/apple/swift-argument-parser",
17 + "state" : {
18 + "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382",
19 + "version" : "1.8.2"
20 + }
21 + },
22 + {
23 + "identity" : "swift-collections",
24 + "kind" : "remoteSourceControl",
25 + "location" : "https://github.com/apple/swift-collections",
26 + "state" : {
27 + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a",
28 + "version" : "1.6.0"
29 + }
30 + }
31 + ],
32 + "version" : 3
33 +}
added MetrikaKit/Package.swift +96 −0
@@ -0,0 +1,96 @@
1 +// swift-tools-version: 6.0
2 +//
3 +// Package.swift
4 +// Metrika
5 +//
6 +// Author: Simon-Pierre Boucher
7 +// Contact: contact@spboucher.ai
8 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
9 +//
10 +
11 +import PackageDescription
12 +
13 +let strictConcurrency: [SwiftSetting] = [
14 + .swiftLanguageMode(.v6)
15 +]
16 +
17 +let package = Package(
18 + name: "MetrikaKit",
19 + platforms: [.macOS(.v14)],
20 + products: [
21 + .library(
22 + name: "MetrikaKit",
23 + targets: [
24 + "ZQParser", "ZQPlanner", "ZQEngine", "ZQData",
25 + "ZQStats", "ZQGPU", "ZQGraphics", "ZQPlugins",
26 + ]
27 + ),
28 + .executable(name: "metrika-cli", targets: ["MetrikaCLI"]),
29 + ],
30 + dependencies: [
31 + .package(url: "https://github.com/duckdb/duckdb-swift", from: "1.0.0"),
32 + .package(url: "https://github.com/apple/swift-collections", from: "1.1.0"),
33 + .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"),
34 + ],
35 + targets: [
36 + .target(
37 + name: "ZQParser",
38 + swiftSettings: strictConcurrency
39 + ),
40 + .target(
41 + name: "ZQData",
42 + dependencies: [
43 + .product(name: "DuckDB", package: "duckdb-swift"),
44 + .product(name: "Collections", package: "swift-collections"),
45 + ],
46 + swiftSettings: strictConcurrency
47 + ),
48 + .target(
49 + name: "ZQStats",
50 + dependencies: ["ZQData"],
51 + swiftSettings: strictConcurrency
52 + ),
53 + .target(
54 + name: "ZQGPU",
55 + dependencies: ["ZQData"],
56 + swiftSettings: strictConcurrency
57 + ),
58 + .target(
59 + name: "ZQPlanner",
60 + dependencies: ["ZQParser", "ZQData"],
61 + swiftSettings: strictConcurrency
62 + ),
63 + .target(
64 + name: "ZQGraphics",
65 + dependencies: ["ZQData"],
66 + swiftSettings: strictConcurrency
67 + ),
68 + .target(
69 + name: "ZQPlugins",
70 + dependencies: ["ZQParser", "ZQData"],
71 + swiftSettings: strictConcurrency
72 + ),
73 + .target(
74 + name: "ZQEngine",
75 + dependencies: [
76 + "ZQParser", "ZQPlanner", "ZQData", "ZQStats",
77 + "ZQGPU", "ZQGraphics", "ZQPlugins",
78 + ],
79 + swiftSettings: strictConcurrency
80 + ),
81 + .executableTarget(
82 + name: "MetrikaCLI",
83 + dependencies: [
84 + "ZQEngine",
85 + .product(name: "ArgumentParser", package: "swift-argument-parser"),
86 + ],
87 + swiftSettings: strictConcurrency
88 + ),
89 + .testTarget(
90 + name: "MetrikaKitTests",
91 + dependencies: ["ZQParser", "ZQPlanner", "ZQEngine", "ZQData", "ZQStats", "ZQGPU"],
92 + resources: [.copy("Fixtures")],
93 + swiftSettings: strictConcurrency
94 + ),
95 + ]
96 +)
added MetrikaKit/Sources/MetrikaCLI/MetrikaCLI.swift +65 −0
@@ -0,0 +1,65 @@
1 +//
2 +// MetrikaCLI.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import ArgumentParser
11 +import Foundation
12 +import ZQEngine
13 +
14 +/// Companion command-line tool: runs ZQL commands or `.zyq` do-files
15 +/// against a headless session. Interactive REPL when no input is given.
16 +@main
17 +struct MetrikaCLI: AsyncParsableCommand {
18 + static let configuration = CommandConfiguration(
19 + commandName: "metrika-cli",
20 + abstract: "Metrika statistical engine — command-line interface."
21 + )
22 +
23 + @Option(
24 + name: [.short, .customLong("execute")],
25 + help: "Execute a ZQL command (repeatable, runs in order)."
26 + )
27 + var execute: [String] = []
28 +
29 + @Argument(help: "A .zyq do-file to run.")
30 + var doFile: String?
31 +
32 + mutating func run() async throws {
33 + let session = try ZQSession()
34 +
35 + if let doFile {
36 + let url = URL(fileURLWithPath: (doFile as NSString).expandingTildeInPath)
37 + let text = try String(contentsOf: url, encoding: .utf8)
38 + let result = try await session.executeScript(text)
39 + print(result.text)
40 + return
41 + }
42 +
43 + if !execute.isEmpty {
44 + for command in execute {
45 + let result = try await session.execute(command)
46 + if !result.text.isEmpty { print(result.text) }
47 + }
48 + return
49 + }
50 +
51 + // REPL
52 + print("Metrika — type ZQL commands, 'exit' to quit.")
53 + while true {
54 + print(". ", terminator: "")
55 + guard let line = readLine() else { break }
56 + if line == "exit" || line == "quit" { break }
57 + do {
58 + let result = try await session.execute(line)
59 + if !result.text.isEmpty { print(result.text) }
60 + } catch {
61 + print("error: \(error)")
62 + }
63 + }
64 + }
65 +}
added MetrikaKit/Sources/ZQData/DataFrame.swift +163 −0
@@ -0,0 +1,163 @@
1 +//
2 +// DataFrame.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Column payload. Numeric data is Float64 end-to-end (CLAUDE.md §5: all
11 +/// CPU compute in Float64) with an explicit validity mask mirroring Arrow
12 +/// semantics: `missing[i] == true` means observation `i` is missing, and
13 +/// the corresponding value slot is undefined.
14 +public enum ZQColumnData: Equatable, Sendable {
15 + case float64(values: [Double], missing: [Bool])
16 + case string([String?])
17 +
18 + public var count: Int {
19 + switch self {
20 + case .float64(let values, _): return values.count
21 + case .string(let values): return values.count
22 + }
23 + }
24 +
25 + public var missingCount: Int {
26 + switch self {
27 + case .float64(_, let missing): return missing.count { $0 }
28 + case .string(let values): return values.count { $0 == nil }
29 + }
30 + }
31 +
32 + public var isNumeric: Bool {
33 + if case .float64 = self { return true }
34 + return false
35 + }
36 +}
37 +
38 +/// A named column of a data frame.
39 +public struct ZQColumn: Equatable, Sendable {
40 + public var name: String
41 + public var data: ZQColumnData
42 +
43 + public init(name: String, data: ZQColumnData) {
44 + self.name = name
45 + self.data = data
46 + }
47 +
48 + public var count: Int { data.count }
49 +
50 + /// Numeric view: values with their validity mask, or nil for string
51 + /// columns.
52 + public var numeric: (values: [Double], missing: [Bool])? {
53 + if case .float64(let values, let missing) = data {
54 + return (values, missing)
55 + }
56 + return nil
57 + }
58 +}
59 +
60 +public struct ZQDataError: Error, Equatable, Sendable, CustomStringConvertible {
61 + public var message: String
62 + public init(_ message: String) { self.message = message }
63 + public var description: String { message }
64 +}
65 +
66 +/// In-memory data frame: the working dataset of a session. Materialized
67 +/// from DuckDB query results; columnar, immutable value semantics.
68 +public struct ZQDataFrame: Equatable, Sendable {
69 + public private(set) var columns: [ZQColumn]
70 + public let rowCount: Int
71 +
72 + public init(columns: [ZQColumn] = []) throws(ZQDataError) {
73 + let counts = Set(columns.map(\.count))
74 + guard counts.count <= 1 else {
75 + throw ZQDataError("columns have inconsistent lengths: \(counts.sorted())")
76 + }
77 + self.columns = columns
78 + self.rowCount = counts.first ?? 0
79 + }
80 +
81 + public var isEmpty: Bool { columns.isEmpty || rowCount == 0 }
82 + public var columnNames: [String] { columns.map(\.name) }
83 +
84 + public func column(named name: String) -> ZQColumn? {
85 + columns.first { $0.name == name }
86 + }
87 +
88 + public func requireColumn(_ name: String) throws(ZQDataError) -> ZQColumn {
89 + guard let column = column(named: name) else {
90 + throw ZQDataError("variable '\(name)' not found")
91 + }
92 + return column
93 + }
94 +
95 + public func requireNumeric(
96 + _ name: String
97 + ) throws(ZQDataError) -> (values: [Double], missing: [Bool]) {
98 + guard let numeric = try requireColumn(name).numeric else {
99 + throw ZQDataError("variable '\(name)' is a string variable")
100 + }
101 + return numeric
102 + }
103 +
104 + /// Adds a new column; fails if the name already exists (Stata `generate`
105 + /// semantics — use `replaceColumn` for `replace`).
106 + public mutating func addColumn(_ column: ZQColumn) throws(ZQDataError) {
107 + guard self.column(named: column.name) == nil else {
108 + throw ZQDataError("variable '\(column.name)' already defined")
109 + }
110 + guard columns.isEmpty || column.count == rowCount else {
111 + throw ZQDataError(
112 + "column '\(column.name)' has \(column.count) rows, expected \(rowCount)"
113 + )
114 + }
115 + columns.append(column)
116 + }
117 +
118 + /// Replaces an existing column in place (Stata `replace`).
119 + public mutating func replaceColumn(_ column: ZQColumn) throws(ZQDataError) {
120 + guard let index = columns.firstIndex(where: { $0.name == column.name }) else {
121 + throw ZQDataError("variable '\(column.name)' not found")
122 + }
123 + guard column.count == rowCount else {
124 + throw ZQDataError(
125 + "column '\(column.name)' has \(column.count) rows, expected \(rowCount)"
126 + )
127 + }
128 + columns[index] = column
129 + }
130 +
131 + public mutating func dropColumns(_ names: [String]) throws(ZQDataError) {
132 + for name in names {
133 + guard columns.contains(where: { $0.name == name }) else {
134 + throw ZQDataError("variable '\(name)' not found")
135 + }
136 + }
137 + columns.removeAll { names.contains($0.name) }
138 + }
139 +
140 + /// Row subset by keep-mask (`mask[i] == true` keeps observation `i`).
141 + public func filtered(by mask: [Bool]) throws(ZQDataError) -> ZQDataFrame {
142 + guard mask.count == rowCount else {
143 + throw ZQDataError("filter mask length \(mask.count) != \(rowCount) rows")
144 + }
145 + let kept = (0..<rowCount).filter { mask[$0] }
146 + let newColumns = columns.map { column -> ZQColumn in
147 + switch column.data {
148 + case .float64(let values, let missing):
149 + return ZQColumn(
150 + name: column.name,
151 + data: .float64(
152 + values: kept.map { values[$0] },
153 + missing: kept.map { missing[$0] }
154 + )
155 + )
156 + case .string(let values):
157 + return ZQColumn(name: column.name, data: .string(kept.map { values[$0] }))
158 + }
159 + }
160 + // Lengths are uniform by construction.
161 + return try! ZQDataFrame(columns: newColumns)
162 + }
163 +}
added MetrikaKit/Sources/ZQData/DuckDBStore.swift +153 −0
@@ -0,0 +1,153 @@
1 +//
2 +// DuckDBStore.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import DuckDB
11 +import Foundation
12 +
13 +/// DuckDB-backed data store (CLAUDE.md §6). File loading goes through
14 +/// DuckDB readers (`read_parquet`, `read_csv`) and query results are
15 +/// materialized into `ZQDataFrame` columns: every numeric database type
16 +/// widens to Float64, text stays as strings.
17 +public actor ZQDataStore {
18 + private let database: Database
19 + private let connection: Connection
20 +
21 + public init() throws {
22 + self.database = try Database(store: .inMemory)
23 + self.connection = try database.connect()
24 + }
25 +
26 + // MARK: - Loading
27 +
28 + /// Loads a dataset file into a data frame. Format is inferred from the
29 + /// file extension: .parquet, .csv, .tsv, .json, .arrow.
30 + public func load(contentsOf url: URL) throws -> ZQDataFrame {
31 + let path = url.path
32 + guard FileManager.default.fileExists(atPath: path) else {
33 + throw ZQDataError("file not found: \(path)")
34 + }
35 + let escaped = path.replacingOccurrences(of: "'", with: "''")
36 + let reader: String
37 + switch url.pathExtension.lowercased() {
38 + case "parquet", "pq":
39 + reader = "read_parquet('\(escaped)')"
40 + case "csv", "tsv", "txt":
41 + reader = "read_csv('\(escaped)')"
42 + case "json", "ndjson", "jsonl":
43 + reader = "read_json('\(escaped)')"
44 + case "arrow", "feather", "ipc":
45 + reader = "read_ipc('\(escaped)')"
46 + case let ext:
47 + throw ZQDataError("unsupported file format '.\(ext)'")
48 + }
49 + return try dataFrame(fromQuery: "SELECT * FROM \(reader)")
50 + }
51 +
52 + /// Runs an arbitrary SQL query and materializes the result.
53 + public func dataFrame(fromQuery sql: String) throws -> ZQDataFrame {
54 + let result: ResultSet
55 + do {
56 + result = try connection.query(sql)
57 + } catch {
58 + throw ZQDataError("query failed: \(error)")
59 + }
60 + var columns: [ZQColumn] = []
61 + for index in 0..<result.columnCount {
62 + let name = result.columnName(at: index)
63 + let untyped = result.column(at: index)
64 + columns.append(materialize(untyped, name: name))
65 + }
66 + return try ZQDataFrame(columns: columns)
67 + }
68 +
69 + private func materialize(_ column: Column<Void>, name: String) -> ZQColumn {
70 + switch column.underlyingDatabaseType {
71 + case .double, .float, .decimal:
72 + return numericColumn(name: name, column.cast(to: Double.self).map { $0 })
73 + case .tinyint, .smallint, .integer, .bigint,
74 + .utinyint, .usmallint, .uinteger:
75 + let values = column.cast(to: Int64.self).map { $0.map(Double.init) }
76 + return numericColumn(name: name, values)
77 + case .ubigint:
78 + let values = column.cast(to: UInt64.self).map { $0.map(Double.init) }
79 + return numericColumn(name: name, values)
80 + case .boolean:
81 + let values = column.cast(to: Bool.self).map { $0.map { $0 ? 1.0 : 0.0 } }
82 + return numericColumn(name: name, values)
83 + default:
84 + // Text, dates, and everything else: keep the string rendering.
85 + let values = column.cast(to: String.self).map { $0 }
86 + return ZQColumn(name: name, data: .string(Array(values)))
87 + }
88 + }
89 +
90 + private func numericColumn(name: String, _ optionals: [Double?]) -> ZQColumn {
91 + var values = [Double](repeating: 0, count: optionals.count)
92 + var missing = [Bool](repeating: false, count: optionals.count)
93 + for (i, value) in optionals.enumerated() {
94 + if let value, !value.isNaN {
95 + values[i] = value
96 + } else {
97 + missing[i] = true
98 + values[i] = .nan
99 + }
100 + }
101 + return ZQColumn(name: name, data: .float64(values: values, missing: missing))
102 + }
103 +
104 + // MARK: - Saving
105 +
106 + /// Writes a data frame to disk via DuckDB `COPY`. Format inferred from
107 + /// the extension (.parquet or .csv).
108 + public func save(_ frame: ZQDataFrame, to url: URL) throws {
109 + let format: String
110 + switch url.pathExtension.lowercased() {
111 + case "parquet", "pq": format = "PARQUET"
112 + case "csv": format = "CSV, HEADER"
113 + case let ext: throw ZQDataError("unsupported save format '.\(ext)'")
114 + }
115 +
116 + try register(frame, as: "__metrika_save")
117 + defer { _ = try? connection.query("DROP TABLE IF EXISTS __metrika_save") }
118 + let escaped = url.path.replacingOccurrences(of: "'", with: "''")
119 + _ = try connection.query(
120 + "COPY __metrika_save TO '\(escaped)' (FORMAT \(format))"
121 + )
122 + }
123 +
124 + /// Registers a data frame as a DuckDB table (used by save and by SQL
125 + /// pushdown of `gen`/`collapse` expressions).
126 + private func register(_ frame: ZQDataFrame, as table: String) throws {
127 + _ = try? connection.query("DROP TABLE IF EXISTS \(table)")
128 + let declarations = frame.columns.map { column in
129 + let type = column.data.isNumeric ? "DOUBLE" : "VARCHAR"
130 + return "\"\(column.name)\" \(type)"
131 + }
132 + guard !declarations.isEmpty else {
133 + throw ZQDataError("nothing to save: dataset has no variables")
134 + }
135 + _ = try connection.query(
136 + "CREATE TABLE \(table) (\(declarations.joined(separator: ", ")))"
137 + )
138 +
139 + let appender = try Appender(connection: connection, table: table)
140 + for row in 0..<frame.rowCount {
141 + for column in frame.columns {
142 + switch column.data {
143 + case .float64(let values, let missing):
144 + try appender.append(missing[row] ? nil : values[row])
145 + case .string(let values):
146 + try appender.append(values[row])
147 + }
148 + }
149 + try appender.endRow()
150 + }
151 + try appender.flush()
152 + }
153 +}
added MetrikaKit/Sources/ZQData/ZQDataModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQDataModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQData.
11 +public enum ZQDataModule {
12 + public static let name = "ZQData"
13 +}
added MetrikaKit/Sources/ZQEngine/ExpressionEvaluator.swift +332 −0
@@ -0,0 +1,332 @@
1 +//
2 +// ExpressionEvaluator.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +import ZQData
12 +import ZQParser
13 +
14 +/// Vectorized evaluation of a `ZQExpression` against the working dataset.
15 +///
16 +/// Missing-value semantics: any arithmetic or comparison touching a
17 +/// missing operand yields missing; undefined operations (division by
18 +/// zero, log of a non-positive number) yield missing. The `missing(x)`
19 +/// function itself never returns missing. In `if` qualifiers a missing
20 +/// condition excludes the observation.
21 +struct ExpressionEvaluator {
22 + struct EvaluationError: Error, CustomStringConvertible {
23 + let message: String
24 + var description: String { message }
25 + }
26 +
27 + /// A column vector or a broadcastable scalar.
28 + enum Value {
29 + case numeric(values: [Double], missing: [Bool])
30 + case strings([String?])
31 + case numericScalar(Double)
32 + case stringScalar(String)
33 + case missingScalar
34 + }
35 +
36 + let frame: ZQDataFrame
37 +
38 + func evaluate(_ expression: ZQExpression) throws -> Value {
39 + switch expression {
40 + case .number(let value):
41 + return .numericScalar(value)
42 + case .string(let value):
43 + return .stringScalar(value)
44 + case .missing:
45 + return .missingScalar
46 + case .variable(let name):
47 + guard let column = frame.column(named: name) else {
48 + throw EvaluationError(message: "variable '\(name)' not found")
49 + }
50 + switch column.data {
51 + case .float64(let values, let missing):
52 + return .numeric(values: values, missing: missing)
53 + case .string(let values):
54 + return .strings(values)
55 + }
56 + case .unary(let op, let operand):
57 + return try evaluateUnary(op: op, operand: operand)
58 + case .binary(let op, let lhs, let rhs):
59 + return try evaluateBinary(op: op, lhs: lhs, rhs: rhs)
60 + case .call(let name, let arguments):
61 + return try evaluateCall(name: name, arguments: arguments)
62 + }
63 + }
64 +
65 + /// Evaluates to a full-length numeric column (broadcasting scalars).
66 + func evaluateNumericColumn(
67 + _ expression: ZQExpression
68 + ) throws -> (values: [Double], missing: [Bool]) {
69 + switch try evaluate(expression) {
70 + case .numeric(let values, let missing):
71 + return (values, missing)
72 + case .numericScalar(let value):
73 + return (
74 + [Double](repeating: value, count: frame.rowCount),
75 + [Bool](repeating: false, count: frame.rowCount)
76 + )
77 + case .missingScalar:
78 + return (
79 + [Double](repeating: .nan, count: frame.rowCount),
80 + [Bool](repeating: true, count: frame.rowCount)
81 + )
82 + case .strings, .stringScalar:
83 + throw EvaluationError(message: "type mismatch: expected a numeric expression")
84 + }
85 + }
86 +
87 + /// Evaluates an `if` qualifier into a keep-mask. Missing → excluded.
88 + func evaluateCondition(_ expression: ZQExpression) throws -> [Bool] {
89 + let (values, missing) = try evaluateNumericColumn(expression)
90 + return (0..<values.count).map { !missing[$0] && values[$0] != 0 }
91 + }
92 +
93 + // MARK: - Operators
94 +
95 + private func evaluateUnary(op: String, operand: ZQExpression) throws -> Value {
96 + let value = try evaluate(operand)
97 + switch op {
98 + case "-":
99 + return try mapNumeric(value) { -$0 }
100 + case "!":
101 + return try mapNumeric(value) { $0 == 0 ? 1 : 0 }
102 + default:
103 + throw EvaluationError(message: "unknown unary operator '\(op)'")
104 + }
105 + }
106 +
107 + private func evaluateBinary(
108 + op: String, lhs: ZQExpression, rhs: ZQExpression
109 + ) throws -> Value {
110 + let left = try evaluate(lhs)
111 + let right = try evaluate(rhs)
112 +
113 + // String equality against a literal or another string column.
114 + if isString(left) || isString(right) {
115 + guard op == "==" || op == "!=" else {
116 + throw EvaluationError(
117 + message: "operator '\(op)' is not defined for strings"
118 + )
119 + }
120 + return try compareStrings(left, right, equal: op == "==")
121 + }
122 +
123 + let combine: (Double, Double) -> Double
124 + switch op {
125 + case "+": combine = (+)
126 + case "-": combine = (-)
127 + case "*": combine = (*)
128 + case "/": combine = { $1 == 0 ? .nan : $0 / $1 }
129 + case "^": combine = { pow($0, $1) }
130 + case "==": combine = { $0 == $1 ? 1 : 0 }
131 + case "!=": combine = { $0 != $1 ? 1 : 0 }
132 + case "<": combine = { $0 < $1 ? 1 : 0 }
133 + case "<=": combine = { $0 <= $1 ? 1 : 0 }
134 + case ">": combine = { $0 > $1 ? 1 : 0 }
135 + case ">=": combine = { $0 >= $1 ? 1 : 0 }
136 + case "&": combine = { ($0 != 0 && $1 != 0) ? 1 : 0 }
137 + case "|": combine = { ($0 != 0 || $1 != 0) ? 1 : 0 }
138 + default:
139 + throw EvaluationError(message: "unknown operator '\(op)'")
140 + }
141 + return try zipNumeric(left, right, combine)
142 + }
143 +
144 + private func evaluateCall(name: String, arguments: [ZQExpression]) throws -> Value {
145 + // missing(x): 1/0 indicator, itself never missing.
146 + if name == "missing" || name == "mi" {
147 + guard arguments.count == 1 else {
148 + throw EvaluationError(message: "missing() takes exactly one argument")
149 + }
150 + switch try evaluate(arguments[0]) {
151 + case .numeric(_, let missing):
152 + return .numeric(
153 + values: missing.map { $0 ? 1.0 : 0.0 },
154 + missing: [Bool](repeating: false, count: missing.count)
155 + )
156 + case .strings(let values):
157 + return .numeric(
158 + values: values.map { $0 == nil ? 1.0 : 0.0 },
159 + missing: [Bool](repeating: false, count: values.count)
160 + )
161 + case .missingScalar:
162 + return .numericScalar(1)
163 + case .numericScalar, .stringScalar:
164 + return .numericScalar(0)
165 + }
166 + }
167 +
168 + let unaryFunctions: [String: @Sendable (Double) -> Double] = [
169 + "ln": { $0 > 0 ? Foundation.log($0) : .nan },
170 + "log": { $0 > 0 ? Foundation.log($0) : .nan },
171 + "log10": { $0 > 0 ? Foundation.log10($0) : .nan },
172 + "exp": Foundation.exp,
173 + "sqrt": { $0 >= 0 ? $0.squareRoot() : .nan },
174 + "abs": abs,
175 + "floor": { $0.rounded(.down) },
176 + "ceil": { $0.rounded(.up) },
177 + "round": { $0.rounded(.toNearestOrAwayFromZero) },
178 + "int": { $0.rounded(.towardZero) },
179 + ]
180 + if let function = unaryFunctions[name] {
181 + guard arguments.count == 1 else {
182 + throw EvaluationError(message: "\(name)() takes exactly one argument")
183 + }
184 + return try mapNumeric(try evaluate(arguments[0]), function)
185 + }
186 +
187 + if name == "max" || name == "min" {
188 + guard arguments.count >= 2 else {
189 + throw EvaluationError(message: "\(name)() takes at least two arguments")
190 + }
191 + var result = try evaluate(arguments[0])
192 + for argument in arguments.dropFirst() {
193 + let next = try evaluate(argument)
194 + result = try zipNumeric(result, next, name == "max" ? max : min)
195 + }
196 + return result
197 + }
198 +
199 + throw EvaluationError(message: "unknown function '\(name)()'")
200 + }
201 +
202 + // MARK: - Helpers
203 +
204 + private func isString(_ value: Value) -> Bool {
205 + switch value {
206 + case .strings, .stringScalar: return true
207 + default: return false
208 + }
209 + }
210 +
211 + private func compareStrings(
212 + _ lhs: Value, _ rhs: Value, equal: Bool
213 + ) throws -> Value {
214 + func stringAt(_ value: Value, _ index: Int) -> String?? {
215 + switch value {
216 + case .strings(let values): return values[index]
217 + case .stringScalar(let value): return value
218 + default: return Optional<String?>.none
219 + }
220 + }
221 + let count: Int
222 + switch (lhs, rhs) {
223 + case (.strings(let values), _), (_, .strings(let values)):
224 + count = values.count
225 + default:
226 + // scalar vs scalar
227 + if case .stringScalar(let a) = lhs, case .stringScalar(let b) = rhs {
228 + return .numericScalar((a == b) == equal ? 1 : 0)
229 + }
230 + throw EvaluationError(message: "type mismatch in string comparison")
231 + }
232 + var values = [Double](repeating: 0, count: count)
233 + let missing = [Bool](repeating: false, count: count)
234 + for i in 0..<count {
235 + guard let a = stringAt(lhs, i), let b = stringAt(rhs, i) else {
236 + throw EvaluationError(message: "type mismatch in string comparison")
237 + }
238 + switch (a, b) {
239 + case (nil, nil), (nil, _), (_, nil):
240 + // Stata treats "" as the string missing value; comparisons
241 + // against it still evaluate.
242 + let equalValues = (a ?? "") == (b ?? "")
243 + values[i] = equalValues == equal ? 1 : 0
244 + default:
245 + values[i] = (a! == b!) == equal ? 1 : 0
246 + }
247 + }
248 + return .numeric(values: values, missing: missing)
249 + }
250 +
251 + private func mapNumeric(
252 + _ value: Value, _ transform: (Double) -> Double
253 + ) throws -> Value {
254 + switch value {
255 + case .numeric(let values, var missing):
256 + var result = [Double](repeating: .nan, count: values.count)
257 + for i in 0..<values.count where !missing[i] {
258 + result[i] = transform(values[i])
259 + if result[i].isNaN { missing[i] = true }
260 + }
261 + return .numeric(values: result, missing: missing)
262 + case .numericScalar(let value):
263 + let result = transform(value)
264 + return result.isNaN ? .missingScalar : .numericScalar(result)
265 + case .missingScalar:
266 + return .missingScalar
267 + case .strings, .stringScalar:
268 + throw EvaluationError(message: "type mismatch: expected a numeric operand")
269 + }
270 + }
271 +
272 + private func zipNumeric(
273 + _ lhs: Value, _ rhs: Value, _ combine: (Double, Double) -> Double
274 + ) throws -> Value {
275 + switch (lhs, rhs) {
276 + case (.numericScalar(let a), .numericScalar(let b)):
277 + let result = combine(a, b)
278 + return result.isNaN ? .missingScalar : .numericScalar(result)
279 + case (.missingScalar, _), (_, .missingScalar):
280 + if case .numeric(let values, _) = lhs {
281 + return .numeric(
282 + values: [Double](repeating: .nan, count: values.count),
283 + missing: [Bool](repeating: true, count: values.count)
284 + )
285 + }
286 + if case .numeric(let values, _) = rhs {
287 + return .numeric(
288 + values: [Double](repeating: .nan, count: values.count),
289 + missing: [Bool](repeating: true, count: values.count)
290 + )
291 + }
292 + return .missingScalar
293 + default:
294 + let (leftValues, leftMissing) = try broadcast(lhs)
295 + let (rightValues, rightMissing) = try broadcast(rhs)
296 + guard leftValues.count == rightValues.count else {
297 + throw EvaluationError(message: "operand length mismatch")
298 + }
299 + var values = [Double](repeating: .nan, count: leftValues.count)
300 + var missing = [Bool](repeating: true, count: leftValues.count)
301 + for i in 0..<leftValues.count where !leftMissing[i] && !rightMissing[i] {
302 + let result = combine(leftValues[i], rightValues[i])
303 + if !result.isNaN {
304 + values[i] = result
305 + missing[i] = false
306 + }
307 + }
308 + return .numeric(values: values, missing: missing)
309 + }
310 + }
311 +
312 + private func broadcast(
313 + _ value: Value
314 + ) throws -> (values: [Double], missing: [Bool]) {
315 + switch value {
316 + case .numeric(let values, let missing):
317 + return (values, missing)
318 + case .numericScalar(let scalar):
319 + return (
320 + [Double](repeating: scalar, count: frame.rowCount),
321 + [Bool](repeating: false, count: frame.rowCount)
322 + )
323 + case .missingScalar:
324 + return (
325 + [Double](repeating: .nan, count: frame.rowCount),
326 + [Bool](repeating: true, count: frame.rowCount)
327 + )
328 + case .strings, .stringScalar:
329 + throw EvaluationError(message: "type mismatch: expected a numeric operand")
330 + }
331 + }
332 +}
added MetrikaKit/Sources/ZQEngine/Session.swift +956 −0
@@ -0,0 +1,956 @@
1 +//
2 +// Session.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +import ZQData
12 +import ZQGPU
13 +import ZQGraphics
14 +import ZQParser
15 +import ZQPlanner
16 +import ZQPlugins
17 +import ZQStats
18 +
19 +public struct ZQEngineError: Error, Equatable, Sendable, CustomStringConvertible {
20 + public var message: String
21 + public init(_ message: String) { self.message = message }
22 + public var description: String { message }
23 +}
24 +
25 +/// One interactive session: working dataset, RNG state, panel settings,
26 +/// logging. Every command mutation — console, do-file, or plugin — goes
27 +/// through `execute(_:)` so all frontends share one execution path
28 +/// (CLAUDE.md §7).
29 +public actor ZQSession {
30 + public private(set) var frame: ZQDataFrame
31 + public private(set) var lastPlot: ZQPlotSpec?
32 + public private(set) var seed: UInt64 = 123_456_789
33 + public private(set) var panelVariable: String?
34 + public private(set) var timeVariable: String?
35 +
36 + private let store: ZQDataStore
37 + private let parser = ZQCommandParser()
38 + private let planner = ZQPlanner()
39 + private let scriptCommands: [String: ZQScriptCommand]
40 + private var logFileURL: URL?
41 + private var scriptDepth = 0
42 +
43 + public init(discoverUserCommands: Bool = true) throws {
44 + self.store = try ZQDataStore()
45 + self.frame = try ZQDataFrame()
46 + if discoverUserCommands {
47 + var commands: [String: ZQScriptCommand] = [:]
48 + for script in ZQPluginDiscovery.scriptCommands() {
49 + commands[script.verb] = script
50 + }
51 + self.scriptCommands = commands
52 + } else {
53 + self.scriptCommands = [:]
54 + }
55 + }
56 +
57 + // MARK: - Entry point
58 +
59 + public func execute(_ line: String) async throws -> ZQResult {
60 + let result: ZQResult
61 + do {
62 + result = try await run(line)
63 + } catch {
64 + try? appendToLog(command: line, output: "error: \(error)")
65 + throw error
66 + }
67 + try? appendToLog(command: line, output: result.text)
68 + return result
69 + }
70 +
71 + /// Executes a do-file: runs each line, concatenating output. Stops at
72 + /// the first error, reporting the failing line number.
73 + public func executeScript(_ text: String) async throws -> ZQResult {
74 + var outputs: [String] = []
75 + var scalars: [String: Double] = [:]
76 + for (index, rawLine) in text.split(
77 + separator: "\n", omittingEmptySubsequences: false
78 + ).enumerated() {
79 + let line = String(rawLine)
80 + do {
81 + let result = try await execute(line)
82 + if !result.text.isEmpty {
83 + outputs.append(". \(line)\n\(result.text)")
84 + }
85 + scalars.merge(result.scalars) { _, new in new }
86 + } catch {
87 + throw ZQEngineError("line \(index + 1): \(error)")
88 + }
89 + }
90 + return ZQResult(text: outputs.joined(separator: "\n\n"), scalars: scalars)
91 + }
92 +
93 + private func run(_ line: String) async throws -> ZQResult {
94 + guard let command = try parser.parse(line) else {
95 + return ZQResult(text: "")
96 + }
97 +
98 + // User script commands shadow nothing built-in; check after parse
99 + // failure would be better UX, but the verb table already resolved.
100 + let plan = planner.plan(command, rowCount: frame.rowCount)
101 + return try await dispatch(plan.command)
102 + }
103 +
104 + private func dispatch(_ command: ZQCommand) async throws -> ZQResult {
105 + if let script = scriptCommands[command.verb] {
106 + guard scriptDepth < 8 else {
107 + throw ZQEngineError("user command recursion too deep")
108 + }
109 + scriptDepth += 1
110 + defer { scriptDepth -= 1 }
111 + return try await executeScript(script.lines.joined(separator: "\n"))
112 + }
113 +
114 + switch command.verb {
115 + case "use": return try await handleUse(command)
116 + case "save": return try await handleSave(command)
117 + case "clear": return handleClear()
118 + case "describe": return handleDescribe()
119 + case "summarize": return try handleSummarize(command)
120 + case "generate": return try handleGenerate(command, replace: false)
121 + case "replace": return try handleGenerate(command, replace: true)
122 + case "drop": return try handleDropKeep(command, keep: false)
123 + case "keep": return try handleDropKeep(command, keep: true)
124 + case "count": return try handleCount(command)
125 + case "list": return try handleList(command)
126 + case "regress": return try handleRegress(command)
127 + case "set": return try handleSet(command)
128 + case "xtset": return try handleXTSet(command)
129 + case "display": return try handleDisplay(command)
130 + case "bootstrap": return try await handleBootstrap(command)
131 + case "graph": return try handleGraph(command)
132 + case "log": return try handleLog(command)
133 + default:
134 + throw ZQEngineError("command '\(command.verb)' is not implemented yet")
135 + }
136 + }
137 +
138 + // MARK: - Data commands
139 +
140 + private func handleUse(_ command: ZQCommand) async throws -> ZQResult {
141 + guard let path = command.argument else {
142 + throw ZQEngineError("use: file name required")
143 + }
144 + let url = URL(fileURLWithPath: (path as NSString).expandingTildeInPath)
145 + frame = try await store.load(contentsOf: url)
146 + return ZQResult(
147 + text: "(\(frame.rowCount) observations, \(frame.columns.count) variables loaded from \(url.lastPathComponent))",
148 + scalars: ["N": Double(frame.rowCount), "k": Double(frame.columns.count)]
149 + )
150 + }
151 +
152 + private func handleSave(_ command: ZQCommand) async throws -> ZQResult {
153 + guard let path = command.argument else {
154 + throw ZQEngineError("save: file name required")
155 + }
156 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
157 + let url = URL(fileURLWithPath: (path as NSString).expandingTildeInPath)
158 + if FileManager.default.fileExists(atPath: url.path),
159 + !command.hasOption("replace") {
160 + throw ZQEngineError("file \(url.lastPathComponent) already exists — add ', replace' to overwrite")
161 + }
162 + try await store.save(frame, to: url)
163 + return ZQResult(text: "file \(url.lastPathComponent) saved")
164 + }
165 +
166 + private func handleClear() -> ZQResult {
167 + frame = try! ZQDataFrame()
168 + panelVariable = nil
169 + timeVariable = nil
170 + return ZQResult(text: "")
171 + }
172 +
173 + private func handleDescribe() -> ZQResult {
174 + guard !frame.isEmpty else {
175 + return ZQResult(text: "Contains data: 0 observations")
176 + }
177 + var lines = [
178 + "Contains data: \(frame.rowCount) observations, \(frame.columns.count) variables",
179 + "",
180 + TableFormatter.pad("Variable", 12) + " " +
181 + TableFormatter.pad("Type", 8, right: false) + " Missing",
182 + String(repeating: "-", count: 34),
183 + ]
184 + for column in frame.columns {
185 + let type = column.data.isNumeric ? "float64" : "string"
186 + lines.append(
187 + TableFormatter.pad(column.name, 12) + " " +
188 + TableFormatter.pad(type, 8, right: false) + " " +
189 + TableFormatter.pad("\(column.data.missingCount)", 7)
190 + )
191 + }
192 + return ZQResult(
193 + text: lines.joined(separator: "\n"),
194 + scalars: ["N": Double(frame.rowCount), "k": Double(frame.columns.count)]
195 + )
196 + }
197 +
198 + private func handleSummarize(_ command: ZQCommand) throws -> ZQResult {
199 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
200 + let mask = try observationMask(command)
201 + let detail = command.hasOption("detail")
202 +
203 + var names: [String] = command.varlist.flatMap(\.referencedNames)
204 + if names.isEmpty {
205 + names = frame.columns.filter(\.data.isNumeric).map(\.name)
206 + }
207 +
208 + var lines = [
209 + TableFormatter.pad("Variable", 12) + " | " +
210 + TableFormatter.pad("Obs", 9) + " " +
211 + TableFormatter.pad("Mean", 11) + " " +
212 + TableFormatter.pad("Std. dev.", 11) + " " +
213 + TableFormatter.pad("Min", 10) + " " +
214 + TableFormatter.pad("Max", 10),
215 + String(repeating: "-", count: 13) + "+" + String(repeating: "-", count: 62),
216 + ]
217 + var scalars: [String: Double] = [:]
218 +
219 + for name in names {
220 + let (values, missing) = try frame.requireNumeric(name)
221 + let restrictedMissing = (0..<values.count).map { missing[$0] || !mask[$0] }
222 + let summary = ZQSummarize.summary(
223 + name: name, values: values, missing: restrictedMissing, detail: detail
224 + )
225 + if let summary {
226 + lines.append(
227 + TableFormatter.pad(name, 12) + " | " +
228 + TableFormatter.pad("\(summary.observationCount)", 9) + " " +
229 + TableFormatter.pad(TableFormatter.general(summary.mean), 11) + " " +
230 + TableFormatter.pad(TableFormatter.general(summary.standardDeviation), 11) + " " +
231 + TableFormatter.pad(TableFormatter.general(summary.minimum), 10) + " " +
232 + TableFormatter.pad(TableFormatter.general(summary.maximum), 10)
233 + )
234 + if let detailBlock = summary.detail {
235 + for p in [1, 5, 10, 25, 50, 75, 90, 95, 99] {
236 + lines.append(
237 + TableFormatter.pad("p\(p)", 16) + " " +
238 + TableFormatter.pad(
239 + TableFormatter.general(detailBlock.percentiles[p] ?? .nan), 11
240 + )
241 + )
242 + }
243 + lines.append(
244 + TableFormatter.pad("skewness", 16) + " " +
245 + TableFormatter.pad(TableFormatter.general(detailBlock.skewness), 11)
246 + )
247 + lines.append(
248 + TableFormatter.pad("kurtosis", 16) + " " +
249 + TableFormatter.pad(TableFormatter.general(detailBlock.kurtosis), 11)
250 + )
251 + }
252 + scalars["N"] = Double(summary.observationCount)
253 + scalars["mean"] = summary.mean
254 + scalars["sd"] = summary.standardDeviation
255 + scalars["min"] = summary.minimum
256 + scalars["max"] = summary.maximum
257 + scalars["Var"] = summary.variance
258 + } else {
259 + lines.append(
260 + TableFormatter.pad(name, 12) + " | " +
261 + TableFormatter.pad("0", 9)
262 + )
263 + }
264 + }
265 + return ZQResult(text: lines.joined(separator: "\n"), scalars: scalars)
266 + }
267 +
268 + private func handleGenerate(_ command: ZQCommand, replace: Bool) throws -> ZQResult {
269 + guard let assignment = command.assignment else {
270 + throw ZQEngineError("\(replace ? "replace" : "generate"): syntax is 'newvar = expression'")
271 + }
272 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
273 +
274 + let evaluator = ExpressionEvaluator(frame: frame)
275 + var (values, missing) = try evaluator.evaluateNumericColumn(assignment.expression)
276 + let mask = try observationMask(command)
277 +
278 + if replace {
279 + let (oldValues, oldMissing) = try frame.requireNumeric(assignment.target)
280 + var changed = 0
281 + for i in 0..<frame.rowCount {
282 + if mask[i] {
283 + let differs = oldMissing[i] != missing[i]
284 + || (!missing[i] && oldValues[i] != values[i])
285 + if differs { changed += 1 }
286 + } else {
287 + values[i] = oldValues[i]
288 + missing[i] = oldMissing[i]
289 + }
290 + }
291 + try frame.replaceColumn(ZQColumn(
292 + name: assignment.target,
293 + data: .float64(values: values, missing: missing)
294 + ))
295 + return ZQResult(text: "(\(changed) real changes made)")
296 + } else {
297 + var missingCount = 0
298 + for i in 0..<frame.rowCount where !mask[i] {
299 + values[i] = .nan
300 + missing[i] = true
301 + }
302 + for flag in missing where flag { missingCount += 1 }
303 + try frame.addColumn(ZQColumn(
304 + name: assignment.target,
305 + data: .float64(values: values, missing: missing)
306 + ))
307 + let note = missingCount > 0 ? "(\(missingCount) missing values generated)" : ""
308 + return ZQResult(text: note)
309 + }
310 + }
311 +
312 + private func handleDropKeep(_ command: ZQCommand, keep: Bool) throws -> ZQResult {
313 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
314 +
315 + // Observation form: `drop if expr` / `keep if expr` / `keep in 1/10`.
316 + if command.varlist.isEmpty, command.condition != nil || command.range != nil {
317 + let mask = try observationMask(command)
318 + let keepMask = keep ? mask : mask.map { !$0 }
319 + let before = frame.rowCount
320 + frame = try frame.filtered(by: keepMask)
321 + return ZQResult(text: "(\(before - frame.rowCount) observations deleted)")
322 + }
323 +
324 + // Variable form: `drop x y` / `keep x y`.
325 + let names = command.varlist.flatMap(\.referencedNames)
326 + guard !names.isEmpty else {
327 + throw ZQEngineError("\(keep ? "keep" : "drop"): specify variables or an 'if' condition")
328 + }
329 + if keep {
330 + let dropped = frame.columnNames.filter { !names.contains($0) }
331 + try frame.dropColumns(dropped)
332 + } else {
333 + try frame.dropColumns(names)
334 + }
335 + return ZQResult(text: "")
336 + }
337 +
338 + private func handleCount(_ command: ZQCommand) throws -> ZQResult {
339 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
340 + let mask = try observationMask(command)
341 + let count = mask.count { $0 }
342 + return ZQResult(text: " \(count)", scalars: ["N": Double(count)])
343 + }
344 +
345 + private func handleList(_ command: ZQCommand) throws -> ZQResult {
346 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
347 + let mask = try observationMask(command)
348 + var names = command.varlist.flatMap(\.referencedNames)
349 + if names.isEmpty { names = frame.columnNames }
350 + let columns = try names.map { try frame.requireColumn($0) }
351 +
352 + let rows = (0..<frame.rowCount).filter { mask[$0] }.prefix(50)
353 + var lines = [names.map { TableFormatter.pad($0, 12) }.joined(separator: " ")]
354 + for row in rows {
355 + let cells = columns.map { column -> String in
356 + switch column.data {
357 + case .float64(let values, let missing):
358 + return TableFormatter.pad(
359 + missing[row] ? "." : TableFormatter.general(values[row]), 12
360 + )
361 + case .string(let values):
362 + return TableFormatter.pad(values[row] ?? "", 12)
363 + }
364 + }
365 + lines.append(cells.joined(separator: " "))
366 + }
367 + return ZQResult(text: lines.joined(separator: "\n"))
368 + }
369 +
370 + // MARK: - Settings
371 +
372 + private func handleSet(_ command: ZQCommand) throws -> ZQResult {
373 + switch command.subverb {
374 + case "seed":
375 + guard let text = command.argument, let value = UInt64(text) else {
376 + throw ZQEngineError("set seed: integer seed required")
377 + }
378 + seed = value
379 + return ZQResult(text: "")
380 + default:
381 + throw ZQEngineError("set: unknown setting '\(command.subverb ?? "")'")
382 + }
383 + }
384 +
385 + private func handleXTSet(_ command: ZQCommand) throws -> ZQResult {
386 + let names = command.varlist.flatMap(\.referencedNames)
387 + guard (1...2).contains(names.count) else {
388 + throw ZQEngineError("xtset: syntax is 'xtset panelvar [timevar]'")
389 + }
390 + for name in names {
391 + _ = try frame.requireColumn(name)
392 + }
393 + panelVariable = names[0]
394 + timeVariable = names.count > 1 ? names[1] : nil
395 + var text = "Panel variable: \(names[0])"
396 + if let timeVariable { text += "\nTime variable: \(timeVariable)" }
397 + return ZQResult(text: text)
398 + }
399 +
400 + private func handleDisplay(_ command: ZQCommand) throws -> ZQResult {
401 + guard let assignment = command.assignment else {
402 + throw ZQEngineError("display: expression required")
403 + }
404 + let evaluator = ExpressionEvaluator(frame: frame)
405 + switch try evaluator.evaluate(assignment.expression) {
406 + case .numericScalar(let value):
407 + return ZQResult(text: TableFormatter.general(value, significant: 10))
408 + case .stringScalar(let value):
409 + return ZQResult(text: value)
410 + case .missingScalar:
411 + return ZQResult(text: ".")
412 + case .numeric(let values, let missing):
413 + guard let first = values.first else { return ZQResult(text: ".") }
414 + return ZQResult(
415 + text: missing[0] ? "." : TableFormatter.general(first, significant: 10)
416 + )
417 + case .strings(let values):
418 + return ZQResult(text: values.first.flatMap { $0 } ?? "")
419 + }
420 + }
421 +
422 + private func handleLog(_ command: ZQCommand) throws -> ZQResult {
423 + if command.argument == "close" || command.subverb == "close" {
424 + logFileURL = nil
425 + return ZQResult(text: "(log closed)")
426 + }
427 + guard var path = command.argument else {
428 + throw ZQEngineError("log: syntax is 'log using filename' or 'log close'")
429 + }
430 + if path == "close" {
431 + logFileURL = nil
432 + return ZQResult(text: "(log closed)")
433 + }
434 + if path.hasPrefix("using ") { path = String(path.dropFirst(6)) }
435 + let url = URL(fileURLWithPath: (path as NSString).expandingTildeInPath)
436 + if !FileManager.default.fileExists(atPath: url.path) {
437 + FileManager.default.createFile(atPath: url.path, contents: nil)
438 + }
439 + logFileURL = url
440 + return ZQResult(text: "(log started: \(url.path))")
441 + }
442 +
443 + private func appendToLog(command: String, output: String) throws {
444 + guard let logFileURL else { return }
445 + let entry = ". \(command)\n\(output)\n\n"
446 + let handle = try FileHandle(forWritingTo: logFileURL)
447 + defer { try? handle.close() }
448 + try handle.seekToEnd()
449 + try handle.write(contentsOf: Data(entry.utf8))
450 + }
451 +
452 + // MARK: - Estimation
453 +
454 + /// Assembled estimation sample after if/in restriction and listwise
455 + /// deletion (§6: explicit report line for dropped observations).
456 + struct RegressionSample {
457 + var responseName: String
458 + var y: [Double]
459 + var predictors: [(name: String, values: [Double])]
460 + var clusterLabels: [Int]?
461 + var droppedMissing: Int
462 + }
463 +
464 + private func buildRegressionSample(
465 + _ command: ZQCommand, clusterVariable: String?
466 + ) throws -> RegressionSample {
467 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
468 + guard let first = command.varlist.first else {
469 + throw ZQEngineError("\(command.verb): dependent variable required")
470 + }
471 + guard case .simple(let responseName) = first else {
472 + throw ZQEngineError("\(command.verb): dependent variable cannot use factor notation")
473 + }
474 +
475 + let mask = try observationMask(command)
476 + let (yAll, yMissing) = try frame.requireNumeric(responseName)
477 +
478 + // Collect raw regressor sources (numeric columns) for missingness.
479 + struct Source {
480 + var spec: ZQVarSpec
481 + var columns: [(name: String, values: [Double], missing: [Bool])]
482 + }
483 + var sources: [Source] = []
484 + for spec in command.varlist.dropFirst() {
485 + var columns: [(String, [Double], [Bool])] = []
486 + for name in spec.referencedNames {
487 + let (values, missing) = try frame.requireNumeric(name)
488 + columns.append((name, values, missing))
489 + }
490 + sources.append(Source(spec: spec, columns: columns))
491 + }
492 +
493 + var clusterColumn: ZQColumn?
494 + if let clusterVariable {
495 + clusterColumn = try frame.requireColumn(clusterVariable)
496 + }
497 +
498 + // Listwise deletion over response, regressors, and cluster var.
499 + var keptRows: [Int] = []
500 + var droppedMissing = 0
501 + for i in 0..<frame.rowCount where mask[i] {
502 + var complete = !yMissing[i]
503 + if complete {
504 + outer: for source in sources {
505 + for column in source.columns where column.missing[i] {
506 + complete = false
507 + break outer
508 + }
509 + }
510 + }
511 + if complete, let clusterColumn {
512 + switch clusterColumn.data {
513 + case .float64(_, let missing): complete = !missing[i]
514 + case .string(let values): complete = values[i] != nil
515 + }
516 + }
517 + if complete {
518 + keptRows.append(i)
519 + } else {
520 + droppedMissing += 1
521 + }
522 + }
523 + guard !keptRows.isEmpty else {
524 + throw ZQEngineError("no observations")
525 + }
526 +
527 + let y = keptRows.map { yAll[$0] }
528 +
529 + // Expand specs on the estimation sample (plan-time expansion).
530 + var predictors: [(name: String, values: [Double])] = []
531 + for source in sources {
532 + switch source.spec {
533 + case .simple(let name):
534 + let (values, _) = try frame.requireNumeric(name)
535 + predictors.append((name, keptRows.map { values[$0] }))
536 + case .factor(let op, let name):
537 + let (values, missing) = try frame.requireNumeric(name)
538 + switch op {
539 + case "i", "ib":
540 + let sampleValues = keptRows.map { values[$0] }
541 + let sampleMissing = keptRows.map { missing[$0] }
542 + let expanded = ZQFactorExpansion.expand(
543 + name: name, values: sampleValues, missing: sampleMissing
544 + )
545 + guard !expanded.isEmpty else {
546 + throw ZQEngineError("factor variable '\(name)' has a single level")
547 + }
548 + predictors.append(contentsOf: expanded)
549 + case "c":
550 + predictors.append((name, keptRows.map { values[$0] }))
551 + default:
552 + throw ZQEngineError("unsupported factor operator '\(op).'")
553 + }
554 + case .interaction(let parts, _):
555 + // v0.1: continuous-by-continuous interactions only.
556 + var label: [String] = []
557 + var product = [Double](repeating: 1, count: keptRows.count)
558 + for part in parts {
559 + guard case .factor(let op, let name) = part, op == "c" else {
560 + guard case .simple(let name) = part else {
561 + throw ZQEngineError("factor interactions beyond c.#c. arrive with xtreg (v0.2)")
562 + }
563 + let (values, _) = try frame.requireNumeric(name)
564 + for (j, row) in keptRows.enumerated() { product[j] *= values[row] }
565 + label.append(name)
566 + continue
567 + }
568 + let (values, _) = try frame.requireNumeric(name)
569 + for (j, row) in keptRows.enumerated() { product[j] *= values[row] }
570 + label.append("c.\(name)")
571 + }
572 + predictors.append((label.joined(separator: "#"), product))
573 + }
574 + }
575 +
576 + // Cluster labels → dense integer codes.
577 + var clusterLabels: [Int]?
578 + if let clusterColumn {
579 + var codes: [Int] = []
580 + codes.reserveCapacity(keptRows.count)
581 + switch clusterColumn.data {
582 + case .float64(let values, _):
583 + var mapping: [Double: Int] = [:]
584 + for row in keptRows {
585 + let value = values[row]
586 + if let code = mapping[value] {
587 + codes.append(code)
588 + } else {
589 + mapping[value] = mapping.count
590 + codes.append(mapping.count - 1)
591 + }
592 + }
593 + case .string(let values):
594 + var mapping: [String: Int] = [:]
595 + for row in keptRows {
596 + let value = values[row] ?? ""
597 + if let code = mapping[value] {
598 + codes.append(code)
599 + } else {
600 + mapping[value] = mapping.count
601 + codes.append(mapping.count - 1)
602 + }
603 + }
604 + }
605 + clusterLabels = codes
606 + }
607 +
608 + return RegressionSample(
609 + responseName: responseName,
610 + y: y,
611 + predictors: predictors,
612 + clusterLabels: clusterLabels,
613 + droppedMissing: droppedMissing
614 + )
615 + }
616 +
617 + private func varianceEstimator(
618 + _ command: ZQCommand, clusterLabels: [Int]?
619 + ) throws -> ZQVarianceEstimator {
620 + if let clusterLabels { return .cluster(clusterLabels) }
621 + if command.hasOption("hc2") { return .hc2 }
622 + if command.hasOption("hc3") { return .hc3 }
623 + if command.hasOption("robust") || command.hasOption("r") { return .hc1 }
624 + if let vce = command.option("vce")?.firstArgument {
625 + switch vce {
626 + case "robust": return .hc1
627 + case "hc2": return .hc2
628 + case "hc3": return .hc3
629 + default:
630 + throw ZQEngineError("vce(\(vce)) is not supported")
631 + }
632 + }
633 + return .classical
634 + }
635 +
636 + private func handleRegress(_ command: ZQCommand) throws -> ZQResult {
637 + let clusterVariable = command.option("cluster")?.firstArgument
638 + ?? clusterFromVCE(command)
639 + let sample = try buildRegressionSample(command, clusterVariable: clusterVariable)
640 + let variance = try varianceEstimator(command, clusterLabels: sample.clusterLabels)
641 + let level = try confidenceLevel(command)
642 +
643 + let result = try ZQOLS.fit(
644 + y: sample.y,
645 + predictors: sample.predictors,
646 + includeConstant: !command.hasOption("noconstant"),
647 + variance: variance,
648 + confidenceLevel: level
649 + )
650 +
651 + var header: [String] = []
652 + switch variance {
653 + case .classical: header.append("Linear regression")
654 + case .cluster: header.append("Linear regression (cluster-robust)")
655 + default: header.append("Linear regression (robust)")
656 + }
657 + if sample.droppedMissing > 0 {
658 + header.append("(\(sample.droppedMissing) observations dropped due to missing values)")
659 + }
660 +
661 + var stats = [
662 + ("Number of obs", TableFormatter.general(Double(result.observationCount))),
663 + ("R-squared", TableFormatter.fixed(result.rSquared, decimals: 4)),
664 + ("Adj R-squared", TableFormatter.fixed(result.adjustedRSquared, decimals: 4)),
665 + ("Root MSE", TableFormatter.general(result.rootMSE, significant: 6)),
666 + ]
667 + if let f = result.fStatistic, let df = result.fDF, let p = result.fPValue {
668 + stats.insert(
669 + ("F(\(Int(df.0)), \(Int(df.1)))", TableFormatter.fixed(f, decimals: 2)),
670 + at: 1
671 + )
672 + stats.insert(("Prob > F", TableFormatter.fixed(p, decimals: 4)), at: 2)
673 + }
674 + if let g = result.clusterCount {
675 + stats.append(("Clusters", "\(g)"))
676 + }
677 + for (label, value) in stats {
678 + header.append(
679 + TableFormatter.pad(label, 46, right: false) + "= " +
680 + TableFormatter.pad(value, 12)
681 + )
682 + }
683 +
684 + var lines = header
685 + lines.append("")
686 + let widths = [12, 12, 11, 8, 8, 22]
687 + lines.append(
688 + TableFormatter.pad(sample.responseName, widths[0]) + " | " +
689 + TableFormatter.pad("Coefficient", widths[1]) + " " +
690 + TableFormatter.pad("Std. err.", widths[2]) + " " +
691 + TableFormatter.pad("t", widths[3]) + " " +
692 + TableFormatter.pad("P>|t|", widths[4]) + " " +
693 + TableFormatter.pad("[\(Int(level * 100))% conf. interval]", widths[5])
694 + )
695 + lines.append(TableFormatter.rule(widths))
696 + var scalars: [String: Double] = [
697 + "N": Double(result.observationCount),
698 + "r2": result.rSquared,
699 + "r2_a": result.adjustedRSquared,
700 + "rmse": result.rootMSE,
701 + "df_r": result.inferenceDF,
702 + ]
703 + if let f = result.fStatistic { scalars["F"] = f }
704 + if let g = result.clusterCount { scalars["N_clust"] = Double(g) }
705 +
706 + for coefficient in result.coefficients {
707 + lines.append(
708 + TableFormatter.pad(coefficient.name, widths[0]) + " | " +
709 + TableFormatter.pad(TableFormatter.general(coefficient.estimate), widths[1]) + " " +
710 + TableFormatter.pad(TableFormatter.general(coefficient.standardError), widths[2]) + " " +
711 + TableFormatter.pad(TableFormatter.fixed(coefficient.tStatistic, decimals: 2), widths[3]) + " " +
712 + TableFormatter.pad(TableFormatter.fixed(coefficient.pValue, decimals: 3), widths[4]) + " " +
713 + TableFormatter.pad(
714 + TableFormatter.general(coefficient.confidenceLower) + " " +
715 + TableFormatter.general(coefficient.confidenceUpper),
716 + widths[5]
717 + )
718 + )
719 + scalars["b_\(coefficient.name)"] = coefficient.estimate
720 + scalars["se_\(coefficient.name)"] = coefficient.standardError
721 + }
722 + return ZQResult(text: lines.joined(separator: "\n"), scalars: scalars)
723 + }
724 +
725 + private func clusterFromVCE(_ command: ZQCommand) -> String? {
726 + guard let vce = command.option("vce"),
727 + vce.arguments.count == 2,
728 + vce.arguments[0] == "cluster" else { return nil }
729 + return vce.arguments[1]
730 + }
731 +
732 + private func confidenceLevel(_ command: ZQCommand) throws -> Double {
733 + guard let text = command.option("level")?.firstArgument else { return 0.95 }
734 + guard let value = Double(text), value > 0, value < 100 else {
735 + throw ZQEngineError("level() must be between 0 and 100")
736 + }
737 + return value / 100
738 + }
739 +
740 + // MARK: - Bootstrap prefix
741 +
742 + private func handleBootstrap(_ command: ZQCommand) async throws -> ZQResult {
743 + guard let body = command.body else {
744 + throw ZQEngineError("bootstrap: syntax is 'bootstrap, reps(#): command'")
745 + }
746 + guard body.verb == "regress" else {
747 + throw ZQEngineError("bootstrap currently supports 'regress' bodies only")
748 + }
749 + guard let repsText = command.option("reps")?.firstArgument,
750 + let reps = Int(repsText), reps >= 2 else {
751 + throw ZQEngineError("bootstrap: reps(#) with # ≥ 2 required")
752 + }
753 + if let seedText = command.option("seed")?.firstArgument {
754 + guard let value = UInt64(seedText) else {
755 + throw ZQEngineError("bootstrap: invalid seed")
756 + }
757 + seed = value
758 + }
759 +
760 + let sample = try buildRegressionSample(body, clusterVariable: nil)
761 + let includeConstant = !body.hasOption("noconstant")
762 + let point = try ZQOLS.fit(
763 + y: sample.y,
764 + predictors: sample.predictors,
765 + includeConstant: includeConstant,
766 + variance: .classical
767 + )
768 +
769 + let n = sample.y.count
770 + let k = point.coefficients.count
771 + let generator = Philox4x32(seed: seed)
772 + let y = sample.y
773 + let predictors = sample.predictors
774 +
775 + // Pairs bootstrap: replicate r regenerates its indices from the
776 + // Philox counter stream — deterministic and order-independent, so
777 + // replicates can be computed in parallel chunks.
778 + let chunkCount = min(
779 + max(1, ProcessInfo.processInfo.activeProcessorCount), reps
780 + )
781 + let chunkSize = (reps + chunkCount - 1) / chunkCount
782 + let draws: [[Double]] = try await withThrowingTaskGroup(
783 + of: [(Int, [Double])].self
784 + ) { group in
785 + for chunk in 0..<chunkCount {
786 + let lower = chunk * chunkSize
787 + let upper = min(reps, lower + chunkSize)
788 + guard lower < upper else { continue }
789 + group.addTask {
790 + var results: [(Int, [Double])] = []
791 + results.reserveCapacity(upper - lower)
792 + for replicate in lower..<upper {
793 + let indices = ZQResampling.pairsBootstrapIndices(
794 + replicate: replicate, sampleSize: n, generator: generator
795 + )
796 + let resampledY = indices.map { y[$0] }
797 + let resampledPredictors = predictors.map { column in
798 + (name: column.name, values: indices.map { column.values[$0] })
799 + }
800 + let fit = try ZQOLS.fit(
801 + y: resampledY,
802 + predictors: resampledPredictors,
803 + includeConstant: includeConstant,
804 + variance: .classical
805 + )
806 + results.append((replicate, fit.coefficients.map(\.estimate)))
807 + }
808 + return results
809 + }
810 + }
811 + var collected = [[Double]](repeating: [], count: reps)
812 + for try await chunkResults in group {
813 + for (replicate, estimates) in chunkResults {
814 + collected[replicate] = estimates
815 + }
816 + }
817 + return collected
818 + }
819 +
820 + // Bootstrap standard errors: sd of replicate estimates.
821 + var lines = [
822 + "Bootstrap results (pairs) Replications = " +
823 + TableFormatter.pad("\(reps)", 10),
824 + " Number of obs = " +
825 + TableFormatter.pad("\(n)", 10),
826 + "",
827 + ]
828 + let widths = [12, 12, 12, 8, 8, 22]
829 + lines.append(
830 + TableFormatter.pad(sample.responseName, widths[0]) + " | " +
831 + TableFormatter.pad("Observed", widths[1]) + " " +
832 + TableFormatter.pad("Bootstrap", widths[2]) + " " +
833 + TableFormatter.pad("z", widths[3]) + " " +
834 + TableFormatter.pad("P>|z|", widths[4]) + " " +
835 + TableFormatter.pad("[95% conf. interval]", widths[5])
836 + )
837 + lines.append(
838 + TableFormatter.pad("", widths[0]) + " | " +
839 + TableFormatter.pad("coefficient", widths[1]) + " " +
840 + TableFormatter.pad("std. err.", widths[2])
841 + )
842 + lines.append(TableFormatter.rule(widths))
843 +
844 + var scalars: [String: Double] = ["reps": Double(reps), "N": Double(n)]
845 + for j in 0..<k {
846 + let estimates = draws.map { $0[j] }
847 + let mean = estimates.reduce(0, +) / Double(reps)
848 + let variance = estimates.reduce(0) { $0 + ($1 - mean) * ($1 - mean) }
849 + / Double(reps - 1)
850 + let se = variance.squareRoot()
851 + let observed = point.coefficients[j].estimate
852 + let z = observed / se
853 + let p = 2 * (1 - ZQDistributions.normalCDF(abs(z)))
854 + let critical = 1.959963984540054 // Φ⁻¹(0.975)
855 + lines.append(
856 + TableFormatter.pad(point.coefficients[j].name, widths[0]) + " | " +
857 + TableFormatter.pad(TableFormatter.general(observed), widths[1]) + " " +
858 + TableFormatter.pad(TableFormatter.general(se), widths[2]) + " " +
859 + TableFormatter.pad(TableFormatter.fixed(z, decimals: 2), widths[3]) + " " +
860 + TableFormatter.pad(TableFormatter.fixed(p, decimals: 3), widths[4]) + " " +
861 + TableFormatter.pad(
862 + TableFormatter.general(observed - critical * se) + " " +
863 + TableFormatter.general(observed + critical * se),
864 + widths[5]
865 + )
866 + )
867 + scalars["b_\(point.coefficients[j].name)"] = observed
868 + scalars["se_\(point.coefficients[j].name)"] = se
869 + }
870 + return ZQResult(text: lines.joined(separator: "\n"), scalars: scalars)
871 + }
872 +
873 + // MARK: - Graphics
874 +
875 + private func handleGraph(_ command: ZQCommand) throws -> ZQResult {
876 + guard !frame.isEmpty else { throw ZQEngineError("no data in memory") }
877 + let kind: ZQPlotSpec.Kind
878 + switch command.subverb {
879 + case "scatter": kind = .scatter
880 + case "line": kind = .line
881 + case "histogram": kind = .histogram
882 + case let other:
883 + throw ZQEngineError("graph \(other ?? ""): not supported yet")
884 + }
885 +
886 + let names = command.varlist.flatMap(\.referencedNames)
887 + guard names.count == 2 else {
888 + throw ZQEngineError("graph \(command.subverb ?? ""): syntax is 'graph \(command.subverb ?? "kind") yvar xvar'")
889 + }
890 + let mask = try observationMask(command)
891 + let (yValues, yMissing) = try frame.requireNumeric(names[0])
892 + let (xValues, xMissing) = try frame.requireNumeric(names[1])
893 +
894 + func series(label: String, rows: [Int]) -> ZQPlotSpec.Series {
895 + ZQPlotSpec.Series(
896 + label: label,
897 + x: rows.map { xValues[$0] },
898 + y: rows.map { yValues[$0] }
899 + )
900 + }
901 +
902 + let complete = (0..<frame.rowCount).filter {
903 + mask[$0] && !yMissing[$0] && !xMissing[$0]
904 + }
905 +
906 + var allSeries: [ZQPlotSpec.Series] = []
907 + if let byName = command.option("by")?.firstArgument {
908 + let (byValues, byMissing) = try frame.requireNumeric(byName)
909 + let levels = ZQFactorExpansion.levels(
910 + values: complete.map { byValues[$0] },
911 + missing: complete.map { byMissing[$0] }
912 + )
913 + for level in levels {
914 + let rows = complete.filter { !byMissing[$0] && byValues[$0] == level }
915 + let label = level == level.rounded()
916 + ? "\(byName)=\(Int(level))"
917 + : "\(byName)=\(level)"
918 + allSeries.append(series(label: label, rows: rows))
919 + }
920 + } else {
921 + allSeries.append(series(label: names[0], rows: complete))
922 + }
923 +
924 + lastPlot = ZQPlotSpec(
925 + kind: kind,
926 + xLabel: names[1],
927 + yLabel: names[0],
928 + series: allSeries
929 + )
930 + let pointCount = allSeries.reduce(0) { $0 + $1.x.count }
931 + return ZQResult(text: "(plot created: \(pointCount) points)")
932 + }
933 +
934 + // MARK: - Qualifiers
935 +
936 + /// Combined if/in keep-mask over current observations.
937 + private func observationMask(_ command: ZQCommand) throws -> [Bool] {
938 + var mask = [Bool](repeating: true, count: frame.rowCount)
939 + if let condition = command.condition {
940 + let evaluator = ExpressionEvaluator(frame: frame)
941 + mask = try evaluator.evaluateCondition(condition)
942 + }
943 + if let range = command.range {
944 + let n = frame.rowCount
945 + func resolve(_ bound: Int) -> Int {
946 + bound >= 0 ? bound : n + bound + 1
947 + }
948 + let lower = max(1, resolve(range.lower))
949 + let upper = min(n, resolve(range.upper))
950 + for i in 0..<n where i + 1 < lower || i + 1 > upper {
951 + mask[i] = false
952 + }
953 + }
954 + return mask
955 + }
956 +}
added MetrikaKit/Sources/ZQEngine/TableFormatter.swift +48 −0
@@ -0,0 +1,48 @@
1 +//
2 +// TableFormatter.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +
12 +/// Console table rendering: monospaced, right-aligned numeric columns in
13 +/// the Stata output tradition.
14 +enum TableFormatter {
15 +
16 + /// Stata-flavored general format: up to `significant` significant
17 + /// digits, no trailing zeros, leading "0." collapsed to ".".
18 + static func general(_ value: Double, significant: Int = 7) -> String {
19 + if value.isNaN { return "." }
20 + if value == 0 { return "0" }
21 + var text = String(format: "%.\(significant)g", value)
22 + if text.contains("."), !text.contains("e"), !text.contains("E") {
23 + while text.hasSuffix("0") { text.removeLast() }
24 + if text.hasSuffix(".") { text.removeLast() }
25 + }
26 + if text.hasPrefix("0.") { text.removeFirst() }
27 + if text.hasPrefix("-0.") { text = "-" + text.dropFirst(2) }
28 + return text
29 + }
30 +
31 + static func fixed(_ value: Double, decimals: Int) -> String {
32 + value.isNaN ? "." : String(format: "%.\(decimals)f", value)
33 + }
34 +
35 + static func pad(_ text: String, _ width: Int, right: Bool = true) -> String {
36 + if text.count >= width { return text }
37 + let padding = String(repeating: " ", count: width - text.count)
38 + return right ? padding + text : text + padding
39 + }
40 +
41 + /// Renders rows of cells with per-column widths; the first column is
42 + /// left-padded to its width and separated by " | ".
43 + static func rule(_ widths: [Int]) -> String {
44 + let first = String(repeating: "-", count: widths[0] + 1)
45 + let rest = widths.dropFirst().reduce(0) { $0 + $1 } + (widths.count - 1) * 2
46 + return first + "+" + String(repeating: "-", count: rest + 1)
47 + }
48 +}
added MetrikaKit/Sources/ZQEngine/ZQEngineModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQEngineModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQEngine.
11 +public enum ZQEngineModule {
12 + public static let name = "ZQEngine"
13 +}
added MetrikaKit/Sources/ZQGPU/Philox.swift +101 −0
@@ -0,0 +1,101 @@
1 +//
2 +// Philox.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Philox4x32-10 counter-based random number generator (Salmon et al.,
11 +/// SC'11). Every 128-bit counter maps to four independent 32-bit outputs,
12 +/// so draws are addressable by index — the property that makes CPU and
13 +/// GPU streams identical for the same seed (CLAUDE.md §5: reproducibility
14 +/// is a release blocker). This pure-Swift implementation is the reference
15 +/// the MLX kernel path must match bit-for-bit.
16 +public struct Philox4x32: Sendable {
17 + private static let multiplier0: UInt32 = 0xD251_1F53
18 + private static let multiplier1: UInt32 = 0xCD9E_8D57
19 + private static let weyl0: UInt32 = 0x9E37_79B9
20 + private static let weyl1: UInt32 = 0xBB67_AE85
21 + private static let rounds = 10
22 +
23 + public let key: (UInt32, UInt32)
24 +
25 + /// `set seed` maps the 64-bit seed directly onto the Philox key.
26 + public init(seed: UInt64) {
27 + self.key = (UInt32(truncatingIfNeeded: seed),
28 + UInt32(truncatingIfNeeded: seed >> 32))
29 + }
30 +
31 + /// One Philox block: 128-bit counter → four 32-bit words.
32 + public func block(counter: (UInt32, UInt32, UInt32, UInt32))
33 + -> (UInt32, UInt32, UInt32, UInt32) {
34 + var c = counter
35 + var k = key
36 + for _ in 0..<Self.rounds {
37 + let product0 = UInt64(Self.multiplier0) * UInt64(c.0)
38 + let product1 = UInt64(Self.multiplier1) * UInt64(c.2)
39 + let high0 = UInt32(truncatingIfNeeded: product0 >> 32)
40 + let low0 = UInt32(truncatingIfNeeded: product0)
41 + let high1 = UInt32(truncatingIfNeeded: product1 >> 32)
42 + let low1 = UInt32(truncatingIfNeeded: product1)
43 + c = (high1 ^ c.1 ^ k.0, low1, high0 ^ c.3 ^ k.1, low0)
44 + k.0 = k.0 &+ Self.weyl0
45 + k.1 = k.1 &+ Self.weyl1
46 + }
47 + return c
48 + }
49 +
50 + /// The i-th 32-bit word of the stream (block = i/4, lane = i%4).
51 + public func word(at index: UInt64) -> UInt32 {
52 + let blockIndex = index / 4
53 + let lane = Int(index % 4)
54 + let output = block(counter: (
55 + UInt32(truncatingIfNeeded: blockIndex),
56 + UInt32(truncatingIfNeeded: blockIndex >> 32),
57 + 0, 0
58 + ))
59 + switch lane {
60 + case 0: return output.0
61 + case 1: return output.1
62 + case 2: return output.2
63 + default: return output.3
64 + }
65 + }
66 +
67 + /// Uniform Double in [0, 1) from the i-th draw, using 53 bits built
68 + /// from two consecutive 32-bit words.
69 + public func uniform(at index: UInt64) -> Double {
70 + let high = UInt64(word(at: index &* 2))
71 + let low = UInt64(word(at: index &* 2 &+ 1))
72 + let bits53 = ((high << 32) | low) >> 11
73 + return Double(bits53) * (1.0 / 9_007_199_254_740_992.0) // 2^-53
74 + }
75 +
76 + /// Uniform integer in 0..<bound from the i-th draw (used for
77 + /// bootstrap resampling indices). Bounded by 64-bit multiply-shift;
78 + /// the tiny modulo bias (< 2⁻³² for realistic n) is acceptable and,
79 + /// critically, is computed identically on CPU and GPU.
80 + public func integer(at index: UInt64, bound: Int) -> Int {
81 + precondition(bound > 0)
82 + let word = UInt64(self.word(at: index))
83 + return Int((word &* UInt64(bound)) >> 32)
84 + }
85 +}
86 +
87 +/// Bootstrap resampling indices: replicate r draws its n indices from
88 +/// dedicated counter positions, so any subset of replicates can be
89 +/// regenerated independently — on either backend.
90 +public enum ZQResampling {
91 + public static func pairsBootstrapIndices(
92 + replicate: Int, sampleSize: Int, generator: Philox4x32
93 + ) -> [Int] {
94 + let base = UInt64(replicate) &* UInt64(sampleSize)
95 + var indices = [Int](repeating: 0, count: sampleSize)
96 + for i in 0..<sampleSize {
97 + indices[i] = generator.integer(at: base &+ UInt64(i), bound: sampleSize)
98 + }
99 + return indices
100 + }
101 +}
added MetrikaKit/Sources/ZQGPU/ZQGPUModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQGPUModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQGPU.
11 +public enum ZQGPUModule {
12 + public static let name = "ZQGPU"
13 +}
added MetrikaKit/Sources/ZQGraphics/PlotSpec.swift +43 −0
@@ -0,0 +1,43 @@
1 +//
2 +// PlotSpec.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import ZQData
11 +
12 +/// Backend-neutral plot description produced by `graph` commands. The app
13 +/// layer renders it with Swift Charts, or the Metal point renderer once
14 +/// the series crosses ~1M points (CLAUDE.md §7).
15 +public struct ZQPlotSpec: Equatable, Sendable {
16 + public enum Kind: String, Equatable, Sendable {
17 + case scatter, line, histogram, kdensity
18 + }
19 +
20 + public struct Series: Equatable, Sendable {
21 + public var label: String
22 + public var x: [Double]
23 + public var y: [Double]
24 +
25 + public init(label: String, x: [Double], y: [Double]) {
26 + self.label = label
27 + self.x = x
28 + self.y = y
29 + }
30 + }
31 +
32 + public var kind: Kind
33 + public var xLabel: String
34 + public var yLabel: String
35 + public var series: [Series]
36 +
37 + public init(kind: Kind, xLabel: String, yLabel: String, series: [Series]) {
38 + self.kind = kind
39 + self.xLabel = xLabel
40 + self.yLabel = yLabel
41 + self.series = series
42 + }
43 +}
added MetrikaKit/Sources/ZQGraphics/ZQGraphicsModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQGraphicsModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQGraphics.
11 +public enum ZQGraphicsModule {
12 + public static let name = "ZQGraphics"
13 +}
added MetrikaKit/Sources/ZQParser/AST.swift +167 −0
@@ -0,0 +1,167 @@
1 +//
2 +// AST.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// A variable specification appearing in a varlist. Factor notation is
11 +/// preserved verbatim; expansion into indicator columns happens at plan
12 +/// time (CLAUDE.md §4), never at parse time.
13 +public indirect enum ZQVarSpec: Equatable, Sendable {
14 + /// Plain variable reference: `price`
15 + case simple(String)
16 + /// Factor-operator variable: `i.region`, `c.age`, `ib2.group`
17 + case factor(op: String, name: String)
18 + /// Interaction: `c.age#c.age`, `i.region#c.price`
19 + case interaction([ZQVarSpec], full: Bool)
20 +
21 + /// The underlying dataset column names this spec references.
22 + public var referencedNames: [String] {
23 + switch self {
24 + case .simple(let n): return [n]
25 + case .factor(_, let n): return [n]
26 + case .interaction(let parts, _): return parts.flatMap(\.referencedNames)
27 + }
28 + }
29 +}
30 +
31 +/// Expression AST used for `if` qualifiers, `gen`/`replace` right-hand
32 +/// sides, and option arguments.
33 +public indirect enum ZQExpression: Equatable, Sendable {
34 + case number(Double)
35 + case string(String)
36 + case variable(String)
37 + /// Stata-style missing-value literal `.`
38 + case missing
39 + case unary(op: String, ZQExpression)
40 + case binary(op: String, ZQExpression, ZQExpression)
41 + case call(name: String, arguments: [ZQExpression])
42 +}
43 +
44 +/// Observation range qualifier: `in 5/200`. Negative values count from the
45 +/// end (`in -10/-1`), 1-based like Stata.
46 +public struct ZQRange: Equatable, Sendable {
47 + public var lower: Int
48 + public var upper: Int
49 +
50 + public init(lower: Int, upper: Int) {
51 + self.lower = lower
52 + self.upper = upper
53 + }
54 +}
55 +
56 +/// Weight qualifier: `[aweight = expr]`.
57 +public struct ZQWeight: Equatable, Sendable {
58 + public enum Kind: String, Equatable, Sendable {
59 + case aweight, fweight, pweight, iweight
60 + }
61 +
62 + public var kind: Kind
63 + public var expression: ZQExpression
64 +
65 + public init(kind: Kind, expression: ZQExpression) {
66 + self.kind = kind
67 + self.expression = expression
68 + }
69 +}
70 +
71 +/// A trailing option: bare flag (`robust`, `detail`) or parenthesized
72 +/// (`cluster(id)`, `reps(100000)`, `by(region year)`).
73 +public struct ZQOption: Equatable, Sendable {
74 + public var name: String
75 + /// Raw argument tokens re-rendered as strings; interpretation is
76 + /// command-specific and happens at plan/execution time.
77 + public var arguments: [String]
78 +
79 + public init(name: String, arguments: [String] = []) {
80 + self.name = name
81 + self.arguments = arguments
82 + }
83 +
84 + public var firstArgument: String? { arguments.first }
85 +}
86 +
87 +/// Assignment payload for `gen`/`replace`-style commands:
88 +/// `gen log_rev = ln(revenue)`.
89 +public struct ZQAssignment: Equatable, Sendable {
90 + public var target: String
91 + public var expression: ZQExpression
92 +
93 + public init(target: String, expression: ZQExpression) {
94 + self.target = target
95 + self.expression = expression
96 + }
97 +}
98 +
99 +/// A fully parsed ZQL command:
100 +/// `command [varlist] [if expr] [in range] [weight] [, options]`,
101 +/// optionally wrapped by a prefix command (`bootstrap, reps(1000): reg …`).
102 +public struct ZQCommand: Equatable, Sendable {
103 + /// Canonical verb (abbreviations already resolved: `reg` → `regress`).
104 + public var verb: String
105 + /// Sub-verb for compound commands (`graph scatter …` → "scatter").
106 + public var subverb: String?
107 + public var varlist: [ZQVarSpec]
108 + /// `gen`/`replace` assignment, mutually exclusive with a plain varlist.
109 + public var assignment: ZQAssignment?
110 + /// Raw argument such as a file path (`use sales.parquet`).
111 + public var argument: String?
112 + public var condition: ZQExpression?
113 + public var range: ZQRange?
114 + public var weight: ZQWeight?
115 + public var options: [ZQOption]
116 + /// Inner command of a prefix construct: for
117 + /// `bootstrap, reps(1000): reg y x`, the outer command is `bootstrap`
118 + /// and `body` holds the parsed `regress` command.
119 + public var body: ZQCommand? {
120 + get { bodyBox?.value }
121 + set { bodyBox = newValue.map(Box.init) }
122 + }
123 +
124 + private var bodyBox: Box<ZQCommand>?
125 +
126 + public init(
127 + verb: String,
128 + subverb: String? = nil,
129 + varlist: [ZQVarSpec] = [],
130 + assignment: ZQAssignment? = nil,
131 + argument: String? = nil,
132 + condition: ZQExpression? = nil,
133 + range: ZQRange? = nil,
134 + weight: ZQWeight? = nil,
135 + options: [ZQOption] = [],
136 + body: ZQCommand? = nil
137 + ) {
138 + self.verb = verb
139 + self.subverb = subverb
140 + self.varlist = varlist
141 + self.assignment = assignment
142 + self.argument = argument
143 + self.condition = condition
144 + self.range = range
145 + self.weight = weight
146 + self.options = options
147 + self.bodyBox = body.map(Box.init)
148 + }
149 +
150 + public func option(_ name: String) -> ZQOption? {
151 + options.first { $0.name == name }
152 + }
153 +
154 + public func hasOption(_ name: String) -> Bool {
155 + option(name) != nil
156 + }
157 +}
158 +
159 +/// Reference box that lets `ZQCommand` nest itself as a prefix body.
160 +private final class Box<T>: @unchecked Sendable {
161 + let value: T
162 + init(_ value: T) { self.value = value }
163 +}
164 +
165 +extension Box: Equatable where T: Equatable {
166 + static func == (lhs: Box<T>, rhs: Box<T>) -> Bool { lhs.value == rhs.value }
167 +}
added MetrikaKit/Sources/ZQParser/Grammar/CommandParser.swift +480 −0
@@ -0,0 +1,480 @@
1 +//
2 +// CommandParser.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +
12 +/// Parser for one ZQL command line:
13 +///
14 +/// command [varlist] [if expr] [in range] [weight] [, options]
15 +///
16 +/// Prefix commands (`bootstrap, reps(1000): reg y x`) recurse on the text
17 +/// after the colon. File-taking verbs (`use`, `save`, …) treat everything
18 +/// before the first top-level comma as a raw path.
19 +public struct ZQCommandParser: Sendable {
20 + public let verbTable: ZQVerbTable
21 +
22 + public init(verbTable: ZQVerbTable = .builtin) {
23 + self.verbTable = verbTable
24 + }
25 +
26 + /// Parses a single line. Returns nil for blank lines and comments.
27 + public func parse(_ line: String) throws(ZQParseError) -> ZQCommand? {
28 + let trimmed = line.trimmingCharacters(in: .whitespaces)
29 + if trimmed.isEmpty || trimmed.hasPrefix("//") || trimmed.hasPrefix("*") {
30 + return nil
31 + }
32 +
33 + // Extract the verb word to decide the parsing strategy.
34 + let verbWord = String(trimmed.prefix { !$0.isWhitespace && $0 != "," })
35 + let verbColumn = columnOf(word: verbWord, in: line)
36 + guard let verb = verbTable.resolve(verbWord.lowercased()) else {
37 + throw ZQParseError(
38 + message: "unknown command '\(verbWord)'",
39 + column: verbColumn,
40 + suggestion: verbTable.closest(to: verbWord.lowercased())
41 + )
42 + }
43 +
44 + if verbTable.prefixVerbs.contains(verb) {
45 + return try parsePrefixCommand(verb: verb, line: line, trimmed: trimmed)
46 + }
47 + if verbTable.fileVerbs.contains(verb) {
48 + return try parseFileCommand(verb: verb, trimmed: trimmed)
49 + }
50 + return try parseStandardCommand(verb: verb, line: line, trimmed: trimmed)
51 + }
52 +
53 + // MARK: - Prefix commands
54 +
55 + private func parsePrefixCommand(
56 + verb: String, line: String, trimmed: String
57 + ) throws(ZQParseError) -> ZQCommand {
58 + // Split on the first top-level colon: `bootstrap, reps(1000): reg y x`
59 + guard let colonIndex = topLevelColonIndex(in: trimmed) else {
60 + throw ZQParseError(
61 + message: "prefix command '\(verb)' requires ': command'",
62 + column: trimmed.count + 1
63 + )
64 + }
65 + let head = String(trimmed[trimmed.startIndex..<colonIndex])
66 + let tail = String(trimmed[trimmed.index(after: colonIndex)...])
67 + .trimmingCharacters(in: .whitespaces)
68 +
69 + var command = try parseStandardCommand(verb: verb, line: head, trimmed: head)
70 + guard let body = try parse(tail) else {
71 + throw ZQParseError(
72 + message: "prefix command '\(verb)' has an empty body",
73 + column: trimmed.count + 1
74 + )
75 + }
76 + command.body = body
77 + return command
78 + }
79 +
80 + /// Index of the first colon that is not nested inside parentheses,
81 + /// brackets, or a string literal.
82 + private func topLevelColonIndex(in text: String) -> String.Index? {
83 + var depth = 0
84 + var inString = false
85 + var index = text.startIndex
86 + while index < text.endIndex {
87 + let c = text[index]
88 + if c == "\"" { inString.toggle() }
89 + if !inString {
90 + switch c {
91 + case "(", "[": depth += 1
92 + case ")", "]": depth -= 1
93 + case ":" where depth == 0: return index
94 + default: break
95 + }
96 + }
97 + index = text.index(after: index)
98 + }
99 + return nil
100 + }
101 +
102 + // MARK: - File commands
103 +
104 + private func parseFileCommand(
105 + verb: String, trimmed: String
106 + ) throws(ZQParseError) -> ZQCommand {
107 + // Everything after the verb and before the first top-level comma is
108 + // a raw path (quoted paths keep spaces); options follow the comma.
109 + let afterVerb = trimmed.drop { !$0.isWhitespace }
110 + var path = String(afterVerb).trimmingCharacters(in: .whitespaces)
111 + var options: [ZQOption] = []
112 +
113 + if let commaIndex = topLevelCommaIndex(in: path) {
114 + let optionText = String(path[path.index(after: commaIndex)...])
115 + path = String(path[path.startIndex..<commaIndex])
116 + .trimmingCharacters(in: .whitespaces)
117 + let lexer = ZQLexer()
118 + let tokens = try lexer.tokenize(optionText)
119 + var cursor = ExpressionParser(tokens: tokens)
120 + options = try parseOptions(&cursor)
121 + }
122 +
123 + // `use file` / `import file` accept an optional leading `using`.
124 + for keyword in ["using ", "using\t"] where path.hasPrefix(keyword) {
125 + path = String(path.dropFirst(keyword.count))
126 + .trimmingCharacters(in: .whitespaces)
127 + }
128 + if path.hasPrefix("\""), path.hasSuffix("\""), path.count >= 2 {
129 + path = String(path.dropFirst().dropLast())
130 + }
131 +
132 + return ZQCommand(
133 + verb: verb,
134 + argument: path.isEmpty ? nil : path,
135 + options: options
136 + )
137 + }
138 +
139 + private func topLevelCommaIndex(in text: String) -> String.Index? {
140 + var depth = 0
141 + var inString = false
142 + var index = text.startIndex
143 + while index < text.endIndex {
144 + let c = text[index]
145 + if c == "\"" { inString.toggle() }
146 + if !inString {
147 + switch c {
148 + case "(", "[": depth += 1
149 + case ")", "]": depth -= 1
150 + case "," where depth == 0: return index
151 + default: break
152 + }
153 + }
154 + index = text.index(after: index)
155 + }
156 + return nil
157 + }
158 +
159 + // MARK: - Standard commands
160 +
161 + private func parseStandardCommand(
162 + verb: String, line: String, trimmed: String
163 + ) throws(ZQParseError) -> ZQCommand {
164 + let lexer = ZQLexer()
165 + let tokens = try lexer.tokenize(trimmed)
166 + var cursor = ExpressionParser(tokens: tokens)
167 +
168 + // Consume the (already resolved) verb token.
169 + _ = cursor.advance()
170 + var command = ZQCommand(verb: verb)
171 +
172 + // Compound verbs: `graph scatter y x`
173 + if verbTable.compoundVerbs.contains(verb) {
174 + guard case .identifier(let sub) = cursor.current.kind else {
175 + throw ZQParseError(
176 + message: "'\(verb)' requires a sub-command",
177 + column: cursor.current.column
178 + )
179 + }
180 + _ = cursor.advance()
181 + command.subverb = sub
182 + }
183 +
184 + // `set seed 42` → subverb "seed", argument "42"
185 + if verb == "set" {
186 + guard case .identifier(let sub) = cursor.current.kind else {
187 + throw ZQParseError(
188 + message: "'set' requires a parameter name",
189 + column: cursor.current.column
190 + )
191 + }
192 + _ = cursor.advance()
193 + command.subverb = sub
194 + if case .number(let value) = cursor.current.kind {
195 + _ = cursor.advance()
196 + command.argument = renderNumber(value)
197 + } else if case .identifier(let word) = cursor.current.kind {
198 + _ = cursor.advance()
199 + command.argument = word
200 + }
201 + try expectEnd(&cursor)
202 + return command
203 + }
204 +
205 + // `display 2 + 2` — a bare expression command.
206 + if verb == "display" {
207 + let expression = try cursor.parseExpression()
208 + command.assignment = ZQAssignment(target: "", expression: expression)
209 + try expectEnd(&cursor)
210 + return command
211 + }
212 +
213 + // Assignment verbs: `gen log_rev = ln(revenue)`
214 + if verbTable.assignmentVerbs.contains(verb) {
215 + guard case .identifier(let target) = cursor.current.kind else {
216 + throw ZQParseError(
217 + message: "'\(verb)' requires a variable name",
218 + column: cursor.current.column
219 + )
220 + }
221 + _ = cursor.advance()
222 + try cursor.expect(.op("="))
223 + let expression = try cursor.parseExpression()
224 + command.assignment = ZQAssignment(target: target, expression: expression)
225 + } else {
226 + command.varlist = try parseVarlist(&cursor)
227 + }
228 +
229 + // Qualifiers may appear in any sensible order; Stata fixes the
230 + // order as if → in → weight, which we enforce implicitly.
231 + if consumeKeyword(&cursor, "if") {
232 + command.condition = try cursor.parseExpression()
233 + }
234 + if consumeKeyword(&cursor, "in") {
235 + command.range = try parseRange(&cursor)
236 + }
237 + if cursor.isKind(.lbracket) {
238 + command.weight = try parseWeight(&cursor)
239 + }
240 + if cursor.consumeIfKind(.comma) {
241 + command.options = try parseOptions(&cursor)
242 + }
243 + try expectEnd(&cursor)
244 + return command
245 + }
246 +
247 + // MARK: - Varlist
248 +
249 + private func parseVarlist(
250 + _ cursor: inout ExpressionParser
251 + ) throws(ZQParseError) -> [ZQVarSpec] {
252 + var specs: [ZQVarSpec] = []
253 + while case .identifier(let name) = cursor.current.kind {
254 + if name == "if" || name == "in" { break }
255 + specs.append(try parseVarSpec(&cursor))
256 + }
257 + return specs
258 + }
259 +
260 + private func parseVarSpec(
261 + _ cursor: inout ExpressionParser
262 + ) throws(ZQParseError) -> ZQVarSpec {
263 + var terms: [ZQVarSpec] = [try parseFactorTerm(&cursor)]
264 + var full = false
265 + while case .op(let symbol) = cursor.current.kind, symbol == "#" || symbol == "##" {
266 + full = full || symbol == "##"
267 + _ = cursor.advance()
268 + terms.append(try parseFactorTerm(&cursor))
269 + }
270 + return terms.count == 1 ? terms[0] : .interaction(terms, full: full)
271 + }
272 +
273 + private func parseFactorTerm(
274 + _ cursor: inout ExpressionParser
275 + ) throws(ZQParseError) -> ZQVarSpec {
276 + guard case .identifier(let first) = cursor.current.kind else {
277 + throw ZQParseError(
278 + message: "expected a variable name, found \(cursor.current.kind.describe)",
279 + column: cursor.current.column
280 + )
281 + }
282 + _ = cursor.advance()
283 + if cursor.isKind(.op(".")) {
284 + _ = cursor.advance()
285 + guard case .identifier(let name) = cursor.current.kind else {
286 + throw ZQParseError(
287 + message: "expected a variable name after '\(first).'",
288 + column: cursor.current.column
289 + )
290 + }
291 + _ = cursor.advance()
292 + return .factor(op: first, name: name)
293 + }
294 + return .simple(first)
295 + }
296 +
297 + // MARK: - Qualifiers
298 +
299 + private func consumeKeyword(_ cursor: inout ExpressionParser, _ keyword: String) -> Bool {
300 + if case .identifier(let word) = cursor.current.kind, word == keyword {
301 + _ = cursor.advance()
302 + return true
303 + }
304 + return false
305 + }
306 +
307 + private func parseRange(
308 + _ cursor: inout ExpressionParser
309 + ) throws(ZQParseError) -> ZQRange {
310 + let lower = try parseRangeBound(&cursor)
311 + try cursor.expect(.op("/"))
312 + let upper = try parseRangeBound(&cursor)
313 + return ZQRange(lower: lower, upper: upper)
314 + }
315 +
316 + private func parseRangeBound(
317 + _ cursor: inout ExpressionParser
318 + ) throws(ZQParseError) -> Int {
319 + var sign = 1
320 + if cursor.consumeIfKind(.op("-")) { sign = -1 }
321 + // `f` and `l` mean first and last observation.
322 + if case .identifier(let word) = cursor.current.kind {
323 + if word == "f" { _ = cursor.advance(); return 1 }
324 + if word == "l" { _ = cursor.advance(); return -1 }
325 + }
326 + guard case .number(let value) = cursor.current.kind,
327 + value == value.rounded() else {
328 + throw ZQParseError(
329 + message: "expected an integer observation number",
330 + column: cursor.current.column
331 + )
332 + }
333 + _ = cursor.advance()
334 + return sign * Int(value)
335 + }
336 +
337 + private func parseWeight(
338 + _ cursor: inout ExpressionParser
339 + ) throws(ZQParseError) -> ZQWeight {
340 + try cursor.expect(.lbracket)
341 + guard case .identifier(let kindWord) = cursor.current.kind else {
342 + throw ZQParseError(
343 + message: "expected a weight type (aweight, fweight, pweight, iweight)",
344 + column: cursor.current.column
345 + )
346 + }
347 + let normalized: ZQWeight.Kind? = switch kindWord {
348 + case "aweight", "aw", "w", "weight": .aweight
349 + case "fweight", "fw": .fweight
350 + case "pweight", "pw": .pweight
351 + case "iweight", "iw": .iweight
352 + default: nil
353 + }
354 + guard let kind = normalized else {
355 + throw ZQParseError(
356 + message: "unknown weight type '\(kindWord)'",
357 + column: cursor.current.column
358 + )
359 + }
360 + _ = cursor.advance()
361 + try cursor.expect(.op("="))
362 + let expression = try cursor.parseExpression()
363 + try cursor.expect(.rbracket)
364 + return ZQWeight(kind: kind, expression: expression)
365 + }
366 +
367 + // MARK: - Options
368 +
369 + private func parseOptions(
370 + _ cursor: inout ExpressionParser
371 + ) throws(ZQParseError) -> [ZQOption] {
372 + var options: [ZQOption] = []
373 + while case .identifier(let name) = cursor.current.kind {
374 + _ = cursor.advance()
375 + var arguments: [String] = []
376 + if cursor.consumeIfKind(.lparen) {
377 + var depth = 1
378 + var pieces: [String] = []
379 + while depth > 0 {
380 + let token = cursor.current
381 + switch token.kind {
382 + case .lparen:
383 + depth += 1
384 + pieces.append("(")
385 + case .rparen:
386 + depth -= 1
387 + if depth > 0 { pieces.append(")") }
388 + case .endOfLine:
389 + throw ZQParseError(
390 + message: "unclosed '(' in option '\(name)'",
391 + column: token.column
392 + )
393 + case .identifier(let word):
394 + pieces.append(word)
395 + case .number(let value):
396 + pieces.append(renderNumber(value))
397 + case .string(let text):
398 + pieces.append("\"\(text)\"")
399 + case .comma:
400 + pieces.append(",")
401 + case .colon:
402 + pieces.append(":")
403 + case .lbracket:
404 + pieces.append("[")
405 + case .rbracket:
406 + pieces.append("]")
407 + case .op(let symbol):
408 + pieces.append(symbol)
409 + }
410 + _ = cursor.advance()
411 + }
412 + // Arguments split on top-level spaces in the original text
413 + // are reassembled here token-by-token: identifiers and
414 + // numbers stand alone, operator glue joins with neighbors.
415 + arguments = assembleOptionArguments(pieces)
416 + }
417 + options.append(ZQOption(name: name, arguments: arguments))
418 + // Optional commas between options are tolerated.
419 + _ = cursor.consumeIfKind(.comma)
420 + }
421 + return options
422 + }
423 +
424 + /// Joins raw option-argument pieces into logical arguments: glue
425 + /// operators (`.`, `=`, comparison, arithmetic) bind to their
426 + /// neighbors, and standalone words split on whitespace boundaries.
427 + private func assembleOptionArguments(_ pieces: [String]) -> [String] {
428 + let glue: Set<String> = [
429 + ".", "=", "==", "!=", "<", "<=", ">", ">=",
430 + "+", "-", "*", "/", "^", "#", "##", "&", "|", "!",
431 + ]
432 + var arguments: [String] = []
433 + var pending = ""
434 + var previousWasGlue = false
435 +
436 + for piece in pieces {
437 + if piece == "," {
438 + if !pending.isEmpty { arguments.append(pending); pending = "" }
439 + previousWasGlue = false
440 + continue
441 + }
442 + if glue.contains(piece) {
443 + pending += piece
444 + previousWasGlue = true
445 + } else if previousWasGlue || pending.isEmpty {
446 + pending += piece
447 + previousWasGlue = false
448 + } else {
449 + arguments.append(pending)
450 + pending = piece
451 + previousWasGlue = false
452 + }
453 + }
454 + if !pending.isEmpty { arguments.append(pending) }
455 + return arguments
456 + }
457 +
458 + // MARK: - Helpers
459 +
460 + private func expectEnd(_ cursor: inout ExpressionParser) throws(ZQParseError) {
461 + guard cursor.isKind(.endOfLine) else {
462 + throw ZQParseError(
463 + message: "unexpected \(cursor.current.kind.describe) after end of command",
464 + column: cursor.current.column
465 + )
466 + }
467 + }
468 +
469 + private func renderNumber(_ value: Double) -> String {
470 + if value == value.rounded(), abs(value) < 1e15 {
471 + return String(Int64(value))
472 + }
473 + return String(value)
474 + }
475 +
476 + private func columnOf(word: String, in line: String) -> Int {
477 + guard let range = line.range(of: word) else { return 1 }
478 + return line.distance(from: line.startIndex, to: range.lowerBound) + 1
479 + }
480 +}
added MetrikaKit/Sources/ZQParser/Grammar/ExpressionParser.swift +128 −0
@@ -0,0 +1,128 @@
1 +//
2 +// ExpressionParser.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Pratt (precedence-climbing) parser for ZQL expressions, shared by `if`
11 +/// qualifiers, `gen`/`replace` right-hand sides, and weight expressions.
12 +///
13 +/// Precedence (low → high):
14 +/// | · & · comparison (== != < <= > >=) · + - · * / · ^ · unary (- !)
15 +struct ExpressionParser {
16 + var tokens: [ZQToken]
17 + var position: Int
18 +
19 + init(tokens: [ZQToken], position: Int = 0) {
20 + self.tokens = tokens
21 + self.position = position
22 + }
23 +
24 + var current: ZQToken { tokens[position] }
25 +
26 + mutating func advance() -> ZQToken {
27 + let token = tokens[position]
28 + if position < tokens.count - 1 { position += 1 }
29 + return token
30 + }
31 +
32 + private static let precedence: [String: Int] = [
33 + "|": 10,
34 + "&": 20,
35 + "==": 30, "!=": 30, "<": 30, "<=": 30, ">": 30, ">=": 30,
36 + "+": 40, "-": 40,
37 + "*": 50, "/": 50,
38 + "^": 60,
39 + ]
40 +
41 + mutating func parseExpression(minPrecedence: Int = 0) throws(ZQParseError) -> ZQExpression {
42 + var left = try parseUnary()
43 +
44 + while case .op(let symbol) = current.kind,
45 + let precedence = Self.precedence[symbol],
46 + precedence >= minPrecedence {
47 + _ = advance()
48 + // ^ is right-associative; everything else is left-associative.
49 + let nextMin = symbol == "^" ? precedence : precedence + 1
50 + let right = try parseExpression(minPrecedence: nextMin)
51 + left = .binary(op: symbol, left, right)
52 + }
53 + return left
54 + }
55 +
56 + private mutating func parseUnary() throws(ZQParseError) -> ZQExpression {
57 + if case .op(let symbol) = current.kind, symbol == "-" || symbol == "!" {
58 + _ = advance()
59 + let operand = try parseUnary()
60 + return .unary(op: symbol, operand)
61 + }
62 + return try parsePrimary()
63 + }
64 +
65 + private mutating func parsePrimary() throws(ZQParseError) -> ZQExpression {
66 + let token = current
67 + switch token.kind {
68 + case .number(let value):
69 + _ = advance()
70 + return .number(value)
71 +
72 + case .string(let value):
73 + _ = advance()
74 + return .string(value)
75 +
76 + case .op("."):
77 + // Bare `.` is the missing-value literal (`if income == .`).
78 + _ = advance()
79 + return .missing
80 +
81 + case .identifier(let name):
82 + _ = advance()
83 + if case .lparen = current.kind {
84 + _ = advance()
85 + var arguments: [ZQExpression] = []
86 + if !isKind(.rparen) {
87 + repeat {
88 + arguments.append(try parseExpression())
89 + } while consumeIfKind(.comma)
90 + }
91 + try expect(.rparen)
92 + return .call(name: name, arguments: arguments)
93 + }
94 + return .variable(name)
95 +
96 + case .lparen:
97 + _ = advance()
98 + let inner = try parseExpression()
99 + try expect(.rparen)
100 + return inner
101 +
102 + default:
103 + throw ZQParseError(
104 + message: "expected an expression, found \(token.kind.describe)",
105 + column: token.column
106 + )
107 + }
108 + }
109 +
110 + func isKind(_ kind: ZQToken.Kind) -> Bool { current.kind == kind }
111 +
112 + mutating func consumeIfKind(_ kind: ZQToken.Kind) -> Bool {
113 + if current.kind == kind {
114 + _ = advance()
115 + return true
116 + }
117 + return false
118 + }
119 +
120 + mutating func expect(_ kind: ZQToken.Kind) throws(ZQParseError) {
121 + guard consumeIfKind(kind) else {
122 + throw ZQParseError(
123 + message: "expected \(kind.describe), found \(current.kind.describe)",
124 + column: current.column
125 + )
126 + }
127 + }
128 +}
added MetrikaKit/Sources/ZQParser/KnownVerbs.swift +109 −0
@@ -0,0 +1,109 @@
1 +//
2 +// KnownVerbs.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Registry of built-in ZQL verbs with their minimum abbreviations,
11 +/// Stata-style: `reg` resolves to `regress`, `su` to `summarize`.
12 +public struct ZQVerbTable: Sendable {
13 + /// canonical verb → minimum abbreviation length
14 + public let verbs: [String: Int]
15 + /// Verbs whose first argument is a raw file path rather than a varlist.
16 + public let fileVerbs: Set<String>
17 + /// Verbs taking a `newvar = expression` assignment.
18 + public let assignmentVerbs: Set<String>
19 + /// Prefix verbs that wrap another command after a colon.
20 + public let prefixVerbs: Set<String>
21 + /// Compound verbs followed by a sub-verb (`graph scatter …`).
22 + public let compoundVerbs: Set<String>
23 +
24 + public static let builtin = ZQVerbTable(
25 + verbs: [
26 + "use": 3,
27 + "save": 4,
28 + "import": 3,
29 + "export": 3,
30 + "summarize": 2,
31 + "tabulate": 3,
32 + "correlate": 3,
33 + "describe": 1,
34 + "list": 1,
35 + "generate": 3,
36 + "replace": 7,
37 + "drop": 4,
38 + "keep": 4,
39 + "regress": 3,
40 + "logit": 5,
41 + "probit": 6,
42 + "poisson": 7,
43 + "ivregress": 9,
44 + "xtreg": 5,
45 + "xtset": 5,
46 + "areg": 4,
47 + "bootstrap": 9,
48 + "permute": 7,
49 + "jackknife": 9,
50 + "graph": 2,
51 + "histogram": 4,
52 + "scatter": 7,
53 + "display": 2,
54 + "log": 3,
55 + "set": 3,
56 + "clear": 5,
57 + "count": 3,
58 + "sort": 4,
59 + "merge": 5,
60 + "append": 6,
61 + "reshape": 7,
62 + "collapse": 8,
63 + "egen": 4,
64 + "predict": 7,
65 + "test": 4,
66 + "help": 4,
67 + ],
68 + fileVerbs: ["use", "save", "import", "export", "log"],
69 + assignmentVerbs: ["generate", "replace", "egen"],
70 + prefixVerbs: ["bootstrap", "permute", "jackknife"],
71 + compoundVerbs: ["graph"]
72 + )
73 +
74 + public init(
75 + verbs: [String: Int],
76 + fileVerbs: Set<String>,
77 + assignmentVerbs: Set<String>,
78 + prefixVerbs: Set<String>,
79 + compoundVerbs: Set<String>
80 + ) {
81 + self.verbs = verbs
82 + self.fileVerbs = fileVerbs
83 + self.assignmentVerbs = assignmentVerbs
84 + self.prefixVerbs = prefixVerbs
85 + self.compoundVerbs = compoundVerbs
86 + }
87 +
88 + /// Resolves a typed verb (possibly abbreviated) to its canonical form.
89 + /// Returns nil when no verb matches unambiguously.
90 + public func resolve(_ typed: String) -> String? {
91 + if let minLength = verbs[typed], typed.count >= minLength { return typed }
92 + if verbs.keys.contains(typed) { return typed }
93 +
94 + let candidates = verbs.filter { verb, minLength in
95 + typed.count >= minLength && verb.hasPrefix(typed)
96 + }
97 + return candidates.count == 1 ? candidates.first!.key : nil
98 + }
99 +
100 + /// Closest known verb by edit distance, for error suggestions.
101 + /// Only returns matches within distance 2 to avoid absurd hints.
102 + public func closest(to typed: String) -> String? {
103 + let scored = verbs.keys
104 + .map { (verb: $0, distance: levenshteinDistance(typed, $0)) }
105 + .sorted { $0.distance < $1.distance }
106 + guard let best = scored.first, best.distance <= 2 else { return nil }
107 + return best.verb
108 + }
109 +}
added MetrikaKit/Sources/ZQParser/Lexer.swift +169 −0
@@ -0,0 +1,169 @@
1 +//
2 +// Lexer.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Tokenizer for a single ZQL command line.
11 +///
12 +/// Numbers support decimal and exponent notation (`1.5e-3`). A `.` is lexed
13 +/// as an operator when not part of a number, which lets the grammar layer
14 +/// assemble factor-variable notation (`i.region`, `c.age#c.age`).
15 +public struct ZQLexer: Sendable {
16 + public init() {}
17 +
18 + public func tokenize(_ line: String) throws(ZQParseError) -> [ZQToken] {
19 + var tokens: [ZQToken] = []
20 + let chars = Array(line)
21 + var i = 0
22 +
23 + func column(_ index: Int) -> Int { index + 1 }
24 +
25 + while i < chars.count {
26 + let c = chars[i]
27 +
28 + if c.isWhitespace { i += 1; continue }
29 +
30 + // Comments: // and * (only at start of line, Stata-style) end the command.
31 + if c == "/", i + 1 < chars.count, chars[i + 1] == "/" { break }
32 + if c == "*", tokens.isEmpty { break }
33 +
34 + // String literal
35 + if c == "\"" {
36 + let start = i
37 + i += 1
38 + var value = ""
39 + while i < chars.count, chars[i] != "\"" {
40 + value.append(chars[i])
41 + i += 1
42 + }
43 + guard i < chars.count else {
44 + throw ZQParseError(
45 + message: "unterminated string literal",
46 + column: column(start)
47 + )
48 + }
49 + i += 1
50 + tokens.append(ZQToken(kind: .string(value), column: column(start)))
51 + continue
52 + }
53 +
54 + // Number: digits, or leading dot followed by a digit
55 + if c.isNumber || (c == "." && i + 1 < chars.count && chars[i + 1].isNumber) {
56 + let start = i
57 + var text = ""
58 + while i < chars.count, chars[i].isNumber || chars[i] == "." {
59 + text.append(chars[i])
60 + i += 1
61 + }
62 + if i < chars.count, chars[i] == "e" || chars[i] == "E" {
63 + var j = i + 1
64 + var exp = String(chars[i])
65 + if j < chars.count, chars[j] == "+" || chars[j] == "-" {
66 + exp.append(chars[j])
67 + j += 1
68 + }
69 + if j < chars.count, chars[j].isNumber {
70 + while j < chars.count, chars[j].isNumber {
71 + exp.append(chars[j])
72 + j += 1
73 + }
74 + text.append(exp)
75 + i = j
76 + }
77 + }
78 + guard let value = Double(text) else {
79 + throw ZQParseError(
80 + message: "invalid numeric literal '\(text)'",
81 + column: column(start)
82 + )
83 + }
84 + tokens.append(ZQToken(kind: .number(value), column: column(start)))
85 + continue
86 + }
87 +
88 + // Identifier: [A-Za-z_][A-Za-z0-9_]*
89 + if c.isLetter || c == "_" {
90 + let start = i
91 + var text = ""
92 + while i < chars.count, chars[i].isLetter || chars[i].isNumber || chars[i] == "_" {
93 + text.append(chars[i])
94 + i += 1
95 + }
96 + tokens.append(ZQToken(kind: .identifier(text), column: column(start)))
97 + continue
98 + }
99 +
100 + // Punctuation & operators
101 + let start = i
102 + func push(_ kind: ZQToken.Kind, advance: Int) {
103 + tokens.append(ZQToken(kind: kind, column: column(start)))
104 + i += advance
105 + }
106 +
107 + switch c {
108 + case ",": push(.comma, advance: 1)
109 + case ":": push(.colon, advance: 1)
110 + case "(": push(.lparen, advance: 1)
111 + case ")": push(.rparen, advance: 1)
112 + case "[": push(.lbracket, advance: 1)
113 + case "]": push(.rbracket, advance: 1)
114 + case "=":
115 + if i + 1 < chars.count, chars[i + 1] == "=" {
116 + push(.op("=="), advance: 2)
117 + } else {
118 + push(.op("="), advance: 1)
119 + }
120 + case "!":
121 + if i + 1 < chars.count, chars[i + 1] == "=" {
122 + push(.op("!="), advance: 2)
123 + } else {
124 + push(.op("!"), advance: 1)
125 + }
126 + case "~":
127 + if i + 1 < chars.count, chars[i + 1] == "=" {
128 + push(.op("!="), advance: 2)
129 + } else {
130 + throw ZQParseError(message: "unexpected character '~'", column: column(start))
131 + }
132 + case "<":
133 + if i + 1 < chars.count, chars[i + 1] == "=" {
134 + push(.op("<="), advance: 2)
135 + } else {
136 + push(.op("<"), advance: 1)
137 + }
138 + case ">":
139 + if i + 1 < chars.count, chars[i + 1] == "=" {
140 + push(.op(">="), advance: 2)
141 + } else {
142 + push(.op(">"), advance: 1)
143 + }
144 + case "&": push(.op("&"), advance: 1)
145 + case "|": push(.op("|"), advance: 1)
146 + case "+": push(.op("+"), advance: 1)
147 + case "-": push(.op("-"), advance: 1)
148 + case "*": push(.op("*"), advance: 1)
149 + case "/": push(.op("/"), advance: 1)
150 + case "^": push(.op("^"), advance: 1)
151 + case "#":
152 + if i + 1 < chars.count, chars[i + 1] == "#" {
153 + push(.op("##"), advance: 2)
154 + } else {
155 + push(.op("#"), advance: 1)
156 + }
157 + case ".": push(.op("."), advance: 1)
158 + default:
159 + throw ZQParseError(
160 + message: "unexpected character '\(c)'",
161 + column: column(start)
162 + )
163 + }
164 + }
165 +
166 + tokens.append(ZQToken(kind: .endOfLine, column: chars.count + 1))
167 + return tokens
168 + }
169 +}
added MetrikaKit/Sources/ZQParser/ParseError.swift +52 −0
@@ -0,0 +1,52 @@
1 +//
2 +// ParseError.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Parse failure with the 1-based source column and, where possible, a
11 +/// suggested fix (CLAUDE.md §4: errors must cite column position and
12 +/// suggest fixes via Levenshtein distance on known verbs).
13 +public struct ZQParseError: Error, Equatable, Sendable, CustomStringConvertible {
14 + public var message: String
15 + public var column: Int
16 + public var suggestion: String?
17 +
18 + public init(message: String, column: Int, suggestion: String? = nil) {
19 + self.message = message
20 + self.column = column
21 + self.suggestion = suggestion
22 + }
23 +
24 + public var description: String {
25 + var text = "parse error at column \(column): \(message)"
26 + if let suggestion {
27 + text += "\n did you mean '\(suggestion)'?"
28 + }
29 + return text
30 + }
31 +}
32 +
33 +/// Levenshtein edit distance, used to suggest the closest known verb for
34 +/// an unrecognized command.
35 +public func levenshteinDistance(_ a: String, _ b: String) -> Int {
36 + let a = Array(a), b = Array(b)
37 + if a.isEmpty { return b.count }
38 + if b.isEmpty { return a.count }
39 +
40 + var previous = Array(0...b.count)
41 + var current = [Int](repeating: 0, count: b.count + 1)
42 +
43 + for i in 1...a.count {
44 + current[0] = i
45 + for j in 1...b.count {
46 + let substitution = previous[j - 1] + (a[i - 1] == b[j - 1] ? 0 : 1)
47 + current[j] = min(previous[j] + 1, current[j - 1] + 1, substitution)
48 + }
49 + swap(&previous, &current)
50 + }
51 + return previous[b.count]
52 +}
added MetrikaKit/Sources/ZQParser/Token.swift +55 −0
@@ -0,0 +1,55 @@
1 +//
2 +// Token.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// A lexical token of the ZQL command language, carrying its 1-based
11 +/// source column so parse errors can point at the offending position.
12 +public struct ZQToken: Equatable, Sendable {
13 + public enum Kind: Equatable, Sendable {
14 + case identifier(String)
15 + case number(Double)
16 + case string(String)
17 + case comma
18 + case colon
19 + case lparen
20 + case rparen
21 + case lbracket
22 + case rbracket
23 + /// Operators: + - * / ^ = == != < <= > >= & | ! # .
24 + case op(String)
25 + case endOfLine
26 + }
27 +
28 + public let kind: Kind
29 + /// 1-based column of the first character of the token.
30 + public let column: Int
31 +
32 + public init(kind: Kind, column: Int) {
33 + self.kind = kind
34 + self.column = column
35 + }
36 +}
37 +
38 +extension ZQToken.Kind {
39 + /// Human-readable rendering used in error messages.
40 + public var describe: String {
41 + switch self {
42 + case .identifier(let s): return "'\(s)'"
43 + case .number(let n): return "number \(n)"
44 + case .string(let s): return "\"\(s)\""
45 + case .comma: return "','"
46 + case .colon: return "':'"
47 + case .lparen: return "'('"
48 + case .rparen: return "')'"
49 + case .lbracket: return "'['"
50 + case .rbracket: return "']'"
51 + case .op(let o): return "'\(o)'"
52 + case .endOfLine: return "end of line"
53 + }
54 + }
55 +}
added MetrikaKit/Sources/ZQParser/ZQParserModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQParserModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQParser.
11 +public enum ZQParserModule {
12 + public static let name = "ZQParser"
13 +}
added MetrikaKit/Sources/ZQPlanner/Planner.swift +106 −0
@@ -0,0 +1,106 @@
1 +//
2 +// Planner.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import ZQData
11 +import ZQParser
12 +
13 +/// Execution backend. The planner chooses automatically; the user never
14 +/// does (CLAUDE.md §5).
15 +public enum ZQBackend: String, Equatable, Sendable {
16 + case cpu, gpu, hybrid
17 +}
18 +
19 +/// A planned command: the parsed AST plus dispatch decision. Factor
20 +/// variables expand here (plan time), not at parse time.
21 +public struct ZQExecutionPlan: Equatable, Sendable {
22 + public var command: ZQCommand
23 + public var backend: ZQBackend
24 +
25 + public init(command: ZQCommand, backend: ZQBackend) {
26 + self.command = command
27 + self.backend = backend
28 + }
29 +}
30 +
31 +public struct ZQPlanner: Sendable {
32 + /// Single-estimation row-count threshold below which the CPU LAPACK
33 + /// path wins (heuristic 1, tuned by Tests/Bench).
34 + public var cpuRowThreshold: Int
35 + /// Independent-replicate threshold at which batched GPU solves win
36 + /// (heuristic 2).
37 + public var gpuReplicateThreshold: Int
38 + /// Whether a GPU backend is available at all. The pure-Swift Philox
39 + /// reference keeps plans reproducible even when this is false.
40 + public var gpuAvailable: Bool
41 +
42 + public init(
43 + cpuRowThreshold: Int = 5_000_000,
44 + gpuReplicateThreshold: Int = 500,
45 + gpuAvailable: Bool = false
46 + ) {
47 + self.cpuRowThreshold = cpuRowThreshold
48 + self.gpuReplicateThreshold = gpuReplicateThreshold
49 + self.gpuAvailable = gpuAvailable
50 + }
51 +
52 + public func plan(_ command: ZQCommand, rowCount: Int) -> ZQExecutionPlan {
53 + ZQExecutionPlan(command: command, backend: backend(for: command, rowCount: rowCount))
54 + }
55 +
56 + private func backend(for command: ZQCommand, rowCount: Int) -> ZQBackend {
57 + guard gpuAvailable else { return .cpu }
58 +
59 + // Heuristic 2: ≥ threshold independent replicates → batched GPU.
60 + if command.verb == "bootstrap" || command.verb == "permute" {
61 + if let reps = command.option("reps")?.firstArgument,
62 + let count = Int(reps), count >= gpuReplicateThreshold {
63 + return .gpu
64 + }
65 + }
66 +
67 + // Heuristic 1: single estimation under the row threshold → CPU.
68 + if rowCount < cpuRowThreshold { return .cpu }
69 +
70 + // Large single estimations stay on CPU until the batched GPU
71 + // estimators land (v0.2); MCMC-style workloads will plan .hybrid.
72 + return .cpu
73 + }
74 +}
75 +
76 +/// Plan-time factor-variable expansion (`i.region` → one indicator column
77 +/// per level, base level omitted).
78 +public enum ZQFactorExpansion {
79 +
80 + /// Distinct sorted levels of a numeric variable, ignoring missing.
81 + public static func levels(values: [Double], missing: [Bool]) -> [Double] {
82 + var seen = Set<Double>()
83 + for i in 0..<values.count where !missing[i] {
84 + seen.insert(values[i])
85 + }
86 + return seen.sorted()
87 + }
88 +
89 + /// Expands one factor spec into named indicator columns against the
90 + /// given data. The lowest level is the omitted base (Stata default).
91 + public static func expand(
92 + name: String, values: [Double], missing: [Bool]
93 + ) -> [(name: String, values: [Double])] {
94 + let allLevels = levels(values: values, missing: missing)
95 + guard allLevels.count > 1 else { return [] }
96 + return allLevels.dropFirst().map { level in
97 + let rendered = level == level.rounded()
98 + ? String(Int(level))
99 + : String(level)
100 + let indicator = values.enumerated().map { index, value in
101 + (!missing[index] && value == level) ? 1.0 : 0.0
102 + }
103 + return (name: "\(rendered).\(name)", values: indicator)
104 + }
105 + }
106 +}
added MetrikaKit/Sources/ZQPlanner/ZQPlannerModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQPlannerModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQPlanner.
11 +public enum ZQPlannerModule {
12 + public static let name = "ZQPlanner"
13 +}
added MetrikaKit/Sources/ZQPlugins/PluginProtocol.swift +128 −0
@@ -0,0 +1,128 @@
1 +//
2 +// PluginProtocol.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +import ZQData
12 +import ZQParser
13 +
14 +/// Declarative syntax specification for a user command: which qualifiers
15 +/// and options the engine should accept and pre-validate.
16 +public struct ZQSyntaxSpec: Equatable, Sendable {
17 + public var acceptsVarlist: Bool
18 + public var acceptsCondition: Bool
19 + public var acceptsRange: Bool
20 + public var acceptsWeight: Bool
21 + /// Known option names; empty means any option is passed through.
22 + public var options: [String]
23 + /// Whether the plugin may mutate the working dataset. Plugins without
24 + /// this receive a read-only view (CLAUDE.md §4).
25 + public var mutates: Bool
26 +
27 + public init(
28 + acceptsVarlist: Bool = true,
29 + acceptsCondition: Bool = true,
30 + acceptsRange: Bool = true,
31 + acceptsWeight: Bool = false,
32 + options: [String] = [],
33 + mutates: Bool = false
34 + ) {
35 + self.acceptsVarlist = acceptsVarlist
36 + self.acceptsCondition = acceptsCondition
37 + self.acceptsRange = acceptsRange
38 + self.acceptsWeight = acceptsWeight
39 + self.options = options
40 + self.mutates = mutates
41 + }
42 +}
43 +
44 +/// Result of executing a command: rendered console text plus named scalar
45 +/// results (Stata's `r()` analog).
46 +public struct ZQResult: Equatable, Sendable {
47 + public var text: String
48 + public var scalars: [String: Double]
49 +
50 + public init(text: String, scalars: [String: Double] = [:]) {
51 + self.text = text
52 + self.scalars = scalars
53 + }
54 +}
55 +
56 +/// Execution context handed to a plugin: the parsed command and a view of
57 +/// the working dataset. The frame is a value copy — mutations stay local
58 +/// unless the plugin declares `mutates: true` and returns a replacement
59 +/// via `ZQContext.replacementFrame`.
60 +public struct ZQContext: Sendable {
61 + public var command: ZQCommand
62 + public var frame: ZQDataFrame
63 + /// Set by mutating plugins; the engine installs it as the new working
64 + /// dataset after successful execution.
65 + public var replacementFrame: ZQDataFrame?
66 +
67 + public init(command: ZQCommand, frame: ZQDataFrame) {
68 + self.command = command
69 + self.frame = frame
70 + self.replacementFrame = nil
71 + }
72 +}
73 +
74 +/// Native plugin protocol (CLAUDE.md §4). Conforming Swift packages are
75 +/// discovered under `~/Library/Application Support/Metrika/Commands/`.
76 +public protocol ZQCommandPlugin: Sendable {
77 + static var verb: String { get }
78 + static var syntax: ZQSyntaxSpec { get }
79 + func execute(_ ctx: ZQContext) async throws -> ZQResult
80 +}
81 +
82 +/// Tier-1 user command: a `.zyq` script (ado-file analog) — a sequence of
83 +/// ZQL lines the engine replays.
84 +public struct ZQScriptCommand: Equatable, Sendable {
85 + public var verb: String
86 + public var fileURL: URL
87 + public var lines: [String]
88 +
89 + public init(verb: String, fileURL: URL, lines: [String]) {
90 + self.verb = verb
91 + self.fileURL = fileURL
92 + self.lines = lines
93 + }
94 +}
95 +
96 +/// Discovers `.zyq` script commands in the user commands directory.
97 +public enum ZQPluginDiscovery {
98 + public static var defaultDirectory: URL {
99 + FileManager.default
100 + .urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]
101 + .appendingPathComponent("Metrika/Commands", isDirectory: true)
102 + }
103 +
104 + public static func scriptCommands(
105 + in directory: URL = defaultDirectory
106 + ) -> [ZQScriptCommand] {
107 + guard let contents = try? FileManager.default.contentsOfDirectory(
108 + at: directory, includingPropertiesForKeys: nil
109 + ) else { return [] }
110 +
111 + return contents
112 + .filter { $0.pathExtension == "zyq" }
113 + .compactMap { url in
114 + guard let text = try? String(contentsOf: url, encoding: .utf8) else {
115 + return nil
116 + }
117 + let lines = text.split(
118 + separator: "\n", omittingEmptySubsequences: false
119 + ).map(String.init)
120 + return ZQScriptCommand(
121 + verb: url.deletingPathExtension().lastPathComponent,
122 + fileURL: url,
123 + lines: lines
124 + )
125 + }
126 + .sorted { $0.verb < $1.verb }
127 + }
128 +}
added MetrikaKit/Sources/ZQPlugins/ZQPluginsModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQPluginsModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQPlugins.
11 +public enum ZQPluginsModule {
12 + public static let name = "ZQPlugins"
13 +}
added MetrikaKit/Sources/ZQStats/Distributions.swift +149 −0
@@ -0,0 +1,149 @@
1 +//
2 +// Distributions.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +
12 +/// Special functions and distribution CDFs used for inference output.
13 +/// Implementations follow the classic Lanczos / continued-fraction
14 +/// formulations and are accurate to well beyond the 1e-10 project
15 +/// tolerance for the parameter ranges used in regression output.
16 +public enum ZQDistributions {
17 +
18 + /// Natural log of the gamma function (Lanczos approximation, g=7).
19 + public static func logGamma(_ x: Double) -> Double {
20 + precondition(x > 0, "logGamma requires x > 0")
21 + let coefficients: [Double] = [
22 + 676.5203681218851, -1259.1392167224028, 771.32342877765313,
23 + -176.61502916214059, 12.507343278686905, -0.13857109526572012,
24 + 9.9843695780195716e-6, 1.5056327351493116e-7,
25 + ]
26 + if x < 0.5 {
27 + // Reflection formula.
28 + return log(.pi / sin(.pi * x)) - logGamma(1 - x)
29 + }
30 + let z = x - 1
31 + var sum = 0.99999999999980993
32 + for (i, c) in coefficients.enumerated() {
33 + sum += c / (z + Double(i) + 1)
34 + }
35 + let t = z + 7.5
36 + return 0.5 * log(2 * .pi) + (z + 0.5) * log(t) - t + log(sum)
37 + }
38 +
39 + /// Regularized incomplete beta function I_x(a, b) via the Lentz
40 + /// continued-fraction algorithm.
41 + public static func incompleteBeta(a: Double, b: Double, x: Double) -> Double {
42 + precondition(a > 0 && b > 0, "incompleteBeta requires a, b > 0")
43 + if x <= 0 { return 0 }
44 + if x >= 1 { return 1 }
45 +
46 + let logBeta = logGamma(a + b) - logGamma(a) - logGamma(b)
47 + let front = exp(logBeta + a * log(x) + b * log(1 - x))
48 +
49 + // Use the symmetry relation to keep the continued fraction in its
50 + // rapidly converging region.
51 + if x < (a + 1) / (a + b + 2) {
52 + return front * betaContinuedFraction(a: a, b: b, x: x) / a
53 + } else {
54 + return 1 - front * betaContinuedFraction(a: b, b: a, x: 1 - x) / b
55 + }
56 + }
57 +
58 + private static func betaContinuedFraction(a: Double, b: Double, x: Double) -> Double {
59 + let tiny = 1e-300
60 + let epsilon = 1e-16
61 + let qab = a + b
62 + let qap = a + 1
63 + let qam = a - 1
64 +
65 + var c = 1.0
66 + var d = 1 - qab * x / qap
67 + if abs(d) < tiny { d = tiny }
68 + d = 1 / d
69 + var h = d
70 +
71 + for m in 1...300 {
72 + let dm = Double(m)
73 + // Even step.
74 + var numerator = dm * (b - dm) * x / ((qam + 2 * dm) * (a + 2 * dm))
75 + d = 1 + numerator * d
76 + if abs(d) < tiny { d = tiny }
77 + c = 1 + numerator / c
78 + if abs(c) < tiny { c = tiny }
79 + d = 1 / d
80 + h *= d * c
81 + // Odd step.
82 + numerator = -(a + dm) * (qab + dm) * x / ((a + 2 * dm) * (qap + 2 * dm))
83 + d = 1 + numerator * d
84 + if abs(d) < tiny { d = tiny }
85 + c = 1 + numerator / c
86 + if abs(c) < tiny { c = tiny }
87 + d = 1 / d
88 + let delta = d * c
89 + h *= delta
90 + if abs(delta - 1) < epsilon { break }
91 + }
92 + return h
93 + }
94 +
95 + /// Standard normal CDF.
96 + public static func normalCDF(_ z: Double) -> Double {
97 + 0.5 * erfc(-z / 2.0.squareRoot())
98 + }
99 +
100 + /// Student t CDF with `df` degrees of freedom.
101 + public static func studentTCDF(_ t: Double, df: Double) -> Double {
102 + precondition(df > 0, "studentTCDF requires df > 0")
103 + let x = df / (df + t * t)
104 + let tail = 0.5 * incompleteBeta(a: df / 2, b: 0.5, x: x)
105 + return t > 0 ? 1 - tail : tail
106 + }
107 +
108 + /// Two-sided p-value for a t statistic, computed directly from the
109 + /// tail integral: 2·P(T > |t|) = I_{df/(df+t²)}(df/2, 1/2). Going
110 + /// through `2·(1 − CDF)` would lose all precision below ~1e-16.
111 + public static func tTestPValue(_ t: Double, df: Double) -> Double {
112 + incompleteBeta(a: df / 2, b: 0.5, x: df / (df + t * t))
113 + }
114 +
115 + /// F distribution CDF with (df1, df2) degrees of freedom.
116 + public static func fCDF(_ f: Double, df1: Double, df2: Double) -> Double {
117 + precondition(df1 > 0 && df2 > 0, "fCDF requires positive df")
118 + if f <= 0 { return 0 }
119 + let x = df1 * f / (df1 * f + df2)
120 + return incompleteBeta(a: df1 / 2, b: df2 / 2, x: x)
121 + }
122 +
123 + /// Upper-tail p-value for an F statistic, computed directly as
124 + /// I_{df2/(df2+df1·f)}(df2/2, df1/2) to preserve precision for large F.
125 + public static func fTestPValue(_ f: Double, df1: Double, df2: Double) -> Double {
126 + if f <= 0 { return 1 }
127 + return incompleteBeta(a: df2 / 2, b: df1 / 2, x: df2 / (df2 + df1 * f))
128 + }
129 +
130 + /// Student t quantile (inverse CDF) via bisection refined with Newton
131 + /// steps — used for confidence intervals.
132 + public static func studentTQuantile(_ p: Double, df: Double) -> Double {
133 + precondition(p > 0 && p < 1, "quantile requires 0 < p < 1")
134 + if abs(p - 0.5) < 1e-15 { return 0 }
135 +
136 + var low = -1000.0, high = 1000.0
137 + var mid = 0.0
138 + for _ in 0..<200 {
139 + mid = 0.5 * (low + high)
140 + if studentTCDF(mid, df: df) < p {
141 + low = mid
142 + } else {
143 + high = mid
144 + }
145 + if high - low < 1e-13 * max(1, abs(mid)) { break }
146 + }
147 + return 0.5 * (low + high)
148 + }
149 +}
added MetrikaKit/Sources/ZQStats/LinearAlgebra.swift +199 −0
@@ -0,0 +1,199 @@
1 +//
2 +// LinearAlgebra.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Accelerate
11 +
12 +/// Thin LAPACK/BLAS wrappers used by the CPU estimators. All matrices are
13 +/// stored column-major (LAPACK convention) as flat [Double] buffers.
14 +enum LinearAlgebra {
15 + struct Failure: Error, CustomStringConvertible {
16 + let routine: String
17 + let info: Int
18 + var description: String { "\(routine) failed with info=\(info)" }
19 + }
20 +
21 + /// QR decomposition of an n×k matrix (n ≥ k) via `dgeqrf`. CLAUDE.md §5:
22 + /// never form X'X — coefficient solves and (X'X)⁻¹ all go through R.
23 + struct QR {
24 + /// Factored matrix as returned by dgeqrf: R in the upper triangle,
25 + /// Householder vectors below the diagonal.
26 + private(set) var factored: [Double]
27 + private(set) var tau: [Double]
28 + let rows: Int
29 + let cols: Int
30 +
31 + init(matrix: [Double], rows: Int, cols: Int) throws {
32 + precondition(matrix.count == rows * cols, "matrix size mismatch")
33 + precondition(rows >= cols, "QR requires rows ≥ cols")
34 + var a = matrix
35 + var m = __CLPK_integer(rows)
36 + var n = __CLPK_integer(cols)
37 + var lda = m
38 + var tau = [Double](repeating: 0, count: cols)
39 + var info: __CLPK_integer = 0
40 +
41 + // Workspace query, then factorization.
42 + var lwork: __CLPK_integer = -1
43 + var workQuery = [Double](repeating: 0, count: 1)
44 + dgeqrf_(&m, &n, &a, &lda, &tau, &workQuery, &lwork, &info)
45 + lwork = __CLPK_integer(workQuery[0])
46 + var work = [Double](repeating: 0, count: max(1, Int(lwork)))
47 + dgeqrf_(&m, &n, &a, &lda, &tau, &work, &lwork, &info)
48 + guard info == 0 else { throw Failure(routine: "dgeqrf", info: Int(info)) }
49 +
50 + self.factored = a
51 + self.tau = tau
52 + self.rows = rows
53 + self.cols = cols
54 + }
55 +
56 + /// Solves min ‖Ax − b‖ using the stored factorization:
57 + /// x = R⁻¹ (Q'b)[0..<k].
58 + func solve(rhs: [Double]) throws -> [Double] {
59 + precondition(rhs.count == rows, "rhs size mismatch")
60 + var qtb = rhs
61 + var side: Int8 = Int8(UInt8(ascii: "L"))
62 + var trans: Int8 = Int8(UInt8(ascii: "T"))
63 + var m = __CLPK_integer(rows)
64 + var nrhs: __CLPK_integer = 1
65 + var k = __CLPK_integer(cols)
66 + var lda = m
67 + var ldc = m
68 + var info: __CLPK_integer = 0
69 + var a = factored
70 + var tauCopy = tau
71 +
72 + var lwork: __CLPK_integer = -1
73 + var workQuery = [Double](repeating: 0, count: 1)
74 + dormqr_(&side, &trans, &m, &nrhs, &k, &a, &lda, &tauCopy, &qtb, &ldc,
75 + &workQuery, &lwork, &info)
76 + lwork = __CLPK_integer(workQuery[0])
77 + var work = [Double](repeating: 0, count: max(1, Int(lwork)))
78 + dormqr_(&side, &trans, &m, &nrhs, &k, &a, &lda, &tauCopy, &qtb, &ldc,
79 + &work, &lwork, &info)
80 + guard info == 0 else { throw Failure(routine: "dormqr", info: Int(info)) }
81 +
82 + // Back-substitute R x = (Q'b)[0..<k].
83 + var uplo: Int8 = Int8(UInt8(ascii: "U"))
84 + var transN: Int8 = Int8(UInt8(ascii: "N"))
85 + var diag: Int8 = Int8(UInt8(ascii: "N"))
86 + var n = __CLPK_integer(cols)
87 + var solution = Array(qtb[0..<cols])
88 + var ldb = n
89 + dtrtrs_(&uplo, &transN, &diag, &n, &nrhs, &a, &lda, &solution, &ldb, &info)
90 + guard info == 0 else { throw Failure(routine: "dtrtrs", info: Int(info)) }
91 + return solution
92 + }
93 +
94 + /// (X'X)⁻¹ = R⁻¹ R⁻ᵀ, computed from the k×k triangular factor.
95 + func crossProductInverse() throws -> [Double] {
96 + // Extract the k×k upper triangle of R (column-major).
97 + var r = [Double](repeating: 0, count: cols * cols)
98 + for j in 0..<cols {
99 + for i in 0...j {
100 + r[j * cols + i] = factored[j * rows + i]
101 + }
102 + }
103 + var uplo: Int8 = Int8(UInt8(ascii: "U"))
104 + var diag: Int8 = Int8(UInt8(ascii: "N"))
105 + var n = __CLPK_integer(cols)
106 + var lda = n
107 + var info: __CLPK_integer = 0
108 + dtrtri_(&uplo, &diag, &n, &r, &lda, &info)
109 + guard info == 0 else { throw Failure(routine: "dtrtri", info: Int(info)) }
110 +
111 + // XtXinv = Rinv · Rinvᵀ (symmetric k×k).
112 + var result = [Double](repeating: 0, count: cols * cols)
113 + for j in 0..<cols {
114 + for i in 0..<cols {
115 + var sum = 0.0
116 + // Rinv is upper triangular: Rinv[i, l] nonzero for l ≥ i.
117 + for l in max(i, j)..<cols {
118 + sum += r[l * cols + i] * r[l * cols + j]
119 + }
120 + result[j * cols + i] = sum
121 + }
122 + }
123 + return result
124 + }
125 +
126 + /// Explicit thin Q (n×k, column-major) via `dorgqr`, used for
127 + /// leverage values h_i = ‖q_i‖².
128 + func thinQ() throws -> [Double] {
129 + var a = factored
130 + var m = __CLPK_integer(rows)
131 + var n = __CLPK_integer(cols)
132 + var k = n
133 + var lda = m
134 + var info: __CLPK_integer = 0
135 + var tauCopy = tau
136 +
137 + var lwork: __CLPK_integer = -1
138 + var workQuery = [Double](repeating: 0, count: 1)
139 + dorgqr_(&m, &n, &k, &a, &lda, &tauCopy, &workQuery, &lwork, &info)
140 + lwork = __CLPK_integer(workQuery[0])
141 + var work = [Double](repeating: 0, count: max(1, Int(lwork)))
142 + dorgqr_(&m, &n, &k, &a, &lda, &tauCopy, &work, &lwork, &info)
143 + guard info == 0 else { throw Failure(routine: "dorgqr", info: Int(info)) }
144 + return a
145 + }
146 + }
147 +
148 + /// y ← A·x for column-major A (rows×cols).
149 + static func multiply(
150 + matrix a: [Double], rows: Int, cols: Int, vector x: [Double]
151 + ) -> [Double] {
152 + precondition(x.count == cols)
153 + var result = [Double](repeating: 0, count: rows)
154 + cblas_dgemv(
155 + CblasColMajor, CblasNoTrans,
156 + Int32(rows), Int32(cols),
157 + 1.0, a, Int32(rows), x, 1, 0.0, &result, 1
158 + )
159 + return result
160 + }
161 +
162 + /// Symmetric sandwich product B · M · B for k×k column-major matrices.
163 + static func sandwich(bread: [Double], meat: [Double], k: Int) -> [Double] {
164 + precondition(bread.count == k * k && meat.count == k * k)
165 + var bm = [Double](repeating: 0, count: k * k)
166 + cblas_dgemm(
167 + CblasColMajor, CblasNoTrans, CblasNoTrans,
168 + Int32(k), Int32(k), Int32(k),
169 + 1.0, bread, Int32(k), meat, Int32(k),
170 + 0.0, &bm, Int32(k)
171 + )
172 + var result = [Double](repeating: 0, count: k * k)
173 + cblas_dgemm(
174 + CblasColMajor, CblasNoTrans, CblasNoTrans,
175 + Int32(k), Int32(k), Int32(k),
176 + 1.0, bm, Int32(k), bread, Int32(k),
177 + 0.0, &result, Int32(k)
178 + )
179 + return result
180 + }
181 +
182 + /// Solves the symmetric positive-definite system A·x = b (k×k) via
183 + /// Cholesky — used for Wald tests on coefficient subsets.
184 + static func solveSymmetric(
185 + _ a: [Double], k: Int, rhs: [Double]
186 + ) throws -> [Double] {
187 + var factor = a
188 + var solution = rhs
189 + var uplo: Int8 = Int8(UInt8(ascii: "U"))
190 + var n = __CLPK_integer(k)
191 + var nrhs: __CLPK_integer = 1
192 + var lda = n
193 + var ldb = n
194 + var info: __CLPK_integer = 0
195 + dposv_(&uplo, &n, &nrhs, &factor, &lda, &solution, &ldb, &info)
196 + guard info == 0 else { throw Failure(routine: "dposv", info: Int(info)) }
197 + return solution
198 + }
199 +}
added MetrikaKit/Sources/ZQStats/OLS.swift +316 −0
@@ -0,0 +1,316 @@
1 +//
2 +// OLS.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +
12 +/// Variance estimator selection for linear regression.
13 +public enum ZQVarianceEstimator: Equatable, Sendable {
14 + /// Classical homoskedastic σ²(X'X)⁻¹.
15 + case classical
16 + /// Heteroskedasticity-consistent sandwich estimators. `robust` in
17 + /// Stata parlance maps to `.hc1`.
18 + case hc0, hc1, hc2, hc3
19 + /// Cluster-robust: `cluster` holds one group label per observation.
20 + case cluster([Int])
21 +}
22 +
23 +/// One row of a regression coefficient table.
24 +public struct ZQCoefficient: Equatable, Sendable {
25 + public var name: String
26 + public var estimate: Double
27 + public var standardError: Double
28 + public var tStatistic: Double
29 + public var pValue: Double
30 + public var confidenceLower: Double
31 + public var confidenceUpper: Double
32 +}
33 +
34 +/// Full OLS fit result.
35 +public struct ZQOLSResult: Equatable, Sendable {
36 + public var coefficients: [ZQCoefficient]
37 + public var observationCount: Int
38 + public var degreesOfFreedomResidual: Int
39 + /// Degrees of freedom used for t statistics and CIs: n−k classically,
40 + /// G−1 under clustering (Stata convention).
41 + public var inferenceDF: Double
42 + public var rSquared: Double
43 + public var adjustedRSquared: Double
44 + public var rootMSE: Double
45 + public var fStatistic: Double?
46 + public var fPValue: Double?
47 + public var fDF: (Double, Double)? {
48 + get { fDFStorage.map { ($0.first, $0.second) } }
49 + set { fDFStorage = newValue.map { FDF($0.0, $0.1) } }
50 + }
51 + public var clusterCount: Int?
52 + public var residuals: [Double]
53 +
54 + private struct FDF: Equatable, Sendable {
55 + let first: Double
56 + let second: Double
57 + init(_ first: Double, _ second: Double) {
58 + self.first = first
59 + self.second = second
60 + }
61 + var asTuple: (Double, Double) { (first, second) }
62 + }
63 +
64 + private var fDFStorage: FDF?
65 +
66 + public init(
67 + coefficients: [ZQCoefficient],
68 + observationCount: Int,
69 + degreesOfFreedomResidual: Int,
70 + inferenceDF: Double,
71 + rSquared: Double,
72 + adjustedRSquared: Double,
73 + rootMSE: Double,
74 + fStatistic: Double?,
75 + fPValue: Double?,
76 + fDF: (Double, Double)?,
77 + clusterCount: Int?,
78 + residuals: [Double]
79 + ) {
80 + self.coefficients = coefficients
81 + self.observationCount = observationCount
82 + self.degreesOfFreedomResidual = degreesOfFreedomResidual
83 + self.inferenceDF = inferenceDF
84 + self.rSquared = rSquared
85 + self.adjustedRSquared = adjustedRSquared
86 + self.rootMSE = rootMSE
87 + self.fStatistic = fStatistic
88 + self.fPValue = fPValue
89 + self.fDFStorage = fDF.map { FDF($0.0, $0.1) }
90 + self.clusterCount = clusterCount
91 + self.residuals = residuals
92 + }
93 +}
94 +
95 +public struct ZQStatsError: Error, Equatable, Sendable, CustomStringConvertible {
96 + public var message: String
97 + public init(_ message: String) { self.message = message }
98 + public var description: String { message }
99 +}
100 +
101 +/// Ordinary least squares on fully observed (already listwise-deleted)
102 +/// data. Solved by LAPACK QR (dgeqrf/dormqr) — X'X is never formed for
103 +/// the coefficient path (CLAUDE.md §5).
104 +public enum ZQOLS {
105 +
106 + /// - Parameters:
107 + /// - y: response, length n
108 + /// - predictors: named regressor columns (length n each), intercept
109 + /// excluded — it is appended automatically as `_cons` unless
110 + /// `includeConstant` is false.
111 + public static func fit(
112 + y: [Double],
113 + predictors: [(name: String, values: [Double])],
114 + includeConstant: Bool = true,
115 + variance: ZQVarianceEstimator = .classical,
116 + confidenceLevel: Double = 0.95
117 + ) throws -> ZQOLSResult {
118 + let n = y.count
119 + var names = predictors.map(\.name)
120 + if includeConstant { names.append("_cons") }
121 + let k = names.count
122 + guard n > k else {
123 + throw ZQStatsError("insufficient observations: n=\(n), k=\(k)")
124 + }
125 + for column in predictors where column.values.count != n {
126 + throw ZQStatsError("regressor '\(column.name)' has wrong length")
127 + }
128 +
129 + // Design matrix, column-major.
130 + var x = [Double]()
131 + x.reserveCapacity(n * k)
132 + for column in predictors { x.append(contentsOf: column.values) }
133 + if includeConstant { x.append(contentsOf: [Double](repeating: 1, count: n)) }
134 +
135 + let qr: LinearAlgebra.QR
136 + let beta: [Double]
137 + let xtxInverse: [Double]
138 + do {
139 + qr = try LinearAlgebra.QR(matrix: x, rows: n, cols: k)
140 + beta = try qr.solve(rhs: y)
141 + xtxInverse = try qr.crossProductInverse()
142 + } catch {
143 + throw ZQStatsError(
144 + "design matrix is rank deficient or QR failed: \(error)"
145 + )
146 + }
147 +
148 + // Residuals and fit statistics.
149 + let fitted = LinearAlgebra.multiply(matrix: x, rows: n, cols: k, vector: beta)
150 + var residuals = [Double](repeating: 0, count: n)
151 + var rss = 0.0
152 + for i in 0..<n {
153 + residuals[i] = y[i] - fitted[i]
154 + rss += residuals[i] * residuals[i]
155 + }
156 + let meanY = y.reduce(0, +) / Double(n)
157 + let tss = y.reduce(0) { $0 + ($1 - meanY) * ($1 - meanY) }
158 + let dfResidual = n - k
159 + let sigma2 = rss / Double(dfResidual)
160 + let rSquared = tss > 0 ? 1 - rss / tss : .nan
161 + let dfModel = k - (includeConstant ? 1 : 0)
162 + let adjustedRSquared =
163 + 1 - (1 - rSquared) * Double(n - (includeConstant ? 1 : 0)) / Double(dfResidual)
164 +
165 + // Covariance matrix.
166 + var clusterCount: Int? = nil
167 + let vce: [Double]
168 + switch variance {
169 + case .classical:
170 + vce = xtxInverse.map { $0 * sigma2 }
171 +
172 + case .hc0, .hc1, .hc2, .hc3:
173 + let leverage: [Double]?
174 + if variance == .hc2 || variance == .hc3 {
175 + let q = try qr.thinQ()
176 + var h = [Double](repeating: 0, count: n)
177 + for j in 0..<k {
178 + for i in 0..<n {
179 + let value = q[j * n + i]
180 + h[i] += value * value
181 + }
182 + }
183 + leverage = h
184 + } else {
185 + leverage = nil
186 + }
187 + var weights = [Double](repeating: 0, count: n)
188 + for i in 0..<n {
189 + let e2 = residuals[i] * residuals[i]
190 + switch variance {
191 + case .hc0: weights[i] = e2
192 + case .hc1: weights[i] = e2 * Double(n) / Double(dfResidual)
193 + case .hc2: weights[i] = e2 / (1 - leverage![i])
194 + case .hc3: weights[i] = e2 / ((1 - leverage![i]) * (1 - leverage![i]))
195 + default: break
196 + }
197 + }
198 + let meat = weightedCrossProduct(x: x, n: n, k: k, weights: weights)
199 + vce = LinearAlgebra.sandwich(bread: xtxInverse, meat: meat, k: k)
200 +
201 + case .cluster(let groups):
202 + guard groups.count == n else {
203 + throw ZQStatsError("cluster variable has wrong length")
204 + }
205 + var scores: [Int: [Double]] = [:]
206 + for i in 0..<n {
207 + var u = scores[groups[i]] ?? [Double](repeating: 0, count: k)
208 + for j in 0..<k {
209 + u[j] += x[j * n + i] * residuals[i]
210 + }
211 + scores[groups[i]] = u
212 + }
213 + let g = scores.count
214 + guard g > 1 else {
215 + throw ZQStatsError("cluster variable must define at least 2 groups")
216 + }
217 + clusterCount = g
218 + var meat = [Double](repeating: 0, count: k * k)
219 + for u in scores.values {
220 + for j in 0..<k {
221 + for i in 0..<k {
222 + meat[j * k + i] += u[i] * u[j]
223 + }
224 + }
225 + }
226 + // Stata regress small-sample factor: G/(G−1) · (n−1)/(n−k).
227 + let scale = Double(g) / Double(g - 1) * Double(n - 1) / Double(dfResidual)
228 + for index in meat.indices { meat[index] *= scale }
229 + vce = LinearAlgebra.sandwich(bread: xtxInverse, meat: meat, k: k)
230 + }
231 +
232 + // Inference: t statistics against n−k df, or G−1 under clustering.
233 + let inferenceDF: Double
234 + if let g = clusterCount {
235 + inferenceDF = Double(g - 1)
236 + } else {
237 + inferenceDF = Double(dfResidual)
238 + }
239 + let tCritical = ZQDistributions.studentTQuantile(
240 + 0.5 + confidenceLevel / 2, df: inferenceDF
241 + )
242 +
243 + var coefficients: [ZQCoefficient] = []
244 + for j in 0..<k {
245 + let se = vce[j * k + j].squareRoot()
246 + let t = beta[j] / se
247 + coefficients.append(ZQCoefficient(
248 + name: names[j],
249 + estimate: beta[j],
250 + standardError: se,
251 + tStatistic: t,
252 + pValue: ZQDistributions.tTestPValue(t, df: inferenceDF),
253 + confidenceLower: beta[j] - tCritical * se,
254 + confidenceUpper: beta[j] + tCritical * se
255 + ))
256 + }
257 +
258 + // Overall Wald F test on all non-constant coefficients.
259 + var fStatistic: Double? = nil
260 + var fPValue: Double? = nil
261 + var fDF: (Double, Double)? = nil
262 + if dfModel > 0 {
263 + let restricted = Array(0..<dfModel)
264 + var subVce = [Double](repeating: 0, count: dfModel * dfModel)
265 + var subBeta = [Double](repeating: 0, count: dfModel)
266 + for (jj, j) in restricted.enumerated() {
267 + subBeta[jj] = beta[j]
268 + for (ii, i) in restricted.enumerated() {
269 + subVce[jj * dfModel + ii] = vce[j * k + i]
270 + }
271 + }
272 + if let solved = try? LinearAlgebra.solveSymmetric(subVce, k: dfModel, rhs: subBeta) {
273 + let wald = zip(subBeta, solved).reduce(0) { $0 + $1.0 * $1.1 }
274 + let f = wald / Double(dfModel)
275 + fStatistic = f
276 + fDF = (Double(dfModel), inferenceDF)
277 + fPValue = ZQDistributions.fTestPValue(
278 + f, df1: Double(dfModel), df2: inferenceDF
279 + )
280 + }
281 + }
282 +
283 + return ZQOLSResult(
284 + coefficients: coefficients,
285 + observationCount: n,
286 + degreesOfFreedomResidual: dfResidual,
287 + inferenceDF: inferenceDF,
288 + rSquared: rSquared,
289 + adjustedRSquared: adjustedRSquared,
290 + rootMSE: sigma2.squareRoot(),
291 + fStatistic: fStatistic,
292 + fPValue: fPValue,
293 + fDF: fDF,
294 + clusterCount: clusterCount,
295 + residuals: residuals
296 + )
297 + }
298 +
299 + /// meat = Σ_i w_i · x_i x_iᵀ for column-major X.
300 + private static func weightedCrossProduct(
301 + x: [Double], n: Int, k: Int, weights: [Double]
302 + ) -> [Double] {
303 + var meat = [Double](repeating: 0, count: k * k)
304 + for j in 0..<k {
305 + for i in j..<k {
306 + var sum = 0.0
307 + for row in 0..<n {
308 + sum += weights[row] * x[i * n + row] * x[j * n + row]
309 + }
310 + meat[j * k + i] = sum
311 + meat[i * k + j] = sum
312 + }
313 + }
314 + return meat
315 + }
316 +}
added MetrikaKit/Sources/ZQStats/Summarize.swift +121 −0
@@ -0,0 +1,121 @@
1 +//
2 +// Summarize.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +
12 +/// Descriptive statistics for one variable, Stata `summarize` semantics.
13 +public struct ZQSummary: Equatable, Sendable {
14 + public var name: String
15 + public var observationCount: Int
16 + public var mean: Double
17 + /// Sample variance (n−1 denominator).
18 + public var variance: Double
19 + public var standardDeviation: Double
20 + public var minimum: Double
21 + public var maximum: Double
22 + /// Detail block (percentiles, skewness, kurtosis) — only populated for
23 + /// `summarize, detail`.
24 + public var detail: Detail?
25 +
26 + public struct Detail: Equatable, Sendable {
27 + /// Percentiles keyed by percent (1, 5, 10, 25, 50, 75, 90, 95, 99),
28 + /// computed with Stata's definition: for np/100 integral, the mean
29 + /// of adjacent order statistics; otherwise the next order statistic.
30 + public var percentiles: [Int: Double]
31 + /// Skewness m₃/m₂^{3/2} and kurtosis m₄/m₂² on central moments
32 + /// with n denominators (Stata definitions).
33 + public var skewness: Double
34 + public var kurtosis: Double
35 + public var smallest: [Double]
36 + public var largest: [Double]
37 + }
38 +}
39 +
40 +public enum ZQSummarize {
41 +
42 + /// Summarizes non-missing values. Returns nil when all values are
43 + /// missing (Stata prints a zero-observation row in that case; the
44 + /// caller renders it).
45 + public static func summary(
46 + name: String, values: [Double], missing: [Bool], detail: Bool = false
47 + ) -> ZQSummary? {
48 + var data: [Double] = []
49 + data.reserveCapacity(values.count)
50 + for i in 0..<values.count where !missing[i] {
51 + data.append(values[i])
52 + }
53 + let n = data.count
54 + guard n > 0 else { return nil }
55 +
56 + var sum = 0.0
57 + for v in data { sum += v }
58 + let mean = sum / Double(n)
59 +
60 + var m2 = 0.0, m3 = 0.0, m4 = 0.0
61 + var minimum = data[0], maximum = data[0]
62 + for v in data {
63 + let d = v - mean
64 + let d2 = d * d
65 + m2 += d2
66 + m3 += d2 * d
67 + m4 += d2 * d2
68 + if v < minimum { minimum = v }
69 + if v > maximum { maximum = v }
70 + }
71 + let variance = n > 1 ? m2 / Double(n - 1) : 0
72 +
73 + var summary = ZQSummary(
74 + name: name,
75 + observationCount: n,
76 + mean: mean,
77 + variance: variance,
78 + standardDeviation: variance.squareRoot(),
79 + minimum: minimum,
80 + maximum: maximum,
81 + detail: nil
82 + )
83 +
84 + if detail {
85 + let sorted = data.sorted()
86 + var percentiles: [Int: Double] = [:]
87 + for p in [1, 5, 10, 25, 50, 75, 90, 95, 99] {
88 + percentiles[p] = stataPercentile(sorted: sorted, percent: Double(p))
89 + }
90 + let mc2 = m2 / Double(n)
91 + let mc3 = m3 / Double(n)
92 + let mc4 = m4 / Double(n)
93 + summary.detail = ZQSummary.Detail(
94 + percentiles: percentiles,
95 + skewness: mc2 > 0 ? mc3 / pow(mc2, 1.5) : .nan,
96 + kurtosis: mc2 > 0 ? mc4 / (mc2 * mc2) : .nan,
97 + smallest: Array(sorted.prefix(4)),
98 + largest: Array(sorted.suffix(4))
99 + )
100 + }
101 + return summary
102 + }
103 +
104 + /// Stata percentile definition: with np = n·p/100, if np is integral
105 + /// the percentile is (x₍np₎ + x₍np+1₎)/2, otherwise x₍⌈np⌉₎ (1-based
106 + /// order statistics).
107 + static func stataPercentile(sorted: [Double], percent: Double) -> Double {
108 + let n = sorted.count
109 + precondition(n > 0)
110 + let np = Double(n) * percent / 100
111 + let isIntegral = np == np.rounded()
112 + if isIntegral {
113 + let index = Int(np)
114 + if index >= n { return sorted[n - 1] }
115 + if index < 1 { return sorted[0] }
116 + return 0.5 * (sorted[index - 1] + sorted[index])
117 + }
118 + let index = min(n, max(1, Int(np.rounded(.up))))
119 + return sorted[index - 1]
120 + }
121 +}
added MetrikaKit/Sources/ZQStats/ZQStatsModule.swift +13 −0
@@ -0,0 +1,13 @@
1 +//
2 +// ZQStatsModule.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +/// Module marker for ZQStats.
11 +public enum ZQStatsModule {
12 + public static let name = "ZQStats"
13 +}
added MetrikaKit/Tests/MetrikaKitTests/EngineTests.swift +147 −0
@@ -0,0 +1,147 @@
1 +//
2 +// EngineTests.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Testing
11 +import ZQEngine
12 +
13 +/// End-to-end tests: full command lines through the session, checking the
14 +/// returned scalar results against the R fixtures.
15 +@Suite("ZQEngine end-to-end", .serialized)
16 +struct EngineTests {
17 + let fixtures: Fixtures
18 + let session: ZQSession
19 +
20 + init() async throws {
21 + self.fixtures = try Fixtures()
22 + self.session = try ZQSession(discoverUserCommands: false)
23 + _ = try await session.execute("use \(fixtures.datasetURL.path)")
24 + _ = try await session.execute("gen log_rev = ln(revenue)")
25 + }
26 +
27 + @Test("use reports dataset dimensions")
28 + func loadedDimensions() async throws {
29 + let result = try await session.execute("count")
30 + #expect(result.scalars["N"] == 60)
31 + }
32 +
33 + @Test("regress with listwise deletion matches R")
34 + func regress() async throws {
35 + let result = try await session.execute("reg log_rev price")
36 + #expect(result.scalars["N"] == fixtures["ols_n"])
37 + expectClose(try #require(result.scalars["b_price"]), fixtures["ols_b_price"], "b[price]")
38 + expectClose(try #require(result.scalars["b__cons"]), fixtures["ols_b_cons"], "b[_cons]")
39 + expectClose(
40 + try #require(result.scalars["se_price"]),
41 + fixtures["ols_se_classical_price"], "se[price]"
42 + )
43 + expectClose(try #require(result.scalars["r2"]), fixtures["ols_r2"], "R²")
44 + #expect(result.text.contains("observations dropped due to missing values"))
45 + }
46 +
47 + @Test("regress, robust matches R HC1")
48 + func regressRobust() async throws {
49 + let result = try await session.execute("reg log_rev price, robust")
50 + expectClose(
51 + try #require(result.scalars["se_price"]),
52 + fixtures["ols_se_hc1_price"], "robust se[price]"
53 + )
54 + }
55 +
56 + @Test("regress, cluster matches R with Stata small-sample factor")
57 + func regressCluster() async throws {
58 + let result = try await session.execute("reg log_rev price, cluster(firm_id)")
59 + #expect(result.scalars["N_clust"] == fixtures["ols_G"])
60 + expectClose(
61 + try #require(result.scalars["se_price"]),
62 + fixtures["ols_se_cluster_price"], "cluster se[price]"
63 + )
64 + }
65 +
66 + @Test("factor variable expansion i.region matches R factor()")
67 + func regressFactor() async throws {
68 + let result = try await session.execute("reg log_rev price i.region")
69 + expectClose(try #require(result.scalars["b_price"]), fixtures["ols2_b_price"], "b[price]")
70 + expectClose(
71 + try #require(result.scalars["b_2.region"]),
72 + fixtures["ols2_b_region2"], "b[2.region]"
73 + )
74 + expectClose(
75 + try #require(result.scalars["b_3.region"]),
76 + fixtures["ols2_b_region3"], "b[3.region]"
77 + )
78 + expectClose(try #require(result.scalars["b__cons"]), fixtures["ols2_b_cons"], "b[_cons]")
79 + expectClose(try #require(result.scalars["r2"]), fixtures["ols2_r2"], "R²")
80 + }
81 +
82 + @Test("summarize scalars match R")
83 + func summarize() async throws {
84 + let result = try await session.execute("summarize revenue")
85 + expectClose(
86 + try #require(result.scalars["mean"]),
87 + fixtures["sum_revenue_mean"], "mean"
88 + )
89 + expectClose(try #require(result.scalars["sd"]), fixtures["sum_revenue_sd"], "sd")
90 + }
91 +
92 + @Test("if qualifier and count")
93 + func conditionalCount() async throws {
94 + let all = try await session.execute("count")
95 + let some = try await session.execute("count if missing(price)")
96 + #expect(all.scalars["N"] == 60)
97 + #expect(some.scalars["N"] == fixtures["ols_dropped"])
98 + }
99 +
100 + @Test("bootstrap is reproducible for a fixed seed")
101 + func bootstrapReproducible() async throws {
102 + let first = try await session.execute(
103 + "bootstrap, reps(200) seed(42): reg log_rev price"
104 + )
105 + let second = try await session.execute(
106 + "bootstrap, reps(200) seed(42): reg log_rev price"
107 + )
108 + #expect(first.scalars["se_price"] == second.scalars["se_price"])
109 + // Bootstrap SE should be in the neighborhood of the analytic one.
110 + let analytic = fixtures["ols_se_classical_price"]
111 + let bootstrap = try #require(first.scalars["se_price"])
112 + #expect(bootstrap > analytic / 3 && bootstrap < analytic * 3)
113 + }
114 +
115 + @Test("generate honors if, replace counts changes")
116 + func generateReplace() async throws {
117 + _ = try await session.execute("gen flag = 1 if price > 10")
118 + let count = try await session.execute("count if flag == 1")
119 + let expected = try await session.execute("count if price > 10")
120 + #expect(count.scalars["N"] == expected.scalars["N"])
121 + let replaced = try await session.execute("replace flag = 0 if missing(flag)")
122 + #expect(replaced.text.contains("real changes"))
123 + _ = try await session.execute("drop flag")
124 + }
125 +
126 + @Test("graph scatter produces a plot spec")
127 + func graphScatter() async throws {
128 + _ = try await session.execute("graph scatter log_rev price, by(region)")
129 + let plot = try #require(await session.lastPlot)
130 + #expect(plot.kind == .scatter)
131 + #expect(plot.series.count == 3) // three regions
132 + #expect(plot.xLabel == "price")
133 + }
134 +
135 + @Test("display evaluates scalar expressions")
136 + func display() async throws {
137 + let result = try await session.execute("display 2 + 2 * 3")
138 + #expect(result.text == "8")
139 + }
140 +
141 + @Test("unknown variable yields a helpful error")
142 + func unknownVariable() async throws {
143 + await #expect(throws: (any Error).self) {
144 + _ = try await session.execute("summarize nonexistent_var")
145 + }
146 + }
147 +}
added MetrikaKit/Tests/MetrikaKitTests/FixtureSupport.swift +61 −0
@@ -0,0 +1,61 @@
1 +//
2 +// FixtureSupport.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +import Testing
12 +
13 +/// Golden values produced by Tests/Fixtures/generate.R.
14 +struct Fixtures {
15 + let expected: [String: Double]
16 + let datasetURL: URL
17 +
18 + init() throws {
19 + guard let tsv = Bundle.module.url(
20 + forResource: "expected", withExtension: "tsv", subdirectory: "Fixtures"
21 + ), let csv = Bundle.module.url(
22 + forResource: "regression", withExtension: "csv", subdirectory: "Fixtures"
23 + ) else {
24 + throw NSError(domain: "Fixtures", code: 1, userInfo: [
25 + NSLocalizedDescriptionKey: "fixture resources missing — run Tests/Fixtures/generate.R",
26 + ])
27 + }
28 + var values: [String: Double] = [:]
29 + for line in try String(contentsOf: tsv, encoding: .utf8)
30 + .split(separator: "\n") {
31 + let parts = line.split(separator: "\t")
32 + guard parts.count == 2, let value = Double(parts[1]) else { continue }
33 + values[String(parts[0])] = value
34 + }
35 + self.expected = values
36 + self.datasetURL = csv
37 + }
38 +
39 + subscript(key: String) -> Double {
40 + guard let value = expected[key] else {
41 + Issue.record("missing fixture key '\(key)'")
42 + return .nan
43 + }
44 + return value
45 + }
46 +}
47 +
48 +/// Relative-tolerance comparison (CLAUDE.md §0: CPU paths match reference
49 +/// implementations to at least 1e-10 relative tolerance).
50 +func expectClose(
51 + _ actual: Double, _ expected: Double, rtol: Double = 1e-10,
52 + _ label: String = "", sourceLocation: SourceLocation = #_sourceLocation
53 +) {
54 + let scale = max(abs(expected), abs(actual), 1e-300)
55 + let relative = abs(actual - expected) / scale
56 + #expect(
57 + relative <= rtol,
58 + "\(label): actual \(actual) vs expected \(expected) (rel. err. \(relative))",
59 + sourceLocation: sourceLocation
60 + )
61 +}
added MetrikaKit/Tests/MetrikaKitTests/Fixtures/expected.tsv +47 −0
@@ -0,0 +1,47 @@
1 +sum_revenue_n 60
2 +sum_revenue_mean 22.041739716666665
3 +sum_revenue_var 75.456382944246272
4 +sum_revenue_sd 8.6865633563709341
5 +sum_revenue_min 9.9938909999999996
6 +sum_revenue_max 48.213110999999998
7 +sum_revenue_skewness 0.83927308973767201
8 +sum_revenue_kurtosis 3.0099405407589113
9 +ols_n 57
10 +ols_dropped 3
11 +ols_b_price -0.076945323253264752
12 +ols_b_cons 4.066766357217416
13 +ols_r2 0.81945138939762363
14 +ols_r2a 0.81616868738667137
15 +ols_rmse 0.16467909008395276
16 +ols_F 249.62710190070032
17 +ols_se_classical_price 0.0048700829803485389
18 +ols_se_classical_cons 0.069041588469169066
19 +ols_t_price -15.799591827028372
20 +ols_p_price 4.2432524575354669e-22
21 +ols_ci_lower_price -0.086705187644217502
22 +ols_ci_upper_price -0.067185458862312003
23 +ols_se_hc0_price 0.0046993275536147206
24 +ols_se_hc0_cons 0.070340462372229157
25 +ols_se_hc1_price 0.0047840069340958705
26 +ols_se_hc1_cons 0.071607960053223257
27 +ols_se_hc2_price 0.0048195183325538586
28 +ols_se_hc2_cons 0.072119681978632416
29 +ols_se_hc3_price 0.0049433978526440998
30 +ols_se_hc3_cons 0.073955625269990985
31 +ols_G 12
32 +ols_se_cluster_price 0.0053437360051300847
33 +ols_se_cluster_cons 0.071520043363908228
34 +ols_p_cluster_price 1.7511622894896076e-08
35 +ols2_b_price -0.078097115374682891
36 +ols2_b_region2 0.071445724920196046
37 +ols2_b_region3 0.21232048832355585
38 +ols2_b_cons 3.9851984093144015
39 +ols2_r2 0.87452597383166375
40 +dist_pt_2p5_df10 0.98427657788169554
41 +dist_pt_m1p3_df3 0.14223375436394847
42 +dist_pt_0p05_df57 0.51985140594489099
43 +dist_pf_3p7_2_30 0.96337776983412116
44 +dist_pf_0p5_5_100 0.22441048076451794
45 +dist_qt_0p975_df12 2.1788128296672284
46 +dist_qt_0p995_df4 4.6040948713499921
47 +dist_pnorm_1p64 0.94949741652589625
added MetrikaKit/Tests/MetrikaKitTests/Fixtures/regression.csv +61 −0
@@ -0,0 +1,61 @@
1 +revenue,price,region,firm_id
2 +13.073013,18.7221,1,1
3 +14.603161,19.0561,2,1
4 +37.036281,9.2921,3,1
5 +12.331106,17.4567,1,1
6 +20.199281,14.6262,2,1
7 +18.532705,12.7864,3,2
8 +13.442576,,1,2
9 +30.288376,7.02,2,2
10 +14.146638,14.8549,3,2
11 +15.789387,15.576,1,2
12 +24.085548,11.8661,2,3
13 +17.866099,15.7867,3,3
14 +13.358007,19.0201,1,3
15 +26.695153,8.8314,2,3
16 +20.905369,11.9344,3,3
17 +12.640513,19.1002,1,4
18 +11.071996,19.6734,2,4
19 +48.213111,6.7623,3,4
20 +19.400193,12.125,1,4
21 +22.781459,13.405,2,4
22 +15.854431,18.5605,3,5
23 +27.549049,7.0807,1,5
24 +15.637737,,2,5
25 +15.806717,19.2,3,5
26 +33.600334,6.2366,1,5
27 +22.746857,12.7132,2,6
28 +30.989393,10.8531,3,6
29 +12.510734,18.5861,1,6
30 +15.099717,11.7045,2,6
31 +17.10762,17.5401,3,6
32 +14.293473,16.0639,1,7
33 +15.713512,17.1658,2,7
34 +30.616712,10.8216,3,7
35 +19.841208,15.2775,1,7
36 +36.095565,5.0592,2,7
37 +20.003055,17.4937,3,8
38 +38.151955,5.11,1,8
39 +36.839992,8.1149,2,8
40 +17.29105,18.599,3,8
41 +19.569965,14.1767,1,8
42 +21.934484,,2,9
43 +26.14202,11.5366,3,9
44 +38.421147,5.5615,1,9
45 +10.899566,19.6031,2,9
46 +24.544112,11.4763,3,9
47 +12.655109,19.3636,1,10
48 +15.641427,18.3163,2,10
49 +22.233252,14.5997,3,10
50 +9.993891,19.5645,1,10
51 +16.320549,14.2826,2,10
52 +37.592103,10.0014,3,11
53 +25.093077,10.2012,1,11
54 +25.408455,10.9773,2,11
55 +17.119771,16.7704,3,11
56 +29.198899,5.584,1,11
57 +18.051847,16.2319,2,12
58 +19.195829,15.1592,3,12
59 +28.993336,7.569,1,12
60 +34.011476,8.9163,2,12
61 +27.274985,12.7162,3,12
added MetrikaKit/Tests/MetrikaKitTests/ParserTests.swift +174 −0
@@ -0,0 +1,174 @@
1 +//
2 +// ParserTests.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Testing
11 +@testable import ZQParser
12 +
13 +@Suite("ZQL parser golden tests")
14 +struct ParserTests {
15 + let parser = ZQCommandParser()
16 +
17 + @Test("use file")
18 + func useFile() throws {
19 + let command = try #require(try parser.parse("use sales.parquet"))
20 + #expect(command.verb == "use")
21 + #expect(command.argument == "sales.parquet")
22 + }
23 +
24 + @Test("summarize with detail option")
25 + func summarizeDetail() throws {
26 + let command = try #require(try parser.parse("summarize revenue price, detail"))
27 + #expect(command.verb == "summarize")
28 + #expect(command.varlist == [.simple("revenue"), .simple("price")])
29 + #expect(command.hasOption("detail"))
30 + }
31 +
32 + @Test("verb abbreviation resolution")
33 + func abbreviations() throws {
34 + #expect(try parser.parse("su price")?.verb == "summarize")
35 + #expect(try parser.parse("reg y x")?.verb == "regress")
36 + #expect(try parser.parse("gen z = 1")?.verb == "generate")
37 + #expect(try parser.parse("d")?.verb == "describe")
38 + }
39 +
40 + @Test("gen with expression")
41 + func generate() throws {
42 + let command = try #require(try parser.parse("gen log_rev = ln(revenue)"))
43 + #expect(command.verb == "generate")
44 + let assignment = try #require(command.assignment)
45 + #expect(assignment.target == "log_rev")
46 + #expect(assignment.expression == .call(name: "ln", arguments: [.variable("revenue")]))
47 + }
48 +
49 + @Test("regress with factor variable and options")
50 + func regressFactors() throws {
51 + let command = try #require(try parser.parse("reg log_rev price i.region, robust"))
52 + #expect(command.verb == "regress")
53 + #expect(command.varlist == [
54 + .simple("log_rev"), .simple("price"), .factor(op: "i", name: "region"),
55 + ])
56 + #expect(command.hasOption("robust"))
57 + }
58 +
59 + @Test("cluster option carries its argument")
60 + func clusterOption() throws {
61 + let command = try #require(try parser.parse("xtreg log_rev price, fe cluster(firm_id)"))
62 + #expect(command.verb == "xtreg")
63 + #expect(command.hasOption("fe"))
64 + #expect(command.option("cluster")?.firstArgument == "firm_id")
65 + }
66 +
67 + @Test("bootstrap prefix command")
68 + func bootstrapPrefix() throws {
69 + let command = try #require(
70 + try parser.parse("bootstrap, reps(100000) seed(42): reg log_rev price")
71 + )
72 + #expect(command.verb == "bootstrap")
73 + #expect(command.option("reps")?.firstArgument == "100000")
74 + #expect(command.option("seed")?.firstArgument == "42")
75 + let body = try #require(command.body)
76 + #expect(body.verb == "regress")
77 + #expect(body.varlist == [.simple("log_rev"), .simple("price")])
78 + }
79 +
80 + @Test("xtset panel declaration")
81 + func xtset() throws {
82 + let command = try #require(try parser.parse("xtset firm_id year"))
83 + #expect(command.verb == "xtset")
84 + #expect(command.varlist == [.simple("firm_id"), .simple("year")])
85 + }
86 +
87 + @Test("graph compound verb with by option")
88 + func graphScatter() throws {
89 + let command = try #require(try parser.parse("graph scatter log_rev price, by(region)"))
90 + #expect(command.verb == "graph")
91 + #expect(command.subverb == "scatter")
92 + #expect(command.option("by")?.firstArgument == "region")
93 + }
94 +
95 + @Test("if, in, and weight qualifiers")
96 + func qualifiers() throws {
97 + let command = try #require(
98 + try parser.parse("summarize price if revenue > 100 & !missing(price) in 1/50 [aweight = pop]")
99 + )
100 + #expect(command.range == ZQRange(lower: 1, upper: 50))
101 + #expect(command.weight == ZQWeight(kind: .aweight, expression: .variable("pop")))
102 + let condition = try #require(command.condition)
103 + #expect(condition == .binary(
104 + op: "&",
105 + .binary(op: ">", .variable("revenue"), .number(100)),
106 + .unary(op: "!", .call(name: "missing", arguments: [.variable("price")]))
107 + ))
108 + }
109 +
110 + @Test("continuous interaction c.age#c.age")
111 + func interaction() throws {
112 + let command = try #require(try parser.parse("reg wage c.age#c.age education"))
113 + #expect(command.varlist == [
114 + .simple("wage"),
115 + .interaction(
116 + [.factor(op: "c", name: "age"), .factor(op: "c", name: "age")],
117 + full: false
118 + ),
119 + .simple("education"),
120 + ])
121 + }
122 +
123 + @Test("missing-value literal in expressions")
124 + func missingLiteral() throws {
125 + let command = try #require(try parser.parse("count if income == ."))
126 + #expect(command.condition == .binary(op: "==", .variable("income"), .missing))
127 + }
128 +
129 + @Test("comments and blank lines yield nil")
130 + func comments() throws {
131 + #expect(try parser.parse("") == nil)
132 + #expect(try parser.parse(" ") == nil)
133 + #expect(try parser.parse("// a comment") == nil)
134 + #expect(try parser.parse("* stata-style comment") == nil)
135 + }
136 +
137 + @Test("unknown verb error carries column and suggestion")
138 + func unknownVerbSuggestion() {
139 + do {
140 + _ = try parser.parse("regresss y x")
141 + Issue.record("expected a parse error")
142 + } catch let error as ZQParseError {
143 + #expect(error.column == 1)
144 + #expect(error.suggestion == "regress")
145 + } catch {
146 + Issue.record("unexpected error type: \(error)")
147 + }
148 + }
149 +
150 + @Test("error cites the offending column")
151 + func errorColumn() {
152 + do {
153 + _ = try parser.parse("summarize price, level(")
154 + Issue.record("expected a parse error")
155 + } catch let error as ZQParseError {
156 + // The unclosed '(' is reported at end of line: one past the
157 + // final character of the 23-character command.
158 + #expect(error.column == 24)
159 + } catch {
160 + Issue.record("unexpected error type: \(error)")
161 + }
162 + }
163 +
164 + @Test("operator precedence in expressions")
165 + func precedence() throws {
166 + let command = try #require(try parser.parse("count if a + b * c == d ^ 2 ^ 3"))
167 + // b*c binds before +; ^ is right-associative.
168 + #expect(command.condition == .binary(
169 + op: "==",
170 + .binary(op: "+", .variable("a"), .binary(op: "*", .variable("b"), .variable("c"))),
171 + .binary(op: "^", .variable("d"), .binary(op: "^", .number(2), .number(3)))
172 + ))
173 + }
174 +}
added MetrikaKit/Tests/MetrikaKitTests/PhiloxTests.swift +92 −0
@@ -0,0 +1,92 @@
1 +//
2 +// PhiloxTests.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Testing
11 +import ZQGPU
12 +
13 +@Suite("Philox4x32-10 RNG")
14 +struct PhiloxTests {
15 +
16 + @Test("Random123 known-answer vectors")
17 + func knownAnswers() {
18 + // Zero counter, zero key.
19 + let zero = Philox4x32(seed: 0).block(counter: (0, 0, 0, 0))
20 + #expect(zero.0 == 0x6627_e8d5)
21 + #expect(zero.1 == 0xe169_c58d)
22 + #expect(zero.2 == 0xbc57_ac4c)
23 + #expect(zero.3 == 0x9b00_dbd8)
24 +
25 + // All-ones counter and key (Random123 kat_vectors).
26 + let ones = Philox4x32(seed: .max).block(
27 + counter: (0xffff_ffff, 0xffff_ffff, 0xffff_ffff, 0xffff_ffff)
28 + )
29 + #expect(ones.0 == 0x408f_276d)
30 + #expect(ones.1 == 0x41c8_3b0e)
31 + #expect(ones.2 == 0xa20b_c7c6)
32 + #expect(ones.3 == 0x6d54_51fd)
33 + }
34 +
35 + @Test("same seed produces identical streams — reproducibility blocker")
36 + func determinism() {
37 + let a = Philox4x32(seed: 42)
38 + let b = Philox4x32(seed: 42)
39 + for index in [0, 1, 2, 1_000, 1_000_000] as [UInt64] {
40 + #expect(a.word(at: index) == b.word(at: index))
41 + #expect(a.uniform(at: index) == b.uniform(at: index))
42 + }
43 + }
44 +
45 + @Test("different seeds diverge")
46 + func seedsDiffer() {
47 + let a = Philox4x32(seed: 42)
48 + let b = Philox4x32(seed: 43)
49 + let differing = (0..<64).count { a.word(at: UInt64($0)) != b.word(at: UInt64($0)) }
50 + #expect(differing > 60)
51 + }
52 +
53 + @Test("uniforms are in [0, 1) and pass a coarse mean check")
54 + func uniformRange() {
55 + let generator = Philox4x32(seed: 7)
56 + var sum = 0.0
57 + let n = 100_000
58 + for i in 0..<n {
59 + let u = generator.uniform(at: UInt64(i))
60 + #expect(u >= 0 && u < 1)
61 + sum += u
62 + }
63 + let mean = sum / Double(n)
64 + #expect(abs(mean - 0.5) < 0.005, "mean \(mean) too far from 0.5")
65 + }
66 +
67 + @Test("bounded integers stay in range and hit every value")
68 + func integerBounds() {
69 + let generator = Philox4x32(seed: 11)
70 + var seen = Set<Int>()
71 + for i in 0..<10_000 {
72 + let value = generator.integer(at: UInt64(i), bound: 10)
73 + #expect(value >= 0 && value < 10)
74 + seen.insert(value)
75 + }
76 + #expect(seen.count == 10)
77 + }
78 +
79 + @Test("bootstrap replicates are independent of computation order")
80 + func resamplingAddressable() {
81 + let generator = Philox4x32(seed: 42)
82 + let early = ZQResampling.pairsBootstrapIndices(
83 + replicate: 5, sampleSize: 100, generator: generator
84 + )
85 + // Recomputing replicate 5 in isolation gives the same draw.
86 + let again = ZQResampling.pairsBootstrapIndices(
87 + replicate: 5, sampleSize: 100, generator: Philox4x32(seed: 42)
88 + )
89 + #expect(early == again)
90 + #expect(early.allSatisfy { (0..<100).contains($0) })
91 + }
92 +}
added MetrikaKit/Tests/MetrikaKitTests/StatsTests.swift +218 −0
@@ -0,0 +1,218 @@
1 +//
2 +// StatsTests.swift
3 +// Metrika
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Contact: contact@spboucher.ai
7 +// Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +//
9 +
10 +import Foundation
11 +import Testing
12 +import ZQData
13 +import ZQStats
14 +
15 +/// Numerical tests against R-generated fixtures (Tests/Fixtures/generate.R)
16 +/// at 1e-10 relative tolerance (CLAUDE.md §0).
17 +@Suite("ZQStats vs R fixtures")
18 +struct StatsTests {
19 + let fixtures: Fixtures
20 + let y: [Double] // log(revenue), estimation sample
21 + let price: [Double]
22 + let region: [Double]
23 + let firmID: [Int]
24 +
25 + init() async throws {
26 + self.fixtures = try Fixtures()
27 +
28 + // Load the fixture CSV through the real data path (DuckDB).
29 + let store = try ZQDataStore()
30 + let frame = try await store.load(contentsOf: fixtures.datasetURL)
31 + let (revenue, revenueMissing) = try frame.requireNumeric("revenue")
32 + let (priceAll, priceMissing) = try frame.requireNumeric("price")
33 + let (regionAll, _) = try frame.requireNumeric("region")
34 + let (firmAll, _) = try frame.requireNumeric("firm_id")
35 +
36 + // Listwise deletion, mirroring generate.R.
37 + var y: [Double] = [], price: [Double] = []
38 + var region: [Double] = [], firm: [Int] = []
39 + for i in 0..<frame.rowCount where !priceMissing[i] && !revenueMissing[i] {
40 + y.append(Foundation.log(revenue[i]))
41 + price.append(priceAll[i])
42 + region.append(regionAll[i])
43 + firm.append(Int(firmAll[i]))
44 + }
45 + self.y = y
46 + self.price = price
47 + self.region = region
48 + self.firmID = firm
49 + }
50 +
51 + @Test("summarize matches R moments")
52 + func summarize() async throws {
53 + let store = try ZQDataStore()
54 + let frame = try await store.load(contentsOf: fixtures.datasetURL)
55 + let (values, missing) = try frame.requireNumeric("revenue")
56 + let summary = try #require(ZQSummarize.summary(
57 + name: "revenue", values: values, missing: missing, detail: true
58 + ))
59 + #expect(summary.observationCount == Int(fixtures["sum_revenue_n"]))
60 + expectClose(summary.mean, fixtures["sum_revenue_mean"], "mean")
61 + expectClose(summary.variance, fixtures["sum_revenue_var"], "variance")
62 + expectClose(summary.standardDeviation, fixtures["sum_revenue_sd"], "sd")
63 + expectClose(summary.minimum, fixtures["sum_revenue_min"], "min")
64 + expectClose(summary.maximum, fixtures["sum_revenue_max"], "max")
65 + let detail = try #require(summary.detail)
66 + expectClose(detail.skewness, fixtures["sum_revenue_skewness"], "skewness")
67 + expectClose(detail.kurtosis, fixtures["sum_revenue_kurtosis"], "kurtosis")
68 + }
69 +
70 + @Test("OLS point estimates and classical inference")
71 + func classicalOLS() throws {
72 + let result = try ZQOLS.fit(
73 + y: y, predictors: [("price", price)], variance: .classical
74 + )
75 + #expect(result.observationCount == Int(fixtures["ols_n"]))
76 + let priceCoef = result.coefficients[0]
77 + let constant = result.coefficients[1]
78 + expectClose(priceCoef.estimate, fixtures["ols_b_price"], "b[price]")
79 + expectClose(constant.estimate, fixtures["ols_b_cons"], "b[_cons]")
80 + expectClose(priceCoef.standardError, fixtures["ols_se_classical_price"], "se[price]")
81 + expectClose(constant.standardError, fixtures["ols_se_classical_cons"], "se[_cons]")
82 + expectClose(priceCoef.tStatistic, fixtures["ols_t_price"], "t[price]")
83 + expectClose(priceCoef.pValue, fixtures["ols_p_price"], "p[price]")
84 + expectClose(priceCoef.confidenceLower, fixtures["ols_ci_lower_price"], "ci lower")
85 + expectClose(priceCoef.confidenceUpper, fixtures["ols_ci_upper_price"], "ci upper")
86 + expectClose(result.rSquared, fixtures["ols_r2"], "R²")
87 + expectClose(result.adjustedRSquared, fixtures["ols_r2a"], "adj. R²")
88 + expectClose(result.rootMSE, fixtures["ols_rmse"], "root MSE")
89 + expectClose(try #require(result.fStatistic), fixtures["ols_F"], "F")
90 + }
91 +
92 + @Test("HC0–HC3 sandwich standard errors", arguments: [
93 + ("hc0", ZQVarianceEstimator.hc0),
94 + ("hc1", ZQVarianceEstimator.hc1),
95 + ("hc2", ZQVarianceEstimator.hc2),
96 + ("hc3", ZQVarianceEstimator.hc3),
97 + ])
98 + func robustSE(_ label: String, _ variance: ZQVarianceEstimator) throws {
99 + let result = try ZQOLS.fit(
100 + y: y, predictors: [("price", price)], variance: variance
101 + )
102 + expectClose(
103 + result.coefficients[0].standardError,
104 + fixtures["ols_se_\(label)_price"], "\(label) se[price]"
105 + )
106 + expectClose(
107 + result.coefficients[1].standardError,
108 + fixtures["ols_se_\(label)_cons"], "\(label) se[_cons]"
109 + )
110 + }
111 +
112 + @Test("cluster-robust standard errors (Stata small-sample factor)")
113 + func clusterSE() throws {
114 + let result = try ZQOLS.fit(
115 + y: y, predictors: [("price", price)], variance: .cluster(firmID)
116 + )
117 + #expect(result.clusterCount == Int(fixtures["ols_G"]))
118 + expectClose(
119 + result.coefficients[0].standardError,
120 + fixtures["ols_se_cluster_price"], "cluster se[price]"
121 + )
122 + expectClose(
123 + result.coefficients[1].standardError,
124 + fixtures["ols_se_cluster_cons"], "cluster se[_cons]"
125 + )
126 + expectClose(
127 + result.coefficients[0].pValue,
128 + fixtures["ols_p_cluster_price"], "cluster p[price] (t on G−1 df)"
129 + )
130 + }
131 +
132 + @Test("OLS invariance: permuting observations leaves estimates unchanged")
133 + func permutationInvariance() throws {
134 + let base = try ZQOLS.fit(y: y, predictors: [("price", price)])
135 + var order = Array(0..<y.count)
136 + // Deterministic shuffle (LCG) — no seeding dependency in tests.
137 + var state: UInt64 = 88_172_645_463_325_252
138 + for i in stride(from: order.count - 1, through: 1, by: -1) {
139 + state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407
140 + order.swapAt(i, Int(state % UInt64(i + 1)))
141 + }
142 + let permuted = try ZQOLS.fit(
143 + y: order.map { y[$0] },
144 + predictors: [("price", order.map { price[$0] })]
145 + )
146 + for (a, b) in zip(base.coefficients, permuted.coefficients) {
147 + expectClose(a.estimate, b.estimate, "permuted b[\(a.name)]")
148 + expectClose(a.standardError, b.standardError, "permuted se[\(a.name)]")
149 + }
150 + }
151 +
152 + @Test("OLS invariance: scaling a regressor rescales its coefficient")
153 + func scalingInvariance() throws {
154 + let base = try ZQOLS.fit(y: y, predictors: [("price", price)])
155 + let scaled = try ZQOLS.fit(
156 + y: y, predictors: [("price", price.map { $0 * 100 })]
157 + )
158 + expectClose(
159 + scaled.coefficients[0].estimate * 100,
160 + base.coefficients[0].estimate,
161 + "scaled coefficient"
162 + )
163 + expectClose(scaled.rSquared, base.rSquared, "scaled R²")
164 + }
165 +}
166 +
167 +@Suite("Distribution functions vs R")
168 +struct DistributionTests {
169 + let fixtures = try! Fixtures()
170 +
171 + @Test("Student t CDF")
172 + func studentT() {
173 + expectClose(
174 + ZQDistributions.studentTCDF(2.5, df: 10),
175 + fixtures["dist_pt_2p5_df10"], rtol: 1e-12, "pt(2.5, 10)"
176 + )
177 + expectClose(
178 + ZQDistributions.studentTCDF(-1.3, df: 3),
179 + fixtures["dist_pt_m1p3_df3"], rtol: 1e-12, "pt(-1.3, 3)"
180 + )
181 + expectClose(
182 + ZQDistributions.studentTCDF(0.05, df: 57),
183 + fixtures["dist_pt_0p05_df57"], rtol: 1e-12, "pt(0.05, 57)"
184 + )
185 + }
186 +
187 + @Test("F CDF")
188 + func fisher() {
189 + expectClose(
190 + ZQDistributions.fCDF(3.7, df1: 2, df2: 30),
191 + fixtures["dist_pf_3p7_2_30"], rtol: 1e-12, "pf(3.7, 2, 30)"
192 + )
193 + expectClose(
194 + ZQDistributions.fCDF(0.5, df1: 5, df2: 100),
195 + fixtures["dist_pf_0p5_5_100"], rtol: 1e-12, "pf(0.5, 5, 100)"
196 + )
197 + }
198 +
199 + @Test("t quantile")
200 + func tQuantile() {
201 + expectClose(
202 + ZQDistributions.studentTQuantile(0.975, df: 12),
203 + fixtures["dist_qt_0p975_df12"], rtol: 1e-10, "qt(0.975, 12)"
204 + )
205 + expectClose(
206 + ZQDistributions.studentTQuantile(0.995, df: 4),
207 + fixtures["dist_qt_0p995_df4"], rtol: 1e-10, "qt(0.995, 4)"
208 + )
209 + }
210 +
211 + @Test("normal CDF")
212 + func normal() {
213 + expectClose(
214 + ZQDistributions.normalCDF(1.64),
215 + fixtures["dist_pnorm_1p64"], rtol: 1e-14, "pnorm(1.64)"
216 + )
217 + }
218 +}
added Tests/Bench/bench.sh +54 −0
@@ -0,0 +1,54 @@
1 +#!/bin/bash
2 +#
3 +# bench.sh — Metrika
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Contact: contact@spboucher.ai
7 +# Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +#
9 +# Performance benchmark harness (CLAUDE.md §8). Generates a synthetic
10 +# dataset and times the core pipeline through metrika-cli. CI compares
11 +# against the budget table; regressions >10% fail the build.
12 +#
13 +# Usage: Tests/Bench/bench.sh [rows] (default 1,000,000)
14 +#
15 +set -euo pipefail
16 +export LC_ALL=C
17 +
18 +ROWS="${1:-1000000}"
19 +ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
20 +DATA="$(mktemp -d)/bench.csv"
21 +trap 'rm -rf "$(dirname "$DATA")"' EXIT
22 +
23 +echo "generating $ROWS rows…"
24 +awk -v n="$ROWS" 'BEGIN {
25 + srand(42)
26 + print "revenue,price,region,firm_id"
27 + for (i = 1; i <= n; i++) {
28 + price = 5 + 15 * rand()
29 + region = (i % 3) + 1
30 + revenue = exp(4 - 0.08 * price + 0.1 * region + 0.15 * (rand() - 0.5))
31 + printf "%.6f,%.4f,%d,%d\n", revenue, price, region, (i % 500) + 1
32 + }
33 +}' > "$DATA"
34 +
35 +cd "$ROOT/MetrikaKit"
36 +swift build -c release --product metrika-cli >/dev/null
37 +CLI=".build/release/metrika-cli"
38 +
39 +bench() { # bench <label> <command...>
40 + local label="$1"; shift
41 + local start end
42 + start=$(python3 -c 'import time; print(time.time())')
43 + "$CLI" "$@" >/dev/null
44 + end=$(python3 -c 'import time; print(time.time())')
45 + printf "%-28s %8.3f s\n" "$label" "$(echo "$end - $start" | bc)"
46 +}
47 +
48 +bench "use csv ($ROWS rows)" -e "use $DATA"
49 +bench "use + summarize" -e "use $DATA" -e "summarize"
50 +bench "use + reg (robust)" -e "use $DATA" -e "gen log_rev = ln(revenue)" \
51 + -e "reg log_rev price, robust"
52 +bench "use + bootstrap(1000)" -e "use $DATA" -e "keep in 1/100000" \
53 + -e "gen log_rev = ln(revenue)" \
54 + -e "bootstrap, reps(1000) seed(42): reg log_rev price"
added Tests/Fixtures/generate.R +161 −0
@@ -0,0 +1,161 @@
1 +#!/usr/bin/env Rscript
2 +#
3 +# generate.R — Metrika
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Contact: contact@spboucher.ai
7 +# Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +#
9 +# Generates the golden numerical fixtures for MetrikaKit estimator tests
10 +# (CLAUDE.md §9). Writes a reference dataset (CSV) plus expected values
11 +# (TSV, key<TAB>value at 17 significant digits) into the Swift test
12 +# resources at MetrikaKit/Tests/MetrikaKitTests/Fixtures/.
13 +#
14 +# All expected values are computed from the CSV **after** a write/read
15 +# round-trip so Swift and R consume bit-identical inputs.
16 +#
17 +
18 +out_dir <- file.path(
19 + dirname(dirname(normalizePath(sub("--file=", "", grep("--file=", commandArgs(FALSE), value = TRUE))))),
20 + "..", "MetrikaKit", "Tests", "MetrikaKitTests", "Fixtures"
21 +)
22 +out_dir <- normalizePath(out_dir, mustWork = FALSE)
23 +dir.create(out_dir, recursive = TRUE, showWarnings = FALSE)
24 +
25 +# ---------------------------------------------------------------- dataset
26 +set.seed(42)
27 +n <- 60
28 +region <- rep(1:3, length.out = n)
29 +firm_id <- rep(1:12, each = 5)
30 +price <- round(runif(n, 5, 20), 4)
31 +noise <- round(rnorm(n, 0, 0.15), 6)
32 +log_rev <- 4 - 0.08 * price + 0.10 * (region == 2) + 0.20 * (region == 3) + noise
33 +revenue <- round(exp(log_rev), 6)
34 +
35 +# Inject missing values to exercise listwise deletion.
36 +price_missing <- price
37 +price_missing[c(7, 23, 41)] <- NA
38 +
39 +data <- data.frame(
40 + revenue = revenue,
41 + price = price_missing,
42 + region = region,
43 + firm_id = firm_id
44 +)
45 +csv_path <- file.path(out_dir, "regression.csv")
46 +write.csv(data, csv_path, row.names = FALSE, quote = FALSE, na = "")
47 +
48 +# Round-trip: recompute everything from what was actually written.
49 +data <- read.csv(csv_path)
50 +
51 +sink_path <- file.path(out_dir, "expected.tsv")
52 +lines <- character(0)
53 +emit <- function(key, value) {
54 + lines <<- c(lines, sprintf("%s\t%.17g", key, value))
55 +}
56 +
57 +# ------------------------------------------------------------- summarize
58 +x <- data$revenue
59 +emit("sum_revenue_n", length(x))
60 +emit("sum_revenue_mean", mean(x))
61 +emit("sum_revenue_var", var(x))
62 +emit("sum_revenue_sd", sd(x))
63 +emit("sum_revenue_min", min(x))
64 +emit("sum_revenue_max", max(x))
65 +m2 <- mean((x - mean(x))^2)
66 +m3 <- mean((x - mean(x))^3)
67 +m4 <- mean((x - mean(x))^4)
68 +emit("sum_revenue_skewness", m3 / m2^1.5) # Stata definition
69 +emit("sum_revenue_kurtosis", m4 / m2^2)
70 +
71 +# ------------------------------------------------------------ regression
72 +# Estimation sample: listwise deletion on price.
73 +complete <- !is.na(data$price)
74 +d <- data[complete, ]
75 +d$log_rev <- log(d$revenue)
76 +n_est <- nrow(d)
77 +
78 +X <- cbind(price = d$price, `_cons` = 1)
79 +y <- d$log_rev
80 +k <- ncol(X)
81 +
82 +fit <- lm(log_rev ~ price, data = d)
83 +beta <- coef(fit) # (Intercept), price
84 +resid <- residuals(fit)
85 +XtXinv <- solve(t(X) %*% X)
86 +df_r <- n_est - k
87 +sigma2 <- sum(resid^2) / df_r
88 +
89 +emit("ols_n", n_est)
90 +emit("ols_dropped", sum(!complete))
91 +emit("ols_b_price", beta["price"])
92 +emit("ols_b_cons", beta["(Intercept)"])
93 +emit("ols_r2", summary(fit)$r.squared)
94 +emit("ols_r2a", summary(fit)$adj.r.squared)
95 +emit("ols_rmse", sqrt(sigma2))
96 +emit("ols_F", summary(fit)$fstatistic[["value"]])
97 +
98 +V_classical <- sigma2 * XtXinv
99 +emit("ols_se_classical_price", sqrt(V_classical["price", "price"]))
100 +emit("ols_se_classical_cons", sqrt(V_classical["_cons", "_cons"]))
101 +t_price <- beta["price"] / sqrt(V_classical["price", "price"])
102 +emit("ols_t_price", t_price)
103 +emit("ols_p_price", 2 * pt(-abs(t_price), df_r))
104 +crit <- qt(0.975, df_r)
105 +emit("ols_ci_lower_price", beta["price"] - crit * sqrt(V_classical["price", "price"]))
106 +emit("ols_ci_upper_price", beta["price"] + crit * sqrt(V_classical["price", "price"]))
107 +
108 +# HC0–HC3 sandwich estimators (manual, no packages).
109 +h <- hatvalues(fit)
110 +sandwich_se <- function(w) {
111 + meat <- t(X * w) %*% X
112 + V <- XtXinv %*% meat %*% XtXinv
113 + sqrt(diag(V))
114 +}
115 +se_hc0 <- sandwich_se(resid^2)
116 +se_hc1 <- sandwich_se(resid^2 * n_est / df_r)
117 +se_hc2 <- sandwich_se(resid^2 / (1 - h))
118 +se_hc3 <- sandwich_se(resid^2 / (1 - h)^2)
119 +emit("ols_se_hc0_price", se_hc0["price"])
120 +emit("ols_se_hc0_cons", se_hc0["_cons"])
121 +emit("ols_se_hc1_price", se_hc1["price"])
122 +emit("ols_se_hc1_cons", se_hc1["_cons"])
123 +emit("ols_se_hc2_price", se_hc2["price"])
124 +emit("ols_se_hc2_cons", se_hc2["_cons"])
125 +emit("ols_se_hc3_price", se_hc3["price"])
126 +emit("ols_se_hc3_cons", se_hc3["_cons"])
127 +
128 +# Cluster-robust with Stata regress small-sample factor.
129 +cl <- d$firm_id
130 +u <- rowsum(X * resid, cl)
131 +G <- length(unique(cl))
132 +meat_cl <- (G / (G - 1)) * ((n_est - 1) / df_r) * (t(u) %*% u)
133 +V_cl <- XtXinv %*% meat_cl %*% XtXinv
134 +emit("ols_G", G)
135 +emit("ols_se_cluster_price", sqrt(V_cl["price", "price"]))
136 +emit("ols_se_cluster_cons", sqrt(V_cl["_cons", "_cons"]))
137 +t_cl <- beta["price"] / sqrt(V_cl["price", "price"])
138 +emit("ols_p_cluster_price", 2 * pt(-abs(t_cl), G - 1))
139 +
140 +# Factor-variable regression: log_rev ~ price + i.region (base = 1).
141 +fit2 <- lm(log_rev ~ price + factor(region), data = d)
142 +b2 <- coef(fit2)
143 +emit("ols2_b_price", b2[["price"]])
144 +emit("ols2_b_region2", b2[["factor(region)2"]])
145 +emit("ols2_b_region3", b2[["factor(region)3"]])
146 +emit("ols2_b_cons", b2[["(Intercept)"]])
147 +emit("ols2_r2", summary(fit2)$r.squared)
148 +
149 +# --------------------------------------------------------- distributions
150 +emit("dist_pt_2p5_df10", pt(2.5, 10))
151 +emit("dist_pt_m1p3_df3", pt(-1.3, 3))
152 +emit("dist_pt_0p05_df57", pt(0.05, 57))
153 +emit("dist_pf_3p7_2_30", pf(3.7, 2, 30))
154 +emit("dist_pf_0p5_5_100", pf(0.5, 5, 100))
155 +emit("dist_qt_0p975_df12", qt(0.975, 12))
156 +emit("dist_qt_0p995_df4", qt(0.995, 4))
157 +emit("dist_pnorm_1p64", pnorm(1.64))
158 +
159 +writeLines(lines, sink_path)
160 +cat("wrote", csv_path, "\n")
161 +cat("wrote", sink_path, "with", length(lines), "expected values\n")
added project.yml +37 −0
@@ -0,0 +1,37 @@
1 +name: Metrika
2 +options:
3 + bundleIdPrefix: ai.spboucher
4 + deploymentTarget:
5 + macOS: "14.0"
6 + createIntermediateGroups: true
7 +
8 +packages:
9 + MetrikaKit:
10 + path: MetrikaKit
11 +
12 +targets:
13 + Metrika:
14 + type: application
15 + platform: macOS
16 + sources:
17 + - Metrika
18 + dependencies:
19 + - package: MetrikaKit
20 + product: MetrikaKit
21 + settings:
22 + base:
23 + PRODUCT_BUNDLE_IDENTIFIER: ai.spboucher.metrika
24 + MARKETING_VERSION: 0.1.0
25 + CURRENT_PROJECT_VERSION: 1
26 + SWIFT_VERSION: "6.0"
27 + SWIFT_STRICT_CONCURRENCY: complete
28 + ARCHS: arm64
29 + ONLY_ACTIVE_ARCH: YES
30 + ENABLE_HARDENED_RUNTIME: YES
31 + ENABLE_APP_SANDBOX: YES
32 + CODE_SIGN_ENTITLEMENTS: Metrika/Metrika.entitlements
33 + CODE_SIGN_STYLE: Automatic
34 + GENERATE_INFOPLIST_FILE: YES
35 + ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
36 + INFOPLIST_KEY_NSHumanReadableCopyright: "© 2026 Simon-Pierre Boucher. All rights reserved."
37 + INFOPLIST_KEY_NSPrincipalClass: NSApplication
added scripts/ExportOptions.plist +9 −0
@@ -0,0 +1,9 @@
1 +<?xml version="1.0" encoding="UTF-8"?>
2 +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
3 + "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
4 +<plist version="1.0"><dict>
5 + <key>method</key><string>developer-id</string>
6 + <key>teamID</key><string><TEAM_ID></string>
7 + <key>signingStyle</key><string>automatic</string>
8 + <key>destination</key><string>export</string>
9 +</dict></plist>
added scripts/check_headers.sh +45 −0
@@ -0,0 +1,45 @@
1 +#!/bin/bash
2 +#
3 +# check_headers.sh — Metrika
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Contact: contact@spboucher.ai
7 +# Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +#
9 +# Rejects any source file (.swift, .metal, .sh, .py, .zyq) missing the
10 +# mandatory Metrika file header (CLAUDE.md §0). With --staged, checks only
11 +# files staged in git (pre-commit mode); otherwise checks the whole tree.
12 +#
13 +set -euo pipefail
14 +
15 +ROOT="$(cd "$(dirname "$0")/.." && pwd)"
16 +MODE="${1:-all}"
17 +
18 +if [[ "$MODE" == "--staged" ]]; then
19 + FILES=$(git -C "$ROOT" diff --cached --name-only --diff-filter=ACMR \
20 + | grep -E '\.(swift|metal|sh|py|zyq)$' || true)
21 +else
22 + FILES=$(cd "$ROOT" && find . \
23 + -not -path './.git/*' -not -path '*/.build/*' -not -path '*/build/*' \
24 + -not -path '*/DerivedData/*' -not -path '*/.swiftpm/*' \
25 + \( -name '*.swift' -o -name '*.metal' -o -name '*.sh' -o -name '*.py' -o -name '*.zyq' \) \
26 + | sed 's|^\./||')
27 +fi
28 +
29 +FAIL=0
30 +for f in $FILES; do
31 + [[ -f "$ROOT/$f" ]] || continue
32 + HEAD_BLOCK=$(head -n 12 "$ROOT/$f")
33 + if ! grep -q 'Author: Simon-Pierre Boucher' <<<"$HEAD_BLOCK" \
34 + || ! grep -q 'Contact: contact@spboucher.ai' <<<"$HEAD_BLOCK"; then
35 + echo "❌ missing Metrika header: $f"
36 + FAIL=1
37 + fi
38 +done
39 +
40 +if [[ $FAIL -ne 0 ]]; then
41 + echo ""
42 + echo "Every source file must begin with the Metrika header (see CLAUDE.md §0)."
43 + exit 1
44 +fi
45 +echo "✅ header check passed"
added scripts/install_hooks.sh +14 −0
@@ -0,0 +1,14 @@
1 +#!/bin/bash
2 +#
3 +# install_hooks.sh — Metrika
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Contact: contact@spboucher.ai
7 +# Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +#
9 +# Installs the repo git hooks. Run once after cloning.
10 +#
11 +set -euo pipefail
12 +ROOT="$(cd "$(dirname "$0")/.." && pwd)"
13 +install -m 755 "$ROOT/scripts/pre-commit" "$ROOT/.git/hooks/pre-commit"
14 +echo "✅ pre-commit hook installed"
added scripts/make_icns.sh +41 −0
@@ -0,0 +1,41 @@
1 +#!/bin/bash
2 +#
3 +# make_icns.sh — Metrika
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Contact: contact@spboucher.ai
7 +# Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +#
9 +# Rasterizes Assets/AppIcon/metrika-icon.svg to all required sizes and
10 +# produces Assets/AppIcon/Metrika.icns via iconutil (CLAUDE.md §11).
11 +# Requires rsvg-convert (brew install librsvg); falls back to qlmanage.
12 +#
13 +set -euo pipefail
14 +
15 +ROOT="$(cd "$(dirname "$0")/.." && pwd)"
16 +SVG="$ROOT/Assets/AppIcon/metrika-icon.svg"
17 +ICONSET="$ROOT/Assets/AppIcon/Metrika.iconset"
18 +ICNS="$ROOT/Assets/AppIcon/Metrika.icns"
19 +
20 +[[ -f "$SVG" ]] || { echo "❌ missing $SVG"; exit 1; }
21 +rm -rf "$ICONSET" && mkdir -p "$ICONSET"
22 +
23 +rasterize() { # rasterize <px> <outfile>
24 + local px="$1" out="$2"
25 + if command -v rsvg-convert >/dev/null; then
26 + rsvg-convert -w "$px" -h "$px" "$SVG" -o "$out"
27 + else
28 + qlmanage -t -s "$px" -o "$(dirname "$out")" "$SVG" >/dev/null
29 + mv "$(dirname "$out")/$(basename "$SVG").png" "$out"
30 + fi
31 +}
32 +
33 +# macOS iconset naming: size @1x and @2x pairs.
34 +for size in 16 32 128 256 512; do
35 + rasterize "$size" "$ICONSET/icon_${size}x${size}.png"
36 + rasterize "$((size*2))" "$ICONSET/icon_${size}x${size}@2x.png"
37 +done
38 +
39 +iconutil -c icns "$ICONSET" -o "$ICNS"
40 +rm -rf "$ICONSET"
41 +echo "✅ $ICNS"
added scripts/pre-commit +13 −0
@@ -0,0 +1,13 @@
1 +#!/bin/bash
2 +#
3 +# pre-commit — Metrika
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Contact: contact@spboucher.ai
7 +# Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +#
9 +# Git pre-commit hook: rejects staged source files missing the Metrika
10 +# header (CLAUDE.md §0). Installed by scripts/install_hooks.sh.
11 +#
12 +set -euo pipefail
13 +exec "$(git rev-parse --show-toplevel)/scripts/check_headers.sh" --staged
added scripts/release.sh +59 −0
@@ -0,0 +1,59 @@
1 +#!/bin/bash
2 +#
3 +# release.sh — Metrika
4 +#
5 +# Author: Simon-Pierre Boucher
6 +# Contact: contact@spboucher.ai
7 +# Copyright © 2026 Simon-Pierre Boucher. All rights reserved.
8 +#
9 +# Release pipeline (CLAUDE.md §10): archive → export → verify → DMG →
10 +# sign → notarize → staple.
11 +#
12 +# ⚠️ PLACEHOLDERS: fill <TEAM_ID> and <KEYCHAIN_PROFILE_NAME> from the
13 +# local credentials folder before the first release build. Never commit
14 +# real credentials.
15 +#
16 +set -euo pipefail
17 +
18 +APP="Metrika"
19 +SCHEME="Metrika"
20 +BUILD_DIR="build"
21 +DMG="${APP}.dmg"
22 +
23 +# 1. Archive
24 +xcodebuild -scheme "$SCHEME" -configuration Release -arch arm64 \
25 + -archivePath "$BUILD_DIR/$APP.xcarchive" archive
26 +
27 +# 2. Export with Developer ID
28 +xcodebuild -exportArchive \
29 + -archivePath "$BUILD_DIR/$APP.xcarchive" \
30 + -exportOptionsPlist scripts/ExportOptions.plist \
31 + -exportPath "$BUILD_DIR/export"
32 +
33 +# 3. Verify hardened runtime + entitlements
34 +codesign -dv --verbose=4 "$BUILD_DIR/export/$APP.app"
35 +codesign --verify --deep --strict "$BUILD_DIR/export/$APP.app"
36 +
37 +# 4. Build DMG (create-dmg, background art in Assets/DMG/)
38 +create-dmg \
39 + --volname "$APP" \
40 + --window-size 540 380 \
41 + --icon-size 128 \
42 + --icon "$APP.app" 130 190 \
43 + --app-drop-link 400 190 \
44 + --background "Assets/DMG/background.png" \
45 + "$DMG" "$BUILD_DIR/export/"
46 +
47 +# 5. Sign the DMG itself
48 +codesign --sign "Developer ID Application: Simon-Pierre Boucher (<TEAM_ID>)" "$DMG"
49 +
50 +# 6. Notarize & wait
51 +xcrun notarytool submit "$DMG" \
52 + --keychain-profile "<KEYCHAIN_PROFILE_NAME>" \
53 + --wait
54 +
55 +# 7. Staple
56 +xcrun stapler staple "$DMG"
57 +xcrun stapler validate "$DMG"
58 +
59 +echo "✅ $DMG notarized and stapled."
60