Repository Public
1import Foundation2import SheetCore3import Statistics4import AnalysisKit56/// Correlation matrix with p-values.7enum CorrelationTool {8 static let descriptor: ToolDescriptor = {9 var t = ToolDescriptor(10 id: "relate.correlation", title: L("Correlation Matrix"), group: Common.groupRelate, systemImage: "square.grid.3x3",11 summary: L("Pairwise correlations between numeric columns with the p-value of each coefficient."),12 fields: [13 .range("data", L("Data range")),14 .columns("vars", L("Variables")),15 .choice("method", L("Coefficient"), [FieldOption("pearson", L("Pearson")), FieldOption("spearman", L("Spearman (ranks)")),16 FieldOption("kendall", L("Kendall τ-b"))]),17 .choice("missing", L("Missing values"), [FieldOption("listwise", L("Drop incomplete rows (listwise)")),18 FieldOption("pairwise", L("Use every available pair (pairwise)"))]),19 Common.levelField(),20 .alpha(L("Significance level α")),21 ],22 info: [23 L("Pearson: t test with n − 2 degrees of freedom; confidence interval by Fisher's z transformation."),24 L("Spearman: the same t approximation applied to the rank correlation (no confidence interval)."),25 L("Kendall τ-b: normal approximation with the tie-corrected variance (no confidence interval)."),26 L("p-values are two-sided and are not adjusted for multiple comparisons."),27 ],28 run: run)29 t.demo = { _, v in v.set("data", range: "Regression!A1:G121"); v.set("vars", columns: [0, 1, 2, 3]) }30 return t31 }()3233 static func method(_ v: FormValues) -> CorrelationMethod {34 switch v.choice("method") {35 case "spearman": .spearman36 case "kendall": .kendall37 default: .pearson38 }39 }4041 static func run(_ input: ToolInput) throws -> ToolResult {42 let frame = try input.frame()43 let cols = try input.columns("vars", atLeast: 2)44 let method = method(input.values)45 let alpha = try Common.alpha(input.values), level = try Common.level(input.values)46 let listwise = input.values.choice("missing") != "pairwise"47 var data = cols.map(\.numbers)48 var note: String49 if listwise {50 let (rows, kept) = DataFrame.listwise(data)51 data = kept52 note = L("Listwise deletion: %d of %d rows are complete and were used for every pair.", rows.count, frame.rowCount)53 } else {54 note = L("Pairwise deletion: each pair uses the rows where both values are present.")55 }56 let k = cols.count57 var r = [[Double]](repeating: [Double](repeating: .nan, count: k), count: k)58 var p = r59 var counts = [[Int]](repeating: [Int](repeating: 0, count: k), count: k)60 var tests = [String: HypothesisTest]()61 for i in 0..<k {62 for j in i..<k {63 let (_, pair) = DataFrame.listwise([data[i], data[j]])64 counts[i][j] = pair[0].count; counts[j][i] = pair[0].count65 if i == j { r[i][i] = pair[0].count > 0 ? 1 : .nan; continue }66 guard let t = try? Inference.correlationTest(pair[0], pair[1], method: method, level: level) else { continue }67 r[i][j] = t.estimate?.value ?? .nan; r[j][i] = r[i][j]68 p[i][j] = t.pValue; p[j][i] = t.pValue69 tests["\(i),\(j)"] = t70 try input.checkCancelled()71 }72 }73 let names = cols.map(\.name)74 func matrix(_ title: String, _ value: (Int, Int) -> Value, _ format: CellFormat) -> ReportTable {75 var rows: [[Value]] = [[.text("")] + names.map { .text($0) }]76 for i in 0..<k { rows.append([.text(names[i])] + (0..<k).map { value(i, $0) }) }77 var t = ReportTable(title: title, rows: rows)78 for c in 1...k { t.columnFormats[c] = format }79 return t80 }81 let methodName = descriptor.field("method").flatMap { f -> String? in82 if case .choice(let options, _) = f.kind { return options.first { $0.id == input.values.choice("method") }?.label }83 return nil84 } ?? ""85 var blocks: [ResultBlock] = []86 if k == 2, let t = tests["0,1"] {87 blocks += Common.testBlocks(t, alpha: alpha, null: L("the two variables are not associated (correlation = 0)"), locale: input.locale)88 let (_, pair) = DataFrame.listwise([data[0], data[1]])89 blocks.append(.chart(ChartBlock(title: L("%@ against %@", names[1], names[0]),90 spec: ChartKit.scatter([(names[1], pair[0], pair[1])], xTitle: names[0], yTitle: names[1], trend: method == .pearson),91 summary: L("Scatter plot of %d points; %@ correlation %@.", pair[0].count, methodName, Fmt.fixed(r[0][1], 4, locale: input.locale)))))92 }93 blocks.append(.table(matrix(L("Correlation matrix (%@)", methodName), { .finite(r[$0][$1]) }, .fixed(4))))94 var pTable = matrix(L("p-values (two-sided)"), { i, j in i == j ? .blank : .finite(p[i][j]) }, .pValue)95 let significant = (0..<k).flatMap { i in ((i + 1)..<k).filter { p[i][$0] < alpha }.map { (i, $0) } }96 pTable.note = L("%d of %d pairs have p < α = %@ (no adjustment for multiple comparisons).", significant.count, k * (k - 1) / 2,97 Fmt.fixed(alpha, alpha < 0.01 ? 4 : 2, locale: input.locale))98 blocks.append(.table(pTable))99 blocks.append(.table(matrix(L("Observations per pair"), { .number(Double(counts[$0][$1])) }, .integer)))100 if k > 2 {101 blocks.append(.interpretation(L("At α = %@, the hypothesis of no association is rejected for %d of %d pairs and not rejected for the others; each test is considered on its own.",102 Fmt.fixed(alpha, alpha < 0.01 ? 4 : 2, locale: input.locale), significant.count, k * (k - 1) / 2)))103 }104 blocks.append(.note([note, L("Blank cells, text and error values are treated as missing.")]))105 return ToolResult(title: L("Correlation Matrix"), subtitle: L("%@ — %d variables", frame.label, k), blocks: blocks, sheetName: L("Correlation"))106 }107}108