Repository Public License
1// ORIX Secure2// File: Packages/ORIXSecureEngines/Tests/ORIXSecureEnginesTests/SecurityAssessmentTests.swift3// Description: Security assessment: each automatic finding in each state, checklist semantics (user-reported, expiry, optional4// items), summary arithmetic, and the honesty rules every finding must satisfy.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ORIXSecureModel10import Testing11@testable import ORIXSecureEngines1213@Suite struct SecurityAssessmentTests {14 let engine = SecurityAssessmentEngine()15 let now = Fixture.now1617 private func find(_ assessment: SecurityAssessment, _ id: String) -> Finding? { assessment.all.first { $0.id == id } }1819 @Test func healthyDeviceHasNoAttentionAndNeverClaimsMore() {20 let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], now: now)21 #expect(assessment.actions.isEmpty)22 #expect(find(assessment, "auth.passcode")?.state == .verified)23 #expect(find(assessment, "auth.biometry")?.state == .verified)24 #expect(find(assessment, "app.integrity")?.state == .verified)25 // "No sign observed" is a heuristic, and says what it cannot see.26 #expect(find(assessment, "app.integrity")?.confidence == .heuristic)27 #expect(find(assessment, "app.integrity")?.notInspected.isEmpty == false)28 // Nothing the user has to confirm is counted as verified.29 #expect(assessment.checklist.filter { $0.state == .verified }.isEmpty)30 #expect(assessment.pendingConfirmations.count == ChecklistCatalog.security.filter { !$0.isOptional }.count)31 #expect(!assessment.notAssessable.isEmpty)32 #expect(assessment.notAssessable.contains { $0.id == "malware.scan" })33 #expect(Fixture.assertHonest(assessment.all).isEmpty, "\(Fixture.assertHonest(assessment.all))")34 }3536 @Test func missingPasscodeIsHighSeverityWithEvidence() {37 let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(passcode: .measured(false), enrolled: .measured(false)), responses: [], now: now)38 let finding = find(assessment, "auth.passcode")39 #expect(finding?.state == .attention)40 #expect(finding?.severity == .high)41 #expect(finding?.confidence == .measured)42 #expect(finding?.evidence.first?.source?.contains("LAContext") == true)43 #expect(assessment.actions.first?.id == "auth.passcode")44 #expect(find(assessment, "auth.biometry")?.severity == .low)45 }4647 @Test func unreadableSignalsBecomeNotAvailableNotDefaults() {48 let signals = Fixture.signals(passcode: .unavailable(.failed, "x"), enrolled: .unavailable(.unsupportedOnDevice, "no sensor"), enclave: false,49 attest: .unavailable(.simulator, "sim"), debugger: .unavailable(.failed, "x"), checks: 0,50 protection: .unavailable(.simulator, "sim"), captured: .unavailable(.notRequested, "n/a"))51 let assessment = engine.assess(device: Fixture.device(simulator: true), signals: signals, responses: [], now: now)52 for id in ["auth.passcode", "auth.biometry", "device.secureEnclave", "app.attest", "app.debugger", "app.integrity", "app.dataProtection", "device.screenCapture"] {53 #expect(find(assessment, id)?.state == .notAvailable, "\(id)")54 }55 #expect(assessment.summary.attention == 0)56 #expect(Fixture.assertHonest(assessment.all).isEmpty)57 }5859 @Test func integrityIndicatorsAreHeuristicAndListed() {60 let indicators = [IntegrityIndicator(id: "path:/var/jb", observation: "/var/jb", source: "FileManager.fileExists"),61 IntegrityIndicator(id: "dyld:frida", observation: "/usr/lib/FridaGadget.dylib", source: "_dyld_get_image_name")]62 let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(indicators: indicators), responses: [], now: now)63 let finding = find(assessment, "app.integrity")64 #expect(finding?.state == .attention)65 #expect(finding?.severity == .high)66 #expect(finding?.confidence == .heuristic)67 #expect(finding?.evidence.map(\.value) == ["/var/jb", "/usr/lib/FridaGadget.dylib"])68 #expect(finding?.explanation.contains("expected") == true)69 }7071 @Test func debuggerScreenCaptureProtectionAndLockout() {72 let signals = Fixture.signals(lockedOut: true, debugger: .measured(true), protection: .measured(FileProtectionType.completeUntilFirstUserAuthentication.rawValue),73 captured: .measured(true))74 let assessment = engine.assess(device: Fixture.device(), signals: signals, responses: [], now: now)75 #expect(find(assessment, "app.debugger")?.state == .attention)76 #expect(find(assessment, "device.screenCapture")?.state == .attention)77 #expect(find(assessment, "app.dataProtection")?.state == .attention)78 #expect(find(assessment, "auth.biometry")?.title.contains("locked") == true)79 #expect(Fixture.assertHonest(assessment.all).isEmpty, "\(Fixture.assertHonest(assessment.all))")80 }8182 @Test func outdatedSystemIsReportedOnlyAgainstADatedReference() {83 let current = engine.assess(device: Fixture.device(os: "26.5", major: 26, sdk: "26.5"), signals: Fixture.signals(), responses: [], now: now)84 #expect(find(current, "device.os.outdated") == nil)85 let oneBehind = engine.assess(device: Fixture.device(os: "26.5", major: 26, sdk: "27.2"), signals: Fixture.signals(), responses: [], now: now)86 #expect(find(oneBehind, "device.os.outdated")?.severity == .low)87 let twoBehind = engine.assess(device: Fixture.device(os: "18.7", major: 18, sdk: "27.2"), signals: Fixture.signals(), responses: [], now: now)88 #expect(find(twoBehind, "device.os.outdated")?.severity == .moderate)89 let unknown = engine.assess(device: Fixture.device(sdk: nil), signals: Fixture.signals(), responses: [], now: now)90 #expect(find(unknown, "device.os.outdated") == nil)91 // The installed version is attached to the update item as evidence, which still needs the user's confirmation.92 let update = find(current, "sec.updates.current")93 #expect(update?.state == .userConfirmationRequired)94 #expect(update?.evidence.contains { $0.value == "iOS 26.5" } == true)95 }9697 @Test func restartSuggestionUsesTheBootDate() {98 let recent = engine.assess(device: Fixture.device(bootDaysAgo: 2), signals: Fixture.signals(), responses: [], now: now)99 #expect(find(recent, "device.restart")?.state == .verified)100 let old = engine.assess(device: Fixture.device(bootDaysAgo: 45), signals: Fixture.signals(), responses: [], now: now)101 #expect(find(old, "device.restart")?.state == .attention)102 #expect(find(old, "device.restart")?.severity == .low)103 #expect(find(old, "device.restart")?.title.contains("45") == true)104 }105106 @Test func contributedFindingsAreCarriedAndCounted() {107 let extra = Finding(id: "net.proxy", category: .network, title: "Proxy", state: .attention, severity: .moderate, confidence: .measured,108 verification: .publicAPI, evidence: [Evidence("Proxy", "10.0.0.1:8080")], explanation: "e", recommendation: "r")109 let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], contributed: [extra], now: now)110 #expect(assessment.actions.map(\.id) == ["net.proxy"])111 #expect(assessment.summary.attention == 1)112 #expect(assessment.summary.highestSeverity == .moderate)113 }114115 @Test func summaryCountsStatesAndCoverageIsNotASecurityLevel() {116 let responses = ChecklistCatalog.security.filter { !$0.isOptional }.map { ChecklistResponse(itemID: $0.id, answer: $0.expected, answeredAt: now) }117 let full = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: responses, now: now)118 #expect(full.summary.coverage == 1)119 #expect(full.summary.awaitingConfirmation == 0)120 #expect(full.summary.userConfirmed == responses.count)121 #expect(full.summary.total == full.all.count)122 let none = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], now: now)123 let coverage = none.summary.coverage ?? 0124 #expect(coverage > 0 && coverage < 1)125 #expect(AssessmentSummary(findings: []).coverage == nil)126 #expect(AssessmentSummary.methodology.first?.contains("does not compute a security score") == true)127 }128129 @Test func assessmentRoundTripsThroughJSON() throws {130 let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], now: now)131 let data = try JSONEncoder().encode(assessment)132 let decoded = try JSONDecoder().decode(SecurityAssessment.self, from: data)133 #expect(decoded == assessment)134 }135}136137@Suite struct ChecklistTests {138 let engine = SecurityChecklistEngine()139 let now = Fixture.now140141 @Test func catalogIsWellFormed() {142 let items = ChecklistCatalog.all143 #expect(Set(items.map(\.id)).count == items.count)144 #expect(ChecklistCatalog.security.count >= 15)145 #expect(ChecklistCatalog.privacyItems.count >= 12)146 for item in items {147 #expect(!item.title.isEmpty && !item.question.isEmpty && !item.why.isEmpty && !item.action.isEmpty, "\(item.id)")148 #expect(item.path.count >= 2, "\(item.id)")149 #expect(item.question.hasSuffix("?") || item.question.contains("?"), "\(item.id)")150 #expect(item.expected == .yes || item.expected == .no, "\(item.id)")151 #expect(item.isOptional == (item.importance == .none), "\(item.id): optional items carry no severity, others do")152 #expect(item.validityDays >= 30)153 #expect(item.id.hasPrefix(item.domain == .security ? "sec." : "priv."), "\(item.id)")154 }155 #expect(ChecklistCatalog.item("sec.sdp")?.group == .deviceAccess)156 #expect(ChecklistCatalog.item("nope") == nil)157 }158159 @Test func everyAnswerMapsToTheRightState() throws {160 let item = try #require(ChecklistCatalog.item("sec.account.2fa"))161 let unanswered = engine.finding(for: item, response: nil, now: now)162 #expect(unanswered.state == .userConfirmationRequired)163 #expect(unanswered.confidence == Confidence.none)164 #expect(unanswered.verification == .notVerifiable)165166 let good = engine.finding(for: item, response: .init(itemID: item.id, answer: .yes, answeredAt: now), now: now)167 #expect(good.state == .userConfirmed)168 #expect(good.confidence == .userReported)169 #expect(good.verification == .userConfirmed)170 #expect(good.severity == Severity.none)171 #expect(good.evidence.contains { $0.source == "user" })172173 let bad = engine.finding(for: item, response: .init(itemID: item.id, answer: .no, answeredAt: now), now: now)174 #expect(bad.state == .attention)175 #expect(bad.severity == .high)176 #expect(bad.confidence == .userReported)177 #expect(bad.recommendation == item.action)178179 let unsure = engine.finding(for: item, response: .init(itemID: item.id, answer: .notSure, answeredAt: now), now: now)180 #expect(unsure.state == .userConfirmationRequired)181 let notApplicable = engine.finding(for: item, response: .init(itemID: item.id, answer: .notApplicable, answeredAt: now), now: now)182 #expect(notApplicable.state == .notAvailable)183 #expect(Fixture.assertHonest([unanswered, good, bad, unsure, notApplicable]).isEmpty)184 }185186 @Test func answersExpire() throws {187 let item = try #require(ChecklistCatalog.item("sec.updates.current"))188 #expect(item.validityDays == 30)189 let fresh = ChecklistResponse(itemID: item.id, answer: .yes, answeredAt: now.addingTimeInterval(-29 * 86_400))190 let stale = ChecklistResponse(itemID: item.id, answer: .yes, answeredAt: now.addingTimeInterval(-31 * 86_400))191 #expect(engine.finding(for: item, response: fresh, now: now).state == .userConfirmed)192 let expired = engine.finding(for: item, response: stale, now: now)193 #expect(expired.state == .userConfirmationRequired)194 #expect(expired.explanation.contains("30"))195 }196197 @Test func optionalItemsNeverBecomeProblems() throws {198 let item = try #require(ChecklistCatalog.item("sec.lockdown"))199 #expect(engine.finding(for: item, response: nil, now: now).state == .notAssessed)200 let off = engine.finding(for: item, response: .init(itemID: item.id, answer: .no, answeredAt: now), now: now)201 #expect(off.state == .userConfirmed)202 #expect(off.severity == Severity.none)203 #expect(off.explanation.contains("optional"))204 }205206 @Test func latestResponseWins() {207 let item = ChecklistCatalog.security[0]208 let responses = [ChecklistResponse(itemID: item.id, answer: .no, answeredAt: now.addingTimeInterval(-100)),209 ChecklistResponse(itemID: item.id, answer: .yes, answeredAt: now)]210 let findings = engine.findings(for: [item], responses: responses, now: now)211 #expect(findings.first?.state == .userConfirmed)212 }213214 @Test func everyCatalogItemProducesHonestFindingsForEveryAnswer() {215 var all: [Finding] = []216 for item in ChecklistCatalog.all {217 all.append(engine.finding(for: item, response: nil, now: now))218 for answer in ChecklistAnswer.allCases {219 all.append(engine.finding(for: item, response: .init(itemID: item.id, answer: answer, answeredAt: now), now: now))220 }221 }222 #expect(all.count == ChecklistCatalog.all.count * 5)223 let problems = Fixture.assertHonest(all)224 #expect(problems.isEmpty, "\(problems.prefix(5))")225 #expect(!all.contains { $0.state == .verified }, "a user-assisted item must never be presented as verified by the app")226 }227}228