SPB Git

Repository Public License

SecurityAssessmentTests.swift 13.7 KB 228 lines swift

1// ORIX Secure2// File: Packages/ORIXSecureEngines/Tests/ORIXSecureEnginesTests/SecurityAssessmentTests.swift3// Description: Security assessment: each automatic finding in each state, checklist semantics (user-reported, expiry, optional4//              items), summary arithmetic, and the honesty rules every finding must satisfy.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ORIXSecureModel10import Testing11@testable import ORIXSecureEngines1213@Suite struct SecurityAssessmentTests {14    let engine = SecurityAssessmentEngine()15    let now = Fixture.now1617    private func find(_ assessment: SecurityAssessment, _ id: String) -> Finding? { assessment.all.first { $0.id == id } }1819    @Test func healthyDeviceHasNoAttentionAndNeverClaimsMore() {20        let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], now: now)21        #expect(assessment.actions.isEmpty)22        #expect(find(assessment, "auth.passcode")?.state == .verified)23        #expect(find(assessment, "auth.biometry")?.state == .verified)24        #expect(find(assessment, "app.integrity")?.state == .verified)25        // "No sign observed" is a heuristic, and says what it cannot see.26        #expect(find(assessment, "app.integrity")?.confidence == .heuristic)27        #expect(find(assessment, "app.integrity")?.notInspected.isEmpty == false)28        // Nothing the user has to confirm is counted as verified.29        #expect(assessment.checklist.filter { $0.state == .verified }.isEmpty)30        #expect(assessment.pendingConfirmations.count == ChecklistCatalog.security.filter { !$0.isOptional }.count)31        #expect(!assessment.notAssessable.isEmpty)32        #expect(assessment.notAssessable.contains { $0.id == "malware.scan" })33        #expect(Fixture.assertHonest(assessment.all).isEmpty, "\(Fixture.assertHonest(assessment.all))")34    }3536    @Test func missingPasscodeIsHighSeverityWithEvidence() {37        let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(passcode: .measured(false), enrolled: .measured(false)), responses: [], now: now)38        let finding = find(assessment, "auth.passcode")39        #expect(finding?.state == .attention)40        #expect(finding?.severity == .high)41        #expect(finding?.confidence == .measured)42        #expect(finding?.evidence.first?.source?.contains("LAContext") == true)43        #expect(assessment.actions.first?.id == "auth.passcode")44        #expect(find(assessment, "auth.biometry")?.severity == .low)45    }4647    @Test func unreadableSignalsBecomeNotAvailableNotDefaults() {48        let signals = Fixture.signals(passcode: .unavailable(.failed, "x"), enrolled: .unavailable(.unsupportedOnDevice, "no sensor"), enclave: false,49                                      attest: .unavailable(.simulator, "sim"), debugger: .unavailable(.failed, "x"), checks: 0,50                                      protection: .unavailable(.simulator, "sim"), captured: .unavailable(.notRequested, "n/a"))51        let assessment = engine.assess(device: Fixture.device(simulator: true), signals: signals, responses: [], now: now)52        for id in ["auth.passcode", "auth.biometry", "device.secureEnclave", "app.attest", "app.debugger", "app.integrity", "app.dataProtection", "device.screenCapture"] {53            #expect(find(assessment, id)?.state == .notAvailable, "\(id)")54        }55        #expect(assessment.summary.attention == 0)56        #expect(Fixture.assertHonest(assessment.all).isEmpty)57    }5859    @Test func integrityIndicatorsAreHeuristicAndListed() {60        let indicators = [IntegrityIndicator(id: "path:/var/jb", observation: "/var/jb", source: "FileManager.fileExists"),61                          IntegrityIndicator(id: "dyld:frida", observation: "/usr/lib/FridaGadget.dylib", source: "_dyld_get_image_name")]62        let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(indicators: indicators), responses: [], now: now)63        let finding = find(assessment, "app.integrity")64        #expect(finding?.state == .attention)65        #expect(finding?.severity == .high)66        #expect(finding?.confidence == .heuristic)67        #expect(finding?.evidence.map(\.value) == ["/var/jb", "/usr/lib/FridaGadget.dylib"])68        #expect(finding?.explanation.contains("expected") == true)69    }7071    @Test func debuggerScreenCaptureProtectionAndLockout() {72        let signals = Fixture.signals(lockedOut: true, debugger: .measured(true), protection: .measured(FileProtectionType.completeUntilFirstUserAuthentication.rawValue),73                                      captured: .measured(true))74        let assessment = engine.assess(device: Fixture.device(), signals: signals, responses: [], now: now)75        #expect(find(assessment, "app.debugger")?.state == .attention)76        #expect(find(assessment, "device.screenCapture")?.state == .attention)77        #expect(find(assessment, "app.dataProtection")?.state == .attention)78        #expect(find(assessment, "auth.biometry")?.title.contains("locked") == true)79        #expect(Fixture.assertHonest(assessment.all).isEmpty, "\(Fixture.assertHonest(assessment.all))")80    }8182    @Test func outdatedSystemIsReportedOnlyAgainstADatedReference() {83        let current = engine.assess(device: Fixture.device(os: "26.5", major: 26, sdk: "26.5"), signals: Fixture.signals(), responses: [], now: now)84        #expect(find(current, "device.os.outdated") == nil)85        let oneBehind = engine.assess(device: Fixture.device(os: "26.5", major: 26, sdk: "27.2"), signals: Fixture.signals(), responses: [], now: now)86        #expect(find(oneBehind, "device.os.outdated")?.severity == .low)87        let twoBehind = engine.assess(device: Fixture.device(os: "18.7", major: 18, sdk: "27.2"), signals: Fixture.signals(), responses: [], now: now)88        #expect(find(twoBehind, "device.os.outdated")?.severity == .moderate)89        let unknown = engine.assess(device: Fixture.device(sdk: nil), signals: Fixture.signals(), responses: [], now: now)90        #expect(find(unknown, "device.os.outdated") == nil)91        // The installed version is attached to the update item as evidence, which still needs the user's confirmation.92        let update = find(current, "sec.updates.current")93        #expect(update?.state == .userConfirmationRequired)94        #expect(update?.evidence.contains { $0.value == "iOS 26.5" } == true)95    }9697    @Test func restartSuggestionUsesTheBootDate() {98        let recent = engine.assess(device: Fixture.device(bootDaysAgo: 2), signals: Fixture.signals(), responses: [], now: now)99        #expect(find(recent, "device.restart")?.state == .verified)100        let old = engine.assess(device: Fixture.device(bootDaysAgo: 45), signals: Fixture.signals(), responses: [], now: now)101        #expect(find(old, "device.restart")?.state == .attention)102        #expect(find(old, "device.restart")?.severity == .low)103        #expect(find(old, "device.restart")?.title.contains("45") == true)104    }105106    @Test func contributedFindingsAreCarriedAndCounted() {107        let extra = Finding(id: "net.proxy", category: .network, title: "Proxy", state: .attention, severity: .moderate, confidence: .measured,108                            verification: .publicAPI, evidence: [Evidence("Proxy", "10.0.0.1:8080")], explanation: "e", recommendation: "r")109        let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], contributed: [extra], now: now)110        #expect(assessment.actions.map(\.id) == ["net.proxy"])111        #expect(assessment.summary.attention == 1)112        #expect(assessment.summary.highestSeverity == .moderate)113    }114115    @Test func summaryCountsStatesAndCoverageIsNotASecurityLevel() {116        let responses = ChecklistCatalog.security.filter { !$0.isOptional }.map { ChecklistResponse(itemID: $0.id, answer: $0.expected, answeredAt: now) }117        let full = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: responses, now: now)118        #expect(full.summary.coverage == 1)119        #expect(full.summary.awaitingConfirmation == 0)120        #expect(full.summary.userConfirmed == responses.count)121        #expect(full.summary.total == full.all.count)122        let none = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], now: now)123        let coverage = none.summary.coverage ?? 0124        #expect(coverage > 0 && coverage < 1)125        #expect(AssessmentSummary(findings: []).coverage == nil)126        #expect(AssessmentSummary.methodology.first?.contains("does not compute a security score") == true)127    }128129    @Test func assessmentRoundTripsThroughJSON() throws {130        let assessment = engine.assess(device: Fixture.device(), signals: Fixture.signals(), responses: [], now: now)131        let data = try JSONEncoder().encode(assessment)132        let decoded = try JSONDecoder().decode(SecurityAssessment.self, from: data)133        #expect(decoded == assessment)134    }135}136137@Suite struct ChecklistTests {138    let engine = SecurityChecklistEngine()139    let now = Fixture.now140141    @Test func catalogIsWellFormed() {142        let items = ChecklistCatalog.all143        #expect(Set(items.map(\.id)).count == items.count)144        #expect(ChecklistCatalog.security.count >= 15)145        #expect(ChecklistCatalog.privacyItems.count >= 12)146        for item in items {147            #expect(!item.title.isEmpty && !item.question.isEmpty && !item.why.isEmpty && !item.action.isEmpty, "\(item.id)")148            #expect(item.path.count >= 2, "\(item.id)")149            #expect(item.question.hasSuffix("?") || item.question.contains("?"), "\(item.id)")150            #expect(item.expected == .yes || item.expected == .no, "\(item.id)")151            #expect(item.isOptional == (item.importance == .none), "\(item.id): optional items carry no severity, others do")152            #expect(item.validityDays >= 30)153            #expect(item.id.hasPrefix(item.domain == .security ? "sec." : "priv."), "\(item.id)")154        }155        #expect(ChecklistCatalog.item("sec.sdp")?.group == .deviceAccess)156        #expect(ChecklistCatalog.item("nope") == nil)157    }158159    @Test func everyAnswerMapsToTheRightState() throws {160        let item = try #require(ChecklistCatalog.item("sec.account.2fa"))161        let unanswered = engine.finding(for: item, response: nil, now: now)162        #expect(unanswered.state == .userConfirmationRequired)163        #expect(unanswered.confidence == Confidence.none)164        #expect(unanswered.verification == .notVerifiable)165166        let good = engine.finding(for: item, response: .init(itemID: item.id, answer: .yes, answeredAt: now), now: now)167        #expect(good.state == .userConfirmed)168        #expect(good.confidence == .userReported)169        #expect(good.verification == .userConfirmed)170        #expect(good.severity == Severity.none)171        #expect(good.evidence.contains { $0.source == "user" })172173        let bad = engine.finding(for: item, response: .init(itemID: item.id, answer: .no, answeredAt: now), now: now)174        #expect(bad.state == .attention)175        #expect(bad.severity == .high)176        #expect(bad.confidence == .userReported)177        #expect(bad.recommendation == item.action)178179        let unsure = engine.finding(for: item, response: .init(itemID: item.id, answer: .notSure, answeredAt: now), now: now)180        #expect(unsure.state == .userConfirmationRequired)181        let notApplicable = engine.finding(for: item, response: .init(itemID: item.id, answer: .notApplicable, answeredAt: now), now: now)182        #expect(notApplicable.state == .notAvailable)183        #expect(Fixture.assertHonest([unanswered, good, bad, unsure, notApplicable]).isEmpty)184    }185186    @Test func answersExpire() throws {187        let item = try #require(ChecklistCatalog.item("sec.updates.current"))188        #expect(item.validityDays == 30)189        let fresh = ChecklistResponse(itemID: item.id, answer: .yes, answeredAt: now.addingTimeInterval(-29 * 86_400))190        let stale = ChecklistResponse(itemID: item.id, answer: .yes, answeredAt: now.addingTimeInterval(-31 * 86_400))191        #expect(engine.finding(for: item, response: fresh, now: now).state == .userConfirmed)192        let expired = engine.finding(for: item, response: stale, now: now)193        #expect(expired.state == .userConfirmationRequired)194        #expect(expired.explanation.contains("30"))195    }196197    @Test func optionalItemsNeverBecomeProblems() throws {198        let item = try #require(ChecklistCatalog.item("sec.lockdown"))199        #expect(engine.finding(for: item, response: nil, now: now).state == .notAssessed)200        let off = engine.finding(for: item, response: .init(itemID: item.id, answer: .no, answeredAt: now), now: now)201        #expect(off.state == .userConfirmed)202        #expect(off.severity == Severity.none)203        #expect(off.explanation.contains("optional"))204    }205206    @Test func latestResponseWins() {207        let item = ChecklistCatalog.security[0]208        let responses = [ChecklistResponse(itemID: item.id, answer: .no, answeredAt: now.addingTimeInterval(-100)),209                         ChecklistResponse(itemID: item.id, answer: .yes, answeredAt: now)]210        let findings = engine.findings(for: [item], responses: responses, now: now)211        #expect(findings.first?.state == .userConfirmed)212    }213214    @Test func everyCatalogItemProducesHonestFindingsForEveryAnswer() {215        var all: [Finding] = []216        for item in ChecklistCatalog.all {217            all.append(engine.finding(for: item, response: nil, now: now))218            for answer in ChecklistAnswer.allCases {219                all.append(engine.finding(for: item, response: .init(itemID: item.id, answer: answer, answeredAt: now), now: now))220            }221        }222        #expect(all.count == ChecklistCatalog.all.count * 5)223        let problems = Fixture.assertHonest(all)224        #expect(problems.isEmpty, "\(problems.prefix(5))")225        #expect(!all.contains { $0.state == .verified }, "a user-assisted item must never be presented as verified by the app")226    }227}228
Files main
Menu

Theme