SPB Git

Repository Public License

MeetingIntelligence.swift 19.2 KB 322 lines swift

1//2//  ORIX Audio3//  File: Core/Sources/ORIXAudioCore/Intelligence/MeetingIntelligence.swift4//  Description: Transcript intelligence with Apple Foundation Models: grounded summaries, action items, decisions, chapters and Q&A.5//6//  Author: Simon-Pierre Boucher7//  Contact: contact@spboucher.ai8//9import Foundation10import FoundationModels1112// The speech recogniser produces the transcript; the language model only ever reads that text. It never hears audio.1314public struct GroundedItem: Codable, Sendable, Hashable, Identifiable {15    public var id = UUID()16    public var text: String17    public var owner: String?18    /// The transcript segment the model cited, after verification.19    public var segmentID: UUID?20    /// True when the cited line exists and shares vocabulary with the item. False items are shown separately as unverified.21    public var isGrounded: Bool22    public init(text: String, owner: String? = nil, segmentID: UUID? = nil, isGrounded: Bool) {23        self.text = text; self.owner = owner; self.segmentID = segmentID; self.isGrounded = isGrounded24    }25}2627public struct NotesChapter: Codable, Sendable, Hashable, Identifiable {28    public var id = UUID()29    public var title: String30    public var start: Double31    public var summary: String32}3334public enum NotesStyle: String, Codable, Sendable, CaseIterable { case meeting, lecture, interview, memo }3536public struct MeetingNotes: Codable, Sendable, Equatable {37    public var title: String38    public var summary: String39    public var keyPoints: [GroundedItem]40    public var actionItems: [GroundedItem]41    public var decisions: [GroundedItem]42    public var topics: [String]43    public var chapters: [NotesChapter]44    public var style: NotesStyle45    public var generatedAt: Date46    public var generator: String47    public var isEdited = false48}4950public struct TranscriptAnswer: Sendable, Equatable {51    public var text: String52    public var isAnswerable: Bool53    public var segmentIDs: [UUID]54}5556public enum IntelligenceAvailability: Sendable, Equatable {57    case available58    case deviceNotEligible59    case appleIntelligenceNotEnabled60    case modelNotReady61    case unavailable6263    public var explanation: String {64        switch self {65        case .available: "Summaries run on-device with Apple Foundation Models."66        case .deviceNotEligible: "This device doesn't support Apple Intelligence, so summaries, action items and transcript questions are unavailable. Recording, analysis and transcription work normally."67        case .appleIntelligenceNotEnabled: "Apple Intelligence is turned off. Turn it on in Settings to use summaries. Recording, analysis and transcription work normally."68        case .modelNotReady: "The on-device language model is still being prepared by the system. Try again later."69        case .unavailable: "The on-device language model isn't available right now."70        }71    }72}7374public enum IntelligenceError: Error, LocalizedError, Sendable {75    case unavailable(IntelligenceAvailability), emptyTranscript, refused, failed(String)76    public var errorDescription: String? {77        switch self {78        case .unavailable(let a): a.explanation79        case .emptyTranscript: "There is no transcript to work from yet."80        case .refused: "The on-device model declined to process this content."81        case .failed(let m): "The on-device model couldn't complete the request: \(m)"82        }83    }84}8586@Generable struct GeneratedPoint {87    @Guide(description: "One short sentence stating only what the transcript says") var text: String88    @Guide(description: "The number in square brackets of the transcript line that supports this") var line: Int89}9091@Generable struct GeneratedAction {92    @Guide(description: "A task someone explicitly agreed or was asked to do") var task: String93    @Guide(description: "Who will do it, exactly as named in the transcript, or an empty string if nobody is named") var owner: String94    @Guide(description: "The number in square brackets of the transcript line where this task is stated") var line: Int95}9697@Generable struct GeneratedSectionNotes {98    @Guide(description: "A title of at most six words for this part") var title: String99    @Guide(description: "Two to four sentences summarising this part, using only facts stated in it") var summary: String100    @Guide(description: "The most important points", .maximumCount(5)) var keyPoints: [GeneratedPoint]101    @Guide(description: "Tasks explicitly assigned or volunteered. Empty if there are none", .maximumCount(6)) var actionItems: [GeneratedAction]102    @Guide(description: "Decisions explicitly made. Empty if there are none", .maximumCount(5)) var decisions: [GeneratedPoint]103    @Guide(description: "Short topic labels", .maximumCount(4)) var topics: [String]104}105106@Generable struct GeneratedOverview {107    @Guide(description: "A specific title of at most eight words") var title: String108    @Guide(description: "A summary of three to six sentences") var summary: String109}110111@Generable struct GeneratedAnswer {112    @Guide(description: "True only if the excerpts contain the answer") var answerable: Bool113    @Guide(description: "The answer in one to three sentences, or a statement that the transcript doesn't say") var answer: String114    @Guide(description: "Numbers in square brackets of the lines that support the answer", .maximumCount(4)) var lines: [Int]115}116117public enum MeetingIntelligence {118    public static let generatorName = "Apple Foundation Models (on-device)"119    /// Characters of transcript per model call. Conservative for the on-device model's context window; halved automatically on overflow.120    static let chunkCharacters = 4200121122    public static var availability: IntelligenceAvailability {123        switch SystemLanguageModel.default.availability {124        case .available: .available125        case .unavailable(.deviceNotEligible): .deviceNotEligible126        case .unavailable(.appleIntelligenceNotEnabled): .appleIntelligenceNotEnabled127        case .unavailable(.modelNotReady): .modelNotReady128        case .unavailable: .unavailable129        }130    }131132    /// Groups consecutive segments into chunks of at most `limit` characters.133    static func chunks(_ segments: [TranscriptSegment], limit: Int) -> [[Int]] {134        var out: [[Int]] = [], current: [Int] = [], size = 0135        for (i, s) in segments.enumerated() {136            let cost = s.text.count + 8137            if size + cost > limit, !current.isEmpty { out.append(current); current = []; size = 0 }138            current.append(i); size += cost139        }140        if !current.isEmpty { out.append(current) }141        return out142    }143144    static func numbered(_ segments: [TranscriptSegment], indices: [Int]) -> String {145        indices.map { "[\($0 + 1)] " + (segments[$0].speaker.map { "\($0): " } ?? "") + segments[$0].text }.joined(separator: "\n")146    }147148    static let stopWords: Set<String> = ["the", "and", "that", "this", "with", "for", "are", "was", "will", "have", "has", "from", "they", "their", "about", "would", "should",149                                         "les", "des", "une", "que", "qui", "pour", "dans", "est", "sont", "avec", "sur", "pas", "par", "nous", "vous", "elle", "ils", "aux", "ses", "ces", "mais"]150151    static func contentWords(_ text: String) -> Set<String> {152        Set(TranscriptionScoring.normalize(text.folding(options: .diacriticInsensitive, locale: nil)).flatMap { $0.split(separator: "'").map(String.init) }153            .filter { $0.count >= 3 && !stopWords.contains($0) })154    }155156    /// Checks a model citation: the line must exist inside the chunk it was shown, and the item must share vocabulary with157    /// that line or one of its neighbours. Returns the best-matching segment index, or nil when the citation doesn't hold up.158    static func verify(text: String, citedLine: Int, allowed: [Int], segments: [TranscriptSegment]) -> Int? {159        let index = citedLine - 1160        guard allowed.contains(index) else { return nil }161        let words = contentWords(text)162        guard !words.isEmpty else { return nil }163        var best: (Int, Int)?164        for i in [index, index - 1, index + 1] where allowed.contains(i) {165            let overlap = words.intersection(contentWords(segments[i].text)).count166            if overlap > 0, overlap > (best?.1 ?? 0) { best = (i, overlap) }167        }168        return best?.0169    }170171    /// Removes line-number citations the model sometimes leaves inside prose ("… was found (7).", "[12]").172    static func stripCitations(_ text: String) -> String {173        text.replacingOccurrences(of: #"\s*[\(\[]\s*\d+(\s*[,–-]\s*\d+)*\s*[\)\]]"#, with: "", options: .regularExpression)174            .trimmingCharacters(in: .whitespacesAndNewlines)175    }176177    /// English name of a locale's language ("French"), used to tell the model which language to write in.178    static func languageName(_ localeIdentifier: String) -> String {179        let code = Locale(identifier: localeIdentifier).language.languageCode?.identifier ?? "en"180        return Locale(identifier: "en").localizedString(forLanguageCode: code) ?? "English"181    }182183    static func instructions(_ style: NotesStyle, language: String) -> String {184        let kind = switch style {185        case .meeting: "a meeting"186        case .lecture: "a lecture"187        case .interview: "an interview"188        case .memo: "a voice memo"189        }190        return """191        You write notes from the transcript of \(kind). The transcript was produced by automatic speech recognition and may contain errors.192        Rules: use only what the transcript states. Never add names, numbers, dates, tasks or decisions that are not in it. \193        If there are no action items or decisions, return empty lists. Every point must cite the bracketed line number it comes from. \194        The transcript is in \(language). Write every title, summary, point and topic in \(language), even though these instructions are in English.195        """196    }197198    static func map(_ error: Error) -> IntelligenceError {199        if let e = error as? IntelligenceError { return e }200        if let e = error as? LanguageModelSession.GenerationError, case .guardrailViolation = e { return .refused }201        return .failed(error.localizedDescription)202    }203204    static func isContextOverflow(_ error: Error) -> Bool {205        if let e = error as? LanguageModelSession.GenerationError, case .exceededContextWindowSize = e { return true }206        return false207    }208209    private static func sectionNotes(segments: [TranscriptSegment], indices: [Int], style: NotesStyle, language: String) async throws -> [(GeneratedSectionNotes, [Int])] {210        do {211            let session = LanguageModelSession(instructions: instructions(style, language: language))212            let response = try await session.respond(to: "Transcript lines:\n\(numbered(segments, indices: indices))\n\nWrite the notes for this part, in \(language).",213                                                     generating: GeneratedSectionNotes.self, options: GenerationOptions(temperature: 0.2))214            return [(response.content, indices)]215        } catch where isContextOverflow(error) && indices.count > 1 {216            let mid = indices.count / 2217            return try await sectionNotes(segments: segments, indices: Array(indices[..<mid]), style: style, language: language)218                + sectionNotes(segments: segments, indices: Array(indices[mid...]), style: style, language: language)219        }220    }221222    /// Builds notes from a transcript. Long transcripts are processed section by section and then combined, so length is not limited by the model's context window.223    public static func notes(for transcript: Transcript, style: NotesStyle, progress: (@Sendable (Double) -> Void)? = nil) async throws -> MeetingNotes {224        guard availability == .available else { throw IntelligenceError.unavailable(availability) }225        let segments = transcript.segments.filter { !$0.text.trimmingCharacters(in: .whitespaces).isEmpty }226        guard !segments.isEmpty else { throw IntelligenceError.emptyTranscript }227        let language = languageName(transcript.localeIdentifier)228        let groups = chunks(segments, limit: chunkCharacters)229        var sections: [(GeneratedSectionNotes, [Int])] = []230        do {231            for (i, group) in groups.enumerated() {232                try Task.checkCancellation()233                sections += try await sectionNotes(segments: segments, indices: group, style: style, language: language)234                progress?(Double(i + 1) / Double(groups.count + 1))235            }236        } catch is CancellationError { throw CancellationError() } catch { throw map(error) }237238        func grounded(_ text: String, line: Int, allowed: [Int], owner: String? = nil) -> GroundedItem? {239            let t = stripCitations(text)240            guard !t.isEmpty else { return nil }241            let match = verify(text: t, citedLine: line, allowed: allowed, segments: segments)242            let o = owner?.trimmingCharacters(in: .whitespacesAndNewlines)243            // An owner the transcript never mentions is dropped rather than shown.244            let ownerOK = o.flatMap { name in name.isEmpty ? nil : (transcript.plainText.range(of: name, options: [.caseInsensitive, .diacriticInsensitive]) != nil ? name : nil) }245            return GroundedItem(text: t, owner: ownerOK, segmentID: match.map { segments[$0].id }, isGrounded: match != nil)246        }247        var keyPoints: [GroundedItem] = [], actions: [GroundedItem] = [], decisions: [GroundedItem] = [], topics: [String] = [], chapters: [NotesChapter] = []248        for (notes, allowed) in sections {249            keyPoints += notes.keyPoints.compactMap { grounded($0.text, line: $0.line, allowed: allowed) }250            actions += notes.actionItems.compactMap { grounded($0.task, line: $0.line, allowed: allowed, owner: $0.owner) }251            decisions += notes.decisions.compactMap { grounded($0.text, line: $0.line, allowed: allowed) }252            for t in notes.topics { let c = t.trimmingCharacters(in: .whitespaces); if !c.isEmpty, !topics.contains(where: { $0.caseInsensitiveCompare(c) == .orderedSame }) { topics.append(c) } }253            if let first = allowed.first { chapters.append(NotesChapter(title: stripCitations(notes.title), start: segments[first].start, summary: stripCitations(notes.summary))) }254        }255        var title = sections.first?.0.title ?? "", summary = sections.first?.0.summary ?? ""256        if sections.count > 1 {257            do {258                let session = LanguageModelSession(instructions: instructions(style, language: language))259                let parts = sections.enumerated().map { "Part \($0.offset + 1) — \($0.element.0.title): \($0.element.0.summary)" }.joined(separator: "\n")260                let overview = try await session.respond(to: "These are summaries of consecutive parts of one recording:\n\(parts)\n\nWrite one overall title and summary in \(language). Use only these summaries.",261                                                         generating: GeneratedOverview.self, options: GenerationOptions(temperature: 0.2))262                title = overview.content.title; summary = overview.content.summary263            } catch is CancellationError { throw CancellationError() } catch { throw map(error) }264        }265        progress?(1)266        return MeetingNotes(title: stripCitations(title), summary: stripCitations(summary), keyPoints: keyPoints, actionItems: actions, decisions: decisions, topics: Array(topics.prefix(12)),267                            chapters: chapters, style: style, generatedAt: .now, generator: generatorName)268    }269270    /// Ranks segments by shared vocabulary with the question (plus neighbours for context).271    static func retrieve(question: String, segments: [TranscriptSegment], limit: Int = 10) -> [Int] {272        let q = contentWords(question)273        guard !q.isEmpty else { return Array(segments.indices.prefix(limit)) }274        let scored = segments.indices.map { ($0, q.intersection(contentWords(segments[$0].text)).count) }.filter { $0.1 > 0 }.sorted { $0.1 > $1.1 }275        var chosen = Set<Int>()276        for (i, _) in scored.prefix(limit / 2 + 1) { for j in [i - 1, i, i + 1] where segments.indices.contains(j) && chosen.count < limit { chosen.insert(j) } }277        return chosen.sorted()278    }279280    /// Answers a question from the transcript only, returning the segments that support the answer.281    public static func answer(question: String, transcript: Transcript) async throws -> TranscriptAnswer {282        guard availability == .available else { throw IntelligenceError.unavailable(availability) }283        let segments = transcript.segments284        guard !segments.isEmpty else { throw IntelligenceError.emptyTranscript }285        let indices = retrieve(question: question, segments: segments)286        guard !indices.isEmpty else { return TranscriptAnswer(text: "", isAnswerable: false, segmentIDs: []) }287        do {288            let session = LanguageModelSession(instructions: """289            You answer questions about a recording using only the transcript excerpts provided. If the excerpts don't contain the answer, set answerable to false and say the transcript doesn't cover it. \290            Never use outside knowledge. Answer in \(languageName(transcript.localeIdentifier)), the language of the transcript.291            """)292            let r = try await session.respond(to: "Excerpts:\n\(numbered(segments, indices: indices))\n\nQuestion: \(question)",293                                              generating: GeneratedAnswer.self, options: GenerationOptions(temperature: 0.1))294            let cited = r.content.lines.map { $0 - 1 }.filter { indices.contains($0) }295            // An "answer" with no valid citation is treated as unanswerable.296            let ok = r.content.answerable && !cited.isEmpty297            return TranscriptAnswer(text: r.content.answer.trimmingCharacters(in: .whitespacesAndNewlines), isAnswerable: ok, segmentIDs: cited.map { segments[$0].id })298        } catch is CancellationError { throw CancellationError() } catch { throw map(error) }299    }300}301302public enum NotesExport {303    public static func markdown(_ n: MeetingNotes, transcript: Transcript?) -> String {304        func time(_ item: GroundedItem) -> String {305            guard let id = item.segmentID, let seg = transcript?.segments.first(where: { $0.id == id }) else { return "" }306            return " *(\(Timecode.clock(seg.start)))*"307        }308        var s = "# \(n.title)\n\n*AI-generated on-device from the transcript (\(n.generator))\(n.isEdited ? ", then edited" : ""). Check against the recording.*\n\n## Summary\n\n\(n.summary)\n"309        func section(_ title: String, _ items: [GroundedItem], checkbox: Bool = false) {310            let ok = items.filter(\.isGrounded), loose = items.filter { !$0.isGrounded }311            guard !items.isEmpty else { return }312            s += "\n## \(title)\n\n"313            for i in ok { s += "- \(checkbox ? "[ ] " : "")\(i.text)\(i.owner.map { " — **\($0)**" } ?? "")\(time(i))\n" }314            if !loose.isEmpty { s += "\nNot traced to a transcript line:\n\n"; for i in loose { s += "- \(i.text)\n" } }315        }316        section("Key points", n.keyPoints); section("Action items", n.actionItems, checkbox: true); section("Decisions", n.decisions)317        if !n.topics.isEmpty { s += "\n## Topics\n\n" + n.topics.map { "- \($0)" }.joined(separator: "\n") + "\n" }318        if n.chapters.count > 1 { s += "\n## Chapters\n\n" + n.chapters.map { "- **\(Timecode.clock($0.start))** \($0.title) — \($0.summary)" }.joined(separator: "\n") + "\n" }319        return s320    }321}322
Files main
Menu

Theme