Repository Public License
1//2// ORIX Audio3// File: Core/Sources/ORIXAudioCore/Intelligence/MeetingIntelligence.swift4// Description: Transcript intelligence with Apple Foundation Models: grounded summaries, action items, decisions, chapters and Q&A.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8//9import Foundation10import FoundationModels1112// The speech recogniser produces the transcript; the language model only ever reads that text. It never hears audio.1314public struct GroundedItem: Codable, Sendable, Hashable, Identifiable {15 public var id = UUID()16 public var text: String17 public var owner: String?18 /// The transcript segment the model cited, after verification.19 public var segmentID: UUID?20 /// True when the cited line exists and shares vocabulary with the item. False items are shown separately as unverified.21 public var isGrounded: Bool22 public init(text: String, owner: String? = nil, segmentID: UUID? = nil, isGrounded: Bool) {23 self.text = text; self.owner = owner; self.segmentID = segmentID; self.isGrounded = isGrounded24 }25}2627public struct NotesChapter: Codable, Sendable, Hashable, Identifiable {28 public var id = UUID()29 public var title: String30 public var start: Double31 public var summary: String32}3334public enum NotesStyle: String, Codable, Sendable, CaseIterable { case meeting, lecture, interview, memo }3536public struct MeetingNotes: Codable, Sendable, Equatable {37 public var title: String38 public var summary: String39 public var keyPoints: [GroundedItem]40 public var actionItems: [GroundedItem]41 public var decisions: [GroundedItem]42 public var topics: [String]43 public var chapters: [NotesChapter]44 public var style: NotesStyle45 public var generatedAt: Date46 public var generator: String47 public var isEdited = false48}4950public struct TranscriptAnswer: Sendable, Equatable {51 public var text: String52 public var isAnswerable: Bool53 public var segmentIDs: [UUID]54}5556public enum IntelligenceAvailability: Sendable, Equatable {57 case available58 case deviceNotEligible59 case appleIntelligenceNotEnabled60 case modelNotReady61 case unavailable6263 public var explanation: String {64 switch self {65 case .available: "Summaries run on-device with Apple Foundation Models."66 case .deviceNotEligible: "This device doesn't support Apple Intelligence, so summaries, action items and transcript questions are unavailable. Recording, analysis and transcription work normally."67 case .appleIntelligenceNotEnabled: "Apple Intelligence is turned off. Turn it on in Settings to use summaries. Recording, analysis and transcription work normally."68 case .modelNotReady: "The on-device language model is still being prepared by the system. Try again later."69 case .unavailable: "The on-device language model isn't available right now."70 }71 }72}7374public enum IntelligenceError: Error, LocalizedError, Sendable {75 case unavailable(IntelligenceAvailability), emptyTranscript, refused, failed(String)76 public var errorDescription: String? {77 switch self {78 case .unavailable(let a): a.explanation79 case .emptyTranscript: "There is no transcript to work from yet."80 case .refused: "The on-device model declined to process this content."81 case .failed(let m): "The on-device model couldn't complete the request: \(m)"82 }83 }84}8586@Generable struct GeneratedPoint {87 @Guide(description: "One short sentence stating only what the transcript says") var text: String88 @Guide(description: "The number in square brackets of the transcript line that supports this") var line: Int89}9091@Generable struct GeneratedAction {92 @Guide(description: "A task someone explicitly agreed or was asked to do") var task: String93 @Guide(description: "Who will do it, exactly as named in the transcript, or an empty string if nobody is named") var owner: String94 @Guide(description: "The number in square brackets of the transcript line where this task is stated") var line: Int95}9697@Generable struct GeneratedSectionNotes {98 @Guide(description: "A title of at most six words for this part") var title: String99 @Guide(description: "Two to four sentences summarising this part, using only facts stated in it") var summary: String100 @Guide(description: "The most important points", .maximumCount(5)) var keyPoints: [GeneratedPoint]101 @Guide(description: "Tasks explicitly assigned or volunteered. Empty if there are none", .maximumCount(6)) var actionItems: [GeneratedAction]102 @Guide(description: "Decisions explicitly made. Empty if there are none", .maximumCount(5)) var decisions: [GeneratedPoint]103 @Guide(description: "Short topic labels", .maximumCount(4)) var topics: [String]104}105106@Generable struct GeneratedOverview {107 @Guide(description: "A specific title of at most eight words") var title: String108 @Guide(description: "A summary of three to six sentences") var summary: String109}110111@Generable struct GeneratedAnswer {112 @Guide(description: "True only if the excerpts contain the answer") var answerable: Bool113 @Guide(description: "The answer in one to three sentences, or a statement that the transcript doesn't say") var answer: String114 @Guide(description: "Numbers in square brackets of the lines that support the answer", .maximumCount(4)) var lines: [Int]115}116117public enum MeetingIntelligence {118 public static let generatorName = "Apple Foundation Models (on-device)"119 /// Characters of transcript per model call. Conservative for the on-device model's context window; halved automatically on overflow.120 static let chunkCharacters = 4200121122 public static var availability: IntelligenceAvailability {123 switch SystemLanguageModel.default.availability {124 case .available: .available125 case .unavailable(.deviceNotEligible): .deviceNotEligible126 case .unavailable(.appleIntelligenceNotEnabled): .appleIntelligenceNotEnabled127 case .unavailable(.modelNotReady): .modelNotReady128 case .unavailable: .unavailable129 }130 }131132 /// Groups consecutive segments into chunks of at most `limit` characters.133 static func chunks(_ segments: [TranscriptSegment], limit: Int) -> [[Int]] {134 var out: [[Int]] = [], current: [Int] = [], size = 0135 for (i, s) in segments.enumerated() {136 let cost = s.text.count + 8137 if size + cost > limit, !current.isEmpty { out.append(current); current = []; size = 0 }138 current.append(i); size += cost139 }140 if !current.isEmpty { out.append(current) }141 return out142 }143144 static func numbered(_ segments: [TranscriptSegment], indices: [Int]) -> String {145 indices.map { "[\($0 + 1)] " + (segments[$0].speaker.map { "\($0): " } ?? "") + segments[$0].text }.joined(separator: "\n")146 }147148 static let stopWords: Set<String> = ["the", "and", "that", "this", "with", "for", "are", "was", "will", "have", "has", "from", "they", "their", "about", "would", "should",149 "les", "des", "une", "que", "qui", "pour", "dans", "est", "sont", "avec", "sur", "pas", "par", "nous", "vous", "elle", "ils", "aux", "ses", "ces", "mais"]150151 static func contentWords(_ text: String) -> Set<String> {152 Set(TranscriptionScoring.normalize(text.folding(options: .diacriticInsensitive, locale: nil)).flatMap { $0.split(separator: "'").map(String.init) }153 .filter { $0.count >= 3 && !stopWords.contains($0) })154 }155156 /// Checks a model citation: the line must exist inside the chunk it was shown, and the item must share vocabulary with157 /// that line or one of its neighbours. Returns the best-matching segment index, or nil when the citation doesn't hold up.158 static func verify(text: String, citedLine: Int, allowed: [Int], segments: [TranscriptSegment]) -> Int? {159 let index = citedLine - 1160 guard allowed.contains(index) else { return nil }161 let words = contentWords(text)162 guard !words.isEmpty else { return nil }163 var best: (Int, Int)?164 for i in [index, index - 1, index + 1] where allowed.contains(i) {165 let overlap = words.intersection(contentWords(segments[i].text)).count166 if overlap > 0, overlap > (best?.1 ?? 0) { best = (i, overlap) }167 }168 return best?.0169 }170171 /// Removes line-number citations the model sometimes leaves inside prose ("… was found (7).", "[12]").172 static func stripCitations(_ text: String) -> String {173 text.replacingOccurrences(of: #"\s*[\(\[]\s*\d+(\s*[,–-]\s*\d+)*\s*[\)\]]"#, with: "", options: .regularExpression)174 .trimmingCharacters(in: .whitespacesAndNewlines)175 }176177 /// English name of a locale's language ("French"), used to tell the model which language to write in.178 static func languageName(_ localeIdentifier: String) -> String {179 let code = Locale(identifier: localeIdentifier).language.languageCode?.identifier ?? "en"180 return Locale(identifier: "en").localizedString(forLanguageCode: code) ?? "English"181 }182183 static func instructions(_ style: NotesStyle, language: String) -> String {184 let kind = switch style {185 case .meeting: "a meeting"186 case .lecture: "a lecture"187 case .interview: "an interview"188 case .memo: "a voice memo"189 }190 return """191 You write notes from the transcript of \(kind). The transcript was produced by automatic speech recognition and may contain errors.192 Rules: use only what the transcript states. Never add names, numbers, dates, tasks or decisions that are not in it. \193 If there are no action items or decisions, return empty lists. Every point must cite the bracketed line number it comes from. \194 The transcript is in \(language). Write every title, summary, point and topic in \(language), even though these instructions are in English.195 """196 }197198 static func map(_ error: Error) -> IntelligenceError {199 if let e = error as? IntelligenceError { return e }200 if let e = error as? LanguageModelSession.GenerationError, case .guardrailViolation = e { return .refused }201 return .failed(error.localizedDescription)202 }203204 static func isContextOverflow(_ error: Error) -> Bool {205 if let e = error as? LanguageModelSession.GenerationError, case .exceededContextWindowSize = e { return true }206 return false207 }208209 private static func sectionNotes(segments: [TranscriptSegment], indices: [Int], style: NotesStyle, language: String) async throws -> [(GeneratedSectionNotes, [Int])] {210 do {211 let session = LanguageModelSession(instructions: instructions(style, language: language))212 let response = try await session.respond(to: "Transcript lines:\n\(numbered(segments, indices: indices))\n\nWrite the notes for this part, in \(language).",213 generating: GeneratedSectionNotes.self, options: GenerationOptions(temperature: 0.2))214 return [(response.content, indices)]215 } catch where isContextOverflow(error) && indices.count > 1 {216 let mid = indices.count / 2217 return try await sectionNotes(segments: segments, indices: Array(indices[..<mid]), style: style, language: language)218 + sectionNotes(segments: segments, indices: Array(indices[mid...]), style: style, language: language)219 }220 }221222 /// Builds notes from a transcript. Long transcripts are processed section by section and then combined, so length is not limited by the model's context window.223 public static func notes(for transcript: Transcript, style: NotesStyle, progress: (@Sendable (Double) -> Void)? = nil) async throws -> MeetingNotes {224 guard availability == .available else { throw IntelligenceError.unavailable(availability) }225 let segments = transcript.segments.filter { !$0.text.trimmingCharacters(in: .whitespaces).isEmpty }226 guard !segments.isEmpty else { throw IntelligenceError.emptyTranscript }227 let language = languageName(transcript.localeIdentifier)228 let groups = chunks(segments, limit: chunkCharacters)229 var sections: [(GeneratedSectionNotes, [Int])] = []230 do {231 for (i, group) in groups.enumerated() {232 try Task.checkCancellation()233 sections += try await sectionNotes(segments: segments, indices: group, style: style, language: language)234 progress?(Double(i + 1) / Double(groups.count + 1))235 }236 } catch is CancellationError { throw CancellationError() } catch { throw map(error) }237238 func grounded(_ text: String, line: Int, allowed: [Int], owner: String? = nil) -> GroundedItem? {239 let t = stripCitations(text)240 guard !t.isEmpty else { return nil }241 let match = verify(text: t, citedLine: line, allowed: allowed, segments: segments)242 let o = owner?.trimmingCharacters(in: .whitespacesAndNewlines)243 // An owner the transcript never mentions is dropped rather than shown.244 let ownerOK = o.flatMap { name in name.isEmpty ? nil : (transcript.plainText.range(of: name, options: [.caseInsensitive, .diacriticInsensitive]) != nil ? name : nil) }245 return GroundedItem(text: t, owner: ownerOK, segmentID: match.map { segments[$0].id }, isGrounded: match != nil)246 }247 var keyPoints: [GroundedItem] = [], actions: [GroundedItem] = [], decisions: [GroundedItem] = [], topics: [String] = [], chapters: [NotesChapter] = []248 for (notes, allowed) in sections {249 keyPoints += notes.keyPoints.compactMap { grounded($0.text, line: $0.line, allowed: allowed) }250 actions += notes.actionItems.compactMap { grounded($0.task, line: $0.line, allowed: allowed, owner: $0.owner) }251 decisions += notes.decisions.compactMap { grounded($0.text, line: $0.line, allowed: allowed) }252 for t in notes.topics { let c = t.trimmingCharacters(in: .whitespaces); if !c.isEmpty, !topics.contains(where: { $0.caseInsensitiveCompare(c) == .orderedSame }) { topics.append(c) } }253 if let first = allowed.first { chapters.append(NotesChapter(title: stripCitations(notes.title), start: segments[first].start, summary: stripCitations(notes.summary))) }254 }255 var title = sections.first?.0.title ?? "", summary = sections.first?.0.summary ?? ""256 if sections.count > 1 {257 do {258 let session = LanguageModelSession(instructions: instructions(style, language: language))259 let parts = sections.enumerated().map { "Part \($0.offset + 1) — \($0.element.0.title): \($0.element.0.summary)" }.joined(separator: "\n")260 let overview = try await session.respond(to: "These are summaries of consecutive parts of one recording:\n\(parts)\n\nWrite one overall title and summary in \(language). Use only these summaries.",261 generating: GeneratedOverview.self, options: GenerationOptions(temperature: 0.2))262 title = overview.content.title; summary = overview.content.summary263 } catch is CancellationError { throw CancellationError() } catch { throw map(error) }264 }265 progress?(1)266 return MeetingNotes(title: stripCitations(title), summary: stripCitations(summary), keyPoints: keyPoints, actionItems: actions, decisions: decisions, topics: Array(topics.prefix(12)),267 chapters: chapters, style: style, generatedAt: .now, generator: generatorName)268 }269270 /// Ranks segments by shared vocabulary with the question (plus neighbours for context).271 static func retrieve(question: String, segments: [TranscriptSegment], limit: Int = 10) -> [Int] {272 let q = contentWords(question)273 guard !q.isEmpty else { return Array(segments.indices.prefix(limit)) }274 let scored = segments.indices.map { ($0, q.intersection(contentWords(segments[$0].text)).count) }.filter { $0.1 > 0 }.sorted { $0.1 > $1.1 }275 var chosen = Set<Int>()276 for (i, _) in scored.prefix(limit / 2 + 1) { for j in [i - 1, i, i + 1] where segments.indices.contains(j) && chosen.count < limit { chosen.insert(j) } }277 return chosen.sorted()278 }279280 /// Answers a question from the transcript only, returning the segments that support the answer.281 public static func answer(question: String, transcript: Transcript) async throws -> TranscriptAnswer {282 guard availability == .available else { throw IntelligenceError.unavailable(availability) }283 let segments = transcript.segments284 guard !segments.isEmpty else { throw IntelligenceError.emptyTranscript }285 let indices = retrieve(question: question, segments: segments)286 guard !indices.isEmpty else { return TranscriptAnswer(text: "", isAnswerable: false, segmentIDs: []) }287 do {288 let session = LanguageModelSession(instructions: """289 You answer questions about a recording using only the transcript excerpts provided. If the excerpts don't contain the answer, set answerable to false and say the transcript doesn't cover it. \290 Never use outside knowledge. Answer in \(languageName(transcript.localeIdentifier)), the language of the transcript.291 """)292 let r = try await session.respond(to: "Excerpts:\n\(numbered(segments, indices: indices))\n\nQuestion: \(question)",293 generating: GeneratedAnswer.self, options: GenerationOptions(temperature: 0.1))294 let cited = r.content.lines.map { $0 - 1 }.filter { indices.contains($0) }295 // An "answer" with no valid citation is treated as unanswerable.296 let ok = r.content.answerable && !cited.isEmpty297 return TranscriptAnswer(text: r.content.answer.trimmingCharacters(in: .whitespacesAndNewlines), isAnswerable: ok, segmentIDs: cited.map { segments[$0].id })298 } catch is CancellationError { throw CancellationError() } catch { throw map(error) }299 }300}301302public enum NotesExport {303 public static func markdown(_ n: MeetingNotes, transcript: Transcript?) -> String {304 func time(_ item: GroundedItem) -> String {305 guard let id = item.segmentID, let seg = transcript?.segments.first(where: { $0.id == id }) else { return "" }306 return " *(\(Timecode.clock(seg.start)))*"307 }308 var s = "# \(n.title)\n\n*AI-generated on-device from the transcript (\(n.generator))\(n.isEdited ? ", then edited" : ""). Check against the recording.*\n\n## Summary\n\n\(n.summary)\n"309 func section(_ title: String, _ items: [GroundedItem], checkbox: Bool = false) {310 let ok = items.filter(\.isGrounded), loose = items.filter { !$0.isGrounded }311 guard !items.isEmpty else { return }312 s += "\n## \(title)\n\n"313 for i in ok { s += "- \(checkbox ? "[ ] " : "")\(i.text)\(i.owner.map { " — **\($0)**" } ?? "")\(time(i))\n" }314 if !loose.isEmpty { s += "\nNot traced to a transcript line:\n\n"; for i in loose { s += "- \(i.text)\n" } }315 }316 section("Key points", n.keyPoints); section("Action items", n.actionItems, checkbox: true); section("Decisions", n.decisions)317 if !n.topics.isEmpty { s += "\n## Topics\n\n" + n.topics.map { "- \($0)" }.joined(separator: "\n") + "\n" }318 if n.chapters.count > 1 { s += "\n## Chapters\n\n" + n.chapters.map { "- **\(Timecode.clock($0.start))** \($0.title) — \($0.summary)" }.joined(separator: "\n") + "\n" }319 return s320 }321}322