Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatAgents/Sources/WebTool.swift3// Description: fetch_url tool (off by default): http/https only, refuses private and loopback destinations (also after redirects), size and time limits, HTML to text.4import Foundation5import LocalChatCore67public enum SSRFGuard {8 /// True when the literal IP address is loopback, private, link-local, CGNAT, multicast, unspecified or otherwise not public.9 public static func isBlockedAddress(_ address: String) -> Bool {10 var v4 = in_addr(), v6 = in6_addr()11 let bare = address.split(separator: "%").first.map(String.init) ?? address12 if inet_pton(AF_INET, bare, &v4) == 1 { return blockedV4(UInt32(bigEndian: v4.s_addr)) }13 if inet_pton(AF_INET6, bare, &v6) == 1 {14 let b = withUnsafeBytes(of: &v6) { Array($0) }15 if b.allSatisfy({ $0 == 0 }) { return true } // ::16 if b[0..<15].allSatisfy({ $0 == 0 }) && b[15] == 1 { return true } // ::117 if b[0] & 0xFE == 0xFC { return true } // fc00::/7 unique local18 if b[0] == 0xFE && b[1] & 0xC0 == 0x80 { return true } // fe80::/10 link-local19 if b[0] == 0xFF { return true } // multicast20 if b[0..<10].allSatisfy({ $0 == 0 }) && b[10] == 0xFF && b[11] == 0xFF { // ::ffff:a.b.c.d21 return blockedV4(UInt32(b[12]) << 24 | UInt32(b[13]) << 16 | UInt32(b[14]) << 8 | UInt32(b[15]))22 }23 if b[0] == 0x00 && b[1] == 0x64 && b[2] == 0xFF && b[3] == 0x9B { // 64:ff9b::/96 NAT6424 return blockedV4(UInt32(b[12]) << 24 | UInt32(b[13]) << 16 | UInt32(b[14]) << 8 | UInt32(b[15]))25 }26 return false27 }28 return true // not an IP literal: callers must resolve first29 }3031 static func blockedV4(_ ip: UInt32) -> Bool {32 let a = UInt8(ip >> 24), b = UInt8((ip >> 16) & 0xFF)33 if a == 0 || a == 10 || a == 127 { return true }34 if a == 100 && (64...127).contains(b) { return true } // CGNAT / Tailscale35 if a == 169 && b == 254 { return true } // link-local, cloud metadata36 if a == 172 && (16...31).contains(b) { return true }37 if a == 192 && b == 168 { return true }38 if a == 192 && b == 0 { return true }39 if a == 198 && (b == 18 || b == 19) { return true }40 return a >= 224 // multicast, reserved, broadcast41 }4243 public static func resolve(_ host: String) -> [String] {44 var hints = addrinfo(); hints.ai_socktype = SOCK_STREAM45 var result: UnsafeMutablePointer<addrinfo>?46 guard getaddrinfo(host, nil, &hints, &result) == 0 else { return [] }47 defer { freeaddrinfo(result) }48 var out: [String] = []; var p = result49 while let info = p {50 var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST))51 if getnameinfo(info.pointee.ai_addr, info.pointee.ai_addrlen, &buffer, socklen_t(buffer.count), nil, 0, NI_NUMERICHOST) == 0 {52 out.append(String(decoding: buffer.prefix(while: { $0 != 0 }).map { UInt8(bitPattern: $0) }, as: UTF8.self))53 }54 p = info.pointee.ai_next55 }56 return out57 }5859 /// Nil when the URL may be fetched, otherwise the reason it is refused. Every address the host resolves to must be public.60 public static func refusal(for url: URL, resolver: (String) -> [String] = SSRFGuard.resolve) -> String? {61 guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else { return "Only http and https URLs are allowed." }62 guard url.user == nil, url.password == nil else { return "URLs with credentials are not allowed." }63 guard var host = url.host?.lowercased(), !host.isEmpty else { return "The URL has no host." }64 if host.hasPrefix("[") && host.hasSuffix("]") { host = String(host.dropFirst().dropLast()) }65 if host == "localhost" || host.hasSuffix(".localhost") || host.hasSuffix(".local") || host.hasSuffix(".internal") {66 return "Requests to local hosts are not allowed."67 }68 var v4 = in_addr(), v6 = in6_addr()69 let isLiteral = inet_pton(AF_INET, host, &v4) == 1 || inet_pton(AF_INET6, host, &v6) == 170 // inet_aton-style forms (2130706433, 0x7f.1, 0177.0.0.1) that some resolvers read as IPv4 addresses71 if !isLiteral && host.range(of: #"^(0x[0-9a-f]+|[0-9]+)(\.(0x[0-9a-f]+|[0-9]+))*$"#, options: .regularExpression) != nil { return "Ambiguous numeric hosts are not allowed." }72 let addresses = isLiteral ? [host] : resolver(host)73 guard !addresses.isEmpty else { return "The host \(host) could not be resolved." }74 if addresses.contains(where: isBlockedAddress) { return "Requests to private, loopback or link-local addresses are not allowed." }75 return nil76 }77}7879public struct FetchedPage: Sendable {80 public var status: Int; public var contentType: String; public var body: Data; public var finalURL: URL; public var truncated: Bool81 public init(status: Int, contentType: String, body: Data, finalURL: URL, truncated: Bool) {82 self.status = status; self.contentType = contentType; self.body = body; self.finalURL = finalURL; self.truncated = truncated83 }84}8586public protocol URLFetching: Sendable {87 func fetch(_ url: URL, maxBytes: Int, timeout: TimeInterval) async throws -> FetchedPage88}8990/// URLSession fetcher that validates the first URL and every redirect target with SSRFGuard. No cookies, no credentials, no cache.91/// Known limit: the address is checked at resolution time, not pinned for the connection (DNS rebinding is not fully excluded).92public final class SafeURLFetcher: NSObject, URLFetching, URLSessionTaskDelegate, Sendable {93 public override init() { super.init() }9495 public func urlSession(_ session: URLSession, task: URLSessionTask, willPerformHTTPRedirection response: HTTPURLResponse,96 newRequest request: URLRequest) async -> URLRequest? {97 guard let url = request.url, SSRFGuard.refusal(for: url) == nil else { return nil }98 return request99 }100101 public func fetch(_ url: URL, maxBytes: Int, timeout: TimeInterval) async throws -> FetchedPage {102 if let reason = SSRFGuard.refusal(for: url) { throw ToolError(reason) }103 let config = URLSessionConfiguration.ephemeral104 config.httpCookieStorage = nil; config.urlCredentialStorage = nil; config.urlCache = nil105 config.timeoutIntervalForRequest = timeout; config.timeoutIntervalForResource = timeout106 let session = URLSession(configuration: config, delegate: self, delegateQueue: nil)107 defer { session.invalidateAndCancel() }108 var request = URLRequest(url: url); request.setValue("LocalChat/1.0", forHTTPHeaderField: "User-Agent")109 request.setValue("text/html,text/plain,application/json;q=0.9,*/*;q=0.5", forHTTPHeaderField: "Accept")110 let (bytes, response) = try await session.bytes(for: request)111 guard let http = response as? HTTPURLResponse else { throw ToolError("Not an HTTP response.") }112 if (300..<400).contains(http.statusCode) { throw ToolError("Redirect to a destination that is not allowed.") }113 var data = Data(); var truncated = false114 data.reserveCapacity(min(maxBytes, 1 << 16))115 for try await byte in bytes {116 if data.count >= maxBytes { truncated = true; break }117 data.append(byte)118 }119 return FetchedPage(status: http.statusCode, contentType: http.value(forHTTPHeaderField: "Content-Type") ?? "", body: data,120 finalURL: http.url ?? url, truncated: truncated)121 }122}123124public enum HTMLText {125 static let entities: [String: String] = ["amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'", "nbsp": " ", "mdash": "—", "ndash": "–",126 "hellip": "…", "rsquo": "’", "lsquo": "‘", "rdquo": "”", "ldquo": "“", "eacute": "é", "egrave": "è",127 "agrave": "à", "ccedil": "ç", "copy": "©", "laquo": "«", "raquo": "»"]128129 public static func extract(_ html: String) -> String {130 var s = html131 for tag in ["script", "style", "noscript", "svg", "head", "template"] {132 s = s.replacingOccurrences(of: "<\(tag)\\b[^>]*>[\\s\\S]*?</\(tag)\\s*>", with: " ", options: [.regularExpression, .caseInsensitive])133 }134 s = s.replacingOccurrences(of: "<!--[\\s\\S]*?-->", with: " ", options: .regularExpression)135 s = s.replacingOccurrences(of: "<(br|/p|/div|/h[1-6]|/li|/tr|/section|/article)\\b[^>]*>", with: "\n", options: [.regularExpression, .caseInsensitive])136 s = s.replacingOccurrences(of: "<li\\b[^>]*>", with: "• ", options: [.regularExpression, .caseInsensitive])137 s = s.replacingOccurrences(of: "<[^>]+>", with: "", options: .regularExpression)138 s = decodeEntities(s)139 s = s.replacingOccurrences(of: "[ \\t\\u{00A0}]+", with: " ", options: .regularExpression)140 s = s.replacingOccurrences(of: " ?\\n ?", with: "\n", options: .regularExpression)141 s = s.replacingOccurrences(of: "\\n{3,}", with: "\n\n", options: .regularExpression)142 return s.trimmingCharacters(in: .whitespacesAndNewlines)143 }144145 static func decodeEntities(_ text: String) -> String {146 guard text.contains("&") else { return text }147 var out = ""; var rest = text[...]148 while let amp = rest.firstIndex(of: "&") {149 out += rest[..<amp]150 let after = rest[rest.index(after: amp)...]151 if let semi = after.prefix(10).firstIndex(of: ";") {152 let name = String(after[..<semi]); var replacement: String?153 if name.hasPrefix("#x") || name.hasPrefix("#X") { replacement = UInt32(name.dropFirst(2), radix: 16).flatMap(Unicode.Scalar.init).map { String($0) } }154 else if name.hasPrefix("#") { replacement = UInt32(name.dropFirst()).flatMap(Unicode.Scalar.init).map { String($0) } }155 else { replacement = entities[name] }156 if let replacement { out += replacement; rest = after[after.index(after: semi)...]; continue }157 }158 out += "&"; rest = after159 }160 return out + rest161 }162}163164struct FetchURLTool: AgentTool {165 let risk: ToolRisk = .read166 let fetcher: any URLFetching167 let definition = ToolDefinition(168 name: "fetch_url", summary: "Downloads a public web page (http/https) and returns its text. Sends the URL to that website.",169 parameters: ["type": "object", "properties": ["url": ["type": "string"]], "required": ["url"]])170 static let maxBytes = 1_500_000, maxCharacters = 20_000171172 func run(arguments: JSONValue, context: ToolContext) async throws -> ToolOutput {173 guard let raw = arguments["url"]?.stringValue, let url = URL(string: raw.trimmingCharacters(in: .whitespaces)) else { throw ToolError("Invalid URL.") }174 if let reason = SSRFGuard.refusal(for: url) { throw ToolError(reason) }175 let page = try await fetcher.fetch(url, maxBytes: Self.maxBytes, timeout: 20)176 guard (200..<300).contains(page.status) else { throw ToolError("The server answered HTTP \(page.status).") }177 let type = page.contentType.lowercased()178 guard type.isEmpty || type.contains("text") || type.contains("json") || type.contains("xml") else { throw ToolError("Unsupported content type: \(page.contentType).") }179 guard let raw = String(data: page.body, encoding: .utf8) ?? String(data: page.body, encoding: .isoLatin1) else { throw ToolError("The page is not text.") }180 var text = type.contains("html") || raw.range(of: "<html", options: .caseInsensitive) != nil ? HTMLText.extract(raw) : raw181 var note = page.truncated ? "\n[… download truncated]" : ""182 if text.count > Self.maxCharacters { text = String(text.prefix(Self.maxCharacters)); note = "\n[… truncated]" }183 return ToolOutput("Content of \(page.finalURL.absoluteString):\n\n" + text + note)184 }185}186