SPB Git

Repository Public

WebTool.swift 12.0 KB 186 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatAgents/Sources/WebTool.swift3// Description: fetch_url tool (off by default): http/https only, refuses private and loopback destinations (also after redirects), size and time limits, HTML to text.4import Foundation5import LocalChatCore67public enum SSRFGuard {8    /// True when the literal IP address is loopback, private, link-local, CGNAT, multicast, unspecified or otherwise not public.9    public static func isBlockedAddress(_ address: String) -> Bool {10        var v4 = in_addr(), v6 = in6_addr()11        let bare = address.split(separator: "%").first.map(String.init) ?? address12        if inet_pton(AF_INET, bare, &v4) == 1 { return blockedV4(UInt32(bigEndian: v4.s_addr)) }13        if inet_pton(AF_INET6, bare, &v6) == 1 {14            let b = withUnsafeBytes(of: &v6) { Array($0) }15            if b.allSatisfy({ $0 == 0 }) { return true }                                  // ::16            if b[0..<15].allSatisfy({ $0 == 0 }) && b[15] == 1 { return true }            // ::117            if b[0] & 0xFE == 0xFC { return true }                                        // fc00::/7 unique local18            if b[0] == 0xFE && b[1] & 0xC0 == 0x80 { return true }                        // fe80::/10 link-local19            if b[0] == 0xFF { return true }                                               // multicast20            if b[0..<10].allSatisfy({ $0 == 0 }) && b[10] == 0xFF && b[11] == 0xFF {      // ::ffff:a.b.c.d21                return blockedV4(UInt32(b[12]) << 24 | UInt32(b[13]) << 16 | UInt32(b[14]) << 8 | UInt32(b[15]))22            }23            if b[0] == 0x00 && b[1] == 0x64 && b[2] == 0xFF && b[3] == 0x9B {             // 64:ff9b::/96 NAT6424                return blockedV4(UInt32(b[12]) << 24 | UInt32(b[13]) << 16 | UInt32(b[14]) << 8 | UInt32(b[15]))25            }26            return false27        }28        return true   // not an IP literal: callers must resolve first29    }3031    static func blockedV4(_ ip: UInt32) -> Bool {32        let a = UInt8(ip >> 24), b = UInt8((ip >> 16) & 0xFF)33        if a == 0 || a == 10 || a == 127 { return true }34        if a == 100 && (64...127).contains(b) { return true }      // CGNAT / Tailscale35        if a == 169 && b == 254 { return true }                    // link-local, cloud metadata36        if a == 172 && (16...31).contains(b) { return true }37        if a == 192 && b == 168 { return true }38        if a == 192 && b == 0 { return true }39        if a == 198 && (b == 18 || b == 19) { return true }40        return a >= 224                                            // multicast, reserved, broadcast41    }4243    public static func resolve(_ host: String) -> [String] {44        var hints = addrinfo(); hints.ai_socktype = SOCK_STREAM45        var result: UnsafeMutablePointer<addrinfo>?46        guard getaddrinfo(host, nil, &hints, &result) == 0 else { return [] }47        defer { freeaddrinfo(result) }48        var out: [String] = []; var p = result49        while let info = p {50            var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST))51            if getnameinfo(info.pointee.ai_addr, info.pointee.ai_addrlen, &buffer, socklen_t(buffer.count), nil, 0, NI_NUMERICHOST) == 0 {52                out.append(String(decoding: buffer.prefix(while: { $0 != 0 }).map { UInt8(bitPattern: $0) }, as: UTF8.self))53            }54            p = info.pointee.ai_next55        }56        return out57    }5859    /// Nil when the URL may be fetched, otherwise the reason it is refused. Every address the host resolves to must be public.60    public static func refusal(for url: URL, resolver: (String) -> [String] = SSRFGuard.resolve) -> String? {61        guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else { return "Only http and https URLs are allowed." }62        guard url.user == nil, url.password == nil else { return "URLs with credentials are not allowed." }63        guard var host = url.host?.lowercased(), !host.isEmpty else { return "The URL has no host." }64        if host.hasPrefix("[") && host.hasSuffix("]") { host = String(host.dropFirst().dropLast()) }65        if host == "localhost" || host.hasSuffix(".localhost") || host.hasSuffix(".local") || host.hasSuffix(".internal") {66            return "Requests to local hosts are not allowed."67        }68        var v4 = in_addr(), v6 = in6_addr()69        let isLiteral = inet_pton(AF_INET, host, &v4) == 1 || inet_pton(AF_INET6, host, &v6) == 170        // inet_aton-style forms (2130706433, 0x7f.1, 0177.0.0.1) that some resolvers read as IPv4 addresses71        if !isLiteral && host.range(of: #"^(0x[0-9a-f]+|[0-9]+)(\.(0x[0-9a-f]+|[0-9]+))*$"#, options: .regularExpression) != nil { return "Ambiguous numeric hosts are not allowed." }72        let addresses = isLiteral ? [host] : resolver(host)73        guard !addresses.isEmpty else { return "The host \(host) could not be resolved." }74        if addresses.contains(where: isBlockedAddress) { return "Requests to private, loopback or link-local addresses are not allowed." }75        return nil76    }77}7879public struct FetchedPage: Sendable {80    public var status: Int; public var contentType: String; public var body: Data; public var finalURL: URL; public var truncated: Bool81    public init(status: Int, contentType: String, body: Data, finalURL: URL, truncated: Bool) {82        self.status = status; self.contentType = contentType; self.body = body; self.finalURL = finalURL; self.truncated = truncated83    }84}8586public protocol URLFetching: Sendable {87    func fetch(_ url: URL, maxBytes: Int, timeout: TimeInterval) async throws -> FetchedPage88}8990/// URLSession fetcher that validates the first URL and every redirect target with SSRFGuard. No cookies, no credentials, no cache.91/// Known limit: the address is checked at resolution time, not pinned for the connection (DNS rebinding is not fully excluded).92public final class SafeURLFetcher: NSObject, URLFetching, URLSessionTaskDelegate, Sendable {93    public override init() { super.init() }9495    public func urlSession(_ session: URLSession, task: URLSessionTask, willPerformHTTPRedirection response: HTTPURLResponse,96                           newRequest request: URLRequest) async -> URLRequest? {97        guard let url = request.url, SSRFGuard.refusal(for: url) == nil else { return nil }98        return request99    }100101    public func fetch(_ url: URL, maxBytes: Int, timeout: TimeInterval) async throws -> FetchedPage {102        if let reason = SSRFGuard.refusal(for: url) { throw ToolError(reason) }103        let config = URLSessionConfiguration.ephemeral104        config.httpCookieStorage = nil; config.urlCredentialStorage = nil; config.urlCache = nil105        config.timeoutIntervalForRequest = timeout; config.timeoutIntervalForResource = timeout106        let session = URLSession(configuration: config, delegate: self, delegateQueue: nil)107        defer { session.invalidateAndCancel() }108        var request = URLRequest(url: url); request.setValue("LocalChat/1.0", forHTTPHeaderField: "User-Agent")109        request.setValue("text/html,text/plain,application/json;q=0.9,*/*;q=0.5", forHTTPHeaderField: "Accept")110        let (bytes, response) = try await session.bytes(for: request)111        guard let http = response as? HTTPURLResponse else { throw ToolError("Not an HTTP response.") }112        if (300..<400).contains(http.statusCode) { throw ToolError("Redirect to a destination that is not allowed.") }113        var data = Data(); var truncated = false114        data.reserveCapacity(min(maxBytes, 1 << 16))115        for try await byte in bytes {116            if data.count >= maxBytes { truncated = true; break }117            data.append(byte)118        }119        return FetchedPage(status: http.statusCode, contentType: http.value(forHTTPHeaderField: "Content-Type") ?? "", body: data,120                           finalURL: http.url ?? url, truncated: truncated)121    }122}123124public enum HTMLText {125    static let entities: [String: String] = ["amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'", "nbsp": " ", "mdash": "—", "ndash": "–",126                                             "hellip": "…", "rsquo": "’", "lsquo": "‘", "rdquo": "”", "ldquo": "“", "eacute": "é", "egrave": "è",127                                             "agrave": "à", "ccedil": "ç", "copy": "©", "laquo": "«", "raquo": "»"]128129    public static func extract(_ html: String) -> String {130        var s = html131        for tag in ["script", "style", "noscript", "svg", "head", "template"] {132            s = s.replacingOccurrences(of: "<\(tag)\\b[^>]*>[\\s\\S]*?</\(tag)\\s*>", with: " ", options: [.regularExpression, .caseInsensitive])133        }134        s = s.replacingOccurrences(of: "<!--[\\s\\S]*?-->", with: " ", options: .regularExpression)135        s = s.replacingOccurrences(of: "<(br|/p|/div|/h[1-6]|/li|/tr|/section|/article)\\b[^>]*>", with: "\n", options: [.regularExpression, .caseInsensitive])136        s = s.replacingOccurrences(of: "<li\\b[^>]*>", with: "• ", options: [.regularExpression, .caseInsensitive])137        s = s.replacingOccurrences(of: "<[^>]+>", with: "", options: .regularExpression)138        s = decodeEntities(s)139        s = s.replacingOccurrences(of: "[ \\t\\u{00A0}]+", with: " ", options: .regularExpression)140        s = s.replacingOccurrences(of: " ?\\n ?", with: "\n", options: .regularExpression)141        s = s.replacingOccurrences(of: "\\n{3,}", with: "\n\n", options: .regularExpression)142        return s.trimmingCharacters(in: .whitespacesAndNewlines)143    }144145    static func decodeEntities(_ text: String) -> String {146        guard text.contains("&") else { return text }147        var out = ""; var rest = text[...]148        while let amp = rest.firstIndex(of: "&") {149            out += rest[..<amp]150            let after = rest[rest.index(after: amp)...]151            if let semi = after.prefix(10).firstIndex(of: ";") {152                let name = String(after[..<semi]); var replacement: String?153                if name.hasPrefix("#x") || name.hasPrefix("#X") { replacement = UInt32(name.dropFirst(2), radix: 16).flatMap(Unicode.Scalar.init).map { String($0) } }154                else if name.hasPrefix("#") { replacement = UInt32(name.dropFirst()).flatMap(Unicode.Scalar.init).map { String($0) } }155                else { replacement = entities[name] }156                if let replacement { out += replacement; rest = after[after.index(after: semi)...]; continue }157            }158            out += "&"; rest = after159        }160        return out + rest161    }162}163164struct FetchURLTool: AgentTool {165    let risk: ToolRisk = .read166    let fetcher: any URLFetching167    let definition = ToolDefinition(168        name: "fetch_url", summary: "Downloads a public web page (http/https) and returns its text. Sends the URL to that website.",169        parameters: ["type": "object", "properties": ["url": ["type": "string"]], "required": ["url"]])170    static let maxBytes = 1_500_000, maxCharacters = 20_000171172    func run(arguments: JSONValue, context: ToolContext) async throws -> ToolOutput {173        guard let raw = arguments["url"]?.stringValue, let url = URL(string: raw.trimmingCharacters(in: .whitespaces)) else { throw ToolError("Invalid URL.") }174        if let reason = SSRFGuard.refusal(for: url) { throw ToolError(reason) }175        let page = try await fetcher.fetch(url, maxBytes: Self.maxBytes, timeout: 20)176        guard (200..<300).contains(page.status) else { throw ToolError("The server answered HTTP \(page.status).") }177        let type = page.contentType.lowercased()178        guard type.isEmpty || type.contains("text") || type.contains("json") || type.contains("xml") else { throw ToolError("Unsupported content type: \(page.contentType).") }179        guard let raw = String(data: page.body, encoding: .utf8) ?? String(data: page.body, encoding: .isoLatin1) else { throw ToolError("The page is not text.") }180        var text = type.contains("html") || raw.range(of: "<html", options: .caseInsensitive) != nil ? HTMLText.extract(raw) : raw181        var note = page.truncated ? "\n[… download truncated]" : ""182        if text.count > Self.maxCharacters { text = String(text.prefix(Self.maxCharacters)); note = "\n[… truncated]" }183        return ToolOutput("Content of \(page.finalURL.absoluteString):\n\n" + text + note)184    }185}186
Files main
Menu

Theme