spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1<!doctype html>2<!-- author: simon-pierre boucher <contact@spboucher.ai> -->3<html lang="en" data-theme="dark">4<head>5<meta charset="utf-8" />6<meta name="viewport" content="width=device-width, initial-scale=1" />7<title>Tendril — web ingestion that reaches what others can't</title>8<meta name="description" content="Tendril is a web ingestion platform (scrape · map) running on residential macOS in WebKit. Deterministic, testable extraction. Try it in the playground." />9<meta property="og:title" content="Tendril" />10<meta property="og:description" content="Web ingestion that reaches what others can't. Scrape and map the web with deterministic, testable extraction." />11<meta property="og:type" content="website" />12<link rel="icon" type="image/svg+xml" href="/logo.svg" />13<link rel="apple-touch-icon" href="/logo.svg" />14<link rel="preconnect" href="https://fonts.googleapis.com" />15<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />16<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=Inter:wght@400;450;500;600&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet" />17<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/styles/github-dark.min.css" />18<style>19 :root {20 --bg: #07100c;21 --bg-2: #0b1712;22 --panel: rgba(18, 32, 26, 0.72);23 --panel-solid: #101b16;24 --line: rgba(120, 200, 160, 0.16);25 --line-strong: rgba(120, 200, 160, 0.32);26 --ink: #e7f3ec;27 --muted: #8fae9f;28 --dim: #5f7a6d;29 --vine: #4ade80;30 --vine-2: #2dd4bf;31 --amber: #f5c451;32 --danger: #ff6b6b;33 --radius: 16px;34 --mono: "JetBrains Mono", ui-monospace, monospace;35 --sans: "Inter", system-ui, sans-serif;36 --display: "Space Grotesk", var(--sans);37 --shadow: 0 24px 60px -30px rgba(0, 0, 0, 0.9);38 }39 * { box-sizing: border-box; }40 html { scroll-behavior: smooth; }41 body {42 margin: 0; background: var(--bg); color: var(--ink); font-family: var(--sans);43 font-size: 15px; line-height: 1.6; -webkit-font-smoothing: antialiased;44 background-image:45 radial-gradient(1100px 600px at 82% -8%, rgba(45, 212, 191, 0.10), transparent 60%),46 radial-gradient(900px 520px at 4% 6%, rgba(74, 222, 128, 0.09), transparent 55%);47 background-attachment: fixed;48 }49 a { color: inherit; text-decoration: none; }50 .wrap { max-width: 1180px; margin: 0 auto; padding: 0 24px; }5152 /* header */53 header {54 position: sticky; top: 0; z-index: 50; backdrop-filter: blur(14px);55 background: linear-gradient(to bottom, rgba(7,16,12,0.9), rgba(7,16,12,0.55));56 border-bottom: 1px solid var(--line);57 }58 .nav { display: flex; align-items: center; gap: 20px; height: 62px; }59 .brand { display: flex; align-items: center; gap: 9px; font-family: var(--display); font-weight: 700; letter-spacing: -0.01em; font-size: 18px; }60 .brand .logo { border-radius: 7px; filter: drop-shadow(0 0 10px rgba(74,222,128,0.35)); }61 .brand b { color: var(--vine); }62 .nav .spacer { flex: 1; }63 .nav a.link { color: var(--muted); font-weight: 500; font-size: 14px; padding: 8px 10px; border-radius: 9px; }64 .nav a.link:hover { color: var(--ink); background: rgba(120,200,160,0.08); }65 .pill {66 font-family: var(--mono); font-size: 12px; color: var(--vine);67 border: 1px solid var(--line-strong); border-radius: 999px; padding: 5px 11px;68 display: inline-flex; align-items: center; gap: 7px; background: rgba(74,222,128,0.06);69 }70 .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--vine); box-shadow: 0 0 9px var(--vine); animation: pulse 2s infinite; }71 @keyframes pulse { 0%,100%{opacity:1} 50%{opacity:.35} }7273 /* hero */74 .hero { padding: 74px 0 40px; position: relative; overflow: hidden; }75 .kicker { font-family: var(--mono); font-size: 12.5px; letter-spacing: 0.18em; text-transform: uppercase; color: var(--vine-2); margin-bottom: 18px; }76 h1 { font-family: var(--display); font-size: clamp(38px, 6vw, 68px); line-height: 1.02; letter-spacing: -0.03em; margin: 0 0 18px; font-weight: 700; }77 h1 .grad { background: linear-gradient(120deg, var(--vine), var(--vine-2) 60%, var(--amber)); -webkit-background-clip: text; background-clip: text; color: transparent; }78 .lede { font-size: 19px; color: var(--muted); max-width: 640px; margin: 0 0 28px; }79 .cta-row { display: flex; gap: 12px; flex-wrap: wrap; align-items: center; }80 .btn {81 font-family: var(--display); font-weight: 600; font-size: 15px; cursor: pointer;82 border-radius: 12px; padding: 12px 20px; border: 1px solid var(--line-strong);83 background: var(--panel-solid); color: var(--ink); transition: .15s; display: inline-flex; align-items: center; gap: 9px;84 }85 .btn:hover { transform: translateY(-1px); border-color: var(--vine); }86 .btn.primary { background: linear-gradient(120deg, var(--vine), var(--vine-2)); color: #05130c; border: none; box-shadow: 0 14px 34px -14px rgba(74,222,128,0.6); }87 .btn.primary:hover { filter: brightness(1.06); }88 .stats { display: flex; gap: 30px; flex-wrap: wrap; margin-top: 40px; }89 .stat .n { font-family: var(--display); font-size: 26px; font-weight: 700; color: var(--ink); }90 .stat .l { font-size: 12.5px; color: var(--dim); font-family: var(--mono); text-transform: uppercase; letter-spacing: 0.08em; }91 .vine-svg { position: absolute; right: -40px; top: 20px; width: 420px; max-width: 46%; opacity: 0.5; pointer-events: none; }92 .vine-svg path { fill: none; stroke: url(#vg); stroke-width: 2; stroke-linecap: round; stroke-dasharray: 1400; stroke-dashoffset: 1400; animation: grow 3.2s ease forwards; }93 @keyframes grow { to { stroke-dashoffset: 0; } }9495 section { padding: 40px 0; }96 .sec-head { display: flex; align-items: baseline; gap: 14px; margin-bottom: 22px; }97 .sec-head h2 { font-family: var(--display); font-size: 26px; letter-spacing: -0.02em; margin: 0; }98 .sec-head .tag { font-family: var(--mono); font-size: 12px; color: var(--dim); }99100 /* playground */101 .pg { display: grid; grid-template-columns: 1fr 1.15fr; gap: 20px; }102 @media (max-width: 900px){ .pg { grid-template-columns: 1fr; } .vine-svg{ display:none } }103 .card { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); box-shadow: var(--shadow); backdrop-filter: blur(8px); }104 .card .hd { padding: 15px 18px; border-bottom: 1px solid var(--line); display: flex; align-items: center; gap: 10px; }105 .card .hd h3 { font-family: var(--display); font-size: 15px; margin: 0; letter-spacing: -0.01em; }106 .card .bd { padding: 18px; }107 .seg { display: inline-flex; background: rgba(10,20,15,0.7); border: 1px solid var(--line); border-radius: 11px; padding: 4px; gap: 3px; }108 .seg button { font-family: var(--mono); font-size: 13px; color: var(--muted); background: none; border: 0; padding: 7px 14px; border-radius: 8px; cursor: pointer; }109 .seg button.on { background: linear-gradient(120deg, rgba(74,222,128,0.9), rgba(45,212,191,0.9)); color: #05130c; font-weight: 600; }110 label.f { display: block; font-size: 12.5px; color: var(--muted); font-family: var(--mono); margin: 16px 0 7px; letter-spacing: 0.02em; }111 input[type=text], input[type=number], select {112 width: 100%; background: rgba(6,14,10,0.85); border: 1px solid var(--line); color: var(--ink);113 border-radius: 10px; padding: 11px 13px; font-family: var(--mono); font-size: 13.5px; outline: none; transition: .15s;114 }115 input:focus, select:focus { border-color: var(--vine); box-shadow: 0 0 0 3px rgba(74,222,128,0.12); }116 .chips { display: flex; flex-wrap: wrap; gap: 8px; }117 .chk { font-family: var(--mono); font-size: 12.5px; display: inline-flex; align-items: center; gap: 7px; padding: 8px 12px; border: 1px solid var(--line); border-radius: 9px; cursor: pointer; color: var(--muted); user-select: none; }118 .chk input { accent-color: var(--vine); }119 .chk.on { border-color: var(--vine); color: var(--ink); background: rgba(74,222,128,0.07); }120 .row2 { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }121 .send-row { display: flex; align-items: center; gap: 12px; margin-top: 20px; }122 .meta-line { font-family: var(--mono); font-size: 12px; color: var(--dim); }123 .badge { font-family: var(--mono); font-size: 11px; padding: 3px 8px; border-radius: 7px; border: 1px solid var(--line); color: var(--muted); }124 .badge.ok { color: var(--vine); border-color: var(--line-strong); }125 .badge.err { color: var(--danger); border-color: rgba(255,107,107,0.4); }126127 .tabs { display: flex; gap: 4px; flex-wrap: wrap; }128 .tabs button { font-family: var(--mono); font-size: 12.5px; color: var(--muted); background: none; border: 1px solid transparent; padding: 6px 11px; border-radius: 8px; cursor: pointer; }129 .tabs button.on { color: var(--ink); border-color: var(--line); background: rgba(10,20,15,0.6); }130 .out { min-height: 260px; max-height: 560px; overflow: auto; }131 .out pre { margin: 0; }132 pre code.hljs { background: transparent !important; font-family: var(--mono); font-size: 12.8px; line-height: 1.55; padding: 16px 18px; display: block; }133 .rendered { padding: 18px; font-size: 14.5px; }134 .rendered h1,.rendered h2,.rendered h3 { font-family: var(--display); line-height: 1.2; }135 .rendered a { color: var(--vine-2); border-bottom: 1px solid rgba(45,212,191,0.3); }136 .rendered pre { background: rgba(6,14,10,0.9); border: 1px solid var(--line); border-radius: 10px; padding: 12px; overflow: auto; }137 .rendered table { border-collapse: collapse; width: 100%; font-size: 13px; }138 .rendered th, .rendered td { border: 1px solid var(--line); padding: 6px 10px; text-align: left; }139 .rendered img { max-width: 100%; border-radius: 8px; }140 .placeholder { color: var(--dim); font-family: var(--mono); font-size: 13px; padding: 40px 18px; text-align: center; }141 .linkrow { display: flex; align-items: center; gap: 10px; padding: 9px 14px; border-bottom: 1px solid var(--line); font-family: var(--mono); font-size: 12.5px; }142 .linkrow a { color: var(--ink); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex: 1; }143 .src { font-size: 10.5px; text-transform: uppercase; letter-spacing: 0.06em; padding: 2px 7px; border-radius: 6px; border: 1px solid var(--line); color: var(--vine-2); }144 .copybtn { font-family: var(--mono); font-size: 11.5px; cursor: pointer; background: rgba(10,20,15,0.6); border: 1px solid var(--line); color: var(--muted); padding: 4px 9px; border-radius: 7px; }145 .copybtn:hover { color: var(--vine); border-color: var(--vine); }146 .snip { position: relative; margin-top: 6px; }147 .snip pre { background: rgba(6,14,10,0.92); border: 1px solid var(--line); border-radius: 10px; margin: 0; overflow: auto; }148 .snip .copysnip { position: absolute; top: 8px; right: 8px; z-index: 2; }149 .spin { width: 15px; height: 15px; border: 2px solid rgba(255,255,255,0.2); border-top-color: var(--vine); border-radius: 50%; animation: sp .7s linear infinite; }150 @keyframes sp { to { transform: rotate(360deg); } }151152 /* docs */153 .ep { border: 1px solid var(--line); border-radius: var(--radius); margin-bottom: 16px; overflow: hidden; background: var(--panel); }154 .ep .top { display: flex; align-items: center; gap: 12px; padding: 15px 18px; cursor: pointer; }155 .method { font-family: var(--mono); font-weight: 600; font-size: 12px; padding: 4px 9px; border-radius: 7px; background: rgba(74,222,128,0.14); color: var(--vine); }156 .method.get { background: rgba(45,212,191,0.14); color: var(--vine-2); }157 .path { font-family: var(--mono); font-size: 14px; }158 .ep .desc { color: var(--muted); font-size: 13px; margin-left: auto; }159 .ep .body { padding: 0 18px 18px; display: none; }160 .ep.open .body { display: block; }161 .ptable { width: 100%; border-collapse: collapse; font-size: 13px; margin: 8px 0 4px; }162 .ptable th { text-align: left; color: var(--dim); font-family: var(--mono); font-weight: 500; font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.06em; padding: 6px 8px; border-bottom: 1px solid var(--line); }163 .ptable td { padding: 7px 8px; border-bottom: 1px solid var(--line); vertical-align: top; }164 .ptable code { font-family: var(--mono); color: var(--vine-2); font-size: 12.5px; }165 .req-opt { font-size: 10.5px; color: var(--amber); font-family: var(--mono); }166 .errtbl { width: 100%; border-collapse: collapse; font-size: 12.5px; }167 .errtbl td, .errtbl th { padding: 6px 10px; border-bottom: 1px solid var(--line); text-align: left; }168 .errtbl code { color: var(--danger); font-family: var(--mono); }169 footer { border-top: 1px solid var(--line); padding: 34px 0 60px; color: var(--dim); font-size: 13px; margin-top: 30px; }170 footer a { color: var(--muted); } footer a:hover { color: var(--vine); }171 .grid3 { display: grid; grid-template-columns: repeat(3,1fr); gap: 16px; }172 @media (max-width:760px){ .grid3{ grid-template-columns:1fr } .row2{ grid-template-columns:1fr } }173 .feat { padding: 18px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--panel); }174 .feat .i { font-size: 22px; margin-bottom: 10px; }175 .feat h4 { font-family: var(--display); margin: 0 0 6px; font-size: 16px; }176 .feat p { margin: 0; color: var(--muted); font-size: 13.5px; }177 .note { font-size: 12.5px; color: var(--amber); font-family: var(--mono); background: rgba(245,196,81,0.07); border: 1px solid rgba(245,196,81,0.25); border-radius: 10px; padding: 10px 13px; margin-top: 14px; }178179 /* result presentation */180 .metrics { display: flex; flex-wrap: wrap; gap: 8px; padding: 4px 2px 14px; align-items: center; }181 .metric { display: inline-flex; align-items: baseline; gap: 6px; font-family: var(--mono); font-size: 12px; padding: 5px 10px; border: 1px solid var(--line); border-radius: 8px; background: rgba(10,20,15,0.5); }182 .metric .k { color: var(--dim); text-transform: uppercase; letter-spacing: 0.05em; font-size: 10px; }183 .metric .v { color: var(--ink); font-weight: 600; }184 .metric.good .v { color: var(--vine); } .metric.bad .v { color: var(--danger); }185 .metacard { display: flex; gap: 14px; padding: 14px; border: 1px solid var(--line); border-radius: 12px; background: linear-gradient(180deg, rgba(45,212,191,0.05), rgba(10,20,15,0.2)); margin: 0 0 8px; }186 .metacard .thumb { width: 92px; height: 92px; border-radius: 10px; object-fit: cover; border: 1px solid var(--line); flex: none; background: rgba(6,14,10,0.6); }187 .metacard .mtitle { font-family: var(--display); font-size: 17px; font-weight: 600; margin: 0 0 4px; letter-spacing: -0.01em; }188 .metacard .mdesc { color: var(--muted); font-size: 13px; margin: 0 0 8px; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; }189 .mchips { display: flex; flex-wrap: wrap; gap: 6px; }190 .mchip { font-family: var(--mono); font-size: 11px; color: var(--vine-2); border: 1px solid var(--line); border-radius: 999px; padding: 2px 9px; }191 .mchip.link { color: var(--muted); } .mchip.link:hover { color: var(--vine); border-color: var(--vine); }192 .toolbar { display: flex; gap: 8px; padding: 10px 2px; flex-wrap: wrap; align-items: center; }193 .tb-btn { font-family: var(--mono); font-size: 12px; cursor: pointer; background: rgba(10,20,15,0.6); border: 1px solid var(--line); color: var(--muted); padding: 6px 11px; border-radius: 8px; }194 .tb-btn:hover { color: var(--vine); border-color: var(--vine); }195 .tb-btn.active { color: var(--vine); border-color: var(--line-strong); background: rgba(74,222,128,0.08); }196 .filterbar { display: flex; gap: 8px; align-items: center; padding: 6px 2px 12px; flex-wrap: wrap; }197 .filterbar input { flex: 1; min-width: 160px; padding: 8px 11px; font-size: 12.5px; }198 .kv { display: grid; grid-template-columns: 150px 1fr; gap: 1px; background: var(--line); border: 1px solid var(--line); border-radius: 10px; overflow: hidden; }199 .kv > div { background: var(--panel-solid); padding: 8px 12px; font-family: var(--mono); font-size: 12.5px; }200 .kv .kk { color: var(--dim); } .kv .vv { color: var(--ink); word-break: break-word; }201 .kv .vv a { color: var(--vine-2); }202 .skel { padding: 16px; } .skel .ln { height: 12px; border-radius: 6px; background: linear-gradient(90deg, rgba(120,200,160,0.06), rgba(120,200,160,0.16), rgba(120,200,160,0.06)); background-size: 200% 100%; animation: shim 1.2s infinite; margin: 10px 0; }203 @keyframes shim { 0%{background-position:200% 0} 100%{background-position:-200% 0} }204 .empty2 { color: var(--dim); font-family: var(--mono); font-size: 12.5px; padding: 24px 16px; text-align: center; }205 .srcbar { display: flex; flex-wrap: wrap; gap: 6px; padding: 4px 2px 12px; }206</style>207</head>208<body>209<header>210 <div class="wrap nav">211 <a class="brand" href="#top"><img class="logo" src="/logo.svg" alt="" width="26" height="26" /> ten<b>·</b>dril</a>212 <span class="spacer"></span>213 <a class="link" href="#playground">Playground</a>214 <a class="link" href="#agents">For agents</a>215 <a class="link" href="#docs">Docs</a>216 <a class="link" href="https://git.spboucher.ai/tendril" target="_blank" rel="noopener">Source</a>217 <span class="pill" id="statusPill"><span class="dot"></span> checking…</span>218 </div>219</header>220221<a id="top"></a>222<div class="hero">223 <div class="wrap">224 <svg class="vine-svg" viewBox="0 0 300 300" aria-hidden="true">225 <defs><linearGradient id="vg" x1="0" y1="0" x2="1" y2="1">226 <stop offset="0" stop-color="#4ade80"/><stop offset="0.6" stop-color="#2dd4bf"/><stop offset="1" stop-color="#f5c451"/>227 </linearGradient></defs>228 <path d="M150 300 C150 240 120 220 150 180 C180 140 120 120 150 80 C170 50 150 30 150 0" />229 <path d="M150 220 C120 210 100 190 85 200 C70 210 80 230 100 225" />230 <path d="M150 150 C185 140 205 120 220 132 C235 144 224 165 202 158" />231 <path d="M150 95 C120 86 104 66 90 78 C78 88 86 108 106 102" />232 </svg>233 <div class="kicker">Web ingestion · scrape · map</div>234 <h1>Reach what other<br />crawlers <span class="grad">can't.</span></h1>235 <p class="lede">Tendril runs on a residential Mac, in WebKit — Safari's real engine — behind a real IP. Extraction is <strong>deterministic and testable</strong>: the same page always yields the same markdown. No cloud fingerprint, no LLM lottery.</p>236 <div class="cta-row">237 <a class="btn primary" href="#playground">▶ Open the playground</a>238 <a class="btn" href="#docs">Read the docs</a>239 </div>240 <div class="stats">241 <div class="stat"><div class="n">3</div><div class="l">escalation tiers</div></div>242 <div class="stat"><div class="n">0</div><div class="l">LLMs in extraction</div></div>243 <div class="stat"><div class="n">~50ms</div><div class="l">Tier 0 p50</div></div>244 <div class="stat"><div class="n">WebKit</div><div class="l">real Safari engine</div></div>245 </div>246 </div>247</div>248249<section class="wrap">250 <div class="grid3">251 <div class="feat"><div class="i">🕸️</div><h4>Browser fidelity</h4><p>WebKit on macOS ARM behind a residential IP — not a headless Chromium signature. Sites that block others often don't block Tendril.</p></div>252 <div class="feat"><div class="i">📐</div><h4>Deterministic extraction</h4><p>JSON-LD, OpenGraph, Readability + density fallback, hand-written Turndown rules. Versioned, golden-file testable output.</p></div>253 <div class="feat"><div class="i">🧭</div><h4>Map without rendering</h4><p>Merge sitemaps, robots directives, homepage links and <code>/llms.txt</code> into one deduplicated URL list in seconds.</p></div>254 </div>255</section>256257<section class="wrap" id="playground">258 <div class="sec-head"><h2>Playground</h2><span class="tag">// live requests against this deployment</span></div>259 <div class="pg">260 <div class="card">261 <div class="hd"><h3>Request</h3><span class="spacer" style="flex:1"></span>262 <div class="seg" id="epseg">263 <button data-ep="scrape" class="on">scrape</button>264 <button data-ep="map">map</button>265 <button data-ep="status">status</button>266 </div>267 </div>268 <div class="bd">269 <div id="urlblock">270 <label class="f">Target URL</label>271 <input type="text" id="url" value="https://www.iana.org" spellcheck="false" placeholder="https://example.com" />272 </div>273274 <div id="scrapeOpts">275 <label class="f">Formats</label>276 <div class="chips" id="formats">277 <label class="chk on"><input type="checkbox" value="markdown" checked>markdown</label>278 <label class="chk"><input type="checkbox" value="links">links</label>279 <label class="chk"><input type="checkbox" value="structured">structured</label>280 <label class="chk"><input type="checkbox" value="html">html</label>281 </div>282 <div class="row2">283 <div><label class="f">Tier</label>284 <select id="tier"><option value="auto">auto</option><option value="http">http (Tier 0)</option><option value="webkit">webkit</option><option value="safari">safari</option></select>285 </div>286 <div><label class="f">Options</label>287 <div class="chips">288 <label class="chk on"><input type="checkbox" id="onlyMain" checked>onlyMainContent</label>289 <label class="chk on"><input type="checkbox" id="respRobots" checked>respectRobots</label>290 </div>291 </div>292 </div>293 </div>294295 <div id="mapOpts" style="display:none">296 <div class="row2">297 <div><label class="f">Search filter</label><input type="text" id="mapSearch" placeholder="optional substring" spellcheck="false"></div>298 <div><label class="f">Limit</label><input type="number" id="mapLimit" value="30" min="1" max="50000"></div>299 </div>300 <label class="f">Options</label>301 <div class="chips">302 <label class="chk"><input type="checkbox" id="sitemapOnly">sitemapOnly</label>303 <label class="chk"><input type="checkbox" id="ignoreSitemap">ignoreSitemap</label>304 <label class="chk"><input type="checkbox" id="subdomains">includeSubdomains</label>305 </div>306 </div>307308 <div id="statusOpts" style="display:none">309 <p class="meta-line">No parameters. Returns per-tier availability and pipeline version.</p>310 </div>311312 <div class="send-row">313 <button class="btn primary" id="send">▶ Send request</button>314 <span id="reqmeta" class="meta-line"></span>315 </div>316 <div class="note" id="autoNote" style="display:none">Tiers 1 & 2 (WebKit/Safari) aren't deployed yet — a page needing escalation under <b>tier: auto</b> returns <code>ERR_TARGET_BLOCKED</code>. Force <b>tier: http</b> for raw Tier 0 output.</div>317 </div>318 </div>319320 <div class="card">321 <div class="hd"><h3>Response</h3><span class="spacer" style="flex:1"></span><span id="respStatus"></span></div>322 <div class="bd" style="padding-top:14px">323 <div class="tabs" id="tabs"></div>324 <div class="out" id="out"><div class="placeholder">Send a request to see the response, rendered markdown, extracted links and ready-to-copy snippets.</div></div>325 </div>326 </div>327 </div>328</section>329330<section class="wrap" id="agents">331 <div class="sec-head"><h2>Give it to your agent</h2><span class="tag">// hand these to an AI agent so it can drive Tendril</span></div>332 <div class="pg">333 <div class="card">334 <div class="hd"><span style="font-size:20px">🔌</span><h3>MCP server</h3><span class="spacer" style="flex:1"></span><span class="badge ok">no install</span></div>335 <div class="bd">336 <p style="margin-top:0;color:var(--muted);font-size:14px">A single-file <b>Model Context Protocol</b> server exposing <code style="color:var(--vine-2)">tendril_scrape</code>, <code style="color:var(--vine-2)">tendril_map</code> and <code style="color:var(--vine-2)">tendril_status</code>. Works in Claude Desktop, Claude Code and any MCP client. Needs only Node ≥ 20.</p>337 <div class="cta-row" style="margin:14px 0">338 <a class="btn primary" href="/downloads/tendril-mcp.mjs" download>↓ tendril-mcp.mjs</a>339 <a class="btn" href="/downloads/tendril-mcp.zip" download>↓ .zip (with README)</a>340 </div>341 <label class="f">Claude Desktop config</label>342 <div class="snip"><button class="copybtn copysnip">⧉ copy</button><pre><code class="hljs language-json">{343 "mcpServers": {344 "tendril": {345 "command": "node",346 "args": ["/absolute/path/to/tendril-mcp.mjs"],347 "env": { "TENDRIL_BASE_URL": "https://www.ten-dril.com" }348 }349 }350}</code></pre></div>351 <label class="f">Claude Code (one line)</label>352 <div class="snip"><button class="copybtn copysnip">⧉ copy</button><pre><code class="hljs language-bash">claude mcp add tendril -- node /absolute/path/to/tendril-mcp.mjs</code></pre></div>353 </div>354 </div>355356 <div class="card">357 <div class="hd"><span style="font-size:20px">📎</span><h3>Agent Skill</h3><span class="spacer" style="flex:1"></span><span class="badge">SKILL.md</span></div>358 <div class="bd">359 <p style="margin-top:0;color:var(--muted);font-size:14px">A Claude <b>Agent Skill</b> that teaches an agent <i>when</i> and <i>how</i> to use Tendril over plain HTTP — endpoints, options, workflow (map → scrape), and error codes. No MCP client required; works with any agent that reads skills or can <code style="color:var(--vine-2)">curl</code>.</p>360 <div class="cta-row" style="margin:14px 0">361 <a class="btn primary" href="/downloads/tendril-skill.zip" download>↓ tendril-skill.zip</a>362 <a class="btn" href="/downloads/tendril-skill.zip">view SKILL.md</a>363 </div>364 <label class="f">Install (Claude Code / agents)</label>365 <div class="snip"><button class="copybtn copysnip">⧉ copy</button><pre><code class="hljs language-bash">unzip tendril-skill.zip -d ~/.claude/skills/366# → ~/.claude/skills/tendril/SKILL.md</code></pre></div>367 <p class="meta-line" style="margin-top:12px">The skill points at <code>https://www.ten-dril.com</code> and needs no API key today.</p>368 </div>369 </div>370 </div>371</section>372373<section class="wrap" id="docs">374 <div class="sec-head"><h2>API reference</h2><span class="tag">// base: this origin · Bearer auth coming with accounts</span></div>375 <div id="epdocs"></div>376377 <div class="sec-head" style="margin-top:30px"><h2>Errors</h2><span class="tag">// §15 taxonomy — one code per failure</span></div>378 <div class="card"><div class="bd">379 <table class="errtbl"><tbody>380 <tr><td><code>ERR_INVALID_URL</code></td><td>400</td><td>Malformed or unsupported scheme</td></tr>381 <tr><td><code>ERR_SSRF_BLOCKED</code></td><td>400</td><td>Resolved to a private/forbidden address</td></tr>382 <tr><td><code>ERR_ROBOTS_DENIED</code></td><td>403</td><td>robots.txt disallows the path</td></tr>383 <tr><td><code>ERR_TARGET_BLOCKED</code></td><td>502</td><td>Tier 0 insufficient; higher tiers unavailable</td></tr>384 <tr><td><code>ERR_UNSUPPORTED_TYPE</code></td><td>415</td><td>Content type not handled</td></tr>385 <tr><td><code>ERR_TIER_TIMEOUT</code></td><td>504</td><td>Render exceeded timeout</td></tr>386 </tbody></table>387 <p class="meta-line" style="margin-top:12px">Every response carries a <code>requestId</code> and an <code>X-Request-Id</code> header for support.</p>388 </div></div>389</section>390391<footer>392 <div class="wrap">393 <div style="display:flex;gap:20px;flex-wrap:wrap;align-items:center">394 <span class="brand"><img class="logo" src="/logo.svg" alt="" width="24" height="24" /> ten·dril</span>395 <span class="spacer" style="flex:1"></span>396 <a href="https://git.spboucher.ai/tendril" target="_blank" rel="noopener">Source</a>397 <a href="/healthz">/healthz</a>398 <a href="mailto:contact@spboucher.ai">contact@spboucher.ai</a>399 </div>400 <p style="margin-top:16px;max-width:640px">Tendril identifies itself with a bot User-Agent and a contact URL, honours <code>robots.txt</code> by default, and rate-limits per host. Stealth here means matching a real browser's fingerprint — not hiding who we are. Built by Simon-Pierre Boucher.</p>401 </div>402</footer>403404<script src="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/highlight.min.js"></script>405<script src="https://cdn.jsdelivr.net/npm/marked@12.0.0/marked.min.js"></script>406<script>407const $ = (s, r=document) => r.querySelector(s);408const $$ = (s, r=document) => [...r.querySelectorAll(s)];409let endpoint = "scrape";410let lastResult = null;411let activeTab = "preview";412413// status pill414fetch("/v1/status").then(r=>r.json()).then(j=>{415 const p = $("#statusPill");416 const http = j?.data?.tiers?.http?.available;417 p.innerHTML = `<span class="dot"></span> ${http ? "live" : "degraded"} · v${j?.data?.pipelineVersion||"?"}`;418}).catch(()=>{ $("#statusPill").innerHTML = '<span class="dot" style="background:#ff6b6b;box-shadow:0 0 9px #ff6b6b"></span> offline'; });419420// endpoint switch421$$("#epseg button").forEach(b => b.onclick = () => {422 $$("#epseg button").forEach(x=>x.classList.remove("on")); b.classList.add("on");423 endpoint = b.dataset.ep;424 $("#scrapeOpts").style.display = endpoint==="scrape" ? "" : "none";425 $("#mapOpts").style.display = endpoint==="map" ? "" : "none";426 $("#statusOpts").style.display = endpoint==="status" ? "" : "none";427 $("#urlblock").style.display = endpoint==="status" ? "none" : "";428});429430// chip toggles431$$(".chk").forEach(c => { const i = c.querySelector("input"); if(i) i.onchange = () => c.classList.toggle("on", i.checked); });432$("#tier").onchange = e => $("#autoNote").style.display = e.target.value==="auto" ? "" : "none";433434function buildBody() {435 if (endpoint === "status") return null;436 const url = $("#url").value.trim();437 if (endpoint === "scrape") {438 const formats = $$("#formats input:checked").map(i=>i.value);439 return { url, formats, tier: $("#tier").value, onlyMainContent: $("#onlyMain").checked, respectRobots: $("#respRobots").checked };440 }441 const b = { url, limit: Number($("#mapLimit").value)||30, sitemapOnly: $("#sitemapOnly").checked, ignoreSitemap: $("#ignoreSitemap").checked, includeSubdomains: $("#subdomains").checked };442 const s = $("#mapSearch").value.trim(); if (s) b.search = s;443 return b;444}445446function snippets(path, body) {447 const origin = location.origin;448 if (!body) return {449 curl: `curl -s ${origin}${path}`,450 python: `import requests\nprint(requests.get("${origin}${path}").json())`,451 js: `const r = await fetch("${origin}${path}");\nconsole.log(await r.json());`452 };453 const j = JSON.stringify(body, null, 2);454 return {455 curl: `curl -s -X POST ${origin}${path} \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(body)}'`,456 python: `import requests\nr = requests.post("${origin}${path}", json=${j.replace(/true/g,"True").replace(/false/g,"False")})\nprint(r.json())`,457 js: `const r = await fetch("${origin}${path}", {\n method: "POST",\n headers: { "content-type": "application/json" },\n body: JSON.stringify(${j})\n});\nconsole.log(await r.json());`458 };459}460461function setTabs(tabs) {462 const t = $("#tabs"); t.innerHTML = "";463 tabs.forEach(name => {464 const b = document.createElement("button");465 b.textContent = name; b.className = name===activeTab ? "on" : "";466 b.onclick = () => { activeTab = name; render(); };467 t.appendChild(b);468 });469}470471function code(text, lang) {472 const pre = document.createElement("pre");473 const c = document.createElement("code");474 c.className = "hljs" + (lang ? " language-"+lang : "");475 c.textContent = text;476 pre.appendChild(c);477 try { hljs.highlightElement(c); } catch(e){}478 return pre;479}480function copyBar(text) {481 const bar = document.createElement("div");482 bar.style.cssText = "display:flex;justify-content:flex-end;padding:8px 12px 0";483 const b = document.createElement("button"); b.className="copybtn"; b.textContent="⧉ copy";484 b.onclick = () => { navigator.clipboard.writeText(text); b.textContent="✓ copied"; setTimeout(()=>b.textContent="⧉ copy",1200); };485 bar.appendChild(b); return bar;486}487488function el(tag, cls, html) { const e = document.createElement(tag); if (cls) e.className = cls; if (html !== undefined) e.innerHTML = html; return e; }489function download(name, text, type) {490 const blob = new Blob([text], { type: type || "text/plain" });491 const a = document.createElement("a"); a.href = URL.createObjectURL(blob); a.download = name; a.click();492 setTimeout(()=>URL.revokeObjectURL(a.href), 2000);493}494function esc(s){ return String(s).replace(/[&<>"]/g, c=>({"&":"&","<":"<",">":">",'"':"""}[c])); }495496function metricsRow() {497 const m = lastResult.meta, d = lastResult.data?.data || {};498 const row = el("div", "metrics");499 const add = (k, v, cls) => { const e = el("div", "metric" + (cls?" "+cls:"")); e.innerHTML = `<span class="k">${k}</span><span class="v">${v}</span>`; row.appendChild(e); };500 add("status", m.httpStatus, m.ok ? "good" : "bad");501 add("time", m.ms + " ms");502 if (endpoint === "scrape") {503 if (d.tierUsed) add("tier", d.tierUsed);504 if (d.cached !== undefined) add("cached", d.cached ? "yes" : "no");505 if (d.timings?.total) add("server", Math.round(d.timings.total) + " ms");506 if (d.metadata?.pipelineVersion) add("pipeline", "v" + d.metadata.pipelineVersion);507 } else if (endpoint === "map") {508 if (d.count !== undefined) add("urls", d.count);509 }510 return row;511}512513function metaCard(md) {514 if (!md || (!md.title && !md.description && !md.image)) return null;515 const card = el("div", "metacard");516 if (md.image) { const img = el("img", "thumb"); img.src = md.image; img.onerror = () => img.remove(); card.appendChild(img); }517 const body = el("div"); body.style.minWidth = "0"; body.style.flex = "1";518 if (md.title) body.appendChild(el("div", "mtitle", esc(md.title)));519 if (md.description) body.appendChild(el("div", "mdesc", esc(md.description)));520 const chips = el("div", "mchips");521 if (md.author) chips.appendChild(el("span", "mchip", "✍ " + esc(md.author)));522 if (md.siteName) chips.appendChild(el("span", "mchip", "◈ " + esc(md.siteName)));523 if (md.publishedTime) chips.appendChild(el("span", "mchip", "🕐 " + esc(String(md.publishedTime).slice(0,10))));524 if (md.lang) chips.appendChild(el("span", "mchip", esc(md.lang)));525 if (md.canonical) { const a = el("a", "mchip link", "↗ canonical"); a.href = md.canonical; a.target = "_blank"; a.rel = "noopener"; chips.appendChild(a); }526 if (chips.children.length) body.appendChild(chips);527 card.appendChild(body);528 return card;529}530531function kvGrid(obj) {532 const grid = el("div", "kv");533 const entries = Object.entries(obj || {}).filter(([,v]) => v !== undefined && v !== null && v !== "");534 if (!entries.length) return el("div", "empty2", "No metadata.");535 for (const [k, v] of entries) {536 grid.appendChild(el("div", "kk", esc(k)));537 const val = Array.isArray(v) ? v.join(", ") : typeof v === "object" ? JSON.stringify(v) : String(v);538 const isUrl = /^https?:\/\//.test(val);539 grid.appendChild(el("div", "vv", isUrl ? `<a href="${esc(val)}" target="_blank" rel="noopener">${esc(val)}</a>` : esc(val)));540 }541 return grid;542}543544function linkList(links, opts) {545 const wrap = el("div");546 const bar = el("div", "filterbar");547 let mode = "all";548 const counts = { all: links.length, internal: links.filter(l=>l.isInternal).length, external: links.filter(l=>l.isInternal===false).length };549 const listEl = el("div");550 const search = el("input"); search.type = "text"; search.placeholder = "filter these results…"; search.spellcheck = false;551 const btns = [];552 const mk = (id, label) => { const b = el("button", "tb-btn" + (id==="all"?" active":""), label); b.onclick = () => { mode = id; btns.forEach(x=>x.classList.toggle("active", x===b)); paint(); }; btns.push(b); return b; };553 if (opts.showInternal) { bar.appendChild(mk("all", `All ${counts.all}`)); bar.appendChild(mk("internal", `Internal ${counts.internal}`)); bar.appendChild(mk("external", `External ${counts.external}`)); }554 bar.appendChild(search);555 const dl = el("button", "tb-btn", "↓ .txt"); dl.onclick = () => download("tendril-links.txt", links.map(l=>l.url).join("\n")); bar.appendChild(dl);556 wrap.appendChild(bar); wrap.appendChild(listEl);557 function paint() {558 const q = search.value.trim().toLowerCase();559 let rows = links;560 if (mode === "internal") rows = rows.filter(l=>l.isInternal);561 if (mode === "external") rows = rows.filter(l=>l.isInternal===false);562 if (q) rows = rows.filter(l => l.url.toLowerCase().includes(q) || (l.text||"").toLowerCase().includes(q));563 listEl.innerHTML = "";564 if (!rows.length) { listEl.appendChild(el("div", "empty2", "No matching links.")); return; }565 rows.slice(0, 500).forEach(l => {566 const r = el("div", "linkrow");567 const badge = opts.source ? `<span class="src">${esc(l.source)}</span>` : (l.isInternal!==undefined ? `<span class="src">${l.isInternal?"internal":"external"}</span>` : "");568 const extra = l.lastModified ? `<span class="meta-line">${esc(String(l.lastModified).slice(0,10))}</span>` : "";569 r.innerHTML = `${badge}<a href="${esc(l.url)}" target="_blank" rel="noopener" title="${esc(l.text||"")}">${esc(l.url)}</a>${extra}`;570 listEl.appendChild(r);571 });572 if (rows.length > 500) listEl.appendChild(el("div", "empty2", `+${rows.length-500} more (download to see all)`));573 }574 search.oninput = paint; paint();575 return wrap;576}577578function srcBreakdown(links) {579 const by = {}; links.forEach(l => by[l.source] = (by[l.source]||0)+1);580 const bar = el("div", "srcbar");581 Object.entries(by).sort((a,b)=>b[1]-a[1]).forEach(([s,n]) => bar.appendChild(el("span", "mchip", `${s} · ${n}`)));582 return bar;583}584585function render() {586 const out = $("#out"); out.innerHTML = "";587 if (!lastResult) return;588 const { data, raw, snips } = lastResult;589 const d = data?.data || {};590 out.appendChild(metricsRow());591592 if (data?.success === false) {593 out.appendChild(el("div", "empty2", `⚠ ${data.error?.code || "error"} — ${esc(data.error?.message || "")}`));594 out.appendChild(code(raw, "json"));595 return;596 }597598 if (["curl","python","js"].includes(activeTab)) {599 const lang = activeTab==="curl"?"bash":activeTab==="python"?"python":"javascript";600 const w = el("div", "snip"); const b = el("button","copybtn copysnip","⧉ copy");601 b.onclick = () => { navigator.clipboard.writeText(snips[activeTab]); b.textContent="✓ copied"; setTimeout(()=>b.textContent="⧉ copy",1200); };602 w.appendChild(b); w.appendChild(code(snips[activeTab], lang)); out.appendChild(w); return;603 }604 if (activeTab === "json") { out.appendChild(code(raw, "json")); return; }605606 if (endpoint === "map") {607 const links = d.links || [];608 if (!links.length) { out.appendChild(el("div","empty2","No URLs discovered.")); return; }609 out.appendChild(srcBreakdown(links));610 out.appendChild(linkList(links, { source: true }));611 return;612 }613 if (endpoint === "status") { out.appendChild(code(JSON.stringify(data, null, 2), "json")); return; }614615 // scrape616 if (activeTab === "links") { out.appendChild(linkList(d.links || [], { showInternal: true })); return; }617 if (activeTab === "metadata") { out.appendChild(kvGrid(d.metadata)); return; }618 if (activeTab === "structured") { out.appendChild(code(JSON.stringify(d.structured || {}, null, 2), "json")); return; }619620 // preview621 const mc = metaCard(d.metadata); if (mc) out.appendChild(mc);622 if (d.markdown) {623 const tb = el("div", "toolbar");624 const words = d.markdown.trim().split(/\s+/).length;625 tb.appendChild(el("span", "meta-line", `${words.toLocaleString()} words · ${d.markdown.length.toLocaleString()} chars`));626 const sp = el("span"); sp.style.flex = "1"; tb.appendChild(sp);627 const cp = el("button","tb-btn","⧉ copy markdown"); cp.onclick=()=>{navigator.clipboard.writeText(d.markdown);cp.textContent="✓ copied";setTimeout(()=>cp.textContent="⧉ copy markdown",1200)}; tb.appendChild(cp);628 const dm = el("button","tb-btn","↓ .md"); dm.onclick=()=>download("tendril.md", d.markdown, "text/markdown"); tb.appendChild(dm);629 out.appendChild(tb);630 const div = el("div", "rendered"); div.innerHTML = marked.parse(d.markdown);631 div.querySelectorAll("pre code").forEach(c=>{try{hljs.highlightElement(c)}catch(e){}});632 out.appendChild(div);633 } else if (d.html) {634 out.appendChild(code(d.html.slice(0, 20000), "xml"));635 } else {636 out.appendChild(code(raw, "json"));637 }638}639640async function send() {641 const path = "/v1/" + endpoint;642 const body = buildBody();643 const snips = snippets(path, body);644 const btn = $("#send"); btn.disabled = true; btn.innerHTML = '<span class="spin"></span> running';645 $("#reqmeta").textContent = ""; $("#respStatus").innerHTML = "";646 $("#tabs").innerHTML = "";647 $("#out").innerHTML = '<div class="skel"><div class="ln" style="width:40%"></div><div class="ln" style="width:90%"></div><div class="ln" style="width:80%"></div><div class="ln" style="width:95%"></div><div class="ln" style="width:60%"></div></div>';648 const t0 = performance.now();649 try {650 const res = await fetch(path, endpoint==="status"651 ? { method:"GET" }652 : { method:"POST", headers:{ "content-type":"application/json" }, body: JSON.stringify(body) });653 const raw = await res.text();654 let data; try { data = JSON.parse(raw); } catch { data = raw; }655 const ms = Math.round(performance.now() - t0);656 const ok = res.ok && data?.success !== false;657 $("#respStatus").innerHTML = `<span class="badge ${ok?"ok":"err"}">${res.status}</span> <span class="meta-line">${ms} ms</span>`;658 $("#reqmeta").innerHTML = ok ? `<span class="badge ok">✓ success</span>` : `<span class="badge err">${data?.error?.code||"error"}</span>`;659 const pretty = typeof data === "string" ? data : JSON.stringify(data, null, 2);660 lastResult = { data, raw: pretty, snips, meta: { httpStatus: res.status, ms, ok } };661662 let tabs;663 if (endpoint === "scrape" && ok) {664 tabs = ["preview"];665 if (data?.data?.links) tabs.push("links");666 if (data?.data?.metadata) tabs.push("metadata");667 if (data?.data?.structured) tabs.push("structured");668 tabs.push("json","curl","python","js");669 } else if (endpoint === "map" && ok) {670 tabs = ["preview","json","curl","python","js"];671 } else if (ok) {672 tabs = ["preview","json","curl","python","js"];673 } else {674 tabs = ["json","curl","python","js"];675 }676 activeTab = tabs[0];677 setTabs(tabs); render();678 } catch (e) {679 $("#respStatus").innerHTML = `<span class="badge err">network</span>`;680 $("#out").innerHTML = `<div class="empty2">Request failed: ${esc(e.message)}</div>`;681 } finally {682 btn.disabled = false; btn.innerHTML = "▶ Send request";683 }684}685$("#send").onclick = send;686687// docs688const EPS = [689 { m:"POST", path:"/v1/scrape", desc:"Fetch one page and extract it",690 params:[691 ["url","string","required","The page to fetch"],692 ["formats","string[]","markdown","Any of markdown · html · rawHtml · links · structured"],693 ["tier","enum","auto","auto · http · webkit · safari"],694 ["onlyMainContent","boolean","true","Strip nav/footer/ads, keep the article"],695 ["respectRobots","boolean","true","Honour robots.txt (refused if disallowed)"],696 ["timeout","number","30000","Milliseconds"],697 ],698 ex:{ url:"https://www.iana.org", formats:["markdown","structured"] } },699 { m:"POST", path:"/v1/map", desc:"Discover a site's URLs without rendering",700 params:[701 ["url","string","required","The site root to map"],702 ["search","string","—","Substring filter, ranked by match position"],703 ["limit","number","5000","Max URLs (cap 50000)"],704 ["sitemapOnly","boolean","false","Only sitemaps, skip homepage links"],705 ["ignoreSitemap","boolean","false","Skip sitemaps, links only"],706 ["includeSubdomains","boolean","false","Include subdomains of the apex"],707 ],708 ex:{ url:"https://www.iana.org", limit:20 } },709 { m:"GET", path:"/v1/status", desc:"Per-tier availability + pipeline version", params:[], ex:null },710];711const box = $("#epdocs");712EPS.forEach(ep => {713 const el = document.createElement("div"); el.className = "ep";714 const rows = ep.params.map(p => `<tr><td><code>${p[0]}</code></td><td>${p[1]}</td><td>${p[2]==="required"?'<span class="req-opt">required</span>':p[2]}</td><td>${p[3]}</td></tr>`).join("");715 el.innerHTML = `716 <div class="top"><span class="method ${ep.m.toLowerCase()}">${ep.m}</span><span class="path">${ep.path}</span><span class="desc">${ep.desc}</span></div>717 <div class="body">718 ${ep.params.length ? `<table class="ptable"><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Notes</th></tr></thead><tbody>${rows}</tbody></table>` : '<p class="meta-line">No parameters.</p>'}719 ${ep.ex ? `<div style="margin-top:12px">${""}</div>` : ""}720 </div>`;721 el.querySelector(".top").onclick = () => el.classList.toggle("open");722 if (ep.ex) {723 const pre = code(`curl -s -X POST ${location.origin}${ep.path} \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(ep.ex)}'`, "bash");724 el.querySelector(".body > div:last-child")?.appendChild(pre);725 el.querySelector(".body").appendChild(pre);726 }727 box.appendChild(el);728});729$$(".ep")[0]?.classList.add("open");730731// copy buttons on static snippets + highlight them732$$(".copysnip").forEach(b => {733 b.onclick = () => {734 const pre = b.parentElement.querySelector("pre");735 navigator.clipboard.writeText(pre.innerText);736 b.textContent = "✓ copied"; setTimeout(()=>b.textContent="⧉ copy", 1200);737 };738});739$$(".snip pre code").forEach(c => { try { hljs.highlightElement(c); } catch(e){} });740</script>741</body>742</html>743