| |
1 | +<!doctype html> |
| |
2 | +<!-- author: simon-pierre boucher <contact@spboucher.ai> --> |
| |
3 | +<html lang="en" data-theme="dark"> |
| |
4 | +<head> |
| |
5 | +<meta charset="utf-8" /> |
| |
6 | +<meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
7 | +<title>Tendril — web ingestion that reaches what others can't</title> |
| |
8 | +<meta name="description" content="Tendril is a web ingestion platform (scrape · map) running on residential macOS in WebKit. Deterministic, testable extraction. Try it in the playground." /> |
| |
9 | +<meta property="og:title" content="Tendril" /> |
| |
10 | +<meta property="og:description" content="Web ingestion that reaches what others can't. Scrape and map the web with deterministic, testable extraction." /> |
| |
11 | +<meta property="og:type" content="website" /> |
| |
12 | +<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Ctext y='26' font-size='26'%3E%F0%9F%8C%B1%3C/text%3E%3C/svg%3E" /> |
| |
13 | +<link rel="preconnect" href="https://fonts.googleapis.com" /> |
| |
14 | +<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin /> |
| |
15 | +<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=Inter:wght@400;450;500;600&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet" /> |
| |
16 | +<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/styles/github-dark.min.css" /> |
| |
17 | +<style> |
| |
18 | + :root { |
| |
19 | + --bg: #07100c; |
| |
20 | + --bg-2: #0b1712; |
| |
21 | + --panel: rgba(18, 32, 26, 0.72); |
| |
22 | + --panel-solid: #101b16; |
| |
23 | + --line: rgba(120, 200, 160, 0.16); |
| |
24 | + --line-strong: rgba(120, 200, 160, 0.32); |
| |
25 | + --ink: #e7f3ec; |
| |
26 | + --muted: #8fae9f; |
| |
27 | + --dim: #5f7a6d; |
| |
28 | + --vine: #4ade80; |
| |
29 | + --vine-2: #2dd4bf; |
| |
30 | + --amber: #f5c451; |
| |
31 | + --danger: #ff6b6b; |
| |
32 | + --radius: 16px; |
| |
33 | + --mono: "JetBrains Mono", ui-monospace, monospace; |
| |
34 | + --sans: "Inter", system-ui, sans-serif; |
| |
35 | + --display: "Space Grotesk", var(--sans); |
| |
36 | + --shadow: 0 24px 60px -30px rgba(0, 0, 0, 0.9); |
| |
37 | + } |
| |
38 | + * { box-sizing: border-box; } |
| |
39 | + html { scroll-behavior: smooth; } |
| |
40 | + body { |
| |
41 | + margin: 0; background: var(--bg); color: var(--ink); font-family: var(--sans); |
| |
42 | + font-size: 15px; line-height: 1.6; -webkit-font-smoothing: antialiased; |
| |
43 | + background-image: |
| |
44 | + radial-gradient(1100px 600px at 82% -8%, rgba(45, 212, 191, 0.10), transparent 60%), |
| |
45 | + radial-gradient(900px 520px at 4% 6%, rgba(74, 222, 128, 0.09), transparent 55%); |
| |
46 | + background-attachment: fixed; |
| |
47 | + } |
| |
48 | + a { color: inherit; text-decoration: none; } |
| |
49 | + .wrap { max-width: 1180px; margin: 0 auto; padding: 0 24px; } |
| |
50 | + |
| |
51 | + /* header */ |
| |
52 | + header { |
| |
53 | + position: sticky; top: 0; z-index: 50; backdrop-filter: blur(14px); |
| |
54 | + background: linear-gradient(to bottom, rgba(7,16,12,0.9), rgba(7,16,12,0.55)); |
| |
55 | + border-bottom: 1px solid var(--line); |
| |
56 | + } |
| |
57 | + .nav { display: flex; align-items: center; gap: 20px; height: 62px; } |
| |
58 | + .brand { display: flex; align-items: center; gap: 10px; font-family: var(--display); font-weight: 700; letter-spacing: -0.01em; font-size: 18px; } |
| |
59 | + .brand .leaf { font-size: 20px; filter: drop-shadow(0 0 10px rgba(74,222,128,0.5)); } |
| |
60 | + .brand b { color: var(--vine); } |
| |
61 | + .nav .spacer { flex: 1; } |
| |
62 | + .nav a.link { color: var(--muted); font-weight: 500; font-size: 14px; padding: 8px 10px; border-radius: 9px; } |
| |
63 | + .nav a.link:hover { color: var(--ink); background: rgba(120,200,160,0.08); } |
| |
64 | + .pill { |
| |
65 | + font-family: var(--mono); font-size: 12px; color: var(--vine); |
| |
66 | + border: 1px solid var(--line-strong); border-radius: 999px; padding: 5px 11px; |
| |
67 | + display: inline-flex; align-items: center; gap: 7px; background: rgba(74,222,128,0.06); |
| |
68 | + } |
| |
69 | + .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--vine); box-shadow: 0 0 9px var(--vine); animation: pulse 2s infinite; } |
| |
70 | + @keyframes pulse { 0%,100%{opacity:1} 50%{opacity:.35} } |
| |
71 | + |
| |
72 | + /* hero */ |
| |
73 | + .hero { padding: 74px 0 40px; position: relative; overflow: hidden; } |
| |
74 | + .kicker { font-family: var(--mono); font-size: 12.5px; letter-spacing: 0.18em; text-transform: uppercase; color: var(--vine-2); margin-bottom: 18px; } |
| |
75 | + h1 { font-family: var(--display); font-size: clamp(38px, 6vw, 68px); line-height: 1.02; letter-spacing: -0.03em; margin: 0 0 18px; font-weight: 700; } |
| |
76 | + h1 .grad { background: linear-gradient(120deg, var(--vine), var(--vine-2) 60%, var(--amber)); -webkit-background-clip: text; background-clip: text; color: transparent; } |
| |
77 | + .lede { font-size: 19px; color: var(--muted); max-width: 640px; margin: 0 0 28px; } |
| |
78 | + .cta-row { display: flex; gap: 12px; flex-wrap: wrap; align-items: center; } |
| |
79 | + .btn { |
| |
80 | + font-family: var(--display); font-weight: 600; font-size: 15px; cursor: pointer; |
| |
81 | + border-radius: 12px; padding: 12px 20px; border: 1px solid var(--line-strong); |
| |
82 | + background: var(--panel-solid); color: var(--ink); transition: .15s; display: inline-flex; align-items: center; gap: 9px; |
| |
83 | + } |
| |
84 | + .btn:hover { transform: translateY(-1px); border-color: var(--vine); } |
| |
85 | + .btn.primary { background: linear-gradient(120deg, var(--vine), var(--vine-2)); color: #05130c; border: none; box-shadow: 0 14px 34px -14px rgba(74,222,128,0.6); } |
| |
86 | + .btn.primary:hover { filter: brightness(1.06); } |
| |
87 | + .stats { display: flex; gap: 30px; flex-wrap: wrap; margin-top: 40px; } |
| |
88 | + .stat .n { font-family: var(--display); font-size: 26px; font-weight: 700; color: var(--ink); } |
| |
89 | + .stat .l { font-size: 12.5px; color: var(--dim); font-family: var(--mono); text-transform: uppercase; letter-spacing: 0.08em; } |
| |
90 | + .vine-svg { position: absolute; right: -40px; top: 20px; width: 420px; max-width: 46%; opacity: 0.5; pointer-events: none; } |
| |
91 | + .vine-svg path { fill: none; stroke: url(#vg); stroke-width: 2; stroke-linecap: round; stroke-dasharray: 1400; stroke-dashoffset: 1400; animation: grow 3.2s ease forwards; } |
| |
92 | + @keyframes grow { to { stroke-dashoffset: 0; } } |
| |
93 | + |
| |
94 | + section { padding: 40px 0; } |
| |
95 | + .sec-head { display: flex; align-items: baseline; gap: 14px; margin-bottom: 22px; } |
| |
96 | + .sec-head h2 { font-family: var(--display); font-size: 26px; letter-spacing: -0.02em; margin: 0; } |
| |
97 | + .sec-head .tag { font-family: var(--mono); font-size: 12px; color: var(--dim); } |
| |
98 | + |
| |
99 | + /* playground */ |
| |
100 | + .pg { display: grid; grid-template-columns: 1fr 1.15fr; gap: 20px; } |
| |
101 | + @media (max-width: 900px){ .pg { grid-template-columns: 1fr; } .vine-svg{ display:none } } |
| |
102 | + .card { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); box-shadow: var(--shadow); backdrop-filter: blur(8px); } |
| |
103 | + .card .hd { padding: 15px 18px; border-bottom: 1px solid var(--line); display: flex; align-items: center; gap: 10px; } |
| |
104 | + .card .hd h3 { font-family: var(--display); font-size: 15px; margin: 0; letter-spacing: -0.01em; } |
| |
105 | + .card .bd { padding: 18px; } |
| |
106 | + .seg { display: inline-flex; background: rgba(10,20,15,0.7); border: 1px solid var(--line); border-radius: 11px; padding: 4px; gap: 3px; } |
| |
107 | + .seg button { font-family: var(--mono); font-size: 13px; color: var(--muted); background: none; border: 0; padding: 7px 14px; border-radius: 8px; cursor: pointer; } |
| |
108 | + .seg button.on { background: linear-gradient(120deg, rgba(74,222,128,0.9), rgba(45,212,191,0.9)); color: #05130c; font-weight: 600; } |
| |
109 | + label.f { display: block; font-size: 12.5px; color: var(--muted); font-family: var(--mono); margin: 16px 0 7px; letter-spacing: 0.02em; } |
| |
110 | + input[type=text], input[type=number], select { |
| |
111 | + width: 100%; background: rgba(6,14,10,0.85); border: 1px solid var(--line); color: var(--ink); |
| |
112 | + border-radius: 10px; padding: 11px 13px; font-family: var(--mono); font-size: 13.5px; outline: none; transition: .15s; |
| |
113 | + } |
| |
114 | + input:focus, select:focus { border-color: var(--vine); box-shadow: 0 0 0 3px rgba(74,222,128,0.12); } |
| |
115 | + .chips { display: flex; flex-wrap: wrap; gap: 8px; } |
| |
116 | + .chk { font-family: var(--mono); font-size: 12.5px; display: inline-flex; align-items: center; gap: 7px; padding: 8px 12px; border: 1px solid var(--line); border-radius: 9px; cursor: pointer; color: var(--muted); user-select: none; } |
| |
117 | + .chk input { accent-color: var(--vine); } |
| |
118 | + .chk.on { border-color: var(--vine); color: var(--ink); background: rgba(74,222,128,0.07); } |
| |
119 | + .row2 { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; } |
| |
120 | + .send-row { display: flex; align-items: center; gap: 12px; margin-top: 20px; } |
| |
121 | + .meta-line { font-family: var(--mono); font-size: 12px; color: var(--dim); } |
| |
122 | + .badge { font-family: var(--mono); font-size: 11px; padding: 3px 8px; border-radius: 7px; border: 1px solid var(--line); color: var(--muted); } |
| |
123 | + .badge.ok { color: var(--vine); border-color: var(--line-strong); } |
| |
124 | + .badge.err { color: var(--danger); border-color: rgba(255,107,107,0.4); } |
| |
125 | + |
| |
126 | + .tabs { display: flex; gap: 4px; flex-wrap: wrap; } |
| |
127 | + .tabs button { font-family: var(--mono); font-size: 12.5px; color: var(--muted); background: none; border: 1px solid transparent; padding: 6px 11px; border-radius: 8px; cursor: pointer; } |
| |
128 | + .tabs button.on { color: var(--ink); border-color: var(--line); background: rgba(10,20,15,0.6); } |
| |
129 | + .out { min-height: 260px; max-height: 560px; overflow: auto; } |
| |
130 | + .out pre { margin: 0; } |
| |
131 | + pre code.hljs { background: transparent !important; font-family: var(--mono); font-size: 12.8px; line-height: 1.55; padding: 16px 18px; display: block; } |
| |
132 | + .rendered { padding: 18px; font-size: 14.5px; } |
| |
133 | + .rendered h1,.rendered h2,.rendered h3 { font-family: var(--display); line-height: 1.2; } |
| |
134 | + .rendered a { color: var(--vine-2); border-bottom: 1px solid rgba(45,212,191,0.3); } |
| |
135 | + .rendered pre { background: rgba(6,14,10,0.9); border: 1px solid var(--line); border-radius: 10px; padding: 12px; overflow: auto; } |
| |
136 | + .rendered table { border-collapse: collapse; width: 100%; font-size: 13px; } |
| |
137 | + .rendered th, .rendered td { border: 1px solid var(--line); padding: 6px 10px; text-align: left; } |
| |
138 | + .rendered img { max-width: 100%; border-radius: 8px; } |
| |
139 | + .placeholder { color: var(--dim); font-family: var(--mono); font-size: 13px; padding: 40px 18px; text-align: center; } |
| |
140 | + .linkrow { display: flex; align-items: center; gap: 10px; padding: 9px 14px; border-bottom: 1px solid var(--line); font-family: var(--mono); font-size: 12.5px; } |
| |
141 | + .linkrow a { color: var(--ink); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex: 1; } |
| |
142 | + .src { font-size: 10.5px; text-transform: uppercase; letter-spacing: 0.06em; padding: 2px 7px; border-radius: 6px; border: 1px solid var(--line); color: var(--vine-2); } |
| |
143 | + .copybtn { font-family: var(--mono); font-size: 11.5px; cursor: pointer; background: rgba(10,20,15,0.6); border: 1px solid var(--line); color: var(--muted); padding: 4px 9px; border-radius: 7px; } |
| |
144 | + .copybtn:hover { color: var(--vine); border-color: var(--vine); } |
| |
145 | + .spin { width: 15px; height: 15px; border: 2px solid rgba(255,255,255,0.2); border-top-color: var(--vine); border-radius: 50%; animation: sp .7s linear infinite; } |
| |
146 | + @keyframes sp { to { transform: rotate(360deg); } } |
| |
147 | + |
| |
148 | + /* docs */ |
| |
149 | + .ep { border: 1px solid var(--line); border-radius: var(--radius); margin-bottom: 16px; overflow: hidden; background: var(--panel); } |
| |
150 | + .ep .top { display: flex; align-items: center; gap: 12px; padding: 15px 18px; cursor: pointer; } |
| |
151 | + .method { font-family: var(--mono); font-weight: 600; font-size: 12px; padding: 4px 9px; border-radius: 7px; background: rgba(74,222,128,0.14); color: var(--vine); } |
| |
152 | + .method.get { background: rgba(45,212,191,0.14); color: var(--vine-2); } |
| |
153 | + .path { font-family: var(--mono); font-size: 14px; } |
| |
154 | + .ep .desc { color: var(--muted); font-size: 13px; margin-left: auto; } |
| |
155 | + .ep .body { padding: 0 18px 18px; display: none; } |
| |
156 | + .ep.open .body { display: block; } |
| |
157 | + .ptable { width: 100%; border-collapse: collapse; font-size: 13px; margin: 8px 0 4px; } |
| |
158 | + .ptable th { text-align: left; color: var(--dim); font-family: var(--mono); font-weight: 500; font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.06em; padding: 6px 8px; border-bottom: 1px solid var(--line); } |
| |
159 | + .ptable td { padding: 7px 8px; border-bottom: 1px solid var(--line); vertical-align: top; } |
| |
160 | + .ptable code { font-family: var(--mono); color: var(--vine-2); font-size: 12.5px; } |
| |
161 | + .req-opt { font-size: 10.5px; color: var(--amber); font-family: var(--mono); } |
| |
162 | + .errtbl { width: 100%; border-collapse: collapse; font-size: 12.5px; } |
| |
163 | + .errtbl td, .errtbl th { padding: 6px 10px; border-bottom: 1px solid var(--line); text-align: left; } |
| |
164 | + .errtbl code { color: var(--danger); font-family: var(--mono); } |
| |
165 | + footer { border-top: 1px solid var(--line); padding: 34px 0 60px; color: var(--dim); font-size: 13px; margin-top: 30px; } |
| |
166 | + footer a { color: var(--muted); } footer a:hover { color: var(--vine); } |
| |
167 | + .grid3 { display: grid; grid-template-columns: repeat(3,1fr); gap: 16px; } |
| |
168 | + @media (max-width:760px){ .grid3{ grid-template-columns:1fr } .row2{ grid-template-columns:1fr } } |
| |
169 | + .feat { padding: 18px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--panel); } |
| |
170 | + .feat .i { font-size: 22px; margin-bottom: 10px; } |
| |
171 | + .feat h4 { font-family: var(--display); margin: 0 0 6px; font-size: 16px; } |
| |
172 | + .feat p { margin: 0; color: var(--muted); font-size: 13.5px; } |
| |
173 | + .note { font-size: 12.5px; color: var(--amber); font-family: var(--mono); background: rgba(245,196,81,0.07); border: 1px solid rgba(245,196,81,0.25); border-radius: 10px; padding: 10px 13px; margin-top: 14px; } |
| |
174 | +</style> |
| |
175 | +</head> |
| |
176 | +<body> |
| |
177 | +<header> |
| |
178 | + <div class="wrap nav"> |
| |
179 | + <a class="brand" href="#top"><span class="leaf">🌱</span> ten<b>·</b>dril</a> |
| |
180 | + <span class="spacer"></span> |
| |
181 | + <a class="link" href="#playground">Playground</a> |
| |
182 | + <a class="link" href="#docs">Docs</a> |
| |
183 | + <a class="link" href="https://git.spboucher.ai/tendril" target="_blank" rel="noopener">Source</a> |
| |
184 | + <span class="pill" id="statusPill"><span class="dot"></span> checking…</span> |
| |
185 | + </div> |
| |
186 | +</header> |
| |
187 | + |
| |
188 | +<a id="top"></a> |
| |
189 | +<div class="hero"> |
| |
190 | + <div class="wrap"> |
| |
191 | + <svg class="vine-svg" viewBox="0 0 300 300" aria-hidden="true"> |
| |
192 | + <defs><linearGradient id="vg" x1="0" y1="0" x2="1" y2="1"> |
| |
193 | + <stop offset="0" stop-color="#4ade80"/><stop offset="0.6" stop-color="#2dd4bf"/><stop offset="1" stop-color="#f5c451"/> |
| |
194 | + </linearGradient></defs> |
| |
195 | + <path d="M150 300 C150 240 120 220 150 180 C180 140 120 120 150 80 C170 50 150 30 150 0" /> |
| |
196 | + <path d="M150 220 C120 210 100 190 85 200 C70 210 80 230 100 225" /> |
| |
197 | + <path d="M150 150 C185 140 205 120 220 132 C235 144 224 165 202 158" /> |
| |
198 | + <path d="M150 95 C120 86 104 66 90 78 C78 88 86 108 106 102" /> |
| |
199 | + </svg> |
| |
200 | + <div class="kicker">Web ingestion · scrape · map</div> |
| |
201 | + <h1>Reach what other<br />crawlers <span class="grad">can't.</span></h1> |
| |
202 | + <p class="lede">Tendril runs on a residential Mac, in WebKit — Safari's real engine — behind a real IP. Extraction is <strong>deterministic and testable</strong>: the same page always yields the same markdown. No cloud fingerprint, no LLM lottery.</p> |
| |
203 | + <div class="cta-row"> |
| |
204 | + <a class="btn primary" href="#playground">▶ Open the playground</a> |
| |
205 | + <a class="btn" href="#docs">Read the docs</a> |
| |
206 | + </div> |
| |
207 | + <div class="stats"> |
| |
208 | + <div class="stat"><div class="n">3</div><div class="l">escalation tiers</div></div> |
| |
209 | + <div class="stat"><div class="n">0</div><div class="l">LLMs in extraction</div></div> |
| |
210 | + <div class="stat"><div class="n">~50ms</div><div class="l">Tier 0 p50</div></div> |
| |
211 | + <div class="stat"><div class="n">WebKit</div><div class="l">real Safari engine</div></div> |
| |
212 | + </div> |
| |
213 | + </div> |
| |
214 | +</div> |
| |
215 | + |
| |
216 | +<section class="wrap"> |
| |
217 | + <div class="grid3"> |
| |
218 | + <div class="feat"><div class="i">🕸️</div><h4>Browser fidelity</h4><p>WebKit on macOS ARM behind a residential IP — not a headless Chromium signature. Sites that block others often don't block Tendril.</p></div> |
| |
219 | + <div class="feat"><div class="i">📐</div><h4>Deterministic extraction</h4><p>JSON-LD, OpenGraph, Readability + density fallback, hand-written Turndown rules. Versioned, golden-file testable output.</p></div> |
| |
220 | + <div class="feat"><div class="i">🧭</div><h4>Map without rendering</h4><p>Merge sitemaps, robots directives, homepage links and <code>/llms.txt</code> into one deduplicated URL list in seconds.</p></div> |
| |
221 | + </div> |
| |
222 | +</section> |
| |
223 | + |
| |
224 | +<section class="wrap" id="playground"> |
| |
225 | + <div class="sec-head"><h2>Playground</h2><span class="tag">// live requests against this deployment</span></div> |
| |
226 | + <div class="pg"> |
| |
227 | + <div class="card"> |
| |
228 | + <div class="hd"><h3>Request</h3><span class="spacer" style="flex:1"></span> |
| |
229 | + <div class="seg" id="epseg"> |
| |
230 | + <button data-ep="scrape" class="on">scrape</button> |
| |
231 | + <button data-ep="map">map</button> |
| |
232 | + <button data-ep="status">status</button> |
| |
233 | + </div> |
| |
234 | + </div> |
| |
235 | + <div class="bd"> |
| |
236 | + <div id="urlblock"> |
| |
237 | + <label class="f">Target URL</label> |
| |
238 | + <input type="text" id="url" value="https://www.iana.org" spellcheck="false" placeholder="https://example.com" /> |
| |
239 | + </div> |
| |
240 | + |
| |
241 | + <div id="scrapeOpts"> |
| |
242 | + <label class="f">Formats</label> |
| |
243 | + <div class="chips" id="formats"> |
| |
244 | + <label class="chk on"><input type="checkbox" value="markdown" checked>markdown</label> |
| |
245 | + <label class="chk"><input type="checkbox" value="links">links</label> |
| |
246 | + <label class="chk"><input type="checkbox" value="structured">structured</label> |
| |
247 | + <label class="chk"><input type="checkbox" value="html">html</label> |
| |
248 | + </div> |
| |
249 | + <div class="row2"> |
| |
250 | + <div><label class="f">Tier</label> |
| |
251 | + <select id="tier"><option value="auto">auto</option><option value="http">http (Tier 0)</option><option value="webkit">webkit</option><option value="safari">safari</option></select> |
| |
252 | + </div> |
| |
253 | + <div><label class="f">Options</label> |
| |
254 | + <div class="chips"> |
| |
255 | + <label class="chk on"><input type="checkbox" id="onlyMain" checked>onlyMainContent</label> |
| |
256 | + <label class="chk on"><input type="checkbox" id="respRobots" checked>respectRobots</label> |
| |
257 | + </div> |
| |
258 | + </div> |
| |
259 | + </div> |
| |
260 | + </div> |
| |
261 | + |
| |
262 | + <div id="mapOpts" style="display:none"> |
| |
263 | + <div class="row2"> |
| |
264 | + <div><label class="f">Search filter</label><input type="text" id="mapSearch" placeholder="optional substring" spellcheck="false"></div> |
| |
265 | + <div><label class="f">Limit</label><input type="number" id="mapLimit" value="30" min="1" max="50000"></div> |
| |
266 | + </div> |
| |
267 | + <label class="f">Options</label> |
| |
268 | + <div class="chips"> |
| |
269 | + <label class="chk"><input type="checkbox" id="sitemapOnly">sitemapOnly</label> |
| |
270 | + <label class="chk"><input type="checkbox" id="ignoreSitemap">ignoreSitemap</label> |
| |
271 | + <label class="chk"><input type="checkbox" id="subdomains">includeSubdomains</label> |
| |
272 | + </div> |
| |
273 | + </div> |
| |
274 | + |
| |
275 | + <div id="statusOpts" style="display:none"> |
| |
276 | + <p class="meta-line">No parameters. Returns per-tier availability and pipeline version.</p> |
| |
277 | + </div> |
| |
278 | + |
| |
279 | + <div class="send-row"> |
| |
280 | + <button class="btn primary" id="send">▶ Send request</button> |
| |
281 | + <span id="reqmeta" class="meta-line"></span> |
| |
282 | + </div> |
| |
283 | + <div class="note" id="autoNote" style="display:none">Tiers 1 & 2 (WebKit/Safari) aren't deployed yet — a page needing escalation under <b>tier: auto</b> returns <code>ERR_TARGET_BLOCKED</code>. Force <b>tier: http</b> for raw Tier 0 output.</div> |
| |
284 | + </div> |
| |
285 | + </div> |
| |
286 | + |
| |
287 | + <div class="card"> |
| |
288 | + <div class="hd"><h3>Response</h3><span class="spacer" style="flex:1"></span><span id="respStatus"></span></div> |
| |
289 | + <div class="bd" style="padding-top:14px"> |
| |
290 | + <div class="tabs" id="tabs"></div> |
| |
291 | + <div class="out" id="out"><div class="placeholder">Send a request to see the response, rendered markdown, extracted links and ready-to-copy snippets.</div></div> |
| |
292 | + </div> |
| |
293 | + </div> |
| |
294 | + </div> |
| |
295 | +</section> |
| |
296 | + |
| |
297 | +<section class="wrap" id="docs"> |
| |
298 | + <div class="sec-head"><h2>API reference</h2><span class="tag">// base: this origin · Bearer auth coming with accounts</span></div> |
| |
299 | + <div id="epdocs"></div> |
| |
300 | + |
| |
301 | + <div class="sec-head" style="margin-top:30px"><h2>Errors</h2><span class="tag">// §15 taxonomy — one code per failure</span></div> |
| |
302 | + <div class="card"><div class="bd"> |
| |
303 | + <table class="errtbl"><tbody> |
| |
304 | + <tr><td><code>ERR_INVALID_URL</code></td><td>400</td><td>Malformed or unsupported scheme</td></tr> |
| |
305 | + <tr><td><code>ERR_SSRF_BLOCKED</code></td><td>400</td><td>Resolved to a private/forbidden address</td></tr> |
| |
306 | + <tr><td><code>ERR_ROBOTS_DENIED</code></td><td>403</td><td>robots.txt disallows the path</td></tr> |
| |
307 | + <tr><td><code>ERR_TARGET_BLOCKED</code></td><td>502</td><td>Tier 0 insufficient; higher tiers unavailable</td></tr> |
| |
308 | + <tr><td><code>ERR_UNSUPPORTED_TYPE</code></td><td>415</td><td>Content type not handled</td></tr> |
| |
309 | + <tr><td><code>ERR_TIER_TIMEOUT</code></td><td>504</td><td>Render exceeded timeout</td></tr> |
| |
310 | + </tbody></table> |
| |
311 | + <p class="meta-line" style="margin-top:12px">Every response carries a <code>requestId</code> and an <code>X-Request-Id</code> header for support.</p> |
| |
312 | + </div></div> |
| |
313 | +</section> |
| |
314 | + |
| |
315 | +<footer> |
| |
316 | + <div class="wrap"> |
| |
317 | + <div style="display:flex;gap:20px;flex-wrap:wrap;align-items:center"> |
| |
318 | + <span class="brand"><span class="leaf">🌱</span> ten·dril</span> |
| |
319 | + <span class="spacer" style="flex:1"></span> |
| |
320 | + <a href="https://git.spboucher.ai/tendril" target="_blank" rel="noopener">Source</a> |
| |
321 | + <a href="/healthz">/healthz</a> |
| |
322 | + <a href="mailto:contact@spboucher.ai">contact@spboucher.ai</a> |
| |
323 | + </div> |
| |
324 | + <p style="margin-top:16px;max-width:640px">Tendril identifies itself with a bot User-Agent and a contact URL, honours <code>robots.txt</code> by default, and rate-limits per host. Stealth here means matching a real browser's fingerprint — not hiding who we are. Built by Simon-Pierre Boucher.</p> |
| |
325 | + </div> |
| |
326 | +</footer> |
| |
327 | + |
| |
328 | +<script src="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/highlight.min.js"></script> |
| |
329 | +<script src="https://cdn.jsdelivr.net/npm/marked@12.0.0/marked.min.js"></script> |
| |
330 | +<script> |
| |
331 | +const $ = (s, r=document) => r.querySelector(s); |
| |
332 | +const $$ = (s, r=document) => [...r.querySelectorAll(s)]; |
| |
333 | +let endpoint = "scrape"; |
| |
334 | +let lastResult = null; |
| |
335 | +let activeTab = "preview"; |
| |
336 | + |
| |
337 | +// status pill |
| |
338 | +fetch("/v1/status").then(r=>r.json()).then(j=>{ |
| |
339 | + const p = $("#statusPill"); |
| |
340 | + const http = j?.data?.tiers?.http?.available; |
| |
341 | + p.innerHTML = `<span class="dot"></span> ${http ? "live" : "degraded"} · v${j?.data?.pipelineVersion||"?"}`; |
| |
342 | +}).catch(()=>{ $("#statusPill").innerHTML = '<span class="dot" style="background:#ff6b6b;box-shadow:0 0 9px #ff6b6b"></span> offline'; }); |
| |
343 | + |
| |
344 | +// endpoint switch |
| |
345 | +$$("#epseg button").forEach(b => b.onclick = () => { |
| |
346 | + $$("#epseg button").forEach(x=>x.classList.remove("on")); b.classList.add("on"); |
| |
347 | + endpoint = b.dataset.ep; |
| |
348 | + $("#scrapeOpts").style.display = endpoint==="scrape" ? "" : "none"; |
| |
349 | + $("#mapOpts").style.display = endpoint==="map" ? "" : "none"; |
| |
350 | + $("#statusOpts").style.display = endpoint==="status" ? "" : "none"; |
| |
351 | + $("#urlblock").style.display = endpoint==="status" ? "none" : ""; |
| |
352 | +}); |
| |
353 | + |
| |
354 | +// chip toggles |
| |
355 | +$$(".chk").forEach(c => { const i = c.querySelector("input"); if(i) i.onchange = () => c.classList.toggle("on", i.checked); }); |
| |
356 | +$("#tier").onchange = e => $("#autoNote").style.display = e.target.value==="auto" ? "" : "none"; |
| |
357 | + |
| |
358 | +function buildBody() { |
| |
359 | + if (endpoint === "status") return null; |
| |
360 | + const url = $("#url").value.trim(); |
| |
361 | + if (endpoint === "scrape") { |
| |
362 | + const formats = $$("#formats input:checked").map(i=>i.value); |
| |
363 | + return { url, formats, tier: $("#tier").value, onlyMainContent: $("#onlyMain").checked, respectRobots: $("#respRobots").checked }; |
| |
364 | + } |
| |
365 | + const b = { url, limit: Number($("#mapLimit").value)||30, sitemapOnly: $("#sitemapOnly").checked, ignoreSitemap: $("#ignoreSitemap").checked, includeSubdomains: $("#subdomains").checked }; |
| |
366 | + const s = $("#mapSearch").value.trim(); if (s) b.search = s; |
| |
367 | + return b; |
| |
368 | +} |
| |
369 | + |
| |
370 | +function snippets(path, body) { |
| |
371 | + const origin = location.origin; |
| |
372 | + if (!body) return { |
| |
373 | + curl: `curl -s ${origin}${path}`, |
| |
374 | + python: `import requests\nprint(requests.get("${origin}${path}").json())`, |
| |
375 | + js: `const r = await fetch("${origin}${path}");\nconsole.log(await r.json());` |
| |
376 | + }; |
| |
377 | + const j = JSON.stringify(body, null, 2); |
| |
378 | + return { |
| |
379 | + curl: `curl -s -X POST ${origin}${path} \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(body)}'`, |
| |
380 | + python: `import requests\nr = requests.post("${origin}${path}", json=${j.replace(/true/g,"True").replace(/false/g,"False")})\nprint(r.json())`, |
| |
381 | + js: `const r = await fetch("${origin}${path}", {\n method: "POST",\n headers: { "content-type": "application/json" },\n body: JSON.stringify(${j})\n});\nconsole.log(await r.json());` |
| |
382 | + }; |
| |
383 | +} |
| |
384 | + |
| |
385 | +function setTabs(tabs) { |
| |
386 | + const t = $("#tabs"); t.innerHTML = ""; |
| |
387 | + tabs.forEach(name => { |
| |
388 | + const b = document.createElement("button"); |
| |
389 | + b.textContent = name; b.className = name===activeTab ? "on" : ""; |
| |
390 | + b.onclick = () => { activeTab = name; render(); }; |
| |
391 | + t.appendChild(b); |
| |
392 | + }); |
| |
393 | +} |
| |
394 | + |
| |
395 | +function code(text, lang) { |
| |
396 | + const pre = document.createElement("pre"); |
| |
397 | + const c = document.createElement("code"); |
| |
398 | + c.className = "hljs" + (lang ? " language-"+lang : ""); |
| |
399 | + c.textContent = text; |
| |
400 | + pre.appendChild(c); |
| |
401 | + try { hljs.highlightElement(c); } catch(e){} |
| |
402 | + return pre; |
| |
403 | +} |
| |
404 | +function copyBar(text) { |
| |
405 | + const bar = document.createElement("div"); |
| |
406 | + bar.style.cssText = "display:flex;justify-content:flex-end;padding:8px 12px 0"; |
| |
407 | + const b = document.createElement("button"); b.className="copybtn"; b.textContent="⧉ copy"; |
| |
408 | + b.onclick = () => { navigator.clipboard.writeText(text); b.textContent="✓ copied"; setTimeout(()=>b.textContent="⧉ copy",1200); }; |
| |
409 | + bar.appendChild(b); return bar; |
| |
410 | +} |
| |
411 | + |
| |
412 | +function render() { |
| |
413 | + const out = $("#out"); out.innerHTML = ""; |
| |
414 | + if (!lastResult) return; |
| |
415 | + const { data, raw, snips } = lastResult; |
| |
416 | + |
| |
417 | + if (activeTab === "preview") { |
| |
418 | + if (endpoint === "map") { |
| |
419 | + const links = data?.data?.links || []; |
| |
420 | + if (!links.length) { out.innerHTML = '<div class="placeholder">No links found.</div>'; return; } |
| |
421 | + const head = document.createElement("div"); head.style.cssText="padding:10px 14px;color:var(--dim);font-family:var(--mono);font-size:12px"; |
| |
422 | + head.textContent = `${data.data.count} URLs discovered`; |
| |
423 | + out.appendChild(head); |
| |
424 | + links.forEach(l => { |
| |
425 | + const r = document.createElement("div"); r.className="linkrow"; |
| |
426 | + r.innerHTML = `<span class="src">${l.source}</span><a href="${l.url}" target="_blank" rel="noopener">${l.url}</a>`; |
| |
427 | + out.appendChild(r); |
| |
428 | + }); |
| |
429 | + } else if (endpoint === "status") { |
| |
430 | + out.appendChild(code(JSON.stringify(data, null, 2), "json")); |
| |
431 | + } else { |
| |
432 | + const d = data?.data; |
| |
433 | + if (d?.markdown) { |
| |
434 | + const div = document.createElement("div"); div.className="rendered"; |
| |
435 | + div.innerHTML = marked.parse(d.markdown); |
| |
436 | + out.appendChild(div); |
| |
437 | + } else if (d?.html) { |
| |
438 | + out.appendChild(code(d.html.slice(0, 20000), "xml")); |
| |
439 | + } else { |
| |
440 | + out.appendChild(code(JSON.stringify(data, null, 2), "json")); |
| |
441 | + } |
| |
442 | + } |
| |
443 | + } else if (activeTab === "links") { |
| |
444 | + const links = lastResult.data?.data?.links || []; |
| |
445 | + if (!links.length) { out.innerHTML = '<div class="placeholder">No links in this response.</div>'; return; } |
| |
446 | + links.forEach(l => { |
| |
447 | + const r = document.createElement("div"); r.className="linkrow"; |
| |
448 | + const badge = l.isInternal!==undefined ? `<span class="src">${l.isInternal?"internal":"external"}</span>` : ""; |
| |
449 | + r.innerHTML = `${badge}<a href="${l.url}" target="_blank" rel="noopener">${l.url}</a>`; |
| |
450 | + out.appendChild(r); |
| |
451 | + }); |
| |
452 | + } else if (activeTab === "metadata") { |
| |
453 | + out.appendChild(code(JSON.stringify(lastResult.data?.data?.metadata || {}, null, 2), "json")); |
| |
454 | + } else if (activeTab === "structured") { |
| |
455 | + out.appendChild(code(JSON.stringify(lastResult.data?.data?.structured || {}, null, 2), "json")); |
| |
456 | + } else if (activeTab === "json") { |
| |
457 | + out.appendChild(code(raw, "json")); |
| |
458 | + } else if (["curl","python","js"].includes(activeTab)) { |
| |
459 | + out.appendChild(copyBar(snips[activeTab])); |
| |
460 | + out.appendChild(code(snips[activeTab], activeTab==="curl"?"bash":activeTab==="python"?"python":"javascript")); |
| |
461 | + } |
| |
462 | +} |
| |
463 | + |
| |
464 | +async function send() { |
| |
465 | + const path = "/v1/" + endpoint; |
| |
466 | + const body = buildBody(); |
| |
467 | + const snips = snippets(path, body); |
| |
468 | + const btn = $("#send"); btn.disabled = true; btn.innerHTML = '<span class="spin"></span> running'; |
| |
469 | + $("#reqmeta").textContent = ""; |
| |
470 | + const t0 = performance.now(); |
| |
471 | + try { |
| |
472 | + const res = await fetch(path, endpoint==="status" |
| |
473 | + ? { method:"GET" } |
| |
474 | + : { method:"POST", headers:{ "content-type":"application/json" }, body: JSON.stringify(body) }); |
| |
475 | + const raw = await res.text(); |
| |
476 | + let data; try { data = JSON.parse(raw); } catch { data = raw; } |
| |
477 | + const ms = Math.round(performance.now() - t0); |
| |
478 | + const ok = res.ok && data?.success !== false; |
| |
479 | + $("#respStatus").innerHTML = `<span class="badge ${ok?"ok":"err"}">${res.status}</span> <span class="meta-line">${ms} ms</span>`; |
| |
480 | + $("#reqmeta").innerHTML = ok ? `<span class="badge ok">success</span>` : `<span class="badge err">${data?.error?.code||"error"}</span>`; |
| |
481 | + const pretty = typeof data === "string" ? data : JSON.stringify(data, null, 2); |
| |
482 | + lastResult = { data, raw: pretty, snips }; |
| |
483 | + |
| |
484 | + let tabs; |
| |
485 | + if (endpoint === "scrape") { |
| |
486 | + tabs = ["preview"]; |
| |
487 | + if (data?.data?.links) tabs.push("links"); |
| |
488 | + if (data?.data?.metadata) tabs.push("metadata"); |
| |
489 | + if (data?.data?.structured) tabs.push("structured"); |
| |
490 | + tabs.push("json","curl","python","js"); |
| |
491 | + } else if (endpoint === "map") { |
| |
492 | + tabs = ["preview","json","curl","python","js"]; |
| |
493 | + } else { |
| |
494 | + tabs = ["preview","json","curl","python","js"]; |
| |
495 | + } |
| |
496 | + activeTab = "preview"; |
| |
497 | + setTabs(tabs); render(); |
| |
498 | + } catch (e) { |
| |
499 | + $("#respStatus").innerHTML = `<span class="badge err">network</span>`; |
| |
500 | + $("#out").innerHTML = `<div class="placeholder">Request failed: ${e.message}</div>`; |
| |
501 | + } finally { |
| |
502 | + btn.disabled = false; btn.innerHTML = "▶ Send request"; |
| |
503 | + } |
| |
504 | +} |
| |
505 | +$("#send").onclick = send; |
| |
506 | + |
| |
507 | +// docs |
| |
508 | +const EPS = [ |
| |
509 | + { m:"POST", path:"/v1/scrape", desc:"Fetch one page and extract it", |
| |
510 | + params:[ |
| |
511 | + ["url","string","required","The page to fetch"], |
| |
512 | + ["formats","string[]","markdown","Any of markdown · html · rawHtml · links · structured"], |
| |
513 | + ["tier","enum","auto","auto · http · webkit · safari"], |
| |
514 | + ["onlyMainContent","boolean","true","Strip nav/footer/ads, keep the article"], |
| |
515 | + ["respectRobots","boolean","true","Honour robots.txt (refused if disallowed)"], |
| |
516 | + ["timeout","number","30000","Milliseconds"], |
| |
517 | + ], |
| |
518 | + ex:{ url:"https://www.iana.org", formats:["markdown","structured"] } }, |
| |
519 | + { m:"POST", path:"/v1/map", desc:"Discover a site's URLs without rendering", |
| |
520 | + params:[ |
| |
521 | + ["url","string","required","The site root to map"], |
| |
522 | + ["search","string","—","Substring filter, ranked by match position"], |
| |
523 | + ["limit","number","5000","Max URLs (cap 50000)"], |
| |
524 | + ["sitemapOnly","boolean","false","Only sitemaps, skip homepage links"], |
| |
525 | + ["ignoreSitemap","boolean","false","Skip sitemaps, links only"], |
| |
526 | + ["includeSubdomains","boolean","false","Include subdomains of the apex"], |
| |
527 | + ], |
| |
528 | + ex:{ url:"https://www.iana.org", limit:20 } }, |
| |
529 | + { m:"GET", path:"/v1/status", desc:"Per-tier availability + pipeline version", params:[], ex:null }, |
| |
530 | +]; |
| |
531 | +const box = $("#epdocs"); |
| |
532 | +EPS.forEach(ep => { |
| |
533 | + const el = document.createElement("div"); el.className = "ep"; |
| |
534 | + const rows = ep.params.map(p => `<tr><td><code>${p[0]}</code></td><td>${p[1]}</td><td>${p[2]==="required"?'<span class="req-opt">required</span>':p[2]}</td><td>${p[3]}</td></tr>`).join(""); |
| |
535 | + el.innerHTML = ` |
| |
536 | + <div class="top"><span class="method ${ep.m.toLowerCase()}">${ep.m}</span><span class="path">${ep.path}</span><span class="desc">${ep.desc}</span></div> |
| |
537 | + <div class="body"> |
| |
538 | + ${ep.params.length ? `<table class="ptable"><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Notes</th></tr></thead><tbody>${rows}</tbody></table>` : '<p class="meta-line">No parameters.</p>'} |
| |
539 | + ${ep.ex ? `<div style="margin-top:12px">${""}</div>` : ""} |
| |
540 | + </div>`; |
| |
541 | + el.querySelector(".top").onclick = () => el.classList.toggle("open"); |
| |
542 | + if (ep.ex) { |
| |
543 | + const pre = code(`curl -s -X POST ${location.origin}${ep.path} \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(ep.ex)}'`, "bash"); |
| |
544 | + el.querySelector(".body > div:last-child")?.appendChild(pre); |
| |
545 | + el.querySelector(".body").appendChild(pre); |
| |
546 | + } |
| |
547 | + box.appendChild(el); |
| |
548 | +}); |
| |
549 | +$$(".ep")[0]?.classList.add("open"); |
| |
550 | +</script> |
| |
551 | +</body> |
| |
552 | +</html> |