|
1 |
+<!doctype html> |
|
2 |
+<!-- author: simon-pierre boucher <contact@spboucher.ai> --> |
|
3 |
+<html lang="en" data-theme="dark"> |
|
4 |
+<head> |
|
5 |
+<meta charset="utf-8" /> |
|
6 |
+<meta name="viewport" content="width=device-width, initial-scale=1" /> |
|
7 |
+<title>Tendril — web ingestion that reaches what others can't</title> |
|
8 |
+<meta name="description" content="Tendril is a web ingestion platform (scrape · map) running on residential macOS in WebKit. Deterministic, testable extraction. Try it in the playground." /> |
|
9 |
+<meta property="og:title" content="Tendril" /> |
|
10 |
+<meta property="og:description" content="Web ingestion that reaches what others can't. Scrape and map the web with deterministic, testable extraction." /> |
|
11 |
+<meta property="og:type" content="website" /> |
|
12 |
+<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Ctext y='26' font-size='26'%3E%F0%9F%8C%B1%3C/text%3E%3C/svg%3E" /> |
|
13 |
+<link rel="preconnect" href="https://fonts.googleapis.com" /> |
|
14 |
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin /> |
|
15 |
+<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=Inter:wght@400;450;500;600&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet" /> |
|
16 |
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/styles/github-dark.min.css" /> |
|
17 |
+<style> |
|
18 |
+ :root { |
|
19 |
+ --bg: #07100c; |
|
20 |
+ --bg-2: #0b1712; |
|
21 |
+ --panel: rgba(18, 32, 26, 0.72); |
|
22 |
+ --panel-solid: #101b16; |
|
23 |
+ --line: rgba(120, 200, 160, 0.16); |
|
24 |
+ --line-strong: rgba(120, 200, 160, 0.32); |
|
25 |
+ --ink: #e7f3ec; |
|
26 |
+ --muted: #8fae9f; |
|
27 |
+ --dim: #5f7a6d; |
|
28 |
+ --vine: #4ade80; |
|
29 |
+ --vine-2: #2dd4bf; |
|
30 |
+ --amber: #f5c451; |
|
31 |
+ --danger: #ff6b6b; |
|
32 |
+ --radius: 16px; |
|
33 |
+ --mono: "JetBrains Mono", ui-monospace, monospace; |
|
34 |
+ --sans: "Inter", system-ui, sans-serif; |
|
35 |
+ --display: "Space Grotesk", var(--sans); |
|
36 |
+ --shadow: 0 24px 60px -30px rgba(0, 0, 0, 0.9); |
|
37 |
+ } |
|
38 |
+ * { box-sizing: border-box; } |
|
39 |
+ html { scroll-behavior: smooth; } |
|
40 |
+ body { |
|
41 |
+ margin: 0; background: var(--bg); color: var(--ink); font-family: var(--sans); |
|
42 |
+ font-size: 15px; line-height: 1.6; -webkit-font-smoothing: antialiased; |
|
43 |
+ background-image: |
|
44 |
+ radial-gradient(1100px 600px at 82% -8%, rgba(45, 212, 191, 0.10), transparent 60%), |
|
45 |
+ radial-gradient(900px 520px at 4% 6%, rgba(74, 222, 128, 0.09), transparent 55%); |
|
46 |
+ background-attachment: fixed; |
|
47 |
+ } |
|
48 |
+ a { color: inherit; text-decoration: none; } |
|
49 |
+ .wrap { max-width: 1180px; margin: 0 auto; padding: 0 24px; } |
|
50 |
+ |
|
51 |
+ /* header */ |
|
52 |
+ header { |
|
53 |
+ position: sticky; top: 0; z-index: 50; backdrop-filter: blur(14px); |
|
54 |
+ background: linear-gradient(to bottom, rgba(7,16,12,0.9), rgba(7,16,12,0.55)); |
|
55 |
+ border-bottom: 1px solid var(--line); |
|
56 |
+ } |
|
57 |
+ .nav { display: flex; align-items: center; gap: 20px; height: 62px; } |
|
58 |
+ .brand { display: flex; align-items: center; gap: 10px; font-family: var(--display); font-weight: 700; letter-spacing: -0.01em; font-size: 18px; } |
|
59 |
+ .brand .leaf { font-size: 20px; filter: drop-shadow(0 0 10px rgba(74,222,128,0.5)); } |
|
60 |
+ .brand b { color: var(--vine); } |
|
61 |
+ .nav .spacer { flex: 1; } |
|
62 |
+ .nav a.link { color: var(--muted); font-weight: 500; font-size: 14px; padding: 8px 10px; border-radius: 9px; } |
|
63 |
+ .nav a.link:hover { color: var(--ink); background: rgba(120,200,160,0.08); } |
|
64 |
+ .pill { |
|
65 |
+ font-family: var(--mono); font-size: 12px; color: var(--vine); |
|
66 |
+ border: 1px solid var(--line-strong); border-radius: 999px; padding: 5px 11px; |
|
67 |
+ display: inline-flex; align-items: center; gap: 7px; background: rgba(74,222,128,0.06); |
|
68 |
+ } |
|
69 |
+ .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--vine); box-shadow: 0 0 9px var(--vine); animation: pulse 2s infinite; } |
|
70 |
+ @keyframes pulse { 0%,100%{opacity:1} 50%{opacity:.35} } |
|
71 |
+ |
|
72 |
+ /* hero */ |
|
73 |
+ .hero { padding: 74px 0 40px; position: relative; overflow: hidden; } |
|
74 |
+ .kicker { font-family: var(--mono); font-size: 12.5px; letter-spacing: 0.18em; text-transform: uppercase; color: var(--vine-2); margin-bottom: 18px; } |
|
75 |
+ h1 { font-family: var(--display); font-size: clamp(38px, 6vw, 68px); line-height: 1.02; letter-spacing: -0.03em; margin: 0 0 18px; font-weight: 700; } |
|
76 |
+ h1 .grad { background: linear-gradient(120deg, var(--vine), var(--vine-2) 60%, var(--amber)); -webkit-background-clip: text; background-clip: text; color: transparent; } |
|
77 |
+ .lede { font-size: 19px; color: var(--muted); max-width: 640px; margin: 0 0 28px; } |
|
78 |
+ .cta-row { display: flex; gap: 12px; flex-wrap: wrap; align-items: center; } |
|
79 |
+ .btn { |
|
80 |
+ font-family: var(--display); font-weight: 600; font-size: 15px; cursor: pointer; |
|
81 |
+ border-radius: 12px; padding: 12px 20px; border: 1px solid var(--line-strong); |
|
82 |
+ background: var(--panel-solid); color: var(--ink); transition: .15s; display: inline-flex; align-items: center; gap: 9px; |
|
83 |
+ } |
|
84 |
+ .btn:hover { transform: translateY(-1px); border-color: var(--vine); } |
|
85 |
+ .btn.primary { background: linear-gradient(120deg, var(--vine), var(--vine-2)); color: #05130c; border: none; box-shadow: 0 14px 34px -14px rgba(74,222,128,0.6); } |
|
86 |
+ .btn.primary:hover { filter: brightness(1.06); } |
|
87 |
+ .stats { display: flex; gap: 30px; flex-wrap: wrap; margin-top: 40px; } |
|
88 |
+ .stat .n { font-family: var(--display); font-size: 26px; font-weight: 700; color: var(--ink); } |
|
89 |
+ .stat .l { font-size: 12.5px; color: var(--dim); font-family: var(--mono); text-transform: uppercase; letter-spacing: 0.08em; } |
|
90 |
+ .vine-svg { position: absolute; right: -40px; top: 20px; width: 420px; max-width: 46%; opacity: 0.5; pointer-events: none; } |
|
91 |
+ .vine-svg path { fill: none; stroke: url(#vg); stroke-width: 2; stroke-linecap: round; stroke-dasharray: 1400; stroke-dashoffset: 1400; animation: grow 3.2s ease forwards; } |
|
92 |
+ @keyframes grow { to { stroke-dashoffset: 0; } } |
|
93 |
+ |
|
94 |
+ section { padding: 40px 0; } |
|
95 |
+ .sec-head { display: flex; align-items: baseline; gap: 14px; margin-bottom: 22px; } |
|
96 |
+ .sec-head h2 { font-family: var(--display); font-size: 26px; letter-spacing: -0.02em; margin: 0; } |
|
97 |
+ .sec-head .tag { font-family: var(--mono); font-size: 12px; color: var(--dim); } |
|
98 |
+ |
|
99 |
+ /* playground */ |
|
100 |
+ .pg { display: grid; grid-template-columns: 1fr 1.15fr; gap: 20px; } |
|
101 |
+ @media (max-width: 900px){ .pg { grid-template-columns: 1fr; } .vine-svg{ display:none } } |
|
102 |
+ .card { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); box-shadow: var(--shadow); backdrop-filter: blur(8px); } |
|
103 |
+ .card .hd { padding: 15px 18px; border-bottom: 1px solid var(--line); display: flex; align-items: center; gap: 10px; } |
|
104 |
+ .card .hd h3 { font-family: var(--display); font-size: 15px; margin: 0; letter-spacing: -0.01em; } |
|
105 |
+ .card .bd { padding: 18px; } |
|
106 |
+ .seg { display: inline-flex; background: rgba(10,20,15,0.7); border: 1px solid var(--line); border-radius: 11px; padding: 4px; gap: 3px; } |
|
107 |
+ .seg button { font-family: var(--mono); font-size: 13px; color: var(--muted); background: none; border: 0; padding: 7px 14px; border-radius: 8px; cursor: pointer; } |
|
108 |
+ .seg button.on { background: linear-gradient(120deg, rgba(74,222,128,0.9), rgba(45,212,191,0.9)); color: #05130c; font-weight: 600; } |
|
109 |
+ label.f { display: block; font-size: 12.5px; color: var(--muted); font-family: var(--mono); margin: 16px 0 7px; letter-spacing: 0.02em; } |
|
110 |
+ input[type=text], input[type=number], select { |
|
111 |
+ width: 100%; background: rgba(6,14,10,0.85); border: 1px solid var(--line); color: var(--ink); |
|
112 |
+ border-radius: 10px; padding: 11px 13px; font-family: var(--mono); font-size: 13.5px; outline: none; transition: .15s; |
|
113 |
+ } |
|
114 |
+ input:focus, select:focus { border-color: var(--vine); box-shadow: 0 0 0 3px rgba(74,222,128,0.12); } |
|
115 |
+ .chips { display: flex; flex-wrap: wrap; gap: 8px; } |
|
116 |
+ .chk { font-family: var(--mono); font-size: 12.5px; display: inline-flex; align-items: center; gap: 7px; padding: 8px 12px; border: 1px solid var(--line); border-radius: 9px; cursor: pointer; color: var(--muted); user-select: none; } |
|
117 |
+ .chk input { accent-color: var(--vine); } |
|
118 |
+ .chk.on { border-color: var(--vine); color: var(--ink); background: rgba(74,222,128,0.07); } |
|
119 |
+ .row2 { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; } |
|
120 |
+ .send-row { display: flex; align-items: center; gap: 12px; margin-top: 20px; } |
|
121 |
+ .meta-line { font-family: var(--mono); font-size: 12px; color: var(--dim); } |
|
122 |
+ .badge { font-family: var(--mono); font-size: 11px; padding: 3px 8px; border-radius: 7px; border: 1px solid var(--line); color: var(--muted); } |
|
123 |
+ .badge.ok { color: var(--vine); border-color: var(--line-strong); } |
|
124 |
+ .badge.err { color: var(--danger); border-color: rgba(255,107,107,0.4); } |
|
125 |
+ |
|
126 |
+ .tabs { display: flex; gap: 4px; flex-wrap: wrap; } |
|
127 |
+ .tabs button { font-family: var(--mono); font-size: 12.5px; color: var(--muted); background: none; border: 1px solid transparent; padding: 6px 11px; border-radius: 8px; cursor: pointer; } |
|
128 |
+ .tabs button.on { color: var(--ink); border-color: var(--line); background: rgba(10,20,15,0.6); } |
|
129 |
+ .out { min-height: 260px; max-height: 560px; overflow: auto; } |
|
130 |
+ .out pre { margin: 0; } |
|
131 |
+ pre code.hljs { background: transparent !important; font-family: var(--mono); font-size: 12.8px; line-height: 1.55; padding: 16px 18px; display: block; } |
|
132 |
+ .rendered { padding: 18px; font-size: 14.5px; } |
|
133 |
+ .rendered h1,.rendered h2,.rendered h3 { font-family: var(--display); line-height: 1.2; } |
|
134 |
+ .rendered a { color: var(--vine-2); border-bottom: 1px solid rgba(45,212,191,0.3); } |
|
135 |
+ .rendered pre { background: rgba(6,14,10,0.9); border: 1px solid var(--line); border-radius: 10px; padding: 12px; overflow: auto; } |
|
136 |
+ .rendered table { border-collapse: collapse; width: 100%; font-size: 13px; } |
|
137 |
+ .rendered th, .rendered td { border: 1px solid var(--line); padding: 6px 10px; text-align: left; } |
|
138 |
+ .rendered img { max-width: 100%; border-radius: 8px; } |
|
139 |
+ .placeholder { color: var(--dim); font-family: var(--mono); font-size: 13px; padding: 40px 18px; text-align: center; } |
|
140 |
+ .linkrow { display: flex; align-items: center; gap: 10px; padding: 9px 14px; border-bottom: 1px solid var(--line); font-family: var(--mono); font-size: 12.5px; } |
|
141 |
+ .linkrow a { color: var(--ink); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex: 1; } |
|
142 |
+ .src { font-size: 10.5px; text-transform: uppercase; letter-spacing: 0.06em; padding: 2px 7px; border-radius: 6px; border: 1px solid var(--line); color: var(--vine-2); } |
|
143 |
+ .copybtn { font-family: var(--mono); font-size: 11.5px; cursor: pointer; background: rgba(10,20,15,0.6); border: 1px solid var(--line); color: var(--muted); padding: 4px 9px; border-radius: 7px; } |
|
144 |
+ .copybtn:hover { color: var(--vine); border-color: var(--vine); } |
|
145 |
+ .spin { width: 15px; height: 15px; border: 2px solid rgba(255,255,255,0.2); border-top-color: var(--vine); border-radius: 50%; animation: sp .7s linear infinite; } |
|
146 |
+ @keyframes sp { to { transform: rotate(360deg); } } |
|
147 |
+ |
|
148 |
+ /* docs */ |
|
149 |
+ .ep { border: 1px solid var(--line); border-radius: var(--radius); margin-bottom: 16px; overflow: hidden; background: var(--panel); } |
|
150 |
+ .ep .top { display: flex; align-items: center; gap: 12px; padding: 15px 18px; cursor: pointer; } |
|
151 |
+ .method { font-family: var(--mono); font-weight: 600; font-size: 12px; padding: 4px 9px; border-radius: 7px; background: rgba(74,222,128,0.14); color: var(--vine); } |
|
152 |
+ .method.get { background: rgba(45,212,191,0.14); color: var(--vine-2); } |
|
153 |
+ .path { font-family: var(--mono); font-size: 14px; } |
|
154 |
+ .ep .desc { color: var(--muted); font-size: 13px; margin-left: auto; } |
|
155 |
+ .ep .body { padding: 0 18px 18px; display: none; } |
|
156 |
+ .ep.open .body { display: block; } |
|
157 |
+ .ptable { width: 100%; border-collapse: collapse; font-size: 13px; margin: 8px 0 4px; } |
|
158 |
+ .ptable th { text-align: left; color: var(--dim); font-family: var(--mono); font-weight: 500; font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.06em; padding: 6px 8px; border-bottom: 1px solid var(--line); } |
|
159 |
+ .ptable td { padding: 7px 8px; border-bottom: 1px solid var(--line); vertical-align: top; } |
|
160 |
+ .ptable code { font-family: var(--mono); color: var(--vine-2); font-size: 12.5px; } |
|
161 |
+ .req-opt { font-size: 10.5px; color: var(--amber); font-family: var(--mono); } |
|
162 |
+ .errtbl { width: 100%; border-collapse: collapse; font-size: 12.5px; } |
|
163 |
+ .errtbl td, .errtbl th { padding: 6px 10px; border-bottom: 1px solid var(--line); text-align: left; } |
|
164 |
+ .errtbl code { color: var(--danger); font-family: var(--mono); } |
|
165 |
+ footer { border-top: 1px solid var(--line); padding: 34px 0 60px; color: var(--dim); font-size: 13px; margin-top: 30px; } |
|
166 |
+ footer a { color: var(--muted); } footer a:hover { color: var(--vine); } |
|
167 |
+ .grid3 { display: grid; grid-template-columns: repeat(3,1fr); gap: 16px; } |
|
168 |
+ @media (max-width:760px){ .grid3{ grid-template-columns:1fr } .row2{ grid-template-columns:1fr } } |
|
169 |
+ .feat { padding: 18px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--panel); } |
|
170 |
+ .feat .i { font-size: 22px; margin-bottom: 10px; } |
|
171 |
+ .feat h4 { font-family: var(--display); margin: 0 0 6px; font-size: 16px; } |
|
172 |
+ .feat p { margin: 0; color: var(--muted); font-size: 13.5px; } |
|
173 |
+ .note { font-size: 12.5px; color: var(--amber); font-family: var(--mono); background: rgba(245,196,81,0.07); border: 1px solid rgba(245,196,81,0.25); border-radius: 10px; padding: 10px 13px; margin-top: 14px; } |
|
174 |
+</style> |
|
175 |
+</head> |
|
176 |
+<body> |
|
177 |
+<header> |
|
178 |
+ <div class="wrap nav"> |
|
179 |
+ <a class="brand" href="#top"><span class="leaf">🌱</span> ten<b>·</b>dril</a> |
|
180 |
+ <span class="spacer"></span> |
|
181 |
+ <a class="link" href="#playground">Playground</a> |
|
182 |
+ <a class="link" href="#docs">Docs</a> |
|
183 |
+ <a class="link" href="https://git.spboucher.ai/tendril" target="_blank" rel="noopener">Source</a> |
|
184 |
+ <span class="pill" id="statusPill"><span class="dot"></span> checking…</span> |
|
185 |
+ </div> |
|
186 |
+</header> |
|
187 |
+ |
|
188 |
+<a id="top"></a> |
|
189 |
+<div class="hero"> |
|
190 |
+ <div class="wrap"> |
|
191 |
+ <svg class="vine-svg" viewBox="0 0 300 300" aria-hidden="true"> |
|
192 |
+ <defs><linearGradient id="vg" x1="0" y1="0" x2="1" y2="1"> |
|
193 |
+ <stop offset="0" stop-color="#4ade80"/><stop offset="0.6" stop-color="#2dd4bf"/><stop offset="1" stop-color="#f5c451"/> |
|
194 |
+ </linearGradient></defs> |
|
195 |
+ <path d="M150 300 C150 240 120 220 150 180 C180 140 120 120 150 80 C170 50 150 30 150 0" /> |
|
196 |
+ <path d="M150 220 C120 210 100 190 85 200 C70 210 80 230 100 225" /> |
|
197 |
+ <path d="M150 150 C185 140 205 120 220 132 C235 144 224 165 202 158" /> |
|
198 |
+ <path d="M150 95 C120 86 104 66 90 78 C78 88 86 108 106 102" /> |
|
199 |
+ </svg> |
|
200 |
+ <div class="kicker">Web ingestion · scrape · map</div> |
|
201 |
+ <h1>Reach what other<br />crawlers <span class="grad">can't.</span></h1> |
|
202 |
+ <p class="lede">Tendril runs on a residential Mac, in WebKit — Safari's real engine — behind a real IP. Extraction is <strong>deterministic and testable</strong>: the same page always yields the same markdown. No cloud fingerprint, no LLM lottery.</p> |
|
203 |
+ <div class="cta-row"> |
|
204 |
+ <a class="btn primary" href="#playground">▶ Open the playground</a> |
|
205 |
+ <a class="btn" href="#docs">Read the docs</a> |
|
206 |
+ </div> |
|
207 |
+ <div class="stats"> |
|
208 |
+ <div class="stat"><div class="n">3</div><div class="l">escalation tiers</div></div> |
|
209 |
+ <div class="stat"><div class="n">0</div><div class="l">LLMs in extraction</div></div> |
|
210 |
+ <div class="stat"><div class="n">~50ms</div><div class="l">Tier 0 p50</div></div> |
|
211 |
+ <div class="stat"><div class="n">WebKit</div><div class="l">real Safari engine</div></div> |
|
212 |
+ </div> |
|
213 |
+ </div> |
|
214 |
+</div> |
|
215 |
+ |
|
216 |
+<section class="wrap"> |
|
217 |
+ <div class="grid3"> |
|
218 |
+ <div class="feat"><div class="i">🕸️</div><h4>Browser fidelity</h4><p>WebKit on macOS ARM behind a residential IP — not a headless Chromium signature. Sites that block others often don't block Tendril.</p></div> |
|
219 |
+ <div class="feat"><div class="i">📐</div><h4>Deterministic extraction</h4><p>JSON-LD, OpenGraph, Readability + density fallback, hand-written Turndown rules. Versioned, golden-file testable output.</p></div> |
|
220 |
+ <div class="feat"><div class="i">🧭</div><h4>Map without rendering</h4><p>Merge sitemaps, robots directives, homepage links and <code>/llms.txt</code> into one deduplicated URL list in seconds.</p></div> |
|
221 |
+ </div> |
|
222 |
+</section> |
|
223 |
+ |
|
224 |
+<section class="wrap" id="playground"> |
|
225 |
+ <div class="sec-head"><h2>Playground</h2><span class="tag">// live requests against this deployment</span></div> |
|
226 |
+ <div class="pg"> |
|
227 |
+ <div class="card"> |
|
228 |
+ <div class="hd"><h3>Request</h3><span class="spacer" style="flex:1"></span> |
|
229 |
+ <div class="seg" id="epseg"> |
|
230 |
+ <button data-ep="scrape" class="on">scrape</button> |
|
231 |
+ <button data-ep="map">map</button> |
|
232 |
+ <button data-ep="status">status</button> |
|
233 |
+ </div> |
|
234 |
+ </div> |
|
235 |
+ <div class="bd"> |
|
236 |
+ <div id="urlblock"> |
|
237 |
+ <label class="f">Target URL</label> |
|
238 |
+ <input type="text" id="url" value="https://www.iana.org" spellcheck="false" placeholder="https://example.com" /> |
|
239 |
+ </div> |
|
240 |
+ |
|
241 |
+ <div id="scrapeOpts"> |
|
242 |
+ <label class="f">Formats</label> |
|
243 |
+ <div class="chips" id="formats"> |
|
244 |
+ <label class="chk on"><input type="checkbox" value="markdown" checked>markdown</label> |
|
245 |
+ <label class="chk"><input type="checkbox" value="links">links</label> |
|
246 |
+ <label class="chk"><input type="checkbox" value="structured">structured</label> |
|
247 |
+ <label class="chk"><input type="checkbox" value="html">html</label> |
|
248 |
+ </div> |
|
249 |
+ <div class="row2"> |
|
250 |
+ <div><label class="f">Tier</label> |
|
251 |
+ <select id="tier"><option value="auto">auto</option><option value="http">http (Tier 0)</option><option value="webkit">webkit</option><option value="safari">safari</option></select> |
|
252 |
+ </div> |
|
253 |
+ <div><label class="f">Options</label> |
|
254 |
+ <div class="chips"> |
|
255 |
+ <label class="chk on"><input type="checkbox" id="onlyMain" checked>onlyMainContent</label> |
|
256 |
+ <label class="chk on"><input type="checkbox" id="respRobots" checked>respectRobots</label> |
|
257 |
+ </div> |
|
258 |
+ </div> |
|
259 |
+ </div> |
|
260 |
+ </div> |
|
261 |
+ |
|
262 |
+ <div id="mapOpts" style="display:none"> |
|
263 |
+ <div class="row2"> |
|
264 |
+ <div><label class="f">Search filter</label><input type="text" id="mapSearch" placeholder="optional substring" spellcheck="false"></div> |
|
265 |
+ <div><label class="f">Limit</label><input type="number" id="mapLimit" value="30" min="1" max="50000"></div> |
|
266 |
+ </div> |
|
267 |
+ <label class="f">Options</label> |
|
268 |
+ <div class="chips"> |
|
269 |
+ <label class="chk"><input type="checkbox" id="sitemapOnly">sitemapOnly</label> |
|
270 |
+ <label class="chk"><input type="checkbox" id="ignoreSitemap">ignoreSitemap</label> |
|
271 |
+ <label class="chk"><input type="checkbox" id="subdomains">includeSubdomains</label> |
|
272 |
+ </div> |
|
273 |
+ </div> |
|
274 |
+ |
|
275 |
+ <div id="statusOpts" style="display:none"> |
|
276 |
+ <p class="meta-line">No parameters. Returns per-tier availability and pipeline version.</p> |
|
277 |
+ </div> |
|
278 |
+ |
|
279 |
+ <div class="send-row"> |
|
280 |
+ <button class="btn primary" id="send">▶ Send request</button> |
|
281 |
+ <span id="reqmeta" class="meta-line"></span> |
|
282 |
+ </div> |
|
283 |
+ <div class="note" id="autoNote" style="display:none">Tiers 1 & 2 (WebKit/Safari) aren't deployed yet — a page needing escalation under <b>tier: auto</b> returns <code>ERR_TARGET_BLOCKED</code>. Force <b>tier: http</b> for raw Tier 0 output.</div> |
|
284 |
+ </div> |
|
285 |
+ </div> |
|
286 |
+ |
|
287 |
+ <div class="card"> |
|
288 |
+ <div class="hd"><h3>Response</h3><span class="spacer" style="flex:1"></span><span id="respStatus"></span></div> |
|
289 |
+ <div class="bd" style="padding-top:14px"> |
|
290 |
+ <div class="tabs" id="tabs"></div> |
|
291 |
+ <div class="out" id="out"><div class="placeholder">Send a request to see the response, rendered markdown, extracted links and ready-to-copy snippets.</div></div> |
|
292 |
+ </div> |
|
293 |
+ </div> |
|
294 |
+ </div> |
|
295 |
+</section> |
|
296 |
+ |
|
297 |
+<section class="wrap" id="docs"> |
|
298 |
+ <div class="sec-head"><h2>API reference</h2><span class="tag">// base: this origin · Bearer auth coming with accounts</span></div> |
|
299 |
+ <div id="epdocs"></div> |
|
300 |
+ |
|
301 |
+ <div class="sec-head" style="margin-top:30px"><h2>Errors</h2><span class="tag">// §15 taxonomy — one code per failure</span></div> |
|
302 |
+ <div class="card"><div class="bd"> |
|
303 |
+ <table class="errtbl"><tbody> |
|
304 |
+ <tr><td><code>ERR_INVALID_URL</code></td><td>400</td><td>Malformed or unsupported scheme</td></tr> |
|
305 |
+ <tr><td><code>ERR_SSRF_BLOCKED</code></td><td>400</td><td>Resolved to a private/forbidden address</td></tr> |
|
306 |
+ <tr><td><code>ERR_ROBOTS_DENIED</code></td><td>403</td><td>robots.txt disallows the path</td></tr> |
|
307 |
+ <tr><td><code>ERR_TARGET_BLOCKED</code></td><td>502</td><td>Tier 0 insufficient; higher tiers unavailable</td></tr> |
|
308 |
+ <tr><td><code>ERR_UNSUPPORTED_TYPE</code></td><td>415</td><td>Content type not handled</td></tr> |
|
309 |
+ <tr><td><code>ERR_TIER_TIMEOUT</code></td><td>504</td><td>Render exceeded timeout</td></tr> |
|
310 |
+ </tbody></table> |
|
311 |
+ <p class="meta-line" style="margin-top:12px">Every response carries a <code>requestId</code> and an <code>X-Request-Id</code> header for support.</p> |
|
312 |
+ </div></div> |
|
313 |
+</section> |
|
314 |
+ |
|
315 |
+<footer> |
|
316 |
+ <div class="wrap"> |
|
317 |
+ <div style="display:flex;gap:20px;flex-wrap:wrap;align-items:center"> |
|
318 |
+ <span class="brand"><span class="leaf">🌱</span> ten·dril</span> |
|
319 |
+ <span class="spacer" style="flex:1"></span> |
|
320 |
+ <a href="https://git.spboucher.ai/tendril" target="_blank" rel="noopener">Source</a> |
|
321 |
+ <a href="/healthz">/healthz</a> |
|
322 |
+ <a href="mailto:contact@spboucher.ai">contact@spboucher.ai</a> |
|
323 |
+ </div> |
|
324 |
+ <p style="margin-top:16px;max-width:640px">Tendril identifies itself with a bot User-Agent and a contact URL, honours <code>robots.txt</code> by default, and rate-limits per host. Stealth here means matching a real browser's fingerprint — not hiding who we are. Built by Simon-Pierre Boucher.</p> |
|
325 |
+ </div> |
|
326 |
+</footer> |
|
327 |
+ |
|
328 |
+<script src="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/highlight.min.js"></script> |
|
329 |
+<script src="https://cdn.jsdelivr.net/npm/marked@12.0.0/marked.min.js"></script> |
|
330 |
+<script> |
|
331 |
+const $ = (s, r=document) => r.querySelector(s); |
|
332 |
+const $$ = (s, r=document) => [...r.querySelectorAll(s)]; |
|
333 |
+let endpoint = "scrape"; |
|
334 |
+let lastResult = null; |
|
335 |
+let activeTab = "preview"; |
|
336 |
+ |
|
337 |
+// status pill |
|
338 |
+fetch("/v1/status").then(r=>r.json()).then(j=>{ |
|
339 |
+ const p = $("#statusPill"); |
|
340 |
+ const http = j?.data?.tiers?.http?.available; |
|
341 |
+ p.innerHTML = `<span class="dot"></span> ${http ? "live" : "degraded"} · v${j?.data?.pipelineVersion||"?"}`; |
|
342 |
+}).catch(()=>{ $("#statusPill").innerHTML = '<span class="dot" style="background:#ff6b6b;box-shadow:0 0 9px #ff6b6b"></span> offline'; }); |
|
343 |
+ |
|
344 |
+// endpoint switch |
|
345 |
+$$("#epseg button").forEach(b => b.onclick = () => { |
|
346 |
+ $$("#epseg button").forEach(x=>x.classList.remove("on")); b.classList.add("on"); |
|
347 |
+ endpoint = b.dataset.ep; |
|
348 |
+ $("#scrapeOpts").style.display = endpoint==="scrape" ? "" : "none"; |
|
349 |
+ $("#mapOpts").style.display = endpoint==="map" ? "" : "none"; |
|
350 |
+ $("#statusOpts").style.display = endpoint==="status" ? "" : "none"; |
|
351 |
+ $("#urlblock").style.display = endpoint==="status" ? "none" : ""; |
|
352 |
+}); |
|
353 |
+ |
|
354 |
+// chip toggles |
|
355 |
+$$(".chk").forEach(c => { const i = c.querySelector("input"); if(i) i.onchange = () => c.classList.toggle("on", i.checked); }); |
|
356 |
+$("#tier").onchange = e => $("#autoNote").style.display = e.target.value==="auto" ? "" : "none"; |
|
357 |
+ |
|
358 |
+function buildBody() { |
|
359 |
+ if (endpoint === "status") return null; |
|
360 |
+ const url = $("#url").value.trim(); |
|
361 |
+ if (endpoint === "scrape") { |
|
362 |
+ const formats = $$("#formats input:checked").map(i=>i.value); |
|
363 |
+ return { url, formats, tier: $("#tier").value, onlyMainContent: $("#onlyMain").checked, respectRobots: $("#respRobots").checked }; |
|
364 |
+ } |
|
365 |
+ const b = { url, limit: Number($("#mapLimit").value)||30, sitemapOnly: $("#sitemapOnly").checked, ignoreSitemap: $("#ignoreSitemap").checked, includeSubdomains: $("#subdomains").checked }; |
|
366 |
+ const s = $("#mapSearch").value.trim(); if (s) b.search = s; |
|
367 |
+ return b; |
|
368 |
+} |
|
369 |
+ |
|
370 |
+function snippets(path, body) { |
|
371 |
+ const origin = location.origin; |
|
372 |
+ if (!body) return { |
|
373 |
+ curl: `curl -s ${origin}${path}`, |
|
374 |
+ python: `import requests\nprint(requests.get("${origin}${path}").json())`, |
|
375 |
+ js: `const r = await fetch("${origin}${path}");\nconsole.log(await r.json());` |
|
376 |
+ }; |
|
377 |
+ const j = JSON.stringify(body, null, 2); |
|
378 |
+ return { |
|
379 |
+ curl: `curl -s -X POST ${origin}${path} \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(body)}'`, |
|
380 |
+ python: `import requests\nr = requests.post("${origin}${path}", json=${j.replace(/true/g,"True").replace(/false/g,"False")})\nprint(r.json())`, |
|
381 |
+ js: `const r = await fetch("${origin}${path}", {\n method: "POST",\n headers: { "content-type": "application/json" },\n body: JSON.stringify(${j})\n});\nconsole.log(await r.json());` |
|
382 |
+ }; |
|
383 |
+} |
|
384 |
+ |
|
385 |
+function setTabs(tabs) { |
|
386 |
+ const t = $("#tabs"); t.innerHTML = ""; |
|
387 |
+ tabs.forEach(name => { |
|
388 |
+ const b = document.createElement("button"); |
|
389 |
+ b.textContent = name; b.className = name===activeTab ? "on" : ""; |
|
390 |
+ b.onclick = () => { activeTab = name; render(); }; |
|
391 |
+ t.appendChild(b); |
|
392 |
+ }); |
|
393 |
+} |
|
394 |
+ |
|
395 |
+function code(text, lang) { |
|
396 |
+ const pre = document.createElement("pre"); |
|
397 |
+ const c = document.createElement("code"); |
|
398 |
+ c.className = "hljs" + (lang ? " language-"+lang : ""); |
|
399 |
+ c.textContent = text; |
|
400 |
+ pre.appendChild(c); |
|
401 |
+ try { hljs.highlightElement(c); } catch(e){} |
|
402 |
+ return pre; |
|
403 |
+} |
|
404 |
+function copyBar(text) { |
|
405 |
+ const bar = document.createElement("div"); |
|
406 |
+ bar.style.cssText = "display:flex;justify-content:flex-end;padding:8px 12px 0"; |
|
407 |
+ const b = document.createElement("button"); b.className="copybtn"; b.textContent="⧉ copy"; |
|
408 |
+ b.onclick = () => { navigator.clipboard.writeText(text); b.textContent="✓ copied"; setTimeout(()=>b.textContent="⧉ copy",1200); }; |
|
409 |
+ bar.appendChild(b); return bar; |
|
410 |
+} |
|
411 |
+ |
|
412 |
+function render() { |
|
413 |
+ const out = $("#out"); out.innerHTML = ""; |
|
414 |
+ if (!lastResult) return; |
|
415 |
+ const { data, raw, snips } = lastResult; |
|
416 |
+ |
|
417 |
+ if (activeTab === "preview") { |
|
418 |
+ if (endpoint === "map") { |
|
419 |
+ const links = data?.data?.links || []; |
|
420 |
+ if (!links.length) { out.innerHTML = '<div class="placeholder">No links found.</div>'; return; } |
|
421 |
+ const head = document.createElement("div"); head.style.cssText="padding:10px 14px;color:var(--dim);font-family:var(--mono);font-size:12px"; |
|
422 |
+ head.textContent = `${data.data.count} URLs discovered`; |
|
423 |
+ out.appendChild(head); |
|
424 |
+ links.forEach(l => { |
|
425 |
+ const r = document.createElement("div"); r.className="linkrow"; |
|
426 |
+ r.innerHTML = `<span class="src">${l.source}</span><a href="${l.url}" target="_blank" rel="noopener">${l.url}</a>`; |
|
427 |
+ out.appendChild(r); |
|
428 |
+ }); |
|
429 |
+ } else if (endpoint === "status") { |
|
430 |
+ out.appendChild(code(JSON.stringify(data, null, 2), "json")); |
|
431 |
+ } else { |
|
432 |
+ const d = data?.data; |
|
433 |
+ if (d?.markdown) { |
|
434 |
+ const div = document.createElement("div"); div.className="rendered"; |
|
435 |
+ div.innerHTML = marked.parse(d.markdown); |
|
436 |
+ out.appendChild(div); |
|
437 |
+ } else if (d?.html) { |
|
438 |
+ out.appendChild(code(d.html.slice(0, 20000), "xml")); |
|
439 |
+ } else { |
|
440 |
+ out.appendChild(code(JSON.stringify(data, null, 2), "json")); |
|
441 |
+ } |
|
442 |
+ } |
|
443 |
+ } else if (activeTab === "links") { |
|
444 |
+ const links = lastResult.data?.data?.links || []; |
|
445 |
+ if (!links.length) { out.innerHTML = '<div class="placeholder">No links in this response.</div>'; return; } |
|
446 |
+ links.forEach(l => { |
|
447 |
+ const r = document.createElement("div"); r.className="linkrow"; |
|
448 |
+ const badge = l.isInternal!==undefined ? `<span class="src">${l.isInternal?"internal":"external"}</span>` : ""; |
|
449 |
+ r.innerHTML = `${badge}<a href="${l.url}" target="_blank" rel="noopener">${l.url}</a>`; |
|
450 |
+ out.appendChild(r); |
|
451 |
+ }); |
|
452 |
+ } else if (activeTab === "metadata") { |
|
453 |
+ out.appendChild(code(JSON.stringify(lastResult.data?.data?.metadata || {}, null, 2), "json")); |
|
454 |
+ } else if (activeTab === "structured") { |
|
455 |
+ out.appendChild(code(JSON.stringify(lastResult.data?.data?.structured || {}, null, 2), "json")); |
|
456 |
+ } else if (activeTab === "json") { |
|
457 |
+ out.appendChild(code(raw, "json")); |
|
458 |
+ } else if (["curl","python","js"].includes(activeTab)) { |
|
459 |
+ out.appendChild(copyBar(snips[activeTab])); |
|
460 |
+ out.appendChild(code(snips[activeTab], activeTab==="curl"?"bash":activeTab==="python"?"python":"javascript")); |
|
461 |
+ } |
|
462 |
+} |
|
463 |
+ |
|
464 |
+async function send() { |
|
465 |
+ const path = "/v1/" + endpoint; |
|
466 |
+ const body = buildBody(); |
|
467 |
+ const snips = snippets(path, body); |
|
468 |
+ const btn = $("#send"); btn.disabled = true; btn.innerHTML = '<span class="spin"></span> running'; |
|
469 |
+ $("#reqmeta").textContent = ""; |
|
470 |
+ const t0 = performance.now(); |
|
471 |
+ try { |
|
472 |
+ const res = await fetch(path, endpoint==="status" |
|
473 |
+ ? { method:"GET" } |
|
474 |
+ : { method:"POST", headers:{ "content-type":"application/json" }, body: JSON.stringify(body) }); |
|
475 |
+ const raw = await res.text(); |
|
476 |
+ let data; try { data = JSON.parse(raw); } catch { data = raw; } |
|
477 |
+ const ms = Math.round(performance.now() - t0); |
|
478 |
+ const ok = res.ok && data?.success !== false; |
|
479 |
+ $("#respStatus").innerHTML = `<span class="badge ${ok?"ok":"err"}">${res.status}</span> <span class="meta-line">${ms} ms</span>`; |
|
480 |
+ $("#reqmeta").innerHTML = ok ? `<span class="badge ok">success</span>` : `<span class="badge err">${data?.error?.code||"error"}</span>`; |
|
481 |
+ const pretty = typeof data === "string" ? data : JSON.stringify(data, null, 2); |
|
482 |
+ lastResult = { data, raw: pretty, snips }; |
|
483 |
+ |
|
484 |
+ let tabs; |
|
485 |
+ if (endpoint === "scrape") { |
|
486 |
+ tabs = ["preview"]; |
|
487 |
+ if (data?.data?.links) tabs.push("links"); |
|
488 |
+ if (data?.data?.metadata) tabs.push("metadata"); |
|
489 |
+ if (data?.data?.structured) tabs.push("structured"); |
|
490 |
+ tabs.push("json","curl","python","js"); |
|
491 |
+ } else if (endpoint === "map") { |
|
492 |
+ tabs = ["preview","json","curl","python","js"]; |
|
493 |
+ } else { |
|
494 |
+ tabs = ["preview","json","curl","python","js"]; |
|
495 |
+ } |
|
496 |
+ activeTab = "preview"; |
|
497 |
+ setTabs(tabs); render(); |
|
498 |
+ } catch (e) { |
|
499 |
+ $("#respStatus").innerHTML = `<span class="badge err">network</span>`; |
|
500 |
+ $("#out").innerHTML = `<div class="placeholder">Request failed: ${e.message}</div>`; |
|
501 |
+ } finally { |
|
502 |
+ btn.disabled = false; btn.innerHTML = "▶ Send request"; |
|
503 |
+ } |
|
504 |
+} |
|
505 |
+$("#send").onclick = send; |
|
506 |
+ |
|
507 |
+// docs |
|
508 |
+const EPS = [ |
|
509 |
+ { m:"POST", path:"/v1/scrape", desc:"Fetch one page and extract it", |
|
510 |
+ params:[ |
|
511 |
+ ["url","string","required","The page to fetch"], |
|
512 |
+ ["formats","string[]","markdown","Any of markdown · html · rawHtml · links · structured"], |
|
513 |
+ ["tier","enum","auto","auto · http · webkit · safari"], |
|
514 |
+ ["onlyMainContent","boolean","true","Strip nav/footer/ads, keep the article"], |
|
515 |
+ ["respectRobots","boolean","true","Honour robots.txt (refused if disallowed)"], |
|
516 |
+ ["timeout","number","30000","Milliseconds"], |
|
517 |
+ ], |
|
518 |
+ ex:{ url:"https://www.iana.org", formats:["markdown","structured"] } }, |
|
519 |
+ { m:"POST", path:"/v1/map", desc:"Discover a site's URLs without rendering", |
|
520 |
+ params:[ |
|
521 |
+ ["url","string","required","The site root to map"], |
|
522 |
+ ["search","string","—","Substring filter, ranked by match position"], |
|
523 |
+ ["limit","number","5000","Max URLs (cap 50000)"], |
|
524 |
+ ["sitemapOnly","boolean","false","Only sitemaps, skip homepage links"], |
|
525 |
+ ["ignoreSitemap","boolean","false","Skip sitemaps, links only"], |
|
526 |
+ ["includeSubdomains","boolean","false","Include subdomains of the apex"], |
|
527 |
+ ], |
|
528 |
+ ex:{ url:"https://www.iana.org", limit:20 } }, |
|
529 |
+ { m:"GET", path:"/v1/status", desc:"Per-tier availability + pipeline version", params:[], ex:null }, |
|
530 |
+]; |
|
531 |
+const box = $("#epdocs"); |
|
532 |
+EPS.forEach(ep => { |
|
533 |
+ const el = document.createElement("div"); el.className = "ep"; |
|
534 |
+ const rows = ep.params.map(p => `<tr><td><code>${p[0]}</code></td><td>${p[1]}</td><td>${p[2]==="required"?'<span class="req-opt">required</span>':p[2]}</td><td>${p[3]}</td></tr>`).join(""); |
|
535 |
+ el.innerHTML = ` |
|
536 |
+ <div class="top"><span class="method ${ep.m.toLowerCase()}">${ep.m}</span><span class="path">${ep.path}</span><span class="desc">${ep.desc}</span></div> |
|
537 |
+ <div class="body"> |
|
538 |
+ ${ep.params.length ? `<table class="ptable"><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Notes</th></tr></thead><tbody>${rows}</tbody></table>` : '<p class="meta-line">No parameters.</p>'} |
|
539 |
+ ${ep.ex ? `<div style="margin-top:12px">${""}</div>` : ""} |
|
540 |
+ </div>`; |
|
541 |
+ el.querySelector(".top").onclick = () => el.classList.toggle("open"); |
|
542 |
+ if (ep.ex) { |
|
543 |
+ const pre = code(`curl -s -X POST ${location.origin}${ep.path} \\\n -H 'content-type: application/json' \\\n -d '${JSON.stringify(ep.ex)}'`, "bash"); |
|
544 |
+ el.querySelector(".body > div:last-child")?.appendChild(pre); |
|
545 |
+ el.querySelector(".body").appendChild(pre); |
|
546 |
+ } |
|
547 |
+ box.appendChild(el); |
|
548 |
+}); |
|
549 |
+$$(".ep")[0]?.classList.add("open"); |
|
550 |
+</script> |
|
551 |
+</body> |
|
552 |
+</html> |