spb/datacenterindex
Public
HTML 53.9%
TypeScript 44.5%
JavaScript 0.6%
SQL 0.5%
1"use client";23import { useMemo, useState } from "react";4import { Badge, Button, EmptyState, SearchInput, Tab, TabList, TabPanel, Tabs } from "@/components/ui";5import { cn } from "@/components/ui/cn";6import { DASH, fmtInt } from "@/lib/format";7import { adminFetch, type DevtoolFetchResult } from "@/lib/admin-api";8import { htmlToTextClient, srcDocFor } from "../document-inspect";9import { Checkbox, Code, countMatches, ErrorNote, ExtLink, Field, fmtBytes, fmtMs, Input, JsonView, KV, LevelBadge, Mono, Panel, Select, Spinner } from "../primitives";10import { useToast } from "../toast";11import { domainOf, slugify, type DevtoolShared } from "./state";1213const SOURCE_CAP = 300_000;1415export function StepFetch({ s }: { s: DevtoolShared }) {16 const { draft, update, fetchResult: r, setFetchResult } = s;17 const f = draft.fetch;18 const [busy, setBusy] = useState(false);19 const [error, setError] = useState<string | null>(null);20 const [search, setSearch] = useState("");21 const [tab, setTab] = useState("rendered");22 const toast = useToast();2324 const run = async () => {25 if (!f.url.trim()) return setError("Enter a URL");26 setBusy(true);27 setError(null);28 const res = await adminFetch<DevtoolFetchResult>("devtool/fetch", { method: "POST", body: { url: f.url.trim(), level: f.level, maxLevel: Math.max(f.level, f.maxLevel), renderJs: f.renderJs || undefined, userAgent: f.userAgent, waitForSelector: f.waitForSelector.trim() || undefined }, timeoutMs: 120_000 });29 setBusy(false);30 if (!res.ok) {31 setError(res.error);32 toast({ tone: "danger", title: "Fetch failed", description: res.error });33 return;34 }35 setFetchResult(res.data);36 s.setPreviewResult(null);37 const d = domainOf(res.data.finalUrl || f.url);38 update((cur) => ({ ...cur, meta: { ...cur.meta, domain: cur.meta.domain || d, id: cur.meta.id || slugify(d), name: cur.meta.name || d.replace(/^www\./, ""), homepage: cur.meta.homepage || (d ? `https://${d}/` : "") } }));39 if (res.data.error) toast({ tone: "warn", title: `Fetched with error (HTML ${res.data.status})`, description: res.data.error.message ?? JSON.stringify(res.data.error).slice(0, 160) });40 else toast({ tone: "ok", title: `Fetched ${res.data.status} via ${res.data.fetcher} (L${res.data.level})`, description: `${fmtBytes(res.data.bytes)} in ${fmtMs(res.data.durationMs)}${res.data.credits ? ` · ${res.data.credits} credits` : ""}` });41 };4243 const html = r?.html ?? "";44 const isHtml = /html|xml/i.test(r?.contentType ?? "") || /<html/i.test(html.slice(0, 2000));45 const finalUrl = r?.finalUrl ?? null;46 const rText = r?.text ?? "";47 const srcDoc = useMemo(() => (isHtml && html ? srcDocFor(html, finalUrl) : ""), [html, isHtml, finalUrl]);48 const text = useMemo(() => (rText.length > 0 ? rText : html ? htmlToTextClient(html) : ""), [rText, html]);49 const cls = (r?.classification ?? null) as Record<string, unknown> | null;5051 return (52 <div className="space-y-3">53 <Panel title="1 · Source URL and fetch options" description="Fetched through the same escalation stack as the worker (L1 direct → L2 browser identity → L3 Firecrawl → L4 Scrapfly). Robots and SSRF policies apply; premium levels spend credits.">54 <div className="grid gap-3 lg:grid-cols-[1fr_120px_120px_150px_auto] lg:items-end">55 <Field label="URL" error={error}>56 <Input value={f.url} onChange={(e) => update({ fetch: { ...f, url: e.target.value } })} onKeyDown={(e) => e.key === "Enter" && void run()} placeholder="https://www.example.com/data-centers/…" mono autoFocus={!f.url} />57 </Field>58 <Field label="Start level">59 <Select value={f.level} onChange={(e) => update({ fetch: { ...f, level: Number(e.target.value) } })}>60 <option value={1}>L1 direct</option>61 <option value={2}>L2 browser UA</option>62 <option value={3}>L3 Firecrawl</option>63 <option value={4}>L4 Scrapfly</option>64 </Select>65 </Field>66 <Field label="Max level">67 <Select value={f.maxLevel} onChange={(e) => update({ fetch: { ...f, maxLevel: Number(e.target.value) } })}>68 <option value={1}>L1</option>69 <option value={2}>L2</option>70 <option value={3}>L3</option>71 <option value={4}>L4</option>72 </Select>73 </Field>74 <Field label="User agent">75 <Select value={f.userAgent} onChange={(e) => update({ fetch: { ...f, userAgent: e.target.value as "bot" | "browser" } })}>76 <option value="bot">DataCenterIndexBot</option>77 <option value="browser">Browser (Safari)</option>78 </Select>79 </Field>80 <Button variant="accent" onClick={() => void run()} disabled={busy}>81 {busy ? <Spinner className="text-white" /> : null}82 {busy ? "Fetching…" : "Fetch sample"}83 </Button>84 </div>85 <div className="mt-2 flex flex-wrap items-end gap-3">86 <Checkbox label="render JavaScript (premium levels)" checked={f.renderJs} onChange={(e) => update({ fetch: { ...f, renderJs: e.target.checked } })} />87 <Field label="Wait for selector" className="w-[260px]" hint="premium levels only">88 <Input value={f.waitForSelector} onChange={(e) => update({ fetch: { ...f, waitForSelector: e.target.value } })} placeholder=".facility-list" mono />89 </Field>90 {f.maxLevel >= 3 && (91 <Badge tone="warn" size="sm">92 maxLevel ≥ 3 may spend Firecrawl / Scrapfly credits93 </Badge>94 )}95 </div>96 </Panel>9798 {r && (99 <>100 <div className="grid grid-cols-2 gap-3 sm:grid-cols-4 lg:grid-cols-8">101 <Stat label="HTTP" value={<span className={cn(r.status >= 400 && "text-danger")}>{r.status}</span>} />102 <Stat label="Fetcher" value={r.fetcher} />103 <Stat label="Level" value={<LevelBadge level={r.level} />} />104 <Stat label="Credits" value={<span className={cn(r.credits > 0 && "text-accent")}>{r.credits}</span>} />105 <Stat label="Duration" value={fmtMs(r.durationMs)} />106 <Stat label="Bytes" value={fmtBytes(r.bytes)} />107 <Stat label="Content type" value={<span className="truncate text-[12px]">{r.contentType ?? DASH}</span>} />108 <Stat label="Page type" value={<span className="text-[12px]">{cls && typeof cls.pageType === "string" ? cls.pageType : DASH}</span>} />109 </div>110 {r.error && <ErrorNote error={`Fetch error: ${r.error.message ?? JSON.stringify(r.error)}`} />}111 {r.finalUrl !== r.url && (112 <p className="text-[12px] text-ink-3">113 Redirected to <ExtLink href={r.finalUrl} />114 </p>115 )}116117 <Panel title={r.title ?? "Untitled page"} description={r.description ?? undefined} padded={false}>118 <Tabs value={tab} onValueChange={setTab}>119 <div className="flex flex-col gap-2 border-b border-hair px-3 pt-1 sm:flex-row sm:items-center">120 <TabList label="Fetch result" className="border-b-0">121 {isHtml && <Tab value="rendered">Rendered</Tab>}122 <Tab value="source">HTML source</Tab>123 <Tab value="text">Text</Tab>124 <Tab value="jsonld" count={r.jsonLd.length}>125 JSON-LD126 </Tab>127 <Tab value="embedded" count={r.embeddedJson.length}>128 Embedded JSON129 </Tab>130 <Tab value="classification">Classification</Tab>131 <Tab value="geo">Geo / address</Tab>132 <Tab value="links" count={r.links.length}>133 Links134 </Tab>135 </TabList>136 {(tab === "source" || tab === "text") && (137 <div className="flex items-center gap-2 pb-2 sm:ml-auto sm:pb-0">138 <SearchInput size="sm" value={search} onChange={setSearch} placeholder="Find (e.g. MW, itemprop)" wrapperClassName="w-[240px]" />139 {search && <span className="figure text-[11.5px] text-ink-3">{fmtInt(countMatches(tab === "source" ? html : text, search))} hits</span>}140 </div>141 )}142 </div>143 {isHtml && (144 <TabPanel value="rendered">145 <iframe title="Rendered sample (sandboxed, scripts stripped)" sandbox="" srcDoc={srcDoc} className="h-[65vh] w-full bg-white" />146 </TabPanel>147 )}148 <TabPanel value="source" className="p-2">149 {r.htmlTruncated && <p className="mb-1 text-[11.5px] text-warn">HTML truncated by the API at 1.5 MB.</p>}150 <Code text={html.length > SOURCE_CAP ? `${html.slice(0, SOURCE_CAP)}\n… showing first ${fmtBytes(SOURCE_CAP)} of ${fmtBytes(html.length)}` : html || "(empty body)"} lang="html" search={search} maxHeight={560} wrap />151 </TabPanel>152 <TabPanel value="text" className="p-2">153 <Code text={text.slice(0, SOURCE_CAP) || "(no text)"} lang="text" search={search} maxHeight={560} wrap />154 </TabPanel>155 <TabPanel value="jsonld" className="space-y-2 p-3">156 {r.jsonLd.length === 0 ? (157 <EmptyState compact title="No JSON-LD objects" />158 ) : (159 r.jsonLd.map((o, i) => {160 const t = o && typeof o === "object" ? (o as Record<string, unknown>)["@type"] : undefined;161 return (162 <div key={i}>163 <div className="mb-1 flex items-center gap-2 text-[12px] text-ink-3">164 <Badge size="xs" mono tone="accent">165 {Array.isArray(t) ? t.join(", ") : String(t ?? "object")}166 </Badge>167 <span>use rule type “JSON-LD” with this @type and a JSON path like $.address.addressLocality</span>168 </div>169 <JsonView value={o} maxHeight={280} />170 </div>171 );172 })173 )}174 </TabPanel>175 <TabPanel value="embedded" className="space-y-2 p-3">176 {r.embeddedJson.length === 0 ? (177 <EmptyState compact title="No embedded JSON blobs" description="Looks for __NEXT_DATA__, drupal-settings-json, application/json scripts and window.__STATE__-style assignments." />178 ) : (179 r.embeddedJson.map((b) => (180 <div key={b.id}>181 <div className="mb-1 text-[12px] text-ink-3">182 <Mono className="text-ink">{b.id}</Mono> · preview (2 kB)183 </div>184 <pre className="max-h-[240px] overflow-auto rounded-xs border border-hair bg-paper px-2.5 py-2 font-mono text-[11.5px] leading-[1.5] text-ink-2">{b.preview}</pre>185 </div>186 ))187 )}188 </TabPanel>189 <TabPanel value="classification" className="p-3">190 <div className="grid gap-3 lg:grid-cols-2">191 <KV items={[{ k: "Title", v: r.title }, { k: "Description", v: r.description }, { k: "Published", v: r.published }, { k: "Final URL", v: <ExtLink href={r.finalUrl} /> }, { k: "Attempts", v: r.attempts != null ? <Mono>{JSON.stringify(r.attempts)}</Mono> : DASH }]} />192 <JsonView value={r.classification} maxHeight={260} />193 </div>194 </TabPanel>195 <TabPanel value="geo" className="p-3">196 <div className="grid gap-3 lg:grid-cols-2">197 <div>198 <div className="label-caps mb-1">Detected geo</div>199 {r.geo ? <JsonView value={r.geo} maxHeight={220} /> : <p className="text-[12.5px] text-ink-4">No coordinates detected (geo meta tags, JSON-LD GeoCoordinates, map embeds).</p>}200 </div>201 <div>202 <div className="label-caps mb-1">Detected address</div>203 {r.address ? <JsonView value={r.address} maxHeight={220} /> : <p className="text-[12.5px] text-ink-4">No postal address detected.</p>}204 </div>205 </div>206 </TabPanel>207 <TabPanel value="links" className="p-0">208 {r.links.length === 0 ? (209 <EmptyState compact title="No links" />210 ) : (211 <ul className="max-h-[480px] divide-y divide-hair overflow-auto">212 {r.links.map((l) => (213 <li key={l.href} className="flex items-center gap-2 px-3 py-1.5 text-[12px]">214 <span className="w-[200px] shrink-0 truncate text-ink-2" title={l.text}>215 {l.text || <span className="text-ink-4">(no text)</span>}216 </span>217 <ExtLink href={l.href} className="min-w-0 flex-1" />218 <button type="button" className="shrink-0 text-[11.5px] text-ink-3 hover:text-accent" onClick={() => update({ fetch: { ...f, url: l.href } })}>219 use as URL220 </button>221 <button type="button" className="shrink-0 text-[11.5px] text-ink-3 hover:text-accent" onClick={() => update((d) => ({ ...d, sampleUrls: d.sampleUrls.includes(l.href) ? d.sampleUrls : `${d.sampleUrls.trim()}${d.sampleUrls.trim() ? "\n" : ""}${l.href}` }))}>222 + sample223 </button>224 </li>225 ))}226 </ul>227 )}228 </TabPanel>229 </Tabs>230 </Panel>231 </>232 )}233 </div>234 );235}236237function Stat({ label, value }: { label: string; value: React.ReactNode }) {238 return (239 <div className="min-w-0 rounded-sm border border-hair bg-surface px-2.5 py-1.5">240 <div className="label-caps">{label}</div>241 <div className="figure truncate text-[14px] text-ink">{value}</div>242 </div>243 );244}245