TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import type { Metadata } from "next";2import Link from "next/link";3import { BookOpen, Network } from "lucide-react";4import { PLAN_LIMITS, normalizePlan } from "@fetcha/core";5import { getWorkspace } from "@/lib/session";6import { internalApi, InternalApiError, type CrawlJob } from "@/lib/api";7import { formatBytes, formatDate, formatNumber, timeAgo } from "@/lib/format";8import { API_PUBLIC_URL } from "@/lib/utils";9import { PageHeader } from "@/components/ui/page-header";10import { Alert } from "@/components/ui/alert";11import { Button } from "@/components/ui/button";12import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";13import { CodeBlock } from "@/components/ui/code-block";14import { EmptyState } from "@/components/ui/empty-state";15import { Stat, StatGrid } from "@/components/ui/stat";16import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";17import { CrawlStatusBadge } from "@/components/dashboard/crawls/crawl-status-badge";18import { CreateCrawlDialog } from "@/components/dashboard/crawls/create-crawl-dialog";19import { CancelCrawlButton } from "@/components/dashboard/crawls/cancel-crawl-button";20import { MapTool } from "@/components/dashboard/crawls/map-tool";2122export const dynamic = "force-dynamic";23export const metadata: Metadata = { title: "Crawls" };2425const API_SNIPPET = `# 1. Start a crawl (returns 202 with the job)26curl -X POST ${API_PUBLIC_URL}/v1/crawl \\27 -H "Authorization: Bearer fch_live_YOUR_KEY" \\28 -H "Content-Type: application/json" \\29 -d '{ "url": "https://docs.example.com/", "max_pages": 100, "max_depth": 3, "format": "markdown" }'30# → { "id": "crawl_…", "status": "queued", … }3132# 2. Poll the job, then page through the results33curl ${API_PUBLIC_URL}/v1/crawl/crawl_… -H "Authorization: Bearer fch_live_YOUR_KEY"34curl "${API_PUBLIC_URL}/v1/crawl/crawl_…/pages?limit=100" -H "Authorization: Bearer fch_live_YOUR_KEY"`;3536function seedLabel(job: CrawlJob): string {37 try {38 const u = new URL(job.seed_url);39 return u.host + (u.pathname !== "/" ? u.pathname : "");40 } catch {41 return job.seed_url;42 }43}4445export default async function CrawlsPage() {46 const ws = await getWorkspace();47 const limits = PLAN_LIMITS[normalizePlan(ws.organization.plan)];4849 let jobs: CrawlJob[] = [];50 let loadError: string | null = null;51 try {52 const res = await internalApi.listCrawls(ws.project.id, ws.user.id, 50);53 jobs = Array.isArray(res?.data) ? res.data : [];54 } catch (e) {55 loadError = e instanceof InternalApiError ? e.message : "The Fetcha API service is unreachable.";56 }5758 const active = jobs.filter((j) => j.status === "queued" || j.status === "running");59 const pagesFetched = jobs.reduce((a, j) => a + (j.stats?.fetched ?? 0), 0);60 const bytes = jobs.reduce((a, j) => a + (j.stats?.bytes ?? 0), 0);61 const createButton = <CreateCrawlDialog defaultCountry={ws.project.defaultCountry} maxPages={limits.crawl_max_pages} />;6263 return (64 <div className="flex flex-col gap-5">65 <PageHeader66 eyebrow={ws.project.name}67 title="Crawls"68 description={`Crawl jobs for ${ws.project.name}. A crawl follows links from a seed URL, fetches each page through the routing engine and stores the content as Markdown, text or HTML.`}69 actions={70 <>71 <Button variant="outline" size="sm" asChild>72 <Link href="/docs/crawl">73 <BookOpen className="size-3.5" /> Crawl API74 </Link>75 </Button>76 {createButton}77 </>78 }79 />8081 <StatGrid cols={4}>82 <Stat label="Active jobs" value={formatNumber(active.length)} hint={`up to ${limits.crawl_concurrent_jobs} concurrent`} />83 <Stat label="Jobs" value={formatNumber(jobs.length)} hint="most recent 50" />84 <Stat label="Pages fetched" value={formatNumber(pagesFetched)} hint={`max ${formatNumber(limits.crawl_max_pages)} per job`} />85 <Stat label="Bytes transferred" value={formatBytes(bytes)} hint="all listed jobs" />86 </StatGrid>8788 {loadError ? (89 <Alert variant="danger" title="Could not load crawl jobs">90 {loadError}91 </Alert>92 ) : null}9394 {jobs.length ? (95 <Card className="overflow-hidden">96 <Table>97 <TableHeader>98 <TableRow className="hover:bg-transparent">99 <TableHead>Crawl</TableHead>100 <TableHead>Status</TableHead>101 <TableHead className="text-right">Fetched</TableHead>102 <TableHead className="text-right">OK</TableHead>103 <TableHead className="text-right">Blocked</TableHead>104 <TableHead className="text-right">Failed</TableHead>105 <TableHead className="text-right">Bytes</TableHead>106 <TableHead>Created</TableHead>107 <TableHead className="text-right"> </TableHead>108 </TableRow>109 </TableHeader>110 <TableBody>111 {jobs.map((j) => (112 <TableRow key={j.id}>113 <TableCell className="max-w-[320px]">114 <Link href={`/dashboard/crawls/${j.id}`} className="block min-w-0">115 <span className="block truncate text-[13.5px] font-medium text-fg underline-offset-4 hover:underline" title={j.label ?? j.seed_url}>116 {j.label ?? seedLabel(j)}117 </span>118 <span className="block truncate font-mono text-[11.5px] text-fg-subtle" title={j.seed_url}>119 {j.label ? j.seed_url : j.id}120 </span>121 </Link>122 </TableCell>123 <TableCell>124 <CrawlStatusBadge status={j.status} />125 </TableCell>126 <TableCell className="text-right font-mono tabular">127 {formatNumber(j.stats?.fetched ?? 0)}128 <span className="text-fg-subtle"> / {formatNumber(j.stats?.discovered ?? 0)}</span>129 </TableCell>130 <TableCell className="text-right font-mono tabular text-success">{formatNumber(j.stats?.ok ?? 0)}</TableCell>131 <TableCell className="text-right font-mono tabular">{j.stats?.blocked ? <span className="text-warning">{formatNumber(j.stats.blocked)}</span> : <span className="text-fg-subtle">0</span>}</TableCell>132 <TableCell className="text-right font-mono tabular">{j.stats?.failed ? <span className="text-danger">{formatNumber(j.stats.failed)}</span> : <span className="text-fg-subtle">0</span>}</TableCell>133 <TableCell className="text-right font-mono tabular text-fg-muted">{formatBytes(j.stats?.bytes ?? 0)}</TableCell>134 <TableCell className="whitespace-nowrap text-fg-muted" title={formatDate(j.created_at, { timeStyle: "medium" })}>135 {timeAgo(j.created_at)}136 </TableCell>137 <TableCell className="text-right">{j.status === "queued" || j.status === "running" ? <CancelCrawlButton id={j.id} size="xs" /> : null}</TableCell>138 </TableRow>139 ))}140 </TableBody>141 </Table>142 </Card>143 ) : !loadError ? (144 <EmptyState145 icon={Network}146 title="No crawls yet"147 description="Start one here to try it, or from your code with POST /v1/crawl. Each crawled page is a normal fetch request: quotas, retries, escalation and routing intelligence apply."148 action={createButton}149 />150 ) : null}151152 <div className="grid gap-4 lg:grid-cols-2">153 <Card>154 <CardHeader>155 <CardTitle>How crawls work</CardTitle>156 <CardDescription>Scope, politeness and what you get back.</CardDescription>157 </CardHeader>158 <CardContent className="space-y-3 text-[13px] leading-relaxed text-fg-muted">159 <p>160 <strong className="text-fg">Frontier.</strong> The seed is fetched first; links are extracted, normalised and filtered by <code className="font-mono">same_domain</code>, <code className="font-mono">include_patterns</code> / <code className="font-mono">exclude_patterns</code> and <code className="font-mono">max_depth</code>, then fetched with up to <code className="font-mono">concurrency</code> workers until <code className="font-mono">max_pages</code> is reached.161 </p>162 <p>163 <strong className="text-fg">Politeness.</strong> <code className="font-mono">robots.txt</code> is honoured by default and <code className="font-mono">delay_ms</code> adds a pause between fetches per worker. Sitemaps can seed the frontier with <code className="font-mono">use_sitemap</code>.164 </p>165 <p>166 <strong className="text-fg">Content.</strong> Each page is stored as Markdown (main content, boilerplate removed), text or HTML, with title, description, status, mode (HTTP or browser), bytes and duration. Blocked pages escalate to the managed browser automatically.167 </p>168 <p className="text-fg-subtle">169 Every page also appears in <Link href="/dashboard/requests" className="underline-offset-4 hover:underline">Requests</Link> with source <code className="font-mono">crawl</code>. Jobs are limited to {formatNumber(limits.crawl_max_pages)} pages and {limits.crawl_concurrent_jobs} concurrent jobs per organization.170 </p>171 </CardContent>172 </Card>173 <CodeBlock code={API_SNIPPET} lang="bash" title="Crawls from the API" className="self-start" />174 </div>175176 <MapTool />177 </div>178 );179}180