"use client"; import * as React from "react"; import { useRouter } from "next/navigation"; import { Plus } from "lucide-react"; import { COUNTRIES } from "@fetcha/core/client"; import { createCrawl } from "@/actions/crawls"; import { Alert } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Dialog, DialogClose, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog"; import { Input, Textarea } from "@/components/ui/input"; import { Field, Hint, Label } from "@/components/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; const ANY = "__any__"; const FORMATS = ["markdown", "text", "html"] as const; function splitPatterns(s: string): string[] | undefined { const list = s .split(/[\n,]+/) .map((x) => x.trim()) .filter(Boolean) .slice(0, 50); return list.length ? list : undefined; } export function CreateCrawlDialog({ defaultCountry, maxPages, variant = "primary" }: { defaultCountry?: string | null; maxPages: number; variant?: "primary" | "outline" }) { const router = useRouter(); const [open, setOpen] = React.useState(false); const [pending, startTransition] = React.useTransition(); const [error, setError] = React.useState(null); const [url, setUrl] = React.useState(""); const [label, setLabel] = React.useState(""); const [maxPagesValue, setMaxPagesValue] = React.useState(25); const [maxDepth, setMaxDepth] = React.useState(2); const [format, setFormat] = React.useState<(typeof FORMATS)[number]>("markdown"); const [sameDomain, setSameDomain] = React.useState(true); const [respectRobots, setRespectRobots] = React.useState(true); const [browser, setBrowser] = React.useState(false); const [country, setCountry] = React.useState(defaultCountry && defaultCountry in COUNTRIES ? defaultCountry : ANY); const [include, setInclude] = React.useState(""); const [exclude, setExclude] = React.useState(""); function reset() { setError(null); setUrl(""); setLabel(""); setMaxPagesValue(25); setMaxDepth(2); setFormat("markdown"); setSameDomain(true); setRespectRobots(true); setBrowser(false); setInclude(""); setExclude(""); } function submit(e: React.FormEvent) { e.preventDefault(); setError(null); startTransition(async () => { const res = await createCrawl({ url: url.trim(), label: label.trim() || undefined, max_pages: Math.min(maxPages, Math.max(1, Math.round(maxPagesValue || 1))), max_depth: Math.min(10, Math.max(0, Math.round(maxDepth || 0))), format, same_domain: sameDomain, respect_robots: respectRobots, browser, country: country === ANY ? undefined : country, include_patterns: splitPatterns(include), exclude_patterns: splitPatterns(exclude), }); if (!res.ok) { setError(res.error.message); return; } setOpen(false); reset(); router.push(`/dashboard/crawls/${res.data.id}`); router.refresh(); }); } return ( { setOpen(o); if (!o) setError(null); }} >
Start a crawl Fetcha follows links from the seed URL, fetches each page through the routing engine and stores the content. Every page is a normal request in the log with source crawl. {error ? {error} : null}
setUrl(e.target.value)} placeholder="https://docs.example.com/" autoComplete="off" autoCapitalize="off" spellCheck={false} className="font-mono text-[13px]" /> setMaxPagesValue(Number(e.target.value))} className="font-mono tabular" /> 1–{maxPages.toLocaleString("en-US")}. The seed counts as one. setMaxDepth(Number(e.target.value))} className="font-mono tabular" /> 0 = seed only, up to 10 link hops.