TypeScript 97.5%
SQL 1.4%
Python 0.8%
1"use client";2import * as React from "react";3import { useRouter } from "next/navigation";4import { Plus } from "lucide-react";5import { COUNTRIES } from "@fetcha/core/client";6import { createCrawl } from "@/actions/crawls";7import { Alert } from "@/components/ui/alert";8import { Button } from "@/components/ui/button";9import { Dialog, DialogClose, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog";10import { Input, Textarea } from "@/components/ui/input";11import { Field, Hint, Label } from "@/components/ui/label";12import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";13import { Switch } from "@/components/ui/switch";1415const ANY = "__any__";16const FORMATS = ["markdown", "text", "html"] as const;1718function splitPatterns(s: string): string[] | undefined {19 const list = s20 .split(/[\n,]+/)21 .map((x) => x.trim())22 .filter(Boolean)23 .slice(0, 50);24 return list.length ? list : undefined;25}2627export function CreateCrawlDialog({ defaultCountry, maxPages, variant = "primary" }: { defaultCountry?: string | null; maxPages: number; variant?: "primary" | "outline" }) {28 const router = useRouter();29 const [open, setOpen] = React.useState(false);30 const [pending, startTransition] = React.useTransition();31 const [error, setError] = React.useState<string | null>(null);3233 const [url, setUrl] = React.useState("");34 const [label, setLabel] = React.useState("");35 const [maxPagesValue, setMaxPagesValue] = React.useState(25);36 const [maxDepth, setMaxDepth] = React.useState(2);37 const [format, setFormat] = React.useState<(typeof FORMATS)[number]>("markdown");38 const [sameDomain, setSameDomain] = React.useState(true);39 const [respectRobots, setRespectRobots] = React.useState(true);40 const [browser, setBrowser] = React.useState(false);41 const [country, setCountry] = React.useState<string>(defaultCountry && defaultCountry in COUNTRIES ? defaultCountry : ANY);42 const [include, setInclude] = React.useState("");43 const [exclude, setExclude] = React.useState("");4445 function reset() {46 setError(null);47 setUrl("");48 setLabel("");49 setMaxPagesValue(25);50 setMaxDepth(2);51 setFormat("markdown");52 setSameDomain(true);53 setRespectRobots(true);54 setBrowser(false);55 setInclude("");56 setExclude("");57 }5859 function submit(e: React.FormEvent) {60 e.preventDefault();61 setError(null);62 startTransition(async () => {63 const res = await createCrawl({64 url: url.trim(),65 label: label.trim() || undefined,66 max_pages: Math.min(maxPages, Math.max(1, Math.round(maxPagesValue || 1))),67 max_depth: Math.min(10, Math.max(0, Math.round(maxDepth || 0))),68 format,69 same_domain: sameDomain,70 respect_robots: respectRobots,71 browser,72 country: country === ANY ? undefined : country,73 include_patterns: splitPatterns(include),74 exclude_patterns: splitPatterns(exclude),75 });76 if (!res.ok) {77 setError(res.error.message);78 return;79 }80 setOpen(false);81 reset();82 router.push(`/dashboard/crawls/${res.data.id}`);83 router.refresh();84 });85 }8687 return (88 <Dialog89 open={open}90 onOpenChange={(o) => {91 setOpen(o);92 if (!o) setError(null);93 }}94 >95 <DialogTrigger asChild>96 <Button variant={variant} size="sm">97 <Plus /> New crawl98 </Button>99 </DialogTrigger>100 <DialogContent size="lg">101 <form onSubmit={submit} className="contents">102 <DialogHeader>103 <DialogTitle>Start a crawl</DialogTitle>104 <DialogDescription>105 Fetcha follows links from the seed URL, fetches each page through the routing engine and stores the content. Every page is a normal request in the log with source <code className="font-mono">crawl</code>.106 </DialogDescription>107 </DialogHeader>108 {error ? <Alert variant="danger">{error}</Alert> : null}109 <div className="grid gap-4 sm:grid-cols-2">110 <Field className="sm:col-span-2">111 <Label htmlFor="c-url">Seed URL</Label>112 <Input id="c-url" type="url" inputMode="url" required value={url} onChange={(e) => setUrl(e.target.value)} placeholder="https://docs.example.com/" autoComplete="off" autoCapitalize="off" spellCheck={false} className="font-mono text-[13px]" />113 </Field>114 <Field>115 <Label htmlFor="c-max-pages">Max pages</Label>116 <Input id="c-max-pages" type="number" inputMode="numeric" min={1} max={maxPages} value={maxPagesValue} onChange={(e) => setMaxPagesValue(Number(e.target.value))} className="font-mono tabular" />117 <Hint>1–{maxPages.toLocaleString("en-US")}. The seed counts as one.</Hint>118 </Field>119 <Field>120 <Label htmlFor="c-max-depth">Max depth</Label>121 <Input id="c-max-depth" type="number" inputMode="numeric" min={0} max={10} value={maxDepth} onChange={(e) => setMaxDepth(Number(e.target.value))} className="font-mono tabular" />122 <Hint>0 = seed only, up to 10 link hops.</Hint>123 </Field>124 <Field>125 <Label htmlFor="c-format">Content format</Label>126 <Select value={format} onValueChange={(v) => setFormat(v as (typeof FORMATS)[number])}>127 <SelectTrigger id="c-format">128 <SelectValue />129 </SelectTrigger>130 <SelectContent>131 {FORMATS.map((f) => (132 <SelectItem key={f} value={f} className="font-mono text-[13px]">133 {f}134 </SelectItem>135 ))}136 </SelectContent>137 </Select>138 </Field>139 <Field>140 <Label htmlFor="c-country">Country</Label>141 <Select value={country} onValueChange={setCountry}>142 <SelectTrigger id="c-country">143 <SelectValue />144 </SelectTrigger>145 <SelectContent>146 <SelectItem value={ANY}>Any country</SelectItem>147 {Object.entries(COUNTRIES).map(([code, name]) => (148 <SelectItem key={code} value={code}>149 <span className="font-mono text-[12px] text-fg-subtle">{code}</span> {name}150 </SelectItem>151 ))}152 </SelectContent>153 </Select>154 </Field>155 <Field>156 <Label htmlFor="c-include">Include patterns</Label>157 <Textarea id="c-include" value={include} onChange={(e) => setInclude(e.target.value)} placeholder={"/docs/*\n/regex/"} rows={3} spellCheck={false} className="font-mono text-[12.5px]" />158 <Hint>One per line. Glob with <span className="font-mono">*</span> or <span className="font-mono">/regex/</span>. Empty = everything.</Hint>159 </Field>160 <Field>161 <Label htmlFor="c-exclude">Exclude patterns</Label>162 <Textarea id="c-exclude" value={exclude} onChange={(e) => setExclude(e.target.value)} placeholder={"*/login*\n*.pdf"} rows={3} spellCheck={false} className="font-mono text-[12.5px]" />163 <Hint>URLs matching any pattern are never crawled.</Hint>164 </Field>165 <div className="grid gap-3 rounded-md border border-border bg-bg-subtle/50 p-3 sm:col-span-2">166 <ToggleRow id="c-same-domain" label="Stay on the seed domain" hint="Only follow links on the same registrable host." checked={sameDomain} onChange={setSameDomain} />167 <ToggleRow id="c-robots" label="Respect robots.txt" hint="Skip paths disallowed for the seed host." checked={respectRobots} onChange={setRespectRobots} />168 <ToggleRow id="c-browser" label="Browser rendering" hint="Render every page in the managed browser. Slower; use for JavaScript-only sites. Blocked pages escalate automatically either way." checked={browser} onChange={setBrowser} />169 </div>170 <Field className="sm:col-span-2">171 <Label htmlFor="c-label">Label (optional)</Label>172 <Input id="c-label" value={label} onChange={(e) => setLabel(e.target.value)} placeholder="docs-site-weekly" maxLength={128} autoComplete="off" />173 </Field>174 </div>175 <DialogFooter>176 <DialogClose asChild>177 <Button type="button" variant="ghost">178 Cancel179 </Button>180 </DialogClose>181 <Button type="submit" variant="primary" loading={pending}>182 Start crawl183 </Button>184 </DialogFooter>185 </form>186 </DialogContent>187 </Dialog>188 );189}190191function ToggleRow({ id, label, hint, checked, onChange }: { id: string; label: string; hint?: string; checked: boolean; onChange: (v: boolean) => void }) {192 return (193 <div className="flex items-center justify-between gap-3">194 <div className="min-w-0">195 <Label htmlFor={id}>{label}</Label>196 {hint ? <Hint className="mt-1">{hint}</Hint> : null}197 </div>198 <Switch id={id} checked={checked} onCheckedChange={onChange} aria-label={label} />199 </div>200 );201}202