import type { Metadata } from "next";
import Link from "next/link";
import { BookOpen, Network } from "lucide-react";
import { PLAN_LIMITS, normalizePlan } from "@fetcha/core";
import { getWorkspace } from "@/lib/session";
import { internalApi, InternalApiError, type CrawlJob } from "@/lib/api";
import { formatBytes, formatDate, formatNumber, timeAgo } from "@/lib/format";
import { API_PUBLIC_URL } from "@/lib/utils";
import { PageHeader } from "@/components/ui/page-header";
import { Alert } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { CodeBlock } from "@/components/ui/code-block";
import { EmptyState } from "@/components/ui/empty-state";
import { Stat, StatGrid } from "@/components/ui/stat";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { CrawlStatusBadge } from "@/components/dashboard/crawls/crawl-status-badge";
import { CreateCrawlDialog } from "@/components/dashboard/crawls/create-crawl-dialog";
import { CancelCrawlButton } from "@/components/dashboard/crawls/cancel-crawl-button";
import { MapTool } from "@/components/dashboard/crawls/map-tool";
export const dynamic = "force-dynamic";
export const metadata: Metadata = { title: "Crawls" };
const API_SNIPPET = `# 1. Start a crawl (returns 202 with the job)
curl -X POST ${API_PUBLIC_URL}/v1/crawl \\
-H "Authorization: Bearer fch_live_YOUR_KEY" \\
-H "Content-Type: application/json" \\
-d '{ "url": "https://docs.example.com/", "max_pages": 100, "max_depth": 3, "format": "markdown" }'
# → { "id": "crawl_…", "status": "queued", … }
# 2. Poll the job, then page through the results
curl ${API_PUBLIC_URL}/v1/crawl/crawl_… -H "Authorization: Bearer fch_live_YOUR_KEY"
curl "${API_PUBLIC_URL}/v1/crawl/crawl_…/pages?limit=100" -H "Authorization: Bearer fch_live_YOUR_KEY"`;
function seedLabel(job: CrawlJob): string {
try {
const u = new URL(job.seed_url);
return u.host + (u.pathname !== "/" ? u.pathname : "");
} catch {
return job.seed_url;
}
}
export default async function CrawlsPage() {
const ws = await getWorkspace();
const limits = PLAN_LIMITS[normalizePlan(ws.organization.plan)];
let jobs: CrawlJob[] = [];
let loadError: string | null = null;
try {
const res = await internalApi.listCrawls(ws.project.id, ws.user.id, 50);
jobs = Array.isArray(res?.data) ? res.data : [];
} catch (e) {
loadError = e instanceof InternalApiError ? e.message : "The Fetcha API service is unreachable.";
}
const active = jobs.filter((j) => j.status === "queued" || j.status === "running");
const pagesFetched = jobs.reduce((a, j) => a + (j.stats?.fetched ?? 0), 0);
const bytes = jobs.reduce((a, j) => a + (j.stats?.bytes ?? 0), 0);
const createButton =
Frontier. The seed is fetched first; links are extracted, normalised and filtered by same_domain, include_patterns / exclude_patterns and max_depth, then fetched with up to concurrency workers until max_pages is reached.
Politeness. robots.txt is honoured by default and delay_ms adds a pause between fetches per worker. Sitemaps can seed the frontier with use_sitemap.
Content. Each page is stored as Markdown (main content, boilerplate removed), text or HTML, with title, description, status, mode (HTTP or browser), bytes and duration. Blocked pages escalate to the managed browser automatically.
Every page also appears in Requests with source crawl. Jobs are limited to {formatNumber(limits.crawl_max_pages)} pages and {limits.crawl_concurrent_jobs} concurrent jobs per organization.