import type { Metadata } from 'next'; import Link from 'next/link'; import { Dash, IntelListing, list, str } from '@/components/intelligence/listing-table'; import { Chip } from '@/components/ui/badges'; import { EntityLink, QualityMark } from '@/components/ui/entity'; import { api, safe } from '@/lib/api'; import { fmtInt, fmtValue } from '@/lib/format'; import { routes, SITE_NAME, SITE_URL } from '@/lib/site'; export const metadata: Metadata = { title: 'AI datasets — publisher, modality, size, licence, access, models trained on them', description: 'Training and evaluation datasets in the atlas with publisher, modality, size, licence and access as stated by their hosts, plus how many models the graph records as trained on each and the papers that describe them.', alternates: { canonical: routes.datasets() }, openGraph: { title: `AI datasets | ${SITE_NAME}`, url: `${SITE_URL}${routes.datasets()}`, siteName: SITE_NAME } }; export const revalidate = 300; type Extra = { trained: number; papers: number; access: string | null }; export default async function DatasetsPage({ searchParams }: { searchParams: Promise> }) { const sp = await searchParams; return ( title="Datasets" eyebrow="Datasets" lede="Training and evaluation datasets referenced by models and benchmarks. Modality, size, licence and access are read from dataset cards and hosting pages; the “used to train” count comes from the graph (uses_dataset relations)." basePath="/datasets" searchParams={sp} fetch={(q) => api.explore('dataset', q)} sorts={[ { value: 'updated', label: 'Recently updated' }, { value: 'first_seen', label: 'Recently added' }, { value: 'name', label: 'Name' }, { value: 'quality', label: 'Data quality' }, ]} filters={sp.org ? [{ kind: 'hidden', name: 'org', value: sp.org }] : []} connectors={['huggingface (dataset cards)', 'arxiv (papers naming datasets)', 'model cards']} enrich={async (items) => { const details = await Promise.all(items.map((e) => safe(api.entity(e.slug)))); const m = new Map(); details.forEach((d, i) => { if (!d) return; const trained = (d.relations ?? []).filter((g) => g.predicate === 'uses_dataset' && g.direction === 'in').reduce((n, g) => n + (g.total || g.items.length), 0); const papers = (d.papers?.length ?? 0) + (d.relations ?? []).filter((g) => g.predicate === 'described_by' && g.direction === 'out').reduce((n, g) => n + (g.total || g.items.length), 0); const a = d.attributes ?? {}; const access = typeof a.access === 'string' ? a.access : a.gated === true ? 'gated' : a.gated === false ? 'public' : null; m.set(items[i]!.slug, { trained, papers, access }); }); return m; }} columns={[ { key: 'name', label: 'Dataset', primary: true, render: (e) => ( <> {e.description && {e.description}} ), }, { key: 'publisher', label: 'Publisher', className: 'text-ink-2', render: (e) => (e.organization ? {e.organization.name} : str(e.attributes?.publisher) ?? ) }, { key: 'modality', label: 'Modality', render: (e) => { const m = str(e.attributes?.modality) ? [str(e.attributes.modality) as string] : list(e.attributes?.modalities); return m.length ? {m.slice(0, 3).map((x) => {x})} : ; }, }, { key: 'size', label: 'Size', className: 'tnum text-ink-2', render: (e) => (e.attributes?.size ? fmtValue(e.attributes.size, 'size') : ) }, { key: 'license', label: 'Licence', className: 'max-w-[10rem] truncate text-ink-2', render: (e) => str(e.attributes?.license) ?? }, { key: 'access', label: 'Access', className: 'text-ink-2', render: (e, x) => (x?.access ? {x.access} : str(e.attributes?.access) ?? ) }, { key: 'trained', label: 'Used to train', num: true, className: 'tnum', render: (_e, x) => (x ? (x.trained ? fmtInt(x.trained) : 0) : ) }, { key: 'papers', label: 'Papers', num: true, className: 'tnum', render: (_e, x) => (x ? (x.papers ? fmtInt(x.papers) : 0) : ) }, { key: 'quality', label: 'Quality', num: true, render: (e) => }, ]} note="Many datasets are known only by reference (a model card names them) and carry no attributes yet — the dash means the sources have not stated it. “Used to train” counts inbound uses_dataset relations; “Papers” counts papers linked to the dataset." /> ); }