HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1import type { Metadata } from 'next';2import Link from 'next/link';3import { Dash, IntelListing, list, str } from '@/components/intelligence/listing-table';4import { Chip } from '@/components/ui/badges';5import { EntityLink, QualityMark } from '@/components/ui/entity';6import { api, safe } from '@/lib/api';7import { fmtInt, fmtValue } from '@/lib/format';8import { routes, SITE_NAME, SITE_URL } from '@/lib/site';910export const metadata: Metadata = { title: 'AI datasets — publisher, modality, size, licence, access, models trained on them', description: 'Training and evaluation datasets in the atlas with publisher, modality, size, licence and access as stated by their hosts, plus how many models the graph records as trained on each and the papers that describe them.', alternates: { canonical: routes.datasets() }, openGraph: { title: `AI datasets | ${SITE_NAME}`, url: `${SITE_URL}${routes.datasets()}`, siteName: SITE_NAME } };11export const revalidate = 300;1213type Extra = { trained: number; papers: number; access: string | null };1415export default async function DatasetsPage({ searchParams }: { searchParams: Promise<Record<string, string | undefined>> }) {16 const sp = await searchParams;17 return (18 <IntelListing<Extra>19 title="Datasets"20 eyebrow="Datasets"21 lede="Training and evaluation datasets referenced by models and benchmarks. Modality, size, licence and access are read from dataset cards and hosting pages; the “used to train” count comes from the graph (uses_dataset relations)."22 basePath="/datasets"23 searchParams={sp}24 fetch={(q) => api.explore('dataset', q)}25 sorts={[26 { value: 'updated', label: 'Recently updated' },27 { value: 'first_seen', label: 'Recently added' },28 { value: 'name', label: 'Name' },29 { value: 'quality', label: 'Data quality' },30 ]}31 filters={sp.org ? [{ kind: 'hidden', name: 'org', value: sp.org }] : []}32 connectors={['huggingface (dataset cards)', 'arxiv (papers naming datasets)', 'model cards']}33 enrich={async (items) => {34 const details = await Promise.all(items.map((e) => safe(api.entity(e.slug))));35 const m = new Map<string, Extra>();36 details.forEach((d, i) => {37 if (!d) return;38 const trained = (d.relations ?? []).filter((g) => g.predicate === 'uses_dataset' && g.direction === 'in').reduce((n, g) => n + (g.total || g.items.length), 0);39 const papers = (d.papers?.length ?? 0) + (d.relations ?? []).filter((g) => g.predicate === 'described_by' && g.direction === 'out').reduce((n, g) => n + (g.total || g.items.length), 0);40 const a = d.attributes ?? {};41 const access = typeof a.access === 'string' ? a.access : a.gated === true ? 'gated' : a.gated === false ? 'public' : null;42 m.set(items[i]!.slug, { trained, papers, access });43 });44 return m;45 }}46 columns={[47 {48 key: 'name',49 label: 'Dataset',50 primary: true,51 render: (e) => (52 <>53 <EntityLink e={e} />54 {e.description && <span className="block max-w-md truncate text-xs text-ink-3">{e.description}</span>}55 </>56 ),57 },58 { key: 'publisher', label: 'Publisher', className: 'text-ink-2', render: (e) => (e.organization ? <Link href={routes.entity({ entity_type: 'company', slug: e.organization.slug })} className="hover:text-accent">{e.organization.name}</Link> : str(e.attributes?.publisher) ?? <Dash />) },59 {60 key: 'modality',61 label: 'Modality',62 render: (e) => {63 const m = str(e.attributes?.modality) ? [str(e.attributes.modality) as string] : list(e.attributes?.modalities);64 return m.length ? <span className="flex flex-wrap gap-1">{m.slice(0, 3).map((x) => <Chip key={x}>{x}</Chip>)}</span> : <Dash />;65 },66 },67 { key: 'size', label: 'Size', className: 'tnum text-ink-2', render: (e) => (e.attributes?.size ? fmtValue(e.attributes.size, 'size') : <Dash />) },68 { key: 'license', label: 'Licence', className: 'max-w-[10rem] truncate text-ink-2', render: (e) => str(e.attributes?.license) ?? <Dash /> },69 { key: 'access', label: 'Access', className: 'text-ink-2', render: (e, x) => (x?.access ? <Chip tone={x.access === 'gated' ? 'accent' : 'neutral'}>{x.access}</Chip> : str(e.attributes?.access) ?? <Dash />) },70 { key: 'trained', label: 'Used to train', num: true, className: 'tnum', render: (_e, x) => (x ? (x.trained ? fmtInt(x.trained) : <span className="text-ink-3">0</span>) : <Dash />) },71 { key: 'papers', label: 'Papers', num: true, className: 'tnum', render: (_e, x) => (x ? (x.papers ? fmtInt(x.papers) : <span className="text-ink-3">0</span>) : <Dash />) },72 { key: 'quality', label: 'Quality', num: true, render: (e) => <QualityMark q={e.quality?.score} /> },73 ]}74 note="Many datasets are known only by reference (a model card names them) and carry no attributes yet — the dash means the sources have not stated it. “Used to train” counts inbound uses_dataset relations; “Papers” counts papers linked to the dataset."75 />76 );77}78