{
  "slug": "common-crawl",
  "name": "Common Crawl",
  "aliases": ["CC-MAIN", "Common Crawl Foundation"],
  "vendor": { "name": "Common Crawl Foundation", "slug": "common-crawl", "website": "https://commoncrawl.org/" },
  "homepage_url": "https://commoncrawl.org/",
  "docs_url": "https://commoncrawl.org/get-started",
  "repo_url": "https://github.com/commoncrawl",
  "product_type": "data-service",
  "product_status": "active",
  "summary": "Nonprofit open repository of petabyte-scale web crawl data since 2008, published as WARC, WAT and WET files on AWS Open Data with URL and columnar indexes",
  "description": "Common Crawl is a nonprofit foundation (501(c)(3), founded in 2007) that crawls the web and publishes the results as an open corpus. Crawls are released several times a year, each labeled by period (for example CC-MAIN-2026-43), and each contains raw page captures in WARC files, metadata extracts (WAT) and plain-text extracts (WET). The data is hosted free of charge on Amazon Web Services' open data program and mirrored on some academic clouds; users process it in place with cloud tools or download selected segments.\n\nTo find pages, Common Crawl provides a URL index served through a CDX-style API and a columnar index in Parquet that can be queried with engines such as Athena, DuckDB or Spark. The corpus has become a foundational source for training language models and for research in web science, linguistics and information retrieval.\n\nAccess is free, subject to terms of use that place responsibility for third-party content and downstream uses, including AI training, on the user. Practical limits: each crawl is a sample rather than a complete snapshot, boilerplate and spam are abundant, a growing number of sites opt out, and working with the full dataset requires serious storage and compute.",
  "capabilities": ["Periodic web crawls in WARC, WAT and WET formats", "Free hosting on AWS Open Data", "URL index with CDX-style API", "Columnar Parquet index for SQL querying", "Open-source tooling on GitHub"],
  "workflows": ["Query the columnar index with DuckDB or Athena to select domains", "Extract text for a language corpus from WET files", "Study link structure or technology adoption across crawls"],
  "advantages": ["Free and openly licensed access", "Long time series since 2008", "Standard archive formats and indexes"],
  "limitations": ["Each crawl is a sample, not a full snapshot", "Noisy content requiring heavy filtering", "Large-scale processing costs fall on the user", "Opt-outs reduce coverage over time"],
  "prerequisites": ["Cloud or local compute able to read WARC/Parquet at scale"],
  "audiences": ["NLP and web-science researchers", "Data engineers building corpora", "Search and archive practitioners"],
  "alternatives": ["wayback-machine", "apify", "bright-data"],
  "categories": [{ "slug": "web-archiving", "primary": true }, { "slug": "open-public-data" }],
  "tags": ["open-data", "web-archive", "warc", "parquet", "nonprofit", "llm", "research"],
  "use_cases": ["build-text-corpora", "query-parquet-files"],
  "operating_systems": ["web"],
  "execution_modes": ["cloud"],
  "interfaces": ["api", "web"],
  "deployment_models": ["saas"],
  "integrations": ["Amazon S3 / Athena", "DuckDB", "Apache Spark", "cdx-toolkit"],
  "data_formats": ["WARC", "WAT", "WET", "CDXJ", "Parquet"],
  "languages": [],
  "source_model": "mixed",
  "pricing_model": ["free"],
  "self_hostable": false,
  "sources": [
    { "url": "https://commoncrawl.org/", "type": "official_site", "title": "Common Crawl", "supports": ["name", "summary", "vendor"], "checked_at": "2026-11-02", "note": "Homepage states 501(c)(3) non-profit founded in 2007" },
    { "url": "https://commoncrawl.org/overview", "type": "official_docs", "title": "Common Crawl – Overview", "supports": ["capabilities", "data_formats", "description"], "checked_at": "2026-11-02" },
    { "url": "https://commoncrawl.org/get-started", "type": "official_docs", "title": "Get Started", "supports": ["capabilities", "integrations"], "checked_at": "2026-11-02" },
    { "url": "https://commoncrawl.org/terms-of-use", "type": "policy_page", "title": "Terms of Use", "supports": ["pricing_model", "limitations"], "checked_at": "2026-11-02" }
  ],
  "confidence": "high",
  "discovery_method": "editorial-research",
  "depth": "standard",
  "editorial_status": "reviewed",
  "visibility": "public",
  "featured": false,
  "author": "batch-C"
}