spb/tools-spboucher-ai

1{2  "slug": "common-crawl",3  "name": "Common Crawl",4  "aliases": ["CC-MAIN", "Common Crawl Foundation"],5  "vendor": { "name": "Common Crawl Foundation", "slug": "common-crawl", "website": "https://commoncrawl.org/" },6  "homepage_url": "https://commoncrawl.org/",7  "docs_url": "https://commoncrawl.org/get-started",8  "repo_url": "https://github.com/commoncrawl",9  "product_type": "data-service",10  "product_status": "active",11  "summary": "Nonprofit open repository of petabyte-scale web crawl data since 2008, published as WARC, WAT and WET files on AWS Open Data with URL and columnar indexes",12  "description": "Common Crawl is a nonprofit foundation (501(c)(3), founded in 2007) that crawls the web and publishes the results as an open corpus. Crawls are released several times a year, each labeled by period (for example CC-MAIN-2026-43), and each contains raw page captures in WARC files, metadata extracts (WAT) and plain-text extracts (WET). The data is hosted free of charge on Amazon Web Services' open data program and mirrored on some academic clouds; users process it in place with cloud tools or download selected segments.\n\nTo find pages, Common Crawl provides a URL index served through a CDX-style API and a columnar index in Parquet that can be queried with engines such as Athena, DuckDB or Spark. The corpus has become a foundational source for training language models and for research in web science, linguistics and information retrieval.\n\nAccess is free, subject to terms of use that place responsibility for third-party content and downstream uses, including AI training, on the user. Practical limits: each crawl is a sample rather than a complete snapshot, boilerplate and spam are abundant, a growing number of sites opt out, and working with the full dataset requires serious storage and compute.",13  "capabilities": ["Periodic web crawls in WARC, WAT and WET formats", "Free hosting on AWS Open Data", "URL index with CDX-style API", "Columnar Parquet index for SQL querying", "Open-source tooling on GitHub"],14  "workflows": ["Query the columnar index with DuckDB or Athena to select domains", "Extract text for a language corpus from WET files", "Study link structure or technology adoption across crawls"],15  "advantages": ["Free and openly licensed access", "Long time series since 2008", "Standard archive formats and indexes"],16  "limitations": ["Each crawl is a sample, not a full snapshot", "Noisy content requiring heavy filtering", "Large-scale processing costs fall on the user", "Opt-outs reduce coverage over time"],17  "prerequisites": ["Cloud or local compute able to read WARC/Parquet at scale"],18  "audiences": ["NLP and web-science researchers", "Data engineers building corpora", "Search and archive practitioners"],19  "alternatives": ["wayback-machine", "apify", "bright-data"],20  "categories": [{ "slug": "web-archiving", "primary": true }, { "slug": "open-public-data" }],21  "tags": ["open-data", "web-archive", "warc", "parquet", "nonprofit", "llm", "research"],22  "use_cases": ["build-text-corpora", "query-parquet-files"],23  "operating_systems": ["web"],24  "execution_modes": ["cloud"],25  "interfaces": ["api", "web"],26  "deployment_models": ["saas"],27  "integrations": ["Amazon S3 / Athena", "DuckDB", "Apache Spark", "cdx-toolkit"],28  "data_formats": ["WARC", "WAT", "WET", "CDXJ", "Parquet"],29  "languages": [],30  "source_model": "mixed",31  "pricing_model": ["free"],32  "self_hostable": false,33  "sources": [34    { "url": "https://commoncrawl.org/", "type": "official_site", "title": "Common Crawl", "supports": ["name", "summary", "vendor"], "checked_at": "2026-11-02", "note": "Homepage states 501(c)(3) non-profit founded in 2007" },35    { "url": "https://commoncrawl.org/overview", "type": "official_docs", "title": "Common Crawl – Overview", "supports": ["capabilities", "data_formats", "description"], "checked_at": "2026-11-02" },36    { "url": "https://commoncrawl.org/get-started", "type": "official_docs", "title": "Get Started", "supports": ["capabilities", "integrations"], "checked_at": "2026-11-02" },37    { "url": "https://commoncrawl.org/terms-of-use", "type": "policy_page", "title": "Terms of Use", "supports": ["pricing_model", "limitations"], "checked_at": "2026-11-02" }38  ],39  "confidence": "high",40  "discovery_method": "editorial-research",41  "depth": "standard",42  "editorial_status": "reviewed",43  "visibility": "public",44  "featured": false,45  "author": "batch-C"46}47