spb/tools-spboucher-ai

1{2  "slug": "crawlee",3  "name": "Crawlee",4  "aliases": ["Apify SDK (legacy name)", "crawlee-python"],5  "vendor": { "name": "Apify Technologies s.r.o.", "slug": "apify", "website": "https://apify.com/" },6  "homepage_url": "https://crawlee.dev/",7  "docs_url": "https://crawlee.dev/js/docs/quick-start",8  "repo_url": "https://github.com/apify/crawlee",9  "license_url": "https://github.com/apify/crawlee/blob/master/LICENSE.md",10  "product_type": "library",11  "product_status": "active",12  "summary": "Open-source crawling and scraping library for Node.js/TypeScript and Python that unifies HTTP and headless-browser crawlers with queues, proxies, fingerprints and storage",13  "description": "Crawlee is Apify's open-source web crawling library, released under the Apache 2.0 license, available for Node.js and TypeScript and, since 2024, for Python. It provides a common crawler abstraction over different fetching strategies: plain HTTP crawlers with Cheerio or JSDOM parsing in JavaScript and BeautifulSoup or Parsel in Python, and browser crawlers driven by Playwright or Puppeteer, including an adaptive crawler that switches to a browser only when a page needs JavaScript.\n\nAround that core it supplies the infrastructure most scrapers end up rebuilding: a persistent request queue with deduplication, automatic scaling based on system resources, retries and session management, proxy rotation, realistic browser fingerprints, and datasets and key-value stores with pluggable backends (file system, memory, SQL, Redis). Crawlers run anywhere Node or Python runs and deploy to the Apify platform as Actors with no code changes.\n\nCrawlee suits developers who want Scrapy-like structure with first-class browser support and modern language tooling. The Python port is younger than the JavaScript library and has fewer integrations, and anti-bot evasion still depends on proxy quality.",14  "capabilities": ["Unified HTTP and browser crawler classes (Cheerio, JSDOM, Playwright, Puppeteer, BeautifulSoup, Parsel)", "Persistent request queue with deduplication and autoscaling", "Proxy rotation, sessions and browser fingerprinting", "Datasets and key-value stores with multiple storage backends", "Adaptive crawler that uses a browser only when needed", "One-step deployment to the Apify platform"],15  "workflows": ["Crawl a documentation site to Markdown with an HTTP crawler", "Scrape a React-rendered catalog with PlaywrightCrawler", "Package a crawler as an Apify Actor"],16  "advantages": ["Batteries included for queues, proxies and storage", "Same mental model in JavaScript and Python", "Apache 2.0 license and active development"],17  "limitations": ["Python port is newer with fewer integrations", "Browser crawling is resource-intensive", "Evasion depends on external proxies"],18  "prerequisites": ["Node.js 16 or newer, or Python 3.10 or newer"],19  "audiences": ["Developers building production scrapers", "Teams deploying to Apify"],20  "alternatives": ["scrapy", "puppeteer", "playwright", "apify"],21  "categories": [{ "slug": "data-collection-crawling", "primary": true }],22  "tags": ["crawler", "browser-automation", "typescript", "python", "nodejs", "open-source"],23  "use_cases": ["crawl-websites", "collect-web-data"],24  "operating_systems": ["macos", "linux", "windows"],25  "execution_modes": ["local", "self-hosted", "cloud"],26  "interfaces": ["library", "cli"],27  "deployment_models": ["on-premises", "saas"],28  "integrations": ["Playwright", "Puppeteer", "Cheerio", "BeautifulSoup", "Apify platform", "Redis", "SQL databases"],29  "data_formats": ["JSON", "CSV", "HTML"],30  "languages": ["TypeScript", "JavaScript", "Python"],31  "source_model": "open-source",32  "license_spdx": "Apache-2.0",33  "pricing_model": ["open-source", "free"],34  "self_hostable": true,35  "sources": [36    { "url": "https://crawlee.dev/", "type": "official_site", "title": "Crawlee", "supports": ["name", "summary", "capabilities"], "checked_at": "2026-11-02" },37    { "url": "https://crawlee.dev/python/", "type": "official_docs", "title": "Crawlee for Python", "supports": ["capabilities", "languages", "integrations"], "checked_at": "2026-11-02" },38    { "url": "https://github.com/apify/crawlee/blob/master/LICENSE.md", "type": "license_page", "supports": ["license_spdx", "source_model"], "checked_at": "2026-11-02" }39  ],40  "confidence": "high",41  "discovery_method": "editorial-research",42  "depth": "standard",43  "editorial_status": "reviewed",44  "visibility": "public",45  "featured": false,46  "author": "batch-C"47}48