spb/tools-spboucher-ai

1{2  "slug": "scrapy",3  "name": "Scrapy",4  "aliases": [],5  "vendor": { "name": "Scrapy developers (maintained with Zyte)", "slug": "scrapy", "website": "https://scrapy.org/" },6  "homepage_url": "https://scrapy.org/",7  "docs_url": "https://docs.scrapy.org/en/latest/",8  "repo_url": "https://github.com/scrapy/scrapy",9  "license_url": "https://github.com/scrapy/scrapy/blob/master/LICENSE",10  "product_type": "framework",11  "product_status": "active",12  "summary": "Mature open-source Python framework for writing crawlers (spiders) that fetch pages asynchronously, extract structured items and export them through pipelines",13  "description": "Scrapy is an open-source Python framework for crawling websites and extracting structured data, in development since 2008 and released under the BSD 3-Clause license. Its development has long been associated with the company now called Zyte, which also offers hosted Scrapy Cloud, but the framework itself is community-governed on GitHub.\n\nA Scrapy project defines spiders, classes that declare start URLs and parse callbacks; the engine schedules requests through an asynchronous downloader built on Twisted, applies downloader and spider middlewares (for headers, retries, proxies, cookies, throttling), and routes extracted items through item pipelines for validation, deduplication and storage. Selectors use XPath and CSS, item loaders normalize fields, and feed exports write JSON, JSON Lines, CSV or XML to local files or cloud storage. Built-in features include auto-throttle, robots.txt handling, caching, link extraction, a shell for interactive testing and a stats collector. Extensions add Playwright integration for JavaScript-heavy sites and deployment via Scrapyd or Scrapy Cloud.\n\nScrapy suits data engineers and researchers who need repeatable, large-scale crawls of static or server-rendered sites with clean separation between fetching, parsing and storage. Current releases require Python 3.10 or newer.\n\nLimits: it does not render JavaScript without an add-on, its Twisted-based architecture and project structure impose a learning curve, and anti-bot evasion, proxy pools and distributed crawling are left to extensions or commercial services.",14  "capabilities": ["Asynchronous crawling engine with request scheduling", "Spiders with XPath and CSS selectors and item loaders", "Downloader and spider middlewares for retries, proxies and throttling", "Item pipelines for cleaning and storage", "Feed exports to JSON, CSV, XML and cloud storage", "Interactive shell, caching and crawl statistics", "Deployment through Scrapyd and Scrapy Cloud"],15  "workflows": ["Crawl a listings site nightly and export JSON Lines", "Build a focused crawler for a research corpus", "Add Playwright to a spider for client-rendered pages", "Schedule spiders on Scrapyd or Scrapy Cloud"],16  "advantages": ["Long-lived, well-documented framework", "Clear architecture separating fetch, parse and store", "Permissive BSD license", "Rich extension ecosystem"],17  "limitations": ["No JavaScript rendering without extensions", "Learning curve from Twisted and project structure", "Anti-bot and proxy handling not included", "Python only"],18  "prerequisites": ["Python 3.10 or newer"],19  "audiences": ["Data engineers building crawl pipelines", "Researchers collecting web corpora", "Developers of focused crawlers"],20  "alternatives": ["crawlee", "puppeteer", "playwright", "apify"],21  "comparison_notes": "Crawlee is the nearest modern equivalent, available for Node.js and Python, with browser crawlers, proxy rotation and fingerprinting built in; Scrapy is older, HTTP-first and more widely deployed in Python shops. Puppeteer and Playwright are browser-automation libraries rather than crawl frameworks, usually combined with Scrapy or Crawlee for JavaScript-rendered pages. Apify is a hosted platform where Scrapy spiders can be packaged as Actors, adding scheduling, storage and proxies.",22  "categories": [{ "slug": "data-collection-crawling", "primary": true }],23  "tags": ["crawler", "python", "open-source", "scraping", "xpath", "cli"],24  "use_cases": ["crawl-websites", "collect-web-data"],25  "operating_systems": ["macos", "linux", "windows"],26  "execution_modes": ["local", "self-hosted", "cloud"],27  "interfaces": ["library", "cli"],28  "deployment_models": ["on-premises", "managed-cloud"],29  "integrations": ["Scrapyd", "Zyte Scrapy Cloud", "Playwright (scrapy-playwright)", "Zyte API", "Amazon S3", "Google Cloud Storage"],30  "data_formats": ["JSON", "JSON Lines", "CSV", "XML", "HTML"],31  "languages": ["Python"],32  "source_model": "open-source",33  "license_spdx": "BSD-3-Clause",34  "pricing_model": ["open-source", "free"],35  "self_hostable": true,36  "sources": [37    { "url": "https://scrapy.org/", "type": "official_site", "title": "Scrapy | A Fast and Powerful Scraping and Web Crawling Framework", "supports": ["name", "summary"], "checked_at": "2026-11-02" },38    { "url": "https://docs.scrapy.org/en/latest/", "type": "official_docs", "title": "Scrapy documentation", "supports": ["capabilities", "description", "data_formats"], "checked_at": "2026-11-02" },39    { "url": "https://github.com/scrapy/scrapy/blob/master/LICENSE", "type": "license_page", "supports": ["license_spdx", "source_model"], "checked_at": "2026-11-02" },40    { "url": "https://pypi.org/project/Scrapy/", "type": "package_registry", "title": "Scrapy on PyPI", "supports": ["prerequisites", "product_status"], "checked_at": "2026-11-02" }41  ],42  "confidence": "high",43  "discovery_method": "editorial-research",44  "depth": "deep",45  "editorial_status": "reviewed",46  "visibility": "public",47  "featured": false,48  "author": "batch-C"49}50