spb/tools-spboucher-ai

1{2  "slug": "dask",3  "name": "Dask",4  "aliases": ["dask.distributed"],5  "vendor": { "name": "Dask developers (open-source community)", "slug": "dask", "website": "https://www.dask.org/" },6  "homepage_url": "https://www.dask.org/",7  "docs_url": "https://docs.dask.org/en/stable/",8  "repo_url": "https://github.com/dask/dask",9  "license_url": "https://github.com/dask/dask/blob/main/LICENSE.txt",10  "product_type": "library",11  "product_status": "active",12  "summary": "Python library for parallel and distributed computing that scales pandas, NumPy and custom task graphs from a laptop to a cluster",13  "description": "Dask is an open-source Python library (BSD-3-Clause) for parallel computing that grew out of the PyData ecosystem. It offers parallel versions of familiar collections: Dask DataFrame partitions a pandas DataFrame across many pandas objects, Dask Array does the same for NumPy, Dask Bag handles unstructured records, and Delayed and Futures let arbitrary Python functions be composed into task graphs. A scheduler executes those graphs with threads or processes on one machine, or across a cluster through the distributed scheduler.\n\nDeployment options cover a local cluster in a Python session, HPC job queues such as Slurm, PBS and LSF via Dask-Jobqueue, Kubernetes via the Dask operator, and commercial cloud hosting through Coiled. Xarray, the labeled-array library widely used in climate and geoscience, is a major consumer of Dask arrays.\n\nDask suits analysts who already use pandas and NumPy and need larger-than-memory or multi-node execution with minimal code changes. Limits: DataFrame operations that need shuffles can be slow, not every pandas API is implemented, and debugging distributed task graphs takes practice.",14  "capabilities": ["Parallel DataFrame and Array collections mirroring pandas and NumPy", "Delayed and Futures for custom task graphs", "Single-machine and distributed schedulers", "Deployment on HPC job queues, Kubernetes and cloud", "Diagnostic dashboard for tasks and memory"],15  "workflows": ["Process a Parquet dataset larger than RAM with pandas syntax", "Fan a loop of simulations across a cluster with Delayed", "Run Dask workers inside a Slurm allocation"],16  "advantages": ["Familiar APIs for pandas and NumPy users", "Scales from one laptop to many nodes", "Permissive license and mature ecosystem"],17  "limitations": ["Shuffle-heavy operations can be slow", "Partial coverage of the pandas API", "Distributed debugging is non-trivial"],18  "prerequisites": ["Python 3.10 or newer"],19  "audiences": ["Data scientists outgrowing pandas", "Scientific computing groups on HPC clusters"],20  "alternatives": ["ray", "apache-spark", "polars", "pandas"],21  "categories": [{ "slug": "distributed-computing", "primary": true }, { "slug": "dataframes-arrays" }],22  "tags": ["python", "distributed-computing", "open-source", "parquet", "dataframes"],23  "use_cases": ["distribute-computation", "query-parquet-files"],24  "operating_systems": ["linux", "macos", "windows"],25  "execution_modes": ["local", "self-hosted", "cloud"],26  "interfaces": ["library", "web"],27  "deployment_models": ["on-premises", "managed-cloud"],28  "integrations": ["pandas", "NumPy", "Xarray", "Slurm (Dask-Jobqueue)", "Kubernetes", "Coiled"],29  "data_formats": ["Parquet", "CSV", "HDF5", "Zarr", "JSON"],30  "languages": ["Python"],31  "source_model": "open-source",32  "license_spdx": "BSD-3-Clause",33  "pricing_model": ["open-source", "free"],34  "self_hostable": true,35  "sources": [36    { "url": "https://www.dask.org/", "type": "official_site", "title": "Dask", "supports": ["name", "summary"], "checked_at": "2026-11-02" },37    { "url": "https://docs.dask.org/en/stable/", "type": "official_docs", "title": "Dask documentation", "supports": ["capabilities", "integrations", "description"], "checked_at": "2026-11-02" },38    { "url": "https://github.com/dask/dask/blob/main/LICENSE.txt", "type": "license_page", "supports": ["license_spdx", "source_model"], "checked_at": "2026-11-02" }39  ],40  "confidence": "high",41  "discovery_method": "editorial-research",42  "depth": "standard",43  "editorial_status": "reviewed",44  "visibility": "public",45  "featured": false,46  "author": "batch-C"47}48