spb/tools-spboucher-ai
Public
1{2 "aliases": [3 "py-polars"4 ],5 "workflows": [6 "Replace slow pandas ETL steps in a Python pipeline",7 "Aggregate minute-bar archives from Parquet with lazy scans",8 "Feature engineering before model training",9 "Rust services that need DataFrame operations"10 ],11 "advantages": [12 "Parallel execution without extra configuration",13 "Lazy optimization reduces I/O on large files",14 "MIT license",15 "Consistent, type-strict expression API"16 ],17 "limitations": [18 "API differs from pandas; migration requires rewriting",19 "Single-node unless using Polars Cloud",20 "Some libraries still expect pandas objects",21 "Strict typing requires explicit casts"22 ],23 "prerequisites": [24 "Python 3 or a Rust toolchain; optional cloud credentials for remote scans"25 ],26 "audiences": [27 "Data scientists and quantitative researchers",28 "Data engineers",29 "Rust developers"30 ],31 "alternatives": [32 "pandas",33 "duckdb",34 "apache-datafusion",35 "apache-spark",36 "dask"37 ],38 "tags": [39 "dataframes",40 "rust",41 "python",42 "apache-arrow",43 "parquet",44 "apple-silicon",45 "open-source"46 ],47 "use_cases": [48 "query-parquet-files",49 "analyze-intraday-data",50 "transform-data"51 ],52 "operating_systems": [53 "macos",54 "linux",55 "windows"56 ],57 "deployment_models": [58 "embedded",59 "managed-cloud"60 ],61 "integrations": [62 "Apache Arrow",63 "pandas",64 "NumPy",65 "DuckDB",66 "Jupyter",67 "marimo",68 "Amazon S3",69 "Delta Lake",70 "Apache Iceberg"71 ],72 "data_formats": [73 "Parquet",74 "CSV",75 "JSON",76 "NDJSON",77 "Arrow IPC",78 "Excel",79 "Avro",80 "Delta Lake",81 "Iceberg"82 ],83 "languages": [84 "Rust",85 "Python",86 "JavaScript",87 "R",88 "SQL"89 ],90 "self_hostable": true,91 "discovery_method": "editorial-research",92 "depth": "deep",93 "editorial_status": "reviewed",94 "visibility": "public",95 "featured": false,96 "author": "batch-D",97 "slug": "polars",98 "name": "Polars",99 "vendor": {100 "name": "Polars (Ritchie Vink and contributors)",101 "slug": "polars-project",102 "website": "https://pola.rs/"103 },104 "homepage_url": "https://pola.rs/",105 "docs_url": "https://docs.pola.rs/",106 "repo_url": "https://github.com/pola-rs/polars",107 "license_url": "https://github.com/pola-rs/polars/blob/main/LICENSE",108 "product_type": "library",109 "product_status": "active",110 "summary": "Rust-based DataFrame library with Python, Rust, Node and R bindings offering lazy query optimization, multithreading and Arrow-native memory",111 "description": "Polars is an open-source DataFrame library written in Rust with first-class bindings for Python and Rust and community or official bindings for Node.js, R and others. It stores columns in Apache Arrow memory, executes expressions in parallel across cores, and offers a lazy API in which a chain of operations is compiled into a query plan with predicate and projection pushdown before any data is read. The same expression syntax works eagerly for interactive use. A streaming engine processes datasets larger than memory in batches, and scan functions read Parquet, CSV, JSON, Arrow IPC, Excel and database tables directly, including partitioned Hive-style directories on S3 or other object stores.\n\nThe library is released under the MIT license and developed by the Polars company founded by its creator, which also offers Polars Cloud for running the same queries on remote or distributed compute. In practice Polars is adopted where pandas scripts have become slow or memory-bound: ETL steps, feature engineering, financial time-series work (group_by_dynamic for time buckets, asof joins) and preprocessing before modeling. Conversion to and from pandas, NumPy, PyArrow and DuckDB is cheap because of shared Arrow memory.\n\nCaveats: the API differs from pandas (no index, expression-based column operations), so porting code is a rewrite rather than a swap; some ecosystem libraries still expect pandas objects; and Polars is single-node unless Polars Cloud or another distributed layer is used. Its expression engine is also strict about data types, which exposes errors earlier but demands explicit casting.",112 "capabilities": [113 "Lazy query planning with predicate and projection pushdown",114 "Multithreaded, vectorized execution on Arrow memory",115 "Streaming mode for larger-than-memory data",116 "Scanners for Parquet, CSV, JSON, IPC, Excel, databases and cloud storage",117 "Time-series operations: dynamic group_by, rolling windows, asof joins",118 "SQL interface over DataFrames",119 "Interop with pandas, NumPy, PyArrow and DuckDB",120 "Bindings for Python, Rust, Node.js and R"121 ],122 "comparison_notes": "pandas is the incumbent Python DataFrame library with the largest ecosystem but single-threaded, eager execution and an index-centric API. DuckDB offers comparable single-node performance through SQL and interoperates with Polars via Arrow. DataFusion is the Rust query engine on which other tools build; Polars has its own engine. Spark and Dask distribute work across machines at the cost of cluster setup, which Polars avoids on a single large workstation.",123 "categories": [124 {125 "slug": "dataframes-arrays",126 "primary": true127 },128 {129 "slug": "analytics-engines"130 },131 {132 "slug": "hf-finance-research",133 "label": "Fast time-series aggregation on Apple Silicon"134 }135 ],136 "execution_modes": [137 "local",138 "cloud"139 ],140 "interfaces": [141 "library",142 "sdk"143 ],144 "source_model": "open-source",145 "license_spdx": "MIT",146 "pricing_model": [147 "open-source",148 "free",149 "usage-based"150 ],151 "sources": [152 {153 "url": "https://pola.rs/",154 "type": "official_site",155 "supports": [156 "name",157 "summary",158 "vendor"159 ],160 "checked_at": "2026-11-02",161 "title": "Polars"162 },163 {164 "url": "https://docs.pola.rs/",165 "type": "official_docs",166 "supports": [167 "capabilities",168 "data_formats",169 "languages"170 ],171 "checked_at": "2026-11-02",172 "title": "Polars user guide"173 },174 {175 "url": "https://github.com/pola-rs/polars/blob/main/LICENSE",176 "type": "license_page",177 "supports": [178 "license_spdx",179 "source_model"180 ],181 "checked_at": "2026-11-02",182 "title": "Polars LICENSE (MIT)"183 },184 {185 "url": "https://github.com/pola-rs/polars",186 "type": "repository",187 "supports": [188 "repo_url"189 ],190 "checked_at": "2026-11-02",191 "title": "pola-rs/polars"192 },193 {194 "url": "https://cloud.pola.rs/",195 "type": "official_site",196 "supports": [197 "pricing_model",198 "execution_modes"199 ],200 "checked_at": "2026-11-02",201 "title": "Polars Cloud",202 "note": "Managed/distributed execution offering; pricing details not reviewed"203 }204 ],205 "confidence": "high"206}207