spb/tools-spboucher-ai

1{2  "slug": "duckdb",3  "name": "DuckDB",4  "aliases": ["Duck DB"],5  "vendor": { "name": "DuckDB Foundation / DuckDB Labs", "slug": "duckdb-foundation", "website": "https://duckdb.org" },6  "homepage_url": "https://duckdb.org/",7  "docs_url": "https://duckdb.org/docs/",8  "repo_url": "https://github.com/duckdb/duckdb",9  "license_url": "https://github.com/duckdb/duckdb/blob/main/LICENSE",10  "product_type": "database",11  "product_status": "active",12  "summary": "In-process analytical SQL database that queries Parquet, CSV and Arrow data directly from Python, R, Node or the CLI",13  "description": "DuckDB is an open-source, in-process analytical database developed by DuckDB Labs and stewarded by the DuckDB Foundation. Like SQLite it runs inside the host process with no server to install, but it is built for analytical (OLAP) workloads: a columnar, vectorized execution engine processes large scans and aggregations quickly on a single machine.\n\nIts most common use is as a query engine for files. DuckDB reads and writes Parquet, CSV and JSON directly, including globbed directories of partitioned files on local disk or object storage, and it exchanges data with Apache Arrow, pandas and Polars without copies. Client APIs exist for Python, R, Java, Node.js, Rust, Go and C/C++, and a standalone CLI ships for macOS, Linux and Windows. Extensions add spatial functions, full-text search, Iceberg and Delta table reading, and remote HTTP/S3 access.\n\nIn a research or finance workflow DuckDB typically sits between a Parquet data lake and a notebook or application: one SQL statement can filter a multi-year minute-bar archive, join it with a reference table and hand back an Arrow table for modelling. It is also a convenient embedded store for medium-sized applications that need SQL but not a database server.\n\nLimits are those of a single-node embedded engine: there is no built-in multi-user server, concurrent writers to the same file are restricted, and datasets much larger than memory rely on spilling to disk, so very large or highly concurrent workloads still call for a warehouse or distributed engine. Major-version upgrades can change the on-disk storage format, so pinning versions in pipelines is advisable.",14  "capabilities": ["Vectorized columnar SQL execution in-process", "Direct querying of Parquet, CSV and JSON files and directories", "Zero-copy exchange with Apache Arrow, pandas and Polars", "Extensions: spatial, full-text search, Iceberg/Delta, httpfs (S3)", "Clients for Python, R, Node.js, Java, Rust, Go, C/C++ and a CLI", "Persistent single-file databases and read-only attachments"],15  "workflows": ["Query a Parquet data lake from a notebook", "Replace ad-hoc pandas transformations with SQL", "Embed an analytical store inside a desktop or server application", "Convert and partition files between CSV and Parquet"],16  "advantages": ["No server or configuration to manage", "Excellent single-node performance on analytical scans", "Permissive MIT license", "First-class Arrow interoperability"],17  "limitations": ["Single-node; no distributed execution", "Limited concurrent write access to one database file", "Storage format can change between major versions", "Not designed for high-rate transactional workloads"],18  "prerequisites": ["A host language runtime (Python, R, Node.js…) or the CLI binary"],19  "audiences": ["Data scientists and quantitative researchers", "Backend developers embedding analytics", "Data engineers building file-based pipelines"],20  "alternatives": ["sqlite", "clickhouse", "polars", "apache-datafusion"],21  "comparison_notes": "SQLite is the closest architectural relative (embedded, single file) but is row-oriented and tuned for transactional access; DuckDB targets analytical scans. ClickHouse offers a comparable columnar engine as a networked server with clustering. Polars covers much of the same ground as a DataFrame library rather than through SQL, and the two interoperate through Arrow.",22  "categories": [{ "slug": "analytics-engines", "primary": true }, { "slug": "relational-databases" }, { "slug": "hf-finance-research", "label": "Common engine for Parquet tick/bar archives" }],23  "tags": ["olap", "embedded-database", "parquet", "sql", "apache-arrow", "columnar"],24  "use_cases": ["query-parquet-files", "analyze-intraday-data", "embed-analytics"],25  "operating_systems": ["macos", "linux", "windows"],26  "execution_modes": ["local", "self-hosted"],27  "interfaces": ["cli", "library", "sdk"],28  "deployment_models": ["embedded"],29  "integrations": ["Apache Arrow", "pandas", "Polars", "Jupyter", "dbt", "Apache Iceberg", "Delta Lake", "Amazon S3"],30  "data_formats": ["Parquet", "CSV", "JSON", "Arrow IPC", "Excel (extension)"],31  "languages": ["Python", "R", "JavaScript", "Java", "Rust", "Go", "C", "C++", "SQL"],32  "source_model": "open-source",33  "license_spdx": "MIT",34  "pricing_model": ["open-source", "free"],35  "self_hostable": true,36  "sources": [37    { "url": "https://duckdb.org/", "type": "official_site", "title": "DuckDB – An in-process SQL OLAP database management system", "supports": ["name", "summary", "product_type"], "checked_at": "2026-11-02" },38    { "url": "https://duckdb.org/docs/", "type": "official_docs", "title": "DuckDB Documentation", "supports": ["capabilities", "interfaces", "languages", "data_formats"], "checked_at": "2026-11-02" },39    { "url": "https://github.com/duckdb/duckdb/blob/main/LICENSE", "type": "license_page", "supports": ["license_spdx", "source_model"], "checked_at": "2026-11-02" }40  ],41  "confidence": "high",42  "discovery_method": "editorial-research",43  "depth": "deep",44  "editorial_status": "reviewed",45  "visibility": "public",46  "featured": true,47  "author": "editorial"48}49