spb/tools-spboucher-ai

1{2  "aliases": [3    "Parquet"4  ],5  "workflows": [6    "Archive minute-bar or tick data partitioned by year and symbol",7    "Store ETL outputs for DuckDB, Polars or Spark consumers",8    "Exchange datasets between Python, R and JVM tooling",9    "Underlying files for Iceberg or Delta Lake tables"10  ],11  "advantages": [12    "Compact storage and fast column scans",13    "Supported by virtually all analytical engines",14    "Statistics enable data skipping",15    "Vendor-neutral Apache 2.0 specification"16  ],17  "limitations": [18    "Immutable files; updates require rewrites or a table format",19    "Many small files degrade performance",20    "Feature support varies between implementations",21    "Not suited to row-by-row transactional access"22  ],23  "prerequisites": [24    "A library or engine with a Parquet reader (PyArrow, DuckDB, Polars, Spark…)"25  ],26  "audiences": [27    "Data engineers",28    "Quantitative and empirical researchers",29    "Analysts working on data lakes"30  ],31  "alternatives": [32    "apache-arrow",33    "apache-iceberg",34    "delta-lake",35    "duckdb",36    "sqlite"37  ],38  "tags": [39    "parquet",40    "columnar",41    "data-lake",42    "open-source",43    "apache-arrow"44  ],45  "use_cases": [46    "query-parquet-files",47    "analyze-intraday-data",48    "archive-datasets"49  ],50  "operating_systems": [51    "any"52  ],53  "deployment_models": [54    "embedded"55  ],56  "integrations": [57    "Apache Arrow",58    "DuckDB",59    "Polars",60    "pandas",61    "Apache Spark",62    "Trino",63    "Apache Iceberg",64    "Delta Lake",65    "ClickHouse"66  ],67  "data_formats": [68    "Parquet"69  ],70  "languages": [71    "C++",72    "Java",73    "Rust",74    "Python",75    "Go",76    "R"77  ],78  "self_hostable": true,79  "discovery_method": "editorial-research",80  "depth": "deep",81  "editorial_status": "reviewed",82  "visibility": "public",83  "featured": false,84  "author": "batch-D",85  "slug": "apache-parquet",86  "name": "Apache Parquet",87  "vendor": {88    "name": "Apache Software Foundation",89    "slug": "apache-software-foundation",90    "website": "https://www.apache.org/"91  },92  "homepage_url": "https://parquet.apache.org/",93  "docs_url": "https://parquet.apache.org/docs/",94  "repo_url": "https://github.com/apache/parquet-format",95  "license_url": "https://github.com/apache/parquet-format/blob/master/LICENSE",96  "product_type": "specification",97  "product_status": "active",98  "summary": "Open columnar file format for analytical data with compression, encodings and statistics, readable by nearly every data engine and library",99  "description": "Apache Parquet is a columnar storage file format specified by an Apache Software Foundation project and implemented in C++ (via Apache Arrow), Java (parquet-java), Rust (via arrow-rs), Go and other languages. A Parquet file organizes rows into row groups and, within each, stores every column contiguously as pages with per-column encodings (dictionary, run-length, delta, byte-stream split) and compression codecs (Snappy, Zstandard, gzip, LZ4, Brotli). The footer carries the schema plus min/max statistics, and optional page indexes and Bloom filters let readers skip row groups and pages that cannot match a predicate. Nested data is represented with the Dremel record-shredding model, so lists and structs are first-class.\n\nParquet is the default storage layer for data lakes and lakehouses: Spark, Trino, DuckDB, Polars, pandas (through PyArrow or fastparquet), DataFusion, ClickHouse, BigQuery, Snowflake and Athena all read and write it, and table formats such as Apache Iceberg and Delta Lake manage collections of Parquet files. For a research group it is the natural archive for tick or minute-bar histories, filings extractions and panel datasets: columns compress well, partitioned directories allow year/month pruning, and a single laptop can query hundreds of gigabytes directly.\n\nThe format is write-once: files are immutable and appends or updates mean rewriting files or using a table format on top. Small files and tiny row groups hurt performance, schema evolution needs care across engines, and features such as the newer encodings, page indexes or geospatial logical types are not supported uniformly by every reader, so pinning writer settings for compatibility is common practice. The specification is Apache 2.0.",100  "capabilities": [101    "Columnar layout with row groups, column chunks and pages",102    "Encodings: dictionary, RLE/bit-packing, delta, byte-stream split",103    "Compression codecs including Snappy, Zstandard, gzip and LZ4",104    "Column statistics, page indexes and Bloom filters for predicate pushdown",105    "Nested types via the Dremel shredding model",106    "Logical types for timestamps, decimals, UUIDs and more",107    "Implementations in C++, Java, Rust, Go, Python and others"108  ],109  "comparison_notes": "Apache Arrow is the in-memory counterpart designed for fast conversion to and from Parquet; Arrow IPC files can also be used on disk but with less compression. Iceberg and Delta Lake are table formats layered over Parquet files that add transactions, schema evolution and time travel. DuckDB and SQLite are databases with their own file formats; DuckDB reads Parquet natively and is often used as the query layer over Parquet archives.",110  "categories": [111    {112      "slug": "data-catalogs",113      "primary": true114    },115    {116      "slug": "analytics-engines"117    },118    {119      "slug": "hf-finance-research",120      "label": "Standard archive format for intraday bars"121    }122  ],123  "execution_modes": [124    "local"125  ],126  "interfaces": [127    "library"128  ],129  "source_model": "open-source",130  "license_spdx": "Apache-2.0",131  "pricing_model": [132    "open-source",133    "free"134  ],135  "sources": [136    {137      "url": "https://parquet.apache.org/",138      "type": "official_site",139      "supports": [140        "name",141        "summary"142      ],143      "checked_at": "2026-11-02",144      "title": "Apache Parquet"145    },146    {147      "url": "https://parquet.apache.org/docs/",148      "type": "official_docs",149      "supports": [150        "capabilities"151      ],152      "checked_at": "2026-11-02",153      "title": "Parquet documentation"154    },155    {156      "url": "https://parquet.apache.org/docs/file-format/",157      "type": "official_docs",158      "supports": [159        "capabilities",160        "product_type"161      ],162      "checked_at": "2026-11-02",163      "title": "Parquet file format specification"164    },165    {166      "url": "https://github.com/apache/parquet-format/blob/master/LICENSE",167      "type": "license_page",168      "supports": [169        "license_spdx",170        "source_model"171      ],172      "checked_at": "2026-11-02",173      "title": "parquet-format LICENSE (Apache 2.0)"174    },175    {176      "url": "https://github.com/apache/parquet-format",177      "type": "repository",178      "supports": [179        "repo_url"180      ],181      "checked_at": "2026-11-02",182      "title": "apache/parquet-format"183    }184  ],185  "confidence": "high"186}187