diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index c2f3ef5870..fd6a90a588 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -84,6 +84,7 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.row-group-limit` | Number of rows | 1048576 | The upper bound of the number of entries within a single row group | | `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk | | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | +| `write.parquet.page-index-enabled` | Boolean | True | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation. Set to false to disable | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 56dbdf5c20..ffc34413e8 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3209,6 +3209,11 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: kwargs: dict[str, Any] = { "compression": compression_codec, "compression_level": compression_level, + "write_page_index": property_as_bool( + properties=table_properties, + property_name=TableProperties.PARQUET_PAGE_INDEX_ENABLED, + default=TableProperties.PARQUET_PAGE_INDEX_ENABLED_DEFAULT, + ), "data_page_size": property_as_int( properties=table_properties, property_name=TableProperties.PARQUET_PAGE_SIZE_BYTES, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 84925fd025..f2b7fe25ec 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -164,6 +164,9 @@ class TableProperties: PARQUET_PAGE_ROW_LIMIT = "write.parquet.page-row-limit" PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20000 + PARQUET_PAGE_INDEX_ENABLED = "write.parquet.page-index-enabled" + PARQUET_PAGE_INDEX_ENABLED_DEFAULT = True + PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes" PARQUET_DICT_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024 # 2 MB diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index e0d647c2e9..3f2885bcdb 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1653,18 +1653,18 @@ def test_merge_manifests_file_content(session_catalog: Catalog, arrow_table_with for i in range(3): tbl_a_data_file = tbl_a_entries["data_file"][i] assert tbl_a_data_file["column_sizes"] == [ - (1, 51), - (2, 80), - (3, 130), - (4, 96), - (5, 120), - (6, 96), - (7, 120), - (8, 120), - (9, 120), - (10, 96), - (11, 80), - (12, 111), + (1, 33), + (2, 68), + (3, 76), + (4, 66), + (5, 74), + (6, 66), + (7, 74), + (8, 74), + (9, 74), + (10, 66), + (11, 68), + (12, 69), ] assert tbl_a_data_file["content"] == 0 assert tbl_a_data_file["equality_ids"] is None diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index ca21fd05f0..f3f47db13e 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3713,6 +3713,23 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None: assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2] +def test_append_writes_parquet_page_index(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="id", field_type=LongType(), required=False)) + catalog.create_namespace("default") + + for page_index_enabled, identifier in [ + (True, "default.test_append_page_index_enabled"), + (False, "default.test_append_page_index_disabled"), + ]: + properties = {} if page_index_enabled else {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "false"} + table = catalog.create_table(identifier, schema=schema, properties=properties) + table.append(pa.table({"id": pa.array(range(1000), type=pa.int64())})) + + file_path = next(iter(table.scan().plan_files())).file.file_path + column_chunk = pq.read_metadata(file_path).row_group(0).column(0) + assert column_chunk.has_column_index is page_index_enabled + assert column_chunk.has_offset_index is page_index_enabled + def test_append_counts_nan_values(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="f", field_type=FloatType(), required=False),