From 420a7687e7b8af62cd60c9e84852d5b5ea9ac7e7 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 14:36:55 +0530 Subject: [PATCH 1/6] Add an opt-in property to write Parquet page indexes Readers that support page indexes can skip non-matching pages during predicate evaluation, but PyIceberg never wrote them. Add the write.parquet.page-index-enabled table property, off by default so the current output and file sizes are preserved, and pass it to the Parquet writer. --- mkdocs/docs/configuration.md | 1 + pyiceberg/io/pyarrow.py | 5 +++++ pyiceberg/table/__init__.py | 3 +++ tests/io/test_pyarrow.py | 18 ++++++++++++++++++ 4 files changed, 27 insertions(+) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 14cb9c7467..3997824cd7 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -84,6 +84,7 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.row-group-limit` | Number of rows | 1048576 | The upper bound of the number of entries within a single row group | | `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk | | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | +| `write.parquet.page-index-enabled` | Boolean | False | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..dc5b13c75f 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3160,6 +3160,11 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: kwargs: dict[str, Any] = { "compression": compression_codec, "compression_level": compression_level, + "write_page_index": property_as_bool( + properties=table_properties, + property_name=TableProperties.PARQUET_PAGE_INDEX_ENABLED, + default=TableProperties.PARQUET_PAGE_INDEX_ENABLED_DEFAULT, + ), "data_page_size": property_as_int( properties=table_properties, property_name=TableProperties.PARQUET_PAGE_SIZE_BYTES, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..045fc87081 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -163,6 +163,9 @@ class TableProperties: PARQUET_PAGE_ROW_LIMIT = "write.parquet.page-row-limit" PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20000 + PARQUET_PAGE_INDEX_ENABLED = "write.parquet.page-index-enabled" + PARQUET_PAGE_INDEX_ENABLED_DEFAULT = False + PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes" PARQUET_DICT_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024 # 2 MB diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..821bc2133f 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3596,6 +3596,24 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None: assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2] +def test_append_writes_parquet_page_index(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="id", field_type=LongType(), required=False)) + catalog.create_namespace("default") + + for page_index_enabled, identifier in [ + (False, "default.test_append_page_index_disabled"), + (True, "default.test_append_page_index_enabled"), + ]: + properties = {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "true"} if page_index_enabled else {} + table = catalog.create_table(identifier, schema=schema, properties=properties) + table.append(pa.table({"id": pa.array(range(1000), type=pa.int64())})) + + file_path = next(iter(table.scan().plan_files())).file.file_path + column_chunk = pq.read_metadata(file_path).row_group(0).column(0) + assert column_chunk.has_column_index is page_index_enabled + assert column_chunk.has_offset_index is page_index_enabled + + def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="foo", field_type=StringType(), required=True), From d172ab9de3d898a6699adc7e30984afe61a03e42 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 20:04:05 +0530 Subject: [PATCH 2/6] feat(pyarrow): write parquet page indexes by default --- mkdocs/docs/configuration.md | 2 +- pyiceberg/table/__init__.py | 2 +- tests/io/test_pyarrow.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 3997824cd7..5cee62a6fb 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -84,7 +84,7 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.row-group-limit` | Number of rows | 1048576 | The upper bound of the number of entries within a single row group | | `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk | | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | -| `write.parquet.page-index-enabled` | Boolean | False | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation | +| `write.parquet.page-index-enabled` | Boolean | True | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation. Set to false to disable | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 045fc87081..b81e83d3d4 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -164,7 +164,7 @@ class TableProperties: PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20000 PARQUET_PAGE_INDEX_ENABLED = "write.parquet.page-index-enabled" - PARQUET_PAGE_INDEX_ENABLED_DEFAULT = False + PARQUET_PAGE_INDEX_ENABLED_DEFAULT = True PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes" PARQUET_DICT_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024 # 2 MB diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 821bc2133f..a7058615cc 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3601,10 +3601,10 @@ def test_append_writes_parquet_page_index(catalog: InMemoryCatalog) -> None: catalog.create_namespace("default") for page_index_enabled, identifier in [ - (False, "default.test_append_page_index_disabled"), (True, "default.test_append_page_index_enabled"), + (False, "default.test_append_page_index_disabled"), ]: - properties = {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "true"} if page_index_enabled else {} + properties = {} if page_index_enabled else {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "false"} table = catalog.create_table(identifier, schema=schema, properties=properties) table.append(pa.table({"id": pa.array(range(1000), type=pa.int64())})) From a70fff7a509c72a65d33b6e9e3f2a359d70bd590 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 23:27:14 +0530 Subject: [PATCH 3/6] Revert "feat(pyarrow): write parquet page indexes by default" This reverts commit d172ab9de3d898a6699adc7e30984afe61a03e42. --- mkdocs/docs/configuration.md | 2 +- pyiceberg/table/__init__.py | 2 +- tests/io/test_pyarrow.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 5cee62a6fb..3997824cd7 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -84,7 +84,7 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.row-group-limit` | Number of rows | 1048576 | The upper bound of the number of entries within a single row group | | `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk | | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | -| `write.parquet.page-index-enabled` | Boolean | True | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation. Set to false to disable | +| `write.parquet.page-index-enabled` | Boolean | False | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index b81e83d3d4..045fc87081 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -164,7 +164,7 @@ class TableProperties: PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20000 PARQUET_PAGE_INDEX_ENABLED = "write.parquet.page-index-enabled" - PARQUET_PAGE_INDEX_ENABLED_DEFAULT = True + PARQUET_PAGE_INDEX_ENABLED_DEFAULT = False PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes" PARQUET_DICT_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024 # 2 MB diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a7058615cc..821bc2133f 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3601,10 +3601,10 @@ def test_append_writes_parquet_page_index(catalog: InMemoryCatalog) -> None: catalog.create_namespace("default") for page_index_enabled, identifier in [ - (True, "default.test_append_page_index_enabled"), (False, "default.test_append_page_index_disabled"), + (True, "default.test_append_page_index_enabled"), ]: - properties = {} if page_index_enabled else {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "false"} + properties = {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "true"} if page_index_enabled else {} table = catalog.create_table(identifier, schema=schema, properties=properties) table.append(pa.table({"id": pa.array(range(1000), type=pa.int64())})) From 4fe9775c4fa3406ea37b696417b8fa29b10fcf30 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 00:04:39 +0530 Subject: [PATCH 4/6] feat(pyarrow): write parquet page indexes by default --- mkdocs/docs/configuration.md | 2 +- pyiceberg/table/__init__.py | 2 +- tests/io/test_pyarrow.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 3997824cd7..5cee62a6fb 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -84,7 +84,7 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.row-group-limit` | Number of rows | 1048576 | The upper bound of the number of entries within a single row group | | `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk | | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | -| `write.parquet.page-index-enabled` | Boolean | False | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation | +| `write.parquet.page-index-enabled` | Boolean | True | Write Parquet column and offset indexes so readers can skip non-matching pages during predicate evaluation. Set to false to disable | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 045fc87081..b81e83d3d4 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -164,7 +164,7 @@ class TableProperties: PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20000 PARQUET_PAGE_INDEX_ENABLED = "write.parquet.page-index-enabled" - PARQUET_PAGE_INDEX_ENABLED_DEFAULT = False + PARQUET_PAGE_INDEX_ENABLED_DEFAULT = True PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes" PARQUET_DICT_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024 # 2 MB diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 821bc2133f..a7058615cc 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3601,10 +3601,10 @@ def test_append_writes_parquet_page_index(catalog: InMemoryCatalog) -> None: catalog.create_namespace("default") for page_index_enabled, identifier in [ - (False, "default.test_append_page_index_disabled"), (True, "default.test_append_page_index_enabled"), + (False, "default.test_append_page_index_disabled"), ]: - properties = {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "true"} if page_index_enabled else {} + properties = {} if page_index_enabled else {TableProperties.PARQUET_PAGE_INDEX_ENABLED: "false"} table = catalog.create_table(identifier, schema=schema, properties=properties) table.append(pa.table({"id": pa.array(range(1000), type=pa.int64())})) From eb774781e90e02e9c158f7b76a90131283582a48 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 00:05:29 +0530 Subject: [PATCH 5/6] test(integration): update column sizes for parquet page indexes --- tests/integration/test_writes/test_writes.py | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index fe720440fb..fb666f259a 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1653,18 +1653,18 @@ def test_merge_manifests_file_content(session_catalog: Catalog, arrow_table_with for i in range(3): tbl_a_data_file = tbl_a_entries["data_file"][i] assert tbl_a_data_file["column_sizes"] == [ - (1, 51), - (2, 80), - (3, 130), - (4, 96), - (5, 120), - (6, 96), - (7, 120), - (8, 120), - (9, 120), - (10, 96), - (11, 80), - (12, 111), + (1, 33), + (2, 68), + (3, 76), + (4, 66), + (5, 74), + (6, 66), + (7, 74), + (8, 74), + (9, 74), + (10, 66), + (11, 68), + (12, 69), ] assert tbl_a_data_file["content"] == 0 assert tbl_a_data_file["equality_ids"] is None From 17157da063331f6fd37282651a983858942d1a44 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 18 Nov 2026 19:41:40 +0100 Subject: [PATCH 6/6] Remove whitespace --- tests/io/test_pyarrow.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9241f53ba..f3f47db13e 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3730,7 +3730,6 @@ def test_append_writes_parquet_page_index(catalog: InMemoryCatalog) -> None: assert column_chunk.has_column_index is page_index_enabled assert column_chunk.has_offset_index is page_index_enabled - def test_append_counts_nan_values(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="f", field_type=FloatType(), required=False),