From df5082ff17509fe78ddc030a3f6a26ba5e92ca58 Mon Sep 17 00:00:00 2001 From: linzch3 <20380442+linzch3@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:18:59 +0000 Subject: [PATCH] Use field IDs when any Parquet column has one _HasIds previously required every column to have an ID. A file with IDs on only some columns, such as a Databricks UniForm table with row-tracking columns, fell back to name mapping and failed when physical names differed from the mapped names. Match Java's ParquetSchemaUtil.hasIds (any) and prune fields without IDs, including incomplete nested types. --- pyiceberg/io/pyarrow.py | 58 ++++++++++++++++++------------ tests/io/test_pyarrow.py | 6 ++-- tests/io/test_pyarrow_visitor.py | 62 +++++++++++++++++++++++++++++--- 3 files changed, 98 insertions(+), 28 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a15fe19e71..a3eb6ca10d 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1268,9 +1268,11 @@ def pyarrow_to_schema( ) -> Schema: has_ids = visit_pyarrow(schema, _HasIds()) if has_ids: - return visit_pyarrow( + result = visit_pyarrow( schema, _ConvertToIceberg(downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version) ) + assert isinstance(result, Schema) + return result elif name_mapping is not None: schema_without_ids = _pyarrow_to_schema_without_ids( schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version @@ -1287,10 +1289,12 @@ def _pyarrow_to_schema_without_ids( downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, ) -> Schema: - return visit_pyarrow( + result = visit_pyarrow( schema, _ConvertToIcebergWithoutIDs(downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version), ) + assert isinstance(result, Schema) + return result def _pyarrow_schema_ensure_large_types(schema: pa.Schema) -> pa.Schema: @@ -1448,32 +1452,39 @@ def _get_field_id(field: pa.Field) -> int | None: class _HasIds(PyArrowSchemaVisitor[bool]): + """Return whether any field in the Arrow schema has an Iceberg field ID. + + Matches Java's ParquetSchemaUtil.hasIds, which is true if any column has an ID. + Fields without IDs are later pruned by _ConvertToIceberg. + """ + def schema(self, schema: pa.Schema, struct_result: bool) -> bool: return struct_result def struct(self, struct: pa.StructType, field_results: builtins.list[bool]) -> bool: - return all(field_results) + return any(field_results) def field(self, field: pa.Field, field_result: bool) -> bool: - return all([_get_field_id(field) is not None, field_result]) + return _get_field_id(field) is not None or field_result def list(self, list_type: pa.ListType, element_result: bool) -> bool: element_field = list_type.value_field element_id = _get_field_id(element_field) - return element_result and element_id is not None + return element_result or element_id is not None def map(self, map_type: pa.MapType, key_result: bool, value_result: bool) -> bool: key_field = map_type.key_field key_id = _get_field_id(key_field) value_field = map_type.item_field value_id = _get_field_id(value_field) - return all([key_id is not None, value_id is not None, key_result, value_result]) + return any([key_id is not None, value_id is not None, key_result, value_result]) def primitive(self, primitive: pa.DataType) -> bool: - return True + # IDs live on pa.Field metadata, not on primitive types + return False -class _ConvertToIceberg(PyArrowSchemaVisitor[IcebergType | Schema]): +class _ConvertToIceberg(PyArrowSchemaVisitor[IcebergType | Schema | None]): """Converts PyArrowSchema to Iceberg Schema.""" _field_names: builtins.list[str] @@ -1485,34 +1496,37 @@ def __init__( self._downcast_ns_timestamp_to_us = downcast_ns_timestamp_to_us self._format_version = format_version - def _field_id(self, field: pa.Field) -> int: - if (field_id := _get_field_id(field)) is not None: - return field_id - else: - raise ValueError(f"Cannot convert {field} to Iceberg Field as field_id is empty.") + def _field_id(self, field: pa.Field) -> int | None: + return _get_field_id(field) - def schema(self, schema: pa.Schema, struct_result: StructType) -> Schema: - return Schema(*struct_result.fields) + def schema(self, schema: pa.Schema, struct_result: StructType | None) -> Schema: + return Schema(*(struct_result.fields if struct_result is not None else ())) - def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField]) -> StructType: - return StructType(*field_results) + def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField | None]) -> StructType | None: + fields = tuple(field for field in field_results if field is not None) + return StructType(*fields) if fields else None - def field(self, field: pa.Field, field_result: IcebergType) -> NestedField: + def field(self, field: pa.Field, field_result: IcebergType | None) -> NestedField | None: field_id = self._field_id(field) + if field_id is None or field_result is None: + return None field_doc = doc_str.decode() if (field.metadata and (doc_str := field.metadata.get(PYARROW_FIELD_DOC_KEY))) else None - field_type = field_result - return NestedField(field_id, field.name, field_type, required=not field.nullable, doc=field_doc) + return NestedField(field_id, field.name, field_result, required=not field.nullable, doc=field_doc) - def list(self, list_type: pa.ListType, element_result: IcebergType) -> ListType: + def list(self, list_type: pa.ListType, element_result: IcebergType | None) -> ListType | None: element_field = list_type.value_field element_id = self._field_id(element_field) + if element_id is None or element_result is None: + return None return ListType(element_id, element_result, element_required=not element_field.nullable) - def map(self, map_type: pa.MapType, key_result: IcebergType, value_result: IcebergType) -> MapType: + def map(self, map_type: pa.MapType, key_result: IcebergType | None, value_result: IcebergType | None) -> MapType | None: key_field = map_type.key_field key_id = self._field_id(key_field) value_field = map_type.item_field value_id = self._field_id(value_field) + if key_id is None or value_id is None or key_result is None or value_result is None: + return None return MapType(key_id, key_result, value_id, value_result, value_required=not value_field.nullable) def primitive(self, primitive: pa.DataType) -> PrimitiveType: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9c418e50a..489b3e765e 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3255,10 +3255,11 @@ def test_schema_compatible_missing_nullable_field_nested(table_schema_nested: Sc "person", pa.struct( [ - pa.field("age", pa.int32(), nullable=False), + pa.field("age", pa.int32(), nullable=False, metadata={"PARQUET:field_id": "17"}), ] ), nullable=True, + metadata={"PARQUET:field_id": "15"}, ), ) try: @@ -3276,10 +3277,11 @@ def test_schema_mismatch_missing_required_field_nested(table_schema_nested: Sche "person", pa.struct( [ - pa.field("name", pa.string(), nullable=True), + pa.field("name", pa.string(), nullable=True, metadata={"PARQUET:field_id": "16"}), ] ), nullable=True, + metadata={"PARQUET:field_id": "15"}, ), ) expected = """Mismatch in fields: diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index e98d76e262..9d42634f8d 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -377,7 +377,7 @@ def test_simple_schema_has_missing_ids() -> None: assert not has_ids -def test_simple_schema_has_missing_ids_partial() -> None: +def test_simple_schema_has_partial_ids() -> None: schema = pa.schema( [ pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), @@ -386,7 +386,7 @@ def test_simple_schema_has_missing_ids_partial() -> None: ) visitor = _HasIds() has_ids = visit_pyarrow(schema, visitor) - assert not has_ids + assert has_ids def test_nested_schema_has_missing_ids() -> None: @@ -436,7 +436,7 @@ def test_nested_schema_has_ids() -> None: assert has_ids -def test_nested_schema_has_partial_missing_ids() -> None: +def test_nested_schema_has_partial_ids() -> None: schema = pa.schema( [ pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), @@ -457,7 +457,61 @@ def test_nested_schema_has_partial_missing_ids() -> None: ) visitor = _HasIds() has_ids = visit_pyarrow(schema, visitor) - assert not has_ids + assert has_ids + + +def test_pyarrow_to_schema_prunes_fields_without_ids() -> None: + # A Databricks UniForm file may attach IDs only to data columns, leaving + # row-tracking columns unannotated. Previously _HasIds required every + # column to have an ID and fell back to name mapping, which failed when + # physical names differed from the mapped names. + file_schema = pa.schema( + [ + pa.field("col-2b93b371", pa.string(), metadata={"PARQUET:field_id": "1"}), + pa.field("_row-id-col-922d6355", pa.int64()), + ] + ) + expected = Schema(NestedField(1, "col-2b93b371", StringType(), required=False)) + name_mappings = [None, NameMapping([MappedField(field_id=1, names=["store_id"])])] + for name_mapping in name_mappings: + assert pyarrow_to_schema(file_schema, name_mapping) == expected + + +def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None: + schema = pa.schema( + [ + pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), + pa.field( + "quux", + pa.map_( + pa.field("key", pa.string(), nullable=False, metadata={"PARQUET:field_id": "7"}), + pa.field( + "value", + pa.map_(pa.field("key", pa.string(), nullable=False), pa.field("value", pa.int32())), + nullable=False, + ), + ), + nullable=False, + metadata={"PARQUET:field_id": "6", "doc": "quux doc"}, + ), + pa.field( + "person", + pa.struct( + [ + pa.field("name", pa.string(), metadata={"PARQUET:field_id": "16"}), + pa.field("age", pa.int32()), + ] + ), + metadata={"PARQUET:field_id": "15"}, + ), + ] + ) + actual = str(pyarrow_to_schema(schema)) + expected = """table { + 1: foo: required string (foo doc) + 15: person: optional struct<16: name: optional string> +}""" + assert actual == expected def test_pyarrow_schema_to_schema_missing_ids_and_name_mapping(pyarrow_schema_simple_without_ids: pa.Schema) -> None: