Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 35 additions & 4 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2760,6 +2760,25 @@ def _nan_value_counts_from_arrow(table: pa.Table, stats_columns: dict[int, Stati
return nan_value_counts


def _file_parquet_path_to_id_mapping(parquet_metadata: pq.FileMetaData, file_schema: Schema) -> dict[str, int]:
"""
Compute the mapping of the parquet column paths of a file to the Iceberg IDs of its columns.

Unlike `parquet_path_to_id_mapping`, the paths are the ones stored in the file, so a list element keeps
the name that the file gives it. The IDs come from the schema of the file, so a column keeps its ID
when the table has renamed it.

Args:
parquet_metadata (pyarrow.parquet.FileMetaData): The metadata of the parquet file.
file_schema (pyiceberg.schema.Schema): The Iceberg schema of the file, as returned by
`_check_pyarrow_schema_compatible`.
"""
# The leaf fields of the file schema are in the same depth-first order as the parquet columns.
field_ids = [pair.field_id for pair in pre_order_visit(file_schema, ID2ParquetPathVisitor())]
paths = [parquet_metadata.schema.column(pos).path for pos in range(parquet_metadata.num_columns)]
return dict(zip(paths, field_ids, strict=True))


def data_file_statistics_from_parquet_metadata(
parquet_metadata: pq.FileMetaData,
stats_columns: dict[int, StatisticsCollector],
Expand Down Expand Up @@ -3115,12 +3134,15 @@ def _check_pyarrow_schema_compatible(
provided_schema: pa.Schema,
downcast_ns_timestamp_to_us: bool = False,
format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION,
) -> None:
) -> Schema:
"""
Check if the `requested_schema` is compatible with `provided_schema`.

Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type.

Returns:
The Iceberg schema of `provided_schema`, with the field IDs of its columns.

Raises:
ValueError: If the schemas are not compatible.
"""
Expand All @@ -3143,6 +3165,7 @@ def _check_pyarrow_schema_compatible(
"Update the schema first (hint, use union_by_name)."
) from e
_check_schema_compatible(requested_schema, provided_schema)
return provided_schema


def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_paths: Iterator[str]) -> Iterator[DataFile]:
Expand All @@ -3159,12 +3182,20 @@ def parquet_file_to_data_file(io: FileIO, table_metadata: TableMetadata, file_pa
arrow_schema = parquet_metadata.schema.to_arrow_schema()

schema = table_metadata.schema()
_check_pyarrow_schema_compatible(schema, arrow_schema, format_version=table_metadata.format_version)
file_schema = _check_pyarrow_schema_compatible(schema, arrow_schema, format_version=table_metadata.format_version)

stats_columns = compute_statistics_plan(schema, table_metadata.properties)
parquet_column_mapping = _file_parquet_path_to_id_mapping(parquet_metadata, file_schema)
if additional_paths := sorted(path for path, field_id in parquet_column_mapping.items() if field_id not in stats_columns):
raise ValueError(
f"PyArrow table contains more columns: {', '.join(additional_paths)}. "
"Update the schema first (hint, use union_by_name)."
)

statistics = data_file_statistics_from_parquet_metadata(
parquet_metadata=parquet_metadata,
stats_columns=compute_statistics_plan(schema, table_metadata.properties),
parquet_column_mapping=parquet_path_to_id_mapping(schema),
stats_columns=stats_columns,
parquet_column_mapping=parquet_column_mapping,
)
data_file = DataFile.from_args(
content=DataFileContent.DATA,
Expand Down
117 changes: 117 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3379,6 +3379,123 @@ def test_schema_downcast(table_schema_simple: Schema) -> None:
pytest.fail("Unexpected Exception raised when calling `_check_pyarrow_schema_compatible`")


def test_add_files_matches_columns_by_field_id(catalog: InMemoryCatalog, tmp_path: str) -> None:
# Each file is valid for its table, but its column paths differ from the table's names.
nested_schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(
2,
"address",
StructType(
NestedField(3, "street", StringType(), required=False),
NestedField(4, "zip", LongType(), required=False),
),
required=False,
),
)
list_schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "tags", ListType(3, StringType(), element_required=False), required=False),
)
nested_file = pa.schema(
[
pa.field("id", pa.int64(), metadata={"PARQUET:field_id": "1"}),
pa.field(
"address",
pa.struct(
[
pa.field("street", pa.string(), metadata={"PARQUET:field_id": "3"}),
pa.field("zip", pa.int64(), metadata={"PARQUET:field_id": "4"}),
]
),
metadata={"PARQUET:field_id": "2"},
),
]
)
item_with_ids = pa.schema(
[
pa.field("id", pa.int64(), metadata={"PARQUET:field_id": "1"}),
pa.field(
"tags",
pa.list_(pa.field("item", pa.string(), metadata={"PARQUET:field_id": "3"})),
metadata={"PARQUET:field_id": "2"},
),
]
)
item_without_ids = pa.schema([pa.field("id", pa.int64()), pa.field("tags", pa.list_(pa.field("item", pa.string())))])
nested_data = {"id": [7, 8], "address": [{"street": "x", "zip": 98101}, {"street": "y", "zip": 10001}]}
list_data = {"id": [1, 2], "tags": [["a", "b"], ["c"]]}
nested_rows = [
{"id": 7, "address": {"street": "x", "postal_code": 98101}},
{"id": 8, "address": {"street": "y", "postal_code": 10001}},
]
list_rows = [{"id": 1, "tags": ["a", "b"]}, {"id": 2, "tags": ["c"]}]

catalog.create_namespace("default")
shapes = [
# The table renames address.zip (field ID 4) after the file was written.
(
"renamed_nested",
nested_schema,
("address.zip", "postal_code"),
nested_file,
nested_data,
True,
nested_rows,
{1: 2, 3: 2, 4: 2},
),
# Legacy writers name the list element "item"; the table calls it "element" (field ID 3).
("item_with_ids", list_schema, None, item_with_ids, list_data, False, list_rows, {1: 2, 3: 3}),
("item_without_ids", list_schema, None, item_without_ids, list_data, False, list_rows, {1: 2, 3: 3}),
]
for name, table_schema, rename, file_schema, data, compliant, rows, value_counts in shapes:
identifier = f"default.add_files_{name}"
catalog.create_table(identifier, schema=table_schema)
if rename is not None:
with catalog.load_table(identifier).update_schema() as update:
update.rename_column(*rename)
file_loc = f"{tmp_path}/{name}.parquet"
pq.write_table(pa.table(data, schema=file_schema), file_loc, use_compliant_nested_type=compliant)

catalog.load_table(identifier).add_files([file_loc])

table = catalog.load_table(identifier)
assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == rows
assert dict(table.inspect.files().to_pylist()[0]["value_counts"]) == value_counts


def test_add_files_rejects_columns_missing_from_table(catalog: InMemoryCatalog, tmp_path: str) -> None:
table_schema = Schema(
NestedField(1, "a", LongType(), required=False),
NestedField(2, "b", StringType(), required=False),
)
field_a = pa.field("a", pa.int64(), metadata={"PARQUET:field_id": "1"})
field_b = pa.field("b", pa.string(), metadata={"PARQUET:field_id": "2"})
field_extra = pa.field("extra", pa.int64(), metadata={"PARQUET:field_id": "3"})

catalog.create_namespace("default")
shapes = [
# The file has a column with field ID 3, which the table does not have.
("extra_column", False, pa.schema([field_a, field_b, field_extra]), {"a": [5], "b": ["p"], "extra": [9]}, "extra"),
# The table dropped field ID 1 and added a new column named "a" (field ID 3). The file's "a" is still field ID 1.
# Before this change, its statistics were recorded under the new "a", so a scan for "a IS NULL" pruned the file.
("same_name_new_id", True, pa.schema([field_a, field_b]), {"a": [5], "b": ["p"]}, "a"),
]
for name, drop_and_add_a, file_schema, data, missing in shapes:
identifier = f"default.add_files_rejects_{name}"
catalog.create_table(identifier, schema=table_schema)
if drop_and_add_a:
with catalog.load_table(identifier).update_schema() as update:
update.delete_column("a")
with catalog.load_table(identifier).update_schema() as update:
update.add_column("a", LongType())
file_loc = f"{tmp_path}/{name}.parquet"
pq.write_table(pa.table(data, schema=file_schema), file_loc)

with pytest.raises(ValueError, match=f"PyArrow table contains more columns: {missing}"):
catalog.load_table(identifier).add_files([file_loc])


def test_partition_for_demo() -> None:
test_pa_schema = pa.schema([("year", pa.int64()), ("n_legs", pa.int64()), ("animal", pa.string())])
test_schema = Schema(
Expand Down
Loading