Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pyiceberg/table/update/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,12 @@ def union_by_name(
new_schema: Schema | pa.Schema,
format_version: TableVersion = 2,
) -> UpdateSchema:
"""Merge new_schema into the existing schema by field name.

Raises:
ValueError: When case_sensitive is False and a field lookup searches an existing struct
containing two fields with the same lower-case name.
"""
from pyiceberg.catalog import Catalog

visit_with_partner(
Expand Down
15 changes: 12 additions & 3 deletions pyiceberg/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -543,15 +543,24 @@ def field(self, field_id: int) -> NestedField | None:
return None

def field_by_name(self, name: str, case_sensitive: bool = True) -> NestedField | None:
"""Return the field with the given name, or None when there is no such field.

Raises:
ValueError: When case_sensitive is False and two fields have names that differ only by case.
"""
if case_sensitive:
for field in self.fields:
if field.name == name:
return field
else:
name_lower = name.lower()
# Index every field first, as Java's caseInsensitiveField does, so any lookup fails on a collision
fields_by_lower_name: dict[str, NestedField] = {}
for field in self.fields:
if field.name.lower() == name_lower:
return field
lower_name = field.name.lower()
if (existing := fields_by_lower_name.get(lower_name)) is not None:
raise ValueError(f"Cannot build lower case index: {existing.name} and {field.name} collide")
fields_by_lower_name[lower_name] = field
return fields_by_lower_name.get(name.lower())
return None

def __str__(self) -> str:
Expand Down
39 changes: 39 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -960,6 +960,45 @@ def test_table_metadata_legacy_invalid_compression_codec_on_commit(
assert table.metadata_location.endswith(".metadata.json")


def test_union_by_name_case_insensitive_duplicates(catalog: Catalog) -> None:
catalog.create_namespace_if_not_exists("default")
schema = Schema(NestedField(1, "ID", LongType(), required=False), NestedField(2, "id", LongType(), required=False))
table = catalog.create_table("default.case_variants", schema=schema)
# A case-insensitive union fails on the table's ID and id, and the table is left unchanged
# An id-only schema adds nothing, so only the struct lookup can raise here
id_only = Schema(NestedField(1, "id", LongType(), required=False))
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
with table.update_schema(case_sensitive=False) as update:
update.union_by_name(id_only)
assert catalog.load_table("default.case_variants").schema() == schema

# The case-sensitive union matches id exactly and adds name
incoming = Schema(NestedField(1, "id", LongType(), required=False), NestedField(2, "name", StringType(), required=False))
with table.update_schema() as update:
update.union_by_name(incoming)
assert [field.name for field in catalog.load_table("default.case_variants").schema().fields] == ["ID", "id", "name"]

# Without a collision, a case-insensitive union still matches the existing column
single_schema = Schema(NestedField(1, "ID", LongType(), required=False))
single = catalog.create_table("default.single", schema=single_schema)
with single.update_schema(case_sensitive=False) as update:
update.union_by_name(Schema(NestedField(1, "id", LongType(), required=False)))
assert catalog.load_table("default.single").schema() == single_schema

# The same collision inside a struct column fails the union, and the table is left unchanged
profile = Schema(
NestedField(2, "ID", LongType(), required=False),
NestedField(3, "id", LongType(), required=False),
).as_struct()
nested_schema = Schema(NestedField(1, "profile", profile, required=False))
nested = catalog.create_table("default.nested_variants", schema=nested_schema)
incoming_profile = Schema(NestedField(4, "id", LongType(), required=False)).as_struct()
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
with nested.update_schema(case_sensitive=False) as update:
update.union_by_name(Schema(NestedField(1, "profile", incoming_profile, required=False)))
assert catalog.load_table("default.nested_variants").schema() == nested_schema


@pytest.mark.parametrize("format_version", [1, 2])
def test_write_and_evolve(catalog: Catalog, format_version: int) -> None:
identifier = f"default.arrow_write_data_and_evolve_schema_v{format_version}"
Expand Down
24 changes: 24 additions & 0 deletions tests/test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,30 @@ def test_struct_field_by_name() -> None:
assert type_var.field_by_name("nonexistent_field", case_sensitive=True) is None


def test_struct_field_by_name_case_insensitive_duplicates() -> None:
upper_field = NestedField(1, "ID", LongType(), required=False)
lower_field = NestedField(2, "id", LongType(), required=False)
safe_field = NestedField(3, "safe", LongType(), required=False)
type_var = StructType(upper_field, lower_field, safe_field)

assert type_var.field_by_name("ID", case_sensitive=True) == upper_field
assert type_var.field_by_name("id", case_sensitive=True) == lower_field
assert type_var.field_by_name("safe", case_sensitive=True) == safe_field

# As in Java's caseInsensitiveField, every case-insensitive lookup fails while the index cannot be built
for name in ("ID", "Id", "id", "safe", "nonexistent_field"):
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
type_var.field_by_name(name, case_sensitive=False)

# The same holds for a struct nested in another struct
profile_field = NestedField(4, "profile", type_var, required=False)
assert StructType(profile_field).field_by_name("PROFILE", case_sensitive=False) == profile_field
profile_type = profile_field.field_type
assert isinstance(profile_type, StructType)
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
profile_type.field_by_name("Id", case_sensitive=False)


def test_list_type() -> None:
type_var = ListType(
1,
Expand Down
Loading