diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index a0312745d2..d0da1c087f 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -151,6 +151,12 @@ def union_by_name( new_schema: Schema | pa.Schema, format_version: TableVersion = 2, ) -> UpdateSchema: + """Merge new_schema into the existing schema by field name. + + Raises: + ValueError: When case_sensitive is False and a field lookup searches an existing struct + containing two fields with the same lower-case name. + """ from pyiceberg.catalog import Catalog visit_with_partner( diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 3295beb666..2a26cd954a 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -543,15 +543,24 @@ def field(self, field_id: int) -> NestedField | None: return None def field_by_name(self, name: str, case_sensitive: bool = True) -> NestedField | None: + """Return the field with the given name, or None when there is no such field. + + Raises: + ValueError: When case_sensitive is False and two fields have names that differ only by case. + """ if case_sensitive: for field in self.fields: if field.name == name: return field else: - name_lower = name.lower() + # Index every field first, as Java's caseInsensitiveField does, so any lookup fails on a collision + fields_by_lower_name: dict[str, NestedField] = {} for field in self.fields: - if field.name.lower() == name_lower: - return field + lower_name = field.name.lower() + if (existing := fields_by_lower_name.get(lower_name)) is not None: + raise ValueError(f"Cannot build lower case index: {existing.name} and {field.name} collide") + fields_by_lower_name[lower_name] = field + return fields_by_lower_name.get(name.lower()) return None def __str__(self) -> str: diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..e4f834b016 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -960,6 +960,45 @@ def test_table_metadata_legacy_invalid_compression_codec_on_commit( assert table.metadata_location.endswith(".metadata.json") +def test_union_by_name_case_insensitive_duplicates(catalog: Catalog) -> None: + catalog.create_namespace_if_not_exists("default") + schema = Schema(NestedField(1, "ID", LongType(), required=False), NestedField(2, "id", LongType(), required=False)) + table = catalog.create_table("default.case_variants", schema=schema) + # A case-insensitive union fails on the table's ID and id, and the table is left unchanged + # An id-only schema adds nothing, so only the struct lookup can raise here + id_only = Schema(NestedField(1, "id", LongType(), required=False)) + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + with table.update_schema(case_sensitive=False) as update: + update.union_by_name(id_only) + assert catalog.load_table("default.case_variants").schema() == schema + + # The case-sensitive union matches id exactly and adds name + incoming = Schema(NestedField(1, "id", LongType(), required=False), NestedField(2, "name", StringType(), required=False)) + with table.update_schema() as update: + update.union_by_name(incoming) + assert [field.name for field in catalog.load_table("default.case_variants").schema().fields] == ["ID", "id", "name"] + + # Without a collision, a case-insensitive union still matches the existing column + single_schema = Schema(NestedField(1, "ID", LongType(), required=False)) + single = catalog.create_table("default.single", schema=single_schema) + with single.update_schema(case_sensitive=False) as update: + update.union_by_name(Schema(NestedField(1, "id", LongType(), required=False))) + assert catalog.load_table("default.single").schema() == single_schema + + # The same collision inside a struct column fails the union, and the table is left unchanged + profile = Schema( + NestedField(2, "ID", LongType(), required=False), + NestedField(3, "id", LongType(), required=False), + ).as_struct() + nested_schema = Schema(NestedField(1, "profile", profile, required=False)) + nested = catalog.create_table("default.nested_variants", schema=nested_schema) + incoming_profile = Schema(NestedField(4, "id", LongType(), required=False)).as_struct() + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + with nested.update_schema(case_sensitive=False) as update: + update.union_by_name(Schema(NestedField(1, "profile", incoming_profile, required=False))) + assert catalog.load_table("default.nested_variants").schema() == nested_schema + + @pytest.mark.parametrize("format_version", [1, 2]) def test_write_and_evolve(catalog: Catalog, format_version: int) -> None: identifier = f"default.arrow_write_data_and_evolve_schema_v{format_version}" diff --git a/tests/test_types.py b/tests/test_types.py index eb8ae2ea52..4aab7b56d0 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -182,6 +182,30 @@ def test_struct_field_by_name() -> None: assert type_var.field_by_name("nonexistent_field", case_sensitive=True) is None +def test_struct_field_by_name_case_insensitive_duplicates() -> None: + upper_field = NestedField(1, "ID", LongType(), required=False) + lower_field = NestedField(2, "id", LongType(), required=False) + safe_field = NestedField(3, "safe", LongType(), required=False) + type_var = StructType(upper_field, lower_field, safe_field) + + assert type_var.field_by_name("ID", case_sensitive=True) == upper_field + assert type_var.field_by_name("id", case_sensitive=True) == lower_field + assert type_var.field_by_name("safe", case_sensitive=True) == safe_field + + # As in Java's caseInsensitiveField, every case-insensitive lookup fails while the index cannot be built + for name in ("ID", "Id", "id", "safe", "nonexistent_field"): + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + type_var.field_by_name(name, case_sensitive=False) + + # The same holds for a struct nested in another struct + profile_field = NestedField(4, "profile", type_var, required=False) + assert StructType(profile_field).field_by_name("PROFILE", case_sensitive=False) == profile_field + profile_type = profile_field.field_type + assert isinstance(profile_type, StructType) + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + profile_type.field_by_name("Id", case_sensitive=False) + + def test_list_type() -> None: type_var = ListType( 1,