diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 7504548507..d1c142991e 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -1974,7 +1974,10 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression: if isinstance(self._eval_projection(part.transform.strict_project(part.name, predicate)), AlwaysTrue): return AlwaysTrue() - if isinstance(self._eval_projection(part.transform.project(part.name, predicate)), AlwaysFalse): + inclusive_projection = part.transform.project(part.name, predicate) + if isinstance(inclusive_projection, UnboundPredicate) and isinstance( + self._eval_projection(inclusive_projection), AlwaysFalse + ): return AlwaysFalse() return predicate diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 45c8ffd196..683ba95f89 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -52,6 +52,7 @@ LessThanOrEqual, NotEqualTo, NotStartsWith, + Or, Reference, StartsWith, UnboundPredicate, @@ -800,6 +801,8 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: return _set_apply_transform(name, pred, self.transform(field_type)) elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore if isinstance(pred, BoundLiteralPredicate): + if isinstance(field_type, (IntegerType, LongType)): + return _truncate_integer_projection(name, pred, self.transform(field_type), field_type, self._width) return _truncate_number(name, pred, self.transform(field_type)) elif isinstance(field_type, (BinaryType, StringType)): if isinstance(pred, BoundLiteralPredicate): @@ -825,6 +828,8 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | if isinstance(field_type, (IntegerType, LongType, DecimalType)): if isinstance(pred, BoundLiteralPredicate): + if isinstance(field_type, (IntegerType, LongType)): + return _truncate_integer_strict(name, pred, self.transform(field_type), field_type, self._width) return _truncate_number_strict(name, pred, self.transform(field_type)) elif isinstance(pred, BoundNotIn): return _set_apply_transform(name, pred, self.transform(field_type)) @@ -864,7 +869,7 @@ def transform(self, source: IcebergType) -> Callable[[S | None], S | None]: if isinstance(source, (IntegerType, LongType)): def truncate_func(v: Any) -> Any: - return v - v % self._width + return _wrap_integer(v - v % self._width, source) elif isinstance(source, (StringType, BinaryType)): @@ -1030,6 +1035,88 @@ def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Arr return lambda arr: pa.nulls(len(arr), type=arr.type) +def _integer_bounds(source: IcebergType) -> tuple[int, int]: + """Return the minimum value and the size of the range of an integer type.""" + if isinstance(source, IntegerType): + return -(2**31), 2**32 + return -(2**63), 2**64 + + +def _wrap_integer(value: int, source: IcebergType) -> int: + """Wrap an integer into the range of the source type, like Java's two's complement arithmetic.""" + minimum, modulus = _integer_bounds(source) + return (value - minimum) % modulus + minimum + + +def _truncate_integer_projection( + name: str, + pred: BoundLiteralPredicate, + transform: Callable[[Any | None], Any | None], + source: IcebergType, + width: int, +) -> UnboundPredicate | None: + """Project a predicate on a truncate partition of an integer column. + + The values from the minimum of the source type truncate to a partition value at the top of the + type's range, which does not sort with the other partition values. A range that can match those + values therefore also matches that partition. + """ + minimum, _ = _integer_bounds(source) + floor_mod = minimum % width + if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): + return _truncate_number(name, pred, transform) + + wrapped_minimum = _wrap_integer(minimum - floor_mod, source) + # The first partition value after the block that holds the minimum of the source type + next_block_start = minimum - floor_mod + width + boundary = pred.literal.value + + if isinstance(pred, BoundGreaterThan): + if boundary + 1 < next_block_start: + return Or(GreaterThanOrEqual(Reference(name), next_block_start), EqualTo(Reference(name), wrapped_minimum)) + return _truncate_number(name, pred, transform) + elif isinstance(pred, BoundGreaterThanOrEqual): + if boundary < next_block_start: + return Or(GreaterThanOrEqual(Reference(name), next_block_start), EqualTo(Reference(name), wrapped_minimum)) + return _truncate_number(name, pred, transform) + + projected = _truncate_number(name, pred, transform) + if isinstance(pred, (BoundLessThan, BoundLessThanOrEqual)) and projected is not None: + return Or(projected, EqualTo(Reference(name), wrapped_minimum)) + return projected + + +def _truncate_integer_strict( + name: str, + pred: BoundLiteralPredicate, + transform: Callable[[Any | None], Any | None], + source: IcebergType, + width: int, +) -> UnboundPredicate | None: + """Project a predicate strictly on a truncate partition of an integer column. + + Values from the minimum of the source type truncate to a partition value at the top of the + type's range, which does not sort with the other partition values. No single predicate on the + partition value can account for that, so those ranges have no strict projection. + """ + minimum, _ = _integer_bounds(source) + floor_mod = minimum % width + if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): + return _truncate_number_strict(name, pred, transform) + + first_block_end = minimum - floor_mod + width + if isinstance(pred, BoundLessThan): + if pred.literal.value < first_block_end: + return None + elif isinstance(pred, BoundLessThanOrEqual): + if pred.literal.increment().value < first_block_end: + return None + elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)): + return None + + return _truncate_number_strict(name, pred, transform) + + def _truncate_number( name: str, pred: BoundLiteralPredicate, transform: Callable[[Any | None], Any | None] ) -> UnboundPredicate | None: diff --git a/tests/expressions/test_projection.py b/tests/expressions/test_projection.py index 4d0c2c1346..9c1a3ef2ff 100644 --- a/tests/expressions/test_projection.py +++ b/tests/expressions/test_projection.py @@ -270,6 +270,18 @@ def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: assert expected[index] == expr, predicate +def test_int_truncate_projection_near_minimum(schema: Schema) -> None: + spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "id_trunc")) + project = inclusive_projection(schema, spec) + wrapped_minimum = 9223372036854775806 + + assert project(EqualTo("id", -(2**63))) == EqualTo("id_trunc", wrapped_minimum) + assert project(LessThan("id", 0)) == Or(LessThanOrEqual("id_trunc", -10), EqualTo("id_trunc", wrapped_minimum)) + assert project(GreaterThan("id", -(2**63))) == Or( + GreaterThanOrEqual("id_trunc", -9223372036854775800), EqualTo("id_trunc", wrapped_minimum) + ) + + def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec) -> None: predicates = [ NotNull("data"), @@ -320,8 +332,8 @@ def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpe expected = [ NotNull("id_trunc"), IsNull("id_trunc"), - LessThanOrEqual("id_trunc", 0), - LessThanOrEqual("id_trunc", 10), + Or(LessThanOrEqual("id_trunc", 0), EqualTo("id_trunc", 9223372036854775806)), + Or(LessThanOrEqual("id_trunc", 10), EqualTo("id_trunc", 9223372036854775806)), GreaterThanOrEqual("id_trunc", 10), GreaterThanOrEqual("id_trunc", 10), EqualTo("id_trunc", 10), diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9c418e50a..cdac621e35 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -71,6 +71,7 @@ EqualTo, GreaterThan, IsNull, + LessThan, Not, Or, ) @@ -3663,6 +3664,49 @@ def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None: assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter +def test_scan_filters_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=True), + NestedField(id=2, name="x", field_type=IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_truncate_minimum", schema=schema, partition_spec=spec) + table.append( + pa.Table.from_pylist( + [{"id": 0, "x": -(2**31)}, {"id": 1, "x": -(2**31) + 100}, {"id": 2, "x": 5}], schema=schema.as_arrow() + ) + ) + + assert table.scan(row_filter=EqualTo("x", -(2**31))).to_arrow()["id"].to_pylist() == [0] + assert sorted(table.scan(row_filter=LessThan("x", 0)).to_arrow()["id"].to_pylist()) == [0, 1] + assert sorted(table.scan(row_filter=GreaterThan("x", -(2**31))).to_arrow()["id"].to_pylist()) == [1, 2] + + +def test_count_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=True), + NestedField(id=2, name="x", field_type=IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc")) + catalog.create_namespace("default") + table = catalog.create_table( + "default.test_count_truncate_minimum", + schema=schema, + partition_spec=spec, + properties={"write.metadata.metrics.default": "counts"}, + ) + table.append(pa.Table.from_pylist([{"id": 0, "x": -(2**31)}, {"id": 1, "x": 5}], schema=schema.as_arrow())) + + less_than = table.scan(row_filter=LessThan("x", -(2**31) + 100)) + assert less_than.to_arrow()["id"].to_pylist() == [0] + assert less_than.count() == 1 + + greater_than = table.scan(row_filter=GreaterThan("x", 0)) + assert greater_than.to_arrow()["id"].to_pylist() == [1] + assert greater_than.count() == 1 + + def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="id", field_type=LongType(), required=False), diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 6ab7b52c4c..e8129ad882 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -465,6 +465,11 @@ def test_truncate_integer(type_var: PrimitiveType, input_var: int, expected: int assert trunc.transform(type_var)(input_var) == expected +def test_truncate_integer_minimum_wraps() -> None: + assert TruncateTransform(10).transform(IntegerType())(-(2**31)) == 2147483646 + assert TruncateTransform(10).transform(LongType())(-(2**63)) == 9223372036854775806 + + @pytest.mark.parametrize( "input_var,expected", [ @@ -1504,8 +1509,8 @@ def test_truncate_strict_integer_lower_bound(bound_reference_int: BoundReference _assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "100") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "100") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "100") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1522,8 +1527,8 @@ def test_truncate_strict_integer_upper_bound(bound_reference_int: BoundReference _assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "90") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "90") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "90") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse) @@ -1538,8 +1543,8 @@ def test_truncate_strict_long_lower_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "100") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "100") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "100") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1556,8 +1561,8 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "90") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "90") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "90") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1568,6 +1573,50 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundIn(term=bound_reference_long, literals={value_dec, value, value_inc}), transform, NotIn) +def test_truncate_strict_integer_minimum(bound_reference_int: BoundReference) -> None: + minimum = -(2**31) + first_block_end = minimum - minimum % 1000 + 1000 + transform = TruncateTransform(1000) + + assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_int, literal=literal(minimum))) is None + assert ( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_int, literal=literal(minimum + 1))) + is None + ) + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=literal(0))) is None + assert ( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=literal(0))) is None + ) + _assert_projection_strict( + BoundLessThan(term=bound_reference_int, literal=literal(first_block_end)), + transform, + LessThan, + str(first_block_end), + ) + + +def test_truncate_strict_long_minimum(bound_reference_long: BoundReference) -> None: + minimum = -(2**63) + first_block_end = minimum - minimum % 1000 + 1000 + transform = TruncateTransform(1000) + + assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_long, literal=literal(minimum))) is None + assert ( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_long, literal=literal(minimum + 1))) + is None + ) + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=literal(0))) is None + assert ( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=literal(0))) is None + ) + _assert_projection_strict( + BoundLessThan(term=bound_reference_long, literal=literal(first_block_end)), + transform, + LessThan, + str(first_block_end), + ) + + def test_truncate_strict_decimal_lower_bound(bound_reference_decimal: BoundReference) -> None: dec = DecimalType(9, 2) value = literal("100.00").to(dec)