Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pyiceberg/expressions/visitors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1974,7 +1974,10 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression:
if isinstance(self._eval_projection(part.transform.strict_project(part.name, predicate)), AlwaysTrue):
return AlwaysTrue()

if isinstance(self._eval_projection(part.transform.project(part.name, predicate)), AlwaysFalse):
inclusive_projection = part.transform.project(part.name, predicate)
if isinstance(inclusive_projection, UnboundPredicate) and isinstance(
self._eval_projection(inclusive_projection), AlwaysFalse
):
return AlwaysFalse()

return predicate
Expand Down
89 changes: 88 additions & 1 deletion pyiceberg/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
LessThanOrEqual,
NotEqualTo,
NotStartsWith,
Or,
Reference,
StartsWith,
UnboundPredicate,
Expand Down Expand Up @@ -800,6 +801,8 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None:
return _set_apply_transform(name, pred, self.transform(field_type))
elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore
if isinstance(pred, BoundLiteralPredicate):
if isinstance(field_type, (IntegerType, LongType)):
return _truncate_integer_projection(name, pred, self.transform(field_type), field_type, self._width)
return _truncate_number(name, pred, self.transform(field_type))
elif isinstance(field_type, (BinaryType, StringType)):
if isinstance(pred, BoundLiteralPredicate):
Expand All @@ -825,6 +828,8 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate |

if isinstance(field_type, (IntegerType, LongType, DecimalType)):
if isinstance(pred, BoundLiteralPredicate):
if isinstance(field_type, (IntegerType, LongType)):
return _truncate_integer_strict(name, pred, self.transform(field_type), field_type, self._width)
return _truncate_number_strict(name, pred, self.transform(field_type))
elif isinstance(pred, BoundNotIn):
return _set_apply_transform(name, pred, self.transform(field_type))
Expand Down Expand Up @@ -864,7 +869,7 @@ def transform(self, source: IcebergType) -> Callable[[S | None], S | None]:
if isinstance(source, (IntegerType, LongType)):

def truncate_func(v: Any) -> Any:
return v - v % self._width
return _wrap_integer(v - v % self._width, source)

elif isinstance(source, (StringType, BinaryType)):

Expand Down Expand Up @@ -1030,6 +1035,88 @@ def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Arr
return lambda arr: pa.nulls(len(arr), type=arr.type)


def _integer_bounds(source: IcebergType) -> tuple[int, int]:
"""Return the minimum value and the size of the range of an integer type."""
if isinstance(source, IntegerType):
return -(2**31), 2**32
return -(2**63), 2**64


def _wrap_integer(value: int, source: IcebergType) -> int:
"""Wrap an integer into the range of the source type, like Java's two's complement arithmetic."""
minimum, modulus = _integer_bounds(source)
return (value - minimum) % modulus + minimum


def _truncate_integer_projection(
name: str,
pred: BoundLiteralPredicate,
transform: Callable[[Any | None], Any | None],
source: IcebergType,
width: int,
) -> UnboundPredicate | None:
"""Project a predicate on a truncate partition of an integer column.

The values from the minimum of the source type truncate to a partition value at the top of the
type's range, which does not sort with the other partition values. A range that can match those
values therefore also matches that partition.
"""
minimum, _ = _integer_bounds(source)
floor_mod = minimum % width
if floor_mod == 0 or not isinstance(pred.literal, LongLiteral):
return _truncate_number(name, pred, transform)

wrapped_minimum = _wrap_integer(minimum - floor_mod, source)
# The first partition value after the block that holds the minimum of the source type
next_block_start = minimum - floor_mod + width
boundary = pred.literal.value

if isinstance(pred, BoundGreaterThan):
if boundary + 1 < next_block_start:
return Or(GreaterThanOrEqual(Reference(name), next_block_start), EqualTo(Reference(name), wrapped_minimum))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: the EqualTo here is redundant. wrapped_minimum is always greater than next_block_start (they differ by the size of the type's range minus the width), so GreaterThanOrEqual(Reference(name), next_block_start) already includes it. Returning just that keeps the projection an UnboundPredicate, so the residual evaluator can still use it to prune. Same for the BoundGreaterThanOrEqual branch below.

return _truncate_number(name, pred, transform)
elif isinstance(pred, BoundGreaterThanOrEqual):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No test covers this branch: with if boundary < next_block_start: replaced by if False:, every test still passes, although x >= MIN then skips all ordinary partitions. An assert next to the GreaterThan one in test_int_truncate_projection_near_minimum would cover it:

assert project(GreaterThanOrEqual("id", -(2**63) + 1)) == Or(
    GreaterThanOrEqual("id_trunc", -9223372036854775800), EqualTo("id_trunc", wrapped_minimum)
)

That test projects the LongType id column, so an IntegerType case would help too; only the scan test covers int.

if boundary < next_block_start:
return Or(GreaterThanOrEqual(Reference(name), next_block_start), EqualTo(Reference(name), wrapped_minimum))
return _truncate_number(name, pred, transform)

projected = _truncate_number(name, pred, transform)
if isinstance(pred, (BoundLessThan, BoundLessThanOrEqual)) and projected is not None:
return Or(projected, EqualTo(Reference(name), wrapped_minimum))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: when the boundary falls in the block that holds the minimum (for example x < -2**31 + 100 with truncate[1000]), _truncate_number projects to LessThanOrEqual(name, wrapped_minimum), which keeps almost every partition. Only the wrapped partition can match there, so this could return EqualTo(Reference(name), wrapped_minimum) alone. That is the case test_count_truncate_partitions_near_the_minimum exercises.

return projected


def _truncate_integer_strict(
name: str,
pred: BoundLiteralPredicate,
transform: Callable[[Any | None], Any | None],
source: IcebergType,
width: int,
) -> UnboundPredicate | None:
"""Project a predicate strictly on a truncate partition of an integer column.

Values from the minimum of the source type truncate to a partition value at the top of the
type's range, which does not sort with the other partition values. No single predicate on the
partition value can account for that, so those ranges have no strict projection.
"""
minimum, _ = _integer_bounds(source)
floor_mod = minimum % width
if floor_mod == 0 or not isinstance(pred.literal, LongLiteral):
return _truncate_number_strict(name, pred, transform)

first_block_end = minimum - floor_mod + width
if isinstance(pred, BoundLessThan):
if pred.literal.value < first_block_end:
return None
elif isinstance(pred, BoundLessThanOrEqual):
if pred.literal.increment().value < first_block_end:
return None
elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)):
return None

return _truncate_number_strict(name, pred, transform)


def _truncate_number(
name: str, pred: BoundLiteralPredicate, transform: Callable[[Any | None], Any | None]
) -> UnboundPredicate | None:
Expand Down
16 changes: 14 additions & 2 deletions tests/expressions/test_projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,18 @@ def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None:
assert expected[index] == expr, predicate


def test_int_truncate_projection_near_minimum(schema: Schema) -> None:
spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "id_trunc"))
project = inclusive_projection(schema, spec)
wrapped_minimum = 9223372036854775806

assert project(EqualTo("id", -(2**63))) == EqualTo("id_trunc", wrapped_minimum)
assert project(LessThan("id", 0)) == Or(LessThanOrEqual("id_trunc", -10), EqualTo("id_trunc", wrapped_minimum))
assert project(GreaterThan("id", -(2**63))) == Or(
GreaterThanOrEqual("id_trunc", -9223372036854775800), EqualTo("id_trunc", wrapped_minimum)
)


def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec) -> None:
predicates = [
NotNull("data"),
Expand Down Expand Up @@ -320,8 +332,8 @@ def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpe
expected = [
NotNull("id_trunc"),
IsNull("id_trunc"),
LessThanOrEqual("id_trunc", 0),
LessThanOrEqual("id_trunc", 10),
Or(LessThanOrEqual("id_trunc", 0), EqualTo("id_trunc", 9223372036854775806)),
Or(LessThanOrEqual("id_trunc", 10), EqualTo("id_trunc", 9223372036854775806)),
GreaterThanOrEqual("id_trunc", 10),
GreaterThanOrEqual("id_trunc", 10),
EqualTo("id_trunc", 10),
Expand Down
44 changes: 44 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
EqualTo,
GreaterThan,
IsNull,
LessThan,
Not,
Or,
)
Expand Down Expand Up @@ -3663,6 +3664,49 @@ def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None:
assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter


def test_scan_filters_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=True),
NestedField(id=2, name="x", field_type=IntegerType(), required=False),
)
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc"))
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_truncate_minimum", schema=schema, partition_spec=spec)
table.append(
pa.Table.from_pylist(
[{"id": 0, "x": -(2**31)}, {"id": 1, "x": -(2**31) + 100}, {"id": 2, "x": 5}], schema=schema.as_arrow()
)
)

assert table.scan(row_filter=EqualTo("x", -(2**31))).to_arrow()["id"].to_pylist() == [0]
assert sorted(table.scan(row_filter=LessThan("x", 0)).to_arrow()["id"].to_pylist()) == [0, 1]
assert sorted(table.scan(row_filter=GreaterThan("x", -(2**31))).to_arrow()["id"].to_pylist()) == [1, 2]


def test_count_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=True),
NestedField(id=2, name="x", field_type=IntegerType(), required=False),
)
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc"))
catalog.create_namespace("default")
table = catalog.create_table(
"default.test_count_truncate_minimum",
schema=schema,
partition_spec=spec,
properties={"write.metadata.metrics.default": "counts"},
)
table.append(pa.Table.from_pylist([{"id": 0, "x": -(2**31)}, {"id": 1, "x": 5}], schema=schema.as_arrow()))

less_than = table.scan(row_filter=LessThan("x", -(2**31) + 100))
assert less_than.to_arrow()["id"].to_pylist() == [0]
assert less_than.count() == 1

greater_than = table.scan(row_filter=GreaterThan("x", 0))
assert greater_than.to_arrow()["id"].to_pylist() == [1]
assert greater_than.count() == 1


def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=False),
Expand Down
65 changes: 57 additions & 8 deletions tests/test_transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,11 @@ def test_truncate_integer(type_var: PrimitiveType, input_var: int, expected: int
assert trunc.transform(type_var)(input_var) == expected


def test_truncate_integer_minimum_wraps() -> None:
assert TruncateTransform(10).transform(IntegerType())(-(2**31)) == 2147483646
assert TruncateTransform(10).transform(LongType())(-(2**63)) == 9223372036854775806


@pytest.mark.parametrize(
"input_var,expected",
[
Expand Down Expand Up @@ -1504,8 +1509,8 @@ def test_truncate_strict_integer_lower_bound(bound_reference_int: BoundReference

_assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "100")
_assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100")
_assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "100")
_assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90")
assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None
assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "100")
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse)
value_dec = value.decrement()
Expand All @@ -1522,8 +1527,8 @@ def test_truncate_strict_integer_upper_bound(bound_reference_int: BoundReference

_assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "90")
_assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100")
_assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "90")
_assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90")
assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None
assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "90")
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse)

Expand All @@ -1538,8 +1543,8 @@ def test_truncate_strict_long_lower_bound(bound_reference_long: BoundReference)

_assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "100")
_assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100")
_assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "100")
_assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90")
assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None
assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "100")
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse)
value_dec = value.decrement()
Expand All @@ -1556,8 +1561,8 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference)

_assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "90")
_assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100")
_assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "90")
_assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90")
assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None
assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "90")
_assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse)
value_dec = value.decrement()
Expand All @@ -1568,6 +1573,50 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference)
_assert_projection_strict(BoundIn(term=bound_reference_long, literals={value_dec, value, value_inc}), transform, NotIn)


def test_truncate_strict_integer_minimum(bound_reference_int: BoundReference) -> None:
minimum = -(2**31)
first_block_end = minimum - minimum % 1000 + 1000
transform = TruncateTransform(1000)

assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_int, literal=literal(minimum))) is None
assert (
transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_int, literal=literal(minimum + 1)))
is None
)
assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=literal(0))) is None
assert (
transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=literal(0))) is None
)
_assert_projection_strict(
BoundLessThan(term=bound_reference_int, literal=literal(first_block_end)),
transform,
LessThan,
str(first_block_end),
)


def test_truncate_strict_long_minimum(bound_reference_long: BoundReference) -> None:
minimum = -(2**63)
first_block_end = minimum - minimum % 1000 + 1000
transform = TruncateTransform(1000)

assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_long, literal=literal(minimum))) is None
assert (
transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_long, literal=literal(minimum + 1)))
is None
)
assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=literal(0))) is None
assert (
transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=literal(0))) is None
)
_assert_projection_strict(
BoundLessThan(term=bound_reference_long, literal=literal(first_block_end)),
transform,
LessThan,
str(first_block_end),
)


def test_truncate_strict_decimal_lower_bound(bound_reference_decimal: BoundReference) -> None:
dec = DecimalType(9, 2)
value = literal("100.00").to(dec)
Expand Down
Loading