Repository navigation
Keep rows in truncate partitions near the minimum int or long #4407
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
77612bf
af385e0
4fef874
fda11b8
53a6d3c
7f0ebc5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -52,6 +52,7 @@ | |
| LessThanOrEqual, | ||
| NotEqualTo, | ||
| NotStartsWith, | ||
| Or, | ||
| Reference, | ||
| StartsWith, | ||
| UnboundPredicate, | ||
|
|
@@ -800,6 +801,8 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: | |
| return _set_apply_transform(name, pred, self.transform(field_type)) | ||
| elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore | ||
| if isinstance(pred, BoundLiteralPredicate): | ||
| if isinstance(field_type, (IntegerType, LongType)): | ||
| return _truncate_integer_projection(name, pred, self.transform(field_type), field_type, self._width) | ||
| return _truncate_number(name, pred, self.transform(field_type)) | ||
| elif isinstance(field_type, (BinaryType, StringType)): | ||
| if isinstance(pred, BoundLiteralPredicate): | ||
|
|
@@ -825,6 +828,8 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | | |
|
|
||
| if isinstance(field_type, (IntegerType, LongType, DecimalType)): | ||
| if isinstance(pred, BoundLiteralPredicate): | ||
| if isinstance(field_type, (IntegerType, LongType)): | ||
| return _truncate_integer_strict(name, pred, self.transform(field_type), field_type, self._width) | ||
| return _truncate_number_strict(name, pred, self.transform(field_type)) | ||
| elif isinstance(pred, BoundNotIn): | ||
| return _set_apply_transform(name, pred, self.transform(field_type)) | ||
|
|
@@ -864,7 +869,7 @@ def transform(self, source: IcebergType) -> Callable[[S | None], S | None]: | |
| if isinstance(source, (IntegerType, LongType)): | ||
|
|
||
| def truncate_func(v: Any) -> Any: | ||
| return v - v % self._width | ||
| return _wrap_integer(v - v % self._width, source) | ||
|
|
||
| elif isinstance(source, (StringType, BinaryType)): | ||
|
|
||
|
|
@@ -1030,6 +1035,88 @@ def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Arr | |
| return lambda arr: pa.nulls(len(arr), type=arr.type) | ||
|
|
||
|
|
||
| def _integer_bounds(source: IcebergType) -> tuple[int, int]: | ||
| """Return the minimum value and the size of the range of an integer type.""" | ||
| if isinstance(source, IntegerType): | ||
| return -(2**31), 2**32 | ||
| return -(2**63), 2**64 | ||
|
|
||
|
|
||
| def _wrap_integer(value: int, source: IcebergType) -> int: | ||
| """Wrap an integer into the range of the source type, like Java's two's complement arithmetic.""" | ||
| minimum, modulus = _integer_bounds(source) | ||
| return (value - minimum) % modulus + minimum | ||
|
|
||
|
|
||
| def _truncate_integer_projection( | ||
| name: str, | ||
| pred: BoundLiteralPredicate, | ||
| transform: Callable[[Any | None], Any | None], | ||
| source: IcebergType, | ||
| width: int, | ||
| ) -> UnboundPredicate | None: | ||
| """Project a predicate on a truncate partition of an integer column. | ||
|
|
||
| The values from the minimum of the source type truncate to a partition value at the top of the | ||
| type's range, which does not sort with the other partition values. A range that can match those | ||
| values therefore also matches that partition. | ||
| """ | ||
| minimum, _ = _integer_bounds(source) | ||
| floor_mod = minimum % width | ||
| if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): | ||
| return _truncate_number(name, pred, transform) | ||
|
|
||
| wrapped_minimum = _wrap_integer(minimum - floor_mod, source) | ||
| # The first partition value after the block that holds the minimum of the source type | ||
| next_block_start = minimum - floor_mod + width | ||
| boundary = pred.literal.value | ||
|
|
||
| if isinstance(pred, BoundGreaterThan): | ||
| if boundary + 1 < next_block_start: | ||
| return Or(GreaterThanOrEqual(Reference(name), next_block_start), EqualTo(Reference(name), wrapped_minimum)) | ||
| return _truncate_number(name, pred, transform) | ||
| elif isinstance(pred, BoundGreaterThanOrEqual): | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No test covers this branch: with assert project(GreaterThanOrEqual("id", -(2**63) + 1)) == Or(
GreaterThanOrEqual("id_trunc", -9223372036854775800), EqualTo("id_trunc", wrapped_minimum)
)That test projects the |
||
| if boundary < next_block_start: | ||
| return Or(GreaterThanOrEqual(Reference(name), next_block_start), EqualTo(Reference(name), wrapped_minimum)) | ||
| return _truncate_number(name, pred, transform) | ||
|
|
||
| projected = _truncate_number(name, pred, transform) | ||
| if isinstance(pred, (BoundLessThan, BoundLessThanOrEqual)) and projected is not None: | ||
| return Or(projected, EqualTo(Reference(name), wrapped_minimum)) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: when the boundary falls in the block that holds the minimum (for example |
||
| return projected | ||
|
|
||
|
|
||
| def _truncate_integer_strict( | ||
| name: str, | ||
| pred: BoundLiteralPredicate, | ||
| transform: Callable[[Any | None], Any | None], | ||
| source: IcebergType, | ||
| width: int, | ||
| ) -> UnboundPredicate | None: | ||
| """Project a predicate strictly on a truncate partition of an integer column. | ||
|
|
||
| Values from the minimum of the source type truncate to a partition value at the top of the | ||
| type's range, which does not sort with the other partition values. No single predicate on the | ||
| partition value can account for that, so those ranges have no strict projection. | ||
| """ | ||
| minimum, _ = _integer_bounds(source) | ||
| floor_mod = minimum % width | ||
| if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): | ||
| return _truncate_number_strict(name, pred, transform) | ||
|
|
||
| first_block_end = minimum - floor_mod + width | ||
| if isinstance(pred, BoundLessThan): | ||
| if pred.literal.value < first_block_end: | ||
| return None | ||
| elif isinstance(pred, BoundLessThanOrEqual): | ||
| if pred.literal.increment().value < first_block_end: | ||
| return None | ||
| elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)): | ||
| return None | ||
|
|
||
| return _truncate_number_strict(name, pred, transform) | ||
|
|
||
|
|
||
| def _truncate_number( | ||
| name: str, pred: BoundLiteralPredicate, transform: Callable[[Any | None], Any | None] | ||
| ) -> UnboundPredicate | None: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: the
EqualTohere is redundant.wrapped_minimumis always greater thannext_block_start(they differ by the size of the type's range minus the width), soGreaterThanOrEqual(Reference(name), next_block_start)already includes it. Returning just that keeps the projection anUnboundPredicate, so the residual evaluator can still use it to prune. Same for theBoundGreaterThanOrEqualbranch below.