Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -985,6 +985,7 @@ split_offsets: list<item: int64>
equality_ids: list<item: int32>
child 0, item: int32
sort_order_id: int32
first_row_id: int64
readable_metrics: struct<city: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: large_string, upper_bound: large_string> not null, lat: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: double, upper_bound: double> not null, long: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: double, upper_bound: double> not null>
child 0, city: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: string, upper_bound: string> not null
child 0, column_size: int64
Expand Down Expand Up @@ -1024,6 +1025,7 @@ key_metadata: [[0100,0100]]
split_offsets:[[[],[]]]
equality_ids:[[[],[]]]
sort_order_id:[[[],[]]]
first_row_id:[[null,null]]
readable_metrics: [
-- is_valid: all not null
-- child 0 type: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: large_string, upper_bound: large_string>
Expand Down
32 changes: 29 additions & 3 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1801,6 +1801,14 @@ def _task_to_record_batches(
(index for index, field in enumerate(projected_schema.fields) if field.field_id == ROW_ID.field_id), None
)
filter_after_read = bool(positional_deletes) or row_id_index is not None
last_updated_index = next(
(
index
for index, field in enumerate(projected_schema.fields)
if field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id
),
None,
)

fragment_scanner = ds.Scanner.from_fragment(
fragment=fragment,
Expand Down Expand Up @@ -1856,6 +1864,10 @@ def _task_to_record_batches(
)
if row_id_index is not None and positions is not None:
result_batch = _assign_row_ids(result_batch, row_id_index, positions, task.file.first_row_id)
if last_updated_index is not None:
result_batch = _assign_last_updated_sequence_numbers(
result_batch, last_updated_index, task.file.first_row_id, task.data_sequence_number
)
yield result_batch


Expand All @@ -1871,6 +1883,23 @@ def _assign_row_ids(batch: pa.RecordBatch, row_id_index: int, positions: pa.Arra
return batch.set_column(row_id_index, batch.schema.field(row_id_index), row_ids)


def _assign_last_updated_sequence_numbers(
batch: pa.RecordBatch, last_updated_index: int, first_row_id: int | None, data_sequence_number: int | None
) -> pa.RecordBatch:
"""Replace null last updated sequence numbers with the data sequence number of the file.

When the data file has no first row ID, every last updated sequence number is null.
"""
if first_row_id is None:
sequence_numbers = pa.nulls(len(batch), type=pa.int64())
elif data_sequence_number is None:
# REST scan planning does not return the data sequence number
raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} requires the data sequence number of the file")
else:
sequence_numbers = pc.coalesce(batch.column(last_updated_index), pa.scalar(data_sequence_number, type=pa.int64()))
return batch.set_column(last_updated_index, batch.schema.field(last_updated_index), sequence_numbers)


@dataclass(frozen=True, slots=True)
class _DeleteFileKey:
"""Identity of a delete file or a referenced DV range."""
Expand Down Expand Up @@ -1937,9 +1966,6 @@ def __init__(
*,
dictionary_columns: tuple[str, ...] = (),
) -> None:
if any(field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id for field in projected_schema.fields):
# The data sequence number is not carried on the scan task yet
raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} is not yet supported")
self._table_metadata = table_metadata
self._io = io
self._projected_schema = projected_schema
Expand Down
4 changes: 4 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2328,16 +2328,19 @@ class FileScanTask(ScanTask):
file: DataFile
delete_files: set[DataFile]
residual: BooleanExpression
data_sequence_number: int | None

def __init__(
self,
data_file: DataFile,
delete_files: set[DataFile] | None = None,
residual: BooleanExpression = ALWAYS_TRUE,
data_sequence_number: int | None = None,
) -> None:
self.file = data_file
self.delete_files = delete_files or set()
self.residual = residual
self.data_sequence_number = data_sequence_number

@staticmethod
def from_rest_response(
Expand Down Expand Up @@ -2996,6 +2999,7 @@ def residual_for(data_file: DataFile) -> BooleanExpression:
partition_key=data_entry.data_file.partition,
),
residual=ALWAYS_TRUE if self._ignore_residuals else residual_for(data_entry.data_file),
data_sequence_number=data_entry.sequence_number,
)
for data_entry in data_entries
]
Expand Down
4 changes: 3 additions & 1 deletion pyiceberg/table/inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -784,6 +784,7 @@ def _get_files_from_manifest(
"split_offsets": data_file.split_offsets,
"equality_ids": data_file.equality_ids,
"sort_order_id": data_file.sort_order_id,
"first_row_id": data_file.first_row_id,
"readable_metrics": readable_metrics,
}
)
Expand All @@ -804,7 +805,7 @@ def _get_files_schema(self) -> pa.Schema:
``file_size_in_bytes``, ``column_sizes``, ``value_counts``,
``null_value_counts``, ``nan_value_counts``, ``lower_bounds``,
``upper_bounds``, ``key_metadata``, ``split_offsets``,
``equality_ids``, ``sort_order_id``, and ``readable_metrics``.
``equality_ids``, ``sort_order_id``, ``first_row_id``, and ``readable_metrics``.
"""
import pyarrow as pa

Expand Down Expand Up @@ -853,6 +854,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:
pa.field("split_offsets", pa.list_(pa.int64()), nullable=True),
pa.field("equality_ids", pa.list_(pa.int32()), nullable=True),
pa.field("sort_order_id", pa.int32(), nullable=True),
pa.field("first_row_id", pa.int64(), nullable=True),
pa.field("readable_metrics", pa.struct(readable_metrics_struct), nullable=True),
]
)
Expand Down
4 changes: 4 additions & 0 deletions tests/integration/test_inspect_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None:
"split_offsets",
"equality_ids",
"sort_order_id",
"first_row_id",
"readable_metrics",
]

Expand Down Expand Up @@ -132,6 +133,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None:
"split_offsets",
"equality_ids",
"sort_order_id",
"first_row_id",
]
]
rhs_subset = rhs[
Expand All @@ -145,6 +147,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None:
"split_offsets",
"equality_ids",
"sort_order_id",
"first_row_id",
]
]

Expand Down Expand Up @@ -943,6 +946,7 @@ def inspect_files_asserts(df: pa.Table) -> None:
"split_offsets",
"equality_ids",
"sort_order_id",
"first_row_id",
"readable_metrics",
]

Expand Down
21 changes: 10 additions & 11 deletions tests/integration/test_reads.py
Original file line number Diff line number Diff line change
Expand Up @@ -1301,8 +1301,8 @@ def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSessi

@pytest.mark.integration
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:
identifier = "default.test_read_row_ids_written_by_spark"
def test_read_row_lineage_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:
identifier = "default.test_read_row_lineage_written_by_spark"
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
spark.sql(
f"""
Expand Down Expand Up @@ -1335,17 +1335,16 @@ def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) ->
# Rows in a new file inherit row IDs from their position, so a filter must not drop rows before they are counted
spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (8, 'h'), (9, 'i'), (10, 'j')")

expected = sorted((row.id, row._row_id) for row in spark.sql(f"SELECT id, _row_id FROM {identifier}").collect())
assert all(row_id is not None for _, row_id in expected)
columns = ("id", "_row_id", "_last_updated_sequence_number")
expected = sorted(tuple(row) for row in spark.sql(f"SELECT {', '.join(columns)} FROM {identifier}").collect())
assert all(value is not None for row in expected for value in row)

table = catalog.load_table(identifier)
result = table.scan(selected_fields=("id", "_row_id")).to_arrow()
assert sorted(zip(result["id"].to_pylist(), result["_row_id"].to_pylist(), strict=True)) == expected
def rows(result: pa.Table) -> list[tuple[int, int, int]]:
return sorted(zip(*(result[column].to_pylist() for column in columns), strict=True))

filtered = table.scan(row_filter="id >= 9", selected_fields=("id", "_row_id")).to_arrow()
assert sorted(zip(filtered["id"].to_pylist(), filtered["_row_id"].to_pylist(), strict=True)) == [
(id_, row_id) for id_, row_id in expected if id_ >= 9
]
table = catalog.load_table(identifier)
assert rows(table.scan(selected_fields=columns).to_arrow()) == expected
assert rows(table.scan(row_filter="id >= 9", selected_fields=columns).to_arrow()) == [row for row in expected if row[0] >= 9]


@pytest.mark.integration
Expand Down
75 changes: 69 additions & 6 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1344,12 +1344,6 @@ def test_read_map(schema_map: Schema, file_map: str) -> None:
)


def test_projection_last_updated_sequence_number_not_supported(file_int: str) -> None:
schema = Schema(NestedField(1, "id", IntegerType(), required=False), LAST_UPDATED_SEQUENCE_NUMBER)
with pytest.raises(NotImplementedError, match="Reading _last_updated_sequence_number is not yet supported"):
project(schema, [file_int])


def test_projection_add_column_struct(schema_int: Schema, file_int: str) -> None:
schema = Schema(
# A new ID
Expand Down Expand Up @@ -4501,6 +4495,75 @@ def test_task_to_record_batches_row_ids(
assert list(zip(result["id"].to_pylist(), result[ROW_ID.name].to_pylist(), strict=True)) == expected


def _read_last_updated_sequence_numbers(
tmpdir: str, stored_sequence_numbers: list[int | None] | None, first_row_id: int | None, data_sequence_number: int | None
) -> list[int | None]:
fields = [pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})]
columns = [pa.array([1, 2, 3], type=pa.int32())]
if stored_sequence_numbers is not None:
fields.append(
pa.field(
LAST_UPDATED_SEQUENCE_NUMBER.name,
pa.int64(),
nullable=True,
metadata={PYARROW_PARQUET_FIELD_ID_KEY: str(LAST_UPDATED_SEQUENCE_NUMBER.field_id)},
)
)
columns.append(pa.array(stored_sequence_numbers, type=pa.int64()))
arrow_schema = pa.schema(fields)
data_file = _write_table_to_data_file(
f"{tmpdir}/last_updated_sequence_numbers.parquet", arrow_schema, pa.table(columns, schema=arrow_schema)
)
data_file.first_row_id = first_row_id

table_schema = Schema(NestedField(1, "id", IntegerType(), required=False))
projected_schema = Schema(*table_schema.fields, ROW_ID, LAST_UPDATED_SEQUENCE_NUMBER)
result_batches = list(
_task_to_record_batches(
PyArrowFileIO(),
FileScanTask(data_file, data_sequence_number=data_sequence_number),
bound_row_filter=AlwaysTrue(),
projected_schema=projected_schema,
table_schema=table_schema,
projected_field_ids={1, ROW_ID.field_id, LAST_UPDATED_SEQUENCE_NUMBER.field_id},
positional_deletes=None,
case_sensitive=True,
)
)

result = pa.Table.from_batches(result_batches)
assert result.schema.field(LAST_UPDATED_SEQUENCE_NUMBER.name).type == pa.int64()
return result[LAST_UPDATED_SEQUENCE_NUMBER.name].to_pylist()


@pytest.mark.parametrize(
"stored_sequence_numbers, first_row_id, data_sequence_number, expected",
[
# Last updated sequence numbers are inherited from the data sequence number of the file
(None, 100, 5, [5, 5, 5]),
# Last updated sequence numbers stored in the file take precedence
([None, 2, None], 100, 5, [5, 2, 5]),
# Without a first row ID, last updated sequence numbers are null even when stored in the file
([None, 2, None], None, 5, [None, None, None]),
# Without a first row ID, the data sequence number is not needed
([None, 2, None], None, None, [None, None, None]),
],
)
def test_task_to_record_batches_last_updated_sequence_numbers(
tmpdir: str,
stored_sequence_numbers: list[int | None] | None,
first_row_id: int | None,
data_sequence_number: int | None,
expected: list[int | None],
) -> None:
assert _read_last_updated_sequence_numbers(tmpdir, stored_sequence_numbers, first_row_id, data_sequence_number) == expected


def test_task_to_record_batches_last_updated_sequence_numbers_without_data_sequence_number(tmpdir: str) -> None:
with pytest.raises(NotImplementedError, match="requires the data sequence number"):
_read_last_updated_sequence_numbers(tmpdir, [None, 2, None], first_row_id=100, data_sequence_number=None)


def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None:
"""The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md."""
data_file = DataFile.from_args(
Expand Down
Loading