diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index a732d2c0a3..86665072e7 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -985,6 +985,7 @@ split_offsets: list equality_ids: list child 0, item: int32 sort_order_id: int32 +first_row_id: int64 readable_metrics: struct not null, lat: struct not null, long: struct not null> child 0, city: struct not null child 0, column_size: int64 @@ -1024,6 +1025,7 @@ key_metadata: [[0100,0100]] split_offsets:[[[],[]]] equality_ids:[[[],[]]] sort_order_id:[[[],[]]] +first_row_id:[[null,null]] readable_metrics: [ -- is_valid: all not null -- child 0 type: struct diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 5961da883c..67d7360729 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1801,6 +1801,14 @@ def _task_to_record_batches( (index for index, field in enumerate(projected_schema.fields) if field.field_id == ROW_ID.field_id), None ) filter_after_read = bool(positional_deletes) or row_id_index is not None + last_updated_index = next( + ( + index + for index, field in enumerate(projected_schema.fields) + if field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id + ), + None, + ) fragment_scanner = ds.Scanner.from_fragment( fragment=fragment, @@ -1856,6 +1864,10 @@ def _task_to_record_batches( ) if row_id_index is not None and positions is not None: result_batch = _assign_row_ids(result_batch, row_id_index, positions, task.file.first_row_id) + if last_updated_index is not None: + result_batch = _assign_last_updated_sequence_numbers( + result_batch, last_updated_index, task.file.first_row_id, task.data_sequence_number + ) yield result_batch @@ -1871,6 +1883,23 @@ def _assign_row_ids(batch: pa.RecordBatch, row_id_index: int, positions: pa.Arra return batch.set_column(row_id_index, batch.schema.field(row_id_index), row_ids) +def _assign_last_updated_sequence_numbers( + batch: pa.RecordBatch, last_updated_index: int, first_row_id: int | None, data_sequence_number: int | None +) -> pa.RecordBatch: + """Replace null last updated sequence numbers with the data sequence number of the file. + + When the data file has no first row ID, every last updated sequence number is null. + """ + if first_row_id is None: + sequence_numbers = pa.nulls(len(batch), type=pa.int64()) + elif data_sequence_number is None: + # REST scan planning does not return the data sequence number + raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} requires the data sequence number of the file") + else: + sequence_numbers = pc.coalesce(batch.column(last_updated_index), pa.scalar(data_sequence_number, type=pa.int64())) + return batch.set_column(last_updated_index, batch.schema.field(last_updated_index), sequence_numbers) + + @dataclass(frozen=True, slots=True) class _DeleteFileKey: """Identity of a delete file or a referenced DV range.""" @@ -1937,9 +1966,6 @@ def __init__( *, dictionary_columns: tuple[str, ...] = (), ) -> None: - if any(field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id for field in projected_schema.fields): - # The data sequence number is not carried on the scan task yet - raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} is not yet supported") self._table_metadata = table_metadata self._io = io self._projected_schema = projected_schema diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 4d65084cbf..de6f9e4b6b 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -2328,16 +2328,19 @@ class FileScanTask(ScanTask): file: DataFile delete_files: set[DataFile] residual: BooleanExpression + data_sequence_number: int | None def __init__( self, data_file: DataFile, delete_files: set[DataFile] | None = None, residual: BooleanExpression = ALWAYS_TRUE, + data_sequence_number: int | None = None, ) -> None: self.file = data_file self.delete_files = delete_files or set() self.residual = residual + self.data_sequence_number = data_sequence_number @staticmethod def from_rest_response( @@ -2996,6 +2999,7 @@ def residual_for(data_file: DataFile) -> BooleanExpression: partition_key=data_entry.data_file.partition, ), residual=ALWAYS_TRUE if self._ignore_residuals else residual_for(data_entry.data_file), + data_sequence_number=data_entry.sequence_number, ) for data_entry in data_entries ] diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index d51079fc9d..6ccc6a500d 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -784,6 +784,7 @@ def _get_files_from_manifest( "split_offsets": data_file.split_offsets, "equality_ids": data_file.equality_ids, "sort_order_id": data_file.sort_order_id, + "first_row_id": data_file.first_row_id, "readable_metrics": readable_metrics, } ) @@ -804,7 +805,7 @@ def _get_files_schema(self) -> pa.Schema: ``file_size_in_bytes``, ``column_sizes``, ``value_counts``, ``null_value_counts``, ``nan_value_counts``, ``lower_bounds``, ``upper_bounds``, ``key_metadata``, ``split_offsets``, - ``equality_ids``, ``sort_order_id``, and ``readable_metrics``. + ``equality_ids``, ``sort_order_id``, ``first_row_id``, and ``readable_metrics``. """ import pyarrow as pa @@ -853,6 +854,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("split_offsets", pa.list_(pa.int64()), nullable=True), pa.field("equality_ids", pa.list_(pa.int32()), nullable=True), pa.field("sort_order_id", pa.int32(), nullable=True), + pa.field("first_row_id", pa.int64(), nullable=True), pa.field("readable_metrics", pa.struct(readable_metrics_struct), nullable=True), ] ) diff --git a/tests/integration/test_inspect_table.py b/tests/integration/test_inspect_table.py index 4d8dfbe9bb..7868bb0f25 100644 --- a/tests/integration/test_inspect_table.py +++ b/tests/integration/test_inspect_table.py @@ -101,6 +101,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", "readable_metrics", ] @@ -132,6 +133,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", ] ] rhs_subset = rhs[ @@ -145,6 +147,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", ] ] @@ -943,6 +946,7 @@ def inspect_files_asserts(df: pa.Table) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", "readable_metrics", ] diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index f3e55f48e6..6a524140a2 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -1301,8 +1301,8 @@ def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSessi @pytest.mark.integration @pytest.mark.parametrize("catalog", [lf("session_catalog")]) -def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None: - identifier = "default.test_read_row_ids_written_by_spark" +def test_read_row_lineage_written_by_spark(catalog: Catalog, spark: SparkSession) -> None: + identifier = "default.test_read_row_lineage_written_by_spark" spark.sql(f"DROP TABLE IF EXISTS {identifier}") spark.sql( f""" @@ -1335,17 +1335,16 @@ def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> # Rows in a new file inherit row IDs from their position, so a filter must not drop rows before they are counted spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (8, 'h'), (9, 'i'), (10, 'j')") - expected = sorted((row.id, row._row_id) for row in spark.sql(f"SELECT id, _row_id FROM {identifier}").collect()) - assert all(row_id is not None for _, row_id in expected) + columns = ("id", "_row_id", "_last_updated_sequence_number") + expected = sorted(tuple(row) for row in spark.sql(f"SELECT {', '.join(columns)} FROM {identifier}").collect()) + assert all(value is not None for row in expected for value in row) - table = catalog.load_table(identifier) - result = table.scan(selected_fields=("id", "_row_id")).to_arrow() - assert sorted(zip(result["id"].to_pylist(), result["_row_id"].to_pylist(), strict=True)) == expected + def rows(result: pa.Table) -> list[tuple[int, int, int]]: + return sorted(zip(*(result[column].to_pylist() for column in columns), strict=True)) - filtered = table.scan(row_filter="id >= 9", selected_fields=("id", "_row_id")).to_arrow() - assert sorted(zip(filtered["id"].to_pylist(), filtered["_row_id"].to_pylist(), strict=True)) == [ - (id_, row_id) for id_, row_id in expected if id_ >= 9 - ] + table = catalog.load_table(identifier) + assert rows(table.scan(selected_fields=columns).to_arrow()) == expected + assert rows(table.scan(row_filter="id >= 9", selected_fields=columns).to_arrow()) == [row for row in expected if row[0] >= 9] @pytest.mark.integration diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..161058536c 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -1344,12 +1344,6 @@ def test_read_map(schema_map: Schema, file_map: str) -> None: ) -def test_projection_last_updated_sequence_number_not_supported(file_int: str) -> None: - schema = Schema(NestedField(1, "id", IntegerType(), required=False), LAST_UPDATED_SEQUENCE_NUMBER) - with pytest.raises(NotImplementedError, match="Reading _last_updated_sequence_number is not yet supported"): - project(schema, [file_int]) - - def test_projection_add_column_struct(schema_int: Schema, file_int: str) -> None: schema = Schema( # A new ID @@ -4501,6 +4495,75 @@ def test_task_to_record_batches_row_ids( assert list(zip(result["id"].to_pylist(), result[ROW_ID.name].to_pylist(), strict=True)) == expected +def _read_last_updated_sequence_numbers( + tmpdir: str, stored_sequence_numbers: list[int | None] | None, first_row_id: int | None, data_sequence_number: int | None +) -> list[int | None]: + fields = [pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})] + columns = [pa.array([1, 2, 3], type=pa.int32())] + if stored_sequence_numbers is not None: + fields.append( + pa.field( + LAST_UPDATED_SEQUENCE_NUMBER.name, + pa.int64(), + nullable=True, + metadata={PYARROW_PARQUET_FIELD_ID_KEY: str(LAST_UPDATED_SEQUENCE_NUMBER.field_id)}, + ) + ) + columns.append(pa.array(stored_sequence_numbers, type=pa.int64())) + arrow_schema = pa.schema(fields) + data_file = _write_table_to_data_file( + f"{tmpdir}/last_updated_sequence_numbers.parquet", arrow_schema, pa.table(columns, schema=arrow_schema) + ) + data_file.first_row_id = first_row_id + + table_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) + projected_schema = Schema(*table_schema.fields, ROW_ID, LAST_UPDATED_SEQUENCE_NUMBER) + result_batches = list( + _task_to_record_batches( + PyArrowFileIO(), + FileScanTask(data_file, data_sequence_number=data_sequence_number), + bound_row_filter=AlwaysTrue(), + projected_schema=projected_schema, + table_schema=table_schema, + projected_field_ids={1, ROW_ID.field_id, LAST_UPDATED_SEQUENCE_NUMBER.field_id}, + positional_deletes=None, + case_sensitive=True, + ) + ) + + result = pa.Table.from_batches(result_batches) + assert result.schema.field(LAST_UPDATED_SEQUENCE_NUMBER.name).type == pa.int64() + return result[LAST_UPDATED_SEQUENCE_NUMBER.name].to_pylist() + + +@pytest.mark.parametrize( + "stored_sequence_numbers, first_row_id, data_sequence_number, expected", + [ + # Last updated sequence numbers are inherited from the data sequence number of the file + (None, 100, 5, [5, 5, 5]), + # Last updated sequence numbers stored in the file take precedence + ([None, 2, None], 100, 5, [5, 2, 5]), + # Without a first row ID, last updated sequence numbers are null even when stored in the file + ([None, 2, None], None, 5, [None, None, None]), + # Without a first row ID, the data sequence number is not needed + ([None, 2, None], None, None, [None, None, None]), + ], +) +def test_task_to_record_batches_last_updated_sequence_numbers( + tmpdir: str, + stored_sequence_numbers: list[int | None] | None, + first_row_id: int | None, + data_sequence_number: int | None, + expected: list[int | None], +) -> None: + assert _read_last_updated_sequence_numbers(tmpdir, stored_sequence_numbers, first_row_id, data_sequence_number) == expected + + +def test_task_to_record_batches_last_updated_sequence_numbers_without_data_sequence_number(tmpdir: str) -> None: + with pytest.raises(NotImplementedError, match="requires the data sequence number"): + _read_last_updated_sequence_numbers(tmpdir, [None, 2, None], first_row_id=100, data_sequence_number=None) + + def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None: """The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" data_file = DataFile.from_args(