Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions mkdocs/docs/geospatial.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Both types store values as WKB (Well-Known Binary) bytes.
## Requirements

- Iceberg format version 3 or higher
- Optional: `geoarrow-pyarrow` for GeoArrow extension type metadata and interoperability. Without it, geometry and geography are written as binary in Parquet while the Iceberg schema still preserves the spatial type. Install with `pip install pyiceberg[geoarrow]`.
- Optional: `geoarrow-types` for GeoArrow extension type metadata and interoperability. Without it, geometry and geography are written as binary in Parquet while the Iceberg schema still preserves the spatial type. Install with `pip install pyiceberg[geoarrow]`.

## Usage

Expand Down Expand Up @@ -88,7 +88,7 @@ point_wkb = bytes.fromhex("0101000000000000000000000000000000000000")

3. **Bounds Metrics**: Geometry/geography columns do not currently contribute to data file bounds metrics.

4. **Without geoarrow-pyarrow**: When the `geoarrow-pyarrow` package is not installed, geometry and geography columns are stored as binary without GeoArrow extension type metadata. The Iceberg schema preserves type information, but other tools reading the Parquet files directly may not recognize them as spatial types. Install with `pip install pyiceberg[geoarrow]` for full GeoArrow support.
4. **Without geoarrow-types**: When the `geoarrow-types` package is not installed, geometry and geography columns are stored as binary without GeoArrow extension type metadata. The Iceberg schema preserves type information, but other tools reading the Parquet files directly may not recognize them as spatial types. Install with `pip install pyiceberg[geoarrow]` for full GeoArrow support.

## Format Version

Expand Down
113 changes: 97 additions & 16 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import functools
import importlib
import itertools
import json
import logging
import math
import operator
Expand Down Expand Up @@ -162,6 +163,7 @@
from pyiceberg.transforms import IdentityTransform, TruncateTransform
from pyiceberg.typedef import EMPTY_DICT, Properties, Record, TableVersion
from pyiceberg.types import (
DEFAULT_GEOMETRY_CRS,
BinaryType,
BooleanType,
DateType,
Expand Down Expand Up @@ -856,33 +858,50 @@ def visit_binary(self, _: BinaryType) -> pa.DataType:
def visit_geometry(self, geometry_type: GeometryType) -> pa.DataType:
"""Convert geometry type to PyArrow type.

When geoarrow-pyarrow is available, returns a GeoArrow WKB extension type
When geoarrow-types is available, returns a GeoArrow WKB extension type
with CRS metadata. Otherwise, falls back to large_binary which stores WKB bytes.
"""
try:
import geoarrow.pyarrow as ga

return ga.wkb().with_crs(geometry_type.crs)
import geoarrow.types as ga
from geoarrow.types.type_pyarrow import register_extension_types

register_extension_types()

# Iceberg cannot represent a GeoArrow/GeoPandas/GeoParquet None CRS as None,
# as its default is OGC:CRS84. Instead, we use the SRID:0 string in the
# Iceberg and Parquet specification examples as the sentinel. This follows
# the meaning of srid:0 in the GeoParquet 2.0 specification, PostGIS, and
# GeoPackage, as well as its mapping to the Parquet logical type in arrow-rs
# and Arrow C++.
crs = None if geometry_type.crs.lower() == "srid:0" else geometry_type.crs
return ga.wkb(crs=crs).to_pyarrow()
except ImportError:
return pa.large_binary()

def visit_geography(self, geography_type: GeographyType) -> pa.DataType:
"""Convert geography type to PyArrow type.

When geoarrow-pyarrow is available, returns a GeoArrow WKB extension type
When geoarrow-types is available, returns a GeoArrow WKB extension type
with CRS and edge type metadata. Otherwise, falls back to large_binary which stores WKB bytes.
"""
try:
import geoarrow.pyarrow as ga

wkb_type = ga.wkb().with_crs(geography_type.crs)
# Map Iceberg algorithm to GeoArrow edge type
if geography_type.algorithm == "spherical":
wkb_type = wkb_type.with_edge_type(ga.EdgeType.SPHERICAL)
# "planar" is the default edge type in GeoArrow, no need to set explicitly
return wkb_type
import geoarrow.types as ga
from geoarrow.types.type_pyarrow import register_extension_types

register_extension_types()

# Other unsupported values are validated by ga.wkb(); however, this is a valid value
# that if passed to ga.wkb() will result in the value being interpreted as geometry
# instead of geography.
if geography_type.algorithm == "planar":
raise ValueError("The planar algorithm is not supported for geography types")

crs = None if geography_type.crs.lower() == "srid:0" else geography_type.crs
return ga.wkb(crs=crs, edge_type=geography_type.algorithm).to_pyarrow()
except ImportError:
return pa.large_binary()
except KeyError as e:
raise ValueError(f"Unsupported geography edge interpolation algorithm: {geography_type.algorithm!r}") from e


def _literal_arrow_type(iceberg_type: IcebergType, values: Iterable[Any]) -> pa.DataType:
Expand Down Expand Up @@ -1473,6 +1492,50 @@ def primitive(self, primitive: pa.DataType) -> bool:
return True


def _geoarrow_crs_to_iceberg(crs: object) -> str:
"""Convert GeoArrow CRS metadata to a compact Iceberg CRS identifier."""
# Missing CRS has a specific meaning in GeoArrow that aligns with a
# GeoParquet "null" CRS and a GeoPandas "None" CRS. From the Iceberg
# and Parquet examples and GeoParquet 2.0 guidance, the corresponding
# Iceberg string is "srid:0".
if crs is None:
return "srid:0"

# PROJJSON CRSes need their authority:code extracted, because the Iceberg
# spec forbids storing a PROJJSON string inline in the schema
# (PROJJSON strings are typically several kilobytes and types are repeated
# in metadata frequently).
if isinstance(crs, dict):
crs_id = crs.get("id")
if (
not isinstance(crs_id, dict)
or not isinstance(crs_id.get("authority"), str)
or not crs_id["authority"]
or not isinstance(crs_id.get("code"), (str, int, float))
or isinstance(crs_id["code"], bool)
):
raise ValueError(
"GeoArrow PROJJSON CRS must have an authority/code ID; inline PROJJSON is not supported in Iceberg schemas"
)
crs = f"{crs_id['authority']}:{crs_id['code']}"

if not isinstance(crs, str):
raise ValueError("GeoArrow CRS must be a string, a PROJJSON object, or null")

# Parquet omits these exact identifiers, defaulting to OGC:CRS84. This ensures
# that the Parquet type and the Iceberg type match for the most common two CRS
# values. The two values are implicitly equivalent according to the Coordinate
# Axis Order of the Parquet specification.
if crs in (DEFAULT_GEOMETRY_CRS, "EPSG:4326"):
return DEFAULT_GEOMETRY_CRS

# Otherwise, preserve the string exactly as PyArrow writes it.
if not crs.strip():
raise ValueError("Geospatial CRS must be non-empty")

return crs


class _ConvertToIceberg(PyArrowSchemaVisitor[IcebergType | Schema]):
"""Converts PyArrowSchema to Iceberg Schema."""

Expand Down Expand Up @@ -1588,6 +1651,14 @@ def primitive(self, primitive: pa.DataType) -> PrimitiveType:
return UnknownType()
elif isinstance(primitive, pa.UuidType):
return UUIDType()
elif isinstance(primitive, pa.ExtensionType) and primitive.extension_name == "geoarrow.wkb":
metadata = json.loads(primitive.__arrow_ext_serialize__())
crs = _geoarrow_crs_to_iceberg(metadata.get("crs"))
edges = metadata.get("edges", "planar")
if edges == "planar":
return GeometryType(crs)
else:
return GeographyType(crs, edges)

raise TypeError(f"Unsupported type: {primitive}")

Expand Down Expand Up @@ -1767,6 +1838,16 @@ def _task_to_record_batches(
format_kwargs: dict[str, Any] = {"pre_buffer": True, "buffer_size": ONE_MEGABYTE * 8}
if dictionary_columns and task.file.file_format == FileFormat.PARQUET:
format_kwargs["dictionary_columns"] = dictionary_columns
if (
task.file.file_format == FileFormat.PARQUET
and hasattr(ds.ParquetFragmentScanOptions, "arrow_extensions_enabled")
and any(
isinstance(table_schema.find_type(field_id), (GeometryType, GeographyType))
for field_id in projected_field_ids & table_schema.field_ids
)
):
# Dataset scans disable logical-type extension decoding by default.
format_kwargs["arrow_extensions_enabled"] = True
Comment on lines +1841 to +1850

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure about this part: it seems like it may turn on a switch that had been turned off deliberately (arrow_extensions_enabled is on by default in pyarrow, so I am guessing it was turned off here)

arrow_format = _create_arrow_read_format(task.file.file_format, task.file.key_metadata, **format_kwargs)
with io.new_input(task.file.file_path).open() as fin:
fragment = arrow_format.make_fragment(fin)
Expand Down Expand Up @@ -1946,6 +2027,8 @@ def __init__(
self._table_metadata = table_metadata
self._io = io
self._projected_schema = projected_schema
# Register extension types before inspecting any data file schemas.
self._arrow_schema = schema_to_pyarrow(projected_schema, include_field_ids=False)
self._bound_row_filter = bind(table_metadata.schema(), row_filter, case_sensitive=case_sensitive)
self._case_sensitive = case_sensitive
self._limit = limit
Expand Down Expand Up @@ -1984,14 +2067,12 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
ResolveError: When a required field cannot be found in the file
ValueError: When a field type in the file cannot be projected to the schema type
"""
arrow_schema = schema_to_pyarrow(self._projected_schema, include_field_ids=False)

batches = self.to_record_batches(tasks)
try:
first_batch = next(batches)
except StopIteration:
# Empty
return arrow_schema.empty_table()
return self._arrow_schema.empty_table()

# Note: cannot use pa.Table.from_batches(itertools.chain([first_batch], batches)))
# as different batches can use different schema's (due to large_ types)
Expand Down
36 changes: 21 additions & 15 deletions pyiceberg/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,12 @@
DEFAULT_GEOGRAPHY_CRS = "OGC:CRS84"
DEFAULT_GEOGRAPHY_ALGORITHM = "spherical"

# Regex patterns for parsing geometry and geography type strings
# Matches: geometry, geometry('CRS'), geometry('crs'), geometry("CRS")
GEOMETRY_REGEX = re.compile(r"geometry(?:\(\s*['\"]([^'\"]+)['\"]\s*\))?$")
# Matches: geography, geography('CRS'), geography('crs', 'algo')
GEOGRAPHY_REGEX = re.compile(r"geography(?:\(\s*['\"]([^'\"]+)['\"](?:\s*,\s*['\"]([^'\"]+)['\"])?\s*\))?$")
# Accept Java's unquoted parameters and the existing quoted forms.
_GEOSPATIAL_PARAMETER = r"""('[^']+'|"[^"]+"|[^'",()\s][^'",()]*?)"""
GEOMETRY_REGEX = re.compile(rf"geometry\s*(?:\(\s*{_GEOSPATIAL_PARAMETER}\s*\))?", re.IGNORECASE)
GEOGRAPHY_REGEX = re.compile(
rf"geography\s*(?:\(\s*{_GEOSPATIAL_PARAMETER}\s*(?:,\s*{_GEOSPATIAL_PARAMETER}\s*)?\))?", re.IGNORECASE
)


def transform_dict_value_to_str(d: dict[str, Any]) -> dict[str, str]:
Expand Down Expand Up @@ -105,6 +106,12 @@ def _parse_fixed_type(fixed: Any) -> int:
return fixed


def _parse_geospatial_parameter(value: str | None, default: str) -> str:
if value is None:
return default
return value[1:-1] if value.startswith(("'", '"')) else value.strip()


def _parse_geometry_type(geometry: Any) -> str:
"""Parse geometry type string and return CRS.

Expand All @@ -115,10 +122,9 @@ def _parse_geometry_type(geometry: Any) -> str:
The CRS string (defaults to DEFAULT_GEOMETRY_CRS if not specified).
"""
if isinstance(geometry, str):
match = GEOMETRY_REGEX.match(geometry)
match = GEOMETRY_REGEX.fullmatch(geometry.strip())
if match:
crs = match.group(1)
return crs if crs else DEFAULT_GEOMETRY_CRS
return _parse_geospatial_parameter(match.group(1), DEFAULT_GEOMETRY_CRS)
else:
raise ValidationError(f"Could not parse {geometry} into a GeometryType")
elif isinstance(geometry, dict):
Expand All @@ -137,10 +143,10 @@ def _parse_geography_type(geography: Any) -> tuple[str, str]:
Tuple of (CRS, algorithm) with defaults applied where not specified.
"""
if isinstance(geography, str):
match = GEOGRAPHY_REGEX.match(geography)
match = GEOGRAPHY_REGEX.fullmatch(geography.strip())
if match:
crs = match.group(1) if match.group(1) else DEFAULT_GEOGRAPHY_CRS
algorithm = match.group(2) if match.group(2) else DEFAULT_GEOGRAPHY_ALGORITHM
crs = _parse_geospatial_parameter(match.group(1), DEFAULT_GEOGRAPHY_CRS)
algorithm = _parse_geospatial_parameter(match.group(2), DEFAULT_GEOGRAPHY_ALGORITHM).lower()
return crs, algorithm
else:
raise ValidationError(f"Could not parse {geography} into a GeographyType")
Expand Down Expand Up @@ -188,9 +194,9 @@ def handle_primitive_type(cls, v: Any, handler: ValidatorFunctionWrapHandler) ->
# a CRS string (or CRS, algorithm tuple). If we try to parse those as type
# strings here, we'd re-enter this validator (or raise) instead of letting
# pydantic validate the raw root values.
if cls.__name__ == "GeometryType" and not v.startswith("geometry"):
if cls.__name__ == "GeometryType" and not v.lstrip().lower().startswith("geometry"):
return handler(v)
if cls.__name__ == "GeographyType" and not v.startswith("geography"):
if cls.__name__ == "GeographyType" and not v.lstrip().lower().startswith("geography"):
return handler(v)

if v == "boolean":
Expand Down Expand Up @@ -228,10 +234,10 @@ def handle_primitive_type(cls, v: Any, handler: ValidatorFunctionWrapHandler) ->
if v.startswith("decimal"):
precision, scale = _parse_decimal_type(v)
return DecimalType(precision, scale)
if v.startswith("geometry"):
if v.lstrip().lower().startswith("geometry"):
crs = _parse_geometry_type(v)
return GeometryType(crs)
if v.startswith("geography"):
if v.lstrip().lower().startswith("geography"):
crs, algorithm = _parse_geography_type(v)
return GeographyType(crs, algorithm)
else:
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ pyiceberg-core = ["pyiceberg-core>=0.10.1,<0.11.0"]
datafusion = ["datafusion>=53,<54"]
gcp-auth = ["google-auth>=2.4.0"]
entra-auth = ["azure-identity>=1.25.1"]
geoarrow = ["geoarrow-pyarrow>=0.2.0"]
geoarrow = ["geoarrow-types>=0.3.0"]
encryption = ["cryptography>=42.0.0"]

[dependency-groups]
Expand Down
45 changes: 45 additions & 0 deletions tests/integration/test_reads.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@
from pyiceberg.types import (
BinaryType,
BooleanType,
GeographyType,
GeometryType,
IntegerType,
LongType,
NestedField,
Expand Down Expand Up @@ -1252,6 +1254,49 @@ def test_initial_default(catalog: Catalog, spark: SparkSession) -> None:
assert len(result_table) == 10


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
def test_read_geospatial_types_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:
pytest.importorskip("geoarrow.types")
identifier = "default.test_read_geospatial_types_written_by_spark"
geom_wkb = "01010000000000000000003e400000000000002440" # POINT (30 10)
geog_wkb = "01010000000000000000c051c00000000000004540" # POINT (-71 42)
geospatial_enabled = spark.conf.get("spark.sql.geospatial.enabled", "false")
spark.conf.set("spark.sql.geospatial.enabled", "true")
try:
spark.sql("CREATE NAMESPACE IF NOT EXISTS default")
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
spark.sql(
f"CREATE TABLE {identifier} (id BIGINT, geom GEOMETRY(4326), geog GEOGRAPHY(4326)) "
"USING iceberg TBLPROPERTIES ('format-version'='3')"
)
spark.sql(
f"INSERT INTO {identifier} VALUES "
f"(1, st_setsrid(st_geomfromwkb(X'{geom_wkb}'), 4326), st_setsrid(st_geogfromwkb(X'{geog_wkb}'), 4326)), "
"(2, NULL, NULL)"
)

table = catalog.load_table(identifier)
assert table.schema().find_type("geom") == GeometryType()
assert table.schema().find_type("geog") == GeographyType()

result = table.scan().to_arrow()
assert sorted(result.to_pylist(), key=lambda row: row["id"]) == [
{"id": 1, "geom": bytes.fromhex(geom_wkb), "geog": bytes.fromhex(geog_wkb)},
{"id": 2, "geom": None, "geog": None},
]
for name in ("geom", "geog"):
arrow_type = result.schema.field(name).type
assert isinstance(arrow_type, pa.ExtensionType)
assert arrow_type.extension_name == "geoarrow.wkb"
assert pyarrow_to_schema(result.schema, name_mapping=table.schema().name_mapping, format_version=3) == table.schema()
finally:
try:
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
finally:
spark.conf.set("spark.sql.geospatial.enabled", geospatial_enabled)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:
Expand Down
Loading
Loading