diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index e9d17631fb..af1ab9c55b 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -238,13 +238,23 @@ def _s3(properties: Properties) -> AbstractFileSystem: if s3_additional_kwargs: s3_fs_kwargs["s3_additional_kwargs"] = s3_additional_kwargs - fs = S3FileSystem(**s3_fs_kwargs) - - for event_name, event_function in register_events.items(): - fs.s3.meta.events.unregister(event_name, unique_id=1925) - fs.s3.meta.events.register_last(event_name, event_function, unique_id=1925) - - return fs + if register_events: + + class _SignedS3FileSystem(S3FileSystem): + async def set_session(self, *args: Any, **kwargs: Any) -> Any: + s3 = await super().set_session(*args, **kwargs) + for event_name, event_function in register_events.items(): + s3.meta.events.unregister(event_name, unique_id=1925) + s3.meta.events.register_last(event_name, event_function, unique_id=1925) + return s3 + + fs = _SignedS3FileSystem(**s3_fs_kwargs, skip_instance_cache=True) + for event_name, event_function in register_events.items(): + fs.s3.meta.events.unregister(event_name, unique_id=1925) + fs.s3.meta.events.register_last(event_name, event_function, unique_id=1925) + return fs + + return S3FileSystem(**s3_fs_kwargs) def _gs(properties: Properties) -> AbstractFileSystem: diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index b539edd848..bbc61c7d31 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -1262,3 +1262,69 @@ def auth_header(self) -> str: assert requests_mock.last_request is not None assert requests_mock.last_request.headers["Authorization"] == "Bearer via-manager" assert request.url == new_uri + +def test_s3_signer_isolated_across_file_ios(monkeypatch: pytest.MonkeyPatch) -> None: + from s3fs.core import sync + + from pyiceberg.io.fsspec import SIGNERS, FsspecFileIO + + calls: list[str] = [] + + def make_signer(tag: str): + def _factory(_properties): + def _sign(request, **kwargs): + calls.append(tag) + + return _sign + + return _factory + + monkeypatch.setitem(SIGNERS, "test-signer-a", make_signer("a")) + monkeypatch.setitem(SIGNERS, "test-signer-b", make_signer("b")) + + base_props = {"s3.endpoint": "http://localhost:9000", "s3.region": "us-east-1"} + io_a = FsspecFileIO({**base_props, "s3.signer": "test-signer-a"}) + io_b = FsspecFileIO({**base_props, "s3.signer": "test-signer-b"}) + + fs_a = io_a.get_fs("s3") + fs_b = io_b.get_fs("s3") + + assert fs_a is not fs_b + assert fs_a.s3 is not fs_b.s3 + + sync(fs_a.loop, fs_a.s3.generate_presigned_url, "head_object", Params={"Bucket": "b", "Key": "k"}) + sync(fs_b.loop, fs_b.s3.generate_presigned_url, "head_object", Params={"Bucket": "b", "Key": "k"}) + assert calls == ["a", "b"] + + +def test_s3_signer_re_registered_on_set_session(monkeypatch: pytest.MonkeyPatch) -> None: + from s3fs.core import sync + + from pyiceberg.io.fsspec import SIGNERS, FsspecFileIO + + calls: list[str] = [] + + def _factory(_properties): + def _sign(request, **kwargs): + calls.append("signed") + + return _sign + + monkeypatch.setitem(SIGNERS, "test-signer-refresh", _factory) + + io = FsspecFileIO( + { + "s3.endpoint": "http://localhost:9000", + "s3.region": "us-east-1", + "s3.signer": "test-signer-refresh", + } + ) + fs = io.get_fs("s3") + old_s3 = fs.s3 + + fs._s3 = None + sync(fs.loop, fs.set_session, refresh=True) + assert fs.s3 is not old_s3 + + sync(fs.loop, fs.s3.generate_presigned_url, "head_object", Params={"Bucket": "b", "Key": "k"}) + assert calls == ["signed"] \ No newline at end of file