From 9ac29a6c37cacc01e3bb68294e074b7f2e12ae94 Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:35:07 +0200 Subject: [PATCH 1/5] fix: skip object store prefix markers in list_dir Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> --- changes/4032.bugfix.md | 1 + src/zarr/storage/_obstore.py | 6 +++++- tests/test_store/test_object.py | 16 +++++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 changes/4032.bugfix.md diff --git a/changes/4032.bugfix.md b/changes/4032.bugfix.md new file mode 100644 index 0000000000..4c9476f583 --- /dev/null +++ b/changes/4032.bugfix.md @@ -0,0 +1 @@ +Fixed `ObjectStore.list_dir` for object-store listings that include a directory-marker object matching the requested non-root prefix. diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index ffea523f9f..9e9e2d4239 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -267,7 +267,11 @@ async def _transform_list_dir( for path in chain( list_result["common_prefixes"], map(itemgetter("path"), list_result["objects"]) ): - yield _relativize_path(path=path, prefix=prefix) + if prefix and path in {prefix, f"{prefix}/"}: + continue + relpath = _relativize_path(path=path, prefix=prefix) + if relpath: + yield relpath class _BoundedRequest(TypedDict): diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py index 6a4b796639..651130b55e 100644 --- a/tests/test_store/test_object.py +++ b/tests/test_store/test_object.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 from pathlib import Path -from typing import TypedDict +from typing import Any, TypedDict import pytest @@ -13,6 +13,7 @@ from zarr.core.buffer import Buffer, cpu from zarr.storage import ObjectStore +from zarr.storage._obstore import _transform_list_dir from zarr.testing.stateful import ZarrHierarchyStateMachine from zarr.testing.store import StoreTests @@ -97,6 +98,19 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non assert total_size == len(buf) * 2 +@pytest.mark.parametrize("prefix", ["g", "g/"]) +async def test_transform_list_dir_skips_prefix_marker(prefix: str) -> None: + async def list_result() -> Any: + return { + "common_prefixes": ["g/child/"], + "objects": [{"path": "g"}, {"path": "g/"}, {"path": "g/item"}], + } + + result = [path async for path in _transform_list_dir(list_result(), prefix)] + + assert result == ["child/", "item"] + + @pytest.mark.slow_hypothesis def test_zarr_hierarchy() -> None: sync_store = ObjectStore(MemoryStore()) From ecef745883219da97e17b0498c1c0c0d63f562f2 Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:31:28 +0200 Subject: [PATCH 2/5] Address list_dir prefix marker review Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> --- src/zarr/core/array.py | 1 + src/zarr/storage/_memory.py | 2 +- src/zarr/storage/_obstore.py | 2 +- src/zarr/testing/store.py | 15 +++++++++++++++ tests/test_array.py | 4 +++- tests/test_store/test_object.py | 17 +---------------- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index ea7d125b10..bb078587a5 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -3994,6 +3994,7 @@ async def _shards_initialized( ] store_contents_relative = [ _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents + if not array.store_path.path or key != array.store_path.path ] return tuple( chunk_key for chunk_key in array._iter_shard_keys() if chunk_key in store_contents_relative diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index e867706155..5f5b632d4a 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -219,7 +219,7 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: keys_unique = { key.removeprefix(f"{prefix}/").split("/")[0] for key in self._store_dict - if key.startswith(f"{prefix}/") and key != prefix + if key.startswith(f"{prefix}/") and key not in {prefix, f"{prefix}/"} } for key in keys_unique: diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index 9e9e2d4239..849495b634 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -267,7 +267,7 @@ async def _transform_list_dir( for path in chain( list_result["common_prefixes"], map(itemgetter("path"), list_result["objects"]) ): - if prefix and path in {prefix, f"{prefix}/"}: + if prefix and path == prefix: continue relpath = _relativize_path(path=path, prefix=prefix) if relpath: diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index fb87a69a09..bb29e2dc96 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -554,6 +554,21 @@ async def test_list_dir(self, store: S) -> None: keys_observed = await _collect_aiterator(store.list_dir(f"{root}/")) assert sorted(keys_expected) == sorted(keys_observed) + async def test_list_dir_ignores_prefix_marker(self, store: S) -> None: + data = self.buffer_cls.from_bytes(b"") + root = "foo" + try: + await self.set(store, f"{root}/", data) + await self.set(store, f"{root}/child", data) + except OSError as exc: + pytest.skip(f"store cannot represent directory marker objects: {exc}") + + keys_observed = await _collect_aiterator(store.list_dir(root)) + assert keys_observed == ("child",) + + keys_observed = await _collect_aiterator(store.list_dir(f"{root}/")) + assert keys_observed == ("child",) + async def test_set_if_not_exists(self, store: S) -> None: key = "k" data_buf = self.buffer_cls.from_bytes(b"0000") diff --git a/tests/test_array.py b/tests/test_array.py index 89d7547e78..508cae693f 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -45,7 +45,7 @@ default_serializer_v3, ) from zarr.core.array_spec import ArrayConfig, ArrayConfigParams -from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype +from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, cpu, default_buffer_prototype from zarr.core.chunk_grids import ( SHARDED_INNER_CHUNK_MAX_BYTES, guess_chunks, @@ -447,6 +447,8 @@ async def test_chunks_initialized( arr = zarr.create_array( store, name=path, shape=shape, shards=shard_shape, chunks=chunk_shape, dtype="i1" ) + if path: + await store.set(path, cpu.Buffer.from_bytes(b"")) chunks_accumulated = tuple( accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_shard_keys())) diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py index 651130b55e..4459149771 100644 --- a/tests/test_store/test_object.py +++ b/tests/test_store/test_object.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 from pathlib import Path -from typing import Any, TypedDict +from typing import TypedDict import pytest @@ -13,7 +13,6 @@ from zarr.core.buffer import Buffer, cpu from zarr.storage import ObjectStore -from zarr.storage._obstore import _transform_list_dir from zarr.testing.stateful import ZarrHierarchyStateMachine from zarr.testing.store import StoreTests @@ -97,20 +96,6 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non total_size = await store.getsize_prefix("c") assert total_size == len(buf) * 2 - -@pytest.mark.parametrize("prefix", ["g", "g/"]) -async def test_transform_list_dir_skips_prefix_marker(prefix: str) -> None: - async def list_result() -> Any: - return { - "common_prefixes": ["g/child/"], - "objects": [{"path": "g"}, {"path": "g/"}, {"path": "g/item"}], - } - - result = [path async for path in _transform_list_dir(list_result(), prefix)] - - assert result == ["child/", "item"] - - @pytest.mark.slow_hypothesis def test_zarr_hierarchy() -> None: sync_store = ObjectStore(MemoryStore()) From 88263b90846c843a855926dc0b3db91279207ddb Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:38:05 +0200 Subject: [PATCH 3/5] Format prefix marker follow-up Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> --- src/zarr/core/array.py | 3 ++- tests/test_store/test_object.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index bb078587a5..cfdaab1b65 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -3993,7 +3993,8 @@ async def _shards_initialized( x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path) ] store_contents_relative = [ - _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents + _relativize_path(path=key, prefix=array.store_path.path) + for key in store_contents if not array.store_path.path or key != array.store_path.path ] return tuple( diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py index 4459149771..6a4b796639 100644 --- a/tests/test_store/test_object.py +++ b/tests/test_store/test_object.py @@ -96,6 +96,7 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non total_size = await store.getsize_prefix("c") assert total_size == len(buf) * 2 + @pytest.mark.slow_hypothesis def test_zarr_hierarchy() -> None: sync_store = ObjectStore(MemoryStore()) From a8f54f1b58a7ff365561031e0e6475ac84c3be5d Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Fri, 10 Jul 2026 01:55:33 +0200 Subject: [PATCH 4/5] Test S3 prefix marker handling Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> --- src/zarr/testing/store.py | 15 --------------- tests/test_store/test_object.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index bb29e2dc96..fb87a69a09 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -554,21 +554,6 @@ async def test_list_dir(self, store: S) -> None: keys_observed = await _collect_aiterator(store.list_dir(f"{root}/")) assert sorted(keys_expected) == sorted(keys_observed) - async def test_list_dir_ignores_prefix_marker(self, store: S) -> None: - data = self.buffer_cls.from_bytes(b"") - root = "foo" - try: - await self.set(store, f"{root}/", data) - await self.set(store, f"{root}/child", data) - except OSError as exc: - pytest.skip(f"store cannot represent directory marker objects: {exc}") - - keys_observed = await _collect_aiterator(store.list_dir(root)) - assert keys_observed == ("child",) - - keys_observed = await _collect_aiterator(store.list_dir(f"{root}/")) - assert keys_observed == ("child",) - async def test_set_if_not_exists(self, store: S) -> None: key = "k" data_buf = self.buffer_cls.from_bytes(b"0000") diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py index 6a4b796639..fb5a4dbdcc 100644 --- a/tests/test_store/test_object.py +++ b/tests/test_store/test_object.py @@ -9,9 +9,10 @@ from hypothesis.stateful import ( run_state_machine_as_test, ) -from obstore.store import LocalStore, MemoryStore +from obstore.store import LocalStore, MemoryStore, S3Store from zarr.core.buffer import Buffer, cpu +from zarr.core.sync import _collect_aiterator from zarr.storage import ObjectStore from zarr.testing.stateful import ZarrHierarchyStateMachine from zarr.testing.store import StoreTests @@ -97,6 +98,35 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non assert total_size == len(buf) * 2 +async def test_list_dir_ignores_s3_prefix_marker(moto_server: str) -> None: + boto3 = pytest.importorskip("boto3") + bucket = "object-store-prefix-marker" + client = boto3.client( + "s3", + endpoint_url=moto_server, + region_name="us-east-1", + aws_access_key_id="x", + aws_secret_access_key="x", + ) + client.create_bucket(Bucket=bucket) + client.put_object(Bucket=bucket, Key="g/", Body=b"") + + store = ObjectStore( + S3Store( + bucket=bucket, + endpoint=moto_server, + region="us-east-1", + access_key_id="x", + secret_access_key="x", + client_options={"allow_http": True}, + virtual_hosted_style_request=False, + ) + ) + + assert await _collect_aiterator(store.list_dir("g")) == () + assert await _collect_aiterator(store.list_dir("g/")) == () + + @pytest.mark.slow_hypothesis def test_zarr_hierarchy() -> None: sync_store = ObjectStore(MemoryStore()) From ad7d2ebed39e3a9b71dcb03f7ec23343fe7b6880 Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Fri, 10 Jul 2026 03:52:44 +0200 Subject: [PATCH 5/5] Filter boto3 deprecation in S3 test Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> --- tests/test_store/test_object.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py index fb5a4dbdcc..eef13f7ef2 100644 --- a/tests/test_store/test_object.py +++ b/tests/test_store/test_object.py @@ -1,4 +1,5 @@ # ruff: noqa: E402 +import re from pathlib import Path from typing import TypedDict @@ -98,6 +99,9 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non assert total_size == len(buf) * 2 +@pytest.mark.filterwarnings( + re.escape("ignore:datetime.datetime.utcnow() is deprecated:DeprecationWarning") +) async def test_list_dir_ignores_s3_prefix_marker(moto_server: str) -> None: boto3 = pytest.importorskip("boto3") bucket = "object-store-prefix-marker"