From 2ec80ad4d551c33e34bbb06a5e4260ce5ba4ea7e Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Tue, 13 Jan 2026 11:32:14 +0100 Subject: [PATCH 1/4] tests: reproduce broken s3 iterdir when directory marker exists --- upath/tests/implementations/test_s3.py | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index e5f070fc..cd72fdca 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -226,3 +226,59 @@ def on_collision_rename_file(src, dst): "src/common_prefix/file1.txt", "src/common_prefix/file2.txt", ] + + +@pytest.fixture +def s3_with_explicit_directory_marker(s3_server): + """issue #226: path.iterdir() yields path itself as the first item (with S3) + + Creates a bucket with an explicit directory marker (zero-byte object with + trailing slash) plus files. This simulates folders created via the AWS + Console, which creates actual zero-byte objects as directory markers. + + Given objects on S3: + s3://my-bucket/my-directory/ (zero-byte directory marker) + s3://my-bucket/my-directory/0.txt + s3://my-bucket/my-directory/1.txt + """ + import boto3 + + anon, s3so = s3_server + bucket = "iterdir_issue_226_marker_bucket" + s3_client = boto3.client("s3", **s3so["client_kwargs"]) + s3_client.create_bucket(Bucket=bucket) + # Create an explicit directory marker (zero-byte object with trailing slash) + # Use boto3 directly to ensure the trailing slash is preserved in the key + # This is what the AWS Console creates when you "Create folder" + s3_client.put_object(Bucket=bucket, Key="my-directory/", Body=b"") + # Then create files inside the directory + s3_client.put_object(Bucket=bucket, Key="my-directory/0.txt", Body=b"content 0") + s3_client.put_object(Bucket=bucket, Key="my-directory/1.txt", Body=b"content 1") + yield bucket, anon, s3so + + +@silence_botocore_datetime_deprecation +def test_iterdir_with_explicit_directory_marker__issue_226( + s3_with_explicit_directory_marker, +): + """issue #226: path.iterdir() yields path itself as the first item (with S3) + + See: https://github.com/fsspec/universal_pathlib/issues/226 + See: https://medium.com/cyberark-engineering/the-strange-case-of-amazon-s3-bucket-folders-c8d113a8dd01 # noqa: E501 + """ + bucket, anon, s3so = s3_with_explicit_directory_marker + directory_path = UPath(f"s3://{bucket}/my-directory", anon=anon, **s3so) + + children = list(directory_path.iterdir()) + + assert directory_path not in children + + expected_files = { + directory_path / "0.txt", + directory_path / "1.txt", + } + assert set(children) == expected_files, ( + f"iterdir() should yield only the files in the directory. " + f"Expected: {[str(f) for f in expected_files]}, " + f"Got: {[str(c) for c in children]}" + ) From 8ce0683b2047ba3957d302902453641f5e0502e3 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Tue, 13 Jan 2026 11:32:42 +0100 Subject: [PATCH 2/4] upath.core: adjust iterdir to not yield directory markers --- upath/core.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/upath/core.py b/upath/core.py index 992a2f23..3efd2368 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1237,15 +1237,26 @@ def iterdir(self) -> Iterator[Self]: base_path = base.path if not fs.isdir(base_path): raise NotADirectoryError(str(self)) - for name in fs.listdir(base_path): + name: str + for info in fs.listdir(base_path): # fsspec returns dictionaries - if isinstance(name, dict): - name = name.get("name") + if isinstance(info, dict): + name = info["name"] + else: + name, info = info, {} # type: ignore[assignment] + # skip the base path itself if a zero-length name + name = name.removesuffix(sep) + if ( + name == base_path + and info.get("size", -1) == 0 + and info.get("Key", None) == base_path + sep + ): + continue # skip empty names + # only want the path name with iterdir + _, _, name = name.rpartition(self.parser.sep) if name in {".", ".."}: # Yielding a path object for these makes little sense continue - # only want the path name with iterdir - _, _, name = name.removesuffix(sep).rpartition(self.parser.sep) yield base.with_segments(base_path, name) def __open_reader__(self) -> BinaryIO: From e5a8e39e8e8647fd32946af20c7fd1652399fff0 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 8 Nov 2026 19:44:15 +0100 Subject: [PATCH 3/4] upath.core: handle file/directory name collisions in iterdir On object stores an object key can also be the prefix of other objects. s3fs then lists the name twice (as file and as directory), and once the parent listing is cached, isdir() returns False for that name, so iterdir() yielded duplicates and raised NotADirectoryError when listing the colliding directory (see fsspec/s3fs#999). iterdir() now skips duplicate names and re-checks isdir() once after invalidating the filesystem cache before raising NotADirectoryError. --- upath/core.py | 15 +++++++++--- upath/tests/implementations/test_s3.py | 32 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/upath/core.py b/upath/core.py index 3efd2368..91ac6339 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1236,27 +1236,36 @@ def iterdir(self) -> Iterator[Self]: fs = base.fs base_path = base.path if not fs.isdir(base_path): - raise NotADirectoryError(str(self)) + # cached listings can misreport a directory that collides with a + # file of the same name (i.e. s3fs), so check again without cache + fs.invalidate_cache(base_path) + if not fs.isdir(base_path): + raise NotADirectoryError(str(self)) name: str + seen: set[str] = set() for info in fs.listdir(base_path): # fsspec returns dictionaries if isinstance(info, dict): name = info["name"] else: name, info = info, {} # type: ignore[assignment] - # skip the base path itself if a zero-length name name = name.removesuffix(sep) + # skip the directory marker of the base path itself (i.e. s3) if ( name == base_path and info.get("size", -1) == 0 and info.get("Key", None) == base_path + sep ): - continue # skip empty names + continue # only want the path name with iterdir _, _, name = name.rpartition(self.parser.sep) if name in {".", ".."}: # Yielding a path object for these makes little sense continue + if name in seen: + # a file and a directory can share a name on object stores + continue + seen.add(name) yield base.with_segments(base_path, name) def __open_reader__(self) -> BinaryIO: diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index cd72fdca..d9af59c3 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -282,3 +282,35 @@ def test_iterdir_with_explicit_directory_marker__issue_226( f"Expected: {[str(f) for f in expected_files]}, " f"Got: {[str(c) for c in children]}" ) + + +@pytest.fixture +def s3_with_file_dir_name_collision(s3_server): + """an object key that is also the prefix of other objects + + Given objects on S3: + s3://my-bucket/my-directory (zero-byte object without trailing slash) + s3://my-bucket/my-directory/0.txt + """ + import boto3 + + anon, s3so = s3_server + bucket = "iterdir_file_dir_collision_bucket" + s3_client = boto3.client("s3", **s3so["client_kwargs"]) + s3_client.create_bucket(Bucket=bucket) + s3_client.put_object(Bucket=bucket, Key="my-directory", Body=b"") + s3_client.put_object(Bucket=bucket, Key="my-directory/0.txt", Body=b"content 0") + yield bucket, anon, s3so + + +@silence_botocore_datetime_deprecation +def test_iterdir_with_file_dir_name_collision(s3_with_file_dir_name_collision): + bucket, anon, s3so = s3_with_file_dir_name_collision + root = UPath(f"s3://{bucket}/", anon=anon, **s3so) + + # the listing contains a file and a directory entry with the same name + children = list(root.iterdir()) + assert children == [root / "my-directory"] + + # listing the parent populated s3fs' dircache with both entries + assert list(children[0].iterdir()) == [root / "my-directory" / "0.txt"] From 910c969bbb61370887239172b9460964e1b54aef Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 8 Nov 2026 19:51:07 +0100 Subject: [PATCH 4/4] upath.core: never yield the directory itself in iterdir The directory marker check only matched s3 listings of non-root prefixes (base paths without a trailing separator, with s3fs specific info keys). Listing entries whose name equals the listed directory are never children, so skip them for all filesystems. This also fixes iterdir() yielding a bogus 's3://bucket/bucket' child for buckets with keys starting with '/' (#144). --- upath/core.py | 10 ++++------ upath/tests/implementations/test_s3.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/upath/core.py b/upath/core.py index 91ac6339..f1903ae7 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1243,6 +1243,7 @@ def iterdir(self) -> Iterator[Self]: raise NotADirectoryError(str(self)) name: str seen: set[str] = set() + base_name = base_path.removesuffix(sep) for info in fs.listdir(base_path): # fsspec returns dictionaries if isinstance(info, dict): @@ -1250,12 +1251,9 @@ def iterdir(self) -> Iterator[Self]: else: name, info = info, {} # type: ignore[assignment] name = name.removesuffix(sep) - # skip the directory marker of the base path itself (i.e. s3) - if ( - name == base_path - and info.get("size", -1) == 0 - and info.get("Key", None) == base_path + sep - ): + if name == base_name: + # the directory itself is not a child, i.e. s3 directory + # markers or empty-named prefixes from keys containing "//" continue # only want the path name with iterdir _, _, name = name.rpartition(self.parser.sep) diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index d9af59c3..02a08d24 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -158,7 +158,7 @@ def s3_with_double_fwd_slash_files(s3_server): anon, s3so = s3_server s3 = fsspec.filesystem("s3", anon=False, **s3so) bucket = "double_fwd_slash_bucket" - s3.mkdir(bucket + "/") + s3.makedirs(bucket, exist_ok=True) s3.pipe_file(f"{bucket}//key.txt", b"hello world") try: yield bucket, anon, s3so @@ -314,3 +314,12 @@ def test_iterdir_with_file_dir_name_collision(s3_with_file_dir_name_collision): # listing the parent populated s3fs' dircache with both entries assert list(children[0].iterdir()) == [root / "my-directory" / "0.txt"] + + +@silence_botocore_datetime_deprecation +def test_iterdir_skips_empty_named_prefix(s3_with_double_fwd_slash_files): + """issue #144: a key starting with "/" creates an empty-named prefix""" + bucket, anon, s3so = s3_with_double_fwd_slash_files + root = UPath(f"s3://{bucket}", anon=anon, **s3so) + # the empty-named child can't be represented as a UPath child of root + assert root.joinpath(bucket) not in list(root.iterdir())