diff --git a/docs/concepts/upath.md b/docs/concepts/upath.md index db1a4b5d..2a01c317 100644 --- a/docs/concepts/upath.md +++ b/docs/concepts/upath.md @@ -180,6 +180,30 @@ assert not isinstance(file_path, Path) # False (uses fsspec instead) - Useful for consistent fsspec-based access across all backends - Implements `os.PathLike` protocol +**File URIs vs. fsspec urlpaths:** + +Both families of local paths can be written as strings starting with `file:`, but these +strings follow different rules: + +| | `PosixUPath` / `WindowsUPath` | `FilePath` | +|---|---|---| +| string form | `as_uri()` returns an [RFC 8089](https://www.rfc-editor.org/rfc/rfc8089) file URI, identical to `pathlib.Path.as_uri()` | `str()` / `as_uri()` return an fsspec urlpath: `file://` followed by the literal path | +| special characters | percent-encoded (`a b` → `a%20b`) | kept as is | +| UNC paths (Windows) | `file://server/share/file` | `file:////server/share/file` | +| convert back | `type(p).from_uri(uri)` | `UPath(urlpath)` | + +`UPath("file://...")` always parses an fsspec urlpath and returns a `FilePath`. fsspec does +not interpret the authority part of a `file://` URI, so `UPath("file://server/share/file")` +is the *relative* path `server/share/file` resolved against the current working directory. +To switch a local path between the two implementations, pass the path object directly: + +```python +p = UPath("/tmp/a b.txt") # PosixUPath +f = UPath(p, protocol="file") # FilePath('/tmp/a b.txt') +assert type(p).from_uri(p.as_uri()) == p # file URI round-trip +assert UPath(str(f)) == f # fsspec urlpath round-trip +``` + ### Remote and Cloud Paths For remote filesystems, `UPath` implementations provide the pathlib API backed by fsspec: diff --git a/docs/usage.md b/docs/usage.md index ee8e6a94..17c720fd 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -34,17 +34,25 @@ local_path = UPath(tmp.name) assert isinstance(local_path, (pathlib.PosixPath, pathlib.WindowsPath)) ``` -Want to rely on fsspec for local filesystems too? Use a `file://` URI: +Want to rely on fsspec for local filesystems too? Pass `protocol="file"`: ```python -local_uri = local_path.absolute().as_uri() -# Result: 'file:///tmp/tmpXXXXXX' - -local_upath = UPath(local_uri) +local_upath = UPath(local_path, protocol="file") # Now it uses fsspec backend assert isinstance(local_upath, UPath) +str(local_upath) +# Result: 'file:///tmp/tmpXXXXXX' ``` +!!! warning "`as_uri()` returns an RFC 8089 file URI, not an fsspec urlpath" + `PosixUPath.as_uri()` and `WindowsUPath.as_uri()` behave exactly like + `pathlib.Path.as_uri()`: special characters are percent-encoded and UNC paths use + the URI authority (`file://server/share/file`). fsspec, and therefore + `UPath("file://...")`, treats everything after `file://` literally as a path. + So `UPath(local_path.as_uri())` points to a different file if the path contains + characters like spaces, `#` or `%`, or if it's a UNC path. Use + `type(local_path).from_uri(uri)` to convert a file URI back to a local path. + ### How do I connect to cloud storage or remote filesystems? Just use the appropriate URI scheme. UPath will automatically connect to the right filesystem: diff --git a/upath/implementations/local.py b/upath/implementations/local.py index 0170f13f..7827eb07 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -13,6 +13,7 @@ from typing import Literal from typing import overload from urllib.parse import SplitResult +from urllib.parse import unquote_to_bytes from fsspec import AbstractFileSystem @@ -527,7 +528,26 @@ def full_match( @classmethod def from_uri(cls, uri: str, **storage_options: Any) -> Self: - return UPath(uri, **storage_options) # type: ignore[return-value] + # backport of pathlib.Path.from_uri from python 3.13 + if not uri.startswith("file:"): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == "///": + # Remove empty authority + path = path[2:] + elif path[:12] == "//localhost/": + # Remove 'localhost' authority + path = path[11:] + if path[:3] == "///" or (path[:1] == "/" and path[2:3] in ":|"): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == "|": + # Replace bar with colon in DOS drive + path = path[:1] + ":" + path[2:] + pth = cls(os.fsdecode(unquote_to_bytes(path)), **storage_options) + if not pth.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return pth def is_dir(self, *, follow_symlinks: bool = True) -> bool: if not follow_symlinks: diff --git a/upath/tests/implementations/test_local.py b/upath/tests/implementations/test_local.py index a77c256f..4a0ed187 100644 --- a/upath/tests/implementations/test_local.py +++ b/upath/tests/implementations/test_local.py @@ -10,6 +10,7 @@ from ..cases import BaseTests from ..utils import OverrideMeta +from ..utils import only_on_windows from ..utils import overrides_base from ..utils import skip_on_windows from ..utils import xfail_if_version @@ -53,6 +54,33 @@ def path(self, local_testdir): self.path = UPath(path) +@pytest.mark.parametrize("name", ["file.txt", "a b#%é?.txt"]) +def test_local_from_uri_roundtrip(tmp_path, name): + pth = UPath(tmp_path).joinpath(name) + uri = pth.as_uri() + assert type(pth).from_uri(uri) == pth + assert type(type(pth).from_uri(uri)) is type(pth) + + +@pytest.mark.parametrize("uri", ["s3://bucket/file.txt", "file:relative/file.txt"]) +def test_local_from_uri_invalid(uri): + with pytest.raises(ValueError): + type(UPath(".")).from_uri(uri) + + +@only_on_windows +@pytest.mark.parametrize( + "uri", ["file://server/share/file.txt", "file:////server/share/file.txt"] +) +def test_local_from_uri_unc_paths(uri): + # https://github.com/fsspec/universal_pathlib/issues/557 + from upath.implementations.local import WindowsUPath + + pth = WindowsUPath.from_uri(uri) + assert pth == WindowsUPath(r"\\server\share\file.txt") + assert WindowsUPath.from_uri(pth.as_uri()) == pth + + @pytest.mark.parametrize( "protocol,path", [