Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions docs/concepts/upath.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,30 @@ assert not isinstance(file_path, Path) # False (uses fsspec instead)
- Useful for consistent fsspec-based access across all backends
- Implements `os.PathLike` protocol

**File URIs vs. fsspec urlpaths:**

Both families of local paths can be written as strings starting with `file:`, but these
strings follow different rules:

| | `PosixUPath` / `WindowsUPath` | `FilePath` |
|---|---|---|
| string form | `as_uri()` returns an [RFC 8089](https://www.rfc-editor.org/rfc/rfc8089) file URI, identical to `pathlib.Path.as_uri()` | `str()` / `as_uri()` return an fsspec urlpath: `file://` followed by the literal path |
| special characters | percent-encoded (`a b` → `a%20b`) | kept as is |
| UNC paths (Windows) | `file://server/share/file` | `file:////server/share/file` |
| convert back | `type(p).from_uri(uri)` | `UPath(urlpath)` |

`UPath("file://...")` always parses an fsspec urlpath and returns a `FilePath`. fsspec does
not interpret the authority part of a `file://` URI, so `UPath("file://server/share/file")`
is the *relative* path `server/share/file` resolved against the current working directory.
To switch a local path between the two implementations, pass the path object directly:

```python
p = UPath("/tmp/a b.txt") # PosixUPath
f = UPath(p, protocol="file") # FilePath('/tmp/a b.txt')
assert type(p).from_uri(p.as_uri()) == p # file URI round-trip
assert UPath(str(f)) == f # fsspec urlpath round-trip
```

### Remote and Cloud Paths

For remote filesystems, `UPath` implementations provide the pathlib API backed by fsspec:
Expand Down
18 changes: 13 additions & 5 deletions docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,25 @@ local_path = UPath(tmp.name)
assert isinstance(local_path, (pathlib.PosixPath, pathlib.WindowsPath))
```

Want to rely on fsspec for local filesystems too? Use a `file://` URI:
Want to rely on fsspec for local filesystems too? Pass `protocol="file"`:

```python
local_uri = local_path.absolute().as_uri()
# Result: 'file:///tmp/tmpXXXXXX'

local_upath = UPath(local_uri)
local_upath = UPath(local_path, protocol="file")
# Now it uses fsspec backend
assert isinstance(local_upath, UPath)
str(local_upath)
# Result: 'file:///tmp/tmpXXXXXX'
```

!!! warning "`as_uri()` returns an RFC 8089 file URI, not an fsspec urlpath"
`PosixUPath.as_uri()` and `WindowsUPath.as_uri()` behave exactly like
`pathlib.Path.as_uri()`: special characters are percent-encoded and UNC paths use
the URI authority (`file://server/share/file`). fsspec, and therefore
`UPath("file://...")`, treats everything after `file://` literally as a path.
So `UPath(local_path.as_uri())` points to a different file if the path contains
characters like spaces, `#` or `%`, or if it's a UNC path. Use
`type(local_path).from_uri(uri)` to convert a file URI back to a local path.

### How do I connect to cloud storage or remote filesystems?

Just use the appropriate URI scheme. UPath will automatically connect to the right filesystem:
Expand Down
22 changes: 21 additions & 1 deletion upath/implementations/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from typing import Literal
from typing import overload
from urllib.parse import SplitResult
from urllib.parse import unquote_to_bytes

from fsspec import AbstractFileSystem

Expand Down Expand Up @@ -527,7 +528,26 @@ def full_match(

@classmethod
def from_uri(cls, uri: str, **storage_options: Any) -> Self:
return UPath(uri, **storage_options) # type: ignore[return-value]
# backport of pathlib.Path.from_uri from python 3.13
if not uri.startswith("file:"):
raise ValueError(f"URI does not start with 'file:': {uri!r}")
path = uri[5:]
if path[:3] == "///":
# Remove empty authority
path = path[2:]
elif path[:12] == "//localhost/":
# Remove 'localhost' authority
path = path[11:]
if path[:3] == "///" or (path[:1] == "/" and path[2:3] in ":|"):
# Remove slash before DOS device/UNC path
path = path[1:]
if path[1:2] == "|":
# Replace bar with colon in DOS drive
path = path[:1] + ":" + path[2:]
pth = cls(os.fsdecode(unquote_to_bytes(path)), **storage_options)
if not pth.is_absolute():
raise ValueError(f"URI is not absolute: {uri!r}")
return pth

def is_dir(self, *, follow_symlinks: bool = True) -> bool:
if not follow_symlinks:
Expand Down
28 changes: 28 additions & 0 deletions upath/tests/implementations/test_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from ..cases import BaseTests
from ..utils import OverrideMeta
from ..utils import only_on_windows
from ..utils import overrides_base
from ..utils import skip_on_windows
from ..utils import xfail_if_version
Expand Down Expand Up @@ -53,6 +54,33 @@ def path(self, local_testdir):
self.path = UPath(path)


@pytest.mark.parametrize("name", ["file.txt", "a b#%é?.txt"])
def test_local_from_uri_roundtrip(tmp_path, name):
pth = UPath(tmp_path).joinpath(name)
uri = pth.as_uri()
assert type(pth).from_uri(uri) == pth
assert type(type(pth).from_uri(uri)) is type(pth)


@pytest.mark.parametrize("uri", ["s3://bucket/file.txt", "file:relative/file.txt"])
def test_local_from_uri_invalid(uri):
with pytest.raises(ValueError):
type(UPath(".")).from_uri(uri)


@only_on_windows
@pytest.mark.parametrize(
"uri", ["file://server/share/file.txt", "file:////server/share/file.txt"]
)
def test_local_from_uri_unc_paths(uri):
# https://github.com/fsspec/universal_pathlib/issues/557
from upath.implementations.local import WindowsUPath

pth = WindowsUPath.from_uri(uri)
assert pth == WindowsUPath(r"\\server\share\file.txt")
assert WindowsUPath.from_uri(pth.as_uri()) == pth


@pytest.mark.parametrize(
"protocol,path",
[
Expand Down
Loading