diff --git a/upath/core.py b/upath/core.py index 992a2f23..7ac9167f 100644 --- a/upath/core.py +++ b/upath/core.py @@ -60,19 +60,23 @@ UnsupportedOperation = NotImplementedError """Raised when an unsupported operation is called on a path object.""" -if TYPE_CHECKING: - import upath.implementations as _uimpl - - if sys.version_info >= (3, 11): - from typing import Self - else: +if sys.version_info >= (3, 11): + from typing import Self +else: + try: from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass +if TYPE_CHECKING: from pydantic import GetCoreSchemaHandler from pydantic_core.core_schema import CoreSchema + import upath.implementations as _uimpl + _MT = TypeVar("_MT") - _WT = TypeVar("_WT", bound="WritablePath") + +_WT = TypeVar("_WT", bound="WritablePath") __all__ = [ "UPath", diff --git a/upath/extensions.py b/upath/extensions.py index df51e978..518566a0 100644 --- a/upath/extensions.py +++ b/upath/extensions.py @@ -33,12 +33,15 @@ from upath.types import UPathParser from upath.types import WritablePathLike -if TYPE_CHECKING: - if sys.version_info > (3, 11): - from typing import Self - else: +if sys.version_info >= (3, 11): + from typing import Self +else: + try: from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass +if TYPE_CHECKING: from pydantic import GetCoreSchemaHandler from pydantic_core.core_schema import CoreSchema diff --git a/upath/implementations/cached.py b/upath/implementations/cached.py index b3e23c98..f81c78fb 100644 --- a/upath/implementations/cached.py +++ b/upath/implementations/cached.py @@ -1,15 +1,17 @@ from __future__ import annotations import sys +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING +from typing import Any + +from fsspec import AbstractFileSystem from upath.core import UPath from upath.types import JoinablePathLike if TYPE_CHECKING: - from collections.abc import Mapping - from typing import Any from typing import Literal if sys.version_info >= (3, 11): @@ -17,8 +19,6 @@ else: from typing_extensions import Unpack - from fsspec import AbstractFileSystem - from upath._chain import FSSpecChainParser from upath.types.storage_options import SimpleCacheStorageOptions diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index f75e20b7..e3835d99 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -3,36 +3,35 @@ import sys from collections.abc import Iterator from collections.abc import Sequence -from typing import TYPE_CHECKING from typing import Any +from typing import Literal +from typing import TypeVar from typing import overload from upath import UnsupportedOperation from upath._chain import DEFAULT_CHAIN_PARSER +from upath._chain import FSSpecChainParser from upath._flavour import upath_strip_protocol from upath.core import UPath from upath.types import JoinablePathLike from upath.types import SupportsPathLike from upath.types import WritablePath - -if TYPE_CHECKING: - from typing import Literal - from typing import TypeVar - - if sys.version_info >= (3, 11): - from typing import Self - from typing import Unpack - else: +from upath.types.storage_options import AzureStorageOptions +from upath.types.storage_options import GCSStorageOptions +from upath.types.storage_options import HfStorageOptions +from upath.types.storage_options import S3StorageOptions + +if sys.version_info >= (3, 11): + from typing import Self + from typing import Unpack +else: + try: from typing_extensions import Self from typing_extensions import Unpack + except ImportError: # only needed for resolving annotations at runtime + pass - from upath._chain import FSSpecChainParser - from upath.types.storage_options import AzureStorageOptions - from upath.types.storage_options import GCSStorageOptions - from upath.types.storage_options import HfStorageOptions - from upath.types.storage_options import S3StorageOptions - - _WT = TypeVar("_WT", bound="WritablePath") +_WT = TypeVar("_WT", bound="WritablePath") __all__ = [ "CloudPath", diff --git a/upath/implementations/data.py b/upath/implementations/data.py index 8d30f97c..5f41c5b9 100644 --- a/upath/implementations/data.py +++ b/upath/implementations/data.py @@ -11,14 +11,20 @@ from upath.core import UPath from upath.types import JoinablePathLike +if sys.version_info >= (3, 11): + from typing import Self +else: + try: + from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass + if TYPE_CHECKING: from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser diff --git a/upath/implementations/ftp.py b/upath/implementations/ftp.py index 2e9663ab..365f1c78 100644 --- a/upath/implementations/ftp.py +++ b/upath/implementations/ftp.py @@ -3,24 +3,30 @@ import sys from ftplib import error_perm as FTPPermanentError # nosec B402 from typing import TYPE_CHECKING +from typing import Any from upath.core import UPath from upath.types import UNSET_DEFAULT from upath.types import JoinablePathLike +from upath.types import WritablePathLike + +if sys.version_info >= (3, 11): + from typing import Self +else: + try: + from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass if TYPE_CHECKING: - from typing import Any from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser - from upath.types import WritablePathLike from upath.types.storage_options import FTPStorageOptions __all__ = ["FTPPath"] diff --git a/upath/implementations/http.py b/upath/implementations/http.py index b09e1471..12320a9b 100644 --- a/upath/implementations/http.py +++ b/upath/implementations/http.py @@ -16,14 +16,20 @@ from upath.types import JoinablePathLike from upath.types import StatResultType +if sys.version_info >= (3, 11): + from typing import Self +else: + try: + from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass + if TYPE_CHECKING: from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser diff --git a/upath/implementations/local.py b/upath/implementations/local.py index 7827eb07..e57c8f0b 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -7,10 +7,14 @@ import warnings from collections.abc import Iterator from collections.abc import Sequence +from typing import IO from typing import TYPE_CHECKING from typing import Any +from typing import BinaryIO from typing import Callable from typing import Literal +from typing import TextIO +from typing import TypeVar from typing import overload from urllib.parse import SplitResult from urllib.parse import unquote_to_bytes @@ -36,17 +40,18 @@ from upath.types import WritablePath from upath.types import WritablePathLike -if TYPE_CHECKING: - from typing import IO - from typing import BinaryIO - from typing import TextIO - from typing import TypeVar +if sys.version_info >= (3, 11): + from typing import Self +else: + try: + from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass +if TYPE_CHECKING: if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from pydantic import GetCoreSchemaHandler @@ -54,7 +59,7 @@ from upath.types.storage_options import FileStorageOptions - _WT = TypeVar("_WT", bound="WritablePath") +_WT = TypeVar("_WT", bound="WritablePath") __all__ = [ "LocalPath", diff --git a/upath/implementations/smb.py b/upath/implementations/smb.py index e6b21849..9f6e5ab5 100644 --- a/upath/implementations/smb.py +++ b/upath/implementations/smb.py @@ -10,14 +10,20 @@ from upath.types import JoinablePathLike from upath.types import WritablePathLike +if sys.version_info >= (3, 11): + from typing import Self +else: + try: + from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass + if TYPE_CHECKING: from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser diff --git a/upath/implementations/tar.py b/upath/implementations/tar.py index 55046bf2..4974b699 100644 --- a/upath/implementations/tar.py +++ b/upath/implementations/tar.py @@ -3,6 +3,7 @@ import stat import sys import warnings +from collections.abc import Iterator from typing import TYPE_CHECKING from upath._stat import UPathStatResult @@ -11,15 +12,20 @@ from upath.types import JoinablePathLike from upath.types import StatResultType +if sys.version_info >= (3, 11): + from typing import Self +else: + try: + from typing_extensions import Self + except ImportError: # only needed for resolving annotations at runtime + pass + if TYPE_CHECKING: - from collections.abc import Iterator from typing import Literal if sys.version_info >= (3, 11): - from typing import Self from typing import Unpack else: - from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser diff --git a/upath/tests/test_type_hints.py b/upath/tests/test_type_hints.py new file mode 100644 index 00000000..5473119e --- /dev/null +++ b/upath/tests/test_type_hints.py @@ -0,0 +1,172 @@ +"""tests for resolving type hints of the public UPath API at runtime + +Tools like pydantic, cattrs, tyro or typer call `typing.get_type_hints()` +on classes and their methods. With `from __future__ import annotations` +all annotations are strings, which get evaluated in the namespace of the +defining module, so every name used in a public annotation needs to be +importable at runtime and not only under `TYPE_CHECKING`. + +Not covered on purpose: dunder hooks for optional third-party libraries, +like `__get_pydantic_core_schema__`, keep their annotations unresolvable +to avoid importing those libraries at runtime. +""" + +import ast +import builtins +import inspect +import re +import subprocess +import sys +import typing + +import pytest + +from upath import UPath +from upath.extensions import ProxyUPath +from upath.implementations.cloud import CloudPath +from upath.implementations.local import LocalPath +from upath.implementations.local import PosixUPath +from upath.implementations.local import WindowsUPath +from upath.registry import available_implementations +from upath.registry import get_upath_class +from upath.types import storage_options + +try: + import typing_extensions # noqa: F401 +except ImportError: + HAS_TYPING_EXTENSIONS = False +else: + HAS_TYPING_EXTENSIONS = True + + +def _upath_classes(): + classes = {UPath, CloudPath, LocalPath, PosixUPath, WindowsUPath, ProxyUPath} + for protocol in available_implementations(): + cls = get_upath_class(protocol) + if cls is not None and cls.__module__.startswith("upath."): + classes.add(cls) + return sorted(classes, key=lambda c: f"{c.__module__}.{c.__qualname__}") + + +def _public_members(cls): + """yield (name, function) for public methods and property getters""" + for name in dir(cls): + if name.startswith("_") and name not in {"__init__", "__new__"}: + continue + for base in cls.__mro__: + if name in base.__dict__: + break + else: + continue + if not base.__module__.startswith("upath"): + # members inherited from stdlib pathlib or pathlib_abc + continue + attr = base.__dict__[name] + if isinstance(attr, property): + func = attr.fget + elif isinstance(attr, (classmethod, staticmethod)): + func = attr.__func__ + elif inspect.isfunction(attr): + func = attr + else: + continue + yield name, func + + +def _annotation_namespaces(obj): + """yield (globalns, annotations) like typing.get_type_hints does""" + if isinstance(obj, type): + for base in reversed(obj.__mro__): + module = sys.modules.get(base.__module__) + annotations = base.__dict__.get("__annotations__", {}) + yield getattr(module, "__dict__", {}), annotations + else: + yield obj.__globals__, obj.__annotations__ + + +def _assert_annotation_names_defined(obj): + for globalns, annotations in _annotation_namespaces(obj): + for annotation in annotations.values(): + if isinstance(annotation, typing.ForwardRef): + annotation = annotation.__forward_arg__ + if not isinstance(annotation, str): + continue + for node in ast.walk(ast.parse(annotation, mode="eval")): + if isinstance(node, ast.Name) and not ( + node.id in globalns or hasattr(builtins, node.id) + ): + raise NameError(f"name {node.id!r} is not defined") + + +def _get_type_hints(obj): + try: + try: + return typing.get_type_hints(obj) + except TypeError as err: + # PEP 604 unions (X | Y) can only be evaluated at runtime on + # Python 3.10+, so on older versions check that all names resolve + if sys.version_info >= (3, 10) or "for |" not in str(err): + raise + _assert_annotation_names_defined(obj) + except NameError as err: + match = re.search(r"name '(\w+)' is not defined", str(err)) + if ( + sys.version_info < (3, 11) + and not HAS_TYPING_EXTENSIONS + and match + and match.group(1) in {"Self", "Unpack"} + ): + pytest.skip("Self and Unpack require typing_extensions on Python < 3.11") + raise + + +_CASES = [ + pytest.param(func, id=f"{cls.__qualname__}.{name}") + for cls in _upath_classes() + for name, func in _public_members(cls) +] + + +def test_public_members_are_collected(): + ids = {p.id for p in _CASES} + assert { + "UPath.__init__", + "UPath.joinpath", + "UPath.fs", + "UPath.copy", + "S3Path.__init__", + "PosixUPath.__new__", + "ProxyUPath.with_segments", + } <= ids + + +@pytest.mark.parametrize("func", _CASES) +def test_get_type_hints_public_members(func): + _get_type_hints(func) + + +@pytest.mark.parametrize("cls", _upath_classes(), ids=lambda c: c.__qualname__) +def test_get_type_hints_class(cls): + _get_type_hints(cls) + + +@pytest.mark.parametrize("name", storage_options.__all__) +def test_get_type_hints_storage_options(name): + _get_type_hints(getattr(storage_options, name)) + + +def test_import_without_typing_extensions(): + # typing_extensions is not a runtime dependency of universal_pathlib + code = "\n".join( + [ + "import sys", + "sys.modules['typing_extensions'] = None", + "import upath.extensions", + "import upath.types.storage_options", + "from upath.registry import available_implementations", + "from upath.registry import get_upath_class", + "for protocol in available_implementations():", + " get_upath_class(protocol)", + ] + ) + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/upath/types/storage_options.py b/upath/types/storage_options.py index 0212a25e..c5884702 100644 --- a/upath/types/storage_options.py +++ b/upath/types/storage_options.py @@ -2,17 +2,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from asyncio import AbstractEventLoop +from typing import Any +from typing import Literal from typing import TypedDict -if TYPE_CHECKING: - from asyncio import AbstractEventLoop - from typing import Any - from typing import Literal - - from fsspec.implementations.cache_mapper import AbstractCacheMapper - from fsspec.spec import AbstractFileSystem - +from fsspec.implementations.cache_mapper import AbstractCacheMapper +from fsspec.spec import AbstractFileSystem __all__ = [ "SimpleCacheStorageOptions",