diff --git a/sshfs/spec.py b/sshfs/spec.py index 95f4b4a..2338040 100644 --- a/sshfs/spec.py +++ b/sshfs/spec.py @@ -15,7 +15,7 @@ sync, sync_wrapper, ) -from fsspec.utils import infer_storage_options +from fsspec.utils import infer_storage_options, stringify_path from sshfs.file import SSHFile from sshfs.pools import SFTPSoftChannelPool @@ -62,6 +62,17 @@ def __init__( Options passed to `SSHClientConnection.start_sftp_client` (e.g. env, send_env, path_encoding, path_errors, sftp_version). + + Notes + ----- + Methods called on an existing filesystem instance use its current + SSH connection. The host, username, and port in string URLs are + ignored; operations use only the path on the connected server. + + Bytes paths (and path-likes whose ``__fspath__`` returns bytes) are sent + to the server unchanged: protocol prefixes are not stripped and trailing + slashes are kept. With ``sftp_client_kwargs={"path_encoding": None}``, + paths must be bytes. """ super().__init__(self, **kwargs) @@ -94,6 +105,11 @@ def __init__( @classmethod def _strip_protocol(cls, path): + if isinstance(path, list): + return [cls._strip_protocol(p) for p in path] + path = stringify_path(path) + if isinstance(path, bytes): + return path # Remove components such as host and username from path. inferred_path = infer_storage_options(path)["path"] stripped = super()._strip_protocol(inferred_path) @@ -214,6 +230,8 @@ async def _modified(self, path: str, **kwargs) -> datetime: @wrap_exceptions async def _mv(self, lpath, rpath, **kwargs): + lpath = self._strip_protocol(lpath) + rpath = self._strip_protocol(rpath) async with self._pool.get() as channel: with suppress(SFTPOpUnsupported): return await channel.posix_rename(lpath, rpath) @@ -235,6 +253,7 @@ async def _put_file( callback=None, **kwargs, ): + rpath = self._strip_protocol(rpath) await self._makedirs(self._parent(rpath), exist_ok=True) async with self._pool.get() as channel: await channel.put( @@ -248,6 +267,7 @@ async def _put_file( async def _get_file( self, lpath, rpath, block_size=READ_BLOCK_SIZE, callback=None, **kwargs ): + lpath = self._strip_protocol(lpath) async with self._pool.get() as channel: await channel.get( lpath, @@ -258,11 +278,14 @@ async def _get_file( @wrap_exceptions async def _cp_file(self, lpath, rpath, **kwargs): + lpath = self._strip_protocol(lpath) + rpath = self._strip_protocol(rpath) cmd = f"cp {shlex.quote(lpath)} {shlex.quote(rpath)}" await self._execute(cmd) @wrap_exceptions async def _ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) async with self._pool.get() as channel: file_attrs = await channel.readdir(path) @@ -284,6 +307,7 @@ async def _ls(self, path, detail=True, **kwargs): async def _mkdir( self, path, *, create_parents=True, permissions=511, **kwargs ): + path = self._strip_protocol(path) if create_parents: return await self._makedirs(path, exist_ok=True) @@ -295,6 +319,7 @@ async def _mkdir( async def _makedirs( self, path, *, exist_ok=False, permissions=511, **kwargs ): + path = self._strip_protocol(path) attrs = asyncssh.SFTPAttrs(permissions=permissions) async with self._pool.get() as channel: await channel.makedirs(path, exist_ok=exist_ok, attrs=attrs) @@ -304,6 +329,7 @@ async def _makedirs( @wrap_exceptions async def _rm_file(self, path, **kwargs): + path = self._strip_protocol(path) async with self._pool.get() as channel: await channel.unlink(path) @@ -316,6 +342,7 @@ async def _rmdir( on_error=None, **kwargs, ): + path = self._strip_protocol(path) async with self._pool.get() as channel: if recursive: await channel.rmtree( @@ -325,7 +352,8 @@ async def _rmdir( await channel.rmdir(path) async def _rm(self, path, recursive=False, **kwargs): - if isinstance(path, str): + path = self._strip_protocol(path) + if isinstance(path, (str, bytes)): path = [path] coros = [] @@ -340,6 +368,7 @@ async def _rm(self, path, recursive=False, **kwargs): @wrap_exceptions async def _checksum(self, path): + path = self._strip_protocol(path) system = await self._get_system() if system == "Linux": command = "md5sum" @@ -378,11 +407,13 @@ async def _execute(self, *args, **kwargs): execute = sync_wrapper(_execute) def _open(self, path, *args, **kwargs): + path = self._strip_protocol(path) return SSHFile(self, path, *args, **kwargs) @wrap_exceptions async def _cat_file(self, path, **kwargs): """Asynchronously fetch the contents of a file""" + path = self._strip_protocol(path) async with self._pool.get() as channel: async with channel.open(path, "rb") as f: return await f.read() @@ -390,6 +421,7 @@ async def _cat_file(self, path, **kwargs): @wrap_exceptions async def _pipe_file(self, path, data, chunksize=50 * 2**20, **kwargs): """Asynchronously writes the given data to a remote file in chunks.""" + path = self._strip_protocol(path) await self._makedirs(self._parent(path), exist_ok=True) async with self._pool.get() as channel: diff --git a/tests/test_sshfs.py b/tests/test_sshfs.py index b87a534..494f523 100644 --- a/tests/test_sshfs.py +++ b/tests/test_sshfs.py @@ -1,7 +1,7 @@ import hashlib +import os import posixpath import secrets -import tempfile import warnings from concurrent import futures from datetime import UTC, datetime @@ -35,17 +35,22 @@ def ssh_server(): yield server -@pytest.fixture(scope="session") -def base_remote_dir(): - with tempfile.TemporaryDirectory() as path: - yield path +@pytest.fixture +def plain_remote_dir(tmp_path): + directory = tmp_path / "remote" + directory.mkdir() + return str(directory) -@pytest.fixture -def remote_dir(fs, base_remote_dir, request): - directory = posixpath.join(base_remote_dir, request.node.name).rstrip("/") - fs.mkdir(directory) - yield directory +@pytest.fixture(params=["plain", "ssh", "sftp"]) +def remote_dir(plain_remote_dir, ssh_server, request): + if request.param == "plain": + return plain_remote_dir + + return ( + f"{request.param}://user@" + f"{ssh_server.host}:{ssh_server.port}{plain_remote_dir}" + ) @pytest.fixture @@ -88,9 +93,9 @@ def test_fsspec_registration(ssh_server): assert isinstance(fs, SSHFileSystem) -def test_fsspec_url_parsing(ssh_server, remote_dir, user="user"): +def test_fsspec_url_parsing(ssh_server, plain_remote_dir, user="user"): for ep in list(entry_points(group="fsspec.specs")): - url = f"{ep.name}://{user}@{ssh_server.host}:{ssh_server.port}/{remote_dir}/file" + url = f"{ep.name}://{user}@{ssh_server.host}:{ssh_server.port}/{plain_remote_dir}/file" with fsspec.open(url, "w", client_keys=[USERS[user]]) as file: # Check the underlying file system. file_fs = file.buffer.fs @@ -103,7 +108,20 @@ def test_fsspec_url_parsing(ssh_server, remote_dir, user="user"): } -def test_sftp_client_kwargs(ssh_server, base_remote_dir, user="user"): +@pytest.mark.parametrize("protocol", ["ssh", "sftp"]) +def test_url_host_is_ignored(fs, plain_remote_dir, protocol): + path = plain_remote_dir + "/a.txt" + with fs.open(path, "wb") as stream: + stream.write(b"data") + + url = f"{protocol}://other-host:12345{path}" + assert fs.cat_file(url) == b"data" + + fs.rm(url) + assert not fs.exists(path) + + +def test_sftp_client_kwargs(ssh_server, tmp_path, user="user"): fs = SSHFileSystem( host=ssh_server.host, port=ssh_server.port, @@ -113,14 +131,12 @@ def test_sftp_client_kwargs(ssh_server, base_remote_dir, user="user"): ) assert fs._pool.sftp_client_kwargs == {"sftp_version": 3} - file = posixpath.join(base_remote_dir, "sftp_client_kwargs_probe") + file = posixpath.join(tmp_path, "sftp_client_kwargs_probe") fs.touch(file) assert fs.exists(file) -def test_sftp_client_kwargs_path_encoding( - ssh_server, base_remote_dir, user="user" -): +def test_sftp_client_kwargs_path_encoding(ssh_server, tmp_path, user="user"): # path_encoding=None makes asyncssh deliver remote paths as raw # bytes instead of str, for servers with non-UTF-8 file names (#39). fs = SSHFileSystem( @@ -131,7 +147,7 @@ def test_sftp_client_kwargs_path_encoding( sftp_client_kwargs={"path_encoding": None}, ) - directory = Path(base_remote_dir) / "path_encoding_probe" + directory = tmp_path / "path_encoding_probe" directory.mkdir() (directory / "data.txt").write_bytes(b"data") @@ -140,20 +156,20 @@ def test_sftp_client_kwargs_path_encoding( assert fs.cat_file(encoded + b"/data.txt") == b"data" -def test_info(fs, remote_dir): +def test_info(fs, remote_dir, plain_remote_dir): fs.touch(remote_dir + "/a.txt") details = fs.info(remote_dir + "/a.txt") assert details["type"] == "file" - assert details["name"] == remote_dir + "/a.txt" + assert details["name"] == plain_remote_dir + "/a.txt" assert details["size"] == 0 fs.mkdir(remote_dir + "/dir") details = fs.info(remote_dir + "/dir") assert details["type"] == "directory" - assert details["name"] == remote_dir + "/dir/" + assert details["name"] == plain_remote_dir + "/dir/" details = fs.info(remote_dir + "/dir/") - assert details["name"] == remote_dir + "/dir/" + assert details["name"] == plain_remote_dir + "/dir/" def test_info_timestamps_are_tz_aware_utc(fs, remote_dir): @@ -266,15 +282,18 @@ def test_checksum(fs, remote_dir): assert fs.checksum(remote_dir + "/a.txt") == checksum -def test_ls(fs, remote_dir): +def test_ls(fs, remote_dir, plain_remote_dir): fs.mkdir(remote_dir + "dir/") files = set() + plain_files = set() for no in range(8): file = remote_dir + f"dir/test_{no}" + plain_file = plain_remote_dir + f"dir/test_{no}" fs.touch(file) files.add(file) + plain_files.add(plain_file) - assert set(fs.ls(remote_dir + "dir/", detail=False)) == files + assert set(fs.ls(remote_dir + "dir/", detail=False)) == plain_files dirs = fs.ls(remote_dir + "dir/") expected = [fs.info(file) for file in files] @@ -286,7 +305,7 @@ def test_ls(fs, remote_dir): assert dirs == expected -def test_walk(fs, remote_dir): +def test_walk(fs, remote_dir, plain_remote_dir): fs.mkdir(remote_dir + "/a") fs.mkdir(remote_dir + "/a/b") fs.touch(remote_dir + "/a/f1") @@ -297,8 +316,8 @@ def test_walk(fs, remote_dir): for root, dirs, files in fs.walk(remote_dir + "/a") } assert result == { - remote_dir + "/a": (["b"], ["f1"]), - remote_dir + "/a/b": ([], ["f2"]), + plain_remote_dir + "/a": (["b"], ["f1"]), + plain_remote_dir + "/a/b": ([], ["f2"]), } @@ -315,6 +334,7 @@ def test_strip_protocol(): # An explicit absolute root is preserved so walk("/") lists from the root. assert strip("/") == "/" assert strip("ssh://host/") == "/" + assert strip("sftp://host/") == "/" # Empty and relative paths still resolve against the home directory # instead of being redirected to the root. assert strip("") == "" @@ -323,6 +343,30 @@ def test_strip_protocol(): # Regular absolute paths are unaffected. assert strip("/foo/bar") == "/foo/bar" assert strip("ssh://host/foo/bar") == "/foo/bar" + assert strip("sftp://user:secret@host:2222/foo/bar") == "/foo/bar" + assert strip(b"/foo/\xff") == b"/foo/\xff" + + # pathlib.Path + assert strip(Path("/foo/bar")) == "/foo/bar" + + class PathLike(os.PathLike): + def __init__(self, path): + self.path = path + + def __fspath__(self): + return self.path + + assert strip(PathLike("/foo/bar")) == "/foo/bar" + assert strip(PathLike("ssh://host/foo/bar")) == "/foo/bar" + assert ( + strip(PathLike("sftp://user:secret@host:2222/foo/bar")) == "/foo/bar" + ) + + # Path-like objects returning bytes must preserve non-UTF-8 paths. + assert strip(PathLike(b"/foo/\xff")) == b"/foo/\xff" + + # list + assert strip(["foo/bar/1", "foo/bar/2"]) == ["foo/bar/1", "foo/bar/2"] def test_mkdir(fs, remote_dir): @@ -489,16 +533,28 @@ def read_random_file(name): @pytest.mark.parametrize("file_path", ["a.txt", "dir/a.txt"]) -def test_put_file(fs, remote_dir, file_path): - with tempfile.NamedTemporaryFile() as file: - file.file.write(b"data") - file.file.flush() - fs.put_file(file.name, remote_dir + f"/{file_path}") +def test_put_file(fs, remote_dir, file_path, tmp_path): + local_file = tmp_path / "upload.txt" + local_file.write_bytes(b"data") + fs.put_file(str(local_file), remote_dir + f"/{file_path}") with fs.open(remote_dir + f"/{file_path}") as stream: assert stream.read() == b"data" +@pytest.mark.parametrize("file_path", ["a.txt", "dir/a.txt"]) +def test_get_file(fs, remote_dir, file_path, tmp_path): + remote_file = remote_dir + f"/{file_path}" + fs.makedirs(posixpath.dirname(remote_file), exist_ok=True) + + with fs.open(remote_file, "wb") as stream: + stream.write(b"data") + + local_file = tmp_path / "downloaded.txt" + fs.get_file(remote_file, str(local_file)) + assert local_file.read_bytes() == b"data" + + def test_concurrency_for_raw_commands(fs, remote_dir): with fs.open(remote_dir + "/cp_data", "wb") as stream: stream.write(b"hello!") @@ -521,13 +577,14 @@ def test_modified(fs, remote_dir): assert isinstance(modified, datetime) -def test_cat_file_sync(fs, remote_dir): +def test_cat_file_sync(fs, remote_dir, plain_remote_dir): # Define the content to write to the test file test_content = b"Test content for cat_file" test_file_path = remote_dir + "/test_file.txt" + test_plain_file_path = plain_remote_dir + "/test_file.txt" # Write content to the file synchronously - with open(test_file_path, "wb") as f: + with open(test_plain_file_path, "wb") as f: f.write(test_content) # Use the cat_file method to read the content back synchronously