diff --git a/README.md b/README.md index 0a3921c..7e22b6e 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,14 @@ fs = SSHFileSystem( Note: you can also pass `client_keys` as an argument to `fsspec.open`. +To pass remote filenames as bytes, set +`sftp_client_kwargs={"path_encoding": None}` when creating the filesystem. +Use bytes for remote paths with this setting. Metadata, listing, removal, +uploads, copies, and checksums preserve these paths through SFTP. Checksums +of byte paths stream the file over SFTP, so they require transferring its data. +Inherited fsspec traversal and bulk methods such as `walk`, `find`, `glob`, +`get`, `put`, and `copy` still require string paths. + All operations and their descriptions are specified [here](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem). Here are a few example calls you can make, starting with `info()` which allows you to retrieve the metadata about given path; diff --git a/sshfs/spec.py b/sshfs/spec.py index 2338040..98f73b2 100644 --- a/sshfs/spec.py +++ b/sshfs/spec.py @@ -1,4 +1,5 @@ import asyncio +import hashlib import posixpath import shlex import stat @@ -128,6 +129,15 @@ def _get_kwargs_from_urls(urlpath): out.pop("protocol", None) return out + @classmethod + def _parent(cls, path): + path = cls._strip_protocol(path) + if isinstance(path, bytes): + return posixpath.dirname(path.rstrip(b"/")) or ( + b"/" if path.startswith(b"/") else b"" + ) + return super()._parent(path) + @wrap_exceptions async def _connect( self, @@ -215,9 +225,10 @@ async def _info(self, path, **kwargs): attributes = await channel.stat(path) info = self._decode_attributes(attributes) - path = path.rstrip("/") + separator = b"/" if isinstance(path, bytes) else "/" + path = path.rstrip(separator) if info["type"] == "directory": - path += "/" + path += separator info["name"] = path return info @@ -254,7 +265,9 @@ async def _put_file( **kwargs, ): rpath = self._strip_protocol(rpath) - await self._makedirs(self._parent(rpath), exist_ok=True) + parent = self._parent(rpath) + if parent: + await self._makedirs(parent, exist_ok=True) async with self._pool.get() as channel: await channel.put( lpath, @@ -280,6 +293,10 @@ async def _get_file( async def _cp_file(self, lpath, rpath, **kwargs): lpath = self._strip_protocol(lpath) rpath = self._strip_protocol(rpath) + if isinstance(lpath, bytes) or isinstance(rpath, bytes): + async with self._pool.get() as channel: + await channel.copy(lpath, rpath, follow_symlinks=True) + return cmd = f"cp {shlex.quote(lpath)} {shlex.quote(rpath)}" await self._execute(cmd) @@ -291,7 +308,7 @@ async def _ls(self, path, detail=True, **kwargs): infos = [] for file_attr in file_attrs: - if file_attr.filename in ["", ".", ".."]: + if file_attr.filename in ("", ".", "..", b"", b".", b".."): continue info = self._decode_attributes(file_attr.attrs) info["name"] = posixpath.join(path, file_attr.filename) @@ -369,6 +386,13 @@ async def _rm(self, path, recursive=False, **kwargs): @wrap_exceptions async def _checksum(self, path): path = self._strip_protocol(path) + if isinstance(path, bytes): + digest = hashlib.md5(usedforsecurity=False) + async with self._pool.get() as channel: + async with channel.open(path, "rb") as file: + while chunk := await file.read(READ_BLOCK_SIZE): + digest.update(chunk) + return digest.hexdigest() system = await self._get_system() if system == "Linux": command = "md5sum" @@ -422,7 +446,9 @@ async def _cat_file(self, path, **kwargs): async def _pipe_file(self, path, data, chunksize=50 * 2**20, **kwargs): """Asynchronously writes the given data to a remote file in chunks.""" path = self._strip_protocol(path) - await self._makedirs(self._parent(path), exist_ok=True) + parent = self._parent(path) + if parent: + await self._makedirs(parent, exist_ok=True) async with self._pool.get() as channel: async with channel.open(path, "wb") as f: diff --git a/tests/test_sshfs.py b/tests/test_sshfs.py index 494f523..cd913d5 100644 --- a/tests/test_sshfs.py +++ b/tests/test_sshfs.py @@ -11,7 +11,13 @@ import fsspec import pytest -from asyncssh.sftp import SFTPAttrs, SFTPClient, SFTPFailure, SFTPOpUnsupported +from asyncssh.sftp import ( + SFTPAttrs, + SFTPClient, + SFTPFailure, + SFTPName, + SFTPOpUnsupported, +) from sshfs import SSHFileSystem from sshfs.file import SSHFile @@ -76,6 +82,17 @@ def fs_hard_queue(ssh_server, user="user"): ) +@pytest.fixture +def fs_bytes(ssh_server): + return SSHFileSystem( + host=ssh_server.host, + port=ssh_server.port, + username="user", + client_keys=[USERS["user"]], + sftp_client_kwargs={"path_encoding": None}, + ) + + def strip_keys(info): for key in ["name", "time", "mtime", "atime"]: info.pop(key, None) @@ -156,6 +173,79 @@ def test_sftp_client_kwargs_path_encoding(ssh_server, tmp_path, user="user"): assert fs.cat_file(encoded + b"/data.txt") == b"data" +def test_bytes_metadata_and_remove(fs_bytes, tmp_path): + directory = tmp_path / "byte-paths" + directory.mkdir() + file = directory / "data.txt" + file.write_bytes(b"data") + path = os.fsencode(file) + parent = os.fsencode(directory) + assert fs_bytes.info(path)["name"] == path + assert fs_bytes.info(parent + b"/")["name"] == parent + b"/" + assert fs_bytes.exists(path) and fs_bytes.isfile(path) + assert fs_bytes.isdir(parent) and fs_bytes.size(path) == 4 + assert fs_bytes.modified(path).tzinfo == UTC + fs_bytes.rm(path) + assert not file.exists() + fs_bytes.rm(parent, recursive=True) + assert not directory.exists() + + +def test_bytes_relative_upload_and_download(fs_bytes, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + source = tmp_path / "local-source" + source.write_bytes(b"relative data") + fs_bytes.put_file(str(source), b"remote-file") + assert fs_bytes.info(b"remote-file")["name"] == b"remote-file" + destination = tmp_path / "local-copy" + fs_bytes.get_file(b"remote-file", str(destination)) + assert destination.read_bytes() == b"relative data" + fs_bytes.pipe_file(b"piped-file", b"piped data") + assert fs_bytes.cat_file(b"piped-file") == b"piped data" + + +@pytest.mark.parametrize("operation", ["put", "pipe", "copy", "checksum"]) +def test_bytes_file_operations(fs_bytes, tmp_path, operation): + local = tmp_path / "source" + data = b"file data" * 10000 + local.write_bytes(data) + destination = tmp_path / "nested" / "a file ' $name" + path = os.fsencode(destination) + if operation == "put": + fs_bytes.put_file(str(local), path) + elif operation == "pipe": + fs_bytes.pipe_file(path, data) + elif operation == "copy": + destination.parent.mkdir() + fs_bytes.cp_file(os.fsencode(local), path) + else: + assert ( + fs_bytes.checksum(os.fsencode(local)) + == hashlib.md5(data).hexdigest() + ) + return + assert destination.read_bytes() == data + + +@pytest.mark.parametrize("detail", [True, False]) +def test_bytes_ls_omits_dot_entries(fs_bytes, tmp_path, monkeypatch, detail): + (tmp_path / "data").write_bytes(b"data") + path = os.fsencode(tmp_path) + original = SFTPClient.readdir + + async def readdir(channel, path): + entries = await original(channel, path) + attrs = await channel.stat(path) + return entries + [ + SFTPName(name, attrs=attrs) for name in (b"", b".", b"..") + ] + + monkeypatch.setattr(SFTPClient, "readdir", readdir) + listing = fs_bytes.ls(path, detail=detail) + names = [info["name"] for info in listing] if detail else listing + assert names == [path + b"/data"] + + def test_info(fs, remote_dir, plain_remote_dir): fs.touch(remote_dir + "/a.txt") details = fs.info(remote_dir + "/a.txt")