Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,14 @@ fs = SSHFileSystem(

Note: you can also pass `client_keys` as an argument to `fsspec.open`.

To pass remote filenames as bytes, set
`sftp_client_kwargs={"path_encoding": None}` when creating the filesystem.
Use bytes for remote paths with this setting. Metadata, listing, removal,
uploads, copies, and checksums preserve these paths through SFTP. Checksums
of byte paths stream the file over SFTP, so they require transferring its data.
Inherited fsspec traversal and bulk methods such as `walk`, `find`, `glob`,
`get`, `put`, and `copy` still require string paths.

All operations and their descriptions are specified [here](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem).
Here are a few example calls you can make, starting with `info()` which allows you to retrieve the metadata about given path;

Expand Down
36 changes: 31 additions & 5 deletions sshfs/spec.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import asyncio
import hashlib
import posixpath
import shlex
import stat
Expand Down Expand Up @@ -128,6 +129,15 @@ def _get_kwargs_from_urls(urlpath):
out.pop("protocol", None)
return out

@classmethod
def _parent(cls, path):
path = cls._strip_protocol(path)
if isinstance(path, bytes):
return posixpath.dirname(path.rstrip(b"/")) or (
b"/" if path.startswith(b"/") else b""
)
return super()._parent(path)

@wrap_exceptions
async def _connect(
self,
Expand Down Expand Up @@ -215,9 +225,10 @@ async def _info(self, path, **kwargs):
attributes = await channel.stat(path)

info = self._decode_attributes(attributes)
path = path.rstrip("/")
separator = b"/" if isinstance(path, bytes) else "/"
path = path.rstrip(separator)
if info["type"] == "directory":
path += "/"
path += separator
info["name"] = path
return info

Expand Down Expand Up @@ -254,7 +265,9 @@ async def _put_file(
**kwargs,
):
rpath = self._strip_protocol(rpath)
await self._makedirs(self._parent(rpath), exist_ok=True)
parent = self._parent(rpath)
if parent:
await self._makedirs(parent, exist_ok=True)
async with self._pool.get() as channel:
await channel.put(
lpath,
Expand All @@ -280,6 +293,10 @@ async def _get_file(
async def _cp_file(self, lpath, rpath, **kwargs):
lpath = self._strip_protocol(lpath)
rpath = self._strip_protocol(rpath)
if isinstance(lpath, bytes) or isinstance(rpath, bytes):
async with self._pool.get() as channel:
await channel.copy(lpath, rpath, follow_symlinks=True)
return
cmd = f"cp {shlex.quote(lpath)} {shlex.quote(rpath)}"
await self._execute(cmd)

Expand All @@ -291,7 +308,7 @@ async def _ls(self, path, detail=True, **kwargs):

infos = []
for file_attr in file_attrs:
if file_attr.filename in ["", ".", ".."]:
if file_attr.filename in ("", ".", "..", b"", b".", b".."):
continue
info = self._decode_attributes(file_attr.attrs)
info["name"] = posixpath.join(path, file_attr.filename)
Expand Down Expand Up @@ -369,6 +386,13 @@ async def _rm(self, path, recursive=False, **kwargs):
@wrap_exceptions
async def _checksum(self, path):
path = self._strip_protocol(path)
if isinstance(path, bytes):
digest = hashlib.md5(usedforsecurity=False)
async with self._pool.get() as channel:
async with channel.open(path, "rb") as file:
while chunk := await file.read(READ_BLOCK_SIZE):
digest.update(chunk)
return digest.hexdigest()
system = await self._get_system()
if system == "Linux":
command = "md5sum"
Expand Down Expand Up @@ -422,7 +446,9 @@ async def _cat_file(self, path, **kwargs):
async def _pipe_file(self, path, data, chunksize=50 * 2**20, **kwargs):
"""Asynchronously writes the given data to a remote file in chunks."""
path = self._strip_protocol(path)
await self._makedirs(self._parent(path), exist_ok=True)
parent = self._parent(path)
if parent:
await self._makedirs(parent, exist_ok=True)

async with self._pool.get() as channel:
async with channel.open(path, "wb") as f:
Expand Down
92 changes: 91 additions & 1 deletion tests/test_sshfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,13 @@

import fsspec
import pytest
from asyncssh.sftp import SFTPAttrs, SFTPClient, SFTPFailure, SFTPOpUnsupported
from asyncssh.sftp import (
SFTPAttrs,
SFTPClient,
SFTPFailure,
SFTPName,
SFTPOpUnsupported,
)

from sshfs import SSHFileSystem
from sshfs.file import SSHFile
Expand Down Expand Up @@ -76,6 +82,17 @@ def fs_hard_queue(ssh_server, user="user"):
)


@pytest.fixture
def fs_bytes(ssh_server):
return SSHFileSystem(
host=ssh_server.host,
port=ssh_server.port,
username="user",
client_keys=[USERS["user"]],
sftp_client_kwargs={"path_encoding": None},
)


def strip_keys(info):
for key in ["name", "time", "mtime", "atime"]:
info.pop(key, None)
Expand Down Expand Up @@ -156,6 +173,79 @@ def test_sftp_client_kwargs_path_encoding(ssh_server, tmp_path, user="user"):
assert fs.cat_file(encoded + b"/data.txt") == b"data"


def test_bytes_metadata_and_remove(fs_bytes, tmp_path):
directory = tmp_path / "byte-paths"
directory.mkdir()
file = directory / "data.txt"
file.write_bytes(b"data")
path = os.fsencode(file)
parent = os.fsencode(directory)
assert fs_bytes.info(path)["name"] == path
assert fs_bytes.info(parent + b"/")["name"] == parent + b"/"
assert fs_bytes.exists(path) and fs_bytes.isfile(path)
assert fs_bytes.isdir(parent) and fs_bytes.size(path) == 4
assert fs_bytes.modified(path).tzinfo == UTC
fs_bytes.rm(path)
assert not file.exists()
fs_bytes.rm(parent, recursive=True)
assert not directory.exists()


def test_bytes_relative_upload_and_download(fs_bytes, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
source = tmp_path / "local-source"
source.write_bytes(b"relative data")
fs_bytes.put_file(str(source), b"remote-file")
assert fs_bytes.info(b"remote-file")["name"] == b"remote-file"
destination = tmp_path / "local-copy"
fs_bytes.get_file(b"remote-file", str(destination))
assert destination.read_bytes() == b"relative data"
fs_bytes.pipe_file(b"piped-file", b"piped data")
assert fs_bytes.cat_file(b"piped-file") == b"piped data"


@pytest.mark.parametrize("operation", ["put", "pipe", "copy", "checksum"])
def test_bytes_file_operations(fs_bytes, tmp_path, operation):
local = tmp_path / "source"
data = b"file data" * 10000
local.write_bytes(data)
destination = tmp_path / "nested" / "a file ' $name"
path = os.fsencode(destination)
if operation == "put":
fs_bytes.put_file(str(local), path)
elif operation == "pipe":
fs_bytes.pipe_file(path, data)
elif operation == "copy":
destination.parent.mkdir()
fs_bytes.cp_file(os.fsencode(local), path)
else:
assert (
fs_bytes.checksum(os.fsencode(local))
== hashlib.md5(data).hexdigest()
)
return
assert destination.read_bytes() == data


@pytest.mark.parametrize("detail", [True, False])
def test_bytes_ls_omits_dot_entries(fs_bytes, tmp_path, monkeypatch, detail):
(tmp_path / "data").write_bytes(b"data")
path = os.fsencode(tmp_path)
original = SFTPClient.readdir

async def readdir(channel, path):
entries = await original(channel, path)
attrs = await channel.stat(path)
return entries + [
SFTPName(name, attrs=attrs) for name in (b"", b".", b"..")
]

monkeypatch.setattr(SFTPClient, "readdir", readdir)
listing = fs_bytes.ls(path, detail=detail)
names = [info["name"] for info in listing] if detail else listing
assert names == [path + b"/data"]


def test_info(fs, remote_dir, plain_remote_dir):
fs.touch(remote_dir + "/a.txt")
details = fs.info(remote_dir + "/a.txt")
Expand Down
Loading