Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions docs/fsspec.rst
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,42 @@ Installation

pip install 'pydrive2[fsspec]'

Installing PyDrive2 registers ``gdrive://`` with fsspec. The ``[fsspec]``
extra installs the dependencies needed to use the backend. This registration
takes precedence over fsspec's built-in mapping to ``gdrive-fsspec``. If
both packages are installed, ``fsspec.open("gdrive://...")`` uses PyDrive2's
paths and credentials. To select the other backend, instantiate
``gdrive_fsspec.GoogleDriveFileSystem`` directly.

The first path component is a case-sensitive Drive ID, ``root``, or
``appDataFolder``. Open an existing file by ID, or name a file relative to a
folder that the account can access:

.. code-block:: python

import fsspec

options = {
"use_service_account": True,
"client_json_file_path": "/path/to/service-account.json",
}
with fsspec.open("gdrive://FILE_ID", "rb", **options) as stream:
data = stream.read()

with fsspec.open("gdrive://FOLDER_ID/report.csv", "rt", **options) as stream:
header = stream.readline()

fs = fsspec.filesystem("gdrive", path="FOLDER_ID", **options)
names = fs.ls("FOLDER_ID")

A filesystem instance is scoped to one root. Use URLs under the same folder
ID or root alias in a single ``fsspec.open_files`` call. Sibling URLs reuse
the filesystem and its directory ID cache. Reads by file ID skip name
lookups. Reads are sequential. To create a file, use a folder ID or root
alias followed by a filename. As with the direct PyDrive2 filesystem,
a write creates a Drive item even when a sibling has the same name. All
reads require credentials, including files shared by link.

Local webserver
---------------

Expand Down
56 changes: 41 additions & 15 deletions pydrive2/fs/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,9 +191,11 @@ def _service_auth(
class GDriveFileSystem(AbstractFileSystem):
"""Access to gdrive as an fsspec filesystem"""

protocol = "gdrive"

def __init__(
self,
path,
path="root",
google_auth=None,
client_id=None,
client_secret=None,
Expand All @@ -208,7 +210,8 @@ def __init__(
):
"""Create an instance of GDriveFileSystem.

:param path: gdrive path.
:param path: gdrive path. Defaults to ``"root"`` (the Drive
root), so ``fsspec`` can build the filesystem without one.
:type path: str.
:param google_auth: Authenticated GoogleAuth instance.
:type google_auth: GoogleAuth.
Expand Down Expand Up @@ -271,12 +274,24 @@ def __init__(
self._trash_only = trash_only
self._acknowledge_abuse = acknowledge_abuse

@classmethod
def _get_kwargs_from_urls(cls, path):
# A filesystem caches names relative to a single Drive root.
root, _, _ = cls._strip_protocol(path).partition("/")
return {} if root in {"", "root"} else {"path": root}

def split_path(self, path):
parts = path.replace("//", "/").rstrip("/").split("/", 1)
if len(parts) == 2:
return parts
else:
return parts[0], ""
path = self._strip_protocol(path).replace("//", "/")
root, _, base = path.partition("/")
return root or "root", base

def _split_mounted_path(self, path):
root, base = self.split_path(path)
if root != self.root:
raise ValueError(
f"Path {path!r} is outside Drive root {self.root!r}"
)
return root, base

@wrap_prop(threading.RLock())
@cached_property
Expand Down Expand Up @@ -414,8 +429,7 @@ def _path_to_item_ids(self, path, create=False, use_cache=True):
)

def _get_item_id(self, path, create=False, use_cache=True):
bucket, base = self.split_path(path)
assert bucket == self.root
_, base = self._split_mounted_path(path)

item_ids = self._path_to_item_ids(base, create, use_cache)
if item_ids:
Expand Down Expand Up @@ -448,8 +462,18 @@ def _create_dir(self, parent_id, name, remote_path):
return item["id"]

def exists(self, path):
from pydrive2.files import ApiRequestError

root, base = self._split_mounted_path(path)
try:
self._get_item_id(path)
if not base and root not in {"root", "appDataFolder"}:
self.info(path)
else:
self._get_item_id(path)
except ApiRequestError as exc:
if exc.error.get("code") == 404:
return False
raise
except FileNotFoundError:
return False
else:
Expand All @@ -463,12 +487,11 @@ def info(self, path):
gdrive_file.FetchMetadata(fields=_INFO_FIELDS)

return _file_entry(
posixpath.join(bucket, base.rstrip("/")), gdrive_file
posixpath.join(bucket, base).rstrip("/"), gdrive_file
)

def ls(self, path, detail=False):
bucket, base = self.split_path(path)
assert bucket == self.root
bucket, base = self._split_mounted_path(path)

dir_ids = self._path_to_item_ids(base)
if not dir_ids:
Expand All @@ -488,8 +511,7 @@ def ls(self, path, detail=False):
return [content["name"] for content in contents]

def find(self, path, detail=False, **kwargs):
bucket, base = self.split_path(path)
assert bucket == self.root
bucket, base = self._split_mounted_path(path)

# Make sure the base path is cached and dir_ids below has some
# dirs revelant to this call
Expand Down Expand Up @@ -641,6 +663,10 @@ def cb(value, total):
def _open(self, path, mode, **kwargs):
assert mode in {"rb", "wb"}
if mode == "wb":
if not self._split_mounted_path(path)[1]:
raise ValueError(
"To create a Drive file, specify a folder ID and name"
)
return GDriveBufferedWriter(self, path)
else:
item_id = self._get_item_id(path)
Expand Down
10 changes: 10 additions & 0 deletions pydrive2/test/test_fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,16 @@ def test_open_rw(fs, remote_dir):
assert stream.read() == data


def test_fsspec_urls(fs, remote_dir):
options = {"google_auth": fs.client.auth}
named_url = f"gdrive://{remote_dir}/report.txt"
with fsspec.open(named_url, "wt", **options) as stream:
stream.write("hello")
file_id = fs._get_item_id(remote_dir + "/report.txt")
with fsspec.open(f"gdrive://{file_id}", "rt", **options) as stream:
assert stream.read() == "hello"


def test_concurrent_operations(fs, fs_factory, remote_dir):
# Include an extra dir name to force upload operations creating it
# this way we can also test that only a single directory is created
Expand Down
96 changes: 96 additions & 0 deletions pydrive2/test/test_fs_registration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import subprocess
import sys
from urllib.parse import urlsplit

import fsspec
import pytest

from pydrive2.fs import GDriveFileSystem
from pydrive2.fs.spec import FOLDER_MIME_TYPE
from pydrive2.test.http_mock import error_response, query, response


def test_fsspec_discovers_backend_without_eager_import():
subprocess.run(
[
sys.executable,
"-c",
(
"import fsspec, sys\n"
"assert 'pydrive2.fs' not in sys.modules\n"
"assert 'googleapiclient.discovery' not in sys.modules\n"
"cls = fsspec.get_filesystem_class('gdrive')\n"
"from pydrive2.fs import GDriveFileSystem\n"
"assert cls is GDriveFileSystem"
),
],
check=True,
)


def test_filesystem_factory(mock_drive):
with pytest.raises(ValueError, match="Specify credentials"):
fsspec.filesystem("gdrive")
drive, requests = mock_drive()
fs = fsspec.filesystem("gdrive", google_auth=drive.auth)
assert (fs.root, fs.base) == ("root", "")
mounted = GDriveFileSystem("FoLdEr/sub", google_auth=drive.auth)
assert (mounted.root, mounted.base) == ("FoLdEr", "sub")
assert not requests


def test_open_by_file_id(mock_drive):
drive, requests = mock_drive(({"status": "200"}, b"contents"))
with fsspec.open("gdrive://FiLe_ID", google_auth=drive.auth) as stream:
assert stream.read() == b"contents"
(download,) = requests
assert urlsplit(download[0]).path == "/drive/v3/files/FiLe_ID"
assert query(download)["alt"] == ["media"]


def test_open_files_reuses_folder_lookup(mock_drive):
def entry(name, item_id, mime_type="text/plain"):
return response(
{"files": [{"name": name, "id": item_id, "mimeType": mime_type}]}
)

drive, requests = mock_drive(
response({"driveId": "shared"}),
entry("sub", "sub-ID", FOLDER_MIME_TYPE),
entry("a", "file-a"),
({"status": "200"}, b"one"),
entry("b", "file-b"),
({"status": "200"}, b"two"),
response({"files": []}),
)
prefix = "gdrive://FoLdEr/sub"
options = {"google_auth": drive.auth}
first, second = fsspec.open_files(
[prefix + "/a", prefix + "/b"], **options
)
assert fsspec.open(prefix + "/b", **options).fs is first.fs is second.fs
assert not requests
with first as a, second as b:
assert (a.read(), b.read()) == (b"one", b"two")
assert first.fs.ls(prefix) == []
shared, sub, a_lookup, _, b_lookup, _, listing = requests
assert urlsplit(shared[0]).path.endswith("/FoLdEr")
assert "'FoLdEr' in parents" in query(sub)["q"][0]
assert all(
"'sub-ID' in parents" in query(r)["q"][0]
for r in (a_lookup, b_lookup, listing)
)
assert query(listing)["driveId"] == ["shared"]


def test_missing_file_id(mock_drive):
drive, requests = mock_drive(error_response(404, "notFound"))
fs, path = fsspec.core.url_to_fs(
"gdrive://Missing", google_auth=drive.auth
)
assert not fs.exists(path)
assert len(requests) == 1
with pytest.raises(ValueError, match="folder ID and name"):
fs.open(path, "wb")
with pytest.raises(ValueError, match="outside Drive root"):
fs.ls("gdrive://Other/sub")
3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ typing = ["mypy==2.4.1", "types-httplib2", "types-PyYAML"]
hook-dirs = "pydrive2.__pyinstaller:get_hook_dirs"
tests = "pydrive2.__pyinstaller:get_PyInstaller_tests"

[project.entry-points."fsspec.specs"]
gdrive = "pydrive2.fs:GDriveFileSystem"

[tool.setuptools]
packages=[
"pydrive2",
Expand Down
Loading