Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 27 additions & 16 deletions pydrive2/fs/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,14 @@

import appdirs
from fsspec.spec import AbstractFileSystem
from funcy import cached_property, retry, wrap_prop, wrap_with
from funcy import (
cached_property,
chunks,
distinct,
retry,
wrap_prop,
wrap_with,
)
from tqdm.utils import CallbackIOWrapper

from pydrive2.auth import GoogleAuth, InvalidCredentialsError
Expand All @@ -21,6 +28,7 @@
logger = logging.getLogger(__name__)

FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
_PARENT_QUERY_BATCH_SIZE = 50

COMMON_SETTINGS = {
"get_refresh_token": True,
Expand All @@ -40,6 +48,11 @@ class GDriveAuthError(Exception):
pass


def _quote_query_value(value):
escaped = value.replace("\\", "\\\\").replace("'", "\\'")
return f"'{escaped}'"


def _gdrive_retry(func):
def should_retry(exc):
from pydrive2.files import ApiRequestError
Expand Down Expand Up @@ -317,27 +330,24 @@ def _gdrive_list(self, query):
# Fetch pages until None is received, lazily flatten the thing.
return chain.from_iterable(iter(get_list, None))

def _gdrive_list_ids(self, query_ids):
query = " or ".join(
f"'{query_id}' in parents" for query_id in query_ids
)
query = f"({query}) and trashed=false"
return self._gdrive_list(query)
def _gdrive_list_ids(self, query_ids, *, title=None):
filters = "trashed=false"
if title is not None:
filters += f" and title={_quote_query_value(title)}"
# Drive can reject large parent queries. Query each parent only once.
for batch in chunks(_PARENT_QUERY_BATCH_SIZE, distinct(query_ids)):
parents = " or ".join(
f"{_quote_query_value(parent)} in parents" for parent in batch
)
yield from self._gdrive_list(f"({parents}) and {filters}")

def _get_remote_item_ids(
self, parent_ids, parent_path, title, use_cache=True
):
if not parent_ids:
return None
query = "trashed=false and ({})".format(
" or ".join(
f"'{parent_id}' in parents" for parent_id in parent_ids
)
)
query += " and title='{}'".format(title.replace("'", "\\'"))

res = []
for item in self._gdrive_list(query):
for item in self._gdrive_list_ids(parent_ids, title=title):
# GDrive list API is case insensitive, we need to compare
# all results and pick the ones with the right title
if item["title"] == title:
Expand All @@ -354,7 +364,8 @@ def _get_cached_item_ids(self, path, use_cache):
if not path:
return [self.root]
if use_cache:
return self._ids_cache["dirs"].get(path, [])
# Another caller can add parents while a batched lookup is running.
return self._ids_cache["dirs"].get(path, []).copy()
return []

def _path_to_item_ids(self, path, create=False, use_cache=True):
Expand Down
173 changes: 173 additions & 0 deletions pydrive2/test/test_fs_listing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
import re
from unittest.mock import Mock

import pytest

from pydrive2.fs import GDriveFileSystem
from pydrive2.fs.spec import FOLDER_MIME_TYPE


@pytest.fixture
def listing_fs():
return GDriveFileSystem(
"root", google_auth=Mock(), skip_instance_cache=True
)


@pytest.mark.parametrize("count", [0, 1, 50, 51, 100, 251])
@pytest.mark.parametrize("container", [list, dict.fromkeys, iter])
def test_list_ids_batches(listing_fs, monkeypatch, count, container):
ids = [f"parent-{index}" for index in range(count)]
listed = []

def list_query(query):
parents = re.findall(r"'([^']+)' in parents", query)
# Model an API that rejects overly complex parent queries.
assert 0 < len(parents) <= 50
assert query.endswith(") and trashed=false")
listed.extend(parents)
return iter(parents)

monkeypatch.setattr(listing_fs, "_gdrive_list", list_query)

assert list(listing_fs._gdrive_list_ids(container(ids))) == ids
assert listed == ids


def test_list_ids_lazy(listing_fs, monkeypatch):
consumed = []

def parents():
for index in range(51):
consumed.append(index)
yield str(index)

list_query = Mock(side_effect=[iter(["page-1", "page-2"]), iter(["last"])])
monkeypatch.setattr(listing_fs, "_gdrive_list", list_query)
result = listing_fs._gdrive_list_ids(parents())
assert consumed == []
list_query.assert_not_called()

assert next(result) == "page-1"
assert len(consumed) == 50
assert next(result) == "page-2"
assert list_query.call_count == 1
assert list(result) == ["last"]
assert len(consumed) == 51
assert list_query.call_count == 2


def test_ls_repeated_cached_parent_across_batches(listing_fs, monkeypatch):
parents = [f"id-{i}" for i in range(51)]
listing_fs._cache_path_id("duplicate", *parents, parents[0])
batches = []

def list_file(params):
batch = re.findall(r"'([^']+)' in parents", params["q"])
batches.append(batch)
# Distinct children with the same title must all remain in the listing.
children = [
{
"id": f"child-{parent}",
"title": "file",
"mimeType": "text/plain",
}
for parent in dict.fromkeys(batch)
]
return iter([children[:1], [], children[1:]])

monkeypatch.setattr(listing_fs.client, "ListFile", list_file)
assert listing_fs.ls("root/duplicate") == ["root/duplicate/file"] * 51
assert batches == [parents[:50], parents[50:]]


def test_ls_uses_parents_cached_at_start(listing_fs, monkeypatch):
parents = [f"id-{i}" for i in range(51)]
listing_fs._cache_path_id("dir", *parents)

def list_query(query):
if "'id-0' in parents" in query:
# Simulate another caller caching a parent during the first batch.
listing_fs._cache_path_id("dir", "late-parent")
if "'late-parent' in parents" in query:
return [{"title": "late-file", "mimeType": "text/plain"}]
return []

monkeypatch.setattr(listing_fs, "_gdrive_list", list_query)
assert listing_fs.ls("root/dir") == []
assert listing_fs.ls("root/dir") == ["root/dir/late-file"]


def test_path_lookup_batches_duplicate_folder_names(listing_fs, monkeypatch):
parents = [f"id-{i}" for i in range(51)]
listing_fs._cache_path_id("duplicate", *parents)
title = r"File's\name"
batches = []

def list_file(params):
query = params["q"]
batch = re.findall(r"'([^']+)' in parents", query)
assert 0 < len(batch) <= 50
assert r"title='File\'s\\name'" in query
assert "trashed=false" in query
batches.append(batch)
if parents[-1] in batch:
item_id, item_title = "found", title
else:
item_id, item_title = "wrong-case", title.lower()
item = {
"id": item_id,
"title": item_title,
"mimeType": FOLDER_MIME_TYPE,
}
return iter([[item]])

monkeypatch.setattr(listing_fs.client, "ListFile", list_file)
assert listing_fs.exists(f"root/duplicate/{title}")
assert listing_fs._ids_cache["dirs"][f"duplicate/{title}"] == ["found"]
assert batches == [parents[:50], parents[50:]]


@pytest.mark.parametrize("detail", [False, True])
def test_find_wide_directory(listing_fs, monkeypatch, detail):
folders = [f"folder-{index}" for index in range(251)]

def list_query(query):
parents = re.findall(r"'([^']+)' in parents", query)
assert 0 < len(parents) <= 50
if parents == ["root"]:
return iter(
{
"id": folder,
"title": folder,
"mimeType": FOLDER_MIME_TYPE,
"parents": [{"id": "root"}],
}
for folder in folders
)
return iter(
{
"title": "data",
"mimeType": "application/octet-stream",
"parents": [{"id": parent}],
"fileSize": "7",
"md5Checksum": "checksum",
}
for parent in parents
)

monkeypatch.setattr(listing_fs, "_gdrive_list", list_query)
files = listing_fs.find("root", detail=detail)
expected = [f"root/{folder}/data" for folder in folders]
assert list(files) == expected
if detail:
assert all(
item
== {
"name": path,
"type": "file",
"size": 7,
"checksum": "checksum",
}
for path, item in files.items()
)
Loading