diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dc075f6e..5a39c51e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -10,11 +10,14 @@ Contributing code ~~~~~~~~~~~~~~~~~ If you have improvements to PyDrive2, send us your pull requests! For those -just getting started, Github has a `howto `_. +just getting started, Github has a `howto`_. -Install the development tools with ``python -m pip install -e '.[tests,fsspec,typing]'``. -Run ``ruff check .`` to lint Python files and ``ruff format .`` to format them. -Run ``python -m mypy`` to type-check the package for Python 3.11. +Install the development tools with +``python -m pip install -e '.[tests,fsspec,typing]'``. Run ``ruff check .`` to +lint Python files and ``ruff format .`` to format them. Run ``python -m mypy`` +to type-check the package for Python 3.11. + +.. _howto: https://help.github.com/articles/using-pull-requests/ Live Drive tests for external pull requests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 8d064426..e3b17fc2 100644 --- a/README.rst +++ b/README.rst @@ -1,42 +1,43 @@ |CI| |Conda| |PyPI| -.. |CI| image:: https://github.com/iterative/PyDrive2/workflows/Tests/badge.svg?branch=main +.. |CI| image:: + https://github.com/iterative/PyDrive2/workflows/Tests/badge.svg?branch=main :target: https://github.com/iterative/PyDrive2/actions :alt: GHA Tests -.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/PyDrive2.svg?label=conda&logo=conda-forge +.. |Conda| image:: + https://img.shields.io/conda/v/conda-forge/PyDrive2.svg?label=conda&logo=conda-forge :target: https://anaconda.org/conda-forge/PyDrive2 :alt: Conda-forge -.. |PyPI| image:: https://img.shields.io/pypi/v/PyDrive2.svg?label=pip&logo=PyPI&logoColor=white +.. |PyPI| image:: + https://img.shields.io/pypi/v/PyDrive2.svg?label=pip&logo=PyPI&logoColor=white :target: https://pypi.org/project/PyDrive2 :alt: PyPI PyDrive2 -------- -*PyDrive2* is a wrapper library of -`google-api-python-client `_ -that simplifies many common Google Drive API V2 tasks. It is an actively -maintained fork of `https://pypi.python.org/pypi/PyDrive `_. -By the authors and maintainers of the `Git for Data `_ - DVC -project. +*PyDrive2* is a wrapper library of `google-api-python-client`_ that simplifies +many common Google Drive API V2 tasks. It is an actively maintained fork of +https://pypi.python.org/pypi/PyDrive. By the authors and maintainers of the +`Git for Data `_ - DVC project. Project Info ------------ -- Package: `https://pypi.python.org/pypi/PyDrive2 `_ -- Documentation: `https://docs.iterative.ai/PyDrive2 `_ -- Source: `https://github.com/iterative/PyDrive2 `_ -- Changelog: `https://github.com/iterative/PyDrive2/releases `_ +- Package: https://pypi.python.org/pypi/PyDrive2 +- Documentation: https://docs.iterative.ai/PyDrive2 +- Source: https://github.com/iterative/PyDrive2 +- Changelog: https://github.com/iterative/PyDrive2/releases - `Running tests `_ Features of PyDrive2 -------------------- - Simplifies OAuth2.0 into just few lines with flexible settings. -- Wraps `Google Drive API V2 `_ into - classes of each resource to make your program more object-oriented. +- Wraps `Google Drive API V2`_ into classes of each resource to make your + program more object-oriented. - Helps common operations else than API calls, such as content fetching and pagination control. - Provides `fsspec`_ filesystem implementation. @@ -101,7 +102,8 @@ efficient way. # Created file hello.png with mimeType image/png file3 = drive.CreateFile({'id': file2['id']}) - print('Downloading file %s from Google Drive' % file3['title']) # 'hello.png' + print('Downloading file %s from Google Drive' % file3['title']) + # 'hello.png' file3.GetContentFile('world.png') # Save Drive file as a local file # or download Google Docs files in an export format provided. @@ -122,7 +124,9 @@ File listing pagination made easy # Paginate file lists by specifying number of max results for file_list in drive.ListFile({'maxResults': 10}): - print('Received {} files from Files.list()'.format(len(file_list))) # <= 10 + print( + 'Received {} files from Files.list()'.format(len(file_list)) + ) # <= 10 for file1 in file_list: print('title: {}, id: {}'.format(file1['title'], file1['id'])) @@ -142,7 +146,8 @@ Install PyDrive2 with the required dependencies from pydrive2.fs import GDriveFileSystem - # replace `root` with ID of a drive or directory and give service account access to it + # replace `root` with ID of a drive or directory and give service account + # access to it fs = GDriveFileSystem("root", client_id=my_id, client_secret=my_secret) for root, dnames, fnames in fs.walk("root"): @@ -164,3 +169,7 @@ Thanks to all our contributors! :target: https://github.com/iterative/PyDrive2/graphs/contributors .. _`fsspec`: https://filesystem-spec.readthedocs.io/en/latest/ + +.. _google-api-python-client: + https://github.com/google/google-api-python-client +.. _Google Drive API V2: https://developers.google.com/drive/v2/web/about-sdk diff --git a/docs/filelist.rst b/docs/filelist.rst index ce92f702..41cd4fc9 100644 --- a/docs/filelist.rst +++ b/docs/filelist.rst @@ -1,5 +1,5 @@ File listing made easy -============================= +====================== PyDrive2 handles pagination and returns :class:`~pydrive2.files.GoogleDriveFile` instances. @@ -22,12 +22,15 @@ v3 examples. PyDrive2 uses v2. When adapting a query, check the from pydrive2.drive import GoogleDrive - drive = GoogleDrive(gauth) # Create GoogleDrive instance with authenticated GoogleAuth instance + # Create GoogleDrive instance with authenticated GoogleAuth instance + drive = GoogleDrive(gauth) # Auto-iterate through all files in the root folder. - file_list = drive.ListFile({'q': "'root' in parents and trashed=false"}).GetList() + file_list = drive.ListFile( + {'q': "'root' in parents and trashed=false"} + ).GetList() for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('title: %s, id: %s' % (file1['title'], file1['id'])) You can update the metadata or content of each returned file. @@ -42,9 +45,9 @@ contains up to ten files. # Paginate file lists by specifying number of max results for file_list in drive.ListFile({'q': 'trashed=true', 'maxResults': 10}): - print('Received %s files from Files.list()' % len(file_list)) # <= 10 - for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('Received %s files from Files.list()' % len(file_list)) # <= 10 + for file1 in file_list: + print('title: %s, id: %s' % (file1['title'], file1['id'])) .. _files.list query parameters: diff --git a/docs/filemanagement.rst b/docs/filemanagement.rst index b0dd58cf..ba6d6072 100644 --- a/docs/filemanagement.rst +++ b/docs/filemanagement.rst @@ -1,8 +1,8 @@ File management made easy ========================= -There are many methods to create and update file metadata and contents. -With *PyDrive*, you don't have to care about any of these different API methods. +There are many methods to create and update file metadata and contents. With +*PyDrive*, you don't have to care about any of these different API methods. Edit the metadata or contents of a :class:`~pydrive2.files.GoogleDriveFile` and call :meth:`~pydrive2.files.GoogleDriveFile.Upload` to save the changes. @@ -26,11 +26,13 @@ In this example, ``gauth`` is an authenticated # title: Hello.txt, id: {{FILE_ID}} :meth:`~pydrive2.drive.GoogleDrive.CreateFile` creates a local file object -with the drive's credentials. Call :meth:`~pydrive2.files.GoogleDriveFile.Upload` -to create ``Hello.txt`` on Google Drive. +with the drive's credentials. Call +:meth:`~pydrive2.files.GoogleDriveFile.Upload` to create ``Hello.txt`` on +Google Drive. Delete, Trash and un-Trash files -------------------------------- + You may want to delete, trash, or un-trash a file. To do this use ``Delete()``, ``Trash()`` or ``UnTrash()`` on a GoogleDriveFile object. @@ -92,7 +94,8 @@ PyDrive gives you access to the metadata of an object through file1 = drive.CreateFile({'id': ''}) - # Fetches all basic metadata fields, including file size, last modified etc. + # Fetches all basic metadata fields, including file size, last modified + # etc. file1.FetchMetadata() # Fetches all metadata available. @@ -107,6 +110,7 @@ See the `Drive API v2 file resource`_ for the available fields. Insert permissions __________________ + Insert, retrieving or deleting permissions is illustrated by making a file readable to all who have a link to the file. @@ -159,9 +163,10 @@ For the more advanced user: ``GetPermissions()`` is a shorthand for: Remove a Permission ___________________ + *PyDrive* allows you to remove a specific permission using the -``DeletePermission(permission_id)`` function. This function allows you to delete -one permission at a time by providing the permission's ID. +``DeletePermission(permission_id)`` function. This function allows you to +delete one permission at a time by providing the permission's ID. .. code-block:: python @@ -194,7 +199,7 @@ field in the query below is ``title``. files = drive.ListFile(query).GetList() List revisions -________________ +______________ Revisions can be fetched using the ``GetRevisions()`` function of a ``GoogleDriveFile``, and can be used like so: @@ -221,7 +226,10 @@ Sample code continues from `Download file metadata from file ID`_: .. code-block:: python - file4 = drive.CreateFile({'title':'appdata.json', 'mimeType':'application/json'}) + file4 = drive.CreateFile({ + 'title': 'appdata.json', + 'mimeType': 'application/json', + }) file4.SetContentString('{"firstname": "John", "lastname": "Smith"}') file4.Upload() # Upload file. file4.SetContentString('{"firstname": "Claudio", "lastname": "Afshar"}') @@ -247,10 +255,10 @@ like so: content_string = file4.GetContentString(encoding='ISO-8859-1') Upload data as bytes in memory buffer --------------------------------------- +------------------------------------- -Data can be kept as bytes in an in-memory buffer when we use the ``io`` module’s -Byte IO operations, we can upload files that reside in memory, for +Data can be kept as bytes in an in-memory buffer when we use the ``io`` +module’s Byte IO operations, we can upload files that reside in memory, for example we have a base64 image, we can decode the string and upload it to drive without first saving it to disk and calling :meth:`~pydrive2.files.GoogleDriveFile.SetContentFile`. @@ -339,11 +347,11 @@ See the :download:`complete Python example <../examples/strip_bom_example.py>`. Abusive files ------------- -Files identified as `abusive`_ (malware, etc.) are only downloadable by the owner. -If you see a -'This file has been identified as malware or spam and cannot be downloaded' -error, set 'acknowledge_abuse=True' parameter in `GetContentFile()`. By using -it you indicate that you acknowledge the risks of downloading potential malware. +Files identified as `abusive`_ (malware, etc.) are only downloadable by the +owner. If you see a 'This file has been identified as malware or spam and +cannot be downloaded' error, set 'acknowledge_abuse=True' parameter in +`GetContentFile()`. By using it you indicate that you acknowledge the risks of +downloading potential malware. .. _Drive API v2 file resource: https://developers.google.com/workspace/drive/api/reference/rest/v2/files#resource:-file diff --git a/docs/fsspec.rst b/docs/fsspec.rst index 19bf2bc0..516abebc 100644 --- a/docs/fsspec.rst +++ b/docs/fsspec.rst @@ -39,6 +39,12 @@ someone else's cached credentials: profile="myprofile", ) +PyDrive2 2.x uses a separate default cache. The first interactive connection +after upgrading from 1.x prompts you to sign in. If you supplied a user +credential file explicitly, choose a fresh path. For DVC jobs that use +``GDRIVE_CREDENTIALS_DATA`` for a user account, replace that secret with the +newly saved JSON. Service-account jobs can reuse their configured key. + Writing cached credentials to a file and using it if it already exists (which avoids interactive auth): @@ -76,7 +82,8 @@ Using json keyfile path: from pydrive2.fs import GDriveFileSystem fs = GDriveFileSystem( - # replace with ID of a drive or directory and give service account access to it + # replace with ID of a drive or directory and give service account + # access to it "root", use_service_account=True, client_json_file_path="/path/to/keyfile.json", @@ -89,7 +96,8 @@ Using json keyfile string: from pydrive2.fs import GDriveFileSystem fs = GDriveFileSystem( - # replace with ID of a drive or directory and give service account access to it + # replace with ID of a drive or directory and give service account + # access to it "root", use_service_account=True, client_json=json_string, @@ -101,20 +109,23 @@ If your service account uses `domain-wide delegation`_, set Additional parameters --------------------- -:trash_only (bool): Move files to trash instead of deleting. -:acknowledge_abuse (bool): Acknowledge the risk when downloading a file - flagged as abusive. See :ref:`abusive-files`. +:trash_only (bool): + Move files to trash instead of deleting. +:acknowledge_abuse (bool): + Acknowledge the risk when downloading a file flagged as abusive. + See :ref:`abusive-files`. Using filesystem ---------------- .. code-block:: python - # replace `root` with ID of a drive or directory and give service account access to it + # replace `root` with ID of a drive or directory and give service account + # access to it for root, dnames, fnames in fs.walk("root"): for dname in dnames: print(f"dir: {root}/{dname}") - + for fname in fnames: print(f"file: {root}/{fname}") @@ -125,4 +136,5 @@ about and manipulating files, refer to fsspec docs on .. _`fsspec`: https://filesystem-spec.readthedocs.io/en/latest/ .. _domain-wide delegation: https://developers.google.com/workspace/guides/create-credentials#set-up-domain-wide-delegation -.. _`how to use a filesystem`: https://filesystem-spec.readthedocs.io/en/latest/usage.html#use-a-file-system +.. _`how to use a filesystem`: + https://filesystem-spec.readthedocs.io/en/latest/usage.html#use-a-file-system diff --git a/docs/index.rst b/docs/index.rst index ddb667ff..d576c9e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,15 +6,16 @@ Welcome to PyDrive2's documentation! ==================================== -PyDrive2 is a wrapper library of `google-api-python-client`_ that simplifies many common Google Drive API tasks. +PyDrive2 is a wrapper library of `google-api-python-client`_ that simplifies +many common Google Drive API tasks. Project Info ============ -- Package: `https://pypi.python.org/pypi/PyDrive2 `_ -- Documentation: `https://docs.iterative.ai/PyDrive2 `_ -- Source: `https://github.com/iterative/PyDrive2 `_ -- Changelog: `https://github.com/iterative/PyDrive2/releases `_ +- Package: https://pypi.python.org/pypi/PyDrive2 +- Documentation: https://docs.iterative.ai/PyDrive2 +- Source: https://github.com/iterative/PyDrive2 +- Changelog: https://github.com/iterative/PyDrive2/releases How to install ============== @@ -32,7 +33,8 @@ To install the current development version from GitHub, use: $ pip install git+https://github.com/iterative/PyDrive2.git#egg=PyDrive2 -.. _`google-api-python-client`: https://github.com/google/google-api-python-client +.. _`google-api-python-client`: + https://github.com/google/google-api-python-client Table of Contents ================= diff --git a/docs/oauth.rst b/docs/oauth.rst index 5f53f329..b4767008 100644 --- a/docs/oauth.rst +++ b/docs/oauth.rst @@ -18,25 +18,21 @@ OAuth2.0 is complex and difficult to start with. To make it more simple, The :doc:`quickstart` registers ``http://localhost:8080/`` as the redirect URI for a web application client. With that client, use :meth:`~pydrive2.auth.GoogleAuth.LocalWebserverAuth` and keep port 8080 -available to receive the authorization code. To open the login URL -yourself, call ``gauth.LocalWebserverAuth(launch_browser=False)``. -The browser must be able to reach the local callback. +available to receive the authorization code. -In PyDrive2 1.x, :meth:`~pydrive2.auth.GoogleAuth.CommandLineAuth` requests -the out-of-band redirect URI ``urn:ietf:wg:oauth:2.0:oob``. Even in -*Testing*, this requires an OAuth client that accepts that URI. Google -blocks out-of-band authorization for apps marked *In Production*. -See `Google's out-of-band migration guide`_. - -.. _Google's out-of-band migration guide: - https://developers.google.com/identity/protocols/oauth2/resources/oob-migration +To open the login URL yourself, call +``gauth.LocalWebserverAuth(launch_browser=False)`` or +:meth:`~pydrive2.auth.GoogleAuth.CommandLineAuth`. Both print the URL +and wait for the same local callback. Open the URL in a browser that can +reach the callback port on the machine running PyDrive2. For unattended +jobs, use a service account or saved user credentials with a refresh token. Automatic and custom authentication with *settings.yaml* -------------------------------------------------------- Read this section if you need a custom authentication flow, **such as silent -authentication on a remote machine**. For an example of such a setup have a look -at `Sample settings.yaml`_. +authentication on a remote machine**. For an example of such a setup have a +look at `Sample settings.yaml`_. OAuth is complicated and it requires a lot of settings. By default, when you don't provide any settings, *PyDrive* will automatically set default @@ -84,40 +80,110 @@ These are all the possible fields of a *settings.yaml* file: Fields explained: -:client_config_backend (str): From where to read client configuration(API application settings such as client_id and client_secrets) from. Valid values are 'file', 'settings' and 'service'. **Default**: 'file'. **Required**: No. -:client_config_file (str): When *client_config_backend* is 'file', path to the file containing client configuration. **Default**: 'client_secrets.json'. **Required**: No. -:client_config (dict): Place holding dictionary for client configuration when *client_config_backend* is 'settings'. **Required**: Yes, only if *client_config_backend* is 'settings' and not using *service_config* -:client_config['client_id'] (str): Client ID of the application. **Required**: Yes, only if *client_config_backend* is 'settings' -:client_config['client_secret'] (str): Client secret of the application. **Required**: Yes, only if *client_config_backend* is 'settings' -:client_config['auth_uri'] (str): The authorization server endpoint URI. +:client_config_backend (str): + From where to read client configuration(API application settings such as + client_id and client_secrets) from. Valid values are 'file', 'settings' and + 'service'. + **Default**: 'file'. + **Required**: No. +:client_config_file (str): + When *client_config_backend* is 'file', path to the file containing client + configuration. + **Default**: 'client_secrets.json'. + **Required**: No. +:client_config (dict): + Place holding dictionary for client configuration when + *client_config_backend* is 'settings'. + **Required**: Yes, only if *client_config_backend* is 'settings' and not + using *service_config* +:client_config['client_id'] (str): + Client ID of the application. + **Required**: Yes, only if *client_config_backend* is 'settings' +:client_config['client_secret'] (str): + Client secret of the application. + **Required**: Yes, only if *client_config_backend* is 'settings' +:client_config['auth_uri'] (str): + The authorization server endpoint URI. **Default**: ``https://accounts.google.com/o/oauth2/auth``. **Required**: No. -:client_config['token_uri'] (str): The token server endpoint URI. +:client_config['token_uri'] (str): + The token server endpoint URI. **Default**: ``https://accounts.google.com/o/oauth2/token``. **Required**: No. -:client_config['redirect_uri'] (str): Redirection endpoint URI. - **Default**: ``urn:ietf:wg:oauth:2.0:oob``. **Required**: No. -:client_config['revoke_uri'] (str): Revoke endpoint URI. **Default**: None. **Required**: No. -:service_config (dict): Place holding dictionary for client configuration when *client_config_backend* is 'service' or 'settings' and using service account. **Required**: Yes, only if *client_config_backend* is 'service' or 'settings' and not using *client_config* -:service_config['client_user_email'] (str): Email address of the user the - service account acts as through `domain-wide delegation`_. +:client_config['redirect_uri'] (str): + Redirection endpoint URI. + **Default**: ``http://localhost``. + **Required**: No. +:client_config['revoke_uri'] (str): + Revoke endpoint URI. + **Default**: None. + **Required**: No. +:service_config (dict): + Place holding dictionary for client configuration when + *client_config_backend* is 'service' or 'settings' and using service + account. + **Required**: Yes, only if *client_config_backend* is 'service' or + 'settings' and not using *client_config* +:service_config['client_user_email'] (str): + Email address of the user the service account acts as through + `domain-wide delegation`_. + **Required**: No. +:service_config['client_json_file_path'] (str): + Path to service account `.json` key file. + **Required**: No. +:service_config['client_json_dict'] (dict): + Service account `.json` key file loaded into a dictionary. + **Required**: No. +:service_config['client_json'] (str): + Service account `.json` key file loaded into a string. **Required**: No. -:service_config['client_json_file_path'] (str): Path to service account `.json` key file. **Required**: No. -:service_config['client_json_dict'] (dict): Service account `.json` key file loaded into a dictionary. **Required**: No. -:service_config['client_json'] (str): Service account `.json` key file loaded into a string. **Required**: No. -:save_credentials (bool): True if you want to save credentials. **Default**: False. **Required**: No. -:save_credentials_backend (str): Backend to save credentials to. 'file' and 'dictionary' are the only valid values for now. **Default**: 'file'. **Required**: No. -:save_credentials_file (str): Destination of credentials file. **Required**: Yes, only if *save_credentials_backend* is 'file'. -:save_credentials_dict (dict): Dict to use for storing credentials. **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. -:save_credentials_key (str): Key within the *save_credentials_dict* to store the credentials in. **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. -:get_refresh_token (bool): True if you want to retrieve refresh token along with access token. **Default**: False. **Required**: No. -:oauth_scope (list of str): OAuth scopes to request. +:save_credentials (bool): + Save OAuth user credentials after signing in. A fresh login stays in + memory when this setting is false. When you load saved credentials, + PyDrive2 writes refreshed tokens back to the same file or dictionary key. + **Default**: False. + **Required**: No. +:save_credentials_backend (str): + Backend to save credentials to. 'file' and 'dictionary' are the only valid + values for now. + **Default**: 'file'. + **Required**: No. +:save_credentials_file (str): + Destination of the credentials file. Give the application a writable + directory for the lock and atomic file replacement. In containers, mount + the cache directory so the file can be replaced on refresh. + **Required**: Yes, only if *save_credentials_backend* is 'file'. +:save_credentials_dict (dict): + Dict to use for storing credentials. + **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. +:save_credentials_key (str): + Key within the *save_credentials_dict* to store the credentials in. + **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. +:get_refresh_token (bool): + True if you want to retrieve refresh token along with access token. + **Default**: False. + **Required**: No. +:oauth_scope (list of str): + OAuth scopes to request. **Default**: ``['https://www.googleapis.com/auth/drive']``. **Required**: No. .. _domain-wide delegation: https://developers.google.com/workspace/guides/create-credentials#set-up-domain-wide-delegation +Upgrading from PyDrive2 1.x +___________________________ + +Sign in once with PyDrive2 2.x to create new OAuth user credentials. Keep +your existing OAuth client ID and client secret. If you configured a cache, +choose a new ``save_credentials_file`` or ``save_credentials_key`` before +signing in. For unattended jobs, set ``get_refresh_token: True`` during +sign-in and provision the newly saved credentials before upgrading the job. + +Service accounts authenticate directly with their configured JSON or PKCS12 +key. Keep that key available to the application. If your deployment relied +solely on a saved 1.x service-account cache, provision its signing key. + Sample *settings.yaml* ______________________ @@ -147,8 +213,14 @@ want to integrate your existing website with Drive API. In that case, you can customize an authentication flow as follows: 1. Get the authorization URL from :meth:`~pydrive2.auth.GoogleAuth.GetAuthUrl`. -2. Ask users to visit the authentication Url and grant access to your application. Retrieve authentication code manually by user or automatically by building your own oauth2callback. -3. Pass the authorization code to :meth:`~pydrive2.auth.GoogleAuth.Auth`. +2. Direct the user to the authorization URL. Receive the authorization code at + the redirect URI registered for your application and verify the OAuth state + in the callback. +3. Pass the authorization code to :meth:`~pydrive2.auth.GoogleAuth.Auth` on + the same instance. Keep the flow for each login until the callback so the + exchange uses its original PKCE verifier. If a separate worker handles + the callback, retain the state and verifier in the user's server-side + session and restore the verifier on the flow before exchanging the code. Your *settings.yaml* will work for your customized authentication flow, too. @@ -158,16 +230,19 @@ Here is a sample code for your customized authentication flow from pydrive2.auth import GoogleAuth + # Configure your registered redirect URI in settings.yaml gauth = GoogleAuth() - auth_url = gauth.GetAuthUrl() # Create authentication url user needs to visit - code = AskUserToVisitLinkAndGiveCode(auth_url) # Your customized authentication flow + # Create authentication url user needs to visit + auth_url = gauth.GetAuthUrl() + # Your application handles the redirect + code = ReceiveVerifiedCallback(auth_url) gauth.Auth(code) # Authorize and build service from the code Authentication with a service account --------------------------------------- +------------------------------------- -A `Service account`_ is a special type of Google account intended to represent a -non-human user that needs to authenticate and be authorized to access data in +A `Service account`_ is a special type of Google account intended to represent +a non-human user that needs to authenticate and be authorized to access data in Google APIs. Typically, service accounts are used in scenarios such as: @@ -195,8 +270,7 @@ account the login is automatic. :return: google auth """ # Define the settings dict to use a service account - # We also can use all options available for the settings dict like - # oauth_scope,save_credentials,etc. + # Set oauth_scope here to request specific Drive permissions. settings = { "client_config_backend": "service", "service_config": { @@ -210,4 +284,5 @@ account the login is automatic. return gauth -.. _`Service account`: https://developers.google.com/workspace/guides/create-credentials#service-account +.. _`Service account`: + https://developers.google.com/workspace/guides/create-credentials#service-account diff --git a/docs/pydrive2.rst b/docs/pydrive2.rst index bcbf0fc9..762fbd18 100644 --- a/docs/pydrive2.rst +++ b/docs/pydrive2.rst @@ -42,7 +42,7 @@ pydrive2.settings module :show-inheritance: pydrive2.fs module ------------------------- +------------------ .. autoclass:: pydrive2.fs.GDriveFileSystem :show-inheritance: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index a9a58863..37b32cab 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,24 +1,30 @@ Quickstart -============================= +========== Authentication -------------- -Drive API requires OAuth2.0 for authentication. *PyDrive2* makes your life much easier by handling complex authentication steps for you. + +Drive API requires OAuth2.0 for authentication. *PyDrive2* makes your life much +easier by handling complex authentication steps for you. 1. Go to `APIs Console`_ and make your own project. 2. Search for 'Google Drive API', select the entry, and click 'Enable'. -3. Select 'Credentials' from the left menu, click 'Create Credentials', select 'OAuth client ID'. -4. Now, the product name and consent screen need to be set -> click 'Configure consent screen' and follow the instructions. Once finished: +3. Select 'Credentials' from the left menu, click 'Create Credentials', select + 'OAuth client ID'. +4. Now, the product name and consent screen need to be set -> click 'Configure + consent screen' and follow the instructions. Once finished: - a. Select 'Application type' to be *Web application*. - b. Enter an appropriate name. - c. Input *http://localhost:8080/* for 'Authorized redirect URIs'. - d. Click 'Create'. + a. Select 'Application type' to be *Web application*. + b. Enter an appropriate name. + c. Input *http://localhost:8080/* for 'Authorized redirect URIs'. + d. Click 'Create'. -5. Click 'Download JSON' on the right side of Client ID to download **client_secret_.json**. +5. Click 'Download JSON' on the right side of Client ID to download + **client_secret_.json**. The downloaded file has all authentication information of your application. -**Rename the file to "client_secrets.json" and place it in your working directory.** +**Rename the file to "client_secrets.json" and place it in your working +directory.** Create *quickstart.py* file and copy and paste the following code. @@ -27,9 +33,12 @@ Create *quickstart.py* file and copy and paste the following code. from pydrive2.auth import GoogleAuth gauth = GoogleAuth() - gauth.LocalWebserverAuth() # Creates local webserver and auto handles authentication. + # Creates local webserver and auto handles authentication. + gauth.LocalWebserverAuth() -Run this code with *python quickstart.py* and you will see a web browser asking you for authentication. Click *Accept* and you are done with authentication. For more details, take a look at documentation: `OAuth made easy`_ +Run this code with *python quickstart.py* and you will see a web browser asking +you for authentication. Click *Accept* and you are done with authentication. +For more details, take a look at documentation: `OAuth made easy`_ .. _`APIs Console`: https://console.developers.google.com/iam-admin/projects .. _`OAuth made easy`: /PyDrive2/oauth/ @@ -37,8 +46,9 @@ Run this code with *python quickstart.py* and you will see a web browser asking Creating and Updating Files --------------------------- -There are many methods to create and update file metadata and contents. With *PyDrive2*, all you have to know is -`Upload()`_ method which makes optimal API call for you. Add the following code to your *quickstart.py* and run it. +There are many methods to create and update file metadata and contents. With +*PyDrive2*, all you have to know is `Upload()`_ method which makes optimal API +call for you. Add the following code to your *quickstart.py* and run it. .. code-block:: python @@ -46,12 +56,15 @@ There are many methods to create and update file metadata and contents. With *Py drive = GoogleDrive(gauth) - file1 = drive.CreateFile({'title': 'Hello.txt'}) # Create GoogleDriveFile instance with title 'Hello.txt'. - file1.SetContentString('Hello World!') # Set content of the file from given string. + # Create GoogleDriveFile instance with title 'Hello.txt'. + file1 = drive.CreateFile({'title': 'Hello.txt'}) + # Set content of the file from given string. + file1.SetContentString('Hello World!') file1.Upload() -This code will create a new file with title *Hello.txt* and its content *Hello World!*. You can see and open this - file from `Google Drive`_ if you want. For more details, take a look at documentation: `File management made easy`_ +This code will create a new file with title *Hello.txt* and its content *Hello +World!*. You can see and open this file from `Google Drive`_ if you want. For +more details, take a look at documentation: `File management made easy`_ .. _`Upload()`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.Upload .. _`Google Drive`: https://drive.google.com @@ -60,86 +73,110 @@ This code will create a new file with title *Hello.txt* and its content *Hello W Listing Files ------------- -*PyDrive2* handles paginations and parses response as list of `GoogleDriveFile`_. Let's get title and id of all the files in the root folder of Google Drive. Again, add the following code to *quickstart.py* and execute it. +*PyDrive2* handles paginations and parses response as list of +`GoogleDriveFile`_. Let's get title and id of all the files in the root folder +of Google Drive. Again, add the following code to *quickstart.py* and execute +it. .. code-block:: python # Auto-iterate through all files that matches this query - file_list = drive.ListFile({'q': "'root' in parents and trashed=false"}).GetList() + file_list = drive.ListFile( + {'q': "'root' in parents and trashed=false"} + ).GetList() for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('title: %s, id: %s' % (file1['title'], file1['id'])) Creating a Folder ----------------- -GoogleDrive treats everything as a file and assigns different mimetypes for different file formats. A folder is thus - also a file with a special mimetype. The code below allows you to add a subfolder to an existing folder. +GoogleDrive treats everything as a file and assigns different mimetypes for +different file formats. A folder is thus also a file with a special mimetype. +The code below allows you to add a subfolder to an existing folder. .. code-block:: python def create_folder(parent_folder_id, subfolder_name): - newFolder = drive.CreateFile({'title': subfolder_name, "parents": [{"kind": "drive#fileLink", "id": \ - parent_folder_id}],"mimeType": "application/vnd.google-apps.folder"}) - newFolder.Upload() - return newFolder + newFolder = drive.CreateFile({ + 'title': subfolder_name, + "parents": [{"kind": "drive#fileLink", "id": parent_folder_id}], + "mimeType": "application/vnd.google-apps.folder", + }) + newFolder.Upload() + return newFolder Return File ID via File Title ----------------------------- A common task is providing the Google Drive API with a file id. -``get_id_of_title`` demonstrates a simple workflow to return the id of a file handle by searching the file titles in a -given directory. The function takes two arguments, ``title`` and ``parent_directory_id``. ``title`` is a string that -will be compared against file titles included in a directory identified by the ``parent_directory_id``. +``get_id_of_title`` demonstrates a simple workflow to return the id of a file +handle by searching the file titles in a given directory. The function takes +two arguments, ``title`` and ``parent_directory_id``. ``title`` is a string +that will be compared against file titles included in a directory identified by +the ``parent_directory_id``. .. code-block:: python - def get_id_of_title(title,parent_directory_id): - foldered_list=drive.ListFile({'q': "'"+parent_directory_id+"' in parents and trashed=false"}).GetList() - for file in foldered_list: - if(file['title']==title): - return file['id'] - return None + def get_id_of_title(title, parent_directory_id): + foldered_list = drive.ListFile({ + 'q': "'" + parent_directory_id + "' in parents and trashed=false" + }).GetList() + for file in foldered_list: + if file['title'] == title: + return file['id'] + return None Browse Folders -------------- -This returns a json output of the data in a directory with some important attributes like size, title, parent_id. + +This returns a json output of the data in a directory with some important +attributes like size, title, parent_id. .. code-block:: python - browsed=[] - def folder_browser(folder_list,parent_id): - for element in folder_list: - if type(element) is dict: - print (element['title']) - else: - print (element) - print("Enter Name of Folder You Want to Use\nEnter '/' to use current folder\nEnter ':' to create New Folder and - use that" ) - inp=input() - if inp=='/': - return parent_id - elif inp==':': - print("Enter Name of Folder You Want to Create") - inp=input() - newfolder=create_folder(parent_id,inp) - if not os.path.exists(HOME_DIRECTORY+ROOT_FOLDER_NAME+os.path.sep+USERNAME): - os.makedirs(HOME_DIRECTORY+ROOT_FOLDER_NAME+os.path.sep+USERNAME) - return newfolder['id'] - - else: - folder_selected=inp + browsed = [] + + def folder_browser(folder_list, parent_id): for element in folder_list: - if type(element) is dict: - if element["title"]==folder_selected: - struc=element["list"] - browsed.append(folder_selected) - print("Inside "+folder_selected) - return folder_browser(struc,element['id']) + if type(element) is dict: + print(element['title']) + else: + print(element) + print( + "Enter Name of Folder You Want to Use\n" + "Enter '/' to use current folder\n" + "Enter ':' to create New Folder and use that" + ) + inp = input() + if inp == '/': + return parent_id + elif inp == ':': + print("Enter Name of Folder You Want to Create") + inp = input() + newfolder = create_folder(parent_id, inp) + if not os.path.exists( + HOME_DIRECTORY + ROOT_FOLDER_NAME + os.path.sep + USERNAME + ): + os.makedirs( + HOME_DIRECTORY + ROOT_FOLDER_NAME + os.path.sep + USERNAME + ) + return newfolder['id'] + else: + folder_selected = inp + for element in folder_list: + if type(element) is dict: + if element["title"] == folder_selected: + struc = element["list"] + browsed.append(folder_selected) + print("Inside " + folder_selected) + return folder_browser(struc, element['id']) here ``folder_list`` is the list of folders that is present -You will see title and id of all the files and folders in root folder of your Google Drive. For more details, refer to the documentation: `File listing made easy`_ +You will see title and id of all the files and folders in root folder of your +Google Drive. For more details, refer to the documentation: +`File listing made easy`_ .. _`GoogleDriveFile`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile .. _`File listing made easy`: /PyDrive2/filelist/ diff --git a/pydrive2/__pyinstaller/test_hook-googleapiclient.py b/pydrive2/__pyinstaller/test_hook-googleapiclient.py index 71f16fa7..4fca8fad 100644 --- a/pydrive2/__pyinstaller/test_hook-googleapiclient.py +++ b/pydrive2/__pyinstaller/test_hook-googleapiclient.py @@ -1,5 +1,7 @@ import subprocess +import sys +import pytest from PyInstaller import __main__ as pyi_main _APP_SOURCE = """ @@ -13,6 +15,11 @@ """ +# TODO: Remove this skip once the PyInstaller fix is verified in #399. +@pytest.mark.skipif( + sys.platform == "win32" and sys.version_info[:2] == (3, 15), + reason="PyInstaller omits python3t.dll on Windows with Python 3.15", +) def test_pyi_hook_google_api_client(tmp_path): app_name = "userapp" workpath = tmp_path / "build" diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py new file mode 100644 index 00000000..60e73b84 --- /dev/null +++ b/pydrive2/_credentials.py @@ -0,0 +1,260 @@ +import errno +import json +import os +import stat +import sys +import tempfile +from contextlib import nullcontext +from hashlib import sha256 +from pathlib import Path +from threading import Lock, RLock +from uuid import uuid4 +from weakref import WeakValueDictionary + +from filelock import FileLock +from google.oauth2.credentials import Credentials + +_dictionary_locks: WeakValueDictionary[tuple[int, str], RLock] = ( + WeakValueDictionary() +) +_registry_lock = Lock() + + +class InvalidCredentialsError(IOError): + """Error trying to read or write credentials.""" + + +class CredentialStore: + def __init__(self, destination: Path | tuple[dict[str, str], str]): + self.destination = destination + self.lock: FileLock | RLock + if isinstance(destination, Path): + self.lock = FileLock(str(destination) + ".lock", mode=0o600) + else: + dictionary, key = destination + with _registry_lock: + self.lock = _dictionary_locks.setdefault( + (id(dictionary), key), RLock() + ) + + @property + def backend(self): + return "file" if isinstance(self.destination, Path) else "dictionary" + + def load(self) -> "StoredUserCredentials | None": + if isinstance(self.destination, Path): + serialized = _read_file(self.destination, self.lock) + else: + dictionary, key = self.destination + with self.lock: + serialized = dictionary.get(key) + if not serialized: + return None + + try: + info = json.loads(serialized) + if ( + not isinstance(info, dict) + or info.get("type", "authorized_user") != "authorized_user" + or "_class" in info + or "access_token" in info + or not (info.get("token") or info.get("refresh_token")) + ): + raise ValueError("Expected google-auth user credentials") + if not info.get("refresh_token"): + # The SDK requires these keys even for access-token-only JSON. + info.setdefault("refresh_token", None) + info.setdefault("client_id", None) + info.setdefault("client_secret", None) + credentials = StoredUserCredentials.from_authorized_user_info(info) + if not info.get("expiry") and not credentials.refresh_token: + credentials.expiry = None + credentials._session_id = info.get( + "_pydrive2_session" + ) or _token_identity(credentials) + except (AttributeError, TypeError, ValueError) as exc: + raise InvalidCredentialsError( + "Select a fresh user credential file or dictionary key and " + "sign in again with PyDrive2 2.x." + ) from exc + credentials.store = self + return credentials + + def save(self, credentials: Credentials): + if credentials.refresh_token: + if not credentials.client_id or not credentials.client_secret: + raise InvalidCredentialsError( + "Saving a refresh token requires a client ID and client secret" + ) + elif not credentials.token: + raise InvalidCredentialsError("No user token to save") + if isinstance(self.destination, Path): + self.destination.parent.mkdir(parents=True, exist_ok=True) + with self.lock: + serialized = credentials.to_json() + session = getattr(credentials, "_session_id", None) + if session and session != _token_identity(credentials): + # Keep the grant identifiable when its refresh token rotates. + info = json.loads(serialized) + info["_pydrive2_session"] = session + serialized = json.dumps(info) + if isinstance(self.destination, Path): + _write_file(self.destination, serialized) + else: + dictionary, key = self.destination + dictionary[key] = serialized + + +def _token_identity(credentials): + return sha256( + (credentials.refresh_token or credentials.token).encode() + ).hexdigest() + + +class StoredUserCredentials(Credentials): + store: CredentialStore | None = None + _session_id: str | None = None + _pending_refresh: "StoredUserCredentials | None" = None + + @classmethod + def from_flow(cls, credentials, store): + user = cls( + token=credentials.token, + refresh_token=credentials.refresh_token, + id_token=credentials.id_token, + token_uri=credentials.token_uri, + client_id=credentials.client_id, + client_secret=credentials.client_secret, + scopes=credentials.scopes, + expiry=credentials.expiry, + granted_scopes=credentials.granted_scopes, + ) + user.store = store + user._session_id = _token_identity(user) + return user + + def __getstate__(self): + state = super().__getstate__() + state.pop("store", None) + state.pop("_pending_refresh", None) + return state + + def use_client_config(self, config): + # google-auth discards token_uri from cache JSON. Only a matching + # application configuration may override its trusted Google endpoint. + endpoint = config.get("token_uri") + if endpoint and config.get("client_id") == self.client_id: + self._token_uri = endpoint + + def refresh(self, request): + store = self.store + if store is None: + super().refresh(request) + return + # AuthorizedHttp can refresh during a request, including after a 401. + previous_token = self.token + with store.lock: + if self.token != previous_token and self.valid: + return + cached = store.load() + if ( + cached is None + or cached._session_id != self._session_id + or cached.client_id != self.client_id + ): + # A different login or an explicit deletion owns this location. + self.store = None + super().refresh(request) + return + # If publication failed after rotation, the old refresh token may + # already be invalid. Retry the write before another exchange. + pending = self._pending_refresh + if pending is not None and cached.token == self.token: + cached = pending + else: + pending = None + if not cached.valid or ( + pending is None and cached.token == previous_token + ): + # Only the original client's trusted endpoint may receive tokens. + cached._token_uri = self.token_uri + Credentials.refresh(cached, request) + pending = self._pending_refresh = cached + if pending is not None: + store.save(pending) + self._pending_refresh = None + self.token = cached.token + self.expiry = cached.expiry + self._refresh_token = cached.refresh_token + self._rapt_token = cached.rapt_token + if cached.id_token is not None: + self._id_token = cached.id_token + if cached.granted_scopes is not None: + self._granted_scopes = cached.granted_scopes + + +def _check_path(path): + try: + info = path.lstat() + except FileNotFoundError: + return None + # Windows uses the name-surrogate bit for symlinks and junctions. + if ( + stat.S_ISLNK(info.st_mode) + or getattr(info, "st_reparse_tag", 0) & 0x20000000 + ): + raise InvalidCredentialsError( + "Credentials file cannot be symbolic link" + ) + return info + + +def _read_file(path, lock): + if _check_path(path) is None: + return None + try: + read_lock = lock.acquire() + except OSError as exc: + # A pre-provisioned cache can be mounted read-only. + if exc.errno not in (errno.EACCES, errno.EPERM, errno.EROFS): + raise + read_lock = nullcontext() + try: + with ( + read_lock, + open(path, encoding="utf-8", opener=_open_no_follow) as reader, + ): + if sys.platform == "win32": + # Open normally to hydrate cloud or compressed files. Windows + # lacks O_NOFOLLOW, so verify the opened file before reading. + info = _check_path(path) + if info is None or not os.path.samestat( + info, os.fstat(reader.fileno()) + ): + raise InvalidCredentialsError( + "Credentials file changed while opening" + ) + return reader.read() + except FileNotFoundError: + return None + + +def _open_no_follow(path, flags): + return os.open(path, flags | getattr(os, "O_NOFOLLOW", 0)) + + +def _write_file(path, serialized): + _check_path(path) + if sys.platform == "win32": + from ._windows_credentials import create_for_replace + + name = path.with_name(uuid4().hex + ".tmp") + fd = create_for_replace(path, name) + else: + fd, name = tempfile.mkstemp(dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(serialized) + os.replace(name, path) + finally: + Path(name).unlink(missing_ok=True) diff --git a/pydrive2/_windows_credentials.py b/pydrive2/_windows_credentials.py new file mode 100644 index 00000000..c2edbe4d --- /dev/null +++ b/pydrive2/_windows_credentials.py @@ -0,0 +1,42 @@ +import os +import sys + +assert sys.platform == "win32" + +import msvcrt + +import pywintypes +import win32con +import win32file +import win32security + + +def create_for_replace(path, temporary): + # Apply the DACL at creation, before another process can open the file. + if path.exists(): + descriptor = win32security.GetFileSecurity( + str(path), win32security.DACL_SECURITY_INFORMATION + ) + protected = win32security.SE_DACL_PROTECTED + descriptor.SetSecurityDescriptorControl(protected, protected) + else: + descriptor = ( + win32security.ConvertStringSecurityDescriptorToSecurityDescriptor( + "D:P(A;;FA;;;OW)", win32security.SDDL_REVISION_1 + ) + ) + security = pywintypes.SECURITY_ATTRIBUTES() + security.SECURITY_DESCRIPTOR = descriptor + security.bInheritHandle = False + handle = win32file.CreateFile( + str(temporary), + win32con.GENERIC_WRITE, + 0, + security, + win32con.CREATE_NEW, + win32con.FILE_ATTRIBUTE_NORMAL, + None, + ) + fd = msvcrt.open_osfhandle(int(handle), os.O_WRONLY) + handle.Detach() # The CRT descriptor now owns the Windows handle. + return fd diff --git a/pydrive2/auth.py b/pydrive2/auth.py index ce2b1850..d53a2f18 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -1,25 +1,29 @@ +import errno import json import os import threading import webbrowser +from contextlib import suppress from functools import wraps +from pathlib import Path from typing import ClassVar import httplib2 +from cryptography.hazmat.primitives.serialization import pkcs12 +from google.auth.crypt import RSASigner +from google.auth.exceptions import RefreshError as GoogleRefreshError +from google.oauth2 import credentials as user_credentials +from google.oauth2 import service_account +from google_auth_httplib2 import AuthorizedHttp, Request +from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build -from oauth2client import clientsecrets -from oauth2client._helpers import scopes_to_string -from oauth2client.client import ( - OOB_CALLBACK_URN, - AccessTokenRefreshError, - FlowExchangeError, - OAuth2WebServerFlow, -) -from oauth2client.contrib.dictionary_storage import DictionaryStorage -from oauth2client.file import Storage -from oauth2client.service_account import ServiceAccountCredentials -from oauth2client.tools import ClientRedirectHandler, ClientRedirectServer +from oauthlib.oauth2 import AccessDeniedError, OAuth2Error +from ._credentials import ( + CredentialStore, + InvalidCredentialsError, + StoredUserCredentials, +) from .apiattr import ApiAttribute, ApiAttributeMixin from .settings import ( InvalidConfigError, @@ -33,10 +37,6 @@ class AuthError(Exception): """Base error for authentication/authorization errors.""" -class InvalidCredentialsError(IOError): - """Error trying to read credentials file.""" - - class AuthenticationRejected(AuthError): """User rejected authentication.""" @@ -59,7 +59,14 @@ def _decorated(self, *args, **kwargs): self.auth = GoogleAuth() # Re-create access token if it expired. if self.auth.access_token_expired: - if getattr(self.auth, "auth_method", False) == "service": + if self.auth.credentials is not None and not isinstance( + self.auth.credentials, user_credentials.Credentials + ): + self.auth._refresh_if_expired() + elif ( + self.auth.credentials is None + and self.auth.auth_method == "service" + ): self.auth.ServiceAuth() else: self.auth.LocalWebserverAuth() @@ -82,7 +89,11 @@ def _decorated(self, *args, **kwargs): else: # If HTTP object not specified, create or resuse an HTTP # object from the thread local storage. - if not getattr(self.auth.thread_local, "http", None): + http = getattr(self.auth.thread_local, "http", None) + if http is None or ( + isinstance(http, AuthorizedHttp) + and http.credentials is not self.auth.credentials + ): self.auth.thread_local.http = self.auth.Get_Http_Object() self.http = self.auth.thread_local.http @@ -91,63 +102,8 @@ def _decorated(self, *args, **kwargs): return _decorated -def CheckServiceAuth(decoratee): - """Decorator to authorize service account.""" - - @wraps(decoratee) - def _decorated(self, *args, **kwargs): - self.auth_method = "service" - dirty = False - save_credentials = self.settings.get("save_credentials") - if self.credentials is None and save_credentials: - self.LoadCredentials() - if self.credentials is None: - decoratee(self, *args, **kwargs) - self.Authorize() - dirty = True - elif self.access_token_expired: - self.Refresh() - dirty = True - self.credentials.set_store(self._default_storage) - if dirty and save_credentials: - self.SaveCredentials() - - return _decorated - - -def CheckAuth(decoratee): - """Decorator to check if it requires OAuth2 flow request.""" - - @wraps(decoratee) - def _decorated(self, *args, **kwargs): - dirty = False - code = None - save_credentials = self.settings.get("save_credentials") - if self.credentials is None and save_credentials: - self.LoadCredentials() - if self.flow is None: - self.GetFlow() - if self.credentials is None: - code = decoratee(self, *args, **kwargs) - dirty = True - else: - if self.access_token_expired: - if self.credentials.refresh_token is not None: - self.Refresh() - else: - code = decoratee(self, *args, **kwargs) - dirty = True - if code is not None: - self.Auth(code) - self.credentials.set_store(self._default_storage) - if dirty and save_credentials: - self.SaveCredentials() - - return _decorated - - class GoogleAuth(ApiAttributeMixin): - """Wrapper class for oauth2client library in google-api-python-client. + """Google Drive authentication using google-auth. Loads all settings and credentials from one 'settings.yaml' file and performs common OAuth2.0 related functionality such as authentication @@ -168,11 +124,10 @@ class GoogleAuth(ApiAttributeMixin): "revoke_uri", "redirect_uri", ] - SERVICE_CONFIGS_LIST: ClassVar[list[str]] = ["client_user_email"] settings = ApiAttribute("settings") client_config = ApiAttribute("client_config") flow = ApiAttribute("flow") - credentials = ApiAttribute("credentials") + _credentials = ApiAttribute("credentials") http = ApiAttribute("http") service = ApiAttribute("service") auth_method = ApiAttribute("auth_method") @@ -190,6 +145,7 @@ def __init__( self.http_timeout = http_timeout ApiAttributeMixin.__init__(self) self.thread_local = threading.local() + self._refresh_lock = threading.RLock() self.client_config = {} if settings is None and settings_file: @@ -201,9 +157,21 @@ def __init__( self.settings = settings or self.DEFAULT_SETTINGS ValidateSettings(self.settings) - storages, default = self._InitializeStoragesFromSettings() - self._storages = storages - self._default_storage = default + self._configured_store = self._storage_from_settings() + self._login_store = self._configured_store + + @property + def credentials(self): + return self._credentials + + @credentials.setter + def credentials(self, value): + with self._refresh_lock: + if self._credentials is not value: + self.service = None + if isinstance(self.http, AuthorizedHttp): + self.http = self.http.http + self._credentials = value @property def access_token_expired(self): @@ -211,11 +179,16 @@ def access_token_expired(self): :returns: bool -- True if access token doesn't exist or is expired. """ - if self.credentials is None: - return True - return self.credentials.access_token_expired + return self.credentials is None or not self.credentials.valid + + def _use_flow_credentials(self, credentials): + if self.settings.get("save_credentials"): + credentials = StoredUserCredentials.from_flow( + credentials, self._login_store + ) + self._login_store.save(credentials) + self.credentials = credentials - @CheckAuth def LocalWebserverAuth( self, host_name="localhost", @@ -242,152 +215,153 @@ def LocalWebserverAuth( If not specified, it will listen on the address specified in the host_name parameter. :type bind_addr: str. - :returns: str -- code returned from local web server :raises: AuthenticationRejected, AuthenticationError """ + if self.credentials is None and self.settings.get("save_credentials"): + self._load_credentials(self._login_store) + if not self.access_token_expired: + return + if self.credentials is not None and ( + not isinstance(self.credentials, user_credentials.Credentials) + or self.credentials.refresh_token + or self.credentials.refresh_handler + ): + self._refresh_if_expired() + return if os.getenv("GDRIVE_NON_INTERACTIVE"): raise AuthenticationError( "Non interactive mode (GDRIVE_NON_INTERACTIVE env) is enabled" ) - + if self.flow is None: + self.GetFlow() if port_numbers is None: - port_numbers = [ - 8080, - 8090, - ] # Mutable objects should not be default - # values, as each call's changes are global. - success = False - port_number = 0 + port_numbers = (8080, 8090) + if launch_browser: + try: + webbrowser.get() + except webbrowser.Error: + launch_browser = False for port in port_numbers: - port_number = port try: - httpd = ClientRedirectServer( - (bind_addr or host_name, port), ClientRedirectHandler + credentials = self.flow.run_local_server( + host=host_name, + bind_addr=bind_addr, + port=port, + open_browser=launch_browser, + **self._authorization_params(), ) - except OSError: - pass - else: - success = True - break - if success: - oauth_callback = f"http://{host_name}:{port_number}/" - else: - print( - "Failed to start a local web server. Please check your firewall" - ) - print( - "settings and locally running programs that may be blocking or" - ) - print("using configured ports. Default ports are 8080 and 8090.") - raise AuthenticationError() - self.flow.redirect_uri = oauth_callback - authorize_url = self.GetAuthUrl() - if launch_browser: - webbrowser.open(authorize_url, new=1, autoraise=True) - print("Your browser has been opened to visit:") - else: - print("Open your browser to visit:") - print() - print(" " + authorize_url) - print() - httpd.handle_request() - if "error" in httpd.query_params: - print("Authentication request was rejected") - raise AuthenticationRejected("User rejected authentication") - if "code" in httpd.query_params: - return httpd.query_params["code"] - else: - print( - 'Failed to find "code" in the query parameters of the redirect.' - ) - print("Try command-line authentication") - raise AuthenticationError("No code found in redirect") + except OSError as exc: + if exc.errno not in (errno.EADDRINUSE, errno.EACCES): + raise + continue + except AccessDeniedError as exc: + raise AuthenticationRejected( + "User rejected authentication" + ) from exc + except (OAuth2Error, Warning) as exc: + raise AuthenticationError( + "OAuth2 authorization failed" + ) from exc + self._use_flow_credentials(credentials) + self.Authorize() + return + raise AuthenticationError( + "Could not bind to any configured OAuth callback port" + ) - @CheckAuth def CommandLineAuth(self): - """Authenticate and authorize from user by printing authentication url - retrieving authentication code from command-line. + """Print a login URL and receive the authorization on a local port.""" + return self.LocalWebserverAuth(launch_browser=False) - :returns: str -- code returned from commandline. - """ - self.flow.redirect_uri = OOB_CALLBACK_URN - authorize_url = self.GetAuthUrl() - print("Go to the following link in your browser:") - print() - print(" " + authorize_url) - print() - return input("Enter verification code: ").strip() - - @CheckServiceAuth def ServiceAuth(self): - """Authenticate and authorize using P12 private key, client id - and client email for a Service account. - :raises: AuthError, InvalidConfigError - """ - if set(self.SERVICE_CONFIGS_LIST) - set(self.client_config): + """Authenticate with a service account, optionally acting for a user.""" + self.auth_method = "service" + if not isinstance(self.credentials, service_account.Credentials): self.LoadServiceConfigSettings() - scopes = scopes_to_string(self.settings["oauth_scope"]) - keyfile_name = self.client_config.get("client_json_file_path") - keyfile_dict = self.client_config.get("client_json_dict") - keyfile_json = self.client_config.get("client_json") - - if not keyfile_dict and keyfile_json: - # Compensating for missing ServiceAccountCredentials.from_json_keyfile - keyfile_dict = json.loads(keyfile_json) - - if keyfile_dict: - self.credentials = ( - ServiceAccountCredentials.from_json_keyfile_dict( - keyfile_dict=keyfile_dict, scopes=scopes + config = self.client_config + options = { + "scopes": self.settings["oauth_scope"], + "subject": config.get("client_user_email"), + } + if "client_pkcs12_file_path" in config: + with open(config["client_pkcs12_file_path"], "rb") as stream: + key, _, _ = pkcs12.load_key_and_certificates( + stream.read(), b"notasecret" + ) + if key is None: + raise InvalidCredentialsError( + "No private key in PKCS12 file" + ) + self.credentials = service_account.Credentials( + RSASigner(key), + config["client_service_email"], + "https://oauth2.googleapis.com/token", + **options, ) - ) - elif keyfile_name: - self.credentials = ( - ServiceAccountCredentials.from_json_keyfile_name( - filename=keyfile_name, scopes=scopes + else: + if "client_json_dict" in config: + info = config["client_json_dict"] + elif "client_json" in config: + info = json.loads(config["client_json"]) + else: + with open( + config["client_json_file_path"], encoding="utf-8" + ) as stream: + info = json.load(stream) + info = { + "token_uri": "https://oauth2.googleapis.com/token", + **info, + } + self.credentials = ( + service_account.Credentials.from_service_account_info( + info, **options + ) ) - ) - else: - service_email = self.client_config["client_service_email"] - file_path = self.client_config["client_pkcs12_file_path"] - self.credentials = ServiceAccountCredentials.from_p12_keyfile( - service_account_email=service_email, - filename=file_path, - scopes=scopes, - ) - - user_email = self.client_config.get("client_user_email") - if user_email: - self.credentials = self.credentials.create_delegated( - sub=user_email - ) + self._refresh_if_expired() + if self.service is None: + self.Authorize() - def _InitializeStoragesFromSettings(self): - result = {"file": None, "dictionary": None} + def _storage_from_settings(self): backend = self.settings.get("save_credentials_backend") - save_credentials = self.settings.get("save_credentials") if backend == "file": - credentials_file = self.settings.get("save_credentials_file") - if credentials_file is None: + filename = self.settings.get("save_credentials_file") + if filename is None: raise InvalidConfigError( "Please specify credentials file to read" ) - result[backend] = Storage(credentials_file) - elif backend == "dictionary": - creds_dict = self.settings.get("save_credentials_dict") - if creds_dict is None: + return CredentialStore(Path(filename)) + if backend == "dictionary": + dictionary = self.settings.get("save_credentials_dict") + key = self.settings.get("save_credentials_key") + if dictionary is None: raise InvalidConfigError("Please specify credentials dict") - - creds_key = self.settings.get("save_credentials_key") - if creds_key is None: + if key is None: raise InvalidConfigError("Please specify credentials key") - - result[backend] = DictionaryStorage(creds_dict, creds_key) - elif save_credentials: + return CredentialStore((dictionary, key)) + if self.settings.get("save_credentials"): raise InvalidConfigError( f"Unknown save_credentials_backend: {backend}" ) - return result, result.get(backend) + return None + + def _credential_backend(self, backend): + if backend is None: + backend = self.settings.get("save_credentials_backend") + if backend is None: + raise InvalidConfigError("Please specify credential backend") + if backend not in ("file", "dictionary"): + raise InvalidConfigError("Unknown save_credentials_backend") + return backend + + def _storage_for(self, backend, filename=None): + if filename is not None: + return CredentialStore(Path(filename)) + store = self._configured_store + if store is None or store.backend != backend: + raise InvalidConfigError( + f"Credential backend {backend!r} is not configured" + ) + return store def LoadCredentials(self, backend=None): """Loads credentials or create empty credentials if it doesn't exist. @@ -396,16 +370,10 @@ def LoadCredentials(self, backend=None): :type backend: str. :raises: InvalidConfigError """ - if backend is None: - backend = self.settings.get("save_credentials_backend") - if backend is None: - raise InvalidConfigError("Please specify credential backend") - if backend == "file": + if self._credential_backend(backend) == "file": self.LoadCredentialsFile() - elif backend == "dictionary": - self._LoadCredentialsDictionary() else: - raise InvalidConfigError("Unknown save_credentials_backend") + self._load_credentials(self._storage_for("dictionary")) def LoadCredentialsFile(self, credentials_file=None): """Loads credentials or create empty credentials if it doesn't exist. @@ -416,39 +384,29 @@ def LoadCredentialsFile(self, credentials_file=None): :type credentials_file: str. :raises: InvalidConfigError, InvalidCredentialsError """ - if credentials_file is None: - self._default_storage = self._storages["file"] - if self._default_storage is None: - raise InvalidConfigError( - "Backend `file` is not configured, specify " - "credentials file to read in the settings " - "file or pass an explicit value" + self._load_credentials(self._storage_for("file", credentials_file)) + + def _load_credentials(self, store): + with self._refresh_lock: + credentials = store.load() + if credentials and credentials.refresh_token: + config = self.client_config or self.settings.get( + "client_config", {} ) - else: - self._default_storage = Storage(credentials_file) - - try: - self.credentials = self._default_storage.get() - except OSError: - raise InvalidCredentialsError( - "Credentials file cannot be symbolic link" - ) - - if self.credentials: - self.credentials.set_store(self._default_storage) - - def _LoadCredentialsDictionary(self): - self._default_storage = self._storages["dictionary"] - if self._default_storage is None: - raise InvalidConfigError( - "Backend `dictionary` is not configured, specify " - "credentials dict and key to read in the settings file" - ) - - self.credentials = self._default_storage.get() - - if self.credentials: - self.credentials.set_store(self._default_storage) + if ( + not config + and self.settings["client_config_backend"] == "file" + ): + # A cached login can refresh without a client secrets file. + path = Path(self.settings["client_config_file"]) + with suppress(OSError): + if path.is_file(): + config = self._read_client_config_file(path) + credentials.use_client_config(config) + self.credentials = credentials + self._login_store = store + # A missing user cache starts a user login, even after ServiceAuth. + self.auth_method = None def SaveCredentials(self, backend=None): """Saves credentials according to specified backend. @@ -460,16 +418,10 @@ def SaveCredentials(self, backend=None): :type backend: str. :raises: InvalidConfigError """ - if backend is None: - backend = self.settings.get("save_credentials_backend") - if backend is None: - raise InvalidConfigError("Please specify credential backend") - if backend == "file": + if self._credential_backend(backend) == "file": self.SaveCredentialsFile() - elif backend == "dictionary": - self._SaveCredentialsDictionary() else: - raise InvalidConfigError("Unknown save_credentials_backend") + self._save_credentials("dictionary") def SaveCredentialsFile(self, credentials_file=None): """Saves credentials to the file in JSON format. @@ -478,39 +430,14 @@ def SaveCredentialsFile(self, credentials_file=None): :type credentials_file: str. :raises: InvalidConfigError, InvalidCredentialsError """ - if self.credentials is None: - raise InvalidCredentialsError("No credentials to save") + self._save_credentials("file", credentials_file) - if credentials_file is None: - storage = self._storages["file"] - if storage is None: - raise InvalidConfigError( - "Backend `file` is not configured, specify " - "credentials file to read in the settings " - "file or pass an explicit value" - ) - else: - storage = Storage(credentials_file) - - try: - storage.put(self.credentials) - except OSError: + def _save_credentials(self, backend, filename=None): + if not isinstance(self.credentials, user_credentials.Credentials): raise InvalidCredentialsError( - "Credentials file cannot be symbolic link" - ) - - def _SaveCredentialsDictionary(self): - if self.credentials is None: - raise InvalidCredentialsError("No credentials to save") - - storage = self._storages["dictionary"] - if storage is None: - raise InvalidConfigError( - "Backend `dictionary` is not configured, specify " - "credentials dict and key to write in the settings file" + "Only signed-in user credentials can be saved" ) - - storage.put(self.credentials) + self._storage_for(backend, filename).save(self.credentials) def LoadClientConfig(self, backend=None): """Loads client configuration according to specified backend. @@ -548,45 +475,27 @@ def LoadClientConfigFile(self, client_config_file=None): """ if client_config_file is None: client_config_file = self.settings["client_config_file"] - try: - client_type, client_info = clientsecrets.loadfile( - client_config_file - ) - except clientsecrets.InvalidClientSecretsError as error: - raise InvalidConfigError(f"Invalid client secrets file {error}") - if client_type not in ( - clientsecrets.TYPE_WEB, - clientsecrets.TYPE_INSTALLED, - ): - raise InvalidConfigError( - "Unknown client_type of client config file" - ) - - # General settings. - try: - config_index = [ - "client_id", - "client_secret", - "auth_uri", - "token_uri", - ] - for config in config_index: - self.client_config[config] = client_info[config] - - self.client_config["revoke_uri"] = client_info.get("revoke_uri") - self.client_config["redirect_uri"] = client_info["redirect_uris"][ - 0 - ] - except KeyError: - raise InvalidConfigError("Insufficient client config in file") + self.client_config = self._read_client_config_file(client_config_file) - # Service auth related fields. - service_auth_config = ["client_email"] + @staticmethod + def _read_client_config_file(client_config_file): try: - for config in service_auth_config: - self.client_config[config] = client_info[config] - except KeyError: - pass # The service auth fields are not present, handling code can go here. + with open(client_config_file, encoding="utf-8") as stream: + config = json.load(stream) + (client_type,) = config + if client_type not in ("web", "installed"): + raise ValueError("Expected web or installed client") + info = config[client_type] + return { + "client_id": info["client_id"], + "client_secret": info["client_secret"], + "auth_uri": info["auth_uri"], + "token_uri": info["token_uri"], + "redirect_uri": info["redirect_uris"][0], + "revoke_uri": info.get("revoke_uri"), + } + except (OSError, KeyError, IndexError, TypeError, ValueError) as exc: + raise InvalidConfigError("Invalid client secrets file") from exc def LoadServiceConfigSettings(self): """Loads client configuration from settings. @@ -609,18 +518,18 @@ def LoadServiceConfigSettings(self): f"One of {configs} is required for service authentication" ) + service_config = self.settings["service_config"] + self.client_config["client_user_email"] = service_config.get( + "client_user_email" + ) if config == "client_pkcs12_file_path": - self.SERVICE_CONFIGS_LIST.append("client_service_email") - - for config in self.SERVICE_CONFIGS_LIST: - try: - self.client_config[config] = self.settings["service_config"][ - config - ] - except KeyError: - err = "Insufficient service config in settings" - err += f"\n\nMissing: {config} key." - raise InvalidConfigError(err) + if not service_config.get("client_service_email"): + raise InvalidConfigError( + "Missing client_service_email for PKCS12 key" + ) + self.client_config["client_service_email"] = service_config[ + "client_service_email" + ] def LoadClientConfigSettings(self): """Loads client configuration from settings file. @@ -646,46 +555,62 @@ def GetFlow(self): config in self.client_config for config in self.CLIENT_CONFIGS_LIST ): self.LoadClientConfig() - constructor_kwargs = { - "redirect_uri": self.client_config["redirect_uri"], - "auth_uri": self.client_config["auth_uri"], - "token_uri": self.client_config["token_uri"], - "access_type": "online", - } - if self.client_config["revoke_uri"] is not None: - constructor_kwargs["revoke_uri"] = self.client_config["revoke_uri"] - self.flow = OAuth2WebServerFlow( - self.client_config["client_id"], - self.client_config["client_secret"], - scopes_to_string(self.settings["oauth_scope"]), - **constructor_kwargs, + redirect_uri = self.client_config["redirect_uri"] + if redirect_uri == "urn:ietf:wg:oauth:2.0:oob": + redirect_uri = "http://localhost" + self.flow = InstalledAppFlow.from_client_config( + {"installed": self.client_config}, + scopes=self.settings["oauth_scope"], + redirect_uri=redirect_uri, + autogenerate_code_verifier=True, ) + + def _authorization_params(self): if self.settings.get("get_refresh_token"): - self.flow.params.update( - {"access_type": "offline", "approval_prompt": "force"} - ) + return {"access_type": "offline", "prompt": "consent"} + return {"access_type": "online"} + + def _refresh_if_expired(self): + with self._refresh_lock: + if self.access_token_expired: + self.Refresh() def Refresh(self): """Refreshes the access_token. :raises: RefreshError """ - if self.credentials is None: - raise RefreshError("No credential to refresh.") - if ( - self.credentials.refresh_token is None - and self.auth_method != "service" - ): - raise RefreshError( - "No refresh_token found." - "Please set access_type of OAuth to offline." + with self._refresh_lock: + if self.credentials is None: + raise RefreshError("No credential to refresh.") + user = isinstance(self.credentials, user_credentials.Credentials) + if user and not ( + self.credentials.refresh_token + or self.credentials.refresh_handler + ): + raise RefreshError( + "No refresh_token found. " + "Please set access_type of OAuth to offline." + ) + if self.http is None: + self.http = self._build_http() + http = ( + self.http.http + if isinstance(self.http, AuthorizedHttp) + else self.http ) - if self.http is None: - self.http = self._build_http() - try: - self.credentials.refresh(self.http) - except AccessTokenRefreshError as error: - raise RefreshError(f"Access token refresh failed: {error}") + try: + self.credentials.refresh(Request(http)) + except GoogleRefreshError as exc: + raise RefreshError( + f"Access token refresh failed: {exc}" + ) from exc + if ( + user + and self.settings.get("save_credentials") + and not isinstance(self.credentials, StoredUserCredentials) + ): + self._login_store.save(self.credentials) def GetAuthUrl(self): """Creates authentication url where user visits to grant access. @@ -694,7 +619,8 @@ def GetAuthUrl(self): """ if self.flow is None: self.GetFlow() - return self.flow.step1_get_authorize_url() + url, _ = self.flow.authorization_url(**self._authorization_params()) + return url def Auth(self, code): """Authenticate, authorize, and build service. @@ -716,9 +642,12 @@ def Authenticate(self, code): if self.flow is None: self.GetFlow() try: - self.credentials = self.flow.step2_exchange(code) - except FlowExchangeError as e: - raise AuthenticationError(f"OAuth2 code exchange failed: {e}") + self.flow.fetch_token(code=code) + self._use_flow_credentials(self.flow.credentials) + except (OAuth2Error, Warning) as exc: + raise AuthenticationError( + f"OAuth2 code exchange failed: {exc}" + ) from exc print("Authentication successful.") def _build_http(self): @@ -741,24 +670,24 @@ def Authorize(self): :raises: AuthenticationError """ - if self.access_token_expired: - raise AuthenticationError( - "No valid credentials provided to authorize" + with self._refresh_lock: + if self.access_token_expired: + raise AuthenticationError( + "No valid credentials provided to authorize" + ) + if self.http is None: + self.http = self._build_http() + if isinstance(self.http, AuthorizedHttp): + self.http = self.http.http + self.http = AuthorizedHttp(self.credentials, http=self.http) + self.service = build( + "drive", "v2", http=self.http, cache_discovery=False ) - if self.http is None: - self.http = self._build_http() - self.http = self.credentials.authorize(self.http) - self.service = build( - "drive", "v2", http=self.http, cache_discovery=False - ) - def Get_Http_Object(self): """Create and authorize an httplib2.Http object. Necessary for thread-safety. :return: The http object to be used in each call. :rtype: httplib2.Http """ - http = self._build_http() - http = self.credentials.authorize(http) - return http + return AuthorizedHttp(self.credentials, http=self._build_http()) diff --git a/pydrive2/files.py b/pydrive2/files.py index a611719f..0dddceb3 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -908,7 +908,7 @@ def _DeletePermission(self, permission_id): try: self.auth.service.permissions().delete( fileId=file_id, permissionId=permission_id - ).execute() + ).execute(http=self.http) except errors.HttpError as error: raise ApiRequestError(error) else: diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 6a2b062f..2cee435a 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -14,7 +14,7 @@ from funcy import cached_property, retry, wrap_prop, wrap_with from tqdm.utils import CallbackIOWrapper -from pydrive2.auth import GoogleAuth +from pydrive2.auth import GoogleAuth, InvalidCredentialsError from pydrive2.drive import GoogleDrive from pydrive2.fs.utils import IterStream @@ -74,6 +74,8 @@ def should_retry(exc): def _wrap_errors(): try: yield + except InvalidCredentialsError as exc: + raise GDriveAuthError(str(exc)) from exc except Exception as exc: # Handle AuthenticationError, RefreshError and other auth failures # It's hard to come up with a narrow exception, since PyDrive throws @@ -100,6 +102,7 @@ def _client_auth( if not creds_file: cache_dir = os.path.join( appdirs.user_cache_dir("pydrive2fs", appauthor=False), + "v2", client_id, ) os.makedirs(cache_dir, exist_ok=True) diff --git a/pydrive2/settings.py b/pydrive2/settings.py index 1b4dc112..f986807c 100644 --- a/pydrive2/settings.py +++ b/pydrive2/settings.py @@ -57,7 +57,7 @@ "redirect_uri": { "type": str, "required": True, - "default": "urn:ietf:wg:oauth:2.0:oob", + "default": "http://localhost", }, "revoke_uri": {"type": str, "required": True, "default": None}, }, diff --git a/pydrive2/test/README.rst b/pydrive2/test/README.rst index 4fa2ef13..8dd7be52 100644 --- a/pydrive2/test/README.rst +++ b/pydrive2/test/README.rst @@ -13,66 +13,67 @@ those tests locally, configure a service account as described below. 1. Copy settings files to the :code:`pydrive2/test/settings/local` directory: -:: + :: - cd pydrive2/test/settings && cp *.yaml local + cd pydrive2/test/settings && cp *.yaml local 2. Setup a Google service account for your Google Cloud Project: - - Sign into the `Google API Console - `_ - - Select or `Create a new - `_ - project. - - `Enable the Drive API - `_ from the **APIs & - Services** **Dashboard** (left sidebar), click on **+ ENABLE APIS AND - SERVICES**. Find and select the "Google Drive API" in the API Library, and - click on the **ENABLE** button. - - Go back to **IAM & Admin** in the left - sidebar, and select **Service Accounts**. Click **+ CREATE SERVICE - ACCOUNT**, on the next screen, enter **Service account name** e.g. "PyDrive - tests", and click **Create**. Select **Continue** at the next **Service - account permissions** page, click at **+ CREATE KEY**, select **JSON** and - **Create**. Save generated :code:`.json` key file at your local disk. - - Copy downloaded :code:`json` file to :code:`/tmp/pydrive2/credentials.json` - directory. - -3. Optional. If you would like to use your own an OAuth client ID follow the steps: - - Under `Google API Console `_ select - **APIs & Services** from the left sidebar, and select **OAuth consent screen**. - Chose a **User Type** and click **CREATE**. On the next screen, enter an - **Application name** e.g. "PyDrive tests", and click the **Save** (scroll to - bottom). - - From the left sidebar, select **Credentials**, and click the - **Create credentials** dropdown to select **OAuth client ID**. Chose **Other** - and click **Create** to proceed with a default client name. At **Credentials** - screen find a list of your **OAuth 2.0 Client IDs**, click download icon in - front of your OAuth client id created previously. You should be prompted to - download :code:`client_secret_xxx_.json` file. - - Copy downloaded :code:`.json` file into :code:`pydrive2/test` directory - and rename to :code:`client_secrets.json`. - - Replace {{ }} sections - in :code:`pydrive2/test/settings/local/test_oauth_test_02.yaml` with the - relevant values of :code:`client_id` and :code:`client_secret` from your - **client_secrets.json** file. -4. Setup virtual environment (recommended optional step): + - Sign into the `Google API Console`_ + - Select or `Create a new`_ project. + - `Enable the Drive API`_ from the **APIs & Services** **Dashboard** (left + sidebar), click on **+ ENABLE APIS AND SERVICES**. Find and select the + "Google Drive API" in the API Library, and click on the **ENABLE** button. + - Go back to **IAM & Admin** in the left sidebar, and select + **Service Accounts**. Click **+ CREATE SERVICE ACCOUNT**, on the next + screen, enter **Service account name** e.g. "PyDrive tests", and click + **Create**. Select **Continue** at the next + **Service account permissions** page, click at **+ CREATE KEY**, select + **JSON** and **Create**. Save generated :code:`.json` key file at your + local disk. + - Copy downloaded :code:`json` file to + :code:`/tmp/pydrive2/credentials.json` directory. + +3. Optional. If you would like to use your own an OAuth client ID follow the + steps: + + - Under `Google API Console`_ select **APIs & Services** from the left + sidebar, and select **OAuth consent screen**. Chose a **User Type** and + click **CREATE**. On the next screen, enter an **Application name** e.g. + "PyDrive tests", and click the **Save** (scroll to bottom). + - From the left sidebar, select **Credentials**, and click the + **Create credentials** dropdown to select **OAuth client ID**. Chose + **Other** and click **Create** to proceed with a default client name. + At **Credentials** screen find a list of your **OAuth 2.0 Client IDs**, + click download icon in front of your OAuth client id created previously. + You should be prompted to download :code:`client_secret_xxx_.json` file. + - Copy downloaded :code:`.json` file into :code:`pydrive2/test` directory + and rename to :code:`client_secrets.json`. + - Replace {{ }} sections in + :code:`pydrive2/test/settings/local/test_oauth_test_02.yaml` with the + relevant values of :code:`client_id` and :code:`client_secret` from your + **client_secrets.json** file. -:: +4. Setup virtual environment (recommended optional step): + :: - virtualenv -p python .env - source .env/bin/activate + virtualenv -p python .env + source .env/bin/activate 5. Install :code:`tests` deps from the root directory of the project: -:: + :: - pip install -e .[tests,fsspec] + pip install -e .[tests,fsspec] +6. Run tests: -5. Run tests: + :: -:: + py.test -v -s - py.test -v -s +.. _Google API Console: https://console.developers.google.com +.. _Create a new: + https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project +.. _Enable the Drive API: https://developers.google.com/drive/api/v2/about-sdk diff --git a/pydrive2/test/settings/test_oauth_test_06.yaml b/pydrive2/test/settings/test_oauth_test_06.yaml deleted file mode 100644 index 8ecb81be..00000000 --- a/pydrive2/test/settings/test_oauth_test_06.yaml +++ /dev/null @@ -1,11 +0,0 @@ -client_config_backend: service -service_config: - client_service_email: your-service-account-email - client_pkcs12_file_path: your-file-path.p12 - -save_credentials: True -save_credentials_backend: file -save_credentials_file: credentials/6.dat - -oauth_scope: - - https://www.googleapis.com/auth/drive diff --git a/pydrive2/test/settings/test_oauth_test_07.yaml b/pydrive2/test/settings/test_oauth_test_07.yaml deleted file mode 100644 index a8065155..00000000 --- a/pydrive2/test/settings/test_oauth_test_07.yaml +++ /dev/null @@ -1,10 +0,0 @@ -client_config_backend: service -service_config: - client_json_file_path: /tmp/pydrive2/credentials.json - -save_credentials: True -save_credentials_backend: file -save_credentials_file: credentials/7.dat - -oauth_scope: - - https://www.googleapis.com/auth/drive diff --git a/pydrive2/test/settings/test_oauth_test_09.yaml b/pydrive2/test/settings/test_oauth_test_09.yaml deleted file mode 100644 index 09eca824..00000000 --- a/pydrive2/test/settings/test_oauth_test_09.yaml +++ /dev/null @@ -1,10 +0,0 @@ -client_config_backend: service -service_config: - client_json_file_path: /tmp/pydrive2/credentials.json - -save_credentials: True -save_credentials_backend: file -save_credentials_file: credentials/9.dat - -oauth_scope: - - https://www.googleapis.com/auth/drive diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py new file mode 100644 index 00000000..139fac9c --- /dev/null +++ b/pydrive2/test/test_auth_flows.py @@ -0,0 +1,912 @@ +import errno +import json +import os +import pickle +import subprocess +import sys +import webbrowser +from base64 import urlsafe_b64encode +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from datetime import datetime +from hashlib import sha256 +from http.client import HTTPConnection +from threading import Barrier, Event +from urllib.parse import parse_qs, urlencode, urlparse + +import httplib2 +import pytest +import requests +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.serialization import pkcs12 +from filelock import Timeout +from google.auth import jwt +from google.oauth2.credentials import Credentials +from google_auth_httplib2 import AuthorizedHttp, Request + +from pydrive2._credentials import CredentialStore +from pydrive2.auth import ( + AuthenticationError, + GoogleAuth, + InvalidCredentialsError, +) +from pydrive2.drive import GoogleDrive +from pydrive2.fs.spec import GDriveAuthError, _client_auth +from pydrive2.settings import InvalidConfigError + +TOKEN_URI = "https://oauth2.googleapis.com/token" +EMAIL = "service@example.test" + + +@pytest.fixture(scope="module") +def key(): + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + +@pytest.fixture +def service_info(key): + return { + "type": "service_account", + "client_email": EMAIL, + "private_key_id": "test-key", + "token_uri": TOKEN_URI, + "private_key": key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode(), + } + + +@pytest.fixture +def token_endpoint(monkeypatch): + calls = [] + + def request(http, uri, method="GET", body=None, **kwargs): + assert (uri, method) == (TOKEN_URI, "POST") + calls.append( + parse_qs(body.decode() if isinstance(body, bytes) else body) + ) + return ( + httplib2.Response({"status": "200"}), + json.dumps( + { + "access_token": "fresh", + "expires_in": 3600, + "token_type": "Bearer", + } + ).encode(), + ) + + monkeypatch.setattr(httplib2.Http, "request", request) + return calls + + +@pytest.mark.parametrize("backend", ["file", "dictionary"]) +def test_automatic_http_refresh_is_saved(backend, monkeypatch, tmp_path): + settings = { + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": backend, + } + if backend == "file": + settings["save_credentials_file"] = str(tmp_path / "user.json") + else: + settings.update(save_credentials_dict={}, save_credentials_key="user") + auth = GoogleAuth(settings=settings) + auth.credentials = Credentials( + "old", + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + expiry=datetime(2099, 1, 1), + ) + auth.SaveCredentials() + auth.LoadCredentials() + export = tmp_path / "export.json" + auth.SaveCredentialsFile(export) + http = auth.Get_Http_Object() + access_tokens = [] + + def request(uri, method="GET", headers=None, **kwargs): + if uri == TOKEN_URI: + return ( + httplib2.Response({"status": "200"}), + json.dumps( + { + "access_token": "new", + "refresh_token": "rotated", + "expires_in": 3600, + } + ).encode(), + ) + access_tokens.append(headers["authorization"]) + status = "200" if headers["authorization"] == "Bearer new" else "401" + return httplib2.Response({"status": status}), b"{}" + + monkeypatch.setattr(http.http, "request", request) + assert http.request("https://example.test/drive")[0].status == 200 + assert access_tokens == ["Bearer old", "Bearer new"] + reopened = GoogleAuth(settings=settings) + reopened.LoadCredentials() + assert reopened.credentials.refresh_token == "rotated" + assert json.loads(export.read_text())["refresh_token"] == "refresh" + + +@pytest.mark.parametrize("save_credentials", [False, True]) +def test_login_respects_save_credentials( + save_credentials, tmp_path, mocker, token_endpoint +): + path = tmp_path / "user.json" + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": save_credentials, + "save_credentials_backend": "file", + "save_credentials_file": str(path), + } + ) + auth.LoadCredentialsFile(path) + auth.flow = mocker.Mock() + auth.flow.run_local_server.return_value = Credentials( + "initial", + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + auth.LocalWebserverAuth(launch_browser=False) + assert path.exists() == save_credentials + # The transport can refresh the login before a subsequent LoadCredentials. + http = auth.Get_Http_Object() + http.credentials.refresh(Request(http.http)) + assert len(token_endpoint) == 1 + if save_credentials: + assert json.loads(path.read_text())["token"] == "fresh" + else: + assert not path.exists() + + auth.SaveCredentials() + assert json.loads(path.read_text())["token"] == "fresh" + + +def test_refresh_only_credentials_round_trip(tmp_path, token_endpoint): + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + path = tmp_path / "user.json" + auth.SaveCredentialsFile(path) + + other = GoogleAuth(settings_file=None) + other.LoadCredentialsFile(path) + other.LocalWebserverAuth() + assert other.credentials.valid + assert json.loads(path.read_text())["token"] == "fresh" + assert len(token_endpoint) == 1 + + +def test_refresh_retries_failed_cache_write(tmp_path, mocker, token_endpoint): + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + replace = mocker.patch("os.replace", side_effect=OSError("replace failed")) + with pytest.raises(OSError, match="replace failed"): + auth.Refresh() + mocker.stop(replace) + auth.Refresh() + assert len(token_endpoint) == 1 + assert json.loads(path.read_text())["token"] == "fresh" + + +def test_concurrent_login_refreshes_and_saves_once(mocker, token_endpoint): + saved = {} + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": "dictionary", + "save_credentials_dict": saved, + "save_credentials_key": "user", + } + ) + auth.credentials = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + refresh_if_expired = auth._refresh_if_expired + workers = Barrier(2) + + def refresh_together(): + workers.wait(timeout=10) + refresh_if_expired() + + mocker.patch.object( + auth, "_refresh_if_expired", side_effect=refresh_together + ) + with ThreadPoolExecutor(max_workers=2) as pool: + calls = [pool.submit(auth.LocalWebserverAuth) for _ in range(2)] + for call in calls: + call.result() + + assert len(token_endpoint) == 1 + assert json.loads(saved["user"])["token"] == "fresh" + + +@pytest.mark.parametrize("local_server", [False, True]) +def test_scope_change_raises_authentication_error(mocker, local_server): + auth = GoogleAuth(settings_file=None) + auth.flow = mocker.Mock() + error = Warning("Scope has changed") + if local_server: + auth.flow.run_local_server.side_effect = error + else: + auth.flow.fetch_token.side_effect = error + with pytest.raises(AuthenticationError) as raised: + if local_server: + auth.LocalWebserverAuth(launch_browser=False) + else: + auth.Authenticate("code") + assert raised.value.__cause__ is error + + +@pytest.mark.parametrize( + "source", + [ + "client_json", + "client_json_dict", + "client_json_file_path", + "client_pkcs12_file_path", + ], +) +def test_service_account_signing( + source, service_info, key, tmp_path, token_endpoint +): + config = {"client_user_email": "delegate@example.test"} + if source == "client_json_dict": + config[source] = service_info + elif source == "client_json": + config[source] = json.dumps(service_info) + else: + keyfile = tmp_path / "key" + if source == "client_pkcs12_file_path": + keyfile.write_bytes( + pkcs12.serialize_key_and_certificates( + b"key", + key, + None, + None, + serialization.BestAvailableEncryption(b"notasecret"), + ) + ) + config["client_service_email"] = EMAIL + else: + keyfile.write_text(json.dumps(service_info)) + config[source] = str(keyfile) + settings = { + "client_config_backend": "service", + "service_config": config, + "oauth_scope": ["scope-a", "scope-b"], + "save_credentials": True, + "save_credentials_backend": "dictionary", + "save_credentials_dict": {}, + "save_credentials_key": "account", + } + auth = GoogleAuth(settings=settings) + auth.ServiceAuth() + assertion = token_endpoint[0]["assertion"][0] + claims = jwt.decode(assertion, verify=False) + assert (claims["iss"], claims["sub"], claims["scope"]) == ( + EMAIL, + "delegate@example.test", + "scope-a scope-b", + ) + assert auth.credentials.valid + assert isinstance(auth.http, AuthorizedHttp) + assert auth.http.http is not auth.Get_Http_Object().http + assert 308 not in auth.http.http.redirect_codes + assert not settings["save_credentials_dict"] + with pytest.raises(InvalidCredentialsError): + auth.SaveCredentials() + + +def test_login_options_and_port_fallback(mocker): + auth = GoogleAuth( + settings={ + "client_config_backend": "settings", + "get_refresh_token": True, + "client_config": {"client_id": "id", "client_secret": "secret"}, + } + ) + query = parse_qs(urlparse(auth.GetAuthUrl()).query) + assert query["access_type"] == ["offline"] + assert query["prompt"] == ["consent"] + assert query["code_challenge_method"] == ["S256"] + assert query["state"] + mocker.patch("webbrowser.get", side_effect=webbrowser.Error("No browser")) + login = mocker.patch.object( + auth.flow, + "run_local_server", + side_effect=[ + OSError(errno.EADDRINUSE, "busy"), + Credentials("token"), + ], + ) + auth.LocalWebserverAuth( + port_numbers=[8080, 8090], + host_name="localhost", + bind_addr="127.0.0.1", + ) + assert [call.kwargs["port"] for call in login.call_args_list] == [ + 8080, + 8090, + ] + assert login.call_args.kwargs["open_browser"] is False + assert login.call_args.kwargs["bind_addr"] == "127.0.0.1" + assert auth.service is not None + + +def test_custom_auth_and_online_only_cache(tmp_path, mocker): + path = tmp_path / "online" + auth = GoogleAuth( + settings={ + "client_config_backend": "settings", + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(path), + "client_config": { + "client_id": "id", + "client_secret": "secret", + "redirect_uri": "https://example.test/callback", + }, + } + ) + query = parse_qs(urlparse(auth.GetAuthUrl()).query) + assert query["access_type"] == ["online"] + assert query["redirect_uri"] == ["https://example.test/callback"] + mocker.patch.object(auth.flow, "fetch_token") + auth.flow.oauth2session.token = { + "access_token": "online", + "expires_at": 4102444800, + } + auth.Auth("code") + auth.flow.fetch_token.assert_called_once_with(code="code") + other = GoogleAuth(settings_file=None) + other.LoadCredentialsFile(path) + assert other.credentials.valid + assert other.credentials.refresh_token is None + + +def test_loopback_code_exchange(tmp_path, monkeypatch, mocker): + auth = GoogleAuth( + settings={ + "client_config_backend": "settings", + "client_config": { + "client_id": "client", + "client_secret": "secret", + }, + "get_refresh_token": True, + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(tmp_path / "user.json"), + } + ) + challenge = None + callback = None + + def open_browser(url, **kwargs): + nonlocal challenge, callback + query = parse_qs(urlparse(url).query) + assert query["code_challenge_method"] == ["S256"] + challenge = query["code_challenge"][0] + redirect = urlparse(query["redirect_uri"][0]) + callback = HTTPConnection(redirect.hostname, redirect.port, timeout=5) + params = urlencode({"state": query["state"][0], "code": "approved"}) + callback.request("GET", f"{redirect.path}?{params}") + return True + + def token_response(session, request, **kwargs): + form = parse_qs(request.body) + assert form["code"] == ["approved"] + digest = sha256(form["code_verifier"][0].encode()).digest() + assert urlsafe_b64encode(digest).rstrip(b"=").decode() == challenge + response = requests.Response() + response.request = request + response.status_code = 200 + response._content = json.dumps( + { + "access_token": "access", + "refresh_token": "refresh", + "expires_in": 3600, + "token_type": "Bearer", + } + ).encode() + return response + + monkeypatch.setattr( + webbrowser, "get", lambda *args: mocker.Mock(open=open_browser) + ) + monkeypatch.setattr(requests.Session, "send", token_response) + try: + auth.LocalWebserverAuth(host_name="127.0.0.1", port_numbers=[0]) + assert callback.getresponse().status == 200 + finally: + if callback is not None: + callback.close() + auth.LoadCredentials() + assert auth.credentials.valid + assert auth.credentials.refresh_token == "refresh" + + +def test_failed_file_save_preserves_previous_credentials(tmp_path, mocker): + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + "original", client_id="id", client_secret="secret" + ) + auth.SaveCredentialsFile(path) + original_files = set(tmp_path.iterdir()) + auth.credentials = Credentials("updated") + mocker.patch("os.replace", side_effect=OSError("replace failed")) + with pytest.raises(OSError, match="replace failed"): + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + assert auth.credentials.token == "original" + assert set(tmp_path.iterdir()) == original_files + + +def test_credentials_file_rejects_symlinks(tmp_path): + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials("test") + link = tmp_path / "link" + target = tmp_path / "target" + target.write_text("original") + try: + link.symlink_to(target) + except OSError: + pytest.skip("Symlinks unavailable") + with pytest.raises(InvalidCredentialsError): + auth.SaveCredentialsFile(link) + with pytest.raises(InvalidCredentialsError): + auth.LoadCredentialsFile(link) + assert target.read_text() == "original" + + +def test_save_waits_for_open_credential_reader(tmp_path, monkeypatch): + path = tmp_path / "user.json" + writer = CredentialStore(path) + writer.lock.timeout = 0 + writer.save(Credentials("original")) + reader = GoogleAuth(settings_file=None) + opened, finish = Event(), Event() + + @contextmanager + def hold_reader(*args, **kwargs): + with open(*args, **kwargs) as stream: + opened.set() + assert finish.wait(10) + yield stream + + monkeypatch.setattr( + "pydrive2._credentials.open", hold_reader, raising=False + ) + with ThreadPoolExecutor(max_workers=1) as pool: + loading = pool.submit(reader.LoadCredentialsFile, path) + try: + assert opened.wait(10) + with pytest.raises(Timeout): + writer.save(Credentials("updated")) + finally: + finish.set() + loading.result(timeout=10) + assert reader.credentials.token == "original" + writer.save(Credentials("updated")) + assert writer.load().token == "updated" + + +def test_read_only_and_missing_cache(tmp_path, mocker): + path = tmp_path / "user.json" + store = CredentialStore(path) + acquire = mocker.patch.object( + store.lock, "acquire", side_effect=OSError(errno.EROFS, "read only") + ) + assert store.load() is None + acquire.assert_not_called() + path.write_text('{"token": "provisioned"}') + assert store.load().token == "provisioned" + assert list(tmp_path.iterdir()) == [path] + + +@pytest.mark.parametrize("source", ["file", "assignment"]) +def test_switching_credentials_rebuilds_transports(source, tmp_path, mocker): + auth = GoogleAuth(settings_file=None) + raw_http = httplib2.Http() + auth.http = raw_http + first = Credentials("first") + auth.credentials = first + drive = GoogleDrive(auth) + execute = mocker.patch( + "googleapiclient.http.HttpRequest.execute", + autospec=True, + return_value={}, + ) + drive.GetAbout() + old_service = auth.service + old_thread_http = drive.http + auth.credentials = first + assert auth.service is old_service + + second = Credentials("second") + if source == "file": + path = tmp_path / "second.json" + path.write_text(second.to_json()) + auth.LoadCredentialsFile(path) + else: + auth.credentials = second + assert auth.service is None + file = drive.CreateFile({"id": "test-file"}) + file.DeletePermission("test-permission") + request = execute.call_args.args[0] + assert auth.service is not old_service + assert request.http is auth.http + assert request.http.http is raw_http + assert request.http.credentials is auth.credentials + assert execute.call_args.kwargs["http"] is file.http + assert file.http is not old_thread_http + assert file.http.credentials is auth.credentials + assert auth.credentials.token == "second" + + +def test_expired_user_credentials_after_service_login( + service_info, token_endpoint, mocker +): + auth = GoogleAuth( + settings={ + "client_config_backend": "service", + "service_config": {"client_json_dict": service_info}, + } + ) + auth.ServiceAuth() + auth.credentials = Credentials( + None, + refresh_token="user-refresh", + token_uri=TOKEN_URI, + client_id="id", + client_secret="secret", + ) + mocker.patch("googleapiclient.http.HttpRequest.execute", return_value={}) + GoogleDrive(auth).GetAbout() + assert token_endpoint[-1]["refresh_token"] == ["user-refresh"] + assert auth.http.credentials is auth.credentials + + +def test_reauth_uses_explicit_profile(tmp_path, mocker): + default = tmp_path / "default.json" + profile = tmp_path / "profile.json" + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(default), + } + ) + auth.credentials = Credentials("default-user") + auth.SaveCredentials() + auth.credentials = Credentials("expired", expiry=datetime(2001, 1, 1)) + auth.SaveCredentialsFile(profile) + auth.LoadCredentialsFile(profile) + auth.flow = mocker.Mock() + auth.flow.run_local_server.return_value = Credentials("new-profile-user") + auth.LocalWebserverAuth() + assert json.loads(profile.read_text())["token"] == "new-profile-user" + assert json.loads(default.read_text())["token"] == "default-user" + + +@pytest.mark.parametrize("backend", ["file", "dictionary"]) +def test_shared_cache_refresh_rotation(backend, tmp_path, monkeypatch): + settings = { + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": backend, + } + if backend == "file": + settings["save_credentials_file"] = str(tmp_path / "user.json") + else: + settings.update(save_credentials_dict={}, save_credentials_key="user") + first, second = (GoogleAuth(settings=settings) for _ in range(2)) + first.credentials = Credentials( + None, + refresh_token="original", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + first.SaveCredentials() + first.LoadCredentials() + second.LoadCredentials() + arrivals = Barrier(2) + token_requests = [] + + def request(http, uri, method, body=None, **kwargs): + assert uri == TOKEN_URI + assert parse_qs(body.decode())["refresh_token"] == ["original"] + token_requests.append(uri) + return httplib2.Response({"status": "200"}), json.dumps( + { + "access_token": "fresh", + "refresh_token": "rotated", + "expires_in": 3600, + } + ).encode() + + def refresh(auth): + arrivals.wait(timeout=10) + auth.Refresh() + + monkeypatch.setattr(httplib2.Http, "request", request) + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(refresh, [first, second])) + assert len(token_requests) == 1 + for auth in (first, second): + assert (auth.credentials.token, auth.credentials.refresh_token) == ( + "fresh", + "rotated", + ) + assert auth.credentials.valid + detached = pickle.loads(pickle.dumps(second.credentials)) + assert detached.refresh_token == "rotated" + assert detached.store is None + + +def test_refresh_does_not_overwrite_a_different_login( + tmp_path, token_endpoint +): + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_token="first", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + replacement = GoogleAuth(settings_file=None) + replacement.credentials = Credentials("second-user") + replacement.SaveCredentialsFile(path) + + auth.Refresh() + assert len(token_endpoint) == 1 + assert auth.credentials.token == "fresh" + replacement.LoadCredentialsFile(path) + assert replacement.credentials.token == "second-user" + + +@pytest.mark.parametrize("backend", ["settings", "file"]) +def test_token_endpoint_comes_from_client_configuration( + backend, tmp_path, monkeypatch +): + path = tmp_path / "user.json" + endpoint = "https://proxy.example.test/oauth/token" + config = { + "client_id": "id", + "client_secret": "secret", + "token_uri": endpoint, + } + settings = {"client_config_backend": backend} + if backend == "settings": + settings["client_config"] = config + else: + client_file = tmp_path / "client_secrets.json" + client_file.write_text( + json.dumps( + { + "web": { + **config, + "auth_uri": "https://accounts.example.test/auth", + "redirect_uris": ["http://localhost"], + } + } + ) + ) + settings["client_config_file"] = str(client_file) + auth = GoogleAuth(settings=settings) + auth.credentials = Credentials( + None, + refresh_token="refresh", + token_uri="https://cache-only.example.test/token", + client_id="id", + client_secret="secret", + ) + auth.SaveCredentialsFile(path) + other = GoogleAuth(settings=settings) + other.LoadCredentialsFile(path) + calls = [] + + def request(http, uri, **kwargs): + calls.append(uri) + return httplib2.Response( + {"status": "200"} + ), b'{"access_token":"fresh","expires_in":3600}' + + monkeypatch.setattr(httplib2.Http, "request", request) + other.Refresh() + assert calls == [endpoint] + assert other.client_config == {} + + +@pytest.mark.parametrize("client_file", ["malformed", "service", "other-user"]) +def test_cached_login_ignores_unrelated_client_config( + tmp_path, monkeypatch, client_file +): + monkeypatch.chdir(tmp_path) + path = tmp_path / "user.json" + cached = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri="https://cache-only.example.test/token", + ) + path.write_text(cached.to_json()) + config = tmp_path / "client_secrets.json" + if client_file == "malformed": + config.write_text("{") + elif client_file == "service": + config.write_text('{"type":"service_account"}') + else: + config.write_text( + json.dumps( + { + "installed": { + "client_id": "other", + "client_secret": "other-secret", + "token_uri": "https://other.example.test/token", + "auth_uri": "https://other.example.test/auth", + "redirect_uris": ["http://localhost"], + } + } + ) + ) + auth = GoogleAuth(settings_file=None) + auth.LoadCredentialsFile(path) + assert auth.credentials.refresh_token == "refresh" + assert auth.credentials.token_uri == TOKEN_URI + assert auth.client_config == {} + if client_file != "other-user": + with pytest.raises(InvalidConfigError): + auth.LoadClientConfigFile() + + +@pytest.mark.parametrize("expiry", [None, datetime(2099, 1, 1)]) +def test_token_only_cache(tmp_path, expiry): + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials("token", expiry=expiry) + path = tmp_path / "user.json" + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + assert auth.credentials.valid + assert auth.credentials.expiry == expiry + + +def test_noninteractive_refresh_handler(monkeypatch): + monkeypatch.setenv("GDRIVE_NON_INTERACTIVE", "true") + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_handler=lambda request, scopes=None: ( + "brokered", + datetime(2099, 1, 1), + ), + ) + auth.LocalWebserverAuth() + assert auth.credentials.token == "brokered" + assert auth.credentials.valid + + +def test_fsspec_upgrade_uses_new_cache(tmp_path, mocker): + mocker.patch("appdirs.user_cache_dir", return_value=str(tmp_path)) + legacy = json.dumps({"_class": "OAuth2Credentials", "access_token": "old"}) + old_path = tmp_path / "client" / "default.json" + old_path.parent.mkdir() + old_path.write_text(legacy) + login = mocker.patch( + "google_auth_oauthlib.flow.InstalledAppFlow.run_local_server", + return_value=Credentials("new-token"), + ) + for _ in range(2): + auth = _client_auth("client", "secret") + assert auth.credentials.token == "new-token" + login.assert_called_once() + assert (tmp_path / "v2" / "client" / "default.json").is_file() + for source in ( + {"client_json": legacy}, + {"client_json_file_path": str(old_path)}, + ): + with pytest.raises(GDriveAuthError, match="sign in again") as exc: + _client_auth("client", "secret", **source) + assert isinstance(exc.value.__cause__, InvalidCredentialsError) + assert old_path.read_text() == legacy + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows DACL") +def test_windows_cache_preserves_file_permissions(tmp_path, monkeypatch): + import win32security as security + + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials("first") + auth.SaveCredentialsFile(path) + flags = security.DACL_SECURITY_INFORMATION + + def dacl(filename): + sd = security.GetFileSecurity(str(filename), flags) + return security.ConvertSecurityDescriptorToStringSecurityDescriptor( + sd, 1, flags + ) + + assert "(A;;FA;;;OW)" in dacl(path) + # Add a distinct explicit grant so an always-owner-only replacement also fails. + sd = security.ConvertStringSecurityDescriptorToSecurityDescriptor( + "D:P(A;;FA;;;OW)(A;;FR;;;SY)", 1 + ) + security.SetFileSecurity(str(path), flags, sd) + original = dacl(path) + replace = os.replace + + def check_staged_acl(src, dst): + if dst == path: + assert dacl(src) == original + return replace(src, dst) + + monkeypatch.setattr(os, "replace", check_staged_acl) + auth.credentials = Credentials("updated") + auth.SaveCredentialsFile(path) + assert dacl(path) == original + assert json.loads(path.read_text())["token"] == "updated" + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows file compression") +def test_windows_compressed_cache(tmp_path): + from ctypes import byref, c_int, windll + + import win32api + + if win32api.GetVolumeInformation(tmp_path.anchor)[4] != "NTFS": + pytest.skip("WOF compression requires NTFS") + + path = tmp_path / "user.json" + path.write_bytes(b'{"token":"compressed"}' + b" " * 65536) + subprocess.run( + ["compact.exe", "/C", "/EXE:XPRESS4K", "/F", str(path)], check=True + ) + backed = c_int() + assert ( + windll.wofutil.WofIsExternalFile( + str(path), byref(backed), None, None, None + ) + == 0 + ) + assert backed.value + auth = GoogleAuth(settings_file=None) + auth.LoadCredentialsFile(path) + assert auth.credentials.token == "compressed" diff --git a/pydrive2/test/test_oauth.py b/pydrive2/test/test_oauth.py index 3aa5a127..359adc7a 100644 --- a/pydrive2/test/test_oauth.py +++ b/pydrive2/test/test_oauth.py @@ -4,7 +4,7 @@ import time import pytest -from oauth2client.file import Storage +from google.oauth2.credentials import Credentials from pydrive2.auth import AuthenticationError, GoogleAuth from pydrive2.test.test_util import ( @@ -78,34 +78,6 @@ def test_05_ConfigFromSettingsWithoutOauthScope(): time.sleep(1) -@pytest.mark.skip(reason="P12 authentication is deprecated") -@pytest.mark.drive -def test_06_ServiceAuthFromSavedCredentialsP12File(): - setup_credentials("credentials/6.dat") - ga = GoogleAuth(settings_file_path("test_oauth_test_06.yaml")) - ga.ServiceAuth() - assert not ga.access_token_expired - time.sleep(1) - - -@pytest.mark.drive -def test_07_ServiceAuthFromSavedCredentialsJsonFile(): - # Have an initial auth so that credentials/7.dat gets saved - ga = GoogleAuth(settings_file_path("test_oauth_test_07.yaml")) - credentials_file = ga.settings["save_credentials_file"] - # Delete old credentials file - delete_file(credentials_file) - assert not os.path.exists(credentials_file) - ga.ServiceAuth() - assert os.path.exists(credentials_file) - # Secondary auth should be made only using the previously saved - # login info - ga = GoogleAuth(settings_file_path("test_oauth_test_07.yaml")) - ga.ServiceAuth() - assert not ga.access_token_expired - time.sleep(1) - - @pytest.mark.drive def test_08_ServiceAuthFromJsonFileNoCredentialsSaving(): # Test that no credentials are saved and API is still functional @@ -116,49 +88,27 @@ def test_08_ServiceAuthFromJsonFileNoCredentialsSaving(): time.sleep(1) -@pytest.mark.drive -def test_09_SaveLoadCredentialsUsesDefaultStorage(mocker): - # Test fix for https://github.com/iterative/PyDrive2/issues/163 - # Make sure that Load and Save credentials by default reuse the - # same Storage (since it defined lock which make it TS) - ga = GoogleAuth(settings_file_path("test_oauth_test_09.yaml")) - credentials_file = ga.settings["save_credentials_file"] - # Delete old credentials file - delete_file(credentials_file) - assert not os.path.exists(credentials_file) - spy = mocker.spy(Storage, "__init__") - ga.ServiceAuth() - ga.LoadCredentials() - ga.SaveCredentials() - assert spy.call_count == 0 - - -@pytest.mark.drive -def test_10_ServiceAuthFromSavedCredentialsDictionary(): - creds_dict = {} - settings = { - "client_config_backend": "service", - "service_config": { - "client_json_file_path": "/tmp/pydrive2/credentials.json", - }, - "oauth_scope": ["https://www.googleapis.com/auth/drive"], - "save_credentials": True, - "save_credentials_backend": "dictionary", - "save_credentials_dict": creds_dict, - "save_credentials_key": "creds", - } - ga = GoogleAuth(settings=settings) - ga.ServiceAuth() - assert not ga.access_token_expired - assert creds_dict - first_creds_dict = creds_dict.copy() - # Secondary auth should be made only using the previously saved - # login info - ga = GoogleAuth(settings=settings) - ga.ServiceAuth() - assert not ga.access_token_expired - assert creds_dict == first_creds_dict - time.sleep(1) +def test_09_SaveLoadCredentialsUsesDefaultStorage(mocker, tmp_path): + # Regression for #163: use the same storage lock for loads and saves. + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(tmp_path / "user.json"), + } + ) + auth.credentials = Credentials( + "token", client_id="id", client_secret="secret" + ) + storage = auth._configured_store + save = mocker.spy(storage, "save") + load = mocker.spy(storage, "load") + auth.SaveCredentials() + auth.LoadCredentials() + assert auth.credentials.token == "token" + save.assert_called_once() + load.assert_called_once_with() @pytest.mark.drive diff --git a/pyproject.toml b/pyproject.toml index 5c53e55e..71886cc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,11 +24,15 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ "google-api-python-client>=1.12.5", - "oauth2client>=4.0.0", + "google-auth>=2.15.0", + "google-auth-httplib2>=0.2.0", + "google-auth-oauthlib>=1.5.0", + "filelock>=3.30.0", + "httplib2>=0.19.0", + "oauthlib>=3.0.0", "PyYAML>=3.0", - # https://github.com/iterative/PyDrive2/issues/361 - "cryptography<44", - "pyOpenSSL>=19.1.0,<=24.2.1", + "cryptography>=3.4", + "pywin32>=305; sys_platform == 'win32'", ] license = {text = "Apache License 2.0"} dynamic = ["version"]