Repository: RhodiumGroup/rhg_compute_tools

Issue:
replicate_directory_structure_on_gcs: Add options for authenticating using `gcloud init`
If you set up a session and cache credentials using the gcloud or gsutil tools, we should be able to pull from these by simply creating a client

Discussion:

Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 053b986..dd1fc91 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -8,7 +8,7 @@ v0.2.0
 ------
 
 * Add CLI tools (:issue:`37`). See ``rctools gcs repdirstruc --help`` to start
-* Add new function ``rhg_compute_tools.gcs.replicate_directory_structure_on_gcs`` to copy directory trees into GCS
+* Add new function ``rhg_compute_tools.gcs.replicate_directory_structure_on_gcs`` to copy directory trees into GCS. Users can authenticate with cred_file or with default google credentials (:issue:`51`)
 * Fixes to docstrings and metadata (:issue:`43`) (:issue:`45`)
 
 
diff --git a/rhg_compute_tools/cli.py b/rhg_compute_tools/cli.py
index 0b3841b..5b2537f 100644
--- a/rhg_compute_tools/cli.py
+++ b/rhg_compute_tools/cli.py
@@ -1,5 +1,6 @@
 import click
-from rhg_compute_tools.gcs import replicate_directory_structure_on_gcs
+from rhg_compute_tools.gcs import (replicate_directory_structure_on_gcs,
+                                   authenticated_client)
 
 
 @click.group(
@@ -21,7 +22,6 @@ def gcs():
 @click.argument('src', type=click.Path(exists=True))
 @click.argument('dst', type=click.Path())
 @click.option('-c', '--credentials', type=click.Path(exists=True),
-              envvar='GOOGLE_APPLICATION_CREDENTIALS',
               help='Optional path to GCS credentials file.')
 def repdirstruc(src, dst, credentials):
     """Replicate directory structure onto GCS bucket.
@@ -31,6 +31,9 @@ def repdirstruc(src, dst, credentials):
 
     If --credentials or -c is not explicitly given, checks the
     GOOGLE_APPLICATION_CREDENTIALS environment variable for path to a GCS
-    credentials file.
+    credentials file, or default service accounts for authentication. See
+    https://googleapis.dev/python/google-api-core/latest/auth.html for more
+    details.
     """
-    replicate_directory_structure_on_gcs(src, dst, credentials)
+    client = authenticated_client(credentials)
+    replicate_directory_structure_on_gcs(src, dst, client)
diff --git a/rhg_compute_tools/gcs.py b/rhg_compute_tools/gcs.py
index cfe89b8..971e33c 100644
--- a/rhg_compute_tools/gcs.py
+++ b/rhg_compute_tools/gcs.py
@@ -11,6 +11,34 @@ import subprocess
 import shlex
 
 
+def authenticated_client(credentials=None):
+    """Convenience function to create an authenticated GCS client.
+
+    Parameters
+    ----------
+    credentials : str or None, optional
+        Str path to storage credentials authentication file. If None
+        is passed (default) will create a Client object with no args, using
+        the authorization credentials for the current environment. See the
+        [google cloud storage docs](
+        https://googleapis.dev/python/google-api-core/latest/auth.html)
+        for an overview of the authorization options.
+
+    Returns
+    -------
+    google.cloud.storage.Client
+    """
+    if credentials is None:
+        client = storage.Client()
+    else:
+        creds = service_account.Credentials.from_service_account_file(
+            str(credentials)
+        )
+        client = storage.Client(credentials=creds)
+
+    return client
+
+
 def get_bucket(cred_path):
     '''Return a bucket object from Rhg's GCS system.
 
@@ -47,7 +75,7 @@ def _get_path_types(src, dest):
     return src_gs, dest_gs, dest_gcs
 
 
-def replicate_directory_structure_on_gcs(src, dst, client_or_creds):
+def replicate_directory_structure_on_gcs(src, dst, client):
     """
     Replicate a local directory structure on google cloud storage
 
@@ -60,16 +88,9 @@ def replicate_directory_structure_on_gcs(src, dst, client_or_creds):
     dst : str
         A url for the root directory of the destination, starting with
         `gs://[bucket_name]/`, e.g. `gs://my_bucket/path/to/my/data`
-    client_or_creds : google.cloud.storage.client.Client or str
-        An authenticated :py:class:`google.cloud.storage.client.Client` object,
-        or a str path to storage credentials authentication file.
+    client : google.cloud.storage.client.Client
+        An authenticated :py:class:`google.cloud.storage.client.Client` object.
     """
-    if isinstance(client_or_creds, str):
-        credentials = service_account.Credentials.from_service_account_file(
-            client_or_creds
-        )
-        client_or_creds = storage.Client(credentials=credentials)
-
     if dst.startswith('gs://'):
         dst = dst[5:]
     elif dst.startswith('gcs://'):
@@ -80,7 +101,7 @@ def replicate_directory_structure_on_gcs(src, dst, client_or_creds):
     bucket_name = dst.split('/')[0]
     blob_path = '/'.join(dst.split('/')[1:])
 
-    bucket = client_or_creds.get_bucket(bucket_name)
+    bucket = client.get_bucket(bucket_name)
 
     for d, dirnames, files in os.walk(src):
         dest_path = os.path.join(blob_path, os.path.relpath(d, src))


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 786f38b..5298c45 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -11,9 +11,13 @@ import rhg_compute_tools.cli
 
 @pytest.fixture
 def replicate_directory_structure_on_gcs_stub(mocker):
-    mocker.patch.object(rhg_compute_tools.cli,
-                        'replicate_directory_structure_on_gcs',
-                        new=lambda *args: click.echo(';'.join(args)))
+    mocker.patch.object(
+        rhg_compute_tools.cli,
+        'replicate_directory_structure_on_gcs',
+        new=lambda *args: click.echo(';'.join(args)),
+    )
+    mocker.patch.object(rhg_compute_tools.cli, 'authenticated_client',
+                        new=lambda x: str(x))
 
 
 @pytest.fixture
@@ -36,8 +40,8 @@ def test_repdirstruc(credflag, replicate_directory_structure_on_gcs_stub,
     dst_path = 'gcs://foo/bar'
 
     if credflag is None:
-        monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', cred_path)
         credargs = []
+        cred_path = str(None)
 
     # Run CLI
     runner = CliRunner()
@@ -50,8 +54,9 @@ def test_repdirstruc(credflag, replicate_directory_structure_on_gcs_stub,
     assert result.output == expected_output
 
 
-@pytest.mark.parametrize('credflag', ['-c', '--credentials', None])
-def test_repdirstruc_nocredfile(credflag, replicate_directory_structure_on_gcs_stub,
+@pytest.mark.parametrize('credflag', ['-c', '--credentials'])
+def test_repdirstruc_nocredfile(credflag,
+                                replicate_directory_structure_on_gcs_stub,
                                 tmpdir, monkeypatch):
     """Test rctools gcs repdirstruc for graceful fail when cred file missing
     """
@@ -61,10 +66,6 @@ def test_repdirstruc_nocredfile(credflag, replicate_directory_structure_on_gcs_s
     src_path = str(tmpdir)
     dst_path = 'gcs://foo/bar'
 
-    if credflag is None:
-        monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', cred_path)
-        credargs = []
-
     # Run CLI
     runner = CliRunner()
     result = runner.invoke(
@@ -72,8 +73,8 @@ def test_repdirstruc_nocredfile(credflag, replicate_directory_structure_on_gcs_s
         ['gcs', 'repdirstruc'] + credargs + [src_path, dst_path],
     )
 
-    expected_output = 'Usage: rctools-cli gcs repdirstruc [OPTIONS] SRC DST\nTry' \
-                      ' "rctools-cli gcs repdirstruc -h" for help.\n\nError: ' \
-                      'Invalid value for "-c" / "--credentials": Path ' \
-                      '"_foobar.json" does not exist.\n'
-    assert result.output == expected_output
+    expected = 'Usage: rctools-cli gcs repdirstruc [OPTIONS] SRC DST\nTry' \
+               ' "rctools-cli gcs repdirstruc -h" for help.\n\nError: ' \
+               'Invalid value for "-c" / "--credentials": Path ' \
+               '"_foobar.json" does not exist.\n'
+    assert result.output == expected