Repository: rasterio/rasterio

Issue:
Generalize cloud data support
It's not clear how to add support for clouds other than Amazon's. I'd like to generalize the existing code so that it is clear.  This is not a fix for #991 but will make Google Cloud (and other) support much easier for users to contribute.


Discussion:

Resolving patch:
diff --git a/rasterio/env.py b/rasterio/env.py
--- a/rasterio/env.py
+++ b/rasterio/env.py
@@ -1,19 +1,21 @@
 """Rasterio's GDAL/AWS environment"""
 
+import attr
 from functools import wraps, total_ordering
 import logging
-import threading
 import re
-import attr
-
+import threading
+import warnings
 
 import rasterio
 from rasterio._env import (
     GDALEnv, del_gdal_config, get_gdal_config, set_gdal_config)
 from rasterio.compat import string_types, getargspec
 from rasterio.dtypes import check_dtype
-from rasterio.errors import EnvError, GDALVersionError
-from rasterio.path import parse_path
+from rasterio.errors import (
+    EnvError, GDALVersionError, RasterioDeprecationWarning)
+from rasterio.path import parse_path, UnparsedPath, ParsedPath
+from rasterio.session import Session, AWSSession
 from rasterio.transform import guard_transform
 
 
@@ -106,7 +108,8 @@ def default_options(cls):
     def __init__(
             self, session=None, aws_unsigned=False, aws_access_key_id=None,
             aws_secret_access_key=None, aws_session_token=None,
-            region_name=None, profile_name=None, **options):
+            region_name=None, profile_name=None, session_class=AWSSession,
+            **options):
         """Create a new GDAL/AWS environment.
 
         Note: this class is a context manager. GDAL isn't configured
@@ -115,7 +118,7 @@ def __init__(
         Parameters
         ----------
         session : optional
-            A boto3 session object.
+            A Session object.
         aws_unsigned : bool, optional (default: False)
             If True, requests will be unsigned.
         aws_access_key_id : str, optional
@@ -128,6 +131,8 @@ def __init__(
             A region name, as per boto3.
         profile_name : str, optional
             A shared credentials profile name, as per boto3.
+        session_class : Session, optional
+            A sub-class of Session.
         **options : optional
             A mapping of GDAL configuration options, e.g.,
             `CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.
@@ -142,25 +147,60 @@ def __init__(
         AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
         credentials are handled exclusively by boto3.
 
+        Examples
+        --------
+
+        >>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True):
+        ...     with rasterio.open("https://example.com/a.tif") as src:
+        ...         print(src.profile)
+
+        For access to secured cloud resources, a Rasterio Session or a
+        foreign session object may be passed to the constructor.
+
+        >>> import boto3
+        >>> from rasterio.session import AWSSession
+        >>> boto3_session = boto3.Session(...)
+        >>> with Env(AWSSession(boto3_session)):
+        ...     with rasterio.open("s3://mybucket/a.tif") as src:
+        ...         print(src.profile)
+
         """
         if ('AWS_ACCESS_KEY_ID' in options or
                 'AWS_SECRET_ACCESS_KEY' in options):
             raise EnvError(
                 "GDAL's AWS config options can not be directly set. "
                 "AWS credentials are handled exclusively by boto3.")
-        self.aws_unsigned = aws_unsigned
-        self.aws_access_key_id = aws_access_key_id
-        self.aws_secret_access_key = aws_secret_access_key
-        self.aws_session_token = aws_session_token
-        self.region_name = region_name
-        self.profile_name = profile_name
-        self.session = session
+
+        if session:
+            # Passing a session via keyword argument is the canonical
+            # way to configure access to secured cloud resources.
+            if not isinstance(session, Session):
+                warnings.warn(
+                    "Passing a boto3 session is deprecated. Pass a Rasterio "
+                    "AWSSession object instead.",
+                    RasterioDeprecationWarning
+                )
+                session = AWSSession(session=session)
+            self.session = session
+        else:
+            # Before 1.0, Rasterio only supported AWS. We will special
+            # case AWS in 1.0.x. TODO: warn deprecation in 1.1.
+            warnings.warn(
+                "Passing abstract session keyword arguments is deprecated. "
+                "Pass a Rasterio AWSSession object instead.",
+                RasterioDeprecationWarning
+            )
+            self.session = AWSSession(
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                aws_session_token=aws_session_token,
+                region_name=region_name,
+                profile_name=profile_name,
+                aws_unsigned=aws_unsigned)
 
         self.options = options.copy()
         self.context_options = {}
 
-        self._creds = None
-
     @classmethod
     def from_defaults(cls, *args, **kwargs):
         """Create an environment with default config options
@@ -193,7 +233,7 @@ def is_credentialized(self):
         -------
         bool
         """
-        return bool(self._creds)
+        return hascreds()  # bool(self.session)
 
     def credentialize(self):
         """Get credentials and configure GDAL
@@ -204,54 +244,21 @@ def credentialize(self):
         Returns
         -------
         None
+
         """
         if hascreds():
             pass
-
         else:
-            import boto3
-            if not self.session and not self.aws_access_key_id and not self.profile_name:
-                self.session = boto3.Session()
-            elif not self.session:
-                self.session = boto3.Session(
-                    aws_access_key_id=self.aws_access_key_id,
-                    aws_secret_access_key=self.aws_secret_access_key,
-                    aws_session_token=self.aws_session_token,
-                    region_name=self.region_name,
-                    profile_name=self.profile_name)
-            else:
-                # use self.session
-                pass
-            self._creds = self.session._session.get_credentials()
-
-            # Pass these credentials to the GDAL environment.
-            cred_opts = {}
-
-            if self.aws_unsigned:
-                cred_opts['AWS_NO_SIGN_REQUEST'] = 'YES'
-            else:
-                if self._creds.access_key:  # pragma: no branch
-                    cred_opts['AWS_ACCESS_KEY_ID'] = self._creds.access_key
-                if self._creds.secret_key:  # pragma: no branch
-                    cred_opts['AWS_SECRET_ACCESS_KEY'] = self._creds.secret_key
-                if self._creds.token:
-                    cred_opts['AWS_SESSION_TOKEN'] = self._creds.token
-                if self.session.region_name:
-                    cred_opts['AWS_REGION'] = self.session.region_name
-
+            cred_opts = self.session.get_credential_options()
             self.options.update(**cred_opts)
             setenv(**cred_opts)
 
-    def can_credentialize_on_enter(self):
-        return bool(self.session or self.aws_access_key_id or self.profile_name)
-
     def drivers(self):
         """Return a mapping of registered drivers."""
         return local._env.drivers()
 
     def __enter__(self):
         log.debug("Entering env context: %r", self)
-        # No parent Rasterio environment exists.
         if local._env is None:
             log.debug("Starting outermost env")
             self._has_parent_env = False
@@ -274,8 +281,7 @@ def __enter__(self):
             self.context_options = getenv()
             setenv(**self.options)
 
-        if self.can_credentialize_on_enter():
-            self.credentialize()
+        self.credentialize()
 
         log.debug("Entered env context: %r", self)
         return self
@@ -389,17 +395,14 @@ def wrapper(*args, **kwds):
             env_ctor = Env
         else:
             env_ctor = Env.from_defaults
-        with env_ctor() as wrapper_env:
-            if isinstance(args[0], str):
-                path = parse_path(args[0])
-                scheme = getattr(path, 'scheme', None)
-                if scheme == 's3':
-                    wrapper_env.credentialize()
-                    log.debug("Credentialized: {!r}".format(getenv()))
-                else:
-                    pass
-            else:
-                pass
+
+        if isinstance(args[0], str):
+            session = Session.from_path(args[0])
+        else:
+            session = Session.from_path(None)
+
+        with env_ctor(session=session):
+            log.debug("Credentialized: {!r}".format(getenv()))
             return f(*args, **kwds)
 
     return wrapper
diff --git a/rasterio/rio/env.py b/rasterio/rio/env.py
--- a/rasterio/rio/env.py
+++ b/rasterio/rio/env.py
@@ -11,7 +11,7 @@
 @click.option('--formats', 'key', flag_value='formats', default=True,
               help="Enumerate the available formats.")
 @click.option('--credentials', 'key', flag_value='credentials', default=False,
-              help="Print AWS credentials.")
+              help="Print credentials.")
 @click.pass_context
 def env(ctx, key):
     """Print information about the Rasterio environment."""
@@ -20,7 +20,4 @@ def env(ctx, key):
             for k, v in sorted(env.drivers().items()):
                 click.echo("{0}: {1}".format(k, v))
         elif key == 'credentials':
-            click.echo(json.dumps({
-                'aws_access_key_id': env._creds.access_key,
-                'aws_secret_access_key': env._creds.secret_key,
-                'aws_session_token': env._creds.token}))
+            click.echo(json.dumps(env.session.credentials))
diff --git a/rasterio/session.py b/rasterio/session.py
new file mode 100644
--- /dev/null
+++ b/rasterio/session.py
@@ -0,0 +1,185 @@
+"""Abstraction for sessions in various clouds."""
+
+
+from rasterio.path import parse_path, UnparsedPath, ParsedPath
+
+
+class Session(object):
+    """Base for classes that configure access to secured resources.
+
+    Attributes
+    ----------
+    credentials : dict
+        Keys and values for session credentials.
+
+    Notes
+    -----
+    This class is not intended to be instantiated.
+
+    """
+
+    def get_credential_options(self):
+        """Get credentials as GDAL configuration options
+
+        Returns
+        -------
+        dict
+
+        """
+        return NotImplementedError
+
+    @staticmethod
+    def from_foreign_session(session, cls=None):
+        """Create a session object matching the foreign `session`.
+
+        Parameters
+        ----------
+        session : obj
+            A foreign session object.
+        cls : Session class, optional
+            The class to return.
+
+        Returns
+        -------
+        Session
+
+        """
+        if not cls:
+            return DummySession()
+        else:
+            return cls(session)
+
+    @staticmethod
+    def from_path(path, *args, **kwargs):
+        """Create a session object suited to the data at `path`.
+
+        Parameters
+        ----------
+        path : str
+            A dataset path or identifier.
+        args : sequence
+            Positional arguments for the foreign session constructor.
+        kwargs : dict
+            Keyword arguments for the foreign session constructor.
+
+        Returns
+        -------
+        Session
+
+        """
+        if not path:
+            return DummySession()
+
+        path = parse_path(path)
+
+        if isinstance(path, UnparsedPath) or path.is_local:
+            return DummySession()
+
+        elif path.scheme == "s3" or "amazonaws.com" in path.path:
+            return AWSSession(*args, **kwargs)
+
+        # This factory can be extended to other cloud providers here.
+        # elif path.scheme == "cumulonimbus":  # for example.
+        #     return CumulonimbusSession(*args, **kwargs)
+
+        else:
+            return DummySession()
+
+
+class DummySession(Session):
+    """A dummy session.
+
+    Attributes
+    ----------
+    credentials : dict
+        The session credentials.
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        self._session = None
+        self.credentials = {}
+
+    def get_credential_options(self):
+        """Get credentials as GDAL configuration options
+
+        Returns
+        -------
+        dict
+
+        """
+        return {}
+
+
+class AWSSession(Session):
+    """Configures access to secured resources stored in AWS S3.
+    """
+
+    def __init__(
+            self, session=None, aws_unsigned=False, aws_access_key_id=None,
+            aws_secret_access_key=None, aws_session_token=None,
+            region_name=None, profile_name=None):
+        """Create a new boto3 session
+
+        Parameters
+        ----------
+        session : optional
+            A boto3 session object.
+        aws_unsigned : bool, optional (default: False)
+            If True, requests will be unsigned.
+        aws_access_key_id : str, optional
+            An access key id, as per boto3.
+        aws_secret_access_key : str, optional
+            A secret access key, as per boto3.
+        aws_session_token : str, optional
+            A session token, as per boto3.
+        region_name : str, optional
+            A region name, as per boto3.
+        profile_name : str, optional
+            A shared credentials profile name, as per boto3.
+
+        """
+        import boto3
+
+        if session:
+            self._session = session
+        else:
+            if not aws_access_key_id and not profile_name:
+                self._session = boto3.Session()
+            else:
+                self._session = boto3.Session(
+                    aws_access_key_id=aws_access_key_id,
+                    aws_secret_access_key=aws_secret_access_key,
+                    aws_session_token=aws_session_token,
+                    region_name=region_name,
+                    profile_name=profile_name)
+
+        self._unsigned = aws_unsigned
+        self._creds = self._session._session.get_credentials()
+
+    @property
+    def credentials(self):
+        """The session credentials as a dict"""
+        creds = {}
+        if self._creds.access_key:  # pragma: no branch
+            creds['aws_access_key_id'] = self._creds.access_key
+        if self._creds.secret_key:  # pragma: no branch
+            creds['aws_secret_access_key'] = self._creds.secret_key
+        if self._creds.token:
+            creds['aws_session_token'] = self._creds.token
+        if self._session.region_name:
+            creds['aws_region'] = self._session.region_name
+        return creds
+
+    def get_credential_options(self):
+        """Get credentials as GDAL configuration options
+
+        Returns
+        -------
+        dict
+
+        """
+        if self._unsigned:
+            return {'AWS_NO_SIGN_REQUEST': 'YES'}
+        else:
+            return {k.upper(): v for k, v in self.credentials.items()}


Test patch:
diff --git a/tests/test_env.py b/tests/test_env.py
--- a/tests/test_env.py
+++ b/tests/test_env.py
@@ -167,10 +167,10 @@ def test_aws_session(gdalenv):
         aws_access_key_id='id', aws_secret_access_key='key',
         aws_session_token='token', region_name='null-island-1')
     with rasterio.env.Env(session=aws_session) as s:
-        assert s._creds.access_key == 'id'
-        assert s._creds.secret_key == 'key'
-        assert s._creds.token == 'token'
-        assert s.session.region_name == 'null-island-1'
+        assert s.session._creds.access_key == 'id'
+        assert s.session._creds.secret_key == 'key'
+        assert s.session._creds.token == 'token'
+        assert s.session._session.region_name == 'null-island-1'
 
 
 def test_aws_session_credentials(gdalenv):
diff --git a/tests/test_session.py b/tests/test_session.py
new file mode 100644
--- /dev/null
+++ b/tests/test_session.py
@@ -0,0 +1,109 @@
+"""Tests of session module"""
+
+import pytest
+
+from rasterio.session import DummySession, AWSSession, Session
+
+
+def test_dummy_session():
+    """DummySession works"""
+    sesh = DummySession()
+    assert sesh._session is None
+    assert sesh.get_credential_options() == {}
+
+
+def test_aws_session_class():
+    """AWSSession works"""
+    sesh = AWSSession(aws_access_key_id='foo', aws_secret_access_key='bar')
+    assert sesh._session
+    assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
+    assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
+
+
+def test_aws_session_class_session():
+    """AWSSession works"""
+    boto3 = pytest.importorskip("boto3")
+    sesh = AWSSession(session=boto3.session.Session(aws_access_key_id='foo', aws_secret_access_key='bar'))
+    assert sesh._session
+    assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
+    assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
+
+
+def test_aws_session_class_unsigned():
+    """AWSSession works"""
+    pytest.importorskip("boto3")
+    sesh = AWSSession(aws_unsigned=True)
+    assert sesh._session
+    assert sesh.get_credential_options()['AWS_NO_SIGN_REQUEST'] == 'YES'
+
+
+def test_aws_session_class_profile(tmpdir, monkeypatch):
+    """Confirm that profile_name kwarg works."""
+    pytest.importorskip("boto3")
+    credentials_file = tmpdir.join('credentials')
+    credentials_file.write("[testing]\n"
+                           "aws_access_key_id = foo\n"
+                           "aws_secret_access_key = bar\n"
+                           "aws_session_token = baz")
+    monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(credentials_file))
+    monkeypatch.setenv('AWS_SESSION_TOKEN', 'ignore_me')
+    sesh = AWSSession(profile_name='testing')
+    assert sesh._session
+    assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
+    assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
+    assert sesh.get_credential_options()['AWS_SESSION_TOKEN'] == 'baz'
+    monkeypatch.undo()
+
+
+def test_session_factory_unparsed():
+    """Get a DummySession for unparsed paths"""
+    sesh = Session.from_path("/vsicurl/lolwut")
+    assert isinstance(sesh, DummySession)
+
+
+def test_session_factory_empty():
+    """Get a DummySession for no path"""
+    sesh = Session.from_path("")
+    assert isinstance(sesh, DummySession)
+
+
+def test_session_factory_local():
+    """Get a DummySession for local paths"""
+    sesh = Session.from_path("file:///lolwut")
+    assert isinstance(sesh, DummySession)
+
+
+def test_session_factory_unknown():
+    """Get a DummySession for unknown paths"""
+    sesh = Session.from_path("https://fancy-cloud.com/lolwut")
+    assert isinstance(sesh, DummySession)
+
+
+def test_session_factory_s3():
+    """Get an AWSSession for s3:// paths"""
+    pytest.importorskip("boto3")
+    sesh = Session.from_path("s3://lol/wut")
+    assert isinstance(sesh, AWSSession)
+
+
+def test_session_factory_s3_kwargs():
+    """Get an AWSSession for s3:// paths with keywords"""
+    pytest.importorskip("boto3")
+    sesh = Session.from_path("s3://lol/wut", aws_access_key_id='foo', aws_secret_access_key='bar')
+    assert isinstance(sesh, AWSSession)
+    assert sesh._session.get_credentials().access_key == 'foo'
+    assert sesh._session.get_credentials().secret_key == 'bar'
+
+
+def test_foreign_session_factory_dummy():
+    sesh = Session.from_foreign_session(None)
+    assert isinstance(sesh, DummySession)
+
+
+def test_foreign_session_factory_s3():
+    boto3 = pytest.importorskip("boto3")
+    aws_session = boto3.Session(aws_access_key_id='foo', aws_secret_access_key='bar')
+    sesh = Session.from_foreign_session(aws_session, cls=AWSSession)
+    assert isinstance(sesh, AWSSession)
+    assert sesh._session.get_credentials().access_key == 'foo'
+    assert sesh._session.get_credentials().secret_key == 'bar'