Repository: rasterio/rasterio

Issue:
Rasterio should not directly instantiate Env() outside of @ensure_env
Now that `rasterio.env.ensure_env()` only creates an environment if one does not already exist, we should rely on it for creating a GDAL environment only when necessary.  I think the only remaining instances are:

- `rasterio.transform.transform()`
- `rasterio.open()`
- `rasterio.io.MemoryFile.open()`

The hardest is `rasterio.open()` because it needs `Env()` in order to authenticate with AWS.


Discussion:
I think restructuring the objects associated with Rasterio's handling of the GDAL environment will help the `rasterio.open()` case.  I have tried doing this twice in the past couple months and hit some weird environment related test failures both times.  There are a lot of instances in the tests where we now have an unnecessary `with rasterio.Env()` so it may be worth cleaning those up first.

As far as the implementation goes, I think `rasterio._env.GDALEnv()` should provide the bridge between Python and GDAL by implementing the necessary methods for managing the environment.  `rasterio.env.Env()` should inherit from `GDALEnv()` and handle nested environments and AWS authentication via `boto`.  Rather than keep track of the currently active `GDALEnv()` we can just track the currently active `Env()`, and with the addition of a `current_env()` function we could get access to it in `rasterio.open()`.  We could also get rid of `rasterio.env.getenv()/setenv()/delenv()`.  Some of the thread related changes in https://github.com/mapbox/rasterio/pull/997 might make this slightly harder, but I think its a simpler, cleaner interface that plugs a few leaks.
@geowurster I'm all for cleaner and simpler! I'm on vacation this week but look forward to checking this out when I get back.
Some clarification on the test failures I hit in the past: a bunch of `warp` related tests started failing, and I suspect it has to do with when/where/how `CHECK_WITH_INVERT_PROJ` gets set or inherited.
TODO for 1.0: put `ensure_env` decorator on `rasterio.open` and credentialize in a different way.

Resolving patch:
diff --git a/rasterio/__init__.py b/rasterio/__init__.py
--- a/rasterio/__init__.py
+++ b/rasterio/__init__.py
@@ -25,7 +25,7 @@ def emit(self, record):
 from rasterio.dtypes import (
     bool_, ubyte, uint8, uint16, int16, uint32, int32, float32, float64,
     complex_, check_dtype)
-from rasterio.env import ensure_env, Env
+from rasterio.env import ensure_env_credentialled, Env
 from rasterio.errors import RasterioIOError
 from rasterio.compat import string_types
 from rasterio.io import (
@@ -55,6 +55,7 @@ def emit(self, record):
 log.addHandler(NullHandler())
 
 
+@ensure_env_credentialled
 def open(fp, mode='r', driver=None, width=None, height=None, count=None,
          crs=None, transform=None, dtype=None, nodata=None, sharing=True,
          **kwargs):
@@ -208,30 +209,25 @@ def fp_writer(fp):
 
         # The 'normal' filename or URL path.
         path = parse_path(fp)
-        scheme = getattr(path, 'scheme', None)
-
-        with Env() as env:
-            if scheme == 's3':
-                env.credentialize()
-
-            # Create dataset instances and pass the given env, which will
-            # be taken over by the dataset's context manager if it is not
-            # None.
-            if mode == 'r':
-                s = DatasetReader(path, driver=driver, **kwargs)
-            elif mode == 'r+':
-                s = get_writer_for_path(path)(path, mode, driver=driver, **kwargs)
-            elif mode.startswith("w"):
-                s = get_writer_for_driver(driver)(path, mode, driver=driver,
-                                                  width=width, height=height,
-                                                  count=count, crs=crs,
-                                                  transform=transform,
-                                                  dtype=dtype, nodata=nodata,
-                                                  **kwargs)
-            else:
-                raise ValueError(
-                    "mode must be one of 'r', 'r+', or 'w', not %s" % mode)
-            return s
+
+        # Create dataset instances and pass the given env, which will
+        # be taken over by the dataset's context manager if it is not
+        # None.
+        if mode == 'r':
+            s = DatasetReader(path, driver=driver, **kwargs)
+        elif mode == 'r+':
+            s = get_writer_for_path(path)(path, mode, driver=driver, **kwargs)
+        elif mode.startswith("w"):
+            s = get_writer_for_driver(driver)(path, mode, driver=driver,
+                                              width=width, height=height,
+                                              count=count, crs=crs,
+                                              transform=transform,
+                                              dtype=dtype, nodata=nodata,
+                                              **kwargs)
+        else:
+            raise ValueError(
+                "mode must be one of 'r', 'r+', or 'w', not %s" % mode)
+        return s
 
 
 Band = namedtuple('Band', ['ds', 'bidx', 'dtype', 'shape'])
diff --git a/rasterio/env.py b/rasterio/env.py
--- a/rasterio/env.py
+++ b/rasterio/env.py
@@ -13,6 +13,7 @@
 from rasterio.compat import string_types, getargspec
 from rasterio.dtypes import check_dtype
 from rasterio.errors import EnvError, GDALVersionError
+from rasterio.path import parse_path
 from rasterio.transform import guard_transform
 
 
@@ -140,6 +141,7 @@ def __init__(
         We raise EnvError if the GDAL config options
         AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
         credentials are handled exclusively by boto3.
+
         """
         if ('AWS_ACCESS_KEY_ID' in options or
                 'AWS_SECRET_ACCESS_KEY' in options):
@@ -177,6 +179,7 @@ def from_defaults(cls, *args, **kwargs):
         Notes
         -----
         The items in kwargs will be overlaid on the default values.
+
         """
         options = Env.default_options()
         options.update(**kwargs)
@@ -361,6 +364,47 @@ def wrapper(*args, **kwds):
     return wrapper
 
 
+def ensure_env_credentialled(f):
+    """Ensures a config environment exists and is credentialized
+
+    Parameters
+    ----------
+    f : function
+        A function.
+
+    Returns
+    -------
+    A function wrapper.
+
+    Notes
+    -----
+    The function wrapper checks the first argument of f and
+    credentializes the environment if the first argument is a URI with
+    scheme "s3".
+
+    """
+    @wraps(f)
+    def wrapper(*args, **kwds):
+        if local._env:
+            env_ctor = Env
+        else:
+            env_ctor = Env.from_defaults
+        with env_ctor() as wrapper_env:
+            if isinstance(args[0], str):
+                path = parse_path(args[0])
+                scheme = getattr(path, 'scheme', None)
+                if scheme == 's3':
+                    wrapper_env.credentialize()
+                    log.debug("Credentialized: {!r}".format(getenv()))
+                else:
+                    pass
+            else:
+                pass
+            return f(*args, **kwds)
+
+    return wrapper
+
+
 @attr.s(slots=True)
 @total_ordering
 class GDALVersion(object):


Test patch: