Repository: rasterio/rasterio

Issue:
Rasterio does not properly tear down GDAL environment
If a Rasterio managed environment does not exist, using `with rasterio.Env()` creates an environment that is not fully cleaned up when the context manager exits.  I'm working on a fix.

The simplest test is:

```python
import rasterio as rio
from rasterio._env import get_gdal_config

key = 'CHECK_WITH_INVERT_PROJ'

print(get_gdal_config(key))
with rio.Env():
    print(get_gdal_config(key))
print(get_gdal_config(key))
```

and a more complex test is:

```python
from osgeo import gdal
import rasterio as rio
from rasterio._env import get_gdal_config


key = 'CHECK_WITH_INVERT_PROJ'


def show(position):

    print(position)
    for k in (key, 'opt'):
        print("osgeo.gdal {}={}".format(k, gdal.GetConfigOption(k)))
        print("rasterio   {}={}".format(k, get_gdal_config(k)))
    print("")


show("Before Env()")

with rio.Env():
    show("In Env() 1")

    # Another inner environment
    with rio.Env(opt='val'):
        show("Inner Env()")

    show("Outside inner Env()")

show("Outside Env()")


rio.env.delenv()
show("Post-delete")
```

produces:

```
Before Env()
osgeo.gdal CHECK_WITH_INVERT_PROJ=None
rasterio   CHECK_WITH_INVERT_PROJ=None
osgeo.gdal opt=None
rasterio   opt=None

In Env() 1
osgeo.gdal CHECK_WITH_INVERT_PROJ=ON
rasterio   CHECK_WITH_INVERT_PROJ=True
osgeo.gdal opt=None
rasterio   opt=None

Inner Env()
osgeo.gdal CHECK_WITH_INVERT_PROJ=ON
rasterio   CHECK_WITH_INVERT_PROJ=True
osgeo.gdal opt=val
rasterio   opt=val

Outside inner Env()
osgeo.gdal CHECK_WITH_INVERT_PROJ=ON
rasterio   CHECK_WITH_INVERT_PROJ=True
osgeo.gdal opt=None
rasterio   opt=None

Outside Env()
osgeo.gdal CHECK_WITH_INVERT_PROJ=ON
rasterio   CHECK_WITH_INVERT_PROJ=True
osgeo.gdal opt=None
rasterio   opt=None

Post-delete
osgeo.gdal CHECK_WITH_INVERT_PROJ=None
rasterio   CHECK_WITH_INVERT_PROJ=None
osgeo.gdal opt=None
rasterio   opt=None
```


Discussion:

Resolving patch:
diff --git a/rasterio/env.py b/rasterio/env.py
--- a/rasterio/env.py
+++ b/rasterio/env.py
@@ -1,6 +1,7 @@
 """Rasterio's GDAL/AWS environment"""
 
 from functools import wraps
+import itertools as it
 import logging
 
 from rasterio._env import (
@@ -15,6 +16,32 @@
 # The currently active GDAL/AWS environment is a private attribute.
 _env = None
 
+
+# When the outermost 'rasterio.Env()' executes '__enter__' it probes the
+# GDAL environment to see if any of the supplied config options already
+# exist, the assumption being that they were set with
+# 'osgeo.gdal.SetConfigOption()' or possibly 'rasterio.env.set_gdal_config()'.
+# The discovered options are reinstated when the outermost Rasterio environment
+# exits.  Without this check any environment options that are present in the
+# GDAL environment and are also passed to 'rasterio.Env()' will be unset
+# when 'rasterio.Env()' tears down, regardless of their value.  For example:
+#
+#   from osgeo import gdal
+#   import rasterio
+#
+#   gdal.SetConfigOption('key', 'value')
+#   with rasterio.Env(key='something'):
+#       pass
+#
+# The config option 'key' would be unset when 'Env()' exits.  A more
+# comprehensive solution would also leverage https://trac.osgeo.org/gdal/changeset/37273
+# but this gets Rasterio + older versions of GDAL halfway there.  One major
+# assumption is that environment variables are not set directly with
+# 'osgeo.gdal.SetConfigOption()' OR 'rasterio.env.set_gdal_config()' inside
+# of a 'rasterio.Env()'.
+_discovered_options = None
+
+
 log = logging.getLogger(__name__)
 
 # Rasterio defaults
@@ -141,17 +168,54 @@ def drivers(self):
         return _env.drivers()
 
     def __enter__(self):
+        global _env
+        global _discovered_options
         log.debug("Entering env context: %r", self)
-        defenv()
-        self.context_options = getenv()
+
+        # No parent Rasterio environment exists.
+        if _env is None:
+            logging.debug("Starting outermost env")
+            self._has_parent_env = False
+
+            # See note directly above where _discovered_options is globally
+            # defined.  This MUST happen before calling 'defenv()'.
+            _discovered_options = {}
+            # Don't want to reinstate the "I'M_ON_RASTERIO" option.
+            probe_env = {k for k in default_options if k != "I'M_ON_RASTERIO"}
+            probe_env |= set(self.options.keys())
+            for key in probe_env:
+                val = get_gdal_config(key, normalize=False)
+                if val is not None:
+                    _discovered_options[key] = val
+                    logging.debug("Discovered option: %s=%s", key, val)
+
+            defenv()
+            self.context_options = {}
+        else:
+            self._has_parent_env = True
+            self.context_options = getenv()
         setenv(**self.options)
         log.debug("Entered env context: %r", self)
         return self
 
     def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
+        global _env
+        global _discovered_options
         log.debug("Exiting env context: %r", self)
         delenv()
-        setenv(**self.context_options)
+        if self._has_parent_env:
+            defenv()
+            setenv(**self.context_options)
+        else:
+            logging.debug("Exiting outermost env")
+            # See note directly above where _discovered_options is globally
+            # defined.
+            while _discovered_options:
+                key, val = _discovered_options.popitem()
+                set_gdal_config(key, val, normalize=False)
+                logging.debug(
+                    "Set discovered option back to: '%s=%s", key, val)
+            _discovered_options = None
         log.debug("Exited env context: %r", self)
 
 
@@ -198,6 +262,7 @@ def delenv():
         _env.clear_config_options()
         log.debug("Cleared existing %r options", _env)
     _env.stop()
+    _env = None
 
 
 def ensure_env(f):


Test patch:
diff --git a/tests/test_env.py b/tests/test_env.py
--- a/tests/test_env.py
+++ b/tests/test_env.py
@@ -41,6 +41,41 @@ def test_gdal_config_accessers():
     assert get_gdal_config('foo') is None
 
 
+def test_gdal_config_accessors_no_normalize():
+    """Disables casting keys to upper case and normalizing values to boolean
+    Python values.
+    """
+    assert get_gdal_config('foo') is None
+    set_gdal_config('foo', 'ON', normalize=False)
+    assert get_gdal_config('foo', normalize=False) == 'ON'
+    del_gdal_config('foo')
+    assert get_gdal_config('foo') is None
+
+
+def test_gdal_config_accessors_capitalization():
+    """GDAL normalizes config names to upper case so Rasterio does not
+    need to do it on its own.  This test serves as a canary in case GDAL
+    changes its behavior, which is an important part of reinstating
+    discovered environment variables when ``rasterio.Env()`` starts.
+    GDAL does not alter config values.
+    """
+    assert get_gdal_config('foo') is None
+    assert get_gdal_config('FOO') is None
+
+    set_gdal_config('foo', 'bar')
+
+    assert get_gdal_config('foo') == 'bar'
+    assert get_gdal_config('FOO') == 'bar'
+
+    del_gdal_config('foo')
+    assert get_gdal_config('foo') is None
+    assert get_gdal_config('FOO') is None
+
+    set_gdal_config('upper', 'UPPER')
+    assert get_gdal_config('upper') == 'UPPER'
+    del_gdal_config('upper')
+
+
 # The 'gdalenv' fixture ensures that gdal configuration is deleted
 # at the end of the test, making tests as isolates as GDAL allows.
 
@@ -55,10 +90,14 @@ def test_env_accessors(gdalenv):
     assert get_gdal_config('foo') == '1'
     assert get_gdal_config('bar') == '2'
     delenv()
-    assert getenv() == rasterio.env._env.options == {}
+    with pytest.raises(EnvError):
+        getenv()
     assert get_gdal_config('foo') is None
     assert get_gdal_config('bar') is None
-    rasterio.env._env = None
+
+
+def test_env_accessors_no_env():
+    """Sould all raise an exception."""
     with pytest.raises(EnvError):
         delenv()
     with pytest.raises(EnvError):
@@ -68,11 +107,10 @@ def test_env_accessors(gdalenv):
 
 
 def test_ensure_env_decorator(gdalenv):
+    @ensure_env
     def f():
         return getenv()['WITH_RASTERIO_ENV']
-    wrapper = ensure_env(f)
-    assert wrapper() is True
-    assert 'WITH_RASTERIO_ENV' not in getenv()
+    assert f() is True
 
 
 def test_no_aws_gdal_config(gdalenv):
@@ -83,15 +121,15 @@ def test_no_aws_gdal_config(gdalenv):
         rasterio.Env(AWS_SECRET_ACCESS_KEY='y')
 
 
-def test_env_options(gdalenv):
-    """Test env options."""
+def test_env_defaults(gdalenv):
+    """Test env defaults."""
     env = rasterio.Env(foo='x')
     assert env.options['foo'] == 'x'
     assert not env.context_options
     with env:
-        assert env.context_options['CHECK_WITH_INVERT_PROJ'] is True
-        assert env.context_options['GTIFF_IMPLICIT_JPEG_OVR'] is False
-        assert env.context_options["I'M_ON_RASTERIO"] is True
+        assert get_gdal_config('CHECK_WITH_INVERT_PROJ') is True
+        assert get_gdal_config('GTIFF_IMPLICIT_JPEG_OVR') is False
+        assert get_gdal_config("I'M_ON_RASTERIO") is True
 
 
 def test_aws_session(gdalenv):
@@ -225,3 +263,69 @@ def test_rio_env_credentials_options(tmpdir, monkeypatch, runner):
     assert '"aws_secret_access_key": "bar"' in result.output
     assert '"aws_session_token": "baz"' in result.output
     monkeypatch.undo()
+
+
+def test_ensure_defaults_teardown(gdalenv):
+
+    """This test guards against a regression.  Previously ``rasterio.Env()``
+    would quietly reinstate any ``rasterio.env.default_options`` that was
+    not modified by the environment.
+
+    https://github.com/mapbox/rasterio/issues/968
+    """
+
+    def _check_defaults():
+        for key in default_options.keys():
+            assert get_gdal_config(key) is None
+
+    _check_defaults()
+    with rasterio.Env():
+        pass
+
+    _check_defaults()
+    assert rasterio.env._env is None
+
+
+@pytest.mark.parametrize("key,val", [
+    ('key', 'ON'),
+    ('CHECK_WITH_INVERT_PROJ', 'ON'),
+    ('key', 'OFF'),
+    ('CHECK_WITH_INVERT_PROJ', 'OFF')])
+def test_env_discovery(key, val):
+    """When passing options to ``rasterio.Env()`` Rasterio first checks
+    to see if they were set in the environment and reinstates on exit.
+    The discovered environment should only be reinstated when the outermost
+    environment exits.  It's really important that this test use an
+    environment default.
+    """
+
+    assert rasterio.env._discovered_options is None, \
+        "Something has gone horribly wrong."
+
+    try:
+        # This should persist when all other environment managers exit.
+        set_gdal_config(key, val)
+
+        # Start an environment and overwrite the value that should persist
+        with rasterio.Env(**{key: True}):
+            assert get_gdal_config(key) is True
+            assert rasterio.env._discovered_options == {key: val}
+
+            # Start another nested environment, again overwriting the value
+            # that should persist
+            with rasterio.Env(**{key: False}):
+                assert rasterio.env._discovered_options == {key: val}
+                assert get_gdal_config(key) is False
+
+            # Ensure the outer state is restored.
+            assert rasterio.env._discovered_options == {key: val}
+            assert get_gdal_config(key) is True
+
+        # Ensure the discovered value remains unchanged.
+        assert rasterio.env._discovered_options is None
+        assert get_gdal_config(key, normalize=False) == val
+
+    # Leaving this option in the GDAL environment could cause a problem
+    # for other tests.
+    finally:
+        del_gdal_config(key)
diff --git a/tests/test_write.py b/tests/test_write.py
--- a/tests/test_write.py
+++ b/tests/test_write.py
@@ -285,8 +285,9 @@ def test_write_noncontiguous(tmpdir):
 def test_write_blacklist(tmpdir, driver):
 
     # Skip if we don't have driver support built in.
-    if driver not in Env().drivers():
-        pytest.skip()
+    with Env() as env:
+        if driver not in env.drivers():
+            pytest.skip()
 
     name = str(tmpdir.join("data.test"))
     with pytest.raises(RasterioIOError) as exc_info: