Repository: rasterio/rasterio

Issue:
GDALEnv overriding PROJ_LIB unfriendly with 'internalproj' builds
OK, this is a particular case but I spent some time figuring out what happened.

So my use case is:
- System proj (4.9.2) installed in /usr prefix because of other things needing it
- PROJ master built and installed in /opt/internalproj prefix
- GDAL master
- Rasterio
- PROJ_LIB not defined

When Rasterio starts, it detects that /usr/share/proj exists and then set PROJ_LIB=/usr/share/proj. Then when GDAL uses internalproj, the later looks in  /usr/share/proj instead of  /opt/internalproj/share/proj. The issue is that /usr/share/proj doesn't contain the new proj.db ...

This is a very particular setup admitedly, but is there a reason for PROJDataFinder.search_prefix() at all to set PROJ_LIB when it detects the directory, since PROJ automatically looks into it, this directory being hardcoded in the binary at build time ?



Discussion:
@rouault I see what you mean. We search under python's sys.prefix mainly to help users of the Anaconda system, these users can't rely on a compiled-in path because the libraries and data are moved after compilation, but sys.prefix works.

I'm not sure what the right solution to this problem is.
ok, your explanation makes sense. I wrongly assumed that Python sys.prefix == PROJ installation prefix if {sys.prefix}/share/proj would exist.
A potential solution might be to run a dummy OSRImportFromProj4(hSRS, "+init=epsg:4326") (between CPLPushErrorHandler(CPLQuietErrorHandler) , CPLPopErrorHandler() ) and check if it is successful. If it is, then no need to modify PROJ_LIB.

@rouault would a probing call of OSRImportFromEPGS accomplish the same thing for GDAL_DATA? Are there any data caching gotchas?
> would a probing call of OSRImportFromEPGS accomplish the same thing for GDAL_DATA? 

Yes  (for GDAL 2.5, that will be the same as ImportFromProj4() way, so if you need to set GDAL_DATA for the purpose of other drivers, that won't be a good method )

> Are there any data caching gotchas?

Apparently not when the file is not found (caching may occur when the file is found)

Is there a call that will detect GDAL_DATA that is forward compatible with 2.5?
> Is there a call that will detect GDAL_DATA that is forward compatible with 2.5?

`CPLFindFile("gdal", "header.dxf") != NULL` should do

Resolving patch:
diff --git a/rasterio/compat.py b/rasterio/compat.py
--- a/rasterio/compat.py
+++ b/rasterio/compat.py
@@ -13,6 +13,7 @@
     from urllib.parse import urlparse
     from collections import UserDict
     from inspect import getfullargspec as getargspec
+    from unittest import mock
 else:  # pragma: no cover
     string_types = basestring,
     text_type = unicode
@@ -22,3 +23,4 @@
     from urlparse import urlparse
     from UserDict import UserDict
     from inspect import getargspec
+    import mock
diff --git a/rasterio/env.py b/rasterio/env.py
--- a/rasterio/env.py
+++ b/rasterio/env.py
@@ -606,16 +606,26 @@ def wrapper(*args, **kwds):
 
 if 'GDAL_DATA' not in os.environ:
 
-    path = GDALDataFinder().search()
+    # See https://github.com/mapbox/rasterio/issues/1631.
+    if GDALDataFinder().find_file("header.dxf"):
+        log.debug("GDAL data files are available at built-in paths")
 
-    if path:
-        os.environ['GDAL_DATA'] = path
-        log.debug("GDAL_DATA not found in environment, set to %r.", path)
+    else:
+        path = GDALDataFinder().search()
+
+        if path:
+            os.environ['GDAL_DATA'] = path
+            log.debug("GDAL_DATA not found in environment, set to %r.", path)
 
 if 'PROJ_LIB' not in os.environ:
 
-    path = PROJDataFinder().search()
+    # See https://github.com/mapbox/rasterio/issues/1631.
+    if PROJDataFinder().has_data():
+        log.debug("PROJ data files are available at built-in paths")
+
+    else:
+        path = PROJDataFinder().search()
 
-    if path:
-        os.environ['PROJ_LIB'] = path
-        log.debug("PROJ data not found in environment, set to %r.", path)
+        if path:
+            os.environ['PROJ_LIB'] = path
+            log.debug("PROJ data not found in environment, set to %r.", path)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -348,7 +348,7 @@ def run(self):
 
 # Add futures to 'test' for Python < 3.2.
 if sys.version_info < (3, 2):
-    extra_reqs['test'].append('futures')
+    extra_reqs['test'].extend(['futures', 'mock'])
 
 # Add all extra requirements
 extra_reqs['all'] = list(set(itertools.chain(*extra_reqs.values())))


Test patch:
diff --git a/tests/test_data_paths.py b/tests/test_data_paths.py
--- a/tests/test_data_paths.py
+++ b/tests/test_data_paths.py
@@ -16,12 +16,22 @@ def test_gdal_data():
     assert GDALDataFinder().search() == os.path.join(os.path.dirname(rasterio.__file__), 'gdal_data')
 
 
+def test_gdal_data_find_file():
+    """Find_file shouldn't raise any exceptions"""
+    GDALDataFinder().find_file("header.dxf")
+
+
 @pytest.mark.wheel
 def test_proj_data():
     """Get GDAL data path from a wheel"""
     assert PROJDataFinder().search() == os.path.join(os.path.dirname(rasterio.__file__), 'proj_data')
 
 
+def test_proj_data_has_data():
+    """has_data shouldn't raise any exceptions"""
+    PROJDataFinder().has_data()
+
+
 @pytest.mark.wheel
 def test_env_gdal_data():
     runner = CliRunner()
diff --git a/tests/test_env.py b/tests/test_env.py
--- a/tests/test_env.py
+++ b/tests/test_env.py
@@ -14,6 +14,7 @@
 from rasterio.env import Env, defenv, delenv, getenv, setenv, ensure_env, ensure_env_credentialled
 from rasterio.env import GDALVersion, require_gdal_version
 from rasterio.errors import EnvError, RasterioIOError, GDALVersionError
+from rasterio.compat import mock
 from rasterio.rio.main import main_group
 from rasterio.session import AWSSession, OSSSession
 
@@ -124,13 +125,16 @@ def f():
     assert f() == '/lol/wut'
 
 
-def test_ensure_env_decorator_sets_gdal_data_prefix(gdalenv, monkeypatch, tmpdir):
+@mock.patch("rasterio._env.GDALDataFinder.find_file")
+def test_ensure_env_decorator_sets_gdal_data_prefix(find_file, gdalenv, monkeypatch, tmpdir):
     """ensure_env finds GDAL data under a prefix"""
 
     @ensure_env
     def f():
         return getenv()['GDAL_DATA']
 
+    find_file.return_value = None
+
     tmpdir.ensure("share/gdal/pcs.csv")
     monkeypatch.delenv('GDAL_DATA', raising=False)
     monkeypatch.setattr(sys, 'prefix', str(tmpdir))
@@ -138,12 +142,15 @@ def f():
     assert f() == str(tmpdir.join("share").join("gdal"))
 
 
-def test_ensure_env_decorator_sets_gdal_data_wheel(gdalenv, monkeypatch, tmpdir):
+@mock.patch("rasterio._env.GDALDataFinder.find_file")
+def test_ensure_env_decorator_sets_gdal_data_wheel(find_file, gdalenv, monkeypatch, tmpdir):
     """ensure_env finds GDAL data in a wheel"""
     @ensure_env
     def f():
         return getenv()['GDAL_DATA']
 
+    find_file.return_value = None
+
     tmpdir.ensure("gdal_data/pcs.csv")
     monkeypatch.delenv('GDAL_DATA', raising=False)
     monkeypatch.setattr(_env, '__file__', str(tmpdir.join(os.path.basename(_env.__file__))))