Repository: rasterio/rasterio

Issue:
Accept file objects as well as URLs and connection strings in rasterio.open()?
See https://github.com/Toblerity/Fiona/issues/409 and https://github.com/mapbox/rasterio/issues/839.

I guess I'm sorta coming around to the idea that `rasterio.open()` could accept a file-like object and hide the `MemoryFile` details from users, removing some boilerplate from their programs. I've written a couple of new tests to help this discussion.




Discussion:
https://github.com/mapbox/rasterio/commit/f948aeba09e96d0a36eff11a0ac4b29a2b0942a0#diff-d3388b510bd97235e4741b8417cc41e9R123 is what a user must do now:

```python
with MemoryFile(file_object.read()) as memfile:
    with memfile.open() as src:
        # Use the dataset object `src`.
```

Future usage might be like https://github.com/mapbox/rasterio/commit/f948aeba09e96d0a36eff11a0ac4b29a2b0942a0#diff-d3388b510bd97235e4741b8417cc41e9R135

```python
with rasterio.open(file_object) as src:
    # Use the dataset object `src`.
```

We can't pass a Python file object to GDAL, so the byes of the file object must be read and then used to initialize a `MemoryFile` within `rasterio.open()`. How that `MemoryFile` would be managed isn't clear to me at the moment.

Resolving patch:
diff --git a/rasterio/__init__.py b/rasterio/__init__.py
--- a/rasterio/__init__.py
+++ b/rasterio/__init__.py
@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 
 from collections import namedtuple
+from contextlib import contextmanager
 import logging
 import warnings
 try:
@@ -21,7 +22,7 @@ def emit(self, record):
 from rasterio.errors import RasterioIOError
 from rasterio.compat import string_types
 from rasterio.io import (
-    DatasetReader, get_writer_for_path, get_writer_for_driver)
+    DatasetReader, get_writer_for_path, get_writer_for_driver, MemoryFile)
 from rasterio.profiles import default_gtiff_profile
 from rasterio.transform import Affine, guard_transform
 from rasterio.vfs import parse_path
@@ -65,7 +66,7 @@ def window_index(*args, **kwargs):
 log.addHandler(NullHandler())
 
 
-def open(path, mode='r', driver=None, width=None, height=None,
+def open(fp, mode='r', driver=None, width=None, height=None,
          count=None, crs=None, transform=None, dtype=None, nodata=None,
          **kwargs):
     """Open file at ``path`` in ``mode`` 'r' (read), 'r+' (read and
@@ -82,6 +83,8 @@ def open(path, mode='r', driver=None, width=None, height=None,
 
     Parameters
     ----------
+    fp: string or file
+        A filename or file object opened in binary mode.
     mode: string
         "r" (read), "r+" (read/write), or "w" (write)
     driver: string
@@ -164,8 +167,9 @@ def open(path, mode='r', driver=None, width=None, height=None,
     "tar://"". In this case, the ``path`` must be an absolute path
     within that container.
     """
-    if not isinstance(path, string_types):
-        raise TypeError("invalid path: {0!r}".format(path))
+    if not isinstance(fp, string_types):
+        if not (hasattr(fp, 'read') or hasattr(fp, 'write')):
+            raise TypeError("invalid path or file: {0!r}".format(fp))
     if mode and not isinstance(mode, string_types):
         raise TypeError("invalid mode: {0!r}".format(mode))
     if driver and not isinstance(driver, string_types):
@@ -202,38 +206,69 @@ def open(path, mode='r', driver=None, width=None, height=None,
             "Blacklisted: file cannot be opened by "
             "driver '{0}' in '{1}' mode".format(driver, mode))
 
-    _, _, scheme = parse_path(path)
-
-    with Env() as env:
-        # Get AWS credentials only if we're attempting to access a
-        # raster using the S3 scheme.
-        if scheme == 's3':
-            env.get_aws_credentials()
-            log.debug("AWS credentials have been obtained")
-
-        # Create dataset instances and pass the given env, which will
-        # be taken over by the dataset's context manager if it is not
-        # None.
-        if mode == 'r':
-            s = DatasetReader(path)
-        elif mode == 'r-':
-            warnings.warn("'r-' mode is deprecated, use 'r'",
-                          DeprecationWarning)
-            s = DatasetReader(path)
-        elif mode == 'r+':
-            s = get_writer_for_path(path)(path, mode)
-        elif mode == 'w':
-            s = get_writer_for_driver(driver)(path, mode, driver=driver,
-                                              width=width, height=height,
-                                              count=count, crs=crs,
-                                              transform=transform,
-                                              dtype=dtype, nodata=nodata,
-                                              **kwargs)
-        else:
-            raise ValueError(
-                "mode string must be one of 'r', 'r+', or 'w', not %s" % mode)
-        s.start()
-        return s
+    # Special case for file object argument.
+    if mode =='r' and hasattr(fp, 'read'):
+
+        @contextmanager
+        def fp_reader(fp):
+            memfile = MemoryFile(fp.read())
+            dataset = memfile.open()
+            yield dataset
+            dataset.close()
+            memfile.close()
+
+        return fp_reader(fp)
+
+    elif mode == 'w' and hasattr(fp, 'write'):
+
+        @contextmanager
+        def fp_writer(fp):
+            memfile = MemoryFile()
+            dataset = memfile.open(driver=driver, width=width, height=height,
+                                   count=count, crs=crs, transform=transform,
+                                   dtype=dtype, nodata=nodata, **kwargs)
+            yield dataset
+            dataset.close()
+            memfile.seek(0)
+            fp.write(memfile.read())
+            memfile.close()
+
+        return fp_writer(fp)
+
+    else:
+        # The 'normal' filename path.
+        _, _, scheme = parse_path(fp)
+
+        with Env() as env:
+            # Get AWS credentials only if we're attempting to access a
+            # raster using the S3 scheme.
+            if scheme == 's3':
+                env.get_aws_credentials()
+                log.debug("AWS credentials have been obtained")
+
+            # Create dataset instances and pass the given env, which will
+            # be taken over by the dataset's context manager if it is not
+            # None.
+            if mode == 'r':
+                s = DatasetReader(fp)
+            elif mode == 'r-':
+                warnings.warn("'r-' mode is deprecated, use 'r'",
+                              DeprecationWarning)
+                s = DatasetReader(fp)
+            elif mode == 'r+':
+                s = get_writer_for_path(fp)(fp, mode)
+            elif mode == 'w':
+                s = get_writer_for_driver(driver)(fp, mode, driver=driver,
+                                                  width=width, height=height,
+                                                  count=count, crs=crs,
+                                                  transform=transform,
+                                                  dtype=dtype, nodata=nodata,
+                                                  **kwargs)
+            else:
+                raise ValueError(
+                    "mode must be one of 'r', 'r+', or 'w', not %s" % mode)
+            s.start()
+            return s
 
 
 @ensure_env


Test patch:
diff --git a/tests/test_memoryfile.py b/tests/test_memoryfile.py
--- a/tests/test_memoryfile.py
+++ b/tests/test_memoryfile.py
@@ -120,21 +120,24 @@ def test_read(tmpdir, rgb_file_bytes):
 
 
 @mingdalversion
-def test_rgb_file_object_example(rgb_file_object):
-    """An example of using a file object"""
-    with MemoryFile(rgb_file_object.read()) as memfile:
-        with memfile.open() as src:
-            assert src.driver == 'GTiff'
-            assert src.count == 3
-            assert src.dtypes == ('uint8', 'uint8', 'uint8')
-            assert src.read().shape == (3, 718, 791)
+def test_file_object_read(rgb_file_object):
+    """An example of reading from a file object"""
+    with rasterio.open(rgb_file_object) as src:
+        assert src.driver == 'GTiff'
+        assert src.count == 3
+        assert src.dtypes == ('uint8', 'uint8', 'uint8')
+        assert src.read().shape == (3, 718, 791)
 
 
 @mingdalversion
-@pytest.mark.xfail(reason="file object input is not implemented")
-def test_rgb_file_object(rgb_file_object):
-    """A failing example of using a file object"""
-    with rasterio.open(rgb_file_object) as src:
+def test_test_file_object_write(tmpdir, rgb_data_and_profile):
+    """An example of writing to a file object"""
+    data, profile = rgb_data_and_profile
+    with tmpdir.join('test.tif').open('wb') as fout:
+        with rasterio.open(fout, 'w', **profile) as dst:
+            dst.write(data)
+
+    with rasterio.open(str(tmpdir.join('test.tif'))) as src:
         assert src.driver == 'GTiff'
         assert src.count == 3
         assert src.dtypes == ('uint8', 'uint8', 'uint8')