Repository: rasterio/rasterio

Issue:
In-memory GDAL datasets, sqlite style
GDAL has a concept of in-memory datasets that you create by passing in a filename like `/vsimem/example.tif`. There's a per-process in-memory filesystem at `/vsimem` and a TIFF file in it at `/vsimem/example.tif`. For Rasterio's purposes it could be better to follow the example of sqlite and simply pass, say, `:memory:`. Rasterio could generate a UUID and an appropriate VSIMEM filename to pass to GDAL.

I'm thinking that this could be used to simply the existing code in rio-mbtiles

``` python
    with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
        reproject(rasterio.band(src, src.indexes),
                  rasterio.band(tmp, tmp.indexes))

    data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))

    # Workaround for https://bugs.python.org/issue23349.
    if sys.version_info[0] == 2 and sys.version_info[2] < 10:
        data[:] = data[-1:] + data[:-1]

    return tile, data
```

down to 

``` python
    with rasterio.open(':memory:', 'w', **kwds) as tmp:
        reproject(rasterio.band(src, src.indexes),
                  rasterio.band(tmp, tmp.indexes))
        data = tmp.to_bytes()

    return tile, data
```

Benefits:
- More appropriate abstraction for Rasterio, where `open()` is supposed to be the sole access to datasets.
- Prevents mixups on the vsimem filesystem that could be hard to recover from and debug.
- We can workaround the Python bug for all users within `to_bytes()`.



Discussion:
I've got an update. Here's an example in which you can see the current shape of this feature. A GeoTIFF is created, given imagery, and then put in an S3 bucket without using the OS filesystem:

``` python
import boto3

import rasterio
from rasterio.io import MemoryFile

# Get data and metadata from the Rasterio testing dataset in the OS
# filesystem. We'll use these below to make a new in-memory GeoTIFF.
with rasterio.open('tests/data/RGB.byte.tif') as src:
    data = src.read()
    profile = src.profile

# MemoryFile is like Python's BytesIO, but is backed by a file in
# GDAL's in-memory filesysem ("/vsimem/").
#
# Here we create an empty, writeable in-memory file.
memfile = MemoryFile()

# MemoryFile.open() returns a Rasterio dataset object. If the 
# in-memory file is initially empty, you can pass `driver`, `dtype`,
# `count`, &c parameters to initialize the dataset.
with memfile.open(**profile) as gtiff:
    gtiff.write(data)

# At this point, if you called `memfile.read()` you'd get all the
# bytes of that GeoTIFF, as if you read them from disk. You can 
# pass `memfile` to functions that expect a file-like object.
# boto3's `put_object()` reads 8k bytes at a time from input files,
# so you only have that amount of overhead in addition to the 
# in-memory file's allocation.
client = boto3.client('s3')
client.put_object(Bucket='my_bucket', Key='gtiffs/example.tif', Body=memfile)
```

`MemoryFile` would be immediately useful in rio-mbtiles, which creates PNGs or JPEGs in memory and saves them as blobs to sqlite. It's also designed to do what Fiona's [buffer_to_virtual_file](https://github.com/Toblerity/Fiona/blob/master/fiona/ogrext1.pyx#L1251) does: let you take bytes (maybe read from S3) and open them as a dataset without writing to disk.

I'm feeling pretty good about not creating any new abstractions here. I'm using the existing one of `BytesIO`, and the `open()` method comes from http://py.readthedocs.io/en/latest/path.html#py-path-local-local-file-system-path.


Resolving patch:
diff --git a/rasterio/io.py b/rasterio/io.py
--- a/rasterio/io.py
+++ b/rasterio/io.py
@@ -3,17 +3,23 @@
 Instances of these classes are called dataset objects.
 """
 
+import logging
 import math
+import uuid
 import warnings
 
 from rasterio._base import (
     get_dataset_driver, driver_can_create, driver_can_create_copy)
 from rasterio._io import (
-    DatasetReaderBase, DatasetWriterBase, BufferedDatasetWriterBase)
+    DatasetReaderBase, DatasetWriterBase, BufferedDatasetWriterBase,
+    MemoryFileBase)
 from rasterio import enums, windows
 from rasterio.transform import guard_transform, xy, rowcol
 
 
+log = logging.getLogger(__name__)
+
+
 class TransformMethodsMixin(object):
     """Mixin providing methods for calculations related
     to transforming between rows and columns of the raster
@@ -170,6 +176,7 @@ def __repr__(self):
         return "<{} DatasetReader name='{}' mode='{}'>".format(
             self.closed and 'closed' or 'open', self.name, self.mode)
 
+
 class DatasetWriter(DatasetWriterBase, WindowMethodsMixin,
                     TransformMethodsMixin):
     """An unbuffered data and metadata writer. Its methods write data
@@ -195,6 +202,68 @@ def __repr__(self):
             self.closed and 'closed' or 'open', self.name, self.mode)
 
 
+class MemoryFile(MemoryFileBase):
+    """A BytesIO-like object, backed by an in-memory file.
+
+    This allows formatted files to be read and written without I/O.
+
+    A MemoryFile created with initial bytes becomes immutable. A
+    MemoryFile created without initial bytes may be written to using
+    either file-like or dataset interfaces.
+
+    Examples
+    --------
+
+    A GeoTIFF can be loaded in memory and accessed using the GeoTIFF
+    format driver
+
+    >>> with open('tests/data/RGB.byte.tif', 'rb') as f, \
+    ...         MemoryFile(f.read()) as memfile:
+    ...     with memfile.open() as src:
+    ...         pprint.pprint(src.profile)
+    ...
+    {'count': 3,
+     'crs': CRS({'init': 'epsg:32618'}),
+     'driver': 'GTiff',
+     'dtype': 'uint8',
+     'height': 718,
+     'interleave': 'pixel',
+     'nodata': 0.0,
+     'tiled': False,
+     'transform': Affine(300.0379266750948, 0.0, 101985.0,
+           0.0, -300.041782729805, 2826915.0),
+     'width': 791}
+
+    """
+
+    def open(self, driver=None, width=None, height=None,
+             count=None, crs=None, transform=None, dtype=None, nodata=None,
+             **kwargs):
+        """Open the file and return a Rasterio dataset object.
+
+        If data has already been written, the file is opened in 'r+'
+        mode. Otherwise, the file is opened in 'w' mode.
+        """
+        if self.closed:
+            raise IOError("I/O operation on closed file.")
+        if self.exists():
+            s = get_writer_for_path(self.name)(self.name, 'r+')
+        else:
+            s = get_writer_for_driver(driver)(self.name, 'w', driver=driver,
+                                              width=width, height=height,
+                                              count=count, crs=crs,
+                                              transform=transform, dtype=dtype,
+                                              nodata=nodata, **kwargs)
+        s.start()
+        return s
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.close()
+
+
 def get_writer_for_driver(driver):
     """Return the writer class appropriate for the specified driver."""
     cls = None


Test patch:
diff --git a/tests/test_memoryfile.py b/tests/test_memoryfile.py
new file mode 100644
--- /dev/null
+++ b/tests/test_memoryfile.py
@@ -0,0 +1,114 @@
+"""MemoryFile tests"""
+
+import logging
+
+from packaging.version import parse
+import pytest
+
+import rasterio
+from rasterio.io import MemoryFile
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+# Custom markers.
+mingdalversion = pytest.mark.skipif(
+    parse(rasterio.__gdal_version__) < parse('2.0dev'),
+    reason="MemoryFile requires GDAL 2.0")
+
+
+@pytest.fixture(scope='function')
+def rgb_file_bytes(path_rgb_byte_tif):
+    return open(path_rgb_byte_tif, 'rb').read()
+
+
+@mingdalversion
+def test_initial_bytes(rgb_file_bytes):
+    """MemoryFile contents can initialized from bytes and opened."""
+    with MemoryFile(rgb_file_bytes) as memfile:
+        with memfile.open() as src:
+            assert src.driver == 'GTiff'
+            assert src.count == 3
+            assert src.dtypes == ('uint8', 'uint8', 'uint8')
+            assert src.read().shape == (3, 718, 791)
+
+
+@mingdalversion
+def test_non_initial_bytes(rgb_file_bytes):
+    """MemoryFile contents can be read from bytes and opened."""
+    with MemoryFile() as memfile:
+        assert memfile.write(rgb_file_bytes) == len(rgb_file_bytes)
+        with memfile.open() as src:
+            assert src.driver == 'GTiff'
+            assert src.count == 3
+            assert src.dtypes == ('uint8', 'uint8', 'uint8')
+            assert src.read().shape == (3, 718, 791)
+
+
+@mingdalversion
+def test_non_initial_bytes_in_two(rgb_file_bytes):
+    """MemoryFile contents can be read from bytes in two steps and opened."""
+    with MemoryFile() as memfile:
+        assert memfile.write(rgb_file_bytes[:10]) == 10
+        assert memfile.write(rgb_file_bytes[10:]) == len(rgb_file_bytes) - 10
+        with memfile.open() as src:
+            assert src.driver == 'GTiff'
+            assert src.count == 3
+            assert src.dtypes == ('uint8', 'uint8', 'uint8')
+            assert src.read().shape == (3, 718, 791)
+
+
+@mingdalversion
+def test_non_initial_bytearray(rgb_file_bytes):
+    """MemoryFile contents can be read from bytearray and opened."""
+    with MemoryFile() as memfile:
+        assert memfile.write(bytearray(rgb_file_bytes)) == len(rgb_file_bytes)
+        with memfile.open() as src:
+            assert src.driver == 'GTiff'
+            assert src.count == 3
+            assert src.dtypes == ('uint8', 'uint8', 'uint8')
+            assert src.read().shape == (3, 718, 791)
+
+
+@pytest.fixture(scope='function')
+def rgb_data_and_profile(path_rgb_byte_tif):
+    with rasterio.open(path_rgb_byte_tif) as src:
+        data = src.read()
+        profile = src.profile
+    return data, profile
+
+
+@mingdalversion
+def test_no_initial_bytes(rgb_data_and_profile):
+    """An empty MemoryFile can be opened and written into."""
+    data, profile = rgb_data_and_profile
+
+    with MemoryFile() as memfile:
+        with memfile.open(**profile) as dst:
+            dst.write(data)
+        view = memfile.getbuffer()
+        # Exact size of the in-memory GeoTIFF varies with GDAL
+        # version and configuration.
+        assert view.size > 1000000
+        data = bytearray(view)
+
+    with MemoryFile(data) as memfile:
+        with memfile.open() as src:
+            assert sorted(src.profile.items()) == sorted(profile.items())
+
+
+@mingdalversion
+def test_read(tmpdir, rgb_file_bytes):
+    """Reading from a MemoryFile works"""
+    with MemoryFile(rgb_file_bytes) as memfile:
+        tmptiff = tmpdir.join('test.tif')
+
+        while 1:
+            chunk = memfile.read(8192)
+            if not chunk:
+                break
+            tmptiff.write(chunk, 'ab')
+
+    with rasterio.open(str(tmptiff)) as src:
+        assert src.count == 3