Repository: rasterio/rasterio

Issue:
Make rasterio.io a public module
Continued from #629 

@sgillies, re: the design of `rasterio.open`

> I was aiming for a design like https://docs.python.org/2/library/io.html where the underlying classes come to you as the result of calling a more generic open() function and are almost never instantiated otherwise.

The `open` function always returns an instance of `IOBase`. The `rasterio.open` function always returns an instance of `DatasetReader`. Neither IOBase nor DatasetReader have public constructors and are always subclassed.

The difference between rasterio and the stdlib is that the IO classes are public whereas the rasterio classes are "private" by convention of the leading underscore in `rasterio._io`

What about bringing the `io` module forward as public? Implications of this?



Discussion:
@perrygeo @brendan-ward I'm going to deal with this as part of my sprint to better document dataset objects. Brendan and I discussed importing the Python classes in _io.pyx into a new io.py, leaving all the C functions behind the curtain. An upside of this is that contributors who don't know Cython would be able to help with io.py module docstrings and new functions.

@perrygeo @sgillies Decision point: do we want these classes to be entirely implemented in Cython, and simply surfaced publicly in Python - similar to the Python `io` module

OR

do we want to implement those things that need to be in Cython, and extend from these to add in the Python methods in the Python implementation - similar to what was recently done in the `CRS` class [here](https://github.com/mapbox/rasterio/blob/master/rasterio/crs.py#L8)

I have a preferece for the later; Cython code responsible for GDAL interactions and the Python code defining the public interface, properties and methods which don't require C.

The downside to this approach is that the code gets split into two files which might be slightly confusing.

The upside is that we gain test coverage, sphinx autodoc-ability, line profiling, the ability to modify without recompiling and a clean logical separation between the C/GDAL pieces of the code and the code that deals exclusively in python data structures.

I'm not firm on that preference, just my initial take.

@perrygeo @brendan-ward I'm leaning towards base classes in C, public API classes in Python. This doesn't lower the bar for developers as much as I'd like because we don't currently have a way to install the base classes (from a binary wheel, say) while working on the Python classes in develop mode.


Resolving patch:
diff --git a/rasterio/__init__.py b/rasterio/__init__.py
--- a/rasterio/__init__.py
+++ b/rasterio/__init__.py
@@ -21,6 +21,8 @@ def emit(self, record):
 from rasterio.env import ensure_env, Env
 from rasterio.errors import RasterioIOError
 from rasterio.compat import string_types
+from rasterio.io import (
+    DatasetReader, get_writer_for_path, get_writer_for_driver)
 from rasterio.profiles import default_gtiff_profile
 from rasterio.transform import Affine, guard_transform
 from rasterio.vfs import parse_path
@@ -181,20 +183,18 @@ def open(path, mode='r', driver=None, width=None, height=None,
     # be taken over by the dataset's context manager if it is not
     # None.
     if mode == 'r':
-        from rasterio._io import RasterReader
-        s = RasterReader(path)
-    elif mode == 'r+':
-        from rasterio._io import writer
-        s = writer(path, mode)
+        s = DatasetReader(path)
     elif mode == 'r-':
-        from rasterio._base import DatasetReader
+        warnings.warn("'r-' mode is deprecated, use 'r'", DeprecationWarning)
         s = DatasetReader(path)
+    elif mode == 'r+':
+        s = get_writer_for_path(path)(path, mode)
     elif mode == 'w':
-        from rasterio._io import writer
-        s = writer(path, mode, driver=driver,
-                   width=width, height=height, count=count,
-                   crs=crs, transform=transform, dtype=dtype,
-                   nodata=nodata, **kwargs)
+        s = get_writer_for_driver(driver)(path, mode, driver=driver,
+                                          width=width, height=height,
+                                          count=count, crs=crs,
+                                          transform=transform, dtype=dtype,
+                                          nodata=nodata, **kwargs)
     else:
         raise ValueError(
             "mode string must be one of 'r', 'r+', or 'w', not %s" % mode)
diff --git a/rasterio/io.py b/rasterio/io.py
new file mode 100644
--- /dev/null
+++ b/rasterio/io.py
@@ -0,0 +1,56 @@
+"""Classes capable of reading and writing datasets
+
+Instances of these classes are called dataset objects.
+"""
+
+from rasterio._base import (
+    get_dataset_driver, driver_can_create, driver_can_create_copy)
+from rasterio._io import (
+    DatasetReaderBase, DatasetWriterBase, BufferedDatasetWriterBase)
+
+
+class DatasetReader(DatasetReaderBase):
+    """An unbuffered data and metadata reader"""
+
+    def __repr__(self):
+        return "<{} DatasetReader name='{}' mode='{}'>".format(
+            self.closed and 'closed' or 'open', self.name, self.mode)
+
+
+class DatasetWriter(DatasetWriterBase):
+    """An unbuffered data and metadata writer. Its methods write data
+    directly to disk.
+    """
+
+    def __repr__(self):
+        return "<{} DatasetWriter name='{}' mode='{}'>".format(
+            self.closed and 'closed' or 'open', self.name, self.mode)
+
+
+class BufferedDatasetWriter(BufferedDatasetWriterBase):
+    """Maintains data and metadata in a buffer, writing to disk or
+    network only when `close()` is called.
+
+    This allows incremental updates to datasets using formats that don't
+    otherwise support updates, such as JPEG.
+    """
+
+    def __repr__(self):
+        return "<{} BufferedDatasetWriter name='{}' mode='{}'>".format(
+            self.closed and 'closed' or 'open', self.name, self.mode)
+
+
+def get_writer_for_driver(driver):
+    """Return the writer class appropriate for the specified driver."""
+    cls = None
+    if driver_can_create(driver):
+        cls = DatasetWriter
+    elif driver_can_create_copy(driver):  # pragma: no branch
+        cls = BufferedDatasetWriter
+    return cls
+
+
+def get_writer_for_path(path):
+    """Return the writer class appropriate for the existing dataset."""
+    driver = get_dataset_driver(path)
+    return get_writer_for_driver(driver)
diff --git a/rasterio/plot.py b/rasterio/plot.py
--- a/rasterio/plot.py
+++ b/rasterio/plot.py
@@ -13,7 +13,7 @@
 import numpy as np
 
 import rasterio
-from rasterio._io import RasterReader
+from rasterio.io import DatasetReader
 from rasterio.transform import guard_transform
 from rasterio.compat import zip_longest
 
@@ -79,7 +79,7 @@ def show(source, with_bounds=True, contour=False, contour_label_kws=None,
         arr = source[0].read(source[1])
         if with_bounds:
             kwargs['extent'] = plotting_extent(source[0])
-    elif isinstance(source, RasterReader):
+    elif isinstance(source, DatasetReader):
         if source.count == 1:
             arr = source.read(1, masked=True)
         else:
@@ -196,7 +196,7 @@ def show_hist(source, bins=10, masked=True, title='Histogram', ax=None, **kwargs
 
     Parameters
     ----------
-    source : np.array or RasterReader, rasterio.Band or tuple(dataset, bidx)
+    source : np.array or DatasetReader, rasterio.Band or tuple(dataset, bidx)
         Input data to display.  The first three arrays in multi-dimensional
         arrays are plotted as red, green, and blue.
     bins : int, optional
@@ -214,7 +214,7 @@ def show_hist(source, bins=10, masked=True, title='Histogram', ax=None, **kwargs
     """
     plt = get_plt()
 
-    if isinstance(source, RasterReader):
+    if isinstance(source, DatasetReader):
         arr = source.read(masked=masked)
     elif isinstance(source, (tuple, rasterio.Band)):
         arr = source[0].read(source[1], masked=masked)


Test patch:
diff --git a/tests/test_read.py b/tests/test_read.py
--- a/tests/test_read.py
+++ b/tests/test_read.py
@@ -47,7 +47,7 @@ def test_context(self):
             self.assertEqual(s.meta['crs'], s.crs)
             self.assertEqual(
                 repr(s),
-                "<open RasterReader name='tests/data/RGB.byte.tif' "
+                "<open DatasetReader name='tests/data/RGB.byte.tif' "
                 "mode='r'>")
         self.assertEqual(s.closed, True)
         self.assertEqual(s.count, 3)
@@ -64,7 +64,7 @@ def test_context(self):
              0, 0, 1.0))
         self.assertEqual(
             repr(s),
-            "<closed RasterReader name='tests/data/RGB.byte.tif' "
+            "<closed DatasetReader name='tests/data/RGB.byte.tif' "
             "mode='r'>")
 
     def test_derived_spatial(self):
diff --git a/tests/test_write.py b/tests/test_write.py
--- a/tests/test_write.py
+++ b/tests/test_write.py
@@ -8,7 +8,9 @@
 
 import rasterio
 from rasterio.drivers import blacklist
-from rasterio.errors import RasterioIOError
+from rasterio.env import Env
+from rasterio.errors import RasterioIOError, DriverRegistrationError
+from rasterio._base import driver_supports_mode
 
 
 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
@@ -60,13 +62,13 @@ def test_context(tmpdir):
         assert s.height == 100
         assert s.shape == (100, 100)
         assert s.indexes == (1,)
-        assert repr(s) == "<open RasterUpdater name='%s' mode='w'>" % name
+        assert repr(s) == "<open DatasetWriter name='%s' mode='w'>" % name
     assert s.closed
     assert s.count == 1
     assert s.width == 100
     assert s.height == 100
     assert s.shape == (100, 100)
-    assert repr(s) == "<closed RasterUpdater name='%s' mode='w'>" % name
+    assert repr(s) == "<closed DatasetWriter name='%s' mode='w'>" % name
     info = subprocess.check_output(["gdalinfo", name]).decode('utf-8')
     assert "GTiff" in info
     assert "Size is 100, 100" in info
@@ -278,6 +280,11 @@ def test_write_noncontiguous(tmpdir):
 
 @pytest.mark.parametrize("driver", list(blacklist.keys()))
 def test_write_blacklist(tmpdir, driver):
+
+    # Skip if we don't have driver support built in.
+    if driver not in Env().drivers():
+        pytest.skip()
+
     name = str(tmpdir.join("data.test"))
     with pytest.raises(RasterioIOError) as exc_info:
         rasterio.open(name, 'w', driver=driver, width=100, height=100,