Repository: pydap/pydap

Issue:
Test Erddap responses
Some Erddap servers return invalid DDS responses when a deep object is returned. In that case, we should parse the DDS from the DODS response and extract the data using it.


Discussion:
laliberte: @jameshiebert do you give your thumbs up for this to be merged?
jameshiebert: Yeah, I like the way that you split out the utility functions. Makes the code more clear. In any case, it looks good from my perspective.
laliberte: Ok, I'll go ahead and merge!

Patch:
diff --git a/setup.py b/setup.py
index 0cfe3cc..d6134fa 100644
--- a/setup.py
+++ b/setup.py
@@ -44,6 +44,7 @@ tests_require = functions_extras + cas_extras + server_extras + [
 
 testing_extras = tests_require + [
     'nose',
+    'mock',
     'coverage',
     'requests'
 ]
diff --git a/src/pydap/handlers/dap.py b/src/pydap/handlers/dap.py
index d1b66c7..5cb9b4c 100644
--- a/src/pydap/handlers/dap.py
+++ b/src/pydap/handlers/dap.py
@@ -139,43 +139,9 @@ class BaseProxy(object):
         dds, data = r.body.split(b'\nData:\n', 1)
         dds = dds.decode(r.content_encoding or 'ascii')
 
-        if self.shape:
-            # skip size packing
-            if self.dtype.char in 'SU':
-                data = data[4:]
-            else:
-                data = data[8:]
-
-        # calculate array size
-        shape = tuple(
-            int(np.ceil((s.stop-s.start)/float(s.step))) for s in index)
-        size = int(np.prod(shape))
-
-        if self.dtype == np.byte:
-            return np.fromstring(data[:size], 'B').reshape(shape)
-        elif self.dtype.char in 'SU':
-            out = []
-            for word in range(size):
-                n = np.asscalar(np.fromstring(data[:4], '>I'))  # read length
-                data = data[4:]
-                out.append(data[:n])
-                data = data[n + (-n % 4):]
-            return np.array([text_type(x.decode('ascii'))
-                             for x in out], 'S').reshape(shape)
-        else:
-            try:
-                return np.fromstring(data, self.dtype).reshape(shape)
-            except ValueError as e:
-                if str(e) == 'total size of new array must be unchanged':
-                    # server-side failure.
-                    # it is expected that the user should be mindful of this:
-                    raise RuntimeError(
-                                ('variable {0} could not be properly '
-                                 'retrieved. To avoid this '
-                                 'error consider using open_url(..., '
-                                 'output_grid=False).').format(quote(self.id)))
-                else:
-                    raise
+        # Parse received dataset:
+        dataset = build_dataset(dds, data=data)
+        return dataset[self.id].data
 
     def __len__(self):
         return self.shape[0]
diff --git a/src/pydap/handlers/lib.py b/src/pydap/handlers/lib.py
index 438ac89..72222dc 100644
--- a/src/pydap/handlers/lib.py
+++ b/src/pydap/handlers/lib.py
@@ -196,12 +196,18 @@ def apply_selection(selection, dataset):
     return dataset
 
 
+def degenerate_grid_to_structure(candidate):
+    if isinstance(candidate, GridType):
+        candidate = StructureType(
+            candidate.name, candidate.attributes)
+    return candidate
+
+
 def apply_projection(projection, dataset):
     """Apply a given projection to a dataset.
 
     This function builds and returns a new dataset by adding those variables
     that were requested on the projection.
-
     """
     out = DatasetType(name=dataset.name, attributes=dataset.attributes)
 
@@ -218,9 +224,7 @@ def apply_projection(projection, dataset):
                         # if there are more children to add we need to clear
                         # the candidate so it has only explicitly added
                         # children; also, Grids are degenerated into Structures
-                        if isinstance(candidate, GridType):
-                            candidate = StructureType(
-                                candidate.name, candidate.attributes)
+                        candidate = degenerate_grid_to_structure(candidate)
                         candidate._keys = []
                     target[name] = candidate
                 target, template = target[name], template[name]
diff --git a/src/pydap/model.py b/src/pydap/model.py
index 83180d8..9c11b6e 100644
--- a/src/pydap/model.py
+++ b/src/pydap/model.py
@@ -322,8 +322,18 @@ class StructureType(DapType):
         item.id = '%s.%s' % (self.id, item.name)
 
     def __getitem__(self, key):
-        key = quote(key)
-        return self._dict[key]
+        try:
+            return self._dict[quote(key)]
+        except KeyError:
+            splitted = key.split('.')
+            if len(splitted) > 1:
+                try:
+                    return (self
+                            .__getitem__(splitted[0])['.'.join(splitted[1:])])
+                except KeyError:
+                    return self.__getitem__('.'.join(splitted[1:]))
+            else:
+                raise
 
     def __delitem__(self, key):
         self._dict.__delitem__(key)
diff --git a/src/pydap/parsers/dds.py b/src/pydap/parsers/dds.py
index 7c85d64..19560b4 100644
--- a/src/pydap/parsers/dds.py
+++ b/src/pydap/parsers/dds.py
@@ -3,6 +3,7 @@
 import re
 
 import numpy as np
+from six import text_type
 
 from . import SimpleParser
 from ..model import (DatasetType, BaseType,
@@ -10,7 +11,6 @@ from ..model import (DatasetType, BaseType,
                      GridType)
 from ..lib import quote, STRING
 
-
 typemap = {
     'byte':    np.dtype("B"),
     'int':     np.dtype(">i4"),
@@ -32,9 +32,10 @@ class DDSParser(SimpleParser):
 
     """A parser for the DDS."""
 
-    def __init__(self, dds):
+    def __init__(self, dds, data=None):
         super(DDSParser, self).__init__(dds, re.IGNORECASE)
         self.dds = dds
+        self.data = data
 
     def consume(self, regexp):
         """Consume and return a token."""
@@ -81,6 +82,9 @@ class DDSParser(SimpleParser):
         self.consume(';')
 
         data = DummyData(dtype, shape)
+        if self.data is not None:
+            data, self.data = convert_data_to_array(self.data, shape,
+                                                    dtype, name)
         var = BaseType(name, data, dimensions=dimensions)
 
         return var
@@ -156,12 +160,54 @@ class DDSParser(SimpleParser):
         return grid
 
 
-def build_dataset(dds):
+def build_dataset(dds, data=None):
     """Return a dataset object from a DDS representation."""
-    return DDSParser(dds).parse()
+    return DDSParser(dds, data=data).parse()
 
 
 class DummyData(object):
     def __init__(self, dtype, shape):
         self.dtype = dtype
         self.shape = shape
+
+
+def convert_data_to_array(data, shape, dtype, id):
+    if len(shape) > 0:
+        if dtype.char in 'SU':
+            data = data[4:]
+        else:
+            data = data[8:]
+
+    # calculate array size
+    size = 1
+    if len(shape) > 0:
+        size = int(np.prod(shape))
+
+    nitems = dtype.itemsize * size
+
+    if dtype == np.byte:
+        return np.fromstring(data[:nitems], 'B').reshape(shape), data[nitems:]
+    elif dtype.char in 'SU':
+        out = []
+        for word in range(size):
+            n = np.asscalar(np.fromstring(data[:4], '>I'))  # read itemsize
+            data = data[4:]
+            out.append(data[:n])
+            data = data[n + (-n % 4):]
+        return np.array([text_type(x.decode('ascii'))
+                         for x in out], 'S').reshape(shape), data
+    else:
+        try:
+            return (np.fromstring(data[:nitems], dtype).reshape(shape),
+                    data[nitems:])
+        except ValueError as e:
+            if str(e) == 'total size of new array must be unchanged':
+                # server-side failure.
+                # it is expected that the user should be mindful of this:
+                raise RuntimeError(
+                            ('variable {0} could not be properly '
+                             'retrieved. To avoid this '
+                             'error consider using open_url(..., '
+                             'output_grid=False).').format(quote(id)))
+            else:
+                raise


Test patch:
diff --git a/src/pydap/tests/test_handlers_dap.py b/src/pydap/tests/test_handlers_dap.py
index f6cfe31..34abe45 100644
--- a/src/pydap/tests/test_handlers_dap.py
+++ b/src/pydap/tests/test_handlers_dap.py
@@ -8,6 +8,10 @@ from pydap.tests.datasets import (
     SimpleSequence, SimpleGrid, SimpleArray, VerySimpleSequence)
 
 import unittest
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
 
 
 class TestDapHandler(unittest.TestCase):
@@ -50,6 +54,67 @@ class TestDapHandler(unittest.TestCase):
         self.assertEqual(
             dataset.SimpleGrid.SimpleGrid.data.slice,
             (slice(None), slice(None)))
+        self.assertEqual(
+                repr(dataset.SimpleGrid[:]),
+                "<GridType with array 'SimpleGrid' and maps 'x', 'y'>")
+
+    def test_grid_erddap(self):
+        """Test that dataset has the correct data proxies for grids
+           with the ERDDAP behavior."""
+        with patch('pydap.handlers.lib.degenerate_grid_to_structure',
+                   side_effect=(lambda x: x)) as mock_degenerate:
+            dataset = DAPHandler("http://localhost:8001/", self.app1).dataset
+            self.assertEqual(
+                    repr(dataset.SimpleGrid[:]),
+                    "<GridType with array 'SimpleGrid' and maps 'x', 'y'>")
+            assert mock_degenerate.called
+
+    def test_grid_output_grid_false(self):
+        """Test that dataset has the correct data proxies for grids with
+           option output_grid set to False."""
+        dataset = DAPHandler("http://localhost:8001/", self.app1,
+                             output_grid=False).dataset
+
+        self.assertEqual(dataset.keys(), ["SimpleGrid", "x", "y"])
+        self.assertEqual(
+            dataset.SimpleGrid.keys(), ["SimpleGrid", "x", "y"])
+
+        # test one of the axis
+        self.assertIsInstance(dataset.SimpleGrid.x.data, BaseProxy)
+        self.assertEqual(
+            dataset.SimpleGrid.x.data.baseurl, "http://localhost:8001/")
+        self.assertEqual(dataset.SimpleGrid.x.data.id, "SimpleGrid.x")
+        self.assertEqual(dataset.SimpleGrid.x.data.dtype, np.dtype('>i4'))
+        self.assertEqual(dataset.SimpleGrid.x.data.shape, (3,))
+        self.assertEqual(
+            dataset.SimpleGrid.x.data.slice, (slice(None),))
+
+        # test the grid
+        self.assertIsInstance(dataset.SimpleGrid.SimpleGrid.data, BaseProxy)
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.baseurl,
+            "http://localhost:8001/")
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.id, "SimpleGrid.SimpleGrid")
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.dtype, np.dtype('>i4'))
+        self.assertEqual(dataset.SimpleGrid.SimpleGrid.data.shape, (2, 3))
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.slice,
+            (slice(None), slice(None)))
+        np.testing.assert_array_equal(dataset.SimpleGrid[:],
+                                      [[0, 1, 2], [3, 4, 5]])
+
+    def test_grid_erddap_output_grid_false(self):
+        """Test that dataset has the correct data proxies for grids with
+           option output_grid set to False and with the ERDDAP behavior."""
+        with patch('pydap.handlers.lib.degenerate_grid_to_structure',
+                   side_effect=(lambda x: x)) as mock_degenerate:
+            dataset = DAPHandler("http://localhost:8001/", self.app1,
+                                 output_grid=False).dataset
+            np.testing.assert_array_equal(dataset.SimpleGrid[:],
+                                          [[0, 1, 2], [3, 4, 5]])
+            assert mock_degenerate.called
 
     def test_grid_with_projection(self):
         """Test that a sliced proxy can be created for grids."""