Repository: Unidata/siphon
Pull request #127 (2017-04-26 17:46:19)

Handle more data types when parsing dataset.xml
Before these changes, we only handled types `int` and `float` when parsing dataset.xml - everything else was left as a string and logged a warning. Even with this PR, we still do not handle:

 * Sequence
 * Structure
 * enum
 * opaque
 * object
 * char

because I do not have an example of those, and honestly have no idea what to do with some.

lesserwhirls: Failing on docs?
dopplershift: That should fix it, let's see...
jrleeman: ![](https://media.giphy.com/media/BtedgmzGNCiuk/giphy.gif)

Patch:
diff --git a/docs/conf.py b/docs/conf.py
index fd67c675..468d9fdb 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -60,7 +60,7 @@ sphinx_gallery_conf = {
     'examples_dirs': ['../examples'],
     'gallery_dirs': ['examples'],
     'filename_pattern': '/',
-    'mod_example_dir': 'api/generated'
+    'backreferences_dir': 'api/generated'
 }
 
 # Add any paths that contain templates here, relative to this directory.
diff --git a/siphon/catalog.py b/siphon/catalog.py
index 848db2bb..9500e6a8 100644
--- a/siphon/catalog.py
+++ b/siphon/catalog.py
@@ -369,7 +369,7 @@ class SimpleService(object):
         self.name = service_node.attrib['name']
         self.service_type = service_node.attrib['serviceType']
         self.base = service_node.attrib['base']
-        self.access_urls = dict()
+        self.access_urls = {}
 
 
 class CompoundService(object):
diff --git a/siphon/ncss.py b/siphon/ncss.py
index 3d4fb19a..c4875f2b 100644
--- a/siphon/ncss.py
+++ b/siphon/ncss.py
@@ -271,7 +271,7 @@ class ResponseRegistry(object):
     '''
 
     def __init__(self):
-        self._reg = dict()
+        self._reg = {}
 
     def register(self, mimetype):
         def dec(func):
@@ -300,7 +300,7 @@ def squish(l):
 def combine_dicts(l):
     r'Combine a list of dictionaries into single one'
 
-    ret = dict()
+    ret = {}
     for item in l:
         ret.update(item)
     return ret
@@ -314,8 +314,8 @@ def parse_xml(data, handle_units):
 
 
 def parse_xml_point(elem):
-    point = dict()
-    units = dict()
+    point = {}
+    units = {}
     for data in elem.findall('data'):
         name = data.get('name')
         unit = data.get('units')
@@ -326,7 +326,7 @@ def parse_xml_point(elem):
 
 
 def combine_xml_points(l, units, handle_units):
-    ret = dict()
+    ret = {}
     for item in l:
         for key, value in item.items():
             ret.setdefault(key, []).append(value)
@@ -341,7 +341,7 @@ def combine_xml_points(l, units, handle_units):
 def parse_xml_dataset(elem, handle_units):
     points, units = zip(*[parse_xml_point(p) for p in elem.findall('point')])
     # Group points by the contents of each point
-    datasets = dict()
+    datasets = {}
     for p in points:
         datasets.setdefault(tuple(p.keys()), []).append(p)
 
@@ -393,7 +393,7 @@ def parse_csv_response(data, unit_handler):
 
 
 def parse_csv_header(line):
-    units = dict()
+    units = {}
     names = []
     for var in line.split(','):
         start = var.find('[')
@@ -416,7 +416,7 @@ def parse_csv_dataset(data, handle_units):
     names, units = parse_csv_header(fobj.readline().decode('utf-8'))
     arrs = np.genfromtxt(fobj, dtype=None, names=names, delimiter=',', unpack=True,
                          converters={'date': lambda s: parse_iso_date(s.decode('utf-8'))})
-    d = dict()
+    d = {}
     for f in arrs.dtype.fields:
         dat = arrs[f]
         if dat.dtype == np.object:
diff --git a/siphon/ncss_dataset.py b/siphon/ncss_dataset.py
index 8d1082af..567e108c 100644
--- a/siphon/ncss_dataset.py
+++ b/siphon/ncss_dataset.py
@@ -15,27 +15,96 @@ import numpy as np
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.WARNING)
+log.addHandler(logging.StreamHandler())
 
 
 class _Types(object):
     @staticmethod
     def handle_typed_values(val, type_name, value_type):
-        if value_type == 'int':
+        r'''Translate typed values into the appropriate python object
+
+            Takes an element name, value, and type and returns a list
+            with the string value(s) properly converted to a python type.
+
+            TypedValues are handled in ucar.ma2.DataType in netcdfJava
+            in the DataType enum. Possibilities are:
+
+                "boolean"
+                "byte"
+                "char"
+                "short"
+                "int"
+                "long"
+                "float"
+                "double"
+                "Sequence"
+                "String"
+                "Structure"
+                "enum1"
+                "enum2"
+                "enum4"
+                "opaque"
+                "object"
+
+            All of these are values written as strings in the xml, so simply
+            applying int, float to the values will work in most cases (i.e.
+            the TDS encodes them as string values properly).
+
+            Examle XML element:
+
+            <attribute name="scale_factor" type="double" value="0.0010000000474974513"/>
+
+            Parameters
+            ----------
+            val : string
+                The string representation of the value attribute of the xml element
+
+            type_name : string
+                The string representation of the name attribute of the xml element
+
+            value_type : string
+                The string representation of the type attribute of the xml element
+
+            Returns
+            -------
+            val : list
+                A list containing the properly typed python values.
+            '''
+        if value_type in ['byte', 'short', 'int', 'long']:
             try:
                 val = val.split()
                 val = list(map(int, val))
             except ValueError:
-                logging.warning('Cannot convert %s to float.', val)
-
-        elif value_type == 'float':
+                log.warning('Cannot convert %s to int. Keeping type as str.', val)
+        elif value_type in ['float', 'double']:
             try:
                 val = val.split()
                 val = list(map(float, val))
             except ValueError:
-                logging.warning('Cannot convert %s to float', val)
+                log.warning('Cannot convert %s to float. Keeping type as str.', val)
+        elif value_type == 'boolean':
+            try:
+                # special case for boolean type
+                val = val.split()
+                # values must be either true or false
+                for potential_bool in val:
+                    if potential_bool not in ['true', 'false']:
+                        raise ValueError
+                val = [True if bool == 'true' else False for bool in val]
+            except ValueError:
+                msg = 'Cannot convert values %s to boolean.'
+                msg += ' Keeping type as str.'
+                log.warning(msg, val)
+        elif value_type == 'String':
+            # nothing special for String type
+            pass
         else:
-            logging.warning('%s type %s not understood.', type_name,
-                            value_type)
+            # possibilities - Sequence, Structure, enum, opaque, object,
+            # and char.
+            # Not sure how to handle these as I do not have an example
+            # of how they would show up in dataset.xml
+            log.warning('%s type %s not understood. Keeping as String.',
+                        type_name, value_type)
 
         if not isinstance(val, list):
             val = [val]
@@ -177,8 +246,8 @@ class NCSSDataset(object):
         Parameters
         ----------
         element : :class:`~xml.etree.ElementTree.Element`
-            An :class:`~xml.etree.ElementTree.Element` representing the top level node of an
-            NCSS dataset.xml doc
+            An :class:`~xml.etree.ElementTree.Element` representing the top level
+            node of an NCSS dataset.xml doc
         """
         self._types = _Types()
         self._types_methods = _Types.__dict__
@@ -220,21 +289,22 @@ class NCSSDataset(object):
             return getattr(self._types, handler_name)
         else:
             msg = 'cannot find handler for element {}'.format(handler_name)
-            logging.warning(msg)
+            log.warning(msg)
 
     def _parse_element(self, element):
         element_name = element.tag
 
-        parser = dict(gridSet=self._parse_gridset, axis=self._parse_axis,
-                      coordTransform=self._parse_coordTransform,
-                      LatLonBox=self._parse_LatLonBox, TimeSpan=self._parse_TimeSpan,
-                      AcceptList=self._parse_AcceptList,
-                      featureDataset=self._parse_featureDataset, variable=self._parse_variable)
+        parser = {'gridSet': self._parse_gridset, 'axis': self._parse_axis,
+                  'coordTransform': self._parse_coordTransform,
+                  'LatLonBox': self._parse_LatLonBox, 'TimeSpan': self._parse_TimeSpan,
+                  'AcceptList': self._parse_AcceptList,
+                  'featureDataset': self._parse_featureDataset,
+                  'variable': self._parse_variable}
 
         try:
             parser[element_name](element)
         except KeyError:
-            logging.warning('No parser found for element %s', element_name)
+            log.warning('No parser found for element %s', element_name)
 
     def _parse_gridset(self, element):
         element_name = element.tag
@@ -254,7 +324,7 @@ class NCSSDataset(object):
                 grid_set.setdefault(child_name, {})[grid_name] = tmp
                 self.variables[grid_name] = tmp
             else:
-                logging.warning('Unknown child in %s: %s', element_name, child_name)
+                log.warning('Unknown child in %s: %s', element_name, child_name)
                 grid_set[child.tag] = 'not handled by _parse_gridset'
 
         self.gridsets.update({gridset_name: grid_set})
@@ -340,7 +410,7 @@ class NCSSDataset(object):
                     self.accept_list.setdefault(request_type,
                                                 []).append(return_type)
             else:
-                logging.warning('Cannot have grid=%s and point=%s', grid, point)
+                log.warning('Cannot have grid=%s and point=%s', grid, point)
 
     def _parse_featureDataset(self, element):  # noqa
         handler = self._get_handler(element.tag)


Test patch:
diff --git a/siphon/tests/test_ncss_dataset.py b/siphon/tests/test_ncss_dataset.py
index ec0ba09c..b57acf9f 100644
--- a/siphon/tests/test_ncss_dataset.py
+++ b/siphon/tests/test_ncss_dataset.py
@@ -11,7 +11,6 @@ from siphon.testing import get_recorder
 
 log = logging.getLogger('siphon.ncss_dataset')
 log.setLevel(logging.WARNING)
-log.addHandler(logging.StreamHandler())
 
 recorder = get_recorder(__file__)
 
@@ -45,15 +44,97 @@ class TestSimpleTypes(object):
     def setup_class(cls):
         cls.types = _Types()
 
-    def test_attribute_1(self):
-        'Test parsing a string attribute'
-        xml = '<attribute name="long_name" value="Specified height level above ground"/>'
+    def test_attribute_byte(self):
+        'Test parsing a byte attribute'
+        xml = '<attribute name="missing_value" type="byte" value="1"/>'
         element = ET.fromstring(xml)
-        expected = {'long_name': 'Specified height level above ground'}
+        expected = {'missing_value': [1]}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_invalid_byte(self, capfd):
+        'Test parsing an invalid byte attribute'
+        xml = '<attribute name="missing_value" type="byte" value="a"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': ['a']}
+        actual = self.types.handle_attribute(element)
+        expected_message = 'Cannot convert [\'a\'] to int. Keeping type as str.'
+        assert expected_message in ''.join(capfd.readouterr())
+        assert expected == actual
+
+    def test_attribute_short(self):
+        'Test parsing a short attribute'
+        xml = '<attribute name="missing_value" type="short" value="-999"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': [-999]}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_invalid_short(self, capfd):
+        'Test parsing an invalid short attribute'
+        xml = '<attribute name="missing_value" type="short" value="a"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': ['a']}
+        actual = self.types.handle_attribute(element)
+        expected_message = 'Cannot convert [\'a\'] to int. Keeping type as str.'
+        assert expected_message in ''.join(capfd.readouterr())
+        assert expected == actual
+
+    def test_attribute_int(self):
+        'Test parsing an int attribute'
+        xml = '<attribute name="missing_value" type="int" value="-999"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': [-999]}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_invalid_int(self, capfd):
+        'Test parsing an invalid int attribute'
+        xml = '<attribute name="missing_value" type="int" value="a"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': ['a']}
+        actual = self.types.handle_attribute(element)
+        expected_message = 'Cannot convert [\'a\'] to int. Keeping type as str.'
+        assert expected_message in ''.join(capfd.readouterr())
+        assert expected == actual
+
+    def test_attribute_long(self):
+        'Test parsing a long attribute'
+        xml = '<attribute name="missing_value" type="long" value="-999"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': [-999]}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_invalid_long(self, capfd):
+        'Test parsing a invalid long attribute'
+        xml = '<attribute name="missing_value" type="long" value="a"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': ['a']}
+        actual = self.types.handle_attribute(element)
+        expected_message = 'Cannot convert [\'a\'] to int. Keeping type as str.'
+        assert expected_message in ''.join(capfd.readouterr())
+        assert expected == actual
+
+    def test_attribute_float(self):
+        'Test parsing a float value attribute'
+        xml = '<attribute name="missing_value" type="float" value="-999.0"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': [float(-999.0)]}
         actual = self.types.handle_attribute(element)
         assert expected == actual
 
-    def test_attribute_2(self):
+    def test_attribute_invalid_float(self, capfd):
+        'Test parsing an invalid float value attribute'
+        xml = '<attribute name="missing_value" type="float" value="a"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': ['a']}
+        actual = self.types.handle_attribute(element)
+        expected_message = 'Cannot convert [\'a\'] to float. Keeping type as str.'
+        assert expected_message in ''.join(capfd.readouterr())
+        assert expected == actual
+
+    def test_attribute_float_nan(self):
         'Test parsing a float nan attribute'
         import math
         xml = '<attribute name="missing_value" type="float" value="NaN"/>'
@@ -64,20 +145,77 @@ class TestSimpleTypes(object):
         assert(math.isnan(actual['missing_value'][0]))
         assert(math.isnan(expected['missing_value'][0]))
 
-    def test_attribute_3(self):
-        'Test parsing a float value attribute'
-        xml = '<attribute name="missing_value" type="float" value="-999"/>'
+    def test_attribute_double(self):
+        'Test parsing a double attribute'
+        xml = '<attribute name="missing_value" type="double" value="-999.0"/>'
         element = ET.fromstring(xml)
-        expected = {'missing_value': [float(-999)]}
+        expected = {'missing_value': [-999.0]}
         actual = self.types.handle_attribute(element)
         assert expected == actual
 
-    def test_attribute_4(self):
-        'Test parsing an int attribute'
-        xml = '<attribute name="missing_value" type="int" value="-999"/>'
+    def test_attribute_invalid_double(self, capfd):
+        'Test parsing an invalid double attribute'
+        xml = '<attribute name="missing_value" type="double" value="a"/>'
         element = ET.fromstring(xml)
-        expected = {'missing_value': [-999]}
+        expected = {'missing_value': ['a']}
+        actual = self.types.handle_attribute(element)
+        expected_message = 'Cannot convert [\'a\'] to float. Keeping type as str.'
+        assert expected_message in ''.join(capfd.readouterr())
+        assert expected == actual
+
+    def test_attribute_double_nan(self):
+        'Test parsing a double nan attribute'
+        import math
+        xml = '<attribute name="missing_value" type="double" value="NaN"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': [float('NaN')]}
+        actual = self.types.handle_attribute(element)
+        assert expected.keys() == actual.keys()
+        assert(math.isnan(actual['missing_value'][0]))
+        assert(math.isnan(expected['missing_value'][0]))
+
+    def test_attribute_string_implicit(self):
+        'Test parsing a string attribute'
+        xml = '<attribute name="long_name" value="Specified height level above ground"/>'
+        element = ET.fromstring(xml)
+        expected = {'long_name': 'Specified height level above ground'}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_string_explicit(self):
+        'Test parsing a string attribute'
+        xml = '<attribute name="long_name" type="String" ' \
+              'value="Specified height level above ground"/>'
+        element = ET.fromstring(xml)
+        expected = {'long_name': ['Specified height level above ground']}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_boolean_true(self):
+        'Test parsing a boolean attribute'
+        xml = '<attribute name="missing_value" type="boolean" value="true"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': [True]}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_boolean_false(self):
+        'Test parsing a boolean attribute'
+        xml = '<attribute name="missing_value" type="boolean" value="false"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': [False]}
+        actual = self.types.handle_attribute(element)
+        assert expected == actual
+
+    def test_attribute_boolean_invalid(self, capfd):
+        'Test parsing an invalid boolean attribute'
+        xml = '<attribute name="missing_value" type="boolean" value="a"/>'
+        element = ET.fromstring(xml)
+        expected = {'missing_value': ['a']}
         actual = self.types.handle_attribute(element)
+        expected_message = 'Cannot convert values [\'a\'] to boolean.'
+        expected_message += ' Keeping type as str.'
+        assert expected_message in ''.join(capfd.readouterr())
         assert expected == actual
 
     def test_value_1(self):