Repository: Unidata/siphon
Pull request #237 (2018-07-10 18:10:47)

add case-insensitive strings and dicts for service types
Use case-insensitive comparisons for service names to handle catalog xmls which define either "OPenDAP", "OPENDAP", or "opendap", for example.


Patch:
diff --git a/siphon/catalog.py b/siphon/catalog.py
index 70db5fc0..2215241c 100644
--- a/siphon/catalog.py
+++ b/siphon/catalog.py
@@ -135,6 +135,90 @@ class DatasetCollection(IndexableMapping):
     __repr__ = __str__
 
 
+def _try_lower(arg):
+    try:
+        arg = arg.lower()
+    except (TypeError, AttributeError, ValueError):
+        log.warning('Could not convert %s to lowercase.', arg)
+    return arg
+
+
+class CaseInsensitiveStr(str):
+    """Extend ``str`` to use case-insensitive comparison and lookup."""
+
+    def __init__(self, *args):
+        """Create str with a _lowered property."""
+        self._lowered = _try_lower(self)
+
+    def __hash__(self):
+        """Hash str using _lowered property."""
+        return str.__hash__(self._lowered)
+
+    def __eq__(self, other):
+        """Return true if other is case-insensitive equal to self."""
+        return str.__eq__(self._lowered, _try_lower(other))
+
+    def __gt__(self, other):
+        """Return true if other is case-insensitive greater than self."""
+        return str.__gt__(self._lowered, _try_lower(other))
+
+    def __ge__(self, other):
+        """Return true if other is case-insensitive greater than or equal to self."""
+        return str.__ge__(self._lowered, _try_lower(other))
+
+    def __lt__(self, other):
+        """Return true if other is case-insensitive less than self."""
+        return str.__lt__(self._lowered, _try_lower(other))
+
+    def __le__(self, other):
+        """Return true if other is case-insensitive less than or equal to to self."""
+        return str.__le__(self._lowered, _try_lower(other))
+
+    def __ne__(self, other):
+        """Return true if other is case-insensitive unequal to self."""
+        return str.__ne__(self._lowered, _try_lower(other))
+
+
+class CaseInsensitiveDict(dict):
+    """Extend ``dict`` to use a case-insensitive key set."""
+
+    def __init__(self, *args, **kwargs):
+        """Create a dict with a set of lowercase keys."""
+        super(CaseInsensitiveDict, self).__init__(*args, **kwargs)
+        self._keys_to_lower()
+
+    def __eq__(self, other):
+        """Return true if other is case-insensitive equal to self."""
+        return super(CaseInsensitiveDict, self).__eq__(CaseInsensitiveDict(other))
+
+    def __getitem__(self, key):
+        """Return value from case-insensitive lookup of ``key``."""
+        return super(CaseInsensitiveDict, self).__getitem__(CaseInsensitiveStr(key))
+
+    def __setitem__(self, key, value):
+        """Set value with lowercase ``key``."""
+        super(CaseInsensitiveDict, self).__setitem__(CaseInsensitiveStr(key), value)
+
+    def __delitem__(self, key):
+        """Delete value associated with case-insensitive lookup of ``key``."""
+        return super(CaseInsensitiveDict, self).__delitem__(CaseInsensitiveStr(key))
+
+    def __contains__(self, key):
+        """Return true if key set includes case-insensitive ``key``."""
+        return super(CaseInsensitiveDict, self).__contains__(CaseInsensitiveStr(key))
+
+    def pop(self, key, *args, **kwargs):
+        """Remove and return the value associated with case-insensitive ``key``."""
+        return super(CaseInsensitiveDict, self).pop(CaseInsensitiveStr(key))
+
+    def _keys_to_lower(self):
+        """Convert key set to lowercase."""
+        for k in list(self.keys()):
+            val = super(CaseInsensitiveDict, self).__getitem__(k)
+            super(CaseInsensitiveDict, self).__delitem__(k)
+            self.__setitem__(CaseInsensitiveStr(k), val)
+
+
 class TDSCatalog(object):
     """
     Parse information from a THREDDS Client Catalog.
@@ -221,7 +305,8 @@ class TDSCatalog(object):
             elif (tag_type == 'metadata') or (tag_type == ''):
                 self._process_metadata(child, tag_type)
             elif tag_type == 'service':
-                if child.attrib['serviceType'] != 'Compound':
+                if CaseInsensitiveStr(child.attrib['serviceType'])\
+                        != CaseInsensitiveStr('Compound'):
                     # we do not want to process single services if they
                     # are already contained within a compound service, so
                     # we need to skip over those cases.
@@ -348,14 +433,14 @@ class Dataset(object):
         The name of the :class:`Dataset` element
     url_path : str
         url to the accessible dataset
-    access_urls : dict[str, str]
+    access_urls : CaseInsensitiveDict[str, str]
         A dictionary of access urls whose keywords are the access service
         types defined in the catalog (for example, "OPENDAP", "NetcdfSubset",
         "WMS", etc.
 
     """
 
-    ncssServiceNames = ('NetcdfSubset', 'NetcdfServer')
+    ncssServiceNames = (CaseInsensitiveStr('NetcdfSubset'), CaseInsensitiveStr('NetcdfServer'))
 
     def __init__(self, element_node, catalog_url=''):
         """Initialize the Dataset object.
@@ -438,7 +523,7 @@ class Dataset(object):
             Metadata from the :class:`TDSCatalog`
 
         """
-        all_service_dict = {}
+        all_service_dict = CaseInsensitiveDict({})
         for service in all_services:
             all_service_dict[service.name] = service
             if isinstance(service, CompoundService):
@@ -447,7 +532,7 @@ class Dataset(object):
 
         service_name = metadata.get('serviceName', None)
 
-        access_urls = {}
+        access_urls = CaseInsensitiveDict({})
         server_url = _find_base_tds_url(catalog_url)
 
         # process access urls for datasets that reference top
@@ -531,7 +616,7 @@ class Dataset(object):
         if service is None:
             service = 'CdmRemote' if 'CdmRemote' in self.access_urls else 'OPENDAP'
 
-        if service not in ('CdmRemote', 'OPENDAP'):
+        if service not in (CaseInsensitiveStr('CdmRemote'), CaseInsensitiveStr('OPENDAP')):
             raise ValueError(service + ' is not a valid service for remote_access')
 
         return self.access_with_service(service)
@@ -583,6 +668,7 @@ class Dataset(object):
             An instance appropriate for communicating using ``service``.
 
         """
+        service = CaseInsensitiveStr(service)
         if service == 'CdmRemote':
             from .cdmr import Dataset as CDMRDataset
             provider = CDMRDataset
@@ -634,7 +720,7 @@ class SimpleService(object):
 
         """
         self.name = service_node.attrib['name']
-        self.service_type = service_node.attrib['serviceType']
+        self.service_type = CaseInsensitiveStr(service_node.attrib['serviceType'])
         self.base = service_node.attrib['base']
         self.access_urls = {}
 
@@ -668,7 +754,7 @@ class CompoundService(object):
 
         """
         self.name = service_node.attrib['name']
-        self.service_type = service_node.attrib['serviceType']
+        self.service_type = CaseInsensitiveStr(service_node.attrib['serviceType'])
         self.base = service_node.attrib['base']
         services = []
         subservices = 0


Test patch:
diff --git a/siphon/tests/fixtures/thredds-test-default-5-0 b/siphon/tests/fixtures/thredds-test-default-5-0
new file mode 100644
index 00000000..dcb87bc7
--- /dev/null
+++ b/siphon/tests/fixtures/thredds-test-default-5-0
@@ -0,0 +1,46 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [Siphon (0.7.0+28.g20c017a.dirty)]
+    method: GET
+    uri: http://localhost:8081/thredds/catalog/catalog.xml
+  response:
+    body: {string: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<catalog xmlns=\"http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0\"
+        xmlns:xlink=\"http://www.w3.org/1999/xlink\" name=\"THREDDS Server Default
+        Catalog : You must change this to fit your server!\" version=\"1.2\">\r\n
+        \ <service name=\"all\" serviceType=\"compound\" base=\"\">\r\n    <service
+        name=\"odap\" serviceType=\"OpenDAP\" base=\"/thredds/dodsC/\" />\r\n    <service
+        name=\"dap4\" serviceType=\"DAP4\" base=\"/thredds/dap4/\" />\r\n    <service
+        name=\"http\" serviceType=\"HTTPServer\" base=\"/thredds/fileServer/\" />\r\n
+        \   <service name=\"wcs\" serviceType=\"WCS\" base=\"/thredds/wcs/\" />\r\n
+        \   <service name=\"wms\" serviceType=\"WMS\" base=\"/thredds/wms/\" />\r\n
+        \   <service name=\"ncssGrid\" serviceType=\"NetcdfSubset\" base=\"/thredds/ncss/grid/\"
+        />\r\n    <service name=\"ncssPoint\" serviceType=\"NetcdfSubset\" base=\"/thredds/ncss/point/\"
+        />\r\n    <service name=\"cdmremote\" serviceType=\"CdmRemote\" base=\"/thredds/cdmremote/\"
+        />\r\n    <service name=\"cdmrFeature\" serviceType=\"CdmrFeature\" base=\"/thredds/cdmrfeature/grid/\"
+        />\r\n    <service name=\"iso\" serviceType=\"ISO\" base=\"/thredds/iso/\"
+        />\r\n    <service name=\"ncml\" serviceType=\"NCML\" base=\"/thredds/ncml/\"
+        />\r\n    <service name=\"uddc\" serviceType=\"UDDC\" base=\"/thredds/uddc/\"
+        />\r\n  </service>\r\n  <dataset name=\"Test Grid Dataset\" ID=\"testGrid\"
+        urlPath=\"test/crossSeamProjection.nc\">\r\n    <serviceName>all</serviceName>\r\n
+        \   <dataType>Grid</dataType>\r\n  </dataset>\r\n  <dataset name=\"Test Point
+        Dataset\" ID=\"testPoint\" urlPath=\"test/H.1.1.nc\">\r\n    <serviceName>all</serviceName>\r\n
+        \   <dataType>Point</dataType>\r\n  </dataset>\r\n  <dataset name=\"Test Station
+        Dataset\" ID=\"testStation\" urlPath=\"test/H.2.1.1.nc\">\r\n    <serviceName>all</serviceName>\r\n
+        \   <dataType>Point</dataType>\r\n  </dataset>\r\n  <catalogRef xlink:href=\"/thredds/catalog/testAll/catalog.xml\"
+        xlink:title=\"Test all files in a directory\" ID=\"testDatasetScan\" name=\"Test
+        all files in a directory\">\r\n    <property name=\"DatasetScan\" value=\"true\"
+        />\r\n    <metadata inherited=\"true\">\r\n      <serviceName>all</serviceName>\r\n
+        \   </metadata>\r\n  </catalogRef>\r\n  <catalogRef xlink:href=\"enhancedCatalog.xml\"
+        xlink:title=\"Test Enhanced Catalog\" name=\"Test Enhanced Catalog\" />\r\n</catalog>\r\n"}
+    headers:
+      Content-Language: [en-US]
+      Content-Type: [application/xml;charset=UTF-8]
+      Date: ['Tue, 10 Jul 2018 17:27:42 GMT']
+      Server: [Apache-Coyote/1.1]
+    status: {code: 200, message: OK}
+version: 1
diff --git a/siphon/tests/test_catalog.py b/siphon/tests/test_catalog.py
index 40988cdf..6d1170e1 100644
--- a/siphon/tests/test_catalog.py
+++ b/siphon/tests/test_catalog.py
@@ -43,6 +43,15 @@ def test_access():
     assert 'OPENDAP' in ds.access_urls
 
 
+@recorder.use_cassette('thredds-test-default-5-0')
+def test_access_default_catalog():
+    """Test case-insensitive parsing of access methods in default catalog."""
+    url = ('http://localhost:8081/thredds/catalog/catalog.xml')
+    cat = TDSCatalog(url)
+    ds = list(cat.datasets.values())[0]
+    assert 'OPENDAP' in ds.access_urls
+
+
 @recorder.use_cassette('top_level_20km_rap_catalog')
 def test_virtual_access():
     """Test access of virtual datasets."""
diff --git a/siphon/tests/test_catalog_access.py b/siphon/tests/test_catalog_access.py
index 6c01b347..fb3420cc 100644
--- a/siphon/tests/test_catalog_access.py
+++ b/siphon/tests/test_catalog_access.py
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 """Test dataset access method helpers."""
 
+
+import logging
 import os
 import tempfile
 
@@ -12,6 +14,8 @@ from siphon.catalog import TDSCatalog
 from siphon.ncss import NCSS
 from siphon.testing import get_recorder
 
+log = logging.getLogger('siphon.catalog')
+log.setLevel(logging.WARNING)
 
 recorder = get_recorder(__file__)
 
@@ -141,3 +145,39 @@ def test_dataset_no_handler(nids_url):
     with pytest.raises(ValueError) as err:
         cat.datasets[0].access_with_service('UDDC')
     assert 'is not an access method supported' in str(err.value)
+
+
+@recorder.use_cassette('cat_only_http')
+def test_case_insensitive_access(caplog):
+    """Test case-insensitive parsing of access methods in default catalog."""
+    url = ('http://thredds-test.unidata.ucar.edu/thredds/catalog/noaaport/text/'
+           'tropical/atlantic/hdob/catalog.xml')
+    cat = TDSCatalog(url)
+    access_name = list(cat.datasets[0].access_urls.keys())[0]
+    assert access_name == 'HTTPSERVER'  # test __eq__
+    assert not access_name != 'HTTPSERVER'  # test __eq__
+    assert access_name > 'a'  # test __gt__
+    assert access_name >= 'a'  # test __ge__
+    assert access_name < 'Z'  # test __lt__
+    assert access_name <= 'Z'  # test __le__
+    assert not access_name == 1  # test fail on _try_lower
+    assert 'Could not convert 1 to lowercase.' in caplog.text
+
+
+@recorder.use_cassette('cat_only_http')
+def test_manage_access_types_case_insensitive(caplog):
+    """Test case-insensitive parsing of access methods in default catalog."""
+    url = ('http://thredds-test.unidata.ucar.edu/thredds/catalog/noaaport/text/'
+           'tropical/atlantic/hdob/catalog.xml')
+    cat = TDSCatalog(url)
+    ds = cat.datasets[0]
+    wrong_case_key = 'HTTPSERVER'
+    test_string = 'test'
+    http_url = ('http://thredds-test.unidata.ucar.edu/thredds/fileServer/noaaport/'
+                'text/tropical/atlantic/hdob/High_density_obs_20170824.txt')
+    assert ds.access_urls == {'HTTPSERVER': http_url}  # test __eq__
+    assert ds.access_urls[wrong_case_key] == http_url  # test __getitem___
+    assert wrong_case_key in ds.access_urls  # test __contains__
+    ds.access_urls[wrong_case_key] = test_string
+    assert ds.access_urls[wrong_case_key] == test_string  # test __setitem__
+    assert ds.access_urls.pop(wrong_case_key) == test_string  # test __delitem__