Repository: Unidata/siphon
Pull request #108 (2016-09-07 01:44:08)

Fix for accessing THREDDS catalogs hosted at a Context Path other tha…
…n /thredds

I'm using siphon to access several THREDDS instances hosted at nonstandard context paths - i.e. without /thredds/ anywhere in the URL.

The /thredds/ context is hard coded in Siphon and this causes failures in connecting to these catalogs.  Basically, siphon takes the entire catalog URL and appends the service URL to the end. (I'm taking the siphon URLs and passing them to pydap)

This is  a simple fix that works for every non-standard server I've encountered thus far and passes your tests when run on Python 3.4. 

I've worked on the assumption that the service descriptions in the catalog should always include an absolute path, which includes the context path, making it safe to take everything from the URL  except the actual path (i.e. take scheme, host, username/password, port). Let me know if you see other cases and I can take a look.


Patch:
diff --git a/siphon/catalog.py b/siphon/catalog.py
index 6cb5019a..b82e5085 100644
--- a/siphon/catalog.py
+++ b/siphon/catalog.py
@@ -15,10 +15,10 @@ from .metadata import TDSCatalogMetadata
 from .http_util import create_http_session, urlopen
 
 try:
-    from urlparse import urljoin
+    from urlparse import urljoin, urlparse
 except ImportError:
     # Python 3
-    from urllib.parse import urljoin
+    from urllib.parse import urljoin, urlparse
 
 log = logging.getLogger(__name__)
 log.addHandler(logging.StreamHandler())  # Python 2.7 needs a handler set
@@ -56,7 +56,7 @@ class TDSCatalog(object):
         """
         # top level server url
         self.catalog_url = catalog_url
-        self.base_tds_url = catalog_url.split('/thredds/')[0]
+        self.base_tds_url = _find_base_tds_url(catalog_url)
 
         session = create_http_session()
 
@@ -277,7 +277,7 @@ class Dataset(object):
                 service_name = metadata["serviceName"]
 
         access_urls = {}
-        server_url = catalog_url.split('/thredds/')[0]
+        server_url = _find_base_tds_url(catalog_url)
 
         found_service = None
         if service_name:
@@ -369,6 +369,19 @@ class CompoundService(object):
         self.number_of_subservices = subservices
 
 
+def _find_base_tds_url(catalog_url):
+    """
+    Identify the base URL of the THREDDS server from the catalog URL.
+
+    Will retain URL scheme, host, port and username/password when present.
+    """
+    url_components = urlparse(catalog_url)
+    if url_components.path:
+        return catalog_url.split(url_components.path)[0]
+    else:
+        return catalog_url
+
+
 def _get_latest_cat(catalog_url):
     r"""
     Get the latest dataset catalog from the supplied top level dataset catalog


Test patch:
diff --git a/siphon/tests/fixtures/cat_non_standard_context_path b/siphon/tests/fixtures/cat_non_standard_context_path
new file mode 100644
index 00000000..fe147925
--- /dev/null
+++ b/siphon/tests/fixtures/cat_non_standard_context_path
@@ -0,0 +1,76 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [Siphon (0.4.0+17.gb19adc3.dirty)]
+    method: GET
+    uri: http://ereeftds.bom.gov.au/ereefs/tds/catalog/ereef/mwq/P1A/catalog.xml
+  response:
+    body: {string: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<catalog xmlns=\"\
+        http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0\" xmlns:xlink=\"\
+        http://www.w3.org/1999/xlink\" version=\"1.0.1\">\r\n  <service name=\"nongrid\"\
+        \ serviceType=\"Compound\" base=\"\">\r\n    <service name=\"ncdods\" serviceType=\"\
+        OPENDAP\" base=\"/ereefs/tds/dodsC/\" />\r\n    <service name=\"HTTPServer\"\
+        \ serviceType=\"HTTPServer\" base=\"/ereefs/tds/fileServer/\" />\r\n  </service>\r\
+        \n  <dataset name=\"P1A\" ID=\"mwq/P1A\">\r\n    <metadata inherited=\"true\"\
+        >\r\n      <serviceName>nongrid</serviceName>\r\n      <authority>au.gov.bom</authority>\r\
+        \n      <dataType>GRID</dataType>\r\n      <dataFormat>netCDF</dataFormat>\r\
+        \n      <documentation type=\"rights\">Registered research users only</documentation>\r\
+        \n    </metadata>\r\n    <dataset name=\"A20020101.P1A.ANN_MIM_RMP.nc\" ID=\"\
+        mwq/P1A/A20020101.P1A.ANN_MIM_RMP.nc\" urlPath=\"ereef/mwq/P1A/A20020101.P1A.ANN_MIM_RMP.nc\"\
+        >\r\n      <dataSize units=\"Mbytes\">94.89</dataSize>\r\n      <date type=\"\
+        modified\">2014-03-09T20:51:38Z</date>\r\n    </dataset>\r\n    <dataset name=\"\
+        A20030101.P1A.ANN_MIM_RMP.nc\" ID=\"mwq/P1A/A20030101.P1A.ANN_MIM_RMP.nc\"\
+        \ urlPath=\"ereef/mwq/P1A/A20030101.P1A.ANN_MIM_RMP.nc\">\r\n      <dataSize\
+        \ units=\"Mbytes\">97.71</dataSize>\r\n      <date type=\"modified\">2014-03-09T21:12:03Z</date>\r\
+        \n    </dataset>\r\n    <dataset name=\"A20040101.P1A.ANN_MIM_RMP.nc\" ID=\"\
+        mwq/P1A/A20040101.P1A.ANN_MIM_RMP.nc\" urlPath=\"ereef/mwq/P1A/A20040101.P1A.ANN_MIM_RMP.nc\"\
+        >\r\n      <dataSize units=\"Mbytes\">98.36</dataSize>\r\n      <date type=\"\
+        modified\">2014-03-09T12:39:31Z</date>\r\n    </dataset>\r\n    <dataset name=\"\
+        A20050101.P1A.ANN_MIM_RMP.nc\" ID=\"mwq/P1A/A20050101.P1A.ANN_MIM_RMP.nc\"\
+        \ urlPath=\"ereef/mwq/P1A/A20050101.P1A.ANN_MIM_RMP.nc\">\r\n      <dataSize\
+        \ units=\"Mbytes\">97.27</dataSize>\r\n      <date type=\"modified\">2014-03-09T12:39:52Z</date>\r\
+        \n    </dataset>\r\n    <dataset name=\"A20060101.P1A.ANN_MIM_RMP.nc\" ID=\"\
+        mwq/P1A/A20060101.P1A.ANN_MIM_RMP.nc\" urlPath=\"ereef/mwq/P1A/A20060101.P1A.ANN_MIM_RMP.nc\"\
+        >\r\n      <dataSize units=\"Mbytes\">98.44</dataSize>\r\n      <date type=\"\
+        modified\">2014-03-09T12:43:41Z</date>\r\n    </dataset>\r\n    <dataset name=\"\
+        A20070101.P1A.ANN_MIM_RMP.nc\" ID=\"mwq/P1A/A20070101.P1A.ANN_MIM_RMP.nc\"\
+        \ urlPath=\"ereef/mwq/P1A/A20070101.P1A.ANN_MIM_RMP.nc\">\r\n      <dataSize\
+        \ units=\"Mbytes\">97.62</dataSize>\r\n      <date type=\"modified\">2014-03-09T12:43:09Z</date>\r\
+        \n    </dataset>\r\n    <dataset name=\"A20080101.P1A.ANN_MIM_RMP.nc\" ID=\"\
+        mwq/P1A/A20080101.P1A.ANN_MIM_RMP.nc\" urlPath=\"ereef/mwq/P1A/A20080101.P1A.ANN_MIM_RMP.nc\"\
+        >\r\n      <dataSize units=\"Mbytes\">98.55</dataSize>\r\n      <date type=\"\
+        modified\">2014-03-09T12:44:09Z</date>\r\n    </dataset>\r\n    <dataset name=\"\
+        A20090101.P1A.ANN_MIM_RMP.nc\" ID=\"mwq/P1A/A20090101.P1A.ANN_MIM_RMP.nc\"\
+        \ urlPath=\"ereef/mwq/P1A/A20090101.P1A.ANN_MIM_RMP.nc\">\r\n      <dataSize\
+        \ units=\"Mbytes\">97.67</dataSize>\r\n      <date type=\"modified\">2014-03-09T12:44:10Z</date>\r\
+        \n    </dataset>\r\n    <dataset name=\"A20100101.P1A.ANN_MIM_RMP.nc\" ID=\"\
+        mwq/P1A/A20100101.P1A.ANN_MIM_RMP.nc\" urlPath=\"ereef/mwq/P1A/A20100101.P1A.ANN_MIM_RMP.nc\"\
+        >\r\n      <dataSize units=\"Mbytes\">97.26</dataSize>\r\n      <date type=\"\
+        modified\">2014-03-09T12:44:32Z</date>\r\n    </dataset>\r\n    <dataset name=\"\
+        A20110101.P1A.ANN_MIM_RMP.nc\" ID=\"mwq/P1A/A20110101.P1A.ANN_MIM_RMP.nc\"\
+        \ urlPath=\"ereef/mwq/P1A/A20110101.P1A.ANN_MIM_RMP.nc\">\r\n      <dataSize\
+        \ units=\"Mbytes\">98.59</dataSize>\r\n      <date type=\"modified\">2014-03-09T12:42:33Z</date>\r\
+        \n    </dataset>\r\n    <dataset name=\"A20120101.P1A.ANN_MIM_RMP.nc\" ID=\"\
+        mwq/P1A/A20120101.P1A.ANN_MIM_RMP.nc\" urlPath=\"ereef/mwq/P1A/A20120101.P1A.ANN_MIM_RMP.nc\"\
+        >\r\n      <dataSize units=\"Mbytes\">98.09</dataSize>\r\n      <date type=\"\
+        modified\">2014-03-09T12:40:15Z</date>\r\n    </dataset>\r\n    <dataset name=\"\
+        A20130101.P1A.ANN_MIM_RMP.nc\" ID=\"mwq/P1A/A20130101.P1A.ANN_MIM_RMP.nc\"\
+        \ urlPath=\"ereef/mwq/P1A/A20130101.P1A.ANN_MIM_RMP.nc\">\r\n      <dataSize\
+        \ units=\"Mbytes\">98.19</dataSize>\r\n      <date type=\"modified\">2014-03-09T12:42:53Z</date>\r\
+        \n    </dataset>\r\n    <dataset name=\"A20140101.316.P1A.ANN_MIM_RMP.nc\"\
+        \ ID=\"mwq/P1A/A20140101.316.P1A.ANN_MIM_RMP.nc\" urlPath=\"ereef/mwq/P1A/A20140101.316.P1A.ANN_MIM_RMP.nc\"\
+        >\r\n      <dataSize units=\"Mbytes\">100.6</dataSize>\r\n      <date type=\"\
+        modified\">2015-01-06T01:05:32Z</date>\r\n    </dataset>\r\n  </dataset>\r\
+        \n</catalog>\r\n"}
+    headers:
+      Connection: [close]
+      Content-Language: [en-US]
+      Content-Type: [application/xml;charset=UTF-8]
+      Date: ['Fri, 09 Sep 2016 00:24:21 GMT']
+      Server: [Apache-Coyote/1.1]
+    status: {code: 200, message: OK}
+version: 1
diff --git a/siphon/tests/test_catalog.py b/siphon/tests/test_catalog.py
index c5cac44e..2e205086 100644
--- a/siphon/tests/test_catalog.py
+++ b/siphon/tests/test_catalog.py
@@ -124,3 +124,14 @@ def test_catalog_ref_order():
     assert list(cat.catalog_refs) == ['Forecast Model Data', 'Forecast Products and Analyses',
                                       'Observation Data', 'Radar Data', 'Satellite Data',
                                       'Unidata case studies']
+
+
+@recorder.use_cassette('cat_non_standard_context_path')
+def test_non_standard_context_path():
+    'Test accessing TDS with non-standard Context Path'
+    url = 'http://ereeftds.bom.gov.au/ereefs/tds/catalog/ereef/mwq/P1A/catalog.xml'
+    cat = TDSCatalog(url)
+    ds = cat.datasets['A20020101.P1A.ANN_MIM_RMP.nc']
+    expected = ('http://ereeftds.bom.gov.au/ereefs/tds/dodsC/ereef/mwq/'
+                'P1A/A20020101.P1A.ANN_MIM_RMP.nc')
+    assert ds.access_urls['OPENDAP'] == expected