Repository: bird-house/birdy

Issue:
Cryptic error message when trying to open non-existent netCDF link
## Description

When opening a netCDF link that does not exist, netcdf-c raises this error on stdout, which ends up in the response from birdy. 

## Environment

* Birdy version used, if any: 0.8.0

## Steps to Reproduce

If the site does not exist, we get
```python
>>> import xarray as xr
>>> xr.open_dataset("http://a.b.c/d.nc")
curl error details: 
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/.conda/envs/birdy/lib/python3.7/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    198             try:
--> 199                 file = self._cache[self._key]
    200             except KeyError:

~/.conda/envs/birdy/lib/python3.7/site-packages/xarray/backends/lru_cache.py in __getitem__(self, key)
     52         with self._lock:
---> 53             value = self._cache[key]
     54             self._cache.move_to_end(key)

KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('http://a.b.c/d.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False))]

During handling of the above exception, another exception occurred:
...
```

If the site exists but the link is not valid, we get
```python
syntax error, unexpected WORD_WORD, expecting SCAN_ATTR or SCAN_DATASET or SCAN_ERROR
context: <html^><head><title>404 Not Found</title></head><body bgcolor="white"><center><h1>404 Not Found</h1></center><hr><center>nginx/1.13.6</center></body></html>
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/.conda/envs/birdy/lib/python3.7/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    198             try:
--> 199                 file = self._cache[self._key]
    200             except KeyError:

~/.conda/envs/birdy/lib/python3.7/site-packages/xarray/backends/lru_cache.py in __getitem__(self, key)
     52         with self._lock:
---> 53             value = self._cache[key]
     54             self._cache.move_to_end(key)

KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('http://pavics.ouranos.ca/tmp/a.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False))]

During handling of the above exception, another exception occurred:
...
```


## Additional Information
https://github.com/Ouranosinc/raven/issues/411


Discussion:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 8f9368a..e5d68d4 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,14 @@
 Change History
 **************
 
+0.8.1 (unreleased)
+==================
+
+Changes:
+
+* Before trying to open a netCDF dataset, determine whether link is a valid OPeNDAP endpoint to avoid unnecessarily raising the cryptic ``syntax error, unexpected WORD_WORD, expecting SCAN_ATTR or SCAN_DATASET or SCAN_ERROR``.
+
+
 0.8.0 (2021-05-25)
 ==================
 
diff --git a/birdy/client/converters.py b/birdy/client/converters.py
index ed2ef0a..cb1752c 100644
--- a/birdy/client/converters.py
+++ b/birdy/client/converters.py
@@ -5,7 +5,7 @@ from distutils.version import StrictVersion
 from importlib import import_module
 from pathlib import Path
 from typing import Sequence, Union
-
+from birdy.utils import is_opendap_url
 from owslib.wps import Output
 
 from . import notebook as nb
@@ -174,12 +174,12 @@ class Netcdf4Converter(BaseConverter):  # noqa: D101
     def convert(self):  # noqa: D102
         import netCDF4
 
-        try:
-            # try OpenDAP url
+        # Try to access with OpenDAP url to avoid a download
+        if is_opendap_url(self.url):
             return netCDF4.Dataset(self.url)
-        except IOError:
-            # download the file
-            return netCDF4.Dataset(self.file)
+
+        # Download the file and open the local copy
+        return netCDF4.Dataset(self.file)
 
 
 class XarrayConverter(BaseConverter):  # noqa: D101
@@ -194,12 +194,12 @@ class XarrayConverter(BaseConverter):  # noqa: D101
     def convert(self):  # noqa: D102
         import xarray as xr
 
-        try:
-            # try OpenDAP url
+        # Try to access with OpenDAP url to avoid a download
+        if is_opendap_url(self.url):
             return xr.open_dataset(self.url)
-        except IOError:
-            # download the file
-            return xr.open_dataset(self.file)
+
+        # Download the file and open the local copy
+        return xr.open_dataset(self.file)
 
 
 # TODO: Add test for this.
diff --git a/birdy/utils.py b/birdy/utils.py
index 39cf0e5..06dde6f 100644
--- a/birdy/utils.py
+++ b/birdy/utils.py
@@ -43,6 +43,34 @@ def is_url(url):
         return True
 
 
+def is_opendap_url(url):
+    """
+    Check if a provided url is an OpenDAP url.
+
+    The DAP Standard specifies that a specific tag must be included in the
+    Content-Description header of every request. This tag is one of:
+        "dods-dds" | "dods-das" | "dods-data" | "dods-error"
+
+    So we can check if the header starts with `dods`.
+
+    Note that this might not work with every DAP server implementation.
+    """
+    import requests
+    from requests.exceptions import ConnectionError, InvalidSchema, MissingSchema
+
+    try:
+        content_description = requests.head(url, timeout=5).headers.get(
+            "Content-Description"
+        )
+    except (ConnectionError, MissingSchema, InvalidSchema):
+        return False
+
+    if content_description:
+        return content_description.lower().startswith("dods")
+    else:
+        return False
+
+
 def is_file(path):
     """Return True if `path` is a valid file."""
     if not path:


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 1aa827b..70112fd 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,5 +1,6 @@
 # noqa
 
+import pytest
 from pathlib import Path
 
 from birdy import utils
@@ -114,3 +115,31 @@ class TestGuessType:  # noqa: D101
             ["application/x-netcdf", "application/x-ogc-dods"],
         )
         assert mime == "application/x-ogc-dods"
+
+
+@pytest.mark.online
+def test_is_opendap_url():
+    # This test uses online requests, and the servers are not as stable as hoped.
+    # We should record these requests so that the tests don't break when the servers are down.
+
+    url = (
+        "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/"
+        "birdhouse/nrcan/nrcan_canada_daily_v2/tasmin/nrcan_canada_daily_tasmin_2017.nc"
+    )
+    assert utils.is_opendap_url(url)
+
+    url = url.replace("dodsC", "fileServer")
+    assert not utils.is_opendap_url(url)
+
+    # no Content-Description header
+    # url = "http://test.opendap.org/opendap/netcdf/examples/tos_O1_2001-2002.nc"
+    # assert is_opendap_url(url)
+
+    url = "invalid_schema://something"
+    assert not utils.is_opendap_url(url)
+
+    url = "https://www.example.com"
+    assert not utils.is_opendap_url(url)
+
+    url = "/missing_schema"
+    assert not utils.is_opendap_url(url)