Repository: Unidata/siphon

Issue:
Better Server Error Handling
When hitting a data server that is down, I receive a seemingly random assortment of inconsistent error messages each time I do so. For example, Java errors, even "too many open file" errors. It would be great to be given a single, consistent "Server temporarily unavailable" error so I know I'm doing nothing wrong!

I've attached an example of the "too many open files" error.
![capture](https://user-images.githubusercontent.com/31324211/38883097-93452324-4231-11e8-8a1d-00fcc490aa63.PNG)


Discussion:

Patch:
diff --git a/environment.yml b/environment.yml
index 2278a3d5..2249c72e 100755
--- a/environment.yml
+++ b/environment.yml
@@ -12,6 +12,7 @@ dependencies:
   - jupyter
   - sphinx
   - sphinx-gallery
+  - pillow
   - doc8
   - pytest
   - pytest-cov
diff --git a/examples/Basic_Usage.py b/examples/Basic_Usage.py
index 0d08be78..d86a8aa6 100644
--- a/examples/Basic_Usage.py
+++ b/examples/Basic_Usage.py
@@ -16,7 +16,14 @@ contained within.
 from __future__ import print_function
 
 from siphon.catalog import TDSCatalog
+from siphon.http_util import session_manager
 
 ###########################################
 cat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog.xml')
 print(list(cat.catalog_refs))
+
+###########################################
+# Basic HTTP authentication can also be used by using the HTTP session manager
+# and setting some default options for HTTP sessions
+session_manager.set_session_options(auth=('username', 'password'))
+cat = TDSCatalog('https://rda.ucar.edu/thredds/catalog.xml')
diff --git a/siphon/catalog.py b/siphon/catalog.py
index 2215241c..0ae912d7 100644
--- a/siphon/catalog.py
+++ b/siphon/catalog.py
@@ -18,7 +18,7 @@ except ImportError:
     # Python 3
     from urllib.parse import urljoin, urlparse
 
-from .http_util import create_http_session, urlopen
+from .http_util import session_manager
 from .metadata import TDSCatalogMetadata
 
 logging.basicConfig(level=logging.ERROR)
@@ -250,7 +250,7 @@ class TDSCatalog(object):
             The URL of a THREDDS client catalog
 
         """
-        session = create_http_session()
+        session = session_manager.create_session()
 
         # get catalog.xml file
         resp = session.get(catalog_url)
@@ -488,7 +488,7 @@ class Dataset(object):
         if catalog_url != '':
             resolver_base = catalog_url.split('catalog.xml')[0]
             resolver_url = resolver_base + self.url_path
-            resolver_xml = urlopen(resolver_url)
+            resolver_xml = session_manager.urlopen(resolver_url)
             tree = ET.parse(resolver_xml)
             root = tree.getroot()
             if 'name' in root.attrib:
@@ -682,7 +682,7 @@ class Dataset(object):
             from .ncss import NCSS
             provider = NCSS
         elif service == 'HTTPServer':
-            provider = urlopen
+            provider = session_manager.urlopen
         else:
             raise ValueError(service + ' is not an access method supported by Siphon')
 
diff --git a/siphon/http_util.py b/siphon/http_util.py
index fd1c0218..525d5830 100644
--- a/siphon/http_util.py
+++ b/siphon/http_util.py
@@ -18,8 +18,6 @@ import requests
 
 from . import __version__
 
-user_agent = 'Siphon ({})'.format(__version__)
-
 HTTPError = requests.HTTPError
 
 
@@ -49,49 +47,74 @@ class UTC(tzinfo):
 utc = UTC()
 
 
-def create_http_session():
-    """Create a new HTTP session with our user-agent set.
+class HTTPSessionManager(object):
+    """Manage the creation of sessions for HTTP access."""
 
-    Returns
-    -------
-    session : requests.Session
-        The created session
+    def __init__(self):
+        """Initialize ``HTTPSessionManager``."""
+        self.user_agent = 'Siphon ({})'.format(__version__)
+        self.options = {}
 
-    See Also
-    --------
-    urlopen
+    def set_session_options(self, **kwargs):
+        """Set options for created session instances.
 
-    """
-    ret = requests.Session()
-    ret.headers['User-Agent'] = user_agent
-    return ret
+        Takes keyword arguments and sets them as attributes on the returned
+        :class:`requests.Session` instance.
 
+        See Also
+        --------
+        create_session
 
-def urlopen(url, **kwargs):
-    """GET a file-like object for a URL using HTTP.
+        """
+        self.options = kwargs
 
-    This is a thin wrapper around :meth:`requests.Session.get` that returns a file-like object
-    wrapped around the resulting content.
+    def create_session(self):
+        """Create a new HTTP session with our user-agent set.
 
-    Parameters
-    ----------
-    url : str
-        The URL to request
+        Returns
+        -------
+        session : requests.Session
+            The created session
 
-    kwargs : arbitrary keyword arguments
-        Additional keyword arguments to pass to :meth:`requests.Session.get`.
+        See Also
+        --------
+        urlopen, set_session_options
 
-    Returns
-    -------
-    fobj : file-like object
-        A file-like interface to the content in the response
+        """
+        ret = requests.Session()
+        ret.headers['User-Agent'] = self.user_agent
+        for k, v in self.options.items():
+            setattr(ret, k, v)
+        return ret
 
-    See Also
-    --------
-    :meth:`requests.Session.get`
+    def urlopen(self, url, **kwargs):
+        """GET a file-like object for a URL using HTTP.
 
-    """
-    return BytesIO(create_http_session().get(url, **kwargs).content)
+        This is a thin wrapper around :meth:`requests.Session.get` that returns a file-like
+        object wrapped around the resulting content.
+
+        Parameters
+        ----------
+        url : str
+            The URL to request
+
+        kwargs : arbitrary keyword arguments
+            Additional keyword arguments to pass to :meth:`requests.Session.get`.
+
+        Returns
+        -------
+        fobj : file-like object
+            A file-like interface to the content in the response
+
+        See Also
+        --------
+        :meth:`requests.Session.get`
+
+        """
+        return BytesIO(self.create_session().get(url, **kwargs).content)
+
+
+session_manager = HTTPSessionManager()
 
 
 def parse_iso_date(s):
@@ -352,7 +375,7 @@ class HTTPEndPoint(object):
 
         """
         self._base = url
-        self._session = create_http_session()
+        self._session = session_manager.create_session()
         self._get_metadata()
 
     def get_query(self, query):
@@ -458,9 +481,10 @@ class HTTPEndPoint(object):
                 text = resp.reason
             else:
                 text = resp.text
-            raise requests.HTTPError('Error accessing {0}: {1:d} {2}'.format(resp.request.url,
-                                                                             resp.status_code,
-                                                                             text))
+            raise requests.HTTPError('Error accessing {0}\n'
+                                     'Server Error ({1:d}: {2})'.format(resp.request.url,
+                                                                        resp.status_code,
+                                                                        text))
         return resp
 
     def _get_metadata(self):
diff --git a/siphon/simplewebservice/acis.py b/siphon/simplewebservice/acis.py
index d4c4d67d..9b6ae607 100644
--- a/siphon/simplewebservice/acis.py
+++ b/siphon/simplewebservice/acis.py
@@ -5,7 +5,7 @@
 
 import requests
 
-from ..http_util import create_http_session
+from ..http_util import session_manager
 
 
 def acis_request(method, params):
@@ -51,7 +51,8 @@ def acis_request(method, params):
     timeout = 300 if method == 'MultiStnData' else 60
 
     try:
-        response = create_http_session().post(base_url + method, json=params, timeout=timeout)
+        response = session_manager.create_session().post(base_url + method, json=params,
+                                                         timeout=timeout)
         return response.json()
     except requests.exceptions.Timeout:
         raise AcisApiException('Connection Timeout')


Test patch:
diff --git a/siphon/cdmr/tests/test_ncstream.py b/siphon/cdmr/tests/test_ncstream.py
index 6e02264b..75206ea8 100644
--- a/siphon/cdmr/tests/test_ncstream.py
+++ b/siphon/cdmr/tests/test_ncstream.py
@@ -29,8 +29,8 @@ def get_test_latest_url(query=None):
 @recorder.use_cassette('latest_rap_ncstream_header')
 def get_header_remote():
     """Get a header from a remote data source."""
-    from siphon.http_util import urlopen
-    return urlopen(get_test_latest_url('req=header'))
+    from siphon.http_util import session_manager
+    return session_manager.urlopen(get_test_latest_url('req=header'))
 
 
 @pytest.mark.parametrize('src, result', [(b'\xb6\xe0\x02', 45110), (b'\x17\n\x0b', 23)])
diff --git a/siphon/tests/test_http_util.py b/siphon/tests/test_http_util.py
index 3eebf13d..460d5ab0 100644
--- a/siphon/tests/test_http_util.py
+++ b/siphon/tests/test_http_util.py
@@ -7,8 +7,8 @@ from datetime import datetime, timedelta
 
 import pytest
 
-from siphon.http_util import (create_http_session, DataQuery, HTTPEndPoint, HTTPError,
-                              parse_iso_date, urlopen, utc)
+from siphon.http_util import (DataQuery, HTTPEndPoint, HTTPError,
+                              parse_iso_date, session_manager, utc)
 import siphon.testing
 
 recorder = siphon.testing.get_recorder(__file__)
@@ -17,18 +17,30 @@ recorder = siphon.testing.get_recorder(__file__)
 @recorder.use_cassette('top_thredds_catalog')
 def test_urlopen():
     """Test siphon's urlopen wrapper."""
-    fobj = urlopen('http://thredds-test.unidata.ucar.edu/thredds/catalog.xml')
+    fobj = session_manager.urlopen('http://thredds-test.unidata.ucar.edu/thredds/catalog.xml')
     assert fobj.read(2) == b'<?'
 
 
 @recorder.use_cassette('top_thredds_catalog')
 def test_session():
-    """Test that https sessions contain the proper user agent."""
-    session = create_http_session()
+    """Test that http sessions contain the proper user agent."""
+    session = session_manager.create_session()
     resp = session.get('http://thredds-test.unidata.ucar.edu/thredds/catalog.xml')
     assert resp.request.headers['user-agent'].startswith('Siphon')
 
 
+@recorder.use_cassette('top_thredds_catalog')
+def test_session_options():
+    """Test that http sessions receive proper options."""
+    auth = ('foo', 'bar')
+    session_manager.set_session_options(auth=auth)
+    try:
+        session = session_manager.create_session()
+        assert session.auth == auth
+    finally:
+        session_manager.set_session_options()
+
+
 def test_parse_iso():
     """Test parsing ISO-formatted dates."""
     parsed = parse_iso_date('2015-06-15T12:00:00Z')
diff --git a/siphon/tests/test_ncss_dataset.py b/siphon/tests/test_ncss_dataset.py
index 1b7e3a4f..956f1c5d 100644
--- a/siphon/tests/test_ncss_dataset.py
+++ b/siphon/tests/test_ncss_dataset.py
@@ -6,7 +6,7 @@
 import logging
 import xml.etree.ElementTree as ET
 
-from siphon.http_util import urlopen
+from siphon.http_util import session_manager
 from siphon.ncss_dataset import _Types, NCSSDataset
 from siphon.testing import get_recorder
 
@@ -559,7 +559,7 @@ def test_dataset_elements_full_ncss_station():
     """Test parsing the dataset from a full ncss station page."""
     url = ('http://thredds.ucar.edu/thredds/ncss/nws/synoptic/'
            'ncdecoded/Surface_Synoptic_Point_Data_fc.cdmr/dataset.xml')
-    element = ET.fromstring(urlopen(url).read())
+    element = ET.fromstring(session_manager.urlopen(url).read())
     parsed = NCSSDataset(element)
     assert parsed
 
@@ -570,7 +570,7 @@ def test_dataset_elements_full_ncss_grid():
     url = ('http://thredds.ucar.edu/thredds/ncss/grib/NCEP/GFS/'
            'Global_0p5deg/GFS_Global_0p5deg_20150602_0000.grib2/'
            'dataset.xml')
-    element = ET.fromstring(urlopen(url).read())
+    element = ET.fromstring(session_manager.urlopen(url).read())
     parsed = NCSSDataset(element)
     assert parsed
 
@@ -580,6 +580,6 @@ def test_dataset_parsing_tds5(recwarn):
     """Test parsing the dataset from TDS 5."""
     url = ('http://thredds-test.unidata.ucar.edu/thredds/ncss/grid/casestudies/irma/model/'
            'gfs/GFS_Global_0p5deg_20170903_1200.grib2/dataset.xml')
-    element = ET.fromstring(urlopen(url).read())
+    element = ET.fromstring(session_manager.urlopen(url).read())
     NCSSDataset(element)
     assert len(recwarn) == 0