Repository: pydap/pydap

Issue:
modernize pydap's http client
<!-- Thank you very much for your input! -->

### Description
pydap currently uses Webob for both the server and client. Webob is fine, but pydap could do better both on the server and client. The server and client aspect of pydap are largely decoupled (although we need to spend some time to make sure of this), and so we can make different choices for server and client. For the server the clear choice is FastAPI. However, there are a couple of choices for the client:

* Requests
* httpx
* aiohttp
* others?

We need to spend some time to consider the best alternative for pydap. In particular we should consider:

1. Async support
2. Speed
3. Stable dev
4. Ease of use
5.Authentication 


Other aspects to consider? Here are some basic resources
1. [Best Python HTTP Clients for Web Scraping in 2024](https://medium.com/@datajournal/best-python-http-clients-for-web-scraping-c09f7a356f29)
2. [Python Requests Alternatives](https://www.zenrows.com/alternative/python-requests)
3. [The best python http clients](https://www.scrapingbee.com/blog/best-python-http-clients/)




Discussion:
Mikejmnez: > Do those tests simulate HTTP access?

Yes indeed. You can create a redirect which is nice, and different types of errors. I used it to test that request was indeed following such redirect. 

Patch:
diff --git a/src/pydap/cas/urs.py b/src/pydap/cas/urs.py
index c223db2..ac7fb58 100644
--- a/src/pydap/cas/urs.py
+++ b/src/pydap/cas/urs.py
@@ -1,33 +1,19 @@
-from . import get_cookies
+from warnings import warn
 
 
 def setup_session(username, password, check_url=None, session=None, verify=True):
     """
-    A special call to get_cookies.setup_session that is tailored for
-    URS EARTHDATA at NASA credentials.
-
-    Parameters:
-    ----------
-    username: str
-    password: str
-    check_url: str, None (default)
-    session: `requests.session`, None (default)
-    verify: bool, True (default)
-
-    Example Data Access and Authentification in:
-    see `https://opendap.github.io/documentation/tutorials/`
-
+    This function is deprecated and will be removed in version 1.0.0.
+    Please use new_function instead.
     """
-
-    if session is not None:
-        # URS connections cannot be kept alive at the moment.
-        session.headers.update({"Connection": "close"})
-    session = get_cookies.setup_session(
-        "https://urs.earthdata.nasa.gov",
-        username=username,
-        password=password,
-        session=session,
-        check_url=check_url,
-        verify=verify,
+    warn(
+        "`urs.setup_session` is deprecated and will be removed in the next release. "
+        "Instead, use a `.netrc` file instead for your EDL authentication credentials."
+        "With the `.netrc file`, authentication with URS EARTHDATA at NASA credentials"
+        "is now handled automatically by requests.session object. "
+        "See: \n"
+        "https://opendap.github.io/documentation/tutorials/ClientAuthentication.html",
+        DeprecationWarning,
+        stacklevel=2,
     )
-    return session
+    pass
diff --git a/src/pydap/client.py b/src/pydap/client.py
index fc3f435..5bf5ae8 100644
--- a/src/pydap/client.py
+++ b/src/pydap/client.py
@@ -173,7 +173,6 @@ def open_dods_url(
     """Open a `.dods` response directly, returning a dataset."""
 
     r = pydap.net.GET(url, application, session, timeout=timeout)
-    pydap.net.raise_for_status(r)
 
     dds, data = r.body.split(b"\nData:\n", 1)
     dds = dds.decode(r.content_encoding or "ascii")
@@ -187,7 +186,7 @@ def open_dods_url(
             (scheme, netloc, path[:-4] + "das", params, query, fragment)
         )
         r = pydap.net.GET(dasurl, application, session, timeout=timeout, verify=verify)
-        pydap.net.raise_for_status(r)
+
         das = pydap.parsers.das.parse_das(r.text)
         pydap.parsers.das.add_attributes(dataset, das)
 
diff --git a/src/pydap/handlers/dap.py b/src/pydap/handlers/dap.py
index 8855739..b5d5f57 100644
--- a/src/pydap/handlers/dap.py
+++ b/src/pydap/handlers/dap.py
@@ -23,8 +23,9 @@ from io import BufferedReader, BytesIO
 from itertools import chain
 
 import numpy
+import requests
 from requests.utils import urlparse, urlunparse
-from webob.response import Response
+from webob.response import Response as webob_Response
 
 from pydap.handlers.lib import BaseHandler, ConstraintExpression, IterData
 from pydap.lib import (
@@ -41,7 +42,7 @@ from pydap.lib import (
     walk,
 )
 from pydap.model import BaseType, GridType, SequenceType, StructureType
-from pydap.net import GET, raise_for_status
+from pydap.net import GET
 from pydap.parsers import parse_ce
 from pydap.parsers.das import add_attributes, parse_das
 from pydap.parsers.dds import dds_to_dataset
@@ -155,7 +156,6 @@ class DAPHandler(BaseHandler):
             timeout=self.timeout,
             verify=self.verify,
         )
-        raise_for_status(r)
         dmr = safe_charset_text(r, self.user_charset)
         self.dataset = dmr_to_dataset(dmr)
 
@@ -178,7 +178,7 @@ class DAPHandler(BaseHandler):
             timeout=self.timeout,
             verify=self.verify,
         )
-        raise_for_status(r)
+
         dds = safe_charset_text(r, self.user_charset)
         self.dataset = dds_to_dataset(dds)
 
@@ -201,7 +201,6 @@ class DAPHandler(BaseHandler):
             timeout=self.timeout,
             verify=self.verify,
         )
-        raise_for_status(r)
         das = safe_charset_text(r, self.user_charset)
         add_attributes(self.dataset, parse_das(das))
 
@@ -294,24 +293,36 @@ def get_charset(r, user_charset):
 
 
 def safe_charset_text(r, user_charset):
-    if r.content_encoding == "gzip":
-        return (
-            gzip.GzipFile(fileobj=BytesIO(r.body))
-            .read()
-            .decode(get_charset(r, user_charset))
-        )
-    else:
-        r.charset = get_charset(r, user_charset)
-        return r.text
+    if isinstance(r, webob_Response):
+        if r.content_encoding == "gzip":
+            return (
+                gzip.GzipFile(fileobj=BytesIO(r.body))
+                .read()
+                .decode(get_charset(r, user_charset))
+            )
+        else:
+            r.charset = get_charset(r, user_charset)
+    return r.text
 
 
 def safe_dds_and_data(r, user_charset):
-    if r.content_encoding == "gzip":
-        raw = gzip.GzipFile(fileobj=BytesIO(r.body)).read()
-    else:
-        raw = r.body
-    dds, data = raw.split(b"\nData:\n", 1)
-    return dds.decode(get_charset(r, user_charset)), data
+    """
+    Takes the raw response of a dap2 request and splits it into the dds and data.
+    If the response is gzipped, it is decompressed first.
+    """
+    dds, data = None, None  # initialize
+    if isinstance(r, webob_Response):
+        if r.content_encoding == "gzip":
+            raw = gzip.GzipFile(fileobj=BytesIO(r.body)).read()
+        else:
+            raw = r.body
+        _dds, data = raw.split(b"\nData:\n", 1)
+        dds = _dds.decode(get_charset(r, user_charset))
+    elif isinstance(r, requests.Response):
+        raw = r.content
+        _dds, data = raw.split(b"\nData:\n", 1)
+        dds = _dds.decode(user_charset)
+    return dds, data
 
 
 class BaseProxyDap2(object):
@@ -376,7 +387,6 @@ class BaseProxyDap2(object):
             verify=self.verify,
         )
 
-        raise_for_status(r)
         dds, data = safe_dds_and_data(r, self.user_charset)
 
         # Parse received dataset:
@@ -459,7 +469,6 @@ class BaseProxyDap4(BaseProxyDap2):
             verify=self.verify,
         )
 
-        raise_for_status(r)
         dataset = UNPACKDAP4DATA(r, self.user_charset).dataset
         self.checksum = dataset[self.id].attributes["checksum"]
         self.data = dataset[self.id].data
@@ -580,11 +589,13 @@ class SequenceProxy(object):
             timeout=self.timeout,
             verify=self.verify,
         )
-        raise_for_status(r)
 
-        i = r.app_iter
-        if not hasattr(i, "__next__"):
-            i = iter(i)
+        if isinstance(r, webob_Response):
+            i = r.app_iter
+            if not hasattr(i, "__next__"):
+                i = iter(i)
+        elif isinstance(r, requests.Response):
+            i = r.iter_content()
 
         # Fast forward past the DDS header
         # the pattern could span chunk boundaries though so make sure to check
@@ -865,14 +876,15 @@ class UNPACKDAP4DATA(object):
     Parameters:
     -----------
         r: contains the dap response.
-            May be a Webob.response.Response created from pydap.net.GET if the dataset
-            is remote (from a url), or a `io.BufferedReader` if the data is local within
-            a filesystem. See `pydap.net.get.open_dap_file`
+            May be a Webob.response.Response or requests.response created from
+            pydap.net.GET if the dataset is remote (from a url), or a
+            `io.BufferedReader` if the data is local within a filesystem.
+            See `pydap.net.get.open_dap_file`
     """
 
     def __init__(self, r, user_charset="ascii"):
         self.user_charset = user_charset
-        if isinstance(r, Response):  # a Webob response
+        if isinstance(r, webob_Response):  # a Webob response
             self.r = r
             if self.r.content_encoding == "gzip":
                 self.raw = BytesReader(
@@ -882,8 +894,12 @@ class UNPACKDAP4DATA(object):
                 self.raw = BytesReader(r.body)
         elif isinstance(r, BufferedReader):
             # r comes from reading a local file
-            self.r = Response()  # make empty response
+            self.r = webob_Response()  # make empty response
             self.raw = BytesReader(r.read())
+        elif isinstance(r, requests.Response):
+            # r comes from reading a remote dataset
+            self.r = r
+            self.raw = BytesReader(r.content)
         else:
             raise TypeError(
                 """
@@ -909,7 +925,12 @@ class UNPACKDAP4DATA(object):
         chunk_header = numpy.frombuffer(self.raw.read(4), dtype=">u4")[0]
         dmr_length = chunk_header & 0x00FFFFFF
         chunk_type = (chunk_header >> 24) & 0xFF
-        dmr = self.raw.read(dmr_length).decode(get_charset(self.r, self.user_charset))
+        if isinstance(self.r, webob_Response):
+            dmr = self.raw.read(dmr_length).decode(
+                get_charset(self.r, self.user_charset)
+            )
+        else:
+            dmr = self.raw.read(dmr_length).decode(self.user_charset)
         data = self.raw.data
         # get endianness from first chunk
         _, _, endianness = decode_chunktype(chunk_type)
diff --git a/src/pydap/net.py b/src/pydap/net.py
index b9c6721..dce2cfb 100644
--- a/src/pydap/net.py
+++ b/src/pydap/net.py
@@ -1,85 +1,66 @@
 import ssl
-from contextlib import closing
 
 import requests
-from requests.exceptions import InvalidSchema, MissingSchema, Timeout
+from requests.adapters import HTTPAdapter
+from requests.exceptions import HTTPError
 from requests.utils import urlparse, urlunparse
-from webob.exc import HTTPError
-from webob.request import Request
+from urllib3 import Retry
+from webob.request import Request as webob_Request
 
 from .lib import DEFAULT_TIMEOUT, _quote
 
 
-def GET(url, application=None, session=None, timeout=DEFAULT_TIMEOUT, verify=True):
-    """Open a remote URL returning a webob.response.Response object
-
-    Optional parameters:
-    session: a requests.Session() object (potentially) containing
-             authentication cookies.
-
-    Optionally open a URL to a local WSGI application
+def GET(
+    url,
+    application=None,
+    session=None,
+    timeout=DEFAULT_TIMEOUT,
+    verify=True,
+    **kwargs,
+):
+    """Open a remote URL returning a requests.GET object
+
+    Parameters:
+    -----------
+        url: str
+            open a remote URL
+        application: a WSGI application object | None
+            When set, we are dealing with a local application, and a webob.response is
+            returned. When None, a requests.Response object is returned.
+        session: requests.Session() | None
+            object (potentially) containing authentication cookies.
+        timeout: int | None (default: 512)
+            timeout in seconds.
+        verify: bool (default: True)
+            verify SSL certificates
+        kwargs: dict
+            additional keyword arguments passed to `requests.get`. Optional arguments
+            include `retry_args` and `session_kwargs`, which are only passed to the
+            session object iff session is None.
+
+    Returns:
+        response: requests.Response object | webob.Response object
     """
     if application:
         _, _, path, _, query, fragment = urlparse(url)
         url = urlunparse(("", "", path, "", _quote(query), fragment))
 
-    response = follow_redirect(
-        url, application=application, session=session, timeout=timeout, verify=verify
+    res = create_request(
+        url,
+        application=application,
+        session=session,
+        timeout=timeout,
+        verify=verify,
+        **kwargs,
     )
-    # Decode request response (i.e. gzip)
-    response.decode_content()
-    return response
-
-
-def raise_for_status(response):
-    # Raise error if status is above 300:
-    if response.status_code >= 400:
-        raise HTTPError(
-            detail=response.status + "\n" + response.text,
-            headers=response.headers,
-            comment=response.body,
-        )
-    elif response.status_code >= 300:
-        try:
-            text = response.text
-        except AttributeError:
-            # With this status_code, response.text could
-            # be ill-defined. If the redirect does not set
-            # an encoding (i.e. response.charset is None).
-            # Set the text to empty string:
-            text = ""
-        raise HTTPError(
-            detail=(
-                response.status
-                + "\n"
-                + text
-                + "\n"
-                + "This is redirect error. These should not usually raise "
-                + "an error in pydap beacuse redirects are handled "
-                + "implicitly. If it failed it is likely due to a "
-                + "circular redirect."
-            ),
-            headers=response.headers,
-            comment=response.body,
-        )
-
-
-def follow_redirect(
-    url, application=None, session=None, timeout=DEFAULT_TIMEOUT, verify=True
-):
-    """
-    This function essentially performs the following command:
-    >>> Request.blank(url).get_response(application)  # doctest: +SKIP
-
-    It however makes sure that the request possesses the same cookies and
-    headers as the passed session.
-    """
+    if isinstance(res, webob_Request):
+        res = get_response(res, application, verify=verify)
+        # requests library automatically decodes the response
+        res.decode_content()
+    return res
 
-    req = create_request(url, session=session, timeout=timeout, verify=verify)
-    return get_response(req, application, verify=verify)
 
-
-def get_response(req, application, verify=True):
+def get_response(req, application=None, verify=True):
     """
     If verify=False, use the ssl library to temporarily disable
     ssl verification.
@@ -110,53 +91,74 @@ def get_response(req, application, verify=True):
     return resp
 
 
-def create_request(url, session=None, timeout=DEFAULT_TIMEOUT, verify=True):
-    if session is not None:
-        # If session is set and cookies were loaded using pydap.cas.get_cookies
-        # using the check_url option, then we can legitimately expect that
-        # the connection will go through seamlessly. However, there might be
-        # redirects that might want to modify the cookies. Webob is not
-        # really up to the task here. The approach used here is to
-        # piggy back on the requests library and use it to fetch the
-        # head of the requested url. Requests will follow redirects and
-        # adjust the cookies as needed. We can then use the final url and
-        # the final cookies to set up a webob Request object that will
-        # be guaranteed to have all the needed credentials:
-        return create_request_from_session(url, session, timeout=timeout, verify=verify)
-    else:
-        # If a session object was not passed, we simply pass a new
-        # requests.Session() object. The requests library allows the
-        # handling of redirects that are not naturally handled by Webob.
-        return create_request_from_session(
-            url, requests.Session(), timeout=timeout, verify=verify
-        )
-
+def create_request(
+    url,
+    application=None,
+    session=None,
+    timeout=DEFAULT_TIMEOUT,
+    verify=True,
+    **kwargs,
+):
+    """
+    Creates a requests.get request object for a local or remote url.
+    If application is set, then we are dealing with a local application
+    and we need to create a webob request object. Otherwise, we
+    are dealing with a remote url and we need to create a requests
+    request object.
+
+    Parameters:
+    -----------
+        url: str
+            open a remote URL
+        application: a WSGI application object | None
+            When set, we are dealing with a local application, and a webob.response is
+            returned. When None, a requests.Response object is returned.
+        session: requests.Session() | None
+        timeout: int | None (default: 512)
+            timeout in seconds.
+        verify: bool (default: True)
+            verify SSL certificates
+        kwargs: dict
+            additional keyword arguments passed to `requests.get`. Optional arguments
+            include `retry_args` and `session_kwargs`, which are only passed to the
+            session object iff session is None.
+    """
 
-def create_request_from_session(url, session, timeout=DEFAULT_TIMEOUT, verify=True):
-    try:
-        # Use session to follow redirects:
-        with closing(
-            session.head(url, allow_redirects=True, timeout=timeout, verify=verify)
-        ) as head:
-            req = Request.blank(head.url)
-            req.environ["webob.client.timeout"] = timeout
-
-            # Get cookies from head:
-            cookies_dict = head.cookies.get_dict()
-
-            # Set request cookies to the head cookies:
-            req.headers["Cookie"] = ",".join(
-                name + "=" + cookies_dict[name] for name in cookies_dict
-            )
-            # Set the headers to the session headers:
-            for item in head.request.headers:
-                req.headers[item] = head.request.headers[item]
-            return req
-    except (MissingSchema, InvalidSchema):
-        # Missing schema can occur in tests when the url
-        # is not pointing to any resource. Simply pass.
-        req = Request.blank(url)
+    if application:
+        # local dataset, webob request.
+        req = webob_Request.blank(url)
         req.environ["webob.client.timeout"] = timeout
         return req
-    except Timeout:
-        raise HTTPError("Timeout")
+    else:
+        session_kwargs = kwargs.pop(
+            "session_kwargs", {}
+        )  # Extract session-specific kwargs
+        if session is None:
+            # Create a new session with user-specified kwargs
+            session = requests.Session()
+            for key, value in session_kwargs.items():
+                setattr(session, key, value)
+
+            # Handle retry arguments separately
+            retry_args = kwargs.pop("retry_args", {})
+            retry_args.setdefault("total", 5)
+            retry_args.setdefault("status_forcelist", [500, 502, 503, 504])
+            retry_args.setdefault("backoff_factor", 0.1)
+            retry_args.setdefault("allowed_methods", ["GET"])
+
+            retries = Retry(**retry_args)
+            adapter = HTTPAdapter(max_retries=retries)
+
+            # Mount the adapter to the session
+            session.mount("http://", adapter)
+            session.mount("https://", adapter)
+        req = session.get(
+            url, timeout=timeout, verify=verify, allow_redirects=True, **kwargs
+        )
+        try:
+            req.raise_for_status()
+            return req
+        except HTTPError as http_err:
+            raise HTTPError(
+                f"HTTP Error occurred {http_err} - Failed to fetch data from `{url}`"
+            ) from http_err
diff --git a/src/pydap/server/devel.py b/src/pydap/server/devel.py
index 08e4bbb..29f35a2 100644
--- a/src/pydap/server/devel.py
+++ b/src/pydap/server/devel.py
@@ -7,7 +7,7 @@ import time
 from wsgiref.simple_server import make_server
 
 import numpy as np
-from webob.exc import HTTPError
+from requests.exceptions import HTTPError
 from webob.request import Request
 
 from ..handlers.lib import BaseHandler
@@ -60,7 +60,7 @@ class LocalTestServer(object):
 
     As an instance:
     >>> with LocalTestServer(application) as server:
-    ...     dataset = open_url("http://localhost:%s" % server.port)
+    ...     dataset = open_url("http://localhost:%s" % server.port, protocol='dap2')
     ...     dataset
     ...     print(dataset['byte'].data[:])
     ...     print(dataset['string'].data[:])
@@ -73,7 +73,7 @@ class LocalTestServer(object):
     Or by managing connection and deconnection:
     >>> server = LocalTestServer(application)
     >>> server.start()
-    >>> dataset = open_url("http://localhost:%s" % server.port)
+    >>> dataset = open_url("http://localhost:%s" % server.port, protocol='dap2')
     >>> dataset
     <DatasetType with children 'byte', 'string', 'short'>
     >>> print(dataset['byte'].data[:])
@@ -137,7 +137,7 @@ class LocalTestServer(object):
                 # When checking whether server has started, do
                 # not verify ssl:
                 resp = get_response(
-                    Request.blank(self.url + ".dds"), None, verify=False
+                    Request.blank(self.url + ".dds"), self.application, verify=False
                 )
                 ok = resp.status_code == 200
             except HTTPError:
diff --git a/src/pydap/server/devel_ssl.py b/src/pydap/server/devel_ssl.py
index 337ef69..fc9039b 100644
--- a/src/pydap/server/devel_ssl.py
+++ b/src/pydap/server/devel_ssl.py
@@ -1,9 +1,7 @@
 import multiprocessing
 import sys
 import time
-import warnings
 
-import requests
 from werkzeug.serving import run_simple
 
 from ..handlers.lib import BaseHandler
@@ -46,6 +44,7 @@ class LocalTestServerSSL(LocalTestServer):
 
     Usage:
     >>> import numpy as np
+    >>> import time
     >>> from pydap.handlers.lib import BaseHandler
     >>> from pydap.model import DatasetType, BaseType
     >>> DefaultDataset = DatasetType("Default")
@@ -59,12 +58,14 @@ class LocalTestServerSSL(LocalTestServer):
     >>> from pydap.client import open_url
     >>> application = BaseHandler(DefaultDataset)
     >>> with LocalTestServerSSL(application) as server:
+    ...     time.sleep(0.1)
     ...     dataset = open_url("http://localhost:%s" % server.port)
 
 
     Or by managing connection and deconnection:
     >>> server = LocalTestServerSSL(application)
     >>> server.start()
+    >>> time.sleep(0.1)
     >>> dataset = open_url("http://localhost:%s" % server.port)
     >>> dataset
     <DatasetType with children 'byte', 'string', 'short'>
@@ -119,13 +120,6 @@ class LocalTestServerSSL(LocalTestServer):
     # https://werkzeug.palletsprojects.com/en/2.2.x/serving/#shutting-down-the-server
     def shutdown(self):
         # Shutdown the server:
-        url = "http://0.0.0.0:%s/shutdown"
-        if self._ssl_context is not None:
-            url = url.replace("http", "https")
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            requests.head(url % self.port, verify=False)
         time.sleep(self._wait)
-        # self._server.join()
         self._server.terminate()
         del self._server


Test patch:
diff --git a/src/pydap/tests/test_net.py b/src/pydap/tests/test_net.py
index e59337e..fcc4cfa 100644
--- a/src/pydap/tests/test_net.py
+++ b/src/pydap/tests/test_net.py
@@ -2,11 +2,15 @@
 Test the follow redirects and handling of more complex routing situations
 """
 
+import pytest
 import requests
 import requests_mock
 from webob.request import Request
+from webob.response import Response
 
-from pydap.net import create_request
+from pydap.handlers.lib import BaseHandler
+from pydap.net import GET, create_request, get_response
+from pydap.tests.datasets import SimpleGroup
 
 
 def test_redirect():
@@ -15,40 +19,77 @@ def test_redirect():
     # mock_url is redirected to https:
     redirect_url = "http://www.test2.com/"
     with requests_mock.Mocker() as m:
-        m.register_uri("HEAD", mock_url, text="resp2", status_code=200)
+        m.register_uri(
+            "GET", mock_url, status_code=302, headers={"Location": redirect_url}
+        )
+        m.register_uri("GET", redirect_url, text="Final Destination", status_code=200)
         req = create_request(mock_url)
-        assert len(m.request_history) == 1
-        assert isinstance(req, Request)
-        assert req.headers["Host"] == "www.test.com:80"
+        assert req.url == redirect_url
+        assert req.status_code == 200
+        assert req.text == "Final Destination"
+        assert isinstance(req, requests.Response)
 
     # Without session:
     with requests_mock.Mocker() as m:
         m.register_uri(
-            "HEAD",
+            "GET",
             mock_url,
             text="resp1",
             status_code=301,
             headers={"Location": redirect_url},
         )
-        m.register_uri("HEAD", redirect_url, text="resp2", status_code=200)
+        m.register_uri("GET", redirect_url, text="resp2", status_code=200)
         req = create_request(mock_url)
+        assert isinstance(req, requests.Response)
         # Ensure follow redirect:
         assert len(m.request_history) == 2
-        assert isinstance(req, Request)
-        assert req.headers["Host"] == "www.test2.com:80"
+        assert req.url == redirect_url
+        assert req.status_code == 200
+        assert req.text == "resp2"
 
     # With session:
     with requests_mock.Mocker() as m:
         m.register_uri(
-            "HEAD",
+            "GET",
             mock_url,
             text="resp1",
             status_code=301,
             headers={"Location": redirect_url},
         )
-        m.register_uri("HEAD", redirect_url, text="resp2", status_code=200)
+        m.register_uri("GET", redirect_url, text="resp2", status_code=200)
         req = create_request(mock_url, session=requests.Session())
         # Ensure follow redirect:
         assert len(m.request_history) == 2
-        assert isinstance(req, Request)
-        assert req.headers["Host"] == "www.test2.com:80"
+        assert isinstance(req, requests.Response)
+        assert req.url == redirect_url
+        assert req.status_code == 200
+        assert req.text == "resp2"
+
+
+def test_raise_httperror():
+    """test that raise_for_status raises the correct HTTPerror"""
+    fake_url = "https://httpstat.us/404"  # this url will return a 404
+    with pytest.raises(requests.exceptions.HTTPError):
+        create_request(fake_url)
+
+
+@pytest.fixture
+def appGroup():
+    """Creates an application from the SimpleGroup dataset"""
+    return BaseHandler(SimpleGroup)
+
+
+def test_GET_application(appGroup):
+    """Test that local url are handled properly"""
+    data_url = "http://localhost:8080/"
+    r = GET(data_url + ".dmr", application=appGroup)
+    assert r.status_code == 200
+
+
+def test_create_request_application(appGroup):
+    """Test that local url are handled properly"""
+    req = create_request("/.dmr", application=appGroup, verify=True)
+    assert isinstance(req, Request)
+    resp = get_response(req, application=appGroup, verify=True)
+    assert isinstance(resp, Response)
+    assert resp.status_code == 200
diff --git a/src/pydap/tests/test_server_devel_ssl.py b/src/pydap/tests/test_server_devel_ssl.py
index 8dc5a62..7ccee46 100644
--- a/src/pydap/tests/test_server_devel_ssl.py
+++ b/src/pydap/tests/test_server_devel_ssl.py
@@ -9,6 +9,7 @@ it could work with more data formats.
 
 import ssl
 import sys
+import time
 
 import numpy as np
 import pytest
@@ -48,7 +49,8 @@ def test_open(sequence_type_data):
     TestDataset = DatasetType("Test")
     TestDataset["sequence"] = sequence_type_data
     with LocalTestServerSSL(BaseHandler(TestDataset)) as server:
-        dataset = open_url(server.url)
+        time.sleep(0.1)
+        dataset = open_url(server.url, protocol="dap2")
         seq = dataset["sequence"]
         retrieved_data = [line for line in seq]
 
@@ -70,7 +72,10 @@ def test_verify_open_url(sequence_type_data):
     application = BaseHandler(TestDataset)
     with LocalTestServerSSL(application, ssl_context="adhoc") as server:
         try:
-            open_url(server.url, verify=False, session=requests.Session())
+            time.sleep(0.2)
+            open_url(
+                server.url, verify=False, session=requests.Session(), protocol="dap2"
+            )
         except (ssl.SSLError, requests.exceptions.SSLError):
             pytest.fail("SSLError should not be raised.")