Repository: sunpy/sunpy

Issue:
When deciding which VSO endpoint to connect to we should determine if the soap address in the WDSL is accessible
By default we access the WDSL at http://docs.virtualsolar.org/WSDL/VSOi_rpc_literal.wsdl but the soap endpoint is http://vso.nso.edu/cgi-bin/VSO/PROD/vsoi_wsdl.cgi we should test both somehow


Discussion:
I have been poking at zeep trying to figure out more info on this, getting the address for the NSO WDSL can be done with this wonderfully simple line: `zc.wsdl.services["VSOiService"].ports["nsoVSOi"].binding_options['address']`

We would need to figure out a way of doing this in a way which isn't WDSL specific (i.e. the `'nsoVSOi` part is troublesome).

Resolving patch:
diff --git a/sunpy/net/vso/vso.py b/sunpy/net/vso/vso.py
--- a/sunpy/net/vso/vso.py
+++ b/sunpy/net/vso/vso.py
@@ -38,25 +38,47 @@
 from .table_response import VSOQueryResponseTable
 from .zeep_plugins import SunPyLoggingZeepPlugin
 
-DEFAULT_URL_PORT = [{'url': 'http://docs.virtualsolar.org/WSDL/VSOi_rpc_literal.wsdl',
-                     'port': 'nsoVSOi'},
-                    {'url': 'https://sdac.virtualsolar.org/API/VSOi_rpc_literal.wsdl',
-                     'port': 'sdacVSOi'}]
+DEFAULT_URL_PORT = [
+    {'url': 'http://docs.virtualsolar.org/WSDL/VSOi_rpc_literal.wsdl', 'port': 'nsoVSOi'},
+    {'url': 'https://sdac.virtualsolar.org/API/VSOi_rpc_literal.wsdl', 'port': 'sdacVSOi'},
+    # This connects to the same cgi as the first WSDL file, but if that file
+    # isn't accessible and the SDAC cgi is unreachable, this might still work.
+    {'url': 'https://sdac.virtualsolar.org/API/VSOi_rpc_literal.wsdl', 'port': 'nsoVSOi'},
+]
 
 
 class _Str(str):
-    """ Subclass of string that contains a meta attribute for the
-    record_item associated with the file. """
+    """
+    Subclass of string that contains a meta attribute for the
+    record_item associated with the file.
+    """
     meta = None
 
 
-# ----------------------------------------
 def check_connection(url):
     try:
-        return urlopen(url).getcode() == 200
+        return urlopen(url, timeout=15).getcode() == 200
     except (OSError, HTTPError, URLError) as e:
         warn_user(f"Connection to {url} failed with error {e}. Retrying with different url and port.")
-        return None
+        return False
+
+
+def check_cgi_connection(url):
+    """
+    At the moment there is no way to make an "are you alive" request to the
+    cgi, so we just hit it with a HTTP get and it gives us back a 411 response.
+    This is weird enough that it probably satisfies us for this check.
+    """
+    try:
+        return urlopen(url, timeout=15).getcode() == 411
+    except HTTPError as e:
+        if e.code == 411:
+            return True
+        warn_user(f"Connection to {url} failed with error {e}. Retrying with different url and port.")
+        return False
+    except (OSError, URLError) as e:
+        warn_user(f"Connection to {url} failed with error {e}. Retrying with different url and port.")
+        return False
 
 
 def get_online_vso_url():
@@ -65,6 +87,13 @@ def get_online_vso_url():
     """
     for mirror in DEFAULT_URL_PORT:
         if check_connection(mirror['url']):
+            # Now we get the port URL from the WSDL and test that
+            wsdl = zeep.wsdl.Document(mirror["url"], zeep.Transport())
+            # I think that accessing "VSOiService" here is equivalent to the
+            # set_ns_prefix call in the build_client function below
+            url = wsdl.services["VSOiService"].ports[mirror["port"]].binding_options["address"]
+            if not check_cgi_connection(url):
+                continue
             return mirror
 
 


Test patch:
diff --git a/sunpy/net/vso/tests/test_vso.py b/sunpy/net/vso/tests/test_vso.py
--- a/sunpy/net/vso/tests/test_vso.py
+++ b/sunpy/net/vso/tests/test_vso.py
@@ -1,3 +1,8 @@
+from functools import partial
+from xml.etree import ElementTree
+from urllib.error import URLError, HTTPError
+from urllib.request import urlopen
+
 import pytest
 from parfive import Results
 
@@ -9,17 +14,33 @@
 from sunpy.net.vso import attrs as va
 from sunpy.net.vso.legacy_response import QueryResponse
 from sunpy.net.vso.table_response import VSOQueryResponseTable, iter_sort_response
-from sunpy.net.vso.vso import VSOClient, build_client, get_online_vso_url
+from sunpy.net.vso.vso import (
+    DEFAULT_URL_PORT,
+    VSOClient,
+    build_client,
+    check_cgi_connection,
+    check_connection,
+    get_online_vso_url,
+)
 from sunpy.tests.mocks import MockObject
 from sunpy.time import parse_time
+from sunpy.util.exceptions import SunpyUserWarning
 
 
 class MockQRRecord:
     """
     Used to test sunpy.net.vso.QueryResponse.build_table(...)
     """
-    def __new__(cls, start_time=None, end_time=None, size=0, source='SOHO', instrument='aia',
-                extent=None, fileid="spam"):
+    def __new__(
+        cls,
+        start_time=None,
+        end_time=None,
+        size=0,
+        source='SOHO',
+        instrument='aia',
+        extent=None,
+        fileid="spam"
+    ):
         return MockObject(size=size,
                           time=MockObject(start=start_time, end=end_time),
                           source=source,
@@ -247,6 +268,62 @@ def test_vso_hmi(client, tmpdir):
         assert all([s == series[0] for s in series])
 
 
+def test_check_connection(mocker):
+    mocker.patch('sunpy.net.vso.vso.urlopen',
+                 side_effect=HTTPError('http://notathing.com/', 400, 'Bad Request', {}, None))
+    with pytest.warns(SunpyUserWarning,
+                      match='Connection to http://notathing.com/ failed with error HTTP Error 400: Bad Request.'):
+        assert check_connection('http://notathing.com/') is False
+
+
+def test_check_cgi_connection(mocker):
+    mocker.patch('sunpy.net.vso.vso.urlopen',
+                 side_effect=HTTPError('http://notathing.com/', 400, 'Bad Request', {}, None))
+    with pytest.warns(SunpyUserWarning,
+                      match='Connection to http://notathing.com/ failed with error HTTP Error 400: Bad Request.'):
+        assert check_cgi_connection('http://notathing.com/') is False
+
+    mocker.patch('sunpy.net.vso.vso.urlopen', side_effect=URLError('http://notathing.com/', 400))
+    with pytest.warns(
+            SunpyUserWarning,
+            match='Connection to http://notathing.com/ failed with error <urlopen error http://notathing.com/>.'
+    ):
+        assert check_cgi_connection('http://notathing.com/') is False
+
+
+def fail_to_open_nso_cgi(disallowed_url, url, **kwargs):
+    if url == disallowed_url:
+        raise URLError(disallowed_url, 404)
+    return urlopen(url, **kwargs)
+
+
+@pytest.mark.remote_data
+def test_fallback_if_cgi_offline(mocker):
+    """
+    This test takes the cgi endpoint URL out of the WDSL, and then disables it,
+    forcing the `get_online_vso_url` function to fallback to a secondary mirror.
+    """
+    default_url = DEFAULT_URL_PORT[0]["url"]
+    # Check that without doing anything we get the URL we expect
+    mirror = get_online_vso_url()
+    assert mirror["url"] == default_url
+
+    # Open the WDSL file and extract the cgi URL
+    # Doing this like this means we don't have to hard code it.
+    wdsl = urlopen(default_url).read()
+    t = ElementTree.fromstring(wdsl)
+    ele = t.findall("{http://schemas.xmlsoap.org/wsdl/}service")[0]
+    cgi_url = list(ele.iter("{http://schemas.xmlsoap.org/wsdl/soap/}address"))[0].attrib['location']
+
+    # Now patch out that URL so we can cause it to return an error
+    mocker.patch('sunpy.net.vso.vso.urlopen', side_effect=partial(fail_to_open_nso_cgi, cgi_url))
+
+    with pytest.warns(SunpyUserWarning,
+                      match=f"Connection to {cgi_url} failed with error .* Retrying with different url and port"):
+        mirror = get_online_vso_url()
+    assert mirror["url"] != "http://docs.virtualsolar.org/WSDL/VSOi_rpc_literal.wsdl"
+
+
 def test_get_online_vso_url(mocker):
     """
     No wsdl links returned valid HTTP response? Return None