Repository: sunpy/sunpy

Issue:
Fido change of URL for GOES15/XRS and earlier data
sunpy/fido returns `urlopen error [Errno 60] Operation timed out` when searching for GOES15 XRS data as NOAA has moved it from 
https://satdat.ngdc.noaa.gov/sem/goes/data/science/xrs/goes15/  to https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes15/ 

GOES08 to 15 now at this new url, and so all these need to be changed. GOES16-18 at different url which hasn't changed.

Fido XRS search returns the shorter cadence "non-science" data versions, i.e. https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes15/gxrs-l2-irrad_science/ then subdirectory YYYY/MM


Discussion:
ah no! 🤦‍♀️ 

ok thanks for raising this @ianan! 





Resolving patch:
diff --git a/sunpy/net/dataretriever/sources/goes.py b/sunpy/net/dataretriever/sources/goes.py
--- a/sunpy/net/dataretriever/sources/goes.py
+++ b/sunpy/net/dataretriever/sources/goes.py
@@ -21,25 +21,27 @@
 
 class XRSClient(GenericClient):
     """
-    Provides access to the GOES XRS fits files archive.
+    Provides access to several GOES XRS files archive.
 
-    Searches for GOES XRS data both on NASA servers prior to re-processed
-    GOES 13, 14 and 15 and on the NOAA archive for > GOES 13.
-    For satellite numbers > 13 the XRSClient searches the NOAA archive, and
-    returns the re-processed science-quality data for GOES 13, 14 and 15, and
-    also the new GOES-R series 16 and 17.
+    For older GOES satellites (7 and below), NASA servers are used.
+    For newer GOES satellites (8 and above), NOAA servers are used.
 
-    Note - the new science quality data have scaling factors removed for 13, 14 and 15
-    and they are not added to GOES 16 AND 17. This means the peak flux will be different to
-    the older version of the data, such as those collected from the NASA servers.
+    For GOES 8-15, the data is the re-processed science-quality data.
 
-    See the following readmes about the data
+    .. note::
 
-    * Reprocessed 13, 14, 15 :
-        https://satdat.ngdc.noaa.gov/sem/goes/data/science/xrs/GOES_13-15_XRS_Science-Quality_Data_Readme.pdf
+        The new science quality data have the scaling factors removed for GOES 8-15
+        and they are not added to GOES 16 AND 17 data products.
+        This means the peak flux will be different to the older version of the data,
+        such as those collected from the NASA servers.
 
-    * GOES-R 16, 17 :
-        https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l1b/docs/GOES-R_EXIS_XRS_L1b_Science-Quality_Data_ReadMe.pdf
+    See the following readmes about these data:
+
+    * GOES 1 - 7: https://umbra.nascom.nasa.gov/goes/fits/goes_fits_files_notes.txt
+
+    * Reprocessed 8 - 15: https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/GOES_1-15_XRS_Science-Quality_Data_Readme.pdf
+
+    * GOES-R 16 - 17: https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l1b/docs/GOES-R_EXIS_XRS_L1b_Science-Quality_Data_ReadMe.pdf
 
     Examples
     --------
@@ -51,9 +53,9 @@ class XRSClient(GenericClient):
     Results from 1 Provider:
     <BLANKLINE>
     4 Results from the XRSClient:
-    Source: <13: https://umbra.nascom.nasa.gov/goes/fits
-    13, 14, 15: https://satdat.ngdc.noaa.gov/sem/goes/data/science/
-    16, 17: https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/
+    Source: <8: https://umbra.nascom.nasa.gov/goes/fits
+    8-15: https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/
+    16-17: https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/
     <BLANKLINE>
            Start Time               End Time        Instrument ... Source Provider
     ----------------------- ----------------------- ---------- ... ------ --------
@@ -63,18 +65,16 @@ class XRSClient(GenericClient):
     2016-01-02 00:00:00.000 2016-01-02 23:59:59.999        XRS ...   GOES     NOAA
     <BLANKLINE>
     <BLANKLINE>
-
     """
-    # GOES XRS data from NASA servers up to GOES 15.
-    # The reprocessed 13, 14, 15 data should be taken from NOAA server.
+    # GOES XRS data from NASA servers up to GOES 7.
     baseurl_old = r'https://umbra.nascom.nasa.gov/goes/fits/%Y/go(\d){2}(\d){6,8}\.fits'
     pattern_old = '{}/fits/{year:4d}/go{SatelliteNumber:02d}{}{month:2d}{day:2d}.fits'
-    # GOES XRS 13, 14, 15 from NOAA (re-processed data)
-    baseurl_new = (r"https://satdat.ngdc.noaa.gov/sem/goes/data/science/xrs/"
-                   r"goes{SatelliteNumber}/gxrs-l2-irrad_science/%Y/%m/sci_gxrs-l2-irrad_g{SatelliteNumber}_d%Y%m%d_.*\.nc")
+    # The reprocessed 8-15 data should be taken from NOAA.
+    baseurl_new = (r"https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/"
+                   r"goes{SatelliteNumber:02d}/gxrs-l2-irrad_science/%Y/%m/sci_gxrs-l2-irrad_g{SatelliteNumber:02d}_d%Y%m%d_.*\.nc")
     pattern_new = ("{}/goes{SatelliteNumber:02d}/gxrs-l2-irrad_science/{year:4d}/"
                    "{month:2d}/sci_gxrs-l2-irrad_g{SatelliteNumber:02d}_d{year:4d}{month:2d}{day:2d}_{}.nc")
-    # GOES XRS data for GOES-R Series - 16, 17
+    # GOES-R Series 16-17 XRS data from NOAA.
     baseurl_r = (r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes{SatelliteNumber}"
                  r"/l2/data/xrsf-l2-flx1s_science/%Y/%m/sci_xrsf-l2-flx1s_g{SatelliteNumber}_d%Y%m%d_.*\.nc")
     pattern_r = ("{}/goes/goes{SatelliteNumber:02d}/l2/data/xrsf-l2-flx1s_science/{year:4d}/"
@@ -82,9 +82,9 @@ class XRSClient(GenericClient):
 
     @property
     def info_url(self):
-        return ("<13: https://umbra.nascom.nasa.gov/goes/fits \n"
-                "13, 14, 15: https://satdat.ngdc.noaa.gov/sem/goes/data/science/ \n"
-                "16, 17: https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/")
+        return ("<8: https://umbra.nascom.nasa.gov/goes/fits \n"
+                "8-15: https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/ \n"
+                "16-17: https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/")
 
     def post_search_hook(self, i, matchdict):
         tr = get_timerange_from_exdict(i)
@@ -102,16 +102,15 @@ def post_search_hook(self, i, matchdict):
             rowdict["Provider"] = matchdict["Provider"][0]
         else:
             rowdict["Provider"] = matchdict["Provider"][1]
-
         return rowdict
 
     def search(self, *args, **kwargs):
         matchdict = self._get_match_dict(*args, **kwargs)
         # this is for the case when the timerange overlaps with the provider change.
-        if matchdict["Start Time"] < "2009-09-01" and matchdict["End Time"] >= "2009-09-01":
+        if matchdict["Start Time"] < "2001-03-01" and matchdict["End Time"] >= "2001-03-01":
             matchdict_before, matchdict_after = matchdict.copy(), matchdict.copy()
-            matchdict_after["Start Time"] = parse_time('2009-09-01')
-            matchdict_before["End Time"] = parse_time('2009-08-31')
+            matchdict_after["Start Time"] = parse_time('2001-03-01')
+            matchdict_before["End Time"] = parse_time('2001-03-01')
             metalist_before = self._get_metalist(matchdict_before)
             metalist_after = self._get_metalist(matchdict_after)
             metalist = metalist_before + metalist_after
@@ -139,17 +138,17 @@ def _get_metalist(self, matchdict):
         This makes it easier for when searching for overlapping providers.
         """
         metalist = []
-        # the data before the re-processed GOES 13, 14, 15 data.
-        if (matchdict["End Time"] < "2009-09-01") or (matchdict["End Time"] >= "2009-09-01" and matchdict["Provider"] == ["sdac"]):
+        # The data before the re-processed GOES 8-15 data.
+        if (matchdict["End Time"] < "2001-03-01") or (matchdict["End Time"] >= "2001-03-01" and matchdict["Provider"] == ["sdac"]):
             metalist += self._get_metalist_fn(matchdict, self.baseurl_old, self.pattern_old)
-        # new data from NOAA.
+        # New data from NOAA.
         else:
             if matchdict["End Time"] >= "2017-02-07":
                 for sat in [16, 17]:
                     metalist += self._get_metalist_fn(matchdict,
                                                       self.baseurl_r.format(SatelliteNumber=sat), self.pattern_r)
             if matchdict["End Time"] <= "2020-03-04":
-                for sat in [13, 14, 15]:
+                for sat in [8, 9, 10, 11, 12, 13, 14, 15]:
                     metalist += self._get_metalist_fn(matchdict,
                                                       self.baseurl_new.format(SatelliteNumber=sat), self.pattern_new)
         return metalist
@@ -170,7 +169,6 @@ def register_values(cls):
             attrs.Provider: [('SDAC', 'The Solar Data Analysis Center.'),
                              ('NOAA', 'The National Oceanic and Atmospheric Administration.')],
             attrs.goes.SatelliteNumber: [(str(x), f"GOES Satellite Number {x}") for x in goes_number]}
-
         return adict
 
 
@@ -178,21 +176,16 @@ class SUVIClient(GenericClient):
     """
     Provides access to data from the GOES Solar Ultraviolet Imager (SUVI).
 
-    SUVI data are provided by NOAA at the following url
-    https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/
-    The SUVI instrument was first included on GOES-16. It produces level-1b as
-    well as level-2 data products. Level-2 data products are a weighted average
-    of level-1b product files and therefore provide higher imaging dynamic
+    `SUVI data are provided by NOAA <https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/>`__.
+
+    The SUVI instrument was first included on GOES-16. It produces level 1b as
+    well as level-2 data products. Level 2 data products are a weighted average
+    of level 1b product files and therefore provide higher imaging dynamic
     range than individual images. The exposure time of level 1b images range
-    from 1 s to 0.005 s. SUVI supports the following wavelengths;
-    94, 131, 171, 195, 284, 304 angstrom. If no wavelength is specified, images
-    from all wavelengths are returned.
+    from 1 s to 0.005 s.
 
-    Note
-    ----
-    GOES-16 began providing regular level-1b data on 2018-06-01.  At the time
-    of writing, SUVI on GOES-17 is operational but currently does not provide
-    Level-2 data.
+    SUVI supports the following wavelengths; 94, 131, 171, 195, 284, 304 angstrom.
+    If no wavelength is specified, images from all wavelengths are returned.
 
     Examples
     --------
@@ -215,7 +208,6 @@ class SUVIClient(GenericClient):
     2020-07-10 00:08:00.000 2020-07-10 00:12:00.000       SUVI ...     2      304.0
     <BLANKLINE>
     <BLANKLINE>
-
     """
     baseurl1b = (r'https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes'
                  r'{SatelliteNumber}/l1b/suvi-l1b-{elem:2}{wave:03}/%Y/%m/%d/OR_SUVI-L1b.*\.fits.gz')
@@ -233,8 +225,7 @@ def info_url(self):
         return 'https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes'
 
     def post_search_hook(self, i, matchdict):
-
-        # extracting start times and end times
+        # Extracting start times and end times
         start = Time(datetime(i['year'], i['month'], i['day'], i['hour'], i['minute'], i['second']))
         start.format = 'iso'
         end = Time(datetime(i['year'], i['month'], i['day'], i['ehour'], i['eminute'], i['esecond']))
@@ -306,7 +297,7 @@ def _attrs_module(cls):
     @classmethod
     def register_values(cls):
         from sunpy.net import attrs
-        goes_number = [16, 17]
+        goes_number = [16, 17, 18]
         adict = {attrs.Instrument: [
             ("SUVI", "GOES Solar Ultraviolet Imager.")],
             attrs.goes.SatelliteNumber: [(str(x), f"GOES Satellite Number {x}") for x in goes_number],


Test patch:
diff --git a/sunpy/database/tests/test_tables.py b/sunpy/database/tests/test_tables.py
--- a/sunpy/database/tests/test_tables.py
+++ b/sunpy/database/tests/test_tables.py
@@ -158,7 +158,7 @@ def test_entries_from_fido_search_result(fido_search_result):
     # 2 entries from goes
     assert entries[60] == DatabaseEntry(
         source='GOES', provider='NOAA', physobs='irradiance',
-        fileid='https://satdat.ngdc.noaa.gov/sem/goes/data/science/xrs/goes15/'
+        fileid='https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes15/'
                'gxrs-l2-irrad_science/2012/01/sci_gxrs-l2-irrad_g15_d20120101_v0-0-0.nc',
         observation_time_start=datetime(2012, 1, 1, 0, 0),
         observation_time_end=datetime(2012, 1, 1, 23, 59, 59, 999000),
diff --git a/sunpy/net/dataretriever/sources/tests/test_goes_ud.py b/sunpy/net/dataretriever/sources/tests/test_goes_ud.py
--- a/sunpy/net/dataretriever/sources/tests/test_goes_ud.py
+++ b/sunpy/net/dataretriever/sources/tests/test_goes_ud.py
@@ -26,8 +26,8 @@ def LCClient():
       'https://umbra.nascom.nasa.gov/goes/fits/1995/go07950603.fits',
       'https://umbra.nascom.nasa.gov/goes/fits/1995/go07950605.fits'),
      (Time('2008/06/02 12:00', '2008/06/04'),
-      'https://umbra.nascom.nasa.gov/goes/fits/2008/go1020080602.fits',
-      'https://umbra.nascom.nasa.gov/goes/fits/2008/go1020080604.fits'),
+      'https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes10/gxrs-l2-irrad_science/2008/06/sci_gxrs-l2-irrad_g10_d20080602_v0-0-0.nc',
+      'https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes10/gxrs-l2-irrad_science/2008/06/sci_gxrs-l2-irrad_g10_d20080604_v0-0-0.nc'),
      (Time('2020/08/02', '2020/08/04'),
       'https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/'
       'goes16/l2/data/xrsf-l2-flx1s_science/2020/08/sci_xrsf-l2-flx1s_g16_d20200802_v2-1-0.nc',
@@ -57,13 +57,12 @@ def test_get_overlap_urls(LCClient, timerange, url_start, url_end):
 @pytest.mark.remote_data
 @pytest.mark.parametrize("timerange, url_start, url_end",
                          [(a.Time("2009/08/30 00:10", "2009/09/02"),
-                           "https://umbra.nascom.nasa.gov/goes/fits/2009/go1020090830.fits",
-                           "https://satdat.ngdc.noaa.gov/sem/goes/data/science/xrs/goes14/gxrs-l2-irrad_science/"
-                           "2009/09/sci_gxrs-l2-irrad_g14_d20090902_v0-0-0.nc")])
+                           "https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes10/gxrs-l2-irrad_science/2009/08/sci_gxrs-l2-irrad_g10_d20090830_v0-0-0.nc",
+                           "https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes14/gxrs-l2-irrad_science/2009/09/sci_gxrs-l2-irrad_g14_d20090902_v0-0-0.nc")])
 def test_get_overlap_providers(LCClient, timerange, url_start, url_end):
     qresponse = LCClient.search(timerange)
     urls = [i['url'] for i in qresponse]
-    assert len(urls) == 4
+    assert len(urls) == 6
     assert urls[0] == url_start
     assert urls[-1] == url_end
 
@@ -72,8 +71,7 @@ def test_get_overlap_providers(LCClient, timerange, url_start, url_end):
 @pytest.mark.parametrize("timerange, url_old, url_new",
                          [(a.Time('2013/10/28', '2013/10/29'),
                            "https://umbra.nascom.nasa.gov/goes/fits/2013/go1520131028.fits",
-                           "https://satdat.ngdc.noaa.gov/sem/goes/data/science/xrs/goes13/gxrs-l2-irrad_science/"
-                           "2013/10/sci_gxrs-l2-irrad_g13_d20131028_v0-0-0.nc")])
+                           "https://www.ncei.noaa.gov/data/goes-space-environment-monitor/access/science/xrs/goes13/gxrs-l2-irrad_science/2013/10/sci_gxrs-l2-irrad_g13_d20131028_v0-0-0.nc")])
 def test_old_data_access(timerange, url_old, url_new):
     # test first for old data
     qr = Fido.search(timerange, a.Instrument("XRS"), a.Provider("SDAC"))
@@ -102,23 +100,16 @@ def test_fixed_satellite(LCClient):
     ans1 = LCClient.search(a.Time("2017/01/01 2:00", "2017/01/02 2:10"),
                            a.Instrument.xrs,
                            a.goes.SatelliteNumber.fifteen)
-
     for resp in ans1:
-
         assert "g15" in resp['url']
-
     ans1 = LCClient.search(a.Time("2017/01/01", "2017/01/02 23:00"),
                            a.Instrument.xrs,
                            a.goes.SatelliteNumber(13))
-
     for resp in ans1:
-
         assert "g13" in resp['url']
-
     ans1 = LCClient.search(a.Time("1999/1/13", "1999/1/16"),
                            a.Instrument.xrs,
                            a.goes.SatelliteNumber(8))
-
     for resp in ans1:
         assert "go08" in resp['url']