Repository: sunpy/sunpy

Issue:
SRSClient download bug for 2020 dates
<!--
We know asking good questions takes effort, and we appreciate your time.
Thank you.

Please be aware that everyone has to follow our code of conduct:
https://sunpy.org/coc

These comments are hidden when you submit this github issue.

Please have a search on our GitHub repository to see if a similar issue has already been posted.
If a similar issue is closed, have a quick look to see if you are satisfied by the resolution.
If not please go ahead and open an issue!
-->

### Description

Try to download some SRS files from the year 2020 which appears to work but no files are downloaded. Looking at the source for the data the 2020 tarball has not been create yet which is the root cause. Proposal from @Cadair was to try tarball first and fallback to daily files. Should try to ensure if there is a failure in the download process we don't return 👻  files 

### Expected vs Actual behaviour
The SRS files to be downloaded or an error raised. Currently it fails silently and even gives back a list of file paths that don't exist.

### Steps to Reproduce
<!--
Please include code that reproduces the issue whenever possible.
The best reproductions are self-contained scripts with minimal dependencies.
-->

```python
from sunpy.net import Fido, attrs as a
results = Fido.search(a.Time('2020-01-01', '2020-01-20'), a.Instrument.srs_table)
dl = Fido.fetch(results)
# none of the returned  paths exist
```

### System Details
<!--
We need to know the the package version you are using.
We provide a short function in sunpy (``sunpy.util.system_info()``) that will provide most of the below information.
-->
- SunPy Version: 2.0.5
- Astropy Version:
- Python Version:
- OS information:



Discussion:
Not sure if  #4600 would need to be backported?  

Resolving patch:
diff --git a/sunpy/net/dataretriever/sources/noaa.py b/sunpy/net/dataretriever/sources/noaa.py
--- a/sunpy/net/dataretriever/sources/noaa.py
+++ b/sunpy/net/dataretriever/sources/noaa.py
@@ -3,13 +3,16 @@
 # Google Summer of Code 2014
 import pathlib
 import tarfile
+from datetime import datetime
 from collections import OrderedDict
 
-from astropy import units as u
-from astropy.time import Time, TimeDelta
+from astropy.time import Time
 
+from sunpy import log
 from sunpy.net.dataretriever import GenericClient
+from sunpy.time import TimeRange
 from sunpy.util.parfive_helpers import Downloader
+from sunpy.util.scraper import Scraper
 
 __all__ = ['NOAAIndicesClient', 'NOAAPredictClient', 'SRSClient']
 
@@ -184,27 +187,69 @@ class SRSClient(GenericClient):
     2016-01-01 00:00:00 2016-01-02 00:00:00 NOAA/USAF       SOON        nan
     <BLANKLINE>
     <BLANKLINE>
-
     """
+    BASE_URL = 'ftp://ftp.swpc.noaa.gov/pub/warehouse/'
+    MIN_YEAR = 1996
 
     def _get_url_for_timerange(self, timerange, **kwargs):
+        """
+        Returns a list of urls corresponding to a given time-range.
+        """
         result = list()
-        base_url = 'ftp://ftp.swpc.noaa.gov/pub/warehouse/'
-        total_days = int(timerange.days.value) + 1
-        all_dates = timerange.split(total_days)
-        today_year = int(Time.now().strftime('%Y'))
-        for day in all_dates:
-            end_year = int(day.end.strftime('%Y'))
-            if end_year > today_year or end_year < 1996:
-                continue
-            elif end_year == today_year:
-                suffix = '{}/SRS/{}SRS.txt'.format(
-                    end_year, day.end.strftime('%Y%m%d'))
-            else:
-                suffix = '{}/{}_SRS.tar.gz'.format(
-                    end_year, day.end.strftime('%Y'))
-            url = base_url + suffix
-            result.append(url)
+        # Validate time range srs generated daily since 1996
+        cur_year = Time.now().datetime.year
+        req_start_year = timerange.start.datetime.year
+        req_end_year = timerange.end.datetime.year
+
+        # Return early if both start and end are less than or greater than limits
+        if req_start_year <= req_end_year < self.MIN_YEAR \
+                or req_end_year >= req_start_year > cur_year:
+            return result
+
+        # No point in searching below the min or above max years
+        start_year = max(req_start_year, self.MIN_YEAR)
+        end_year = min(req_end_year, cur_year)
+
+        # Search for tarballs for all years in the query
+        tarball_timerange = TimeRange(f'{start_year}-01-01', f'{end_year}-12-31 23:59:59.999')
+        tarball_urls = dict()
+        tarball_scraper = Scraper(self.BASE_URL + '%Y/%Y_SRS.tar.gz')
+        tarballs = tarball_scraper.filelist(tarball_timerange)
+        max_tarball_year = None
+        for tb_url in tarballs:
+            date = tarball_scraper._extractDateURL(tb_url)
+            year = date.to_datetime().year
+            max_tarball_year = year
+            tarball_urls[year] = tb_url
+            log.debug('SRS tarball found for year %d', year)
+
+        # Create a new time range for the times not covered by tarballs, have to assume tarballs
+        # cover a year, and look for individual srs file for this time range.
+        srs_urls = dict()
+        min_file_year = max_tarball_year if max_tarball_year else start_year
+        min_file_date = (datetime(max_tarball_year, 12, 31, 23, 59, 59) if max_tarball_year else
+                         datetime(start_year, 1, 1, 0, 0, 0))
+        max_file_date = min(timerange.end.datetime, Time.now().datetime)
+        if min_file_date < max_file_date:
+            file_timerange = TimeRange(f'{min_file_year}-01-01', max_file_date)
+            srsfile_scraper = Scraper(self.BASE_URL + '%Y/SRS/%Y%m%dSRS.txt')
+            srsfiles = srsfile_scraper.filelist(file_timerange)
+            for srs_url in srsfiles:
+                date = srsfile_scraper._extractDateURL(srs_url)
+                srs_urls[(date.datetime.year, date.datetime.month, date.datetime.day)] = srs_url
+                log.debug('SRS file found for year %d', date)
+
+        # Now iterate over all days and if the day is in a year we have a tarball for or a day there
+        # is a individual srs file add to the result with corresponding extdict
+        for day in timerange.get_dates():
+            day_ymd = (int(day.strftime('%Y')), int(day.strftime('%m')), int(day.strftime('%d')))
+            extdict = {'year': day_ymd[0], 'month': day_ymd[1], 'day': day_ymd[2]}
+            if self.MIN_YEAR <= day_ymd[0] <= cur_year:
+                if day_ymd[0] in tarball_urls.keys():
+                    result.append((extdict, tarball_urls[day_ymd[0]]))
+                elif day_ymd in srs_urls.keys():
+                    result.append((extdict, srs_urls[day_ymd]))
+
         return result
 
     def fetch(self, qres, path=None, error_callback=None, **kwargs):
@@ -220,25 +265,20 @@ def fetch(self, qres, path=None, error_callback=None, **kwargs):
         -------
         Results Object
         """
-
-        urls = [qrblock.url for qrblock in qres]
-
+        urls = [qrblock.url[1] for qrblock in qres]
         filenames = []
-        local_filenames = []
-
-        for i, [url, qre] in enumerate(zip(urls, qres)):
+        local_filenames = set()
+        for url, qre in zip(urls, qres):
             name = url.split('/')[-1]
-
-            # temporary fix !!! coz All QRBs have same start_time values
-            day = Time(qre.time.start.strftime('%Y-%m-%d')) + TimeDelta(i*u.day)
-
+            start_date = qre.time.start
+            end_date = qre.time.end
             if name not in filenames:
                 filenames.append(name)
-
             if name.endswith('.gz'):
-                local_filenames.append('{}SRS.txt'.format(day.strftime('%Y%m%d')))
+                local_filenames.add('{}SRS.txt'.format(start_date.strftime('%Y%m%d')))
+                local_filenames.add('{}SRS.txt'.format(end_date.strftime('%Y%m%d')))
             else:
-                local_filenames.append(name)
+                local_filenames.add(name)
 
         if path is not None:
             path = pathlib.Path(path)
@@ -252,37 +292,28 @@ def fetch(self, qres, path=None, error_callback=None, **kwargs):
         # OrderedDict is required to maintain ordering because it will be zipped with paths later
         urls = list(OrderedDict.fromkeys(urls))
 
-        dobj = Downloader(max_conn=5)
-
+        downloader = Downloader(max_conn=2)
         for aurl, fname in zip(urls, paths):
-            dobj.enqueue_file(aurl, filename=fname)
+            downloader.enqueue_file(aurl, filename=fname)
 
-        paths = dobj.download()
+        paths = downloader.download()
 
         outfiles = []
         for fname, srs_filename in zip(local_paths, local_filenames):
-
             name = fname.name
-
             past_year = False
-            for i, fname2 in enumerate(paths):
+            for fname2 in paths:
                 fname2 = pathlib.Path(fname2)
-
                 if fname2.name.endswith('.txt'):
                     continue
-
                 year = fname2.name.split('_SRS')[0]
-
                 if year in name:
-                    TarFile = tarfile.open(fname2)
-                    filepath = fname.parent
-                    member = TarFile.getmember('SRS/' + srs_filename)
-                    member.name = name
-                    TarFile.extract(member, path=filepath)
-                    TarFile.close()
-
+                    with tarfile.open(fname2) as open_tar:
+                        filepath = fname.parent
+                        member = open_tar.getmember('SRS/' + srs_filename)
+                        member.name = name
+                        open_tar.extract(member, path=filepath)
                     outfiles.append(fname)
-
                     past_year = True
                     break
 
diff --git a/sunpy/util/scraper.py b/sunpy/util/scraper.py
--- a/sunpy/util/scraper.py
+++ b/sunpy/util/scraper.py
@@ -3,18 +3,21 @@
 """
 import os
 import re
-import datetime
+import calendar
 import warnings
 from ftplib import FTP
+from datetime import datetime
 from urllib.error import HTTPError
 from urllib.parse import urlsplit
 from urllib.request import urlopen
 
 from bs4 import BeautifulSoup
+from dateutil.relativedelta import relativedelta
 
 import astropy.units as u
 from astropy.time import Time, TimeDelta
 
+from sunpy.time import TimeRange
 from sunpy.util.exceptions import SunpyUserWarning
 
 __all__ = ['Scraper']
@@ -70,6 +73,7 @@ class Scraper:
     The ``now`` attribute does not return an existent file, but just how the
     pattern looks with the actual time.
     """
+
     def __init__(self, pattern, regex=False, **kwargs):
         if regex:
             self.pattern = pattern
@@ -81,9 +85,9 @@ def __init__(self, pattern, regex=False, **kwargs):
         self.domain = "{0.scheme}://{0.netloc}/".format(urlsplit(self.pattern))
         milliseconds = re.search(r'\%e', self.pattern)
         if not milliseconds:
-            self.now = datetime.datetime.now().strftime(self.pattern)
+            self.now = datetime.now().strftime(self.pattern)
         else:
-            now = datetime.datetime.now()
+            now = datetime.now()
             milliseconds_ = int(now.microsecond / 1000.)
             self.now = now.strftime('{start}{milli:03d}{end}'.format(
                 start=self.pattern[0:milliseconds.start()],
@@ -112,18 +116,55 @@ def range(self, timerange):
         # find directory structure - without file names
         directorypattern = os.path.dirname(self.pattern) + '/'
         # TODO what if there's not slashes?
-        rangedelta = timerange.dt
         timestep = self._smallerPattern(directorypattern)
         if timestep is None:
             return [directorypattern]
         else:
-            # Number of elements in the time range (including end)
-            n_steps = rangedelta.sec/timestep.sec
-            TotalTimeElements = int(round(n_steps)) + 1
-            directories = [(timerange.start + n * timestep).strftime(directorypattern)
-                           for n in range(TotalTimeElements)]  # TODO if date <= endate
+            directories = []
+            cur = self._date_floor(timerange.start, timestep)
+            end = self._date_floor(timerange.end, timestep) + timestep
+            while cur < end:
+                directories.append(cur.strftime(directorypattern))
+                cur = cur + timestep
+
             return directories
 
+    @staticmethod
+    def _date_floor(date, timestep):
+        """
+        Return the "floor" of the given date and time step.
+
+        Parameters
+        ----------
+        datetime : `datetime.datetime` or `astropy.time.Time`
+            The date to floor
+        timestep : `dateutil.relativedelta.relativedelta`
+            The smallest time step to floor
+        Returns
+        -------
+        `datetime.datetime`
+            The time floored at the given time step
+
+        """
+        date_parts = [int(p) for p in date.strftime('%Y,%m,%d,%H,%M,%S').split(',')]
+        date_parts[-1] = date_parts[-1] % 60
+        date = datetime(*date_parts)
+        orig_time_tup = date.timetuple()
+        time_tup = [orig_time_tup.tm_year, orig_time_tup.tm_mon, orig_time_tup.tm_mday,
+                    orig_time_tup.tm_hour, orig_time_tup.tm_min, orig_time_tup.tm_sec]
+        if timestep == relativedelta(minutes=1):
+            time_tup[-1] = 0
+        elif timestep == relativedelta(hours=1):
+            time_tup[-2:] = [0, 0]
+        elif timestep == relativedelta(days=1):
+            time_tup[-3:] = [0, 0, 0]
+        elif timestep == relativedelta(months=1):
+            time_tup[-4:] = [1, 0, 0, 0]
+        elif timestep == relativedelta(years=1):
+            time_tup[-5:] = [1, 1, 0, 0, 0]
+
+        return datetime(*time_tup)
+
     def _URL_followsPattern(self, url):
         """
         Check whether the url provided follows the pattern.
@@ -316,18 +357,63 @@ def _smallerPattern(self, directoryPattern):
         """
         try:
             if "%S" in directoryPattern:
-                return TimeDelta(1*u.second)
+                return relativedelta(seconds=1)
             elif "%M" in directoryPattern:
-                return TimeDelta(1*u.minute)
+                return relativedelta(minutes=1)
             elif any(hour in directoryPattern for hour in ["%H", "%I"]):
-                return TimeDelta(1*u.hour)
+                return relativedelta(hours=1)
             elif any(day in directoryPattern for day in ["%d", "%j"]):
-                return TimeDelta(1*u.day)
+                return relativedelta(days=1)
             elif any(month in directoryPattern for month in ["%b", "%B", "%m"]):
-                return TimeDelta(31*u.day)
+                return relativedelta(months=1)
             elif any(year in directoryPattern for year in ["%Y", "%y"]):
-                return TimeDelta(365*u.day)
+                return relativedelta(years=1)
             else:
                 return None
         except Exception:
             raise
+
+
+def get_timerange_from_exdict(exdict):
+    """
+    Function to get URL's timerange using extracted metadata.
+    It computes start and end times first using the given
+    dictionary and then returns a timerange.
+
+    Parameters
+    ----------
+    exdict : `dict`
+        Metadata extracted from the file's url.
+
+    Returns
+    -------
+    `~sunpy.time.TimeRange`
+        The time range of the file.
+    """
+    datetypes = ['year', 'month', 'day']
+    timetypes = ['hour', 'minute', 'second', 'millisecond']
+    dtlist = [int(exdict.get(d, 1)) for d in datetypes]
+    dtlist.extend([int(exdict.get(t, 0)) for t in timetypes])
+    startTime = Time(datetime(*dtlist))
+
+    tdelta = 1*u.millisecond
+    if "year" in exdict:
+        if calendar.isleap(int(exdict['year'])):
+            tdelta = 366*u.day
+        else:
+            tdelta = 365*u.day
+    if "month" in exdict:
+        days_in_month = calendar.monthrange(int(exdict['year']), int(exdict['month']))[1]
+        tdelta = days_in_month*u.day
+    if "day" in exdict:
+        tdelta = 1*u.day
+    if "hour" in exdict:
+        tdelta = 1*u.hour
+    if "minute" in exdict:
+        tdelta = 1*u.minute
+    if "second" in exdict:
+        tdelta = 1*u.second
+
+    endTime = startTime + TimeDelta(tdelta) - TimeDelta(1*u.millisecond)
+    file_timerange = TimeRange(startTime, endTime)
+    return file_timerange


Test patch:
diff --git a/sunpy/net/dataretriever/sources/tests/test_noaa.py b/sunpy/net/dataretriever/sources/tests/test_noaa.py
--- a/sunpy/net/dataretriever/sources/tests/test_noaa.py
+++ b/sunpy/net/dataretriever/sources/tests/test_noaa.py
@@ -137,13 +137,13 @@ def test_fetch(mock_wait, mock_search, mock_enqueue, tmp_path, LCClient):
                                           path / "observed-solar-cycle-indices.json"))
 
 
-@no_vso
 @mock.patch('sunpy.net.dataretriever.sources.noaa.NOAAIndicesClient.search',
             return_value=mock_query_object('2012/10/4', '2012/10/6'))
 # The return value of download is irrelevant
 @mock.patch('parfive.Downloader.download',
             return_value=None)
 @mock.patch('parfive.Downloader.enqueue_file')
+@no_vso
 def test_fido(mock_wait, mock_search, mock_enqueue, tmp_path, LCClient):
     path = tmp_path / "sub"
     path.mkdir()
@@ -159,6 +159,7 @@ def test_fido(mock_wait, mock_search, mock_enqueue, tmp_path, LCClient):
                                           path / "observed-solar-cycle-indices.json"))
 
 
+@no_vso
 @pytest.mark.remote_data
 def test_srs_tar_unpack():
     qr = Fido.search(a.Instrument("soon") & a.Time("2015/01/01", "2015/01/01T23:59:29"))
@@ -167,33 +168,49 @@ def test_srs_tar_unpack():
     assert res.data[0].endswith("20150101SRS.txt")
 
 
+@no_vso
 @pytest.mark.remote_data
 def test_srs_tar_unpack_midyear():
     qr = Fido.search(a.Instrument("soon") & a.Time("2011/06/07", "2011/06/08T23:59:29"))
     res = Fido.fetch(qr)
     assert len(res) == 2
-    assert res.data[0].endswith("20110607SRS.txt")
-    assert res.data[-1].endswith("20110608SRS.txt")
+    files = sorted(res.data)
+    assert files[0].endswith("20110607SRS.txt")
+    assert files[-1].endswith("20110608SRS.txt")
+
+
+@no_vso
+@pytest.mark.remote_data
+@mock.patch('ftplib.FTP.nlst', side_effect=[[''], ['20200101SRS.txt', '20200102SRS.txt']])
+def test_srs_missing_tarball(mock_ftp_nlst):
+    qr = Fido.search(a.Time('2020-01-01', '2020-01-02'), a.Instrument.srs_table)
+    urls = [qr.get_response(0).blocks[i].url[1] for i in range(2)]
+    assert urls[0].endswith('20200101SRS.txt')
+    assert urls[1].endswith('20200102SRS.txt')
 
 
+@no_vso
 @pytest.mark.remote_data
 def test_srs_current_year():
     year = datetime.date.today().year
-    qr = Fido.search(a.Instrument("soon") & a.Time(f"{year}/01/01", f"{year}/01/01T23:59:29"))
+    qr = Fido.search(a.Instrument("soon") & a.Time(f"{year}/02/01", f"{year}/02/01T23:59:29"))
     res = Fido.fetch(qr)
     assert len(res) == 1
-    assert res.data[0].endswith(f"{year}0101SRS.txt")
+    assert res.data[0].endswith(f"{year}0201SRS.txt")
 
 
+@no_vso
 @pytest.mark.remote_data
 def test_srs_save_path(tmpdir):
     qr = Fido.search(a.Instrument.srs_table, a.Time("2016/10/01", "2016/10/02"))
     files = Fido.fetch(qr, path=str(tmpdir))
     assert len(files) == 2
+    files = sorted(files)
     assert files[0].endswith("20161001SRS.txt")
     assert files[1].endswith("20161002SRS.txt")
 
 
+@pytest.mark.remote_data
 @pytest.mark.filterwarnings('ignore:ERFA function')
 def test_srs_out_of_range(SRSClient):
     res = SRSClient.search(a.Time('1995/01/01', '1995/02/01'))
@@ -202,6 +219,16 @@ def test_srs_out_of_range(SRSClient):
     assert len(res) == 0
 
 
+@pytest.mark.remote_data
+@pytest.mark.filterwarnings('ignore:ERFA function')
+def test_srs_start_or_end_out_of_range(SRSClient):
+    res = SRSClient.search(a.Time('1995/12/30', '1996/01/02'))
+    assert len(res) == 1
+    cur_year = datetime.date.today().year
+    res = SRSClient.search(a.Time(f'{cur_year}/01/01', f'{cur_year+2}/01/01'))
+    assert len(res) > 0
+
+
 def test_attr_reg():
     assert a.Instrument.noaa_indices == a.Instrument("NOAA-Indices")
     assert a.Instrument.noaa_predict == a.Instrument("NOAA-Predict")
diff --git a/sunpy/net/tests/strategies.py b/sunpy/net/tests/strategies.py
--- a/sunpy/net/tests/strategies.py
+++ b/sunpy/net/tests/strategies.py
@@ -55,7 +55,7 @@ def offline_instruments():
     Returns a strategy for any instrument that does not need the internet to do
     a query.
     """
-    offline_instr = ['noaa-indices', 'noaa-predict', 'soon']
+    offline_instr = ['noaa-indices', 'noaa-predict']
     offline_instr = st.builds(a.Instrument, st.sampled_from(offline_instr))
 
     return st.one_of(offline_instr)
@@ -66,7 +66,7 @@ def online_instruments():
     Returns a strategy for any instrument that does need the internet to do
     a query.
     """
-    online_instr = ['lyra', 'goes', 'eve', 'rhessi', 'norh']
+    online_instr = ['lyra', 'goes', 'eve', 'rhessi', 'norh', 'soon']
     online_instr = st.builds(a.Instrument, st.sampled_from(online_instr))
 
     return online_instr
diff --git a/sunpy/net/tests/test_fido.py b/sunpy/net/tests/test_fido.py
--- a/sunpy/net/tests/test_fido.py
+++ b/sunpy/net/tests/test_fido.py
@@ -39,8 +39,6 @@ def offline_query(draw, instrument=offline_instruments()):
     # If we have AttrAnd then we don't have GOES
     if isinstance(query, a.Instrument) and query.value == 'goes':
         query &= draw(goes_time())
-    elif isinstance(query, a.Instrument) and query.value == 'soon':
-        query &= draw(srs_time())
     else:
         query = attr.and_(query, draw(time_attr()))
     return query
@@ -54,6 +52,8 @@ def online_query(draw, instrument=online_instruments()):
         query &= a.Level.zero
     if isinstance(query, a.Instrument) and query.value == 'norh':
         query &= a.Wavelength(17*u.GHz)
+    if isinstance(query, a.Instrument) and query.value == 'soon':
+        query &= draw(srs_time())
 
     return query
 
diff --git a/sunpy/util/tests/test_scraper.py b/sunpy/util/tests/test_scraper.py
--- a/sunpy/util/tests/test_scraper.py
+++ b/sunpy/util/tests/test_scraper.py
@@ -2,19 +2,17 @@
 from unittest.mock import Mock, patch
 
 import pytest
-
-import astropy.units as u
-from astropy.time import TimeDelta
+from dateutil.relativedelta import relativedelta
 
 from sunpy.data.test import rootdir
 from sunpy.time import TimeRange, parse_time
 from sunpy.util.scraper import Scraper
 
 PATTERN_EXAMPLES = [
-    ('%b%y', TimeDelta(31*u.day)),
-    ('%m%y', TimeDelta(31*u.day)),
-    ('%H%d', TimeDelta(1*u.hour)),
-    ('%y%b', TimeDelta(31*u.day)),
+    ('%b%y', relativedelta(months=1)),
+    ('%m%y', relativedelta(months=1)),
+    ('%H%d', relativedelta(hours=1)),
+    ('%y%b', relativedelta(months=1)),
 ]
 
 
@@ -154,7 +152,7 @@ def testURL_patternMillisecondsZeroPadded():
     # Asserts solution to ticket #1954.
     # Milliseconds must be zero-padded in order to match URL lengths.
     now_mock = Mock(return_value=datetime.datetime(2019, 4, 19, 0, 0, 0, 4009))
-    with patch('datetime.datetime', now=now_mock):
+    with patch('sunpy.util.scraper.datetime', now=now_mock):
         s = Scraper('fd_%Y%m%d_%H%M%S_%e.fts')
     now_mock.assert_called_once()
     assert s.now == 'fd_20190419_000000_004.fts'