Repository: sunpy/sunpy

Issue:
FIDO XRSClient query results are mislabeled
When using FIDO to search for and download GOES XRS data, the results from a FIDO.search() are mislabeled, although the actual downloaded files are correct. For example:

    from sunpy.net import Fido
    from sunpy.net import attrs

    files = Fido.search(attrs.Time('2011-06-07','2011-06-09'),attrs.Instrument('goes'))
    files
    
    <sunpy.net.fido_factory.UnifiedResponse object at 0x1155c7ed0>
    Results from 1 Provider:

    3 Results from the XRSClient:
     Start Time           End Time      Source Instrument Wavelength
       str19               str19         str4     str4       str3   
    ------------------- ------------------- ------ ---------- ----------
    2011-06-07 00:00:00 2011-06-09 00:00:00   nasa       goes        nan
    2011-06-07 00:00:00 2011-06-09 00:00:00   nasa       goes        nan
    2011-06-07 00:00:00 2011-06-09 00:00:00   nasa       goes        nan


There are three entries, all with the same start and end time, which is misleading. But, if you download the actual files they are correct.

    Fido.fetch(files)
    [========================================] 
    ['/Users/ainglis/sunpy/data/go1520110608.fits',
     '/Users/ainglis/sunpy/data/go1520110607.fits',
     '/Users/ainglis/sunpy/data/go1520110609.fits']

The actual functionality is correct, so it's mainly a quality of life issue.



Discussion:
@dpshelio didn't you do some work on making it possible to fix this? I am sure we have talked about this before somewhere.

Resolving patch:
diff --git a/sunpy/net/dataretriever/sources/goes.py b/sunpy/net/dataretriever/sources/goes.py
--- a/sunpy/net/dataretriever/sources/goes.py
+++ b/sunpy/net/dataretriever/sources/goes.py
@@ -2,12 +2,15 @@
 # This module was developed under funding provided by
 # Google Summer of Code 2014
 
+import os
 import datetime
 
 from sunpy.time import parse_time, TimeRange
 
 from ..client import GenericClient
 
+from sunpy.extern.six.moves.urllib.parse import urlsplit
+
 from sunpy import config
 TIME_FORMAT = config.get("general", "time_format")
 
@@ -54,6 +57,25 @@ def _get_goes_sat_num(self, date):
             raise ValueError('No operational GOES satellites on {}'.format(
                 date.strftime(TIME_FORMAT)))
 
+    def _get_time_for_url(self, urls):
+        times = []
+        for uri in urls:
+            uripath = urlsplit(uri).path
+
+            # Extract the yymmdd or yyyymmdd timestamp
+            datestamp = os.path.splitext(os.path.split(uripath)[1])[0][4:]
+
+            # 1999-01-15 as an integer.
+            if int(datestamp) < 990115:
+                start = datetime.datetime.strptime(datestamp, "%y%m%d")
+            else:
+                start = datetime.datetime.strptime(datestamp, "%Y%m%d")
+
+            almost_day = datetime.timedelta(days=1, milliseconds=-1)
+            times.append(TimeRange(start, start + almost_day))
+
+        return times
+
     def _get_url_for_timerange(self, timerange, **kwargs):
         """
         Returns a URL to the GOES data for the specified date.
@@ -72,7 +94,9 @@ def _get_url_for_timerange(self, timerange, **kwargs):
         base_url = 'http://umbra.nascom.nasa.gov/goes/fits/'
         start_time = datetime.datetime.combine(timerange.start.date(),
                                                datetime.datetime.min.time())
-        total_days = int(timerange.days.value) + 1
+        # make sure we are counting a day even if only a part of it is in the query range.
+        day_range = TimeRange(timerange.start.date(), timerange.end.date())
+        total_days = int(day_range.days.value) + 1
         result = list()
 
         # Iterate over each day in the input timerange and generate a URL for
diff --git a/sunpy/time/time.py b/sunpy/time/time.py
--- a/sunpy/time/time.py
+++ b/sunpy/time/time.py
@@ -1,7 +1,6 @@
 from __future__ import absolute_import, division, print_function
 import re
-from datetime import datetime
-from datetime import timedelta
+from datetime import datetime, date, time, timedelta
 
 import numpy as np
 import pandas
@@ -170,6 +169,8 @@ def parse_time(time_string, time_format='', **kwargs):
         return time_string._mpl_repr()
     elif isinstance(time_string, datetime) or time_format == 'datetime':
         return time_string
+    elif isinstance(time_string, date):
+        return datetime.combine(time_string, time())
     elif isinstance(time_string, tuple):
         return datetime(*time_string)
     elif time_format == 'utime' or isinstance(time_string, (int, float)):


Test patch:
diff --git a/sunpy/net/dataretriever/tests/test_goes_ud.py b/sunpy/net/dataretriever/tests/test_goes_ud.py
--- a/sunpy/net/dataretriever/tests/test_goes_ud.py
+++ b/sunpy/net/dataretriever/tests/test_goes_ud.py
@@ -1,4 +1,5 @@
 import pytest
+import datetime
 
 from sunpy.time.timerange import TimeRange, parse_time
 from sunpy.net.vso.attrs import Time, Instrument
@@ -66,8 +67,13 @@ def test_fixed_satellite():
 def test_query(time):
     qr1 = LCClient.search(time, Instrument('XRS'))
     assert isinstance(qr1, QueryResponse)
-    assert qr1.time_range().start == time.start
-    assert qr1.time_range().end == time.end
+    # We only compare dates here as the start time of the qr will always be the
+    # start of the day.
+    assert qr1.time_range().start.date() == time.start.date()
+
+    almost_day = datetime.timedelta(days=1, milliseconds=-1)
+    end = datetime.datetime.combine(time.end.date(), datetime.time()) + almost_day
+    assert qr1.time_range().end == end
 
 
 def test_query_error():
@@ -109,3 +115,15 @@ def test_fido(time, instrument):
     assert isinstance(qr, UnifiedResponse)
     response = Fido.fetch(qr)
     assert len(response) == qr._numfile
+
+
+@given(goes_time())
+def test_time_for_url(time):
+    time = time.start.date().strftime("%Y/%m/%d")
+    almost_day = datetime.timedelta(days=1, milliseconds=-1)
+
+    tr = TimeRange(time, almost_day)
+    url = LCClient._get_url_for_timerange(tr)
+    times = LCClient._get_time_for_url(url)
+
+    assert all([tr == t2 for t2 in times])
diff --git a/sunpy/net/tests/test_fido.py b/sunpy/net/tests/test_fido.py
--- a/sunpy/net/tests/test_fido.py
+++ b/sunpy/net/tests/test_fido.py
@@ -77,8 +77,9 @@ def check_response(query, unifiedresp):
         raise ValueError("No Time Specified")
 
     for block in unifiedresp.responses:
+        res_tr = block.time_range()
         for res in block:
-            assert res.time.start in query_tr
+            assert res.time.start in res_tr
             assert query_instr.lower() == res.instrument.lower()