Repository: sunpy/sunpy

Issue:
Fido UnifiedResponse is still not indexing properly
```python
from sunpy.net import Fido, attrs as a
res = Fido.search(a.Time("2016-06-07", "2016-06-08"), a.Instrument('eve') | a.Instrument('goes'))
res[0]
Traceback (most recent call last):
  File "/usr/lib/python3.6/site-packages/xonsh/__amalgam__.py", line 15472, in default
    run_compiled_code(code, self.ctx, None, 'single')
  File "/usr/lib/python3.6/site-packages/xonsh/__amalgam__.py", line 3634, in run_compiled_code
    func(code, glb, loc)
  File "<xonsh-code>", line 1, in <module>
  File "/usr/lib/python3.6/site-packages/sunpy/net/fido_factory.py", line 67, in __getitem__
    return type(self)(ret)
  File "/usr/lib/python3.6/site-packages/sunpy/net/fido_factory.py", line 57, in __init__
    raise Exception("{} is not a valid input to UnifiedResponse.".format(type(lst)))
Exception: <class 'sunpy.net.vso.vso.QueryResponse'> is not a valid input to UnifiedResponse.
```
ping @DanRyanIrish @dpshelio 

UnifiedResponse should support slicing the results objects it contains
If you have 5 clients, you should be able to sub-select the results from client 1 and ignore the others then pass to `Fido.fetch`.

@SolarDrew go!


Discussion:

Resolving patch:
diff --git a/sunpy/net/fido_factory.py b/sunpy/net/fido_factory.py
--- a/sunpy/net/fido_factory.py
+++ b/sunpy/net/fido_factory.py
@@ -6,10 +6,9 @@
 `~sunpy.net.fido_factory.DownloadResponse` objects are returned by ``Fido.fetch``.
 
 """
-# Author: Rishabh Sharma <rishabh.sharma.gunner@gmail.com>
-# This module was developed under funding provided by
-# Google Summer of Code 2014
-from collections import MutableSequence
+# This module was initially developed under funding provided by Google Summer
+# of Code 2014
+from collections import Sequence
 
 from sunpy.util.datatype_factory_base import BasicRegistrationFactory
 from sunpy.util.datatype_factory_base import NoMatchError
@@ -17,31 +16,49 @@
 
 from sunpy.net.dataretriever.clients import CLIENTS
 from sunpy.net.dataretriever.client import QueryResponse
-from sunpy.net.vso import VSOClient
+from sunpy.net.vso import VSOClient, QueryResponse as vsoQueryResponse
+
 from . import attr
 from . import attrs as a
 
 __all__ = ['Fido', 'UnifiedResponse', 'UnifiedDownloaderFactory', 'DownloadResponse']
 
 
-class UnifiedResponse(MutableSequence):
+class UnifiedResponse(Sequence):
     """
-    The object used to store responses from the unified downloader.
+    The object used to store results from `~sunpy.net.UnifiedDownloaderFactory.search`.
+
+    The `~sunpy.net.Fido` object returns results from multiple different
+    clients. So it is always possible to sub-select these results, you can
+    index this object with two indices. The first index is the client index,
+    i.e. corresponding to the results from the `~sunpy.net.vso.VSOClient`. The
+    second index can be used to select records from the results returned from
+    that client, for instance if you only want every second result you could
+    index the second dimension with ``::2``.
+
+    Parameters
+    ----------
+    lst : `object`
+        A single instance or an iterable of ``(QueryResponse, client)`` pairs
+        or ``QueryResponse`` objects with a ``.client`` attribute.
     """
+
     def __init__(self, lst):
         """
         Input to this constructor can be one of a few things:
 
-        1. A list of one UnifiedResponse object
-        2. A list of tuples (QueryResponse, client)
+        1. A ``QueryResponse`` object
+        2. A list of tuples ``(QueryResponse, client)``
         """
 
         tmplst = []
         # numfile is the number of files not the number of results.
         self._numfile = 0
-        if isinstance(lst, QueryResponse):
+        if isinstance(lst, (QueryResponse, vsoQueryResponse)):
             if not hasattr(lst, 'client'):
-                raise("QueryResponse is only a valid input if it has a client attribute.")
+                raise ValueError(
+                    "A {} object is only a valid input to UnifiedResponse if it has a client attribute.".
+                    format(type(lst).__name__))
             tmplst.append(lst)
             self._numfile = len(lst)
         else:
@@ -54,31 +71,72 @@ def __init__(self, lst):
                     tmplst.append(block)
                     self._numfile += len(block)
                 else:
-                    raise Exception("{} is not a valid input to UnifiedResponse.".format(type(lst)))
+                    raise ValueError(
+                        "{} is not a valid input to UnifiedResponse.".format(type(lst)))
 
         self._list = tmplst
 
     def __len__(self):
         return len(self._list)
 
-    def __getitem__(self, aslice):
-        ret = self._list[aslice]
-        if ret:
-            return type(self)(ret)
+    def __iter__(self):
+        return self.responses
+
+    def _handle_record_slice(self, client_resp, record_slice):
+        """
+        Given a slice to be applied to the results from a single client, return
+        an object of the same type as client_resp.
+        """
+        # When we subindex, we want to persist the type of the response object.
+        resp_type = type(client_resp)
+
+        # Make sure we always have an iterable, as most of the response objects
+        # expect one.
+        if isinstance(record_slice, int):
+            resp = [client_resp[record_slice]]
+        else:
+            resp = client_resp[record_slice]
+
+        # Reconstruct a response object with the sub-indexed records.
+        ret = resp_type(resp)
+        # Make sure we pass the client back out again.
+        ret.client = client_resp.client
 
         return ret
 
-    def __delitem__(self, i):
-        del self._list[i]
+    def __getitem__(self, aslice):
+        """
+        Support slicing the UnifiedResponse as a 2D object.
 
-    def __setitem__(self, aslice, v):
-        self._list[aslice] = v
+        The first index is to the client and the second index is the records
+        returned from those clients.
+        """
+        # Just a single int as a slice, we are just indexing client.
+        if isinstance(aslice, (int, slice)):
+            ret = self._list[aslice]
+
+        # Make sure we only have a length two slice.
+        elif isinstance(aslice, tuple):
+            if len(aslice) > 2:
+                raise IndexError("UnifiedResponse objects can only be sliced with one or two indices.")
+
+            # Indexing both client and records, but only for one client.
+            if isinstance(aslice[0], int):
+                client_resp = self._list[aslice[0]]
+                ret = self._handle_record_slice(client_resp, aslice[1])
+
+            # Indexing both client and records for multiple clients.
+            else:
+                intermediate = self._list[aslice[0]]
+                ret = []
+                for client_resp in intermediate:
+                    resp = self._handle_record_slice(client_resp, aslice[1])
+                    ret.append(resp)
 
-    def __iter__(self):
-        return self.responses
+        else:
+            raise IndexError("UnifiedResponse objects must be sliced with integers.")
 
-    def insert(self, i, v):
-        self._list.insert(i, v)
+        return UnifiedResponse(ret)
 
     def get_response(self, i):
         """
@@ -101,21 +159,35 @@ def file_num(self):
         return self._numfile
 
     def _repr_html_(self):
-        ret = ''
+        nprov = len(self)
+        if nprov == 1:
+            ret = 'Results from {} Provider:</br></br>'.format(len(self))
+        else:
+            ret = 'Results from {} Providers:</br></br>'.format(len(self))
         for block in self.responses:
-            ret += "Results from the {}:\n".format(block.client.__class__.__name__)
+            ret += "{} Results from the {}:</br>".format(len(block), block.client.__class__.__name__)
             ret += block._repr_html_()
-            ret += '\n'
+            ret += '</br>'
 
         return ret
 
     def __repr__(self):
         ret = super(UnifiedResponse, self).__repr__()
-        ret += '\n'
+        ret += '\n' + str(self)
+
+        return ret
+
+    def __str__(self):
+        nprov = len(self)
+        if nprov == 1:
+            ret = 'Results from {} Provider:\n\n'.format(len(self))
+        else:
+            ret = 'Results from {} Providers:\n\n'.format(len(self))
         for block in self.responses:
-            ret += "Results from the {}:\n".format(block.client.__class__.__name__)
-            ret += repr(block)
-            ret += '\n'
+            ret += "{} Results from the {}:\n".format(len(block), block.client.__class__.__name__)
+            lines = repr(block).split('\n')
+            ret += '\n'.join(lines[1:])
+            ret += '\n\n'
 
         return ret
 
@@ -189,6 +261,7 @@ class UnifiedDownloaderFactory(BasicRegistrationFactory):
 
     Search and Download data from a variety of supported sources.
     """
+
     def search(self, *query):
         """
         Query for data in form of multiple parameters.
@@ -203,7 +276,7 @@ def search(self, *query):
         Query for data from Nobeyama Radioheliograph and RHESSI
 
         >>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
-                                   a.Instrument('norh') | a.Instrument('rhessi'))
+                                   (a.Instrument('norh') & a.Wavelength(17*u.GHz)) | a.Instrument('rhessi'))
 
         Query for 304 Angstrom SDO AIA data with a cadence of 10 minutes
 
@@ -212,7 +285,7 @@ def search(self, *query):
         >>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
                                    a.Instrument('AIA'),
                                    a.Wavelength(304*u.angstrom, 304*u.angstrom),
-                                   a.Sample(10*u.minute))
+                                   a.vso.Sample(10*u.minute))
 
         Parameters
         ----------
@@ -237,15 +310,17 @@ def search(self, *query):
         query = attr.and_(*query)
         return UnifiedResponse(query_walker.create(query, self))
 
-    def fetch(self, query_result, wait=True, progress=True, **kwargs):
+    # Python 3: this line should be like this
+    # def fetch(self, *query_results, wait=True, progress=True, **kwargs):
+    def fetch(self, *query_results, **kwargs):
         """
-        Downloads the files pointed at by URLs contained within UnifiedResponse
-        object.
+        Download the records represented by
+        `~sunpy.net.fido_factory.UnifiedResponse` objects.
 
         Parameters
         ----------
-        query_result : `sunpy.net.fido_factory.UnifiedResponse`
-            Container returned by query method.
+        query_results : `sunpy.net.fido_factory.UnifiedResponse`
+            Container returned by query method, or multiple.
 
         wait : `bool`
             fetch will wait until the download is complete before returning.
@@ -264,9 +339,12 @@ def fetch(self, query_result, wait=True, progress=True, **kwargs):
         >>> downresp = Fido.get(unifresp)
         >>> file_paths = downresp.wait()
         """
+        wait = kwargs.pop("wait", True)
+        progress = kwargs.pop("progress", True)
         reslist = []
-        for block in query_result.responses:
-            reslist.append(block.client.get(block, **kwargs))
+        for query_result in query_results:
+            for block in query_result.responses:
+                reslist.append(block.client.get(block, **kwargs))
 
         results = DownloadResponse(reslist)
 
@@ -276,8 +354,7 @@ def fetch(self, query_result, wait=True, progress=True, **kwargs):
             return results
 
     def __call__(self, *args, **kwargs):
-        raise TypeError("'{}' object is not callable".format(
-            self.__class__.__name__))
+        raise TypeError("'{}' object is not callable".format(self.__class__.__name__))
 
     def _check_registered_widgets(self, *args):
         """Factory helper function"""
@@ -290,8 +367,7 @@ def _check_registered_widgets(self, *args):
         n_matches = len(candidate_widget_types)
         if n_matches == 0:
             # There is no default client
-            raise NoMatchError(
-                "This query was not understood by any clients. Did you miss an OR?")
+            raise NoMatchError("This query was not understood by any clients. Did you miss an OR?")
         elif n_matches == 2:
             # If two clients have reported they understand this query, and one
             # of them is the VSOClient, then we ignore VSOClient.
@@ -301,10 +377,9 @@ def _check_registered_widgets(self, *args):
         # Finally check that we only have one match.
         if len(candidate_widget_types) > 1:
             candidate_names = [cls.__name__ for cls in candidate_widget_types]
-            raise MultipleMatchError(
-                "The following clients matched this query. "
-                "Please make your query more specific.\n"
-                "{}".format(candidate_names))
+            raise MultipleMatchError("The following clients matched this query. "
+                                     "Please make your query more specific.\n"
+                                     "{}".format(candidate_names))
 
         return candidate_widget_types
 
@@ -320,12 +395,13 @@ def _make_query_to_client(self, *query):
         -------
         response : `~sunpy.net.dataretriever.client.QueryResponse`
 
-        client : Instance of client class
+        client : `object`
+		Instance of client class
         """
         candidate_widget_types = self._check_registered_widgets(*query)
         tmpclient = candidate_widget_types[0]()
         return tmpclient.query(*query), tmpclient
 
 
-Fido = UnifiedDownloaderFactory(registry=CLIENTS,
-                                additional_validation_functions=['_can_handle_query'])
+Fido = UnifiedDownloaderFactory(
+    registry=CLIENTS, additional_validation_functions=['_can_handle_query'])
diff --git a/sunpy/net/vso/__init__.py b/sunpy/net/vso/__init__.py
--- a/sunpy/net/vso/__init__.py
+++ b/sunpy/net/vso/__init__.py
@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 
 # for exposure to from sunpy.net.vso import *
-from sunpy.net.vso.vso import VSOClient, InteractiveVSOClient
+from sunpy.net.vso.vso import VSOClient, InteractiveVSOClient, QueryResponse
 
 # for, e.g.,
 # >>> from sunpy.net import vso
diff --git a/sunpy/net/vso/vso.py b/sunpy/net/vso/vso.py
--- a/sunpy/net/vso/vso.py
+++ b/sunpy/net/vso/vso.py
@@ -119,6 +119,9 @@ def get_online_vso_url(api, url, port):
 
 # TODO: Python 3 this should subclass from UserList
 class QueryResponse(list):
+    """
+    A container for VSO Records returned from VSO Searches.
+    """
 
     def __init__(self, lst, queryresult=None, table=None):
         super(QueryResponse, self).__init__(lst)


Test patch:
diff --git a/sunpy/net/tests/test_fido.py b/sunpy/net/tests/test_fido.py
--- a/sunpy/net/tests/test_fido.py
+++ b/sunpy/net/tests/test_fido.py
@@ -4,12 +4,14 @@
 
 import pytest
 import hypothesis.strategies as st
-from hypothesis import given
+from hypothesis import given, assume
 
 import astropy.units as u
 
 from sunpy.net import attr
+from sunpy.net.vso import attrs as va
 from sunpy.net import Fido, attrs as a
+from sunpy.net.vso.vso import QueryResponse as vsoQueryResponse
 from sunpy.net.fido_factory import DownloadResponse, UnifiedResponse
 from sunpy.net.dataretriever.client import CLIENTS, QueryResponse
 from sunpy.util.datatype_factory_base import NoMatchError, MultipleMatchError
@@ -177,6 +179,22 @@ def test_unifiedresponse_slicing():
     assert isinstance(results[0], UnifiedResponse)
 
 
+def test_unifiedresponse_slicing_reverse():
+    results = Fido.search(
+        a.Time("2012/1/1", "2012/1/5"), a.Instrument("lyra"))
+    assert isinstance(results[::-1], UnifiedResponse)
+    assert len(results[::-1]) == len(results)
+    assert isinstance(results[0, ::-1], UnifiedResponse)
+    assert results[0, ::-1]._list[0] == results._list[0][::-1]
+
+
+def test_vso_unifiedresponse():
+    vrep = vsoQueryResponse([])
+    vrep.client = True
+    uresp = UnifiedResponse(vrep)
+    assert isinstance(uresp, UnifiedResponse)
+
+
 def test_responses():
     results = Fido.search(
         a.Time("2012/1/1", "2012/1/5"), a.Instrument("lyra"))
@@ -193,5 +211,74 @@ def test_repr():
 
     rep = repr(results)
     rep = rep.split('\n')
-    # 6 header lines, the results table and a blank line at the end
-    assert len(rep) == 6 + len(list(results.responses)[0]) + 1
+    # 6 header lines, the results table and two blank lines at the end
+    assert len(rep) == 7 + len(list(results.responses)[0]) + 2
+
+
+def filter_queries(queries):
+    return attr.and_(queries) not in queries
+
+
+@given(st.tuples(offline_query(), offline_query()).filter(filter_queries))
+def test_fido_indexing(queries):
+    query1, query2 = queries
+
+    # This is a work around for an aberration where the filter was not catching
+    # this.
+    assume(query1.attrs[1].start != query2.attrs[1].start)
+
+    res = Fido.search(query1 | query2)
+
+    assert len(res) == 2
+    assert len(res[0]) == 1
+    assert len(res[1]) == 1
+
+    aa = res[0, 0]
+    assert isinstance(aa, UnifiedResponse)
+    assert len(aa) == 1
+    assert len(aa.get_response(0)) == 1
+
+    aa = res[:, 0]
+    assert isinstance(aa, UnifiedResponse)
+    assert len(aa) == 2
+    assert len(aa.get_response(0)) == 1
+
+    aa = res[0, :]
+    assert isinstance(aa, UnifiedResponse)
+    assert len(aa) == 1
+
+    with pytest.raises(IndexError):
+        res[0, 0, 0]
+
+    with pytest.raises(IndexError):
+        res["saldkal"]
+
+    with pytest.raises(IndexError):
+        res[1.0132]
+
+
+@given(st.tuples(offline_query(), offline_query()).filter(filter_queries))
+def test_fido_iter(queries):
+    query1, query2 = queries
+
+    # This is a work around for an aberration where the filter was not catching
+    # this.
+    assume(query1.attrs[1].start != query2.attrs[1].start)
+
+    res = Fido.search(query1 | query2)
+
+    for resp in res:
+        assert isinstance(resp, QueryResponse)
+
+@given(offline_query())
+def test_repr(query):
+    res = Fido.search(query)
+
+    for rep_meth in (res.__repr__, res.__str__, res._repr_html_):
+        if len(res) == 1:
+            assert "Provider" in rep_meth()
+            assert "Providers" not in rep_meth()
+
+        else:
+            assert "Provider" not in rep_meth()
+            assert "Providers" in rep_meth()