Repository: sunpy/sunpy

Issue:
We should return an astropy table from the HEK search method
The list of dicts is really ugly in comparison to an astropy table.


Discussion:
Interesting idea - HEK object returns could be a lot more useful.  For
example, if we return sunpy co-ordinates then plotting of bounding boxes
and object outlines becomes much easier.

*http://www.lmsal.com/hek/VOEvent_Spec.html
<http://www.lmsal.com/hek/VOEvent_Spec.html>*

See also #1792 and #1398

On Fri, Sep 7, 2018 at 12:41 PM Stuart Mumford <notifications@github.com>
wrote:

> The list of dicts is really ugly in comparison to an astropy table.
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/sunpy/sunpy/issues/2740>, or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AA8CFxE7BQhiqf21XZyatltTVVQJ2qpEks5uYml5gaJpZM4Wewfv>
> .
>

Yes, I agree there is much more we could do, but you can construct a `Table` from the current results output *super* easily by throwing it into the `Table(results)` constructor. 

It would also be good I think to remove the empty columns that this creates, and then from there we could add other processing (like having the bounding box columns be `SkyCoord` (which can be Table columns)).
Also interesting to know that HEK returns VO XML. We should look into using Astropy to parse the VO XML straight into a table, without going through a list of Dicts like we do at the moment.
I want to work on this issue as well as #1792 . What i want to know is why it was suggested that the ```parse_hek_response_to_quantities()``` is applied inside get_voevent() and not on results of HEKClient search. What that features requests need?
@Cadair I think for this one have to implement an HEKTable, HEKColumn, HEKRow. because as of now HEKClient.search() method returns response object which in turn can be used to get voe event (by ```get_voevent``` method of Response object)
```
from sunpy.net import hek
tend = '2011/08/09 12:40:29'
tstart = '2011/08/09 07:23:56'
event_type = 'FL'
client = hek.HEKClient()
result = client.search(hek.attrs.Time(tstart,tend),hek.attrs.EventType(event_type))
print(result[0].get_voevent())
```
while if anyone astropy table is returned in HEKClient search then result[0] will be an astropy Row Object with no get_voevent method


Resolving patch:
diff --git a/sunpy/net/hek/hek.py b/sunpy/net/hek/hek.py
--- a/sunpy/net/hek/hek.py
+++ b/sunpy/net/hek/hek.py
@@ -14,6 +14,8 @@
 from datetime import datetime
 from itertools import chain
 
+from astropy.table import Table, Row, Column
+
 from sunpy.net import attr
 from sunpy.util import unique
 from sunpy.net.hek import attrs
@@ -70,7 +72,7 @@ def _download(self, data):
             results.extend(result['result'])
 
             if not result['overmax']:
-                return list(map(Response, results))
+                return HEKTable(results)
             page += 1
 
     def search(self, *query):
@@ -96,13 +98,28 @@ def _merge(self, responses):
         """ Merge responses, removing duplicates. """
         return list(unique(chain.from_iterable(responses), _freeze))
 
+class HEKTable(Table):
+    def __getitem__(self, item):
+        table_item = super().__getitem__(item)
+
+        if table_item.__class__ == Column:
+            table_item.__class__ = HEKColumn
+        elif table_item.__class__ == Row:
+            table_item.__class__ = HEKRow
+
+        return table_item
 
-class Response(dict):
-    """Handles the response from the HEK.  Each Response object is a subclass
-    of the dictionary object.  The dictionary key-value pairs correspond to the
+class HEKColumn(Column):
+    pass
+
+class HEKRow(Row):
+    """
+    Handles the response from the HEK.  Each HEKRow object is a subclass
+    of `astropy.Table.row`.  The column-row key-value pairs correspond to the
     HEK feature/event properties and their values, for that record from the
-    HEK.  Each Response object also has extra properties that relate HEK
-    concepts to VSO concepts."""
+    HEK.  Each HEKRow object also has extra properties that relate HEK
+    concepts to VSO concepts.
+    """
     @property
     def vso_time(self):
         return v_attrs.Time(
@@ -141,3 +158,9 @@ def get_voevent(self, as_dict=True,
             return xml_to_dict(response)
         else:
             return response
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
diff --git a/sunpy/net/hek2vso/hek2vso.py b/sunpy/net/hek2vso/hek2vso.py
--- a/sunpy/net/hek2vso/hek2vso.py
+++ b/sunpy/net/hek2vso/hek2vso.py
@@ -13,6 +13,7 @@
 
 import sys
 from astropy import units
+from astropy.table import Table
 
 from sunpy.net import hek
 from sunpy.net import vso
@@ -34,7 +35,7 @@ def translate_results_to_query(results):
 
     Parameters
     ----------
-    results : `sunpy.net.hek.hek.Response` or list of `sunpy.net.hek.hek.Response`
+    results : `sunpy.net.hek.hek.HEKRow` or `sunpy.net.hek.hek.HEKTable`
         The HEK results from a HEK query to be translated.
 
     Examples
@@ -55,7 +56,7 @@ def translate_results_to_query(results):
     [[<Time(datetime.datetime(2011, 8, 8, 1, 30, 4), datetime.datetime(2011, 8, 10, 0, 0, 4), None)>, <Source(u'SDO')>, <Instrument(u'AIA')>, <Wave(210.99999999999997, 210.99999999999997, 'Angstrom')>], ..., [<Time(datetime.datetime(2011, 8, 9, 8, 1, 21), datetime.datetime(2011, 8, 9, 8, 16, 45), None)>, <Source(u'SDO')>, <Instrument(u'AIA')>, <Wave(303.99999999999994, 303.99999999999994, 'Angstrom')>]]
     """
     queries = []
-    if type(results) is list:
+    if isinstance(results, Table):
         for result in results:
             query = vso_attribute_parse(result)
             queries.append(query)
@@ -74,7 +75,7 @@ def vso_attribute_parse(phrase):
 
     Parameters
     ----------
-    phrase : `dict` containing a `sunpy.net.hek.hek.Response`.
+    phrase : `sunpy.net.hek.hek.HEKRow`.
         The single HEK result to be parsed for VSO attribute data.
 
     Examples
@@ -166,7 +167,7 @@ def translate_and_query(self, hek_results, limit=None, progress=False):
 
         Parameters
         ----------
-        hek_results : `sunpy.net.hek.hek.Response` or list of such Responses
+        hek_results : `sunpy.net.hek.hek.HEKRow` or `sunpy.net.hek.hek.HEKTable`
             The results from a HEK query in the form of a list.
         limit : int
             An approximate limit to the desired number of VSO results.


Test patch:
diff --git a/sunpy/net/tests/test_hek.py b/sunpy/net/tests/test_hek.py
--- a/sunpy/net/tests/test_hek.py
+++ b/sunpy/net/tests/test_hek.py
@@ -85,3 +85,18 @@ def test_err_dummyattr_create():
 def test_err_dummyattr_apply():
     with pytest.raises(TypeError):
         hek.attrs.walker.apply(attr.DummyAttr(), {})
+
+
+@pytest.mark.remote_data
+def test_hek_client():
+    startTime = '2011/08/09 07:23:56'
+    endTime = '2011/08/09 12:40:29'
+    eventType = 'FL'
+
+    hekTime = hek.attrs.Time(startTime, endTime)
+    hekEvent = hek.attrs.EventType(eventType)
+
+    h = hek.HEKClient()
+    hek_query = h.search(hekTime, hekEvent)
+    assert hek_query[0]['event_peaktime'] == hek_query[0].get('event_peaktime')
+    assert hek_query[0].get('') == None
diff --git a/sunpy/net/tests/test_hek2vso.py b/sunpy/net/tests/test_hek2vso.py
--- a/sunpy/net/tests/test_hek2vso.py
+++ b/sunpy/net/tests/test_hek2vso.py
@@ -14,7 +14,8 @@
 import pytest
 
 from astropy import units as u
-
+from astropy import table
+from sunpy.time import parse_time
 from sunpy.net import hek
 from sunpy.net import vso
 from sunpy.net import hek2vso
@@ -51,11 +52,12 @@ def test_translate_results_to_query():
     hek_query = h.search(hekTime, hekEvent)
     vso_query = hek2vso.translate_results_to_query(hek_query)
 
-    if isinstance(hek_query, list):
-        # Comparing length of two lists
-        assert len(hek_query) == len(vso_query)
-        # Comparing types of both queries
-        assert isinstance(hek_query, vso_query)
+    # Comparing length of two lists
+    assert len(hek_query) == len(vso_query)
+    # Comparing types of both queries
+    # Not sure this test makes any sense now
+    assert isinstance(hek_query, table.Table)
+    assert isinstance(vso_query, list)
 
 
 @pytest.mark.remote_data
@@ -66,8 +68,8 @@ def test_vso_attribute_parse():
     vso_query = hek2vso.vso_attribute_parse(hek_query[0])
 
     # Checking Time
-    assert vso_query[0].start == hek_query[0]['event_starttime']
-    assert vso_query[0].end == hek_query[0]['event_endtime']
+    assert vso_query[0].start == parse_time(hek_query[0]['event_starttime'])
+    assert vso_query[0].end == parse_time(hek_query[0]['event_endtime'])
 
     # Checking Observatory
     assert vso_query[1].value == hek_query[0]['obs_observatory']