Repository: sunpy/sunpy

Issue:
Provide ability to read IDL genx files
Most of the files in SSW are genx files which contain a lot of data which would be useful for SunPy. It may be useful to provide this capability to help migrate from IDL. Discuss below whether this is worthwhile for SunPy to provide this.



Discussion:
@wtbarnes @dpshelio Can you link to the relevant IDL code and other info here please?

Here's just a few to get started. I haven't had much luck finding good examples about how to actually read an xdr file (though see last bullet point) though it seems that once you know the structure of the file, it should be relatively easy (though perhaps a bit tedious). 
- [SSW directory with relevant read/write functions](https://darts.isas.jaxa.jp/pub/ssw/gen/idl/io/)
- [Docs for `xdrlib` in Python](https://docs.python.org/2/library/xdrlib.html)
- [Some general (but ancient) info about the eXternal Data Representation (XDR) standard](https://tools.ietf.org/html/rfc1832.html)
- [Old mailing list discussion about reading xdr files with `xdrlib`](https://mail.python.org/pipermail/python-list/2010-August/585550.html)

Haven't had time to mess around with this much yet, but

``` Python
>>> import xdrlib
>>> f = open('/path/to/ssw/sdo/aia/response/aia_V6_all_fullinst.genx','rb').read()
>>> up = xdrlib.Unpacker(f)
```

seems to be a good place to start.


Resolving patch:
diff --git a/sunpy/io/special/__init__.py b/sunpy/io/special/__init__.py
new file mode 100644
--- /dev/null
+++ b/sunpy/io/special/__init__.py
@@ -0,0 +1 @@
+from .genx import *
diff --git a/sunpy/io/special/genx.py b/sunpy/io/special/genx.py
new file mode 100644
--- /dev/null
+++ b/sunpy/io/special/genx.py
@@ -0,0 +1,167 @@
+import xdrlib
+from collections import OrderedDict
+from functools import partial
+import copy
+
+import numpy as np
+
+__all__ = ['read_genx']
+
+class SSWUnpacker(xdrlib.Unpacker):
+    """
+    `xdrlib.Unpacker` customisation to read strings and complex data as written
+    by IDL.
+    """
+    def unpack_string(self):
+        n = self.unpack_uint()
+        if n > 0:
+            n = self.unpack_uint()
+        return self.unpack_fstring(n).decode('utf-8')
+    def unpack_complex(self):
+        return complex(self.unpack_float(), self.unpack_float())
+    def unpack_complex_double(self):
+        return complex(self.unpack_double(), self.unpack_double())
+
+def read_struct_skeleton(xdrdata):
+    """
+    Reads the skeleton of the IDL's structure as written in
+    solarsoft `build_str()` function.
+    """
+    # Read Number of tags
+    ntags = xdrdata.unpack_uint()
+    # Read names of tags
+    tags = [xdrdata.unpack_string() for n in range(ntags)]
+    # Read the tag type
+    tagdict = OrderedDict()
+    for tt in tags:
+        dim = xdrdata.unpack_uint()
+        arr_size = xdrdata.unpack_farray(dim + 2, xdrdata.unpack_int) # [7, 1]
+        typedata = arr_size[-2]
+        if typedata == 8: # it's a structure
+            if dim == 2 and arr_size[0] == 1:
+                # For when structures has been defined with 2 dim but only has one:
+                # bb = replicate({tata: 1, bebe:2, casa:'asa'}, 3)
+                # dd = replicate({tata: 1, bebe:2, casa:'asa'}, 1, 3)
+                # print,size(bb), size(dd)
+                #            1           3           8           3
+                #            2           1           3           8           3
+                dim = 1
+                arr_size[0] = arr_size[1]
+            if arr_size[-1] > 1:
+                tagdict[tt] = np.array([read_struct_skeleton(xdrdata)] * arr_size[-1]).reshape(arr_size[0:-2])
+            else:
+                tagdict[tt] = read_struct_skeleton(xdrdata)
+        else:
+            tagdict[tt] = [dim] + arr_size
+    return tagdict
+
+def struct_to_data(xdrdata, subskeleton):
+    """"
+    Converts the dictionary with the keys and IDL's size output to
+    the data stored in the xdrdata.
+
+    `subskeleton` must contain the size and type of the data that's going to be
+    read in the right order (that's why `OrderedDict` is used). Then the data is
+    read and the `subskeleton` is updated with the data itself.
+    """
+    #http://www.harrisgeospatial.com/docs/SIZE.html
+    types_dict = {
+        2: (xdrdata.unpack_int, np.int16), # int
+        3: (xdrdata.unpack_int, np.int32), # long
+        4: (xdrdata.unpack_float, np.float32),
+        5: (xdrdata.unpack_double, np.float64),
+        6: (xdrdata.unpack_complex, np.complex),
+        7: (xdrdata.unpack_string, None),
+        9: (xdrdata.unpack_complex_double, np.complex64),
+        12: (xdrdata.unpack_uint, np.uint16), # unsign int
+        13: (xdrdata.unpack_uint, np.uint32), # unsign Long int
+        14: (xdrdata.unpack_hyper, np.int64), # Long64
+        15: (xdrdata.unpack_uhyper, np.uint64), # unsign Long64
+    }
+    for key in subskeleton:
+        if isinstance(subskeleton[key], OrderedDict):
+            struct_to_data(xdrdata, subskeleton[key])
+        elif isinstance(subskeleton[key], np.ndarray):
+            testlist = list()
+            struct_shape = subskeleton[key].shape
+            for elem in subskeleton[key].flatten():
+                elem2 = copy.deepcopy(elem)
+                struct_to_data(xdrdata, elem2)
+                testlist.append(elem2)
+            subskeleton[key] = np.array(testlist).reshape(struct_shape)
+        else:
+            sswsize = subskeleton[key]
+            sswtype = sswsize[-2]
+            if sswsize[0] == 0:
+                subskeleton[key] = types_dict[sswtype][0]()
+            else:
+                subskeleton[key] = np.array(xdrdata.unpack_farray(sswsize[-1], types_dict[sswtype][0]),
+                                            dtype=types_dict[sswtype][1]).reshape(sswsize[1:-2][::-1])
+
+def read_genx(filename):
+    """solarsoft genx file reader
+
+    genx files have been used to store calibration data for multiple
+    instruments and distributed within solarsoft. They are stored in XDR
+    format; The External Data Representation Standard file format (XDR) is
+    described in `RFC 1014 <https://tools.ietf.org/html/rfc1014>`_,
+    written by Sun Microsystems, Inc. June 1987
+
+    SolarSoft genx writer creates structures to store the values together with
+    the variable names. It use the `size` IDL function to include the data
+    type, dimension and number of elements that each variable contains.
+
+    Parameters
+    ----------
+    filename : `str`
+        The genx file to be read
+
+    Returns
+    -------
+    output : `OrderedDict`
+        A dictionary with possibly nested dictionaries with the data in
+        the genx file.
+
+    Notes
+    -----
+    The reader aims to maintain the shape and type of the arrays, but take care with the
+    difference in indexing between Python and IDL (row mayor vs column mayor).
+
+    Regarding the type notice that single numbers are converted to python precision, therefore
+    a single integer is converted from 16 to 32/64 bits, and a float from 32 to 64.
+
+    **Strings** read from genx files are assumed to be UTF-8.
+
+    """
+    with open(filename, mode='rb') as xdrfile:
+        xdrdata = SSWUnpacker(xdrfile.read())
+
+    # HEADER information
+    version, xdr = xdrdata.unpack_int(), xdrdata.unpack_int()
+    creation =  xdrdata.unpack_string()
+    if version == 2:
+        arch =  xdrdata.unpack_string()
+        os =  xdrdata.unpack_string()
+        release =  xdrdata.unpack_string()
+    text = xdrdata.unpack_string()
+
+
+    dim = xdrdata.unpack_int() # TODO: I don't think will have dim>1 but need
+                               # to check, if it's larger like savegen has run
+                               # with a multidimensional structure, then
+                               # probably the read_struct_skeleton will have to
+                               # run as in multi-dim structure.
+    arr_size = xdrdata.unpack_farray(dim + 2, xdrdata.unpack_int) # [1, 8, 1] = Main structure for the data
+    mainsize = arr_size[2] # number of upper level strs
+    skeleton = read_struct_skeleton(xdrdata)
+    struct_to_data(xdrdata, skeleton)
+    xdrdata.done()
+    skeleton['HEADER'] = OrderedDict([('VERSION', version), ('XDR', xdr), ('CREATION', creation)])
+    if version == 2:
+        skeleton['HEADER']['IDL_VERSION'] = OrderedDict([('ARCH', arch),
+                                                         ('OS', os),
+                                                         ('RELEASE', release)])
+    skeleton['HEADER']['TEXT'] = text
+    # TODO: for python >= 3.2; so we can keep the original order as how it's stored in the file
+    # skeleton.move_to_end('HEADER', last=False)
+    return skeleton


Test patch:
diff --git a/sunpy/data/test/generated_sample.genx b/sunpy/data/test/generated_sample.genx
new file mode 100644
Binary files /dev/null and b/sunpy/data/test/generated_sample.genx differ
diff --git a/sunpy/io/tests/generate_genx.pro b/sunpy/io/tests/generate_genx.pro
new file mode 100644
--- /dev/null
+++ b/sunpy/io/tests/generate_genx.pro
@@ -0,0 +1,159 @@
+; docformat = 'rst'
+;+
+; NAME:
+;  GENERATE_GENX
+;
+; PURPOSE:
+;  Provide a test file for being tested by SunPy
+;  It needs solarsoft to be run and to create the file
+;
+; EXAMPLE:
+;   idl> .r generate_genx.pro
+;   idl> generate_genx.pro
+;   idl> restgen,file='testing.genx', STRUCT=TESTING, HEAD=HEAD, TEXT=TEXT
+;   idl> help, TESTING,/str
+;   ** Structure MS_216285419006, 9 tags, length=3720, data length=3658:
+;      MYTEXT          STRING    'Waiter to Mr Creosote: 'Finally, monsieur – a wafer-thin mint.''
+;      MYTEXT_ARRAY    STRING    Array[3]
+;      MYTEXT_ARRAY_DIMENSION
+;                      STRING    Array[2, 3]
+;      MYNUMBER        INT              1
+;      MYNUMBER_ARRAY  INT       Array[3]
+;      MYNUMBER_ARRAY_DIMENSION
+;                      INT       Array[2, 3, 4, 5]
+;      MYUINT          UINT             2
+;      MYSTRUCTURE     STRUCT    -> MS_216285419005 Array[1]
+;      MYSTRUCTURE_ARRAY
+;                      STRUCT    -> MS_216285419005 Array[6]
+;   idl> print, TESTING.MYNUMBER_ARRAY_DIMENSION[0,0,0,0]
+;   0
+;   idl> print, TESTING.MYNUMBER_ARRAY_DIMENSION[0,0,0,4]
+;   96
+;   idl> print, TESTING.MYNUMBER_ARRAY_DIMENSION[0,2,2,0]
+;   16
+;   idl> print, TESTING.MYNUMBER_ARRAY_DIMENSION[0,2,3,0]
+;   22
+;   idl> print, TESTING.MYNUMBER_ARRAY_DIMENSION[0,2,3,4]
+;   118
+;   idl> print, size(TESTING.MYSTRUCTURE_ARRAY)
+;           1           6           8           6
+;   idl> print, TESTING.MYSTRUCTURE.MYFARRAY
+;      0.00000      1.00000      2.00000
+;   idl> print, TESTING.MYSTRUCTURE.MYFARRAYD[0,*]
+;      0.00000
+;      2.00000
+;      4.00000
+;   idl> print, TESTING.MYSTRUCTURE.MYDARRAYD[2,1]
+;       5.0000000
+;   idl> print, TESTING.MYSTRUCTURE.NESTEDSTRUCT.MYLARRAYD[1,0,3]
+;       19
+;   idl> print, TESTING.MYSTRUCTURE.NESTEDSTRUCT.MYLARRAYD[0,*,2]
+;       12
+;       14
+;       16
+;   idl> print, TESTING.MYSTRUCTURE.MYCARRAY[1]
+;       (      1.00000,     -9.00000)
+;   idl> print, TESTING.MYSTRUCTURE.MYDCARRAY[2]
+;       (     12.00000,      1.00000)
+;   idl> print, TESTING.MYSTRUCTURE.NESTEDSTRUCT.MYL64NUMBER
+;       9223372036854775807
+;   idl> print, TESTING.MYSTRUCTURE.NESTEDSTRUCT.MYUL64NUMBER
+;       18446744073709551615
+;   idl> print, size(TESTING.MYSTRUCTURE.NESTEDSTRUCT.MYLARRAYD[0,*,2])
+;       2           1           3           3           3
+;   idl> help, HEAD, TEXT,/str
+;   ** Structure MS_216109884001, 4 tags, length=72, data length=72:
+;     VERSION         LONG                 2
+;     XDR             LONG                 1
+;     CREATION        STRING    'Sat Oct 29 08:15:08 2016'
+;     IDL_VERSION     STRUCT    -> IDLV3_VERSION Array[1]
+;     TEXT            STRING    = 'Sample crazy file to test the sunpy genx reader'
+;-
+pro generate_genx
+
+  ;; variables to create:
+  ;; Single text element
+  mytext = "Waiter to Mr Creosote: 'Finally, monsieur – a wafer-thin mint.'"
+  ;; Text array (one dimension, 3 elements)
+  mytext_array = replicate(mytext, 3)
+  ;; multi-dimension string
+  mytext_array_dimension = replicate(mytext, 2, 3)
+
+  ;; single numbers (unsigned)
+  mynumber = 1
+  ;; array of one dimension
+  mynumber_array = indgen(3)
+  ;; multiple dimension:
+  mynumber_array_dimension = indgen(2, 3, 4, 5)
+
+  ;; uint
+  myUint = uint(2)
+
+  ;; long numbers
+  myLnumber = 3L
+  ;; array of one dimension
+  myLnumber_array = lindgen(3)
+  ;; multiple dimension:
+  myLnumber_array_dimension = lindgen(2, 3, 4)
+
+  ;; ulong
+  myUlong = ulong(4)
+
+  ;; long 64
+  myL64number = long64(9223372036854775807)
+
+  ;; ulong64
+  myUL64number = ulong64(18446744073709551615)
+
+  ;; float numbers
+  myfloatnumber = 7.
+  ;; array of one dimension
+  myfloatnumber_array = findgen(3)
+  ;; multiple dimension:
+  myfloatnumber_array_dimension = findgen(2, 3)
+
+  ;; double numbers
+  myDnumber = 8D
+  ;; array of one dimension
+  myDnumber_array = dindgen(3)
+  ;; multiple dimension:
+  myDnumber_array_dimension = dindgen(3, 2)
+
+  ;; complex
+  myComplex = complex(9, 10)
+  myComplex_array = complex(findgen(3), findgen(3)-10)
+
+  ;; dcomplex
+  myDComplex = dcomplex(15, 20)
+  myDComplex_array = dcomplex(findgen(3)+10, findgen(3)-1)
+
+  ;; Structure
+ myStructure = {dummy, name: 'floats and doubles', $
+                 myfnumber: myfloatnumber, myfarray: myfloatnumber_array, myfarrayd: myfloatnumber_array_dimension, $
+                 myDnumber: myDnumber, myDarray: myDnumber_array, myDarrayd: myDnumber_array_dimension, $
+                 nestedstruct: {subdummy, spam: 'brian, longs and ulongs', $
+                                myLnumber: myLnumber, myLarray: myLnumber_array, myLarrayd: myLnumber_array_dimension, $
+                                myULnumber: myUlong, myL64number: myL64number, myUL64number: myUL64number}, $
+                 mytext: 'adding also complex', $
+                 randomNumbers: [87, 105, 116, 104, 32, 108, 111, 118, 101, 32, 102, 114, 111, 109, 32, 83, 97, 110, $
+                                 32, 70, 114, 97, 110, 99, 105, 115, 99, 111, 44, 32, 50, 48, 49, 54], $
+                 myCnumber: myComplex, myCarray: myComplex_array, $
+                 myDCnumber: myDComplex, myDCarray: myDComplex_array}
+
+
+  ;; array of structures
+  myStructure_array = replicate(myStructure, 2, 3)
+
+  ;; Let's save them into a genx
+
+  savegen, mytext, mytext_array, mytext_array_dimension, $
+           mynumber, mynumber_array, mynumber_array_dimension, $
+           myUint, $
+           myStructure, $
+           myStructure_array, $
+           names=['mytext', 'mytext_array', 'mytext_array_dimension', $
+                  'mynumber', 'mynumber_array', 'mynumber_array_dimension', $
+                  'myUint', 'myStructure', 'myStructure_array'], $
+           file='testing', /xdr, $
+           text='Sample crazy file to test the sunpy genx reader'
+end
diff --git a/sunpy/io/tests/test_genx.py b/sunpy/io/tests/test_genx.py
new file mode 100644
--- /dev/null
+++ b/sunpy/io/tests/test_genx.py
@@ -0,0 +1,66 @@
+import os
+import datetime
+
+import pytest
+import numpy as np
+
+from sunpy.io.special import genx
+from sunpy.data.test import rootdir
+
+TESTING = genx.read_genx(os.path.join(rootdir, 'generated_sample.genx'))
+
+def test_skeleton():
+    # top level
+    toplevel_dims = {'MYTEXT': 63, 'MYTEXT_ARRAY': 3, 'MYTEXT_ARRAY_DIMENSION': (2,3),
+                     'MYNUMBER': 1, 'MYNUMBER_ARRAY': 3, 'MYNUMBER_ARRAY_DIMENSION': (2, 3, 4, 5),
+                     'MYUINT': 1, 'MYSTRUCTURE': 14, # the elements inside the OrderedDict
+                     'MYSTRUCTURE_ARRAY': 6, 'HEADER': 5}
+    assert sorted(list(TESTING.keys())) == sorted(list(toplevel_dims.keys()))
+    for key, val in toplevel_dims.items():
+        if isinstance(val, tuple):
+            assert TESTING[key].shape == tuple(reversed(val))
+        else:
+            if val > 1:
+                assert len(TESTING[key]) == val
+            else:
+                assert isinstance(TESTING[key], int)
+
+
+def test_array_elements_values():
+    np.testing.assert_allclose(TESTING['MYSTRUCTURE']['MYFARRAY'], np.arange(3.))
+    np.testing.assert_allclose(TESTING['MYSTRUCTURE']['MYFARRAYD'][:, 0], np.arange(6., step=2))
+    assert TESTING['MYSTRUCTURE']['MYDARRAYD'][1, 2] ==  5.
+    assert TESTING['MYSTRUCTURE']['NESTEDSTRUCT']['MYLARRAYD'][3, 0, 1] == 19
+    np.testing.assert_allclose(TESTING['MYSTRUCTURE']['NESTEDSTRUCT']['MYLARRAYD'][2, :, 0], np.arange(12, 17, step=2))
+    assert TESTING['MYSTRUCTURE']['MYCARRAY'][1] == complex(1, -9)
+    assert TESTING['MYSTRUCTURE']['MYDCARRAY'][2] == complex(12, 1)
+    assert TESTING['MYSTRUCTURE']['NESTEDSTRUCT']['MYUL64NUMBER'] == 18446744073709551615
+    assert TESTING['MYSTRUCTURE']['NESTEDSTRUCT']['MYL64NUMBER'] == 9223372036854775807
+
+@pytest.mark.parametrize("slice, value", [((0,0,0,0), 0),
+                                          ((4, 0, 0, 0), 96),
+                                          ((0, 2, 2, 0), 16),
+                                          ((0, 3, 2, 0), 22),
+                                          ((4, 3, 2, 0), 118)])
+def test_value_slice(slice, value):
+    assert TESTING['MYNUMBER_ARRAY_DIMENSION'][slice] ==  value
+
+@pytest.mark.parametrize("myarray, dtype", [(TESTING['MYNUMBER_ARRAY'], np.int16),
+                                            (TESTING['MYNUMBER_ARRAY_DIMENSION'], np.int16),
+                                            (TESTING['MYSTRUCTURE']['MYFARRAY'], np.float32),
+                                            (TESTING['MYSTRUCTURE']['MYFARRAYD'], np.float32),
+                                            (TESTING['MYSTRUCTURE']['MYDARRAY'], np.float64),
+                                            (TESTING['MYSTRUCTURE']['MYDARRAYD'], np.float64),
+                                            (TESTING['MYSTRUCTURE']['NESTEDSTRUCT']['MYLARRAY'], np.int32),
+                                            (TESTING['MYSTRUCTURE']['NESTEDSTRUCT']['MYLARRAYD'], np.int32),
+                                            (TESTING['MYSTRUCTURE']['RANDOMNUMBERS'], np.int16),
+                                            (TESTING['MYSTRUCTURE']['MYCARRAY'], np.complex),
+                                            (TESTING['MYSTRUCTURE']['MYDCARRAY'], np.complex64)])
+def test_type(myarray, dtype):
+    assert myarray.dtype == dtype
+
+
+def test_date():
+    creation_str = TESTING['HEADER']['CREATION']
+    creation = datetime.datetime.strptime(creation_str, '%a %b %d %H:%M:%S %Y')
+    assert int(''.join(chr(x) for x in TESTING['MYSTRUCTURE']['RANDOMNUMBERS'][-4:])) == creation.year