Repository: radis/radis (language: python)

Issue:
HDF5 support for Spectrum objects
Spectrum objects should be stored in HDF5. This would yield much faster read / write times. 

Suggested structure:

- All arrays are stored as HDF5 datasets. 

- All conditions are stored as HDF5 attributes `attrs`. This is basically a Python dictionary 

Suggested tasks: 

- [ ] Create a separate branch! 

Write:

- [ ] (**MAIN TASK!**) Implement a save_hdf5 function in `tools\database.py`. Look at existing `save` function. The `_format_to_jsondict` function may be reused

- [ ] Update `SpecDatabase.add` method in `tools\database.py`: add an option for the output format? 

- [ ] Update `Spectrum.store` and `Spectrum.save` method in `spectrum\spectrum.py`: add an option for the output format?

Read:

- [ ] (**MAIN TASK!**) Add a `_h5_to_spec` function in `tools\database.py`. See `_json_to_spec` function. 

- [ ] The `load_spec` function should be rewritten to point either to `_h5_to_spec`, `_json_to_spec`  in normal mode, or `_json_to_spec` in binary mode. I'd rewrite the `binary=False` argument as `fmt='h5', 'json' or 'binary'`

- [ ] Update `SpecDatabase._load_file`

After:

- [ ] Keep JSON support in a first time, but set to HDF5 by default. 

- [ ] Measure perf difference

- [ ] Add a test similarly to the one in `test\spectrum\test_spectrum.py`: `test_store_functions

- [ ] Mention the HDF5 support in online docs, for instance in `docs\spectrum\howto.rst` : how to save and retrieve a Spectrum object?


Discussion / hints:
Note: the function `radis.misc.cache_file.save_to_hdf` can help: cannot be used as is, but some parts may be reused (ex: `new_dict = _h5_compatible(dict)`) 
(one year old, closed without implementation)  
Re-opening. The discussion in https://github.com/radis/radis/issues/84 improved read/write performances of Spectrum objects by a factor of 2-10, but switching to HDF5 could be x100. 

Good first issue as it does not require particular knowledge of the code structure. 

Fix patch:
diff --git a/radis/io/cache_files.py b/radis/io/cache_files.py
index c7465c936..4a03564ab 100644
--- a/radis/io/cache_files.py
+++ b/radis/io/cache_files.py
@@ -32,7 +32,6 @@
 from os.path import exists, splitext
 from warnings import warn
 
-import pandas as pd
 from packaging.version import parse
 
 import radis
@@ -449,13 +448,19 @@ def check_not_deprecated(
 
 
 def _h5_compatible(a_dict):
-    """Make dictionary ``a_dict`` h5 compatible."""
+    """Make dictionary ``a_dict`` compatible with HDF5 attributes.
+
+    Note that nested dictionaries are not supported, see for instance
+    https://gitlab.com/quantify-os/quantify-core/-/issues/158
+    """
     out = {}
     for k, v in a_dict.items():
         if v is None:
             continue  # dont store None
-        elif is_float(v):
+        elif is_float(v) or isinstance(v, bool):
             out[k] = v
+        # elif isinstance(v, dict):
+        #     raise ValueError(f"Value of key `{k}` is a dictionary and cannot be stored as attribute of an HDF5 file. Delete it, flatten it, or convert it to a string ?")
         else:
             out[k] = str(v)  # convert to str
     return out
@@ -471,36 +476,39 @@ def check_relevancy(
 ):
     """Make sure cache file is relevant.
 
-    Use case: checks that  wavenumber min and
-    wavenumber max in ``metadata`` are relevant for the specified spectral
-    range.
-
-    Parameters
-    ----------
-    file: str
-        a `` .h5``  line database cache file
-    load_only_wavenum_above, relevant_if_metadata_below: dict
-        only load the cached file if the metadata values are above/below
-        the specific values for each key.
-    relevant_if_metadata_above, relevant_if_metadata_below: dict
-        file is relevant if the file metadata value for each key of the dictionary
-        is above/below the value in the dictionary
-
-    Other Parameters
-    ----------------
-    key: str
-        dataset key in storer.
+     Use case: checks that  wavenumber min and
+     wavenumber max in ``metadata`` are relevant for the specified spectral
+     range.
+
+     Parameters
+     ----------
+     file: str
+         a `` .h5``  line database cache file
+     load_only_wavenum_above, relevant_if_metadata_below: dict
+         only load the cached file if the metadata values are above/below
+         the specific values for each key.
+     relevant_if_metadata_above, relevant_if_metadata_below: dict
+         file is relevant if the file metadata value for each key of the dictionary
+         is above/below the value in the dictionary
+
+     Other Parameters
+     ----------------
+     key: str
+         dataset key in storer.
+    engine: ``'h5py'``, ``'pytables'``, ``'vaex'``, ``'auto'``
+        which HDF5 library to use. If ``'auto'``, try to guess. Note: ``'vaex'``
+        uses ``'h5py'`` compatible HDF5.
 
-    Examples
-    --------
-    You want to compute a spectrum in between 2300 and 2500 cm-1. A line database
-    file is relevant only if its metadata says that ``'wavenum_max' > 2300``
-    and ``'wavenum_min'`` < 2500 cm-1.
+     Examples
+     --------
+     You want to compute a spectrum in between 2300 and 2500 cm-1. A line database
+     file is relevant only if its metadata says that ``'wavenum_max' > 2300``
+     and ``'wavenum_min'`` < 2500 cm-1.
 
-        check_relevancy('path/to/file', relevant_if_metadata_above={'wavenum_max':2300},
-                        relevant_if_metadata_below={'wavenum_min':2500})
+         check_relevancy('path/to/file', relevant_if_metadata_above={'wavenum_max':2300},
+                         relevant_if_metadata_below={'wavenum_min':2500})
 
-        the specified value.
+         the specified value.
 
     """
     if engine == "auto":
@@ -541,41 +549,51 @@ def _warn_if_object_columns(df, fname):
 
 
 def save_to_hdf(
-    df, fname, metadata, version=None, key="df", overwrite=True, verbose=True
+    df,
+    fname,
+    metadata,
+    version=None,
+    key="df",
+    overwrite=True,
+    verbose=True,
+    engine="pytables",
 ):
     """Save energy levels or lines to HDF5 file. Add metadata and version.
 
-    Parameters
-    ----------
-    df: a pandas DataFrame
-        data will be stored in the key ``'df'``
-    fname: str
-        ``.h5`` file where to store.
-    metadata: dict
-         dictionary of values that were used to generate the DataFrame. Metadata
-         will be asked again on file load to ensure it hasnt changed. ``None``
-         values are not stored.
-    version: str, or ``None``
-        file version. If ``None``, the current :data:`radis.__version__` is used.
-        On file loading, a warning will be raised if the current version is
-        posterior, or an error if the file version is set to be uncompatible.
-    key: str
-        dataset name. Default ``'df'``
-    overwrite: boolean
-        if ``True``, overwrites file. Else, raise an error if it exists.
-
-    Other Parameters
-    ----------------
-    verbose: bool
-        If >=2, also warns if non numeric values are present (it would make
-        calculations slower)
-
-    Notes
-    -----
-    ``None`` values are not stored
+     Parameters
+     ----------
+     df: a pandas DataFrame
+         data will be stored in the key ``'df'``
+     fname: str
+         ``.h5`` file where to store.
+     metadata: dict
+          dictionary of values that were used to generate the DataFrame. Metadata
+          will be asked again on file load to ensure it hasnt changed. ``None``
+          values are not stored.
+     version: str, or ``None``
+         file version. If ``None``, the current :data:`radis.__version__` is used.
+         On file loading, a warning will be raised if the current version is
+         posterior, or an error if the file version is set to be uncompatible.
+     key: str
+         dataset name. Default ``'df'``
+     overwrite: boolean
+         if ``True``, overwrites file. Else, raise an error if it exists.
+
+     Other Parameters
+     ----------------
+     verbose: bool
+         If >=2, also warns if non numeric values are present (it would make
+         calculations slower)
+    engine: ``'h5py'``, ``'pytables'``, ``'vaex'``, ``'pytables-fixed'``
+        which HDF5 library to use. Note: ``'vaex'``
+        uses ``'h5py'`` compatible HDF5. Default ``pytables``
+
+     Notes
+     -----
+     ``None`` values are not stored
     """
     # Check file
-    assert fname.endswith(".h5")
+    assert fname.endswith(".h5") or fname.endswith(".hdf5")
     assert "version" not in metadata
     # ... 'object' columns slow everything down (not fixed format strings!)
     if verbose >= 2:
@@ -589,7 +607,8 @@ def save_to_hdf(
         raise ValueError("File exist: {0}".format(fname))
 
     # start by exporting dataframe
-    df.to_hdf(fname, key, format="fixed", mode="a", complevel=9, complib="blosc")
+    manager = HDF5Manager(engine)
+    manager.write(fname, df, append=False, key=key)
 
     # Add metadata
 
@@ -598,8 +617,7 @@ def save_to_hdf(
         version = radis.__version__
     metadata.update({"version": version})
 
-    with pd.HDFStore(fname, mode="a") as store:
-        store.get_storer(key).attrs.metadata = metadata
+    manager.add_metadata(fname, metadata)
 
     if verbose >= 3:
         print("... saved {0} with metadata: {1}".format(fname, metadata))
@@ -653,9 +671,20 @@ def some_function(*args):
     return metadata
 
 
-def cache_file_name(fname):
-    """Return the corresponding cache file name for fname."""
-    return splitext(fname)[0] + ".h5"
+def cache_file_name(fname, engine="pytables"):
+    """Return the corresponding cache file name for fname.
+
+    Other Parameters
+    ----------------
+    engine: ``'h5py'``, ``'pytables'``, ``'vaex'``
+       which HDF5 library to use. Default ``pytables``
+    """
+    if engine in ["pytables"]:
+        return splitext(fname)[0] + ".h5"
+    elif engine in ["h5py", "vaex"]:
+        return splitext(fname)[0] + ".hdf5"
+    else:
+        raise ValueError(engine)
 
 
 if __name__ == "__main__":
diff --git a/radis/io/hdf5.py b/radis/io/hdf5.py
index 4f218f005..9d456f82f 100644
--- a/radis/io/hdf5.py
+++ b/radis/io/hdf5.py
@@ -15,33 +15,33 @@
 from tables.exceptions import NoSuchNodeError
 
 
-def update_pytables_to_vaex(fname, remove_initial=False, verbose=True):
+def update_pytables_to_vaex(fname, remove_initial=False, verbose=True, key="df"):
     """Convert a HDF5 file generated from PyTables to a
     Vaex-friendly HDF5 format, preserving metadata"""
     import vaex
 
-    if verbose:
-        print(f"Auto-updating {fname} to a Vaex-compatible HDF5 file")
-    df = pd.read_hdf(fname)
-    df = vaex.from_pandas(df)
     if fname.endswith(".h5"):
         fname_vaex = fname.replace(".h5", ".hdf5")
     else:
         fname_vaex = fname
 
+    if verbose:
+        print(f"Auto-updating {fname} to a Vaex-compatible HDF5 file {fname_vaex}")
+    df = pd.read_hdf(fname)
+    df = vaex.from_pandas(df)
+
+    pytables_manager = HDF5Manager(engine="pytables")
+
     # Read metadata
-    with pd.HDFStore(fname, mode="r") as store:
-        file_metadata = store.get_storer("df").attrs.metadata
+    file_metadata = pytables_manager.read_metadata(fname)
 
     # Write Vaex file
     df.export_hdf5(fname_vaex)
     df.close()  # try to fix file not closed()  TODO: remove?
     del df  # same TODO
 
-    with h5py.File(fname_vaex, "a") as hf:
-        # Add metadata
-        for k, v in file_metadata.items():
-            hf.attrs[k] = v
+    vaex_manager = HDF5Manager(engine="vaex")
+    vaex_manager.add_metadata(fname_vaex, file_metadata)
 
     if verbose:
         print(f"Converted to Vaex's HDF5 format {fname_vaex}")
@@ -69,38 +69,93 @@ def open(self, file, mode="w"):
         else:
             raise NotImplementedError(self.engine)
 
-    def write(self, handler, df, append=True):
+    def write(
+        self,
+        file,
+        df,
+        append=True,
+        key="default",
+        format="table",
+        data_columns=["iso", "wav"],
+    ):
+        """Write dataframe ``df`` to ``file``
+
+        Other Parameters
+        ----------------
+        key: str
+            group to write to. If ``None``, write at root level. If ``'default'``,
+            use engine's default (`/table` for `'vaex'`, `df` for `pytables`,
+            root for `h5py` )
+        data_columns : list
+            only these column names will be searchable directly on disk to
+            load certain lines only. See :py:func:`~radis.io.hdf5.hdf2df`
+        """
         if self.engine == "pytables":
-            DATA_COLUMNS = ["iso", "wav"]
-            """
-            list : only these column names will be searchable directly on disk to
-            only load certain lines. See :py:func:`~radis.io.hdf5.hdf2df`
-            """
-            handler.put(
-                key="df",
-                value=df,
-                append=append,
-                format="table",
-                data_columns=DATA_COLUMNS,
-            )
+            if key == "default":
+                key = "df"
+            with self.open(file, "a" if append else "w") as f:
+                f.put(
+                    key=key,
+                    value=df,
+                    append=append,
+                    format=format,
+                    data_columns=data_columns,
+                )
+        elif self.engine == "pytables-fixed":
+            assert not append
+            # export dataframe
+            df.to_hdf(file, key, format="fixed", mode="w", complevel=9, complib="blosc")
+        elif self.engine == "vaex":
+            if key == "default":
+                key = r"/table"
+            import vaex
+
+            if append == True:
+                raise ValueError(
+                    "Cannot append with 'vaex' engine. Load all files separately using vaex.open('many') then export to a single file"
+                )
+            try:
+                df.export_hdf5(file, group=key, mode="w")
+            except AttributeError:  # case where df is not a Vaex dataFrame but (likely) a Pandas Dataframe
+                vaex.from_pandas(df).export_hdf5(file, group=key, mode="w")
         else:
             raise NotImplementedError(self.engine)
+            # h5py is not designed to write Pandas DataFrames
 
     def load(
-        self, fname, columns=None, where=None, key="df", **store_kwargs
-    ) -> pd.DataFrame:
+        self,
+        fname,
+        columns=None,
+        where=None,
+        key="default",
+        none_if_empty=False,
+        **store_kwargs,
+    ):
         """
         Parameters
         ----------
         columns: list of str
             list of columns to load. If ``None``, returns all columns in the file.
+        where: list of str
+            filtering conditions. Ex::
+
+                "wav > 2300"
 
         Other Parameters
         ----------------
-        key: store key in  ``'pytables'`` mode.
+        key: str
+            group to load from. If ``None``, load from root level. If ``'default'``,
+            use engine's default (`/table` for `'vaex'`, `df` for `pytables`,
+            root for `h5py` )
+
+        Returns
+        -------
+        pd.DataFrame or vaex.DataFrame
         """
 
-        if self.engine == "pytables":
+        if self.engine in ["pytables", "pytables-fixed"]:
+            if key == "default":
+                key = "df"
             try:
                 df = pd.read_hdf(
                     fname, columns=columns, where=where, key=key, **store_kwargs
@@ -110,46 +165,67 @@ def load(
                     raise TypeError(
                         f"radis.io.hdf5.hdf2df can only be used to load specific HDF5 files generated in a 'Table' which allows to select only certain columns or rows. Here the file {fname} is in 'Fixed' format. Regenerate it ? If it's a cache file of a .par file, load the .par file directly ?"
                     )
+                elif "cannot create a storer if the object is not existing" in str(err):
+                    raise TypeError(
+                        f"Missing group `{key}` in {fname}. Maybe the file has been generated by a different HDF5 library than Pytables. Try using `engine='vaex'` in the calling function (hdf2df, etc.)"
+                    )
                 else:
                     raise
             except NoSuchNodeError as err:
                 # Probably tried to read a Vaex/h5py HDF5 file forcing "engine='pytables'"
                 raise AttributeError(
-                    f"file {fname} does not seem to have been generated by Pytables. Try using `engine='vaex'` in hdf2df"
+                    f"file {fname} does not seem to have been generated by Pytables. Try using `engine='vaex'` in the calling function (hdf2df, etc.)"
                 ) from err
 
         elif self.engine == "vaex":
+            if key == "default":
+                key = r"/table"
+
+            import h5py
             import vaex
 
             # Open file
             assert len(store_kwargs) == 0
+            fname_list = fname if isinstance(fname, list) else [fname]
+            # vaex can open several files at the same time
+            # First check group exists
+            for fname in fname_list:
+                try:
+                    with h5py.File(fname, "r") as f:
+                        if key and key not in f:
+                            raise KeyError(
+                                key
+                            )  # before vaex raises the same error; which prints things on the console that cannot be caught
+                except (FileNotFoundError, OSError) as err:
+                    # error message with suggestion on how to convert from existing file
+                    for f in fname_list:
+                        if exists(f.replace(".hdf5", ".h5")):
+                            raise FileNotFoundError(
+                                f"`{f}` not found but `{f.replace('.hdf5', '.h5')}` exists (probably a row-based pytables HDF5 file). Try (1) using engine='pytables' in the calling function (`hdf2df`, `fetch_hitemp`, etc.)  ; (2) delete the file to re-download and re-parse it (this may take a lot of time !) ;  or (3, recommended) set `import radis; radis.config['AUTO_UPDATE_DATABASE']= True` in your script to auto-update to Vaex HDF5 file"
+                            ) from err
+                    raise
+
+            # Now, open with vaex
             try:
-                df = vaex.open(fname)
-            except FileNotFoundError as err:
-                # error message with suggestion on how to convert from existing file
-                fname_list = fname if isinstance(fname, list) else [fname]
-                for f in fname_list:
-                    if exists(f.replace(".hdf5", ".h5")):
-                        raise FileNotFoundError(
-                            f"`{f}` not found but `{f.replace('.hdf5', '.h5')}` exists (probably a row-based pytables HDF5 file). Try (1) using engine='pytables' in `hdf2df` or `fetch_hitemp` ; (2) delete the file to re-download and re-parse it (this may take a lot of time !) ;  or (3, recommended) set `import radis; radis.congig['AUTO_UPDATE_DATABASE']= True` in your script to auto-update to Vaex HDF5 file"
-                        ) from err
-                # else:
-                raise
+                df = vaex.open(fname_list, group=key)
             except OSError as err:
                 raise OSError(
-                    f"Cannot read {fname} with Vaex's HDF5 library (column-based). It may be a file generated by pytables (row-based). Try (1) using engine='pytables' in `hdf2df` or `fetch_hitemp` ; (2) delete the file to re-download and re-parse it (this may take a lot of time !) ;  or (3, recommended) set `import radis; radis.congig['AUTO_UPDATE_DATABASE'] = True` in your script to auto-update to Vaex HDF5 file"
+                    f"Cannot read {fname}, group `{key}` with Vaex HDF5 library (column-based). It may be a file generated by pytables (row-based). Try (1) using engine='pytables' in the calling function (`hdf2df`, `fetch_hitemp`, etc.)  ; (2) delete the file to re-download and re-parse it (this may take a lot of time !) ;  or (3, recommended) set `import radis; radis.config['AUTO_UPDATE_DATABASE'] = True` in your script to auto-update to Vaex HDF5 file"
                 ) from err
 
             return df
 
         elif self.engine == "h5py":
+            # TODO: define default key ?
+            if key == "default":
+                key = None
             import h5py
 
             with h5py.File(fname, "r") as f:
-                if key:
-                    load_from = f[key]
-                else:
+                if key is None:  # load from root level
                     load_from = f
+                else:
+                    load_from = f[key]
                 out = {}
                 for k in load_from.keys():
                     out[k] = f[k][()]
@@ -160,56 +236,121 @@ def load(
 
         return df
 
-    def add_metadata(self, fname: str, metadata: dict):
+    def add_metadata(
+        self, fname: str, metadata: dict, key="default", create_empty_dataset=False
+    ):
+        """
+        Parameters
+        ----------
+        fname: str
+            filename
+        metadata: dict
+            dictionary of metadata to add in group ``key``
+        key: str
+            group to add metadata to. If ``None``, add at root level. If ``'default'``,
+            use engine's default (`/table` for `'vaex'`, `df` for `pytables`,
+            root for `h5py` )
 
-        if self.engine == "pytables":
-            with pd.HDFStore(fname, mode="a", complib="blosc", complevel=9) as f:
+        Other Parameters
+        ----------------
+        create_empty_dataset: bool
+            if True, create an empty dataset to store the metadata as attribute
 
-                # f.get_storer("df").attrs.update(metadata)
-                f.get_storer("df").attrs.metadata = metadata
+        """
+        from radis.io.cache_files import _h5_compatible
+
+        if self.engine in ["pytables", "pytables-fixed"]:
+            if key == "default":
+                key = "df"
+            with pd.HDFStore(fname, mode="a", complib="blosc", complevel=9) as f:
+                if create_empty_dataset:
+                    assert self.engine != "pytables-fixed"
+                    # Not possible to create an empty node directly, so we create a dummy group  # TODO ?
+                    f.put(key, pd.Series([]))
+                f.get_storer(key).attrs.metadata = metadata
 
         elif self.engine == "h5py":
+            if key == "default":
+                key = None
             with h5py.File(fname, "a") as hf:
-                hf.attrs.update(metadata)
+                if create_empty_dataset:
+                    assert key is not None
+                    hf.create_dataset(key, dtype="f")
+                if key is None:  # add metadta at root level
+                    hf.attrs.update(_h5_compatible(metadata))
+                else:
+                    hf[key].attrs.update(_h5_compatible(metadata))
 
         elif self.engine == "vaex":
+            if key == "default":
+                key = r"/table"
+            # Should be able to deal with multiple files at a time
             if isinstance(fname, list):
                 assert isinstance(metadata, list)
                 for f, m in zip(fname, metadata):
                     with h5py.File(f, "a") as hf:
-                        hf.attrs.update(m)
+                        if create_empty_dataset:
+                            assert key is not None
+                            hf.create_dataset(key, dtype="f")
+                        if key is None:  # add metadta at root level
+                            hf.attrs.update(_h5_compatible(m))
+                        else:
+                            hf[key].attrs.update(_h5_compatible(m))
             else:
                 with h5py.File(fname, "a") as hf:
-                    hf.attrs.update(metadata)
+                    if create_empty_dataset:
+                        assert key is not None
+                        hf.create_dataset(key, dtype="f")
+                    if key is None:  # add metadta at root level
+                        hf.attrs.update(_h5_compatible(metadata))
+                    else:
+                        hf[key].attrs.update(_h5_compatible(metadata))
 
         else:
             raise NotImplementedError(self.engine)
 
-    def read_metadata(self, fname: str, key="df") -> dict:
+    def read_metadata(self, fname: str, key="default") -> dict:
         """
         Other Parameters
         ----------------
-        key: store key in  ``'pytables'`` mode.
+        key: str
+            group where to read metadat from. If ``None``, add at root level. If ``'default'``,
+            use engine's default (`/table` for `'vaex'`, `df` for `pytables`,
+            root for `h5py` )
         """
 
-        if self.engine == "pytables":
+        if self.engine in ["pytables", "pytables-fixed"]:
+            if key == "default":
+                key = "df"
             with pd.HDFStore(fname, mode="r", complib="blosc", complevel=9) as f:
-
-                metadata = f.get_storer("df").attrs.metadata
+                metadata = f.get_storer(key).attrs.metadata
 
         elif self.engine == "h5py":
+            if key == "default":
+                key = None
             with h5py.File(fname, "r") as hf:
-                metadata = dict(hf.attrs)
+                if key is None:  # read metadta at root level
+                    metadata = dict(hf.attrs)
+                else:
+                    metadata = dict(hf[key].attrs)
 
         elif self.engine == "vaex":
+            if key == "default":
+                key = r"/table"
             if isinstance(fname, list):
                 metadata = []
                 for f in fname:
                     with h5py.File(f, "r") as hf:
-                        metadata.append(dict(hf.attrs))
+                        if key is None:  # read metadta at root level
+                            metadata.append(dict(hf.attrs))
+                        else:
+                            metadata.append(dict(hf[key].attrs))
             else:
                 with h5py.File(fname, "r") as hf:
-                    metadata = dict(hf.attrs)
+                    if key is None:  # add metadta at root level
+                        metadata = dict(hf.attrs)
+                    else:
+                        metadata = dict(hf[key].attrs)
 
         else:
             raise NotImplementedError(self.engine)
diff --git a/radis/io/hitemp.py b/radis/io/hitemp.py
index d2d7cdfaa..16b67f675 100644
--- a/radis/io/hitemp.py
+++ b/radis/io/hitemp.py
@@ -317,53 +317,48 @@ def parse_to_local_file(
 
             # assert not(exists(local_file))
 
-            with writer.open(local_file) as f:
+            b = np.zeros(chunksize, dtype=dt)  # receives the HITRAN 160-character data.
+
+            for nbytes in iter(lambda: gfile.readinto(b), 0):
+
+                if not b[-1]:
+                    # End of file flag within the chunk (but does not start
+                    # with End of file flag) so nbytes != 0
+                    b = get_last(b)
+
+                df = _ndarray2df(b, columns, linereturnformat)
+
+                # Post-processing :
+                # ... Add local quanta attributes, based on the HITRAN group
+                df = parse_local_quanta(df, molecule)
+
+                # ... Add global quanta attributes, based on the HITRAN class
+                df = parse_global_quanta(df, molecule)
+
+                # Switch 'P', 'Q', 'R' to -1, 0, 1
+                if "branch" in df:
+                    replace_PQR_with_m101(df)
+
+                writer.write(local_file, df, append=True)
+
+                wmin = np.min((wmin, df.wav.min()))
+                wmax = np.max((wmax, df.wav.max()))
+
+                Nlines += len(df)
+                Nlines_tot += len(df)
+                Nlines_raw += len(b)
+                if pbar_Ntot_estimate_factor is None:
+                    pbar_Ntot_message = f"{Ntotal_lines_expected:,} lines"
+                else:
+                    pbar_Ntot_message = f"~{Ntotal_lines_expected:,} lines (estimate)"
+                pb.update(
+                    Nlines_tot,
+                    message=f"  Parsed {Nlines_tot:,} / {pbar_Ntot_message}. Wavenumber range {wmin:.2f}-{wmax:.2f} cm-1 is complete.",
+                )
+                # Reinitialize for next read
                 b = np.zeros(
                     chunksize, dtype=dt
                 )  # receives the HITRAN 160-character data.
-
-                for nbytes in iter(lambda: gfile.readinto(b), 0):
-
-                    if not b[-1]:
-                        # End of file flag within the chunk (but does not start
-                        # with End of file flag) so nbytes != 0
-                        b = get_last(b)
-
-                    df = _ndarray2df(b, columns, linereturnformat)
-
-                    # Post-processing :
-                    # ... Add local quanta attributes, based on the HITRAN group
-                    df = parse_local_quanta(df, molecule)
-
-                    # ... Add global quanta attributes, based on the HITRAN class
-                    df = parse_global_quanta(df, molecule)
-
-                    # Switch 'P', 'Q', 'R' to -1, 0, 1
-                    if "branch" in df:
-                        replace_PQR_with_m101(df)
-
-                    writer.write(f, df, append=True)
-
-                    wmin = np.min((wmin, df.wav.min()))
-                    wmax = np.max((wmax, df.wav.max()))
-
-                    Nlines += len(df)
-                    Nlines_tot += len(df)
-                    Nlines_raw += len(b)
-                    if pbar_Ntot_estimate_factor is None:
-                        pbar_Ntot_message = f"{Ntotal_lines_expected:,} lines"
-                    else:
-                        pbar_Ntot_message = (
-                            f"~{Ntotal_lines_expected:,} lines (estimate)"
-                        )
-                    pb.update(
-                        Nlines_tot,
-                        message=f"  Parsed {Nlines_tot:,} / {pbar_Ntot_message}. Wavenumber range {wmin:.2f}-{wmax:.2f} cm-1 is complete.",
-                    )
-                    # Reinitialize for next read
-                    b = np.zeros(
-                        chunksize, dtype=dt
-                    )  # receives the HITRAN 160-character data.
         if pbar_last:
             pb.update(
                 Nlines_tot,
diff --git a/radis/io/hitran.py b/radis/io/hitran.py
index d3b05c991..d7b84c4c0 100644
--- a/radis/io/hitran.py
+++ b/radis/io/hitran.py
@@ -103,6 +103,7 @@ def hit2df(
     drop_non_numeric=True,
     load_wavenum_min=None,
     load_wavenum_max=None,
+    engine="pytables",
 ):
     """Convert a HITRAN/HITEMP [1]_ file to a Pandas dataframe
 
@@ -128,6 +129,8 @@ def hit2df(
         if not ``'None'``, only load the cached file if it contains data for
         wavenumbers above/below the specified value. See :py:func`~radis.io.cache_files.load_h5_cache_file`.
         Default ``'None'``.
+    engine: 'pytables', 'vaex'
+        format for Hdf5 cache file. Default `pytables`
 
     Returns
     -------
@@ -165,7 +168,7 @@ def hit2df(
     columns = columns_2004
 
     # Use cache file if possible
-    fcache = cache_file_name(fname)
+    fcache = cache_file_name(fname, engine=engine)
     if cache and exists(fcache):
         relevant_if_metadata_above = (
             {"wavenum_max": load_wavenum_min} if load_wavenum_min else {}
@@ -182,6 +185,7 @@ def hit2df(
             current_version=radis.__version__,
             last_compatible_version=radis.config["OLDEST_COMPATIBLE_VERSION"],
             verbose=verbose,
+            engine=engine,
         )
         if df is not None:
             return df
@@ -279,6 +283,7 @@ def hit2df(
                 key="df",
                 overwrite=True,
                 verbose=verbose,
+                engine=engine,
             )
         except PermissionError:
             if verbose:
diff --git a/radis/io/query.py b/radis/io/query.py
index 08d065513..780914b42 100644
--- a/radis/io/query.py
+++ b/radis/io/query.py
@@ -115,11 +115,13 @@ def fetch_astroquery(
                 use_cached=cache,
                 expected_metadata=expected_metadata,
                 verbose=verbose,
-                engine="pytables",
+                engine="pytables-fixed",
             )
             if exists(fcache):
                 try:
-                    return get_cache_file(fcache, verbose=verbose)
+                    return get_cache_file(
+                        fcache, verbose=verbose, engine="pytables-fixed"
+                    )
                 except Exception as err:
                     if verbose:
                         printr(
@@ -253,6 +255,7 @@ def fetch_astroquery(
                 key="df",
                 overwrite=True,
                 verbose=verbose,
+                engine="pytables-fixed",
             )
         except PermissionError:
             if verbose:
diff --git a/radis/io/spec_hdf.py b/radis/io/spec_hdf.py
new file mode 100644
index 000000000..6576557ea
--- /dev/null
+++ b/radis/io/spec_hdf.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+"""
+Functions to read/write Spectrum objects under HDF5 format
+"""
+
+try:
+    from .hdf5 import HDF5Manager
+except ImportError:
+    from radis.io.hdf5 import HDF5Manager
+
+
+# Convert
+
+
+def spec2hdf(s, file, engine="pytables"):
+    """ Convert Spectrum s to HDF5 file"""
+
+    df_arrays = s.to_pandas(copy=False)
+
+    # Store
+    mgr = HDF5Manager(engine=engine)
+
+    # Store spectral arrays and lines as Datasets
+
+    mgr.write(file, df_arrays, key="arrays", append=False, data_columns=["wavespace"])
+    mgr.add_metadata(file, s.units, key="arrays")
+
+    if s.lines:
+        mgr.write(file, s.lines, key="lines", append=True)
+
+    # Store the rest as metadata
+
+    if s.populations:
+        mgr.add_metadata(
+            file, s.populations, key="populations", create_empty_dataset=True
+        )
+
+    mgr.add_metadata(file, s.conditions, key="conditions", create_empty_dataset=True)
+
+    if s.references:
+        mgr.add_metadata(
+            file, s.references, key="references", create_empty_dataset=True
+        )
+
+
+def hdf2spec(file, wmin=None, wmax=None, wunit=None, columns=None, engine="pytables"):
+    """ Read HDF5 file into a Spectrum object"""
+    from radis import Spectrum
+
+    mgr = HDF5Manager(engine=engine)
+
+    # Range selection
+    # TODO : convert if stored in wavelength/wavenumber
+    if wmin is not None or wmax is not None:
+        if engine == "pytables":
+            where = []
+            if wmin is not None:
+                where.append(f"wavespace > {wmin}")
+            if wmax is not None:
+                where.append(f"wavespace < {wmax}")
+        elif engine == "vaex":
+            #  Selection is done after opening the file in vaex
+            where = None
+        else:
+            raise NotImplementedError(engine)
+    else:
+        where = None
+
+    # Column selection
+    # ... always load 'wavespace'
+    if columns and "wavespace" not in columns:
+        columns.append("wavespace")
+
+    df2 = mgr.load(file, columns=columns, where=where, key="arrays")
+
+    # Convert to Pandas df (if vaex)
+    if engine == "vaex":
+        if wmin is not None and wmax is not None:
+            df2.select((df2.wavespace > wmin) & (df2.wavespace < wmax))
+            selection = True
+        elif wmin is not None:
+            df2.select(df2.wavespace > wmin)
+            selection = True
+        elif wmax is not None:
+            df2.select(df2.wavespace < wmax)
+            selection = True
+        else:
+            selection = False
+        df2 = df2.to_pandas_df(selection=selection, column_names=columns)
+
+    units2 = mgr.read_metadata(file, key="arrays")
+    conditions2 = mgr.read_metadata(file, key="conditions")
+    try:
+        populations2 = mgr.read_metadata(file, key="populations")
+    except KeyError:
+        populations2 = None
+    try:
+        references2 = mgr.read_metadata(file, key="references")
+    except KeyError:
+        references2 = None
+
+    try:
+        lines2 = mgr.load(file, key="lines")
+    except (KeyError, OSError):  # KeyError in PyTables ; OSError in Vaex
+        lines2 = None
+
+    s2 = Spectrum(
+        {k: v.values for k, v in dict(df2).items()},
+        lines=lines2,
+        units=units2,
+        conditions=conditions2,
+        references=references2,
+        populations=populations2,
+    )
+
+    return s2
diff --git a/radis/spectrum/spectrum.py b/radis/spectrum/spectrum.py
index 1ebeb57b3..b42519425 100644
--- a/radis/spectrum/spectrum.py
+++ b/radis/spectrum/spectrum.py
@@ -751,6 +751,41 @@ class directly. Ex::
         s.file = file
         return s
 
+    @classmethod
+    def from_hdf5(
+        self, file, wmin=None, wmax=None, wunit=None, columns=None, engine="pytables"
+    ):
+        """Generates a Spectrum from an HDF5 file. Uses :py:func:`~radis.io.spec_hdf.hdf2spec`
+
+        Other Parameters
+        ----------------
+        wmin, wmax: float
+            range of wmin, wmax to load , using ``wunit`` (if None, load everything)
+        columns: list of str
+            spectral arrays to load (if None, load everything)
+
+        Examples
+        --------
+        ::
+            Spectrum.from_hdf5("rad_hdf.h5", wmin=2100, wmax=2200, columns=['abscoeff', 'emisscoeff'])
+        """
+        from radis.io.spec_hdf import hdf2spec
+
+        s = hdf2spec(
+            file, wmin=wmin, wmax=wmax, wunit=wunit, columns=columns, engine=engine
+        )
+
+        # Store filename
+        s.file = file
+        return s
+
+    @classmethod
+    def from_spec(self, file):
+        """Generates a Spectrum from a .spec [json] file. Uses :py:func:`~radis.tools.database.load_spec`"""
+        from radis.tools.database import load_spec
+
+        return load_spec(file)
+
     # Public functions
     # %% ======================================================================
     # ----------------
@@ -3052,6 +3087,43 @@ def save(self, *args, **kwargs):
 
         return self.store(*args, **kwargs)
 
+    def to_json(self, *args, **kwargs):
+        """Alias to Spectrum.store(compress=False).
+
+        See Spectrum.store for documentation
+        """
+
+        return self.store(compress=False, *args, **kwargs)
+
+    def to_hdf5(self, file, engine="pytables"):
+        """Stores the Spectrum under HDF5 format. Uses :py:func:`~radis.io.spec_hdf.spec2hdf`"""
+        from radis.io.spec_hdf import spec2hdf
+
+        return spec2hdf(self, file, engine=engine)
+
+    def to_pandas(s, copy=True):
+        """Convert a Spectrum to a Pandas DataFrame
+
+        Returns
+        -------
+        pd.DataFrame : pandas DataFrame where columns are spectral arrays, and
+            units are stored in attributes ``df.attrs``
+
+        Notes
+        -----
+        Pandas does not support units yet. pint-pandas is an advanced
+        project but not fully working.
+        See discussion in https://github.com/pandas-dev/pandas/issues/10349
+
+        For the moment, we store units as metadata"""
+        import pandas as pd
+
+        df = pd.DataFrame(s._q, copy=copy)
+
+        df.attrs = s.units
+
+        return df
+
     def resample(
         self,
         w_new,


Test patch:
diff --git a/radis/test/io/test_hdf5.py b/radis/test/io/test_hdf5.py
index e6edca831..031a8b328 100644
--- a/radis/test/io/test_hdf5.py
+++ b/radis/test/io/test_hdf5.py
@@ -48,7 +48,7 @@ def test_hdf5_io_engines(*args, **kwargs):
 
     # Test h5py engine : add_metadata ; load ; read_metadata
     manager = HDF5Manager(engine="h5py")
-    manager.add_metadata("test_h5py.h5", metadata0)
+    manager.add_metadata("test_h5py.h5", metadata0, key=None)
     df = manager.load("test_h5py.h5", key=None)
     assert (df == df0).all().all()
     assert manager.read_metadata("test_h5py.h5") == metadata0
@@ -56,13 +56,13 @@ def test_hdf5_io_engines(*args, **kwargs):
     # Test vaex engine : add_metadata ; load ; read_
     # .. also able to read h5py files
     manager = HDF5Manager(engine="vaex")
-    manager.add_metadata("test_h5py.h5", metadata0)
+    manager.add_metadata("test_h5py.h5", metadata0, key=None)
     df = manager.load("test_h5py.h5", key=None)
     # this time; df is a vaex DataFrame
     # ... it keeps the file open so we should close the file handle
     assert (df.to_pandas_df() == df0).all().all()
     df.close()
-    assert manager.read_metadata("test_h5py.h5") == metadata0
+    assert manager.read_metadata("test_h5py.h5", key=None) == metadata0
 
 
 @pytest.mark.needs_connection