Repository: matchms/matchms
Pull request #52 (2020-05-27 15:23:46)

Add import and export functions for json files
Importing/exporting spectra to json files is faster than mgf (see #15). It also is now supported by GNPS.
Their json file format, however, is not how I would implement it and differs in many places from the mgf/pyteomics field names, so we have to do some parsing of the metadata.
- added ``save_as_json``, ``load_from_json`` to exporting and importing
- added test that saves and loads spectrum from and to json files

jspaaks: See also:
- step 1 in this notebook https://github.com/matchms/notebooks/blob/9a8f94db0250c2cc540b83de87147c08fac34187/tutorial.ipynb and
- my comment https://github.com/matchms/notebooks/pull/12#issue-395224959.
jspaaks: I tried using specrtum_utils, but its plot function requires that we install, e.g. ``rdkit``. To me, this seemed unnecessary for a simple stem plot. Therefore, I copy-pasted some code from @florian-huber that already made a stem plot using ``matplotlib``.

This is part of https://github.com/matchms/matchms/pull/67. Once that is merged, this issue should be closed.



Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 87e0bb32..36b6bb09 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,9 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+
 - Faster cosine score implementation using numba [#29](https://github.com/matchms/matchms/issues/29)
 - Cosine score based on Hungarian algorithm [#40](https://github.com/matchms/matchms/pull/40)
 - Modified cosine score [#26](https://github.com/matchms/matchms/issues/26)
+- Import and export of spectrums from json files [#15](https://github.com/matchms/matchms/issues/15)
 - Doc strings for many methods [#49](https://github.com/matchms/matchms/issues/49)
 - Examples in doc strings which are tested on CI
 
@@ -29,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Fix verify step in conda publish workflow
+- Fixed mixed up loss intensity order. [#20](https://github.com/matchms/matchms/issues/20)
 
 ## [0.3.3] - 2020-05-27
 
diff --git a/matchms/exporting/__init__.py b/matchms/exporting/__init__.py
index 7295813d..ad6d1c76 100644
--- a/matchms/exporting/__init__.py
+++ b/matchms/exporting/__init__.py
@@ -1,4 +1,8 @@
+from .save_as_json import save_as_json
 from .save_as_mgf import save_as_mgf
 
 
-__all__ = ["save_as_mgf"]
+__all__ = [
+    "save_as_json",
+    "save_as_mgf",
+]
diff --git a/matchms/exporting/save_as_json.py b/matchms/exporting/save_as_json.py
new file mode 100644
index 00000000..327bf5c5
--- /dev/null
+++ b/matchms/exporting/save_as_json.py
@@ -0,0 +1,41 @@
+import json
+from typing import List
+import numpy
+from ..Spectrum import Spectrum
+
+
+def save_as_json(spectrums: List[Spectrum], filename: str):
+    """Save spectrum(s) as json file.
+
+    :py:attr:`~matchms.Spectrum.losses` of spectrum will not be saved.
+
+    Arguments:
+    ----------
+    spectrums:
+        Expected input is a list of  :py:class:`~matchms.Spectrum.Spectrum` objects.
+    filename:
+        Provide filename to save spectrum(s).
+    """
+    if not isinstance(spectrums, list):
+        # Assume that input was single Spectrum
+        spectrums = [spectrums]
+
+    # Write to json file
+    with open(filename, 'w') as fout:
+        json.dump(spectrums, fout, cls=SpectrumJSONEncoder)
+
+
+class SpectrumJSONEncoder(json.JSONEncoder):
+    # pylint: disable=method-hidden
+    # See https://github.com/PyCQA/pylint/issues/414 for reference
+    def default(self, o):
+        """JSON Encoder which can encode a :py:class:`~matchms.Spectrum.Spectrum` object"""
+        if isinstance(o, Spectrum):
+            spec = o.clone()
+            peaks_list = numpy.vstack((spec.peaks.mz, spec.peaks.intensities)).T.tolist()
+
+            # Convert matchms.Spectrum() into dictionaries
+            spectrum_dict = {key: spec.metadata[key] for key in spec.metadata}
+            spectrum_dict["peaks_json"] = peaks_list
+            return spectrum_dict
+        return json.JSONEncoder.default(self, o)
diff --git a/matchms/importing/__init__.py b/matchms/importing/__init__.py
index 5abaeec8..ea37e299 100644
--- a/matchms/importing/__init__.py
+++ b/matchms/importing/__init__.py
@@ -1,8 +1,10 @@
 from .load_adducts import load_adducts
+from .load_from_json import load_from_json
 from .load_from_mgf import load_from_mgf
 
 
 __all__ = [
+    "load_from_json",
     "load_from_mgf",
     "load_adducts"
 ]
diff --git a/matchms/importing/load_from_json.py b/matchms/importing/load_from_json.py
new file mode 100644
index 00000000..32cff6bb
--- /dev/null
+++ b/matchms/importing/load_from_json.py
@@ -0,0 +1,95 @@
+import ast
+import json
+from typing import Union
+import numpy
+from ..Spectrum import Spectrum
+
+
+def load_from_json(filename):
+    """Load spectrum(s) from json file.
+
+JSON document formatted like the `GNPS Spectra library <https://gnps-external.ucsd.edu/gnpslibrary>`_
+    Spectrums with zero peaks will be skipped.
+
+    Args:
+    ----
+    filename: str
+        Provide filename for json file containing spectrum(s).
+    """
+    with open(filename, 'rb') as fin:
+        spectrums = []
+        for spectrum_dict in json.load(fin):
+            spectrum = as_spectrum(spectrum_dict)
+            if spectrum is not None:
+                spectrums.append(spectrum)
+
+    return spectrums
+
+
+def as_spectrum(dct: dict) -> Union[dict, Spectrum, None]:
+    """A :py:func:`json.load` object_hook to convert dictionary shaped like
+    spectrum into :py:class:`~matchms.Spectrum.Spectrum` object.
+
+    Parameters
+    ----------
+    dct
+        Dictionary shaped like spectrum
+    Returns
+    -------
+    A Spectrum or None when no peaks where found.
+    """
+    # Recognize Spectrum by peaks_json key
+    if 'peaks_json' in dct:
+        return dict2spectrum(dct)
+    return None
+
+
+def dict2spectrum(spectrum_dict: dict) -> Union[Spectrum, None]:
+    """Convert dictionary to a :py:class:`~matchms.Spectrum.Spectrum` object.
+
+    Parameters
+    ----------
+    spectrum_dict
+        Dictionary shaped like a single JSON object from the `GNPS Spectra library <https://gnps-external.ucsd.edu/gnpslibrary>`_
+
+    Returns
+    -------
+    A Spectrum or None when no peaks where found.
+    """
+    not_metadata_fields = ["peaks_json"]
+    parse_fieldnames = dict(inchi_aux="inchiaux",
+                            ion_mode="ionmode")
+
+    def get_peaks_list(spectrum_dict, fieldname):
+        peaks_list = spectrum_dict.get(fieldname)
+        if isinstance(peaks_list, list):
+            return peaks_list
+        # Handle peaks list when stored as string
+        if isinstance(peaks_list, str):
+            return ast.literal_eval(peaks_list)
+        return []
+
+    def parse_fieldname(key):
+        """Add options to read GNPS style json files."""
+        key_parsed = key.lower()
+        key_parsed = parse_fieldnames.get(key_parsed, key_parsed)
+        return key_parsed
+
+    metadata_dict = {parse_fieldname(key): spectrum_dict[key]
+                     for key in spectrum_dict if key not in not_metadata_fields}
+    peaks_list = get_peaks_list(spectrum_dict, "peaks_json")
+    if len(peaks_list) > 0 and metadata_dict:
+        mz = numpy.array(peaks_list)[:, 0]
+        intensities = numpy.array(peaks_list)[:, 1]
+
+        # Sort by mz (if not sorted already)
+        if not numpy.all(mz[:-1] <= mz[1:]):
+            idx_sorted = numpy.argsort(mz)
+            mz = mz[idx_sorted]
+            intensities = intensities[idx_sorted]
+        return Spectrum(mz=mz,
+                        intensities=intensities,
+                        metadata=metadata_dict)
+    print("Empty spectrum found (no peaks in 'peaks_json').",
+          "Will not be imported.")
+    return None


Test patch:
diff --git a/tests/test_save_as_json_load_from_json.py b/tests/test_save_as_json_load_from_json.py
new file mode 100644
index 00000000..1625cdef
--- /dev/null
+++ b/tests/test_save_as_json_load_from_json.py
@@ -0,0 +1,95 @@
+import json
+import os
+import numpy
+from matchms import Spectrum
+from matchms.exporting import save_as_json
+from matchms.importing import load_from_json
+
+
+def test_save_and_load_json_single_spectrum(tmp_path):
+    """Test saving spectrum to .json file"""
+    spectrum = Spectrum(mz=numpy.array([100, 200, 300], dtype="float"),
+                        intensities=numpy.array([10, 10, 500], dtype="float"),
+                        metadata={"charge": -1,
+                                  "inchi": '"InChI=1S/C6H12"',
+                                  "precursor_mz": 222.2,
+                                  "test_field": "test"})
+    # Write to test file
+    filename = os.path.join(tmp_path, "test.json")
+    save_as_json(spectrum, filename)
+
+    # test if file exists
+    assert os.path.isfile(filename)
+
+    # Test if content of json file is correct
+    spectrum_import = load_from_json(filename)[0]
+    assert spectrum_import == spectrum, "Original and saved+loaded spectrum not identical"
+
+
+def test_save_and_load_json_spectrum_list(tmp_path):
+    """Test saving spectrum list to .json file"""
+    spectrum1 = Spectrum(mz=numpy.array([100, 200, 300], dtype="float"),
+                         intensities=numpy.array([10, 10, 500], dtype="float"),
+                         metadata={"test_field": "test1"})
+
+    spectrum2 = Spectrum(mz=numpy.array([100, 200, 300], dtype="float"),
+                         intensities=numpy.array([10, 10, 500], dtype="float"),
+                         metadata={"test_field": "test2"})
+    # Write to test file
+    filename = os.path.join(tmp_path, "test.json")
+    save_as_json([spectrum1, spectrum2], filename)
+
+    # test if file exists
+    assert os.path.isfile(filename)
+
+    # Test if content of json file is correct
+    spectrum_imports = load_from_json(filename)
+    assert spectrum_imports[0] == spectrum1, "Original and saved+loaded spectrum not identical"
+    assert spectrum_imports[1] == spectrum2, "Original and saved+loaded spectrum not identical"
+
+
+def test_load_from_json_zero_peaks(tmp_path):
+    spectrum1 = Spectrum(mz=numpy.array([], dtype="float"),
+                         intensities=numpy.array([], dtype="float"),
+                         metadata={"test_field": "test1"})
+
+    filename = tmp_path / "test.json"
+
+    save_as_json([spectrum1], filename)
+
+    # test if file exists
+    assert os.path.isfile(filename)
+
+    # Test if content of json file is correct
+    spectrum_imports = load_from_json(filename)
+    assert len(spectrum_imports) == 0, "Spectrum without peaks should be skipped"
+
+
+def test_load_from_json_with_minimal_json(tmp_path):
+    filename = tmp_path / "test.json"
+    body = '[{"test_field": "test1", "peaks_json": [[100.0, 10.0], [200.0, 10.0], [300.0, 500.0]]}]'
+
+    with open(filename, 'w') as f:
+        f.write(body)
+
+    spectrum_imports = load_from_json(filename)
+
+    expected = Spectrum(mz=numpy.array([100, 200, 300], dtype="float"),
+                        intensities=numpy.array([10, 10, 500], dtype="float"),
+                        metadata={"test_field": "test1"})
+    assert spectrum_imports == [expected], "Loaded JSON document not identical to expected Spectrum"
+
+
+def test_save_as_json_with_minimal_json(tmp_path):
+    filename = tmp_path / "test.json"
+    spectrum1 = Spectrum(mz=numpy.array([100, 200, 300], dtype="float"),
+                         intensities=numpy.array([10, 10, 500], dtype="float"),
+                         metadata={"test_field": "test1"})
+
+    save_as_json([spectrum1], filename)
+
+    with open(filename) as f:
+        spectrum_imports = json.load(f)
+
+    expected = [{"test_field": "test1", "peaks_json": [[100.0, 10.0], [200.0, 10.0], [300.0, 500.0]]}]
+    assert spectrum_imports == expected, "Saved Spectrum not identical to expected JSON Document"