Repository: matchms/matchms
Pull request #401 (2023-04-17 18:39:50)

switch to to_dict() method
@hechth @zargham-ahmad
I added some changes. Let me know what you think of this.

I also checked the failing exporting_metadata test which happend because "accession" and "db#" are now both being mapped to "spectrum_id" according to your key_conversion table update (seems fine to, but I am not the best person to judge this).

If the changes look OK, the `update_keys()` method is in principle no longer needed.


Patch:
diff --git a/.github/workflows/CI_build.yml b/.github/workflows/CI_build.yml
index 465351bf..f9b2f91c 100644
--- a/.github/workflows/CI_build.yml
+++ b/.github/workflows/CI_build.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Build package and create dev environment
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[dev]
+          pip install -e .[dev,chemistry]
       - name: Show pip list
         run: |
           pip list
@@ -67,10 +67,10 @@ jobs:
     needs: first_check
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.8
       - name: Python info
         run: |
           which python
@@ -110,54 +110,30 @@ jobs:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
-        python-version: ['3.7', '3.8', '3.9']
+        python-version: ['3.7', '3.8', '3.9', '3.10']
         exclude:
           - os: ubuntu-latest
             python-version: 3.9
     steps:
       - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
         with:
-          fetch-depth: "0"
-      - name: Create conda environment
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
           python-version: ${{ matrix.python-version }}
-      - name: Show conda config
-        shell: bash -l {0}
-        run: |
-          conda info
-          conda list
-          conda config --show-sources
-          conda config --show
-          conda env list
       - name: Python info
-        shell: bash -l {0}
         run: |
           which python
           python --version
-      - name: Show environment variables
-        shell: bash -l {0}
-        run: |
-          env | sort
-      - name: Install conda dependencies
-        shell: bash -l {0}
-        run: |
-          conda install -c conda-forge rdkit
-      - name: Install dev dependencies
-        shell: bash -l {0}
+      - name: Install dependencies (includinv dev + chemistry)
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[dev]
-      - name: Show conda list
-        shell: bash -l {0}
-        run: conda list
+          pip install -e .[dev,chemistry]
       - name: Show pip list
-        shell: bash -l {0}
-        run: pip list
+        run: |
+          pip list
       - name: Run tests
-        shell: bash -l {0}
-        run: pytest
+        run: |
+          pytest
 
 
   test_with_conda:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f9f78501..25c00c74 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Added function to infer filetype when loading spectra
 
 ### Changed
 
diff --git a/README.rst b/README.rst
index 66a60b92..ca66c274 100644
--- a/README.rst
+++ b/README.rst
@@ -125,7 +125,7 @@ Installation
 
 Prerequisites:  
 
-- Python 3.7, 3.8 or 3.9, (3.10 and 3.11 should work as well, but are not yet tested systematically)
+- Python 3.7, 3.8, 3.9 or 3.10, (higher versions should work as well, but are not yet tested systematically)
 - Anaconda (recommended)
 
 We recommend installing matchms in a new virtual environment to avoid dependency clashes
@@ -136,12 +136,12 @@ We recommend installing matchms in a new virtual environment to avoid dependency
   conda activate matchms
   conda install --channel bioconda --channel conda-forge matchms
 
-Alternatively, matchms can also be installed using ``pip`` but users will then either have to install ``rdkit`` on their own or won't be able to use the entire functionality. Without ``rdkit`` installed several filter functions related to processing and cleaning chemical metadata will not run.
-To install matchms with ``pip`` simply run
+Alternatively, matchms can also be installed using ``pip``. In the most basic version matchms will then come without ``rdkit`` so that several filter functions related to processing and cleaning chemical metadata will not run. To include ``rdkit`` install matchms as ``matchms[chemistry]``:
 
 .. code-block:: console
 
-  pip install matchms
+  pip install matchms  # simple install w/o rdkit
+  pip install matchms[chemistry]  # full install including rdkit
 
 matchms ecosystem -> additional functionalities
 ===============================================
diff --git a/matchms/Metadata.py b/matchms/Metadata.py
index 595e24a0..7b5e06d9 100644
--- a/matchms/Metadata.py
+++ b/matchms/Metadata.py
@@ -5,7 +5,7 @@ from .filtering.add_precursor_mz import _add_precursor_mz_metadata
 from .filtering.add_retention import _add_retention
 from .filtering.interpret_pepmass import _interpret_pepmass_metadata
 from .filtering.make_charge_int import _convert_charge_to_int
-from .utils import load_known_key_conversions, load_export_key_conversions
+from .utils import load_export_key_conversions, load_known_key_conversions
 
 
 _key_regex_replacements = {r"\s": "_",
@@ -113,19 +113,6 @@ class Metadata:
 
         self.data = metadata_filtered
 
-    def update_keys(self, key_conversions: dict):
-
-        rem_list = [x for x in self._data.keys() if x not in key_conversions]
-        for key in rem_list:
-            del self._data[key]
-        
-        self._data.force_lower_case = False
-        self._data.key_regex_replacements = None
-        self._data.key_replacements = key_conversions
-
-        if '' in self._data.keys():
-            del self._data['']
-
     # ------------------------------
     # Getters and Setters
     # ------------------------------
@@ -157,6 +144,26 @@ class Metadata:
         """
         return self._data.items()
 
+    def to_dict(self, export_style: str = "matchms"):
+        """Returns a regular Python dictionary containing the metadata entries.
+
+        Parameters
+        ----------
+        export_style:
+            Specifies the naming style of the metadata fields.
+            Default is "matchms".
+        """
+        if export_style == "matchms":
+            return dict(self._data)
+        key_conversions = load_export_key_conversions(export_style=export_style)
+        keep_list = [x for x in self._data.keys() if x in key_conversions]
+        converted_dict = {}
+        for key in keep_list:
+            value = self._data[key]
+            if value != "":
+                converted_dict[key_conversions[key]] = self._data[key]
+        return converted_dict
+
     def __getitem__(self, key=None):
         return self.get(key)
 
diff --git a/matchms/Spectrum.py b/matchms/Spectrum.py
index 0d8a7612..d19a4ebd 100644
--- a/matchms/Spectrum.py
+++ b/matchms/Spectrum.py
@@ -182,10 +182,9 @@ class Spectrum:
         if "fingerprint" in spectrum_dict:
             spectrum_dict["fingerprint"] = spectrum_dict["fingerprint"].tolist()
         return spectrum_dict
-    
-    def update_keys(self, key_conversions: dict):
-        self._metadata.update_keys(key_conversions)
-        return self
+
+    def metadata_dict(self, export_style: str = "matchms") -> dict:
+        return self._metadata.to_dict(export_style)
 
     @property
     def mz(self):
diff --git a/matchms/exporting/metadata_export.py b/matchms/exporting/metadata_export.py
index c6330ab2..e42e019d 100644
--- a/matchms/exporting/metadata_export.py
+++ b/matchms/exporting/metadata_export.py
@@ -20,7 +20,7 @@ def _get_metadata_dict(spectrum: Spectrum, include_fields: Optional[List[str]] =
     if not isinstance(include_fields, list):
         print("'Include_fields' must be 'all' or list of keys.")
         return None
-        
+
     return {key: spectrum.metadata[key] for key in spectrum.metadata.keys()
             & include_fields}
 
@@ -72,13 +72,18 @@ def export_metadata_as_csv(spectra: List[Spectrum], filename: str,
         for data in metadata:
             writer.writerow(data)
 
+
 def _subset_metadata(include_fields: List[str], metadata: np.array, columns: Set[str]) -> Tuple[np.array, Set[str]]:
     """Subset metadata to 'include_fields' and return intersection of columns.
 
-    Args:
-        include_fields (List[str]): Columns to include.
-        metadata (np.array): Data to subset.
-        columns (set[str]): Set of columns present in data
+    Parameters
+    ----------
+    include_fields:
+        Columns to include.
+    metadata:
+        Data to subset.
+    columns:
+        Set of columns present in data
 
     Returns:
         Tuple[np.array, set[str]]: Subset data and columns.
@@ -89,8 +94,10 @@ def _subset_metadata(include_fields: List[str], metadata: np.array, columns: Set
 def get_metadata_as_array(spectra: List[Spectrum]) -> Tuple[np.array, List[str]]:
     """Extract union of all metadata as numpy array from all spectra.
 
-    Args:
-        spectra (List[Spectrum]): Spectra from which to collect metadata.
+    Parameters
+    ----------
+    spectra:
+        Spectra from which to collect metadata.
 
     Returns:
         Tuple[np.array, List[str]]: Metadata and union of all columns detected in all spectra.
@@ -102,8 +109,8 @@ def get_metadata_as_array(spectra: List[Spectrum]) -> Tuple[np.array, List[str]]
     values = []
 
     for s in spectra:
-        v = tuple((s.get(k) for k in keys))
-        values.append(v)
-    
-    d = np.array(values, dtype=[(k, np.chararray) for k in keys])
-    return d, keys
\ No newline at end of file
+        value = tuple((s.get(k) for k in keys))
+        values.append(value)
+
+    values_array = np.array(values, dtype=[(k, np.chararray) for k in keys])
+    return values_array, keys
diff --git a/matchms/exporting/save_as_msp.py b/matchms/exporting/save_as_msp.py
index 4a1b0eed..1ac0c467 100644
--- a/matchms/exporting/save_as_msp.py
+++ b/matchms/exporting/save_as_msp.py
@@ -1,8 +1,6 @@
 import logging
 import os
 from typing import IO, Dict, List, Union
-
-from matchms.utils import load_export_key_conversions
 from ..Fragments import Fragments
 from ..Spectrum import Spectrum
 
@@ -61,17 +59,16 @@ def save_as_msp(spectrums: List[Spectrum], filename: str,
                        filename.split(".")[-1])
     spectrums = _ensure_list(spectrums)
 
-    if style != "matchms":
-        conversions = load_export_key_conversions(export_style = style)
-        spectrums = list(map(lambda x: x.clone().update_keys(conversions), spectrums))
-
     with open(filename, mode, encoding="utf-8") as outfile:
         for spectrum in spectrums:
-            _write_spectrum(spectrum, outfile, write_peak_comments)
+            _write_spectrum(spectrum, outfile, write_peak_comments, style)
 
 
-def _write_spectrum(spectrum: Spectrum, outfile: IO, write_peak_comments: bool):
-    _write_metadata(spectrum.metadata, outfile)
+def _write_spectrum(spectrum: Spectrum,
+                    outfile: IO,
+                    write_peak_comments: bool,
+                    export_style: str = "matchms"):
+    _write_metadata(spectrum.metadata_dict(export_style), outfile)
     if write_peak_comments is True:
         _write_peaks(spectrum.peaks, spectrum.peak_comments, outfile)
     else:
diff --git a/matchms/importing/__init__.py b/matchms/importing/__init__.py
index de18f101..f18921d1 100644
--- a/matchms/importing/__init__.py
+++ b/matchms/importing/__init__.py
@@ -21,6 +21,7 @@ from .load_from_mzml import load_from_mzml
 from .load_from_mzxml import load_from_mzxml
 from .load_from_usi import load_from_usi
 from .load_scores import scores_from_json, scores_from_pickle
+from .load_spectra import load_from_pickle, load_spectra
 
 
 __all__ = [
@@ -30,6 +31,8 @@ __all__ = [
     "load_from_mzml",
     "load_from_mzxml",
     "load_from_usi",
+    "load_spectra",
+    "load_from_pickle",
     "scores_from_json",
     "scores_from_pickle",
 ]
diff --git a/matchms/importing/load_spectra.py b/matchms/importing/load_spectra.py
new file mode 100644
index 00000000..e2943406
--- /dev/null
+++ b/matchms/importing/load_spectra.py
@@ -0,0 +1,62 @@
+import os
+import pickle
+from typing import Any, Generator, Optional
+from matchms.importing import (load_from_json, load_from_mgf, load_from_msp,
+                               load_from_mzml, load_from_mzxml, load_from_usi)
+from matchms.typing import SpectrumType
+
+
+def load_spectra(file: str, ftype: Optional[str] = None) -> Generator[SpectrumType, None, None]:
+    """Loads spectra from your spectrum file into memory as matchms Spectrum object
+
+    The following file extensions can be loaded in with this function:
+    "mzML", "json", "mgf", "msp", "mzxml", "usi" and "pickle".
+    A pickled file is expected to directly contain a list of matchms spectrum objects.
+
+    Args:
+    -----
+    file:
+        Path to file containing spectra, with file extension "mzML", "json", "mgf", "msp",
+        "mzxml", "usi" or "pickle"
+    ftype:
+        Optional. Filetype
+    """
+    assert os.path.exists(file), f"The specified file: {file} does not exists"
+
+    if ftype is None:
+        ftype = os.path.splitext(file)[1].lower()[1:]
+    else:
+        ftype = ftype.lower()
+
+    if ftype == "mzml":
+        return load_from_mzml(file)
+    if ftype == "json":
+        return load_from_json(file)
+    if ftype == "mgf":
+        return load_from_mgf(file)
+    if ftype == "msp":
+        return load_from_msp(file)
+    if ftype == "mzxml":
+        return load_from_mzxml(file)
+    if ftype == "usi":
+        return load_from_usi(file)
+    if ftype == "pickle":
+        spectra = load_from_pickle(file)
+        assert isinstance(spectra, list), "Expected list of spectra"
+        assert isinstance(spectra[0], SpectrumType), "Expected list of spectra"
+        return spectra
+    assert False, f"File extension of file: {file} is not recognized"
+
+
+def load_from_pickle(filename: str) -> Any:
+    """Load spectra stored in pickle
+
+    Args:
+        filename (str): Pickled file with spectra.
+
+    Returns:
+        Any: Unpickled object. Should be a list of Spectra.
+    """
+    with open(filename, 'rb') as file:
+        loaded_object = pickle.load(file)
+    return loaded_object
diff --git a/matchms/utils.py b/matchms/utils.py
index ffd38a84..9abbd034 100644
--- a/matchms/utils.py
+++ b/matchms/utils.py
@@ -62,4 +62,4 @@ def _load_key_conversions(file: str, source: str, target: str) -> dict:
         for row in reader:
             key_conversions[row[source]] = row[target]
 
-    return key_conversions
\ No newline at end of file
+    return key_conversions


Test patch:
diff --git a/tests/test_exporting_metadata.py b/tests/test_exporting_metadata.py
index 7a2e2329..916b213d 100644
--- a/tests/test_exporting_metadata.py
+++ b/tests/test_exporting_metadata.py
@@ -30,8 +30,8 @@ def test_get_metadata_dict():
 def test_get_metadata_as_array(spectra):
     actual, colnames = get_metadata_as_array(spectra)
     assert len(actual) == 10
-    assert len(colnames) == 24
-    
+    assert len(colnames) == 23
+
 
 def test_export_as_csv(tmp_path, spectra):
     expected = os.path.join(os.path.dirname(__file__), "expected_metadata.csv")    
diff --git a/tests/test_key_conversion.py b/tests/test_key_conversion.py
index 5ea83dce..2ab743af 100644
--- a/tests/test_key_conversion.py
+++ b/tests/test_key_conversion.py
@@ -1,8 +1,7 @@
 import pytest
-
-from matchms.utils import load_export_key_conversions
 from .builder_Spectrum import SpectrumBuilder
 
+
 @pytest.mark.parametrize("input_dict, expected, export_style", [
     [{"precursor_mz": 101.01}, {"PrecursorMZ": 101.01}, "nist"],
     [{"precursormz": 101.01}, {"PRECURSORMZ": 101.01}, "riken"],
@@ -12,6 +11,6 @@ from .builder_Spectrum import SpectrumBuilder
     [{"ionmode": "Negative"}, {"IONMODE": "Negative"}, "riken"]])
 def test_key_conversion(input_dict, expected, export_style):
     spectrum = SpectrumBuilder().with_metadata(input_dict).build()
-    spectrum.update_keys(load_export_key_conversions(export_style = export_style))
+    actual = spectrum.metadata_dict(export_style)
 
-    assert spectrum.metadata == expected
+    assert actual == expected
diff --git a/tests/test_load_spectra.py b/tests/test_load_spectra.py
new file mode 100644
index 00000000..cbb7d2d1
--- /dev/null
+++ b/tests/test_load_spectra.py
@@ -0,0 +1,30 @@
+import os
+import pytest
+from matchms import Spectrum
+from matchms.importing.load_spectra import load_spectra
+
+
+def test_load_spectra_unknown_file(tmp_path):
+    """Tests if unknown file raises an Assertion error"""
+    with pytest.raises(AssertionError):
+        load_spectra(os.path.join(tmp_path, "file_that_does_not_exist.json"))
+
+
+@pytest.mark.parametrize("filename, ftype, expected_num_spectra", [
+    ["pesticides.mgf", None, 76],
+    ["testdata.mgf", "mgf", 30],
+    ["testdata.mzml", None, 10],
+    ["testdata.mzXML", None, 1],
+    ["massbank_five_spectra.msp", None, 5]
+])
+def test_load_spectra(filename, ftype, expected_num_spectra):
+    """Test if pickled file is loaded in correctly"""
+    tests_root = os.path.join(os.path.dirname(__file__), "../tests")
+    file = os.path.join(tests_root, filename)
+    actual = list(load_spectra(file, ftype=ftype))
+   
+        
+    assert isinstance(actual, list), "expected list of spectra"
+    assert len(actual) == expected_num_spectra
+    for spectrum in actual:
+        assert isinstance(spectrum, Spectrum)
diff --git a/tests/test_metadata.py b/tests/test_metadata.py
index ee4defef..2f88b0db 100644
--- a/tests/test_metadata.py
+++ b/tests/test_metadata.py
@@ -55,6 +55,21 @@ def test_metadata_setitem_getitem(harmonize, set_key, set_value, get_key, get_va
         "Expected different _metadata dictionary."
 
 
+@pytest.mark.parametrize("input_dict, export_style, expected", [
+    [None, "matchms", {}],
+    [{"precursor_mz": 101.01}, "matchms", {"precursor_mz": 101.01}],
+    [{"peptide_modifications": 1}, "massbank", {"COMMENT:PEPTIDE_MODIFICATIONS": 1}],
+    [{"ionmode": "Negative"}, "massbank", {"AC$MASS_SPECTROMETRY:ION_MODE": "Negative"}],
+    [{"compound_name": "Dummy"}, "nist", {"Name": "Dummy"}],
+    [{"compound_name": "Dummy"}, "riken", {"NAME": "Dummy"}],
+    [{"compound_name": "Dummy"}, "gnps", {"NAME": "Dummy"}],
+    ])
+def test_metadata_to_dict(input_dict, export_style, expected):
+    metadata = Metadata(input_dict)
+    assert metadata.to_dict(export_style) == expected, \
+        "Expected different metadata dictionary."
+
+
 @pytest.mark.parametrize("dict1, dict2, expected", [
     [{"precursor_mz": 101.01}, {"precursor_mz": 101.01}, True],
     [{"abc": ["a", 5, 7.01]}, {"abc": ["a", 7.01, 5]}, False],
diff --git a/tests/test_save_as_msp.py b/tests/test_save_as_msp.py
index 7bc52ed3..8e63881b 100644
--- a/tests/test_save_as_msp.py
+++ b/tests/test_save_as_msp.py
@@ -57,15 +57,6 @@ def save_and_reload_spectra(filename, spectra: List[Spectrum], write_peak_commen
     return reloaded_spectra
 
 
-def test_spectrum_none_exception(none_spectrum, filename):
-    """ Test for exception being thrown if the spectrum to be saved. """
-    with pytest.raises(AttributeError) as exception:
-        save_as_msp(none_spectrum, filename)
-
-    message = exception.value.args[0]
-    assert message == "'NoneType' object has no attribute 'metadata'"
-
-
 def test_wrong_filename_exception():
     """ Test for exception being thrown if output file doesn't end with .msp. """
     with tempfile.TemporaryDirectory() as temp_dir: