Repository: matchms/matchms

Issue:
Add retention data filters
As discussed in #251 , adding RI and RT filters to clean the metadata in the spectrum object would be good.

One additional idea would be for the retention time filter to allow passing a conversion factor which converts the RT to seconds and use this as a canonical unit for RT?


Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95f5f529..8d688ee0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,11 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - `Spectrum()` objects now also allows generating hashes, e.g. `hash(spectrum)` [#259](https://github.com/matchms/matchms/pull/259)
 - `Spectrum()` objects can generate `.spectrum_hash()` and `.metadata_hash()` to track changes to peaks or metadata [#259](https://github.com/matchms/matchms/pull/259)
-- 'load_from_mf()` now accepts both a path to a mgf file or a file-like object from a preloaded MGF file [#258](https://github.com/matchms/matchms/pull/258)
-
+- `load_from_mgf()` now accepts both a path to a mgf file or a file-like object from a preloaded MGF file [#258](https://github.com/matchms/matchms/pull/258)
+- Added `add_retention` filters with function `add_retention_time()` and `add_retention_index()` [#265](https://github.com/matchms/matchms/pull/265)
 ### Changed
 
 - Code linting triggered by pylint update [#257](https://github.com/matchms/matchms/pull/257)
+- Refactored `add_parent_mass()` filter can now also handle missing charge entries (if ionmode is known) [#252](https://github.com/matchms/matchms/pull/252)
 
 ## [0.9.2] - 2021-07-20
 
diff --git a/matchms/filtering/__init__.py b/matchms/filtering/__init__.py
index 249386d1..93ccdaa9 100644
--- a/matchms/filtering/__init__.py
+++ b/matchms/filtering/__init__.py
@@ -41,6 +41,8 @@ from .add_fingerprint import add_fingerprint
 from .add_losses import add_losses
 from .add_parent_mass import add_parent_mass
 from .add_precursor_mz import add_precursor_mz
+from .add_retention import add_retention_index
+from .add_retention import add_retention_time
 from .clean_compound_name import clean_compound_name
 from .correct_charge import correct_charge
 from .default_filters import default_filters
@@ -77,6 +79,8 @@ __all__ = [
     "add_losses",
     "add_parent_mass",
     "add_precursor_mz",
+    "add_retention_index",
+    "add_retention_time",
     "clean_compound_name",
     "correct_charge",
     "default_filters",
diff --git a/matchms/filtering/add_parent_mass.py b/matchms/filtering/add_parent_mass.py
index 0f02745c..e7b3bd81 100644
--- a/matchms/filtering/add_parent_mass.py
+++ b/matchms/filtering/add_parent_mass.py
@@ -32,28 +32,56 @@ def add_parent_mass(spectrum_in: SpectrumType, estimate_from_adduct: bool = True
     spectrum = spectrum_in.clone()
     adducts_dict = load_adducts_dict()
 
-    if spectrum.get("parent_mass", None) is None or overwrite_existing_entry:
-        parent_mass = None
-        charge = spectrum.get("charge")
-        adduct = clean_adduct(spectrum.get("adduct"))
-        precursor_mz = spectrum.get("precursor_mz", None)
-        if precursor_mz is None:
-            print("Missing precursor m/z to derive parent mass.")
-            return spectrum
-
-        if estimate_from_adduct and adduct in adducts_dict:
-            multiplier = adducts_dict[adduct]["mass_multiplier"]
-            correction_mass = adducts_dict[adduct]["correction_mass"]
-            parent_mass = precursor_mz * multiplier - correction_mass
-
-        if parent_mass is None and charge is not None and charge != 0:
-            # Otherwise assume adduct of shape [M+xH] or [M-xH]
-            protons_mass = PROTON_MASS * charge
-            precursor_mass = precursor_mz * abs(charge)
-            parent_mass = precursor_mass - protons_mass
-
-        if parent_mass is None:
-            print("Not sufficient spectrum metadata to derive parent mass.")
-        else:
-            spectrum.set("parent_mass", float(parent_mass))
+    if spectrum.get("parent_mass", None) and not overwrite_existing_entry:
+        return spectrum
+
+    parent_mass = None
+    charge = _get_charge(spectrum)
+    adduct = clean_adduct(spectrum.get("adduct"))
+    precursor_mz = spectrum.get("precursor_mz", None)
+    if precursor_mz is None:
+        print("Missing precursor m/z to derive parent mass.")
+        return spectrum
+
+    if estimate_from_adduct and (adduct in adducts_dict):
+        multiplier = adducts_dict[adduct]["mass_multiplier"]
+        correction_mass = adducts_dict[adduct]["correction_mass"]
+        parent_mass = precursor_mz * multiplier - correction_mass
+
+    if (parent_mass is None) and _is_valid_charge(charge):
+        # Assume adduct of shape [M+xH] or [M-xH]
+        protons_mass = PROTON_MASS * charge
+        precursor_mass = precursor_mz * abs(charge)
+        parent_mass = precursor_mass - protons_mass
+
+    if parent_mass is None:
+        print("Not sufficient spectrum metadata to derive parent mass.")
+    else:
+        spectrum.set("parent_mass", float(parent_mass))
     return spectrum
+
+
+def _is_valid_charge(charge):
+    return (charge is not None) and (charge != 0)
+
+
+def _get_charge(spectrum):
+    """Get charge from `Spectrum()` object.
+    In case no valid charge is found, guess +1 or -1 based on ionmode.
+    Else return 0.
+    """
+    charge = spectrum.get("charge")
+    if _is_valid_charge(charge):
+        return charge
+    if spectrum.get('ionmode') == "positive":
+        print("Missing charge entry, but positive ionmode detected. "
+              "Consider prior run of `correct_charge()` filter.")
+        return 1
+    if spectrum.get('ionmode') == "negative":
+        print("Missing charge entry, but negative ionmode detected. "
+              "Consider prior run of `correct_charge()` filter.")
+        return -1
+
+    print("Missing charge and ionmode entries. "
+          "Consider prior run of `derive_ionmode()` and `correct_charge()` filters.")
+    return 0
diff --git a/matchms/filtering/add_precursor_mz.py b/matchms/filtering/add_precursor_mz.py
index 2022653f..6aeb8479 100644
--- a/matchms/filtering/add_precursor_mz.py
+++ b/matchms/filtering/add_precursor_mz.py
@@ -1,19 +1,9 @@
-from typing import Iterable
-from typing import TypeVar
+from matchms.utils import get_first_common_element
 from ..typing import SpectrumType
 
 
-_T = TypeVar('_T')
 _accepted_keys = ["precursor_mz", "precursormz", "precursor_mass"]
 _accepted_types = (float, str, int)
-_convertible_types = (str, int)
-
-
-def get_first_common_element(first: Iterable[_T], second: Iterable[_T]) -> _T:
-    """ Get first common element from two lists.
-    Returns 'None' if there are no common elements.
-    """
-    return next((item for item in first if item in second), None)
 
 
 def add_precursor_mz(spectrum_in: SpectrumType) -> SpectrumType:
diff --git a/matchms/filtering/add_retention.py b/matchms/filtering/add_retention.py
new file mode 100644
index 00000000..f5b38c5e
--- /dev/null
+++ b/matchms/filtering/add_retention.py
@@ -0,0 +1,108 @@
+from typing import Any
+from typing import List
+from typing import Optional
+from matchms.utils import filter_none
+from matchms.utils import get_common_keys
+from ..typing import SpectrumType
+
+
+_retention_time_keys = ["retention_time", "retentiontime", "rt", "scan_start_time", "RT_Query"]
+_retention_index_keys = ["retention_index", "retentionindex", "ri"]
+
+
+def safe_store_value(spectrum: SpectrumType, value: Any, target_key: str) -> SpectrumType:
+    """Helper function to safely store a value in the target key without throwing an exception, but storing 'None' instead.
+
+    Args:
+        spectrum (SpectrumType): Spectrum to which to add 'value' in 'target_key'.
+        value (Any): Value to parse into 'target_key'.
+        target_key (str): Name of the key in which to store the value.
+
+    Returns:
+        SpectrumType: Spectrum with added key.
+    """
+    if value is not None:   # one of accepted keys is present
+        value = safe_convert_to_float(value)
+    spectrum.set(target_key, value)
+    return spectrum
+
+
+def safe_convert_to_float(value: Any) -> Optional[float]:
+    """Safely convert value to float. Return 'None' on failure.
+
+    Args:
+        value (Any): Object to convert to float.
+
+    Returns:
+        Optional[float]: Converted float value or 'None' if conversion is not possible.
+    """
+    if isinstance(value, list) and len(value) == 1:
+        value = value[0]
+    try:
+        value = float(value)
+        rt = value if value >= 0 else None  # discard negative RT values
+    except ValueError:
+        print(f"{value} can't be converted to float.")
+        rt = None
+    return rt
+
+
+def _add_retention(spectrum: SpectrumType, target_key: str, accepted_keys: List[str]) -> SpectrumType:
+    """Add value from one of accepted keys to target key.
+
+    Args:
+        spectrum (SpectrumType): Spectrum from which to read the values.
+        target_key (str): Key under which to store the value.
+        accepted_keys (List[str]): List of accepted keys from which a value will be read (in order).
+
+    Returns:
+        SpectrumType: Spectrum with value from first accepted key stored under target_key.
+    """
+    common_keys = get_common_keys(spectrum.metadata.keys(), accepted_keys)
+    values_for_keys = filter_none([spectrum.get(key) for key in common_keys])
+    values = list(map(safe_convert_to_float, values_for_keys))
+    value = next(filter_none(values), None)
+
+    spectrum = safe_store_value(spectrum, value, target_key)
+    return spectrum
+
+
+def add_retention_time(spectrum_in: SpectrumType) -> SpectrumType:
+    """Add retention time information to the 'retention_time' key as float.
+    Negative values and those not convertible to a float result in 'retention_time'
+    being 'None'.
+
+    Args:
+        spectrum_in (SpectrumType): Spectrum with retention time information.
+
+    Returns:
+        SpectrumType: Spectrum with harmonized retention time information.
+    """
+    if spectrum_in is None:
+        return None
+
+    spectrum = spectrum_in.clone()
+
+    target_key = "retention_time"
+    spectrum = _add_retention(spectrum, target_key, _retention_time_keys)
+    return spectrum
+
+
+def add_retention_index(spectrum_in: SpectrumType) -> SpectrumType:
+    """Add retention index into 'retention_index' key if present.
+
+
+    Args:
+        spectrum_in (SpectrumType): Spectrum with RI information.
+
+    Returns:
+        SpectrumType: Spectrum with RI info stored under 'retention_index'.
+    """
+    if spectrum_in is None:
+        return None
+
+    spectrum = spectrum_in.clone()
+
+    target_key = "retention_index"
+    spectrum = _add_retention(spectrum, target_key, _retention_index_keys)
+    return spectrum
diff --git a/matchms/utils.py b/matchms/utils.py
index c12897a5..6bc394c4 100644
--- a/matchms/utils.py
+++ b/matchms/utils.py
@@ -1,4 +1,6 @@
 import re
+from typing import Iterable
+from typing import List
 from typing import Optional
 import numpy
 from .importing import load_adducts_dict
@@ -303,3 +305,35 @@ def clean_adduct(adduct: str) -> str:
 
     adduct_cleaned = adduct_core[:-len(adduct_charge)] + "]" + adduct_charge
     return adduct_conversion(adduct_cleaned)
+
+
+def get_first_common_element(first: Iterable[str], second: Iterable[str]) -> str:
+    """ Get first common element from two lists.
+    Returns 'None' if there are no common elements.
+    """
+    return next((item for item in first if item in second), None)
+
+
+def get_common_keys(first: List[str], second: List[str]) -> List[str]:
+    """Get common elements of two sets of strings in a case insensitive way.
+
+    Args:
+        first (List[str]): First list of strings.
+        second (List[str]): List of strings to search for matches.
+
+    Returns:
+        List[str]: List of common elements without regarding case of first list.
+    """
+    return [value for value in first if value in second or value.lower() in second]
+
+
+def filter_none(iterable: Iterable) -> Iterable:
+    """Filter iterable to remove 'None' elements.
+
+    Args:
+        iterable (Iterable): Iterable to filter.
+
+    Returns:
+        Iterable: Filtered iterable.
+    """
+    return filter(lambda x: x is not None, iterable)


Test patch:
diff --git a/tests/test_add_parent_mass.py b/tests/test_add_parent_mass.py
index ca411603..3fb5d5f7 100644
--- a/tests/test_add_parent_mass.py
+++ b/tests/test_add_parent_mass.py
@@ -1,6 +1,7 @@
 import numpy
 import pytest
 from matchms import Spectrum
+from matchms.constants import PROTON_MASS
 from matchms.filtering import add_parent_mass
 
 
@@ -90,7 +91,9 @@ def test_add_parent_mass_using_adduct(adduct, expected):
     assert isinstance(spectrum.get("parent_mass"), float), "Expected parent mass to be float."
 
 
-def test_add_parent_mass_overwrite():
+@pytest.mark.parametrize("overwrite, expected", [(True, 442.992724),
+                                                 (False, 443.0)])
+def test_add_parent_mass_overwrite(overwrite, expected):
     """Test if parent mass is replaced by newly calculated value."""
     mz = numpy.array([], dtype='float')
     intensities = numpy.array([], dtype='float')
@@ -102,9 +105,9 @@ def test_add_parent_mass_overwrite():
                            intensities=intensities,
                            metadata=metadata)
 
-    spectrum = add_parent_mass(spectrum_in, overwrite_existing_entry=True)
+    spectrum = add_parent_mass(spectrum_in, overwrite_existing_entry=overwrite)
 
-    assert numpy.allclose(spectrum.get("parent_mass"), 442.992724, atol=1e-4), \
+    assert numpy.allclose(spectrum.get("parent_mass"), expected, atol=1e-4), \
         "Expected parent mass to be replaced by new value."
 
 
@@ -128,3 +131,21 @@ def test_empty_spectrum():
     spectrum = add_parent_mass(spectrum_in)
 
     assert spectrum is None, "Expected different handling of None spectrum."
+
+
+@pytest.mark.parametrize("ionmode, expected", [("positive", 444.0 - PROTON_MASS),
+                                               ("negative", 444.0 + PROTON_MASS)])
+def test_use_of_ionmode(ionmode, expected):
+    """Test when there is no charge given, than the ionmode
+    is used to derive parent mass."""
+    mz = numpy.array([], dtype='float')
+    intensities = numpy.array([], dtype='float')
+    metadata = {"precursor_mz": 444.0, "ionmode": ionmode}
+    spectrum_in = Spectrum(mz=mz,
+                           intensities=intensities,
+                           metadata=metadata)
+
+    spectrum = add_parent_mass(spectrum_in)
+
+    assert spectrum.get("parent_mass") == expected, \
+        "Expected a different parent_mass"
diff --git a/tests/test_add_retention.py b/tests/test_add_retention.py
new file mode 100644
index 00000000..452f91f8
--- /dev/null
+++ b/tests/test_add_retention.py
@@ -0,0 +1,66 @@
+import numpy
+import pytest
+from matchms import Spectrum
+from matchms.filtering import add_retention_index
+from matchms.filtering import add_retention_time
+
+
+@pytest.mark.parametrize("metadata, expected", [
+    [{"retention_time": 100.0}, 100.0],
+    [{"retention_time": "NA"}, None],
+    [{"retention_time": "100.0"}, 100.0],
+    [{"retentiontime": 200}, 200.0],
+    [{"retentiontime": -1}, None],
+    [{"retentiontime": "-1"}, None],
+    [{"rt": 200}, 200.0],
+    [{"RT": 200}, 200.0],
+    [{"RT_Query": 200}, 200.0],
+    [{"nothing": "200"}, None],
+    [{'scan_start_time': 0.629566}, 0.629566],
+    [{'scan_start_time': [0.629566]}, 0.629566],
+    [{"rt": "None", "retentiontime": 12.17}, 12.17]
+])
+def test_add_retention_time(metadata, expected):
+    spectrum_in = Spectrum(mz=numpy.array(
+        [], "float"), intensities=numpy.array([], "float"), metadata=metadata)
+
+    spectrum = add_retention_time(spectrum_in)
+    actual = spectrum.get("retention_time")
+
+    if expected is None:
+        assert actual is None
+    else:
+        assert actual == expected and isinstance(actual, float)
+
+
+@pytest.mark.parametrize("metadata, expected", [
+    [{"retention_index": 100.0}, 100.0],
+    [{"retention_index": "NA"}, None],
+    [{"retention_index": "100.0"}, 100.0],
+    [{"retentionindex": 200}, 200.0],
+    [{"retentionindex": -1}, None],
+    [{"retentionindex": "-1"}, None],
+    [{"ri": 200}, 200.0],
+    [{"RI": 200}, 200.0],
+    [{"nothing": "200"}, None]
+])
+def test_add_retention_index(metadata, expected):
+    spectrum_in = Spectrum(mz=numpy.array(
+        [], "float"), intensities=numpy.array([], "float"), metadata=metadata)
+
+    spectrum = add_retention_index(spectrum_in)
+    actual = spectrum.get("retention_index")
+
+    if expected is None:
+        assert actual is None
+    else:
+        assert actual == expected and isinstance(actual, float)
+
+
+def test_empty_spectrum():
+    spectrum_in = None
+    spectrum = add_retention_time(spectrum_in)
+    assert spectrum is None, "Expected different handling of None spectrum."
+
+    spectrum = add_retention_index(spectrum_in)
+    assert spectrum is None, "Expected different handling of None spectrum."