Repository: matchms/matchms
Pull request #525 (2023-09-06 15:37:23)

Complete library cleaning pipeline
After a second round of testing pipeline on an entire library a few small changes were made:

- Added filters that were missing from default pipelines.
- Also added a small function that prints filter functions available that are currently not used in a yaml file. This makes it easy to find filters that you might have forgotten in your pipeline. 
- Only check if the 2D structures match between smiles and inchi instead of 3D structure. 
- Change warning to logging for changed sign of given charge.
- Change default minimum number of peaks in reduce to number of peaks, since this is unexpected behaviour to remove spectra by default. The removal of spectra with too little peaks should be done by the filter made for that.


Patch:
diff --git a/matchms/Fragments.py b/matchms/Fragments.py
index 52787605..41801dae 100644
--- a/matchms/Fragments.py
+++ b/matchms/Fragments.py
@@ -45,8 +45,6 @@ class Fragments:
         assert self._is_sorted(), "mz values are out of order."
 
     def __eq__(self, other):
-        if other is None:
-            return False
         return \
             self.mz.shape == other.mz.shape and \
             np.allclose(self.mz, other.mz) and \
diff --git a/matchms/Pipeline.py b/matchms/Pipeline.py
index 1f4b241d..e231b46c 100644
--- a/matchms/Pipeline.py
+++ b/matchms/Pipeline.py
@@ -5,6 +5,7 @@ from typing import Iterable, List, Optional, Union
 import yaml
 import matchms.similarity as mssimilarity
 from matchms import calculate_scores
+from matchms.filtering.filter_order_and_default_pipelines import ALL_FILTERS
 from matchms.filtering.SpectrumProcessor import SpectrumProcessor
 from matchms.importing.load_spectra import load_list_of_spectrum_files
 from matchms.logging_functions import (add_logging_to_file,
@@ -424,3 +425,14 @@ class Pipeline:
     @property
     def spectrums_references(self) -> List[SpectrumType]:
         return self._spectrums_references
+
+
+def get_unused_filters(yaml_file):
+    """Prints all filter names that are in ALL_FILTERS, but not in the yaml file"""
+    workflow = load_workflow_from_yaml_file(yaml_file)
+    processor = initialize_spectrum_processor(None, workflow["query_filters"])
+
+    filters_used = [filter_function.__name__ for filter_function in processor.filters]
+    for filter_function in ALL_FILTERS:
+        if filter_function.__name__ not in filters_used:
+            print(filter_function.__name__)
diff --git a/matchms/filtering/SpectrumProcessor.py b/matchms/filtering/SpectrumProcessor.py
index f504257c..f0a9ed1f 100644
--- a/matchms/filtering/SpectrumProcessor.py
+++ b/matchms/filtering/SpectrumProcessor.py
@@ -2,7 +2,7 @@ import inspect
 import logging
 from collections import defaultdict
 from functools import partial
-from typing import Callable, Dict, Optional, Tuple, Union
+from typing import Dict, Optional, Tuple, Union
 import numpy as np
 import pandas as pd
 from tqdm import tqdm
@@ -39,8 +39,7 @@ class SpectrumProcessor:
             for filter_name in PREDEFINED_PIPELINES[predefined_pipeline]:
                 self.add_matchms_filter(filter_name)
 
-    def add_filter(self,
-                   filter_function: Union[Tuple[str, Dict[str, any]], str, Tuple[Callable, Dict[str, any]], Callable]):
+    def add_filter(self, filter_function: Union[Tuple[str, Dict[str, any]], str]):
         """Add a filter to the processing pipeline. Takes both matchms filter names (and parameters)
         as well as custom-made functions.
         """
@@ -89,7 +88,7 @@ class SpectrumProcessor:
         # Sort filters according to their order in self.filter_order
         self.filters.sort(key=lambda f: self.filter_order.index(f.__name__))
 
-    def add_custom_filter(self, filter_function: Union[Tuple[Callable, Dict[str, any]], Callable], filter_params=None, filter_position=None):
+    def add_custom_filter(self, filter_function, filter_params=None, filter_position: Optional[int] = None):
         """
         Add a custom filter function to the processing pipeline.
 
@@ -254,9 +253,8 @@ class ProcessingReport:
     """Class to keep track of spectrum changes during filtering.
     """
     def __init__(self):
-        self.counter_changed_metadata = defaultdict(int)
+        self.counter_changed_spectrum = defaultdict(int)
         self.counter_removed_spectrums = defaultdict(int)
-        self.counter_changed_peaks = defaultdict(int)
         self.counter_number_processed = 0
 
     def add_to_report(self, spectrum_old, spectrum_new: Spectrum,
@@ -266,23 +264,18 @@ class ProcessingReport:
         if spectrum_new is None:
             self.counter_removed_spectrums[filter_function_name] += 1
         else:
-            # Add metadata changes
-            if spectrum_new.metadata != spectrum_old.metadata:
-                self.counter_changed_metadata[filter_function_name] += 1
-            # Add peak changes
-            if spectrum_new.peaks != spectrum_old.peaks or spectrum_new.losses != spectrum_old.losses:
-                self.counter_changed_peaks[filter_function_name] += 1
+            for field, value in spectrum_new.metadata.items():
+                if objects_differ(spectrum_old.get(field), value):
+                    self.counter_changed_spectrum[filter_function_name] += 1
+                    break
 
     def to_dataframe(self):
         """Create Pandas DataFrame Report of counted spectrum changes."""
-        metadata_changed = pd.DataFrame(self.counter_changed_metadata.items(),
-                                        columns=["filter", "changed metadata"])
+        changes = pd.DataFrame(self.counter_changed_spectrum.items(),
+                               columns=["filter", "changed spectra"])
         removed = pd.DataFrame(self.counter_removed_spectrums.items(),
                                columns=["filter", "removed spectra"])
-        peaks_changed = pd.DataFrame(self.counter_changed_peaks.items(),
-                                     columns=["filter", "changed mass spectrum"])
-        processing_report = pd.merge(removed, metadata_changed, how="outer", on="filter")
-        processing_report = pd.merge(processing_report, peaks_changed, how="outer", on="filter")
+        processing_report = pd.merge(removed, changes, how="outer", on="filter")
 
         processing_report = processing_report.set_index("filter").fillna(0)
         return processing_report.astype(int)
@@ -300,9 +293,7 @@ Changes during processing:
     def __repr__(self):
         return f"Report({self.counter_number_processed},\
         {self.counter_removed_spectrums},\
-        {dict(self.counter_removed_spectrums)},\
-        {dict(self.counter_changed_metadata)},\
-        {dict(self.counter_changed_peaks)})"
+        {dict(self.counter_changed_spectrum)})"
 
 
 def objects_differ(obj1, obj2):
diff --git a/matchms/filtering/filter_order_and_default_pipelines.py b/matchms/filtering/filter_order_and_default_pipelines.py
index 814fce05..0761a8b1 100644
--- a/matchms/filtering/filter_order_and_default_pipelines.py
+++ b/matchms/filtering/filter_order_and_default_pipelines.py
@@ -67,6 +67,7 @@ DEFAULT_FILTERS = BASIC_FILTERS \
                      "harmonize_undefined_smiles",
                      "repair_inchi_inchikey_smiles",
                      "normalize_intensities",
+                     "add_retention_time",
                      ]
 FULLY_ANNOTATED_PROCESSING = DEFAULT_FILTERS \
                              + ["clean_adduct",
@@ -75,12 +76,27 @@ FULLY_ANNOTATED_PROCESSING = DEFAULT_FILTERS \
                                 "derive_inchikey_from_inchi",
                                 ("require_correct_ionmode", {"ion_mode_to_keep": "both"}),
                                 ("require_parent_mass_match_smiles", {'mass_tolerance': 0.1}),
+                                ("repair_smiles_of_salts", {'mass_tolerance': 0.1}),
+                                ("repair_precursor_is_parent_mass", {'mass_tolerance': 0.1}),
+                                ("repair_parent_mass_is_mol_wt", {'mass_tolerance': 0.1}),
+                                ("repair_adduct_based_on_smiles", {'mass_tolerance': 0.1}),
                                 "repair_not_matching_annotation",
                                 "require_valid_annotation",
+                                ("derive_smiles_from_pubchem_compound_name_search", {"mass_tolerance": 0.1}),
                                 ]
+
+
+MS2DEEPSCORE_TRAINING = FULLY_ANNOTATED_PROCESSING + \
+                        [("select_by_mz", {"mz_from": 0, "mz_to": 1000}),
+                         ("select_by_relative_intensity", {"intensity_from": 0.001}),
+                         ("reduce_to_number_of_peaks", {"n_max": 1000}),
+                         ("require_minimum_of_high_peaks", {"no_peaks": 5, "intensity_percent": 2.0}),
+                         ]
+
 PREDEFINED_PIPELINES = {
     "minimal": MINIMAL_FILTERS,
     "basic": BASIC_FILTERS,
     "default": DEFAULT_FILTERS,
     "fully_annotated": FULLY_ANNOTATED_PROCESSING,
+    "ms2deepscore": MS2DEEPSCORE_TRAINING,
 }
diff --git a/matchms/filtering/metadata_processing/correct_charge.py b/matchms/filtering/metadata_processing/correct_charge.py
index cd4d6a9b..bc8d60b1 100644
--- a/matchms/filtering/metadata_processing/correct_charge.py
+++ b/matchms/filtering/metadata_processing/correct_charge.py
@@ -44,7 +44,7 @@ def correct_charge(spectrum_in: SpectrumType) -> SpectrumType:
 
     # Correct charge when in conflict with ionmode (trust ionmode more!)
     if np.sign(charge) == 1 and ionmode == 'negative':
-        logger.warning("Changed sign of given charge: %s to match negative ionmode", charge)
+        logger.info("Changed sign of given charge: %s to match negative ionmode", charge)
         charge *= -1
     elif np.sign(charge) == -1 and ionmode == 'positive':
         logger.warning("Changed sign of given charge: %s to match positive ionmode", charge)
diff --git a/matchms/filtering/peak_processing/reduce_to_number_of_peaks.py b/matchms/filtering/peak_processing/reduce_to_number_of_peaks.py
index 216c64e6..7084ec93 100644
--- a/matchms/filtering/peak_processing/reduce_to_number_of_peaks.py
+++ b/matchms/filtering/peak_processing/reduce_to_number_of_peaks.py
@@ -9,7 +9,7 @@ from matchms.typing import SpectrumType
 logger = logging.getLogger("matchms")
 
 
-def reduce_to_number_of_peaks(spectrum_in: SpectrumType, n_required: int = 1, n_max: int = np.inf,
+def reduce_to_number_of_peaks(spectrum_in: SpectrumType, n_required: int = 0, n_max: int = np.inf,
                               ratio_desired: Optional[float] = None) -> SpectrumType:
     """Lowest intensity peaks will be removed when it has more peaks than desired.
 


Test patch:
diff --git a/tests/filtering/test_spectrum_processor.py b/tests/filtering/test_spectrum_processor.py
index 303ae58f..26250acb 100644
--- a/tests/filtering/test_spectrum_processor.py
+++ b/tests/filtering/test_spectrum_processor.py
@@ -8,15 +8,16 @@ from ..builder_Spectrum import SpectrumBuilder
 
 @pytest.fixture
 def spectrums():
-    s1 = SpectrumBuilder().\
-        with_metadata({"charge": "+1", "pepmass": 100}).\
-        with_mz([10, 20, 30]).with_intensities([0.1, 0.4, 10]).build()
-    s2 = SpectrumBuilder().with_metadata({"charge": "-1",
-                                          "pepmass": 102}).\
-        with_mz([10, 20, 30]).with_intensities([0.1, 0.2, 1]).build()
-    s3 = SpectrumBuilder().with_metadata({"charge": -1,
-                                          "pepmass": 104}).\
-        with_mz([10, ]).with_intensities([0.1, ]).build()
+    metadata1 = {"charge": "+1",
+                 "pepmass": 100}
+    metadata2 = {"charge": "-1",
+                 "pepmass": 102}
+    metadata3 = {"charge": -1,
+                 "pepmass": 104}
+
+    s1 = SpectrumBuilder().with_metadata(metadata1).build()
+    s2 = SpectrumBuilder().with_metadata(metadata2).build()
+    s3 = SpectrumBuilder().with_metadata(metadata3).build()
     return [s1, s2, s3]
 
 
@@ -30,6 +31,7 @@ def test_filter_sorting_and_output():
         'clean_compound_name',
         'interpret_pepmass',
         'add_precursor_mz',
+        'add_retention_time',
         'derive_ionmode',
         'correct_charge',
         'require_precursor_mz',
@@ -51,6 +53,7 @@ def test_filter_sorting_and_output():
                         'clean_compound_name',
                         'interpret_pepmass',
                         'add_precursor_mz',
+                        'add_retention_time',
                         'derive_ionmode',
                         'correct_charge',
                         ('require_precursor_mz', {'minimum_accepted_mz': 10.0}),
@@ -135,20 +138,18 @@ def test_filter_spectrums(spectrums):
 
 def test_filter_spectrums_report(spectrums):
     processor = SpectrumProcessor("minimal")
-    processor.add_filter(filter_function=("require_minimum_number_of_peaks", {"n_required": 2}))
-    processor.add_filter(filter_function="add_losses")
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
-    assert len(spectrums) == 2
+    assert len(spectrums) == 3
     actual_masses = [s.get("precursor_mz") for s in spectrums]
-    expected_masses = [100, 102]
+    expected_masses = [100, 102, 104]
     assert actual_masses == expected_masses
     assert report.counter_number_processed == 3
-    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3, 'derive_ionmode': 3}
+    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3, 'derive_ionmode': 3}
     report_df = report.to_dataframe()
-    assert np.all(report_df.loc[["require_minimum_number_of_peaks", "interpret_pepmass", "add_losses"]].values == np.array(
-        [[1, 0, 0],
-         [0, 3, 0],
-         [0, 0, 2]]))
+    assert np.all(report_df.loc[["make_charge_int", "interpret_pepmass", "derive_ionmode"]].values == np.array(
+        [[0, 2],
+         [0, 3],
+         [0, 3]]))
 
 
 def test_processing_report_class(spectrums):
@@ -159,7 +160,7 @@ def test_processing_report_class(spectrums):
         processing_report.add_to_report(s, spectrum_processed, "test_filter")
 
     assert not processing_report.counter_removed_spectrums
-    assert processing_report.counter_changed_metadata == {"test_filter": 3}
+    assert processing_report.counter_changed_spectrum == {"test_filter": 3}
 
 
 def test_adding_custom_filter(spectrums):
@@ -174,7 +175,7 @@ def test_adding_custom_filter(spectrums):
     assert filters[-1].__name__ == "nonsense_inchikey"
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
     assert report.counter_number_processed == 3
-    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3,
+    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3,
                                                'derive_ionmode': 3, 'nonsense_inchikey': 3}
     assert spectrums[0].get("inchikey") == "NONSENSE", "Custom filter not executed properly"
 
@@ -191,7 +192,7 @@ def test_adding_custom_filter_with_parameters(spectrums):
     assert filters[-1].__name__ == "nonsense_inchikey_multiple"
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
     assert report.counter_number_processed == 3
-    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3,
+    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3,
                                                'derive_ionmode': 3, 'nonsense_inchikey_multiple': 3}
     assert spectrums[0].get("inchikey") == "NONSENSENONSENSE", "Custom filter not executed properly"