Repository: matchms/matchms

Issue:
Add peak changes to SpectrumProcessor
Currently we only keep track of changes in the metadata. But not of changes made to the peaks, this should be added to the processing report as well. 

Discussion:

Patch:
diff --git a/matchms/Fragments.py b/matchms/Fragments.py
index 41801dae..52787605 100644
--- a/matchms/Fragments.py
+++ b/matchms/Fragments.py
@@ -45,6 +45,8 @@ class Fragments:
         assert self._is_sorted(), "mz values are out of order."
 
     def __eq__(self, other):
+        if other is None:
+            return False
         return \
             self.mz.shape == other.mz.shape and \
             np.allclose(self.mz, other.mz) and \
diff --git a/matchms/filtering/SpectrumProcessor.py b/matchms/filtering/SpectrumProcessor.py
index 3c0c2659..4c5d3b02 100644
--- a/matchms/filtering/SpectrumProcessor.py
+++ b/matchms/filtering/SpectrumProcessor.py
@@ -1,7 +1,7 @@
 import inspect
 from collections import defaultdict
 from functools import partial
-from typing import Dict, Optional, Tuple, Union
+from typing import Callable, Dict, Optional, Tuple, Union
 import numpy as np
 import pandas as pd
 from tqdm import tqdm
@@ -35,7 +35,8 @@ class SpectrumProcessor:
             for filter_name in PREDEFINED_PIPELINES[predefined_pipeline]:
                 self.add_matchms_filter(filter_name)
 
-    def add_filter(self, filter_function: Union[Tuple[str, Dict[str, any]], str]):
+    def add_filter(self,
+                   filter_function: Union[Tuple[str, Dict[str, any]], str, Tuple[Callable, Dict[str, any]], Callable]):
         """Add a filter to the processing pipeline. Takes both matchms filter names (and parameters)
         as well as custom-made functions.
         """
@@ -73,7 +74,7 @@ class SpectrumProcessor:
         # Sort filters according to their order in self.filter_order
         self.filters.sort(key=lambda f: self.filter_order.index(f.__name__))
 
-    def add_custom_filter(self, filter_function, filter_params=None, filter_position: Optional[int] = None):
+    def add_custom_filter(self, filter_function: Union[Tuple[Callable, Dict[str, any]], Callable], filter_params=None, filter_position=None):
         """
         Add a custom filter function to the processing pipeline.
 
@@ -238,8 +239,9 @@ class ProcessingReport:
     """Class to keep track of spectrum changes during filtering.
     """
     def __init__(self):
-        self.counter_changed_spectrum = defaultdict(int)
+        self.counter_changed_metadata = defaultdict(int)
         self.counter_removed_spectrums = defaultdict(int)
+        self.counter_changed_peaks = defaultdict(int)
         self.counter_number_processed = 0
 
     def add_to_report(self, spectrum_old, spectrum_new: Spectrum,
@@ -249,18 +251,23 @@ class ProcessingReport:
         if spectrum_new is None:
             self.counter_removed_spectrums[filter_function_name] += 1
         else:
-            for field, value in spectrum_new.metadata.items():
-                if objects_differ(spectrum_old.get(field), value):
-                    self.counter_changed_spectrum[filter_function_name] += 1
-                    break
+            # Add metadata changes
+            if spectrum_new.metadata != spectrum_old.metadata:
+                self.counter_changed_metadata[filter_function_name] += 1
+            # Add peak changes
+            if spectrum_new.peaks != spectrum_old.peaks or spectrum_new.losses != spectrum_old.losses:
+                self.counter_changed_peaks[filter_function_name] += 1
 
     def to_dataframe(self):
         """Create Pandas DataFrame Report of counted spectrum changes."""
-        changes = pd.DataFrame(self.counter_changed_spectrum.items(),
-                               columns=["filter", "changed spectra"])
+        metadata_changed = pd.DataFrame(self.counter_changed_metadata.items(),
+                                        columns=["filter", "changed metadata"])
         removed = pd.DataFrame(self.counter_removed_spectrums.items(),
                                columns=["filter", "removed spectra"])
-        processing_report = pd.merge(removed, changes, how="outer", on="filter")
+        peaks_changed = pd.DataFrame(self.counter_changed_peaks.items(),
+                                     columns=["filter", "changed mass spectrum"])
+        processing_report = pd.merge(removed, metadata_changed, how="outer", on="filter")
+        processing_report = pd.merge(processing_report, peaks_changed, how="outer", on="filter")
 
         processing_report = processing_report.set_index("filter").fillna(0)
         return processing_report.astype(int)
@@ -278,7 +285,9 @@ Changes during processing:
     def __repr__(self):
         return f"Report({self.counter_number_processed},\
         {self.counter_removed_spectrums},\
-        {dict(self.counter_changed_spectrum)})"
+        {dict(self.counter_removed_spectrums)},\
+        {dict(self.counter_changed_metadata)},\
+        {dict(self.counter_changed_peaks)})"
 
 
 def objects_differ(obj1, obj2):
diff --git a/matchms/filtering/metadata_processing/repair_not_matching_annotation.py b/matchms/filtering/metadata_processing/repair_not_matching_annotation.py
index 0968f33f..cb26cc88 100644
--- a/matchms/filtering/metadata_processing/repair_not_matching_annotation.py
+++ b/matchms/filtering/metadata_processing/repair_not_matching_annotation.py
@@ -2,7 +2,7 @@ import logging
 from matchms import Spectrum
 from matchms.filtering.filter_utils.smile_inchi_inchikey_conversions import (
     _check_fully_annotated, convert_inchi_to_inchikey, convert_inchi_to_smiles,
-    convert_smiles_to_inchi)
+    convert_smiles_to_inchi, is_valid_inchi, is_valid_smiles)
 from matchms.filtering.metadata_processing.require_parent_mass_match_smiles import \
     _check_smiles_and_parent_mass_match
 
@@ -15,7 +15,7 @@ def repair_not_matching_annotation(spectrum_in: Spectrum):
     Repairs mismatches in a spectrum's annotations related to SMILES, InChI, and InChIKey.
 
     Given a spectrum, this function ensures that the provided SMILES, InChI, and InChIKey
-    annotations are consistent with one another. If there are discrepancies, they are resolved 
+    annotations are consistent with one another. If there are discrepancies, they are resolved
     as follows:
 
     1. If the SMILES and InChI do not match:


Test patch:
diff --git a/tests/filtering/test_spectrum_processor.py b/tests/filtering/test_spectrum_processor.py
index 25b39d39..4143f8df 100644
--- a/tests/filtering/test_spectrum_processor.py
+++ b/tests/filtering/test_spectrum_processor.py
@@ -7,16 +7,15 @@ from ..builder_Spectrum import SpectrumBuilder
 
 @pytest.fixture
 def spectrums():
-    metadata1 = {"charge": "+1",
-                 "pepmass": 100}
-    metadata2 = {"charge": "-1",
-                 "pepmass": 102}
-    metadata3 = {"charge": -1,
-                 "pepmass": 104}
-
-    s1 = SpectrumBuilder().with_metadata(metadata1).build()
-    s2 = SpectrumBuilder().with_metadata(metadata2).build()
-    s3 = SpectrumBuilder().with_metadata(metadata3).build()
+    s1 = SpectrumBuilder().\
+        with_metadata({"charge": "+1", "pepmass": 100}).\
+        with_mz([10, 20, 30]).with_intensities([0.1, 0.4, 10]).build()
+    s2 = SpectrumBuilder().with_metadata({"charge": "-1",
+                                          "pepmass": 102}).\
+        with_mz([10, 20, 30]).with_intensities([0.1, 0.2, 1]).build()
+    s3 = SpectrumBuilder().with_metadata({"charge": -1,
+                                          "pepmass": 104}).\
+        with_mz([10, ]).with_intensities([0.1, ]).build()
     return [s1, s2, s3]
 
 
@@ -135,18 +134,20 @@ def test_filter_spectrums(spectrums):
 
 def test_filter_spectrums_report(spectrums):
     processor = SpectrumProcessor("minimal")
+    processor.add_filter(filter_function=("require_minimum_number_of_peaks", {"n_required": 2}))
+    processor.add_filter(filter_function="add_losses")
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
-    assert len(spectrums) == 3
+    assert len(spectrums) == 2
     actual_masses = [s.get("precursor_mz") for s in spectrums]
-    expected_masses = [100, 102, 104]
+    expected_masses = [100, 102]
     assert actual_masses == expected_masses
     assert report.counter_number_processed == 3
-    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3, 'derive_ionmode': 3}
+    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3, 'derive_ionmode': 3}
     report_df = report.to_dataframe()
-    assert np.all(report_df.loc[["make_charge_int", "interpret_pepmass", "derive_ionmode"]].values == np.array(
-        [[0, 2],
-         [0, 3],
-         [0, 3]]))
+    assert np.all(report_df.loc[["require_minimum_number_of_peaks", "interpret_pepmass", "add_losses"]].values == np.array(
+        [[1, 0, 0],
+         [0, 3, 0],
+         [0, 0, 2]]))
 
 
 def test_processing_report_class(spectrums):
@@ -157,7 +158,7 @@ def test_processing_report_class(spectrums):
         processing_report.add_to_report(s, spectrum_processed, "test_filter")
 
     assert not processing_report.counter_removed_spectrums
-    assert processing_report.counter_changed_spectrum == {"test_filter": 3}
+    assert processing_report.counter_changed_metadata == {"test_filter": 3}
 
 
 def test_adding_custom_filter(spectrums):
@@ -172,7 +173,7 @@ def test_adding_custom_filter(spectrums):
     assert filters[-1].__name__ == "nonsense_inchikey"
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
     assert report.counter_number_processed == 3
-    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3,
+    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3,
                                                'derive_ionmode': 3, 'nonsense_inchikey': 3}
     assert spectrums[0].get("inchikey") == "NONSENSE", "Custom filter not executed properly"
 
@@ -189,7 +190,7 @@ def test_adding_custom_filter_with_parameters(spectrums):
     assert filters[-1].__name__ == "nonsense_inchikey_multiple"
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
     assert report.counter_number_processed == 3
-    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3,
+    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3,
                                                'derive_ionmode': 3, 'nonsense_inchikey_multiple': 3}
     assert spectrums[0].get("inchikey") == "NONSENSENONSENSE", "Custom filter not executed properly"