Repository: matchms/matchms
Pull request #510 (2023-08-28 09:20:05)

Small bug fixing (during running of pipeline)
Some small filter bugfixes:
- fixed get netural mass handling None. Getting smiles = None would result in the pipeline breaking. 
- Add check is_valid_inchikey to not break workflow when inchikey is None
- Remove wrong warnings and make warnings more informative
- Added changelog for previous PR's
- and more...


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dbce4de4..0af1424a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [unreleased]
 
 ### Added
-
+- Additional tests for filter pipeline order
+- ProcessingReport. This adds an overview of the number of spectra changed by each filter step.
 - `repair_not_matching_annotation` filter [#505](https://github.com/matchms/matchms/pull/505)
 - Missing docstring documentions [#507](https://github.com/matchms/matchms/pull/507)
 
@@ -17,6 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Logger warning for rdkit molecule conversion [#507](https://github.com/matchms/matchms/pull/507)
 - Repair_smiles_from_compound_name, now works without matchmsextras [#509](https://github.com/matchms/matchms/pull/509/)
   - pubchempy was added as dependency
+- Default filters are now stored in the yaml file as separate filters
+- Custom filters are added after default filters or at a position specified by the user
+- The file structure of metadata_utils was refactored
 
 ### Fixed
 
diff --git a/matchms/Fragments.py b/matchms/Fragments.py
index 41801dae..52787605 100644
--- a/matchms/Fragments.py
+++ b/matchms/Fragments.py
@@ -45,6 +45,8 @@ class Fragments:
         assert self._is_sorted(), "mz values are out of order."
 
     def __eq__(self, other):
+        if other is None:
+            return False
         return \
             self.mz.shape == other.mz.shape and \
             np.allclose(self.mz, other.mz) and \
diff --git a/matchms/Pipeline.py b/matchms/Pipeline.py
index e8e941b0..1f4b241d 100644
--- a/matchms/Pipeline.py
+++ b/matchms/Pipeline.py
@@ -379,7 +379,6 @@ class Pipeline:
         self._spectrums_queries = load_list_of_spectrum_files(query_files)
 
         self.write_to_logfile(f"Loaded query spectra from {query_files}")
-        self.write_to_logfile(f"Loaded {len(self._spectrums_queries)} query spectra")
 
         # import reference spectra
         if reference_files is None:
@@ -389,7 +388,6 @@ class Pipeline:
         else:
             self._spectrums_references = load_list_of_spectrum_files(reference_files)
             self.write_to_logfile(f"Loaded reference spectra from {reference_files}")
-            self.write_to_logfile(f"Loaded {len(self._spectrums_references)} reference spectra")
 
     # Getter & Setters
     @property
diff --git a/matchms/filtering/SpectrumProcessor.py b/matchms/filtering/SpectrumProcessor.py
index 3c0c2659..4c5d3b02 100644
--- a/matchms/filtering/SpectrumProcessor.py
+++ b/matchms/filtering/SpectrumProcessor.py
@@ -1,7 +1,7 @@
 import inspect
 from collections import defaultdict
 from functools import partial
-from typing import Dict, Optional, Tuple, Union
+from typing import Callable, Dict, Optional, Tuple, Union
 import numpy as np
 import pandas as pd
 from tqdm import tqdm
@@ -35,7 +35,8 @@ class SpectrumProcessor:
             for filter_name in PREDEFINED_PIPELINES[predefined_pipeline]:
                 self.add_matchms_filter(filter_name)
 
-    def add_filter(self, filter_function: Union[Tuple[str, Dict[str, any]], str]):
+    def add_filter(self,
+                   filter_function: Union[Tuple[str, Dict[str, any]], str, Tuple[Callable, Dict[str, any]], Callable]):
         """Add a filter to the processing pipeline. Takes both matchms filter names (and parameters)
         as well as custom-made functions.
         """
@@ -73,7 +74,7 @@ class SpectrumProcessor:
         # Sort filters according to their order in self.filter_order
         self.filters.sort(key=lambda f: self.filter_order.index(f.__name__))
 
-    def add_custom_filter(self, filter_function, filter_params=None, filter_position: Optional[int] = None):
+    def add_custom_filter(self, filter_function: Union[Tuple[Callable, Dict[str, any]], Callable], filter_params=None, filter_position=None):
         """
         Add a custom filter function to the processing pipeline.
 
@@ -238,8 +239,9 @@ class ProcessingReport:
     """Class to keep track of spectrum changes during filtering.
     """
     def __init__(self):
-        self.counter_changed_spectrum = defaultdict(int)
+        self.counter_changed_metadata = defaultdict(int)
         self.counter_removed_spectrums = defaultdict(int)
+        self.counter_changed_peaks = defaultdict(int)
         self.counter_number_processed = 0
 
     def add_to_report(self, spectrum_old, spectrum_new: Spectrum,
@@ -249,18 +251,23 @@ class ProcessingReport:
         if spectrum_new is None:
             self.counter_removed_spectrums[filter_function_name] += 1
         else:
-            for field, value in spectrum_new.metadata.items():
-                if objects_differ(spectrum_old.get(field), value):
-                    self.counter_changed_spectrum[filter_function_name] += 1
-                    break
+            # Add metadata changes
+            if spectrum_new.metadata != spectrum_old.metadata:
+                self.counter_changed_metadata[filter_function_name] += 1
+            # Add peak changes
+            if spectrum_new.peaks != spectrum_old.peaks or spectrum_new.losses != spectrum_old.losses:
+                self.counter_changed_peaks[filter_function_name] += 1
 
     def to_dataframe(self):
         """Create Pandas DataFrame Report of counted spectrum changes."""
-        changes = pd.DataFrame(self.counter_changed_spectrum.items(),
-                               columns=["filter", "changed spectra"])
+        metadata_changed = pd.DataFrame(self.counter_changed_metadata.items(),
+                                        columns=["filter", "changed metadata"])
         removed = pd.DataFrame(self.counter_removed_spectrums.items(),
                                columns=["filter", "removed spectra"])
-        processing_report = pd.merge(removed, changes, how="outer", on="filter")
+        peaks_changed = pd.DataFrame(self.counter_changed_peaks.items(),
+                                     columns=["filter", "changed mass spectrum"])
+        processing_report = pd.merge(removed, metadata_changed, how="outer", on="filter")
+        processing_report = pd.merge(processing_report, peaks_changed, how="outer", on="filter")
 
         processing_report = processing_report.set_index("filter").fillna(0)
         return processing_report.astype(int)
@@ -278,7 +285,9 @@ Changes during processing:
     def __repr__(self):
         return f"Report({self.counter_number_processed},\
         {self.counter_removed_spectrums},\
-        {dict(self.counter_changed_spectrum)})"
+        {dict(self.counter_removed_spectrums)},\
+        {dict(self.counter_changed_metadata)},\
+        {dict(self.counter_changed_peaks)})"
 
 
 def objects_differ(obj1, obj2):
diff --git a/matchms/filtering/filter_utils/derive_precursor_mz_and_parent_mass.py b/matchms/filtering/filter_utils/derive_precursor_mz_and_parent_mass.py
index d673cc64..f776da09 100644
--- a/matchms/filtering/filter_utils/derive_precursor_mz_and_parent_mass.py
+++ b/matchms/filtering/filter_utils/derive_precursor_mz_and_parent_mass.py
@@ -90,7 +90,7 @@ def _get_multiplier_and_correction_mass_from_adduct(adduct: str) -> Tuple[int, f
         multiplier = known_adducts.loc[known_adducts["adduct"] == adduct, "mass_multiplier"].values[0]
         correction_mass = known_adducts.loc[known_adducts["adduct"] == adduct, "correction_mass"].values[0]
     else:
-        multiplier, correction_mass = get_multiplier_and_mass_from_adduct("adduct")
+        multiplier, correction_mass = get_multiplier_and_mass_from_adduct(adduct)
     return multiplier, correction_mass
 
 
diff --git a/matchms/filtering/filter_utils/get_neutral_mass_from_smiles.py b/matchms/filtering/filter_utils/get_neutral_mass_from_smiles.py
index 1a74d5fc..f1b6b65f 100644
--- a/matchms/filtering/filter_utils/get_neutral_mass_from_smiles.py
+++ b/matchms/filtering/filter_utils/get_neutral_mass_from_smiles.py
@@ -72,6 +72,8 @@ def _get_neutral_mass(smiles: str, monoisotopic: bool) -> Optional[float]:
     """
     if not _has_rdkit:
         raise ImportError(rdkit_missing_message)
+    if smiles is None:
+        return None
     mol = Chem.MolFromSmiles(smiles)
     if mol is None:
         logger.warning("No mass could be calculated for smiles: %s, since it is not a valid smiles.", smiles)
diff --git a/matchms/filtering/filter_utils/interpret_unknown_adduct.py b/matchms/filtering/filter_utils/interpret_unknown_adduct.py
index fc82a9b8..b385b095 100644
--- a/matchms/filtering/filter_utils/interpret_unknown_adduct.py
+++ b/matchms/filtering/filter_utils/interpret_unknown_adduct.py
@@ -40,6 +40,9 @@ def get_multiplier_and_mass_from_adduct(adduct: str) -> Tuple[Optional[float], O
     Returns:
         Tuple[Optional[float], Optional[float]]: Multiplier and mass of this adduct.
     """
+    if adduct is None or not isinstance(adduct, str):
+        return None, None
+
     charge = get_charge_of_adduct(adduct)
     if charge is None:
         return None, None
@@ -146,6 +149,7 @@ def get_charge_of_adduct(adduct) -> Optional[int]:
 
     e.g. '[M+H-H2O]2+' -> 2
     """
+
     charge = re.findall((r"\]([0-9]?[+-])"), adduct)
     if len(charge) != 1:
         logger.warning("Charge was found %s times in adduct %s",
diff --git a/matchms/filtering/metadata_processing/clean_adduct.py b/matchms/filtering/metadata_processing/clean_adduct.py
index 982fb01a..57c6ca5c 100644
--- a/matchms/filtering/metadata_processing/clean_adduct.py
+++ b/matchms/filtering/metadata_processing/clean_adduct.py
@@ -20,7 +20,6 @@ def clean_adduct(spectrum_in):
         return None
     spectrum = spectrum_in.clone()
     adduct = spectrum.get("adduct")
-
     cleaned_adduct = _clean_adduct(adduct)
     if adduct != cleaned_adduct:
         spectrum.set("adduct", cleaned_adduct)
diff --git a/matchms/filtering/metadata_processing/correct_charge.py b/matchms/filtering/metadata_processing/correct_charge.py
index 66d7e6f1..cd4d6a9b 100644
--- a/matchms/filtering/metadata_processing/correct_charge.py
+++ b/matchms/filtering/metadata_processing/correct_charge.py
@@ -44,11 +44,11 @@ def correct_charge(spectrum_in: SpectrumType) -> SpectrumType:
 
     # Correct charge when in conflict with ionmode (trust ionmode more!)
     if np.sign(charge) == 1 and ionmode == 'negative':
+        logger.warning("Changed sign of given charge: %s to match negative ionmode", charge)
         charge *= -1
-        logger.warning("Changed sign of given charge to match negative ionmode")
     elif np.sign(charge) == -1 and ionmode == 'positive':
+        logger.warning("Changed sign of given charge: %s to match positive ionmode", charge)
         charge *= -1
-        logger.warning("Changed sign of given charge to match positive ionmode")
 
     spectrum.set("charge", charge)
 
diff --git a/matchms/filtering/metadata_processing/derive_smiles_from_pubchem_compound_name_search.py b/matchms/filtering/metadata_processing/derive_smiles_from_pubchem_compound_name_search.py
index 2ba3bb7b..6002346a 100644
--- a/matchms/filtering/metadata_processing/derive_smiles_from_pubchem_compound_name_search.py
+++ b/matchms/filtering/metadata_processing/derive_smiles_from_pubchem_compound_name_search.py
@@ -1,8 +1,11 @@
 import csv
+import json
 import logging
 import os
+import time
 from functools import lru_cache
 from typing import List, Optional
+from urllib.error import URLError
 import numpy as np
 import pandas as pd
 import pubchempy
@@ -81,32 +84,46 @@ def _get_pubchem_compound_name_annotation(compound_name, csv_file=None) -> List[
     return annotated_compound_names
 
 
-def _pubchem_name_search(compound_name: str, name_search_depth=10,) -> List[dict]:
+def _pubchem_name_search(compound_name: str,
+                         name_search_depth=10,
+                         max_retries=15) -> List[dict]:
     """Search pubmed for compound name"""
-    try:
-        results_pubchem = pubchempy.get_compounds(compound_name,
-                                                  'name',
-                                                  listkey_count=name_search_depth)
-        if len(results_pubchem) == 0 and "_" in compound_name:
-            results_pubchem = pubchempy.get_compounds(compound_name.replace("_", " "),
+    retries = 0
+    max_delay = 3600
+    while retries < max_retries:
+        try:
+            results_pubchem = pubchempy.get_compounds(compound_name,
                                                       'name',
                                                       listkey_count=name_search_depth)
-    except (pubchempy.ServerError, ConnectionError, ConnectionAbortedError, pubchempy.PubChemHTTPError):
-        print("Connection error, trying again")
-        return _pubchem_name_search(compound_name, name_search_depth=name_search_depth)
-
-    extracted_results = []
-    # extract the needed information:
-    for result in results_pubchem:
-        smiles_pubchem = result.isomeric_smiles
-        if smiles_pubchem is None:
-            smiles_pubchem = result.canonical_smiles
-        extracted_results.append({"compound_name": compound_name,
-                                  "smiles": smiles_pubchem,
-                                  "inchi": result.inchi,
-                                  "inchikey": result.inchikey,
-                                  "monoisotopic_mass": float(result.monoisotopic_mass),})
-    return extracted_results
+            if len(results_pubchem) == 0 and "_" in compound_name:
+                results_pubchem = pubchempy.get_compounds(compound_name.replace("_", " "),
+                                                          'name',
+                                                          listkey_count=name_search_depth)
+            extracted_results = []
+            # extract the needed information:
+            for result in results_pubchem:
+                smiles_pubchem = result.isomeric_smiles
+                if smiles_pubchem is None:
+                    smiles_pubchem = result.canonical_smiles
+                extracted_results.append({"compound_name": compound_name,
+                                          "smiles": smiles_pubchem,
+                                          "inchi": result.inchi,
+                                          "inchikey": result.inchikey,
+                                          "monoisotopic_mass": float(result.monoisotopic_mass), })
+            return extracted_results
+        except (pubchempy.ServerError, ConnectionError, ConnectionAbortedError,
+                pubchempy.PubChemHTTPError, URLError):
+            # keep retrying when an connection error occurs
+            delay = 2 ** retries
+            delay = min(max_delay, delay)
+            print(f"Connection error, trying again, after waiting for {delay} seconds")
+            time.sleep(delay)
+        except json.decoder.JSONDecodeError:
+            logger.warning("Compound name: %s resulted in broken json from pubchem", compound_name)
+            return []
+    logger.error("Compound name: %s could not be loaded due to a connection error after %s tries ",
+                 compound_name, str(max_retries))
+    return []
 
 
 def _load_compound_name_annotations(annotated_compound_names_csv, compound_name: str):
@@ -120,7 +137,6 @@ def _load_compound_name_annotations(annotated_compound_names_csv, compound_name:
         "compound_name, smiles, inchi, inchikey, monoisotopic_mass"
 
     matches = annotated_compound_names[annotated_compound_names["compound_name"] == compound_name]
-    matches.replace(np.nan, None, inplace=True)
     return matches.to_dict("records")
 
 
diff --git a/matchms/filtering/metadata_processing/repair_not_matching_annotation.py b/matchms/filtering/metadata_processing/repair_not_matching_annotation.py
index d6955f49..4cd7ee61 100644
--- a/matchms/filtering/metadata_processing/repair_not_matching_annotation.py
+++ b/matchms/filtering/metadata_processing/repair_not_matching_annotation.py
@@ -2,7 +2,8 @@ import logging
 from matchms import Spectrum
 from matchms.filtering.filter_utils.smile_inchi_inchikey_conversions import (
     _check_fully_annotated, convert_inchi_to_inchikey, convert_inchi_to_smiles,
-    convert_smiles_to_inchi)
+    convert_smiles_to_inchi, is_valid_inchi, is_valid_inchikey,
+    is_valid_smiles)
 from matchms.filtering.metadata_processing.require_parent_mass_match_smiles import \
     _check_smiles_and_parent_mass_match
 
@@ -15,7 +16,7 @@ def repair_not_matching_annotation(spectrum_in: Spectrum):
     Repairs mismatches in a spectrum's annotations related to SMILES, InChI, and InChIKey.
 
     Given a spectrum, this function ensures that the provided SMILES, InChI, and InChIKey
-    annotations are consistent with one another. If there are discrepancies, they are resolved 
+    annotations are consistent with one another. If there are discrepancies, they are resolved
     as follows:
 
     1. If the SMILES and InChI do not match:
@@ -25,6 +26,7 @@ def repair_not_matching_annotation(spectrum_in: Spectrum):
         - A new InChIKey is generated from the InChI and replaces the old one.
 
     Warnings and information logs are generated to track changes and potential issues.
+    For correctness of InChIKey entries, only the first 14 characters are considered.
 
     Parameters:
     ----------
@@ -42,20 +44,32 @@ def repair_not_matching_annotation(spectrum_in: Spectrum):
 
     spectrum = spectrum_in.clone()
 
-    if not _check_fully_annotated(spectrum):
-        logger.warning("First run derive inchi_from_smiles, derive_inchikey_from_inchi, and derive_smiles_from_inchi")
-
     smiles = spectrum.get("smiles")
     inchi = spectrum.get("inchi")
     inchikey = spectrum.get("inchikey")
-    parent_mass = spectrum.get("parent_mass")
 
+    if not _check_fully_annotated(spectrum):
+        if is_valid_inchi(inchi) or is_valid_smiles(smiles) or is_valid_inchikey(inchikey):
+            logger.warning("Please first run repair_inchi_from_smiles, repair_smiles_from_inchi and repair_inchikey. "
+                           "The spectrum had partly valid annotations, "
+                           "this shows that these repair functions were not yet run.")
+        else:
+            logger.info("No valid annotation was available for the spectrum, "
+                        "so repair_not_matching_annotation was not run")
+        return spectrum
+
+    parent_mass = spectrum.get("parent_mass")
     inchi_from_smiles = convert_smiles_to_inchi(smiles)
 
     # Check if SMILES and InChI match
     if inchi_from_smiles != inchi:
         smiles_from_inchi = convert_inchi_to_smiles(inchi)
-        spectrum = _handle_smiles_inchi_mismatch(spectrum, smiles, inchi, smiles_from_inchi, inchi_from_smiles, parent_mass)
+        spectrum = _repair_smiles_inchi(spectrum,
+                                        smiles,
+                                        inchi,
+                                        smiles_from_inchi,
+                                        inchi_from_smiles,
+                                        parent_mass)
 
     # Check if the InChIKey matches the InChI
     correct_inchikey = convert_inchi_to_inchikey(spectrum.get("inchi"))
@@ -67,9 +81,9 @@ def repair_not_matching_annotation(spectrum_in: Spectrum):
     return spectrum
 
 
-def _handle_smiles_inchi_mismatch(spectrum, smiles, inchi,
-                                  smiles_from_inchi, inchi_from_smiles,
-                                  parent_mass):
+def _repair_smiles_inchi(spectrum, smiles, inchi,
+                         smiles_from_inchi, inchi_from_smiles,
+                         parent_mass):
     # pylint: disable=too-many-arguments
     smiles_correct = _check_smiles_and_parent_mass_match(smiles, parent_mass, 0.1)
     inchi_correct = _check_smiles_and_parent_mass_match(smiles_from_inchi, parent_mass, 0.1)
diff --git a/matchms/filtering/metadata_processing/repair_smiles_of_salts.py b/matchms/filtering/metadata_processing/repair_smiles_of_salts.py
index 94321075..2b8eb184 100644
--- a/matchms/filtering/metadata_processing/repair_smiles_of_salts.py
+++ b/matchms/filtering/metadata_processing/repair_smiles_of_salts.py
@@ -23,7 +23,6 @@ def repair_smiles_of_salts(spectrum_in,
         return spectrum
 
     if not is_valid_smiles(smiles):
-        logger.warning("No valid smiles, please run require_valid_annotation before")
         return spectrum
     parent_mass = spectrum.get("parent_mass")
     possible_ion_combinations = _create_possible_ions(smiles)
diff --git a/matchms/importing/load_spectra.py b/matchms/importing/load_spectra.py
index 5ffd5834..bdac5279 100644
--- a/matchms/importing/load_spectra.py
+++ b/matchms/importing/load_spectra.py
@@ -68,14 +68,20 @@ def load_from_pickle(filename: str, metadata_harmonization: bool) -> List[Spectr
     return loaded_object
 
 
-def load_list_of_spectrum_files(spectrum_files: Union[List[str],str]) -> List[SpectrumType]:
+def load_list_of_spectrum_files(spectrum_files: Union[List[str], str]
+                                ) -> Union[List[SpectrumType], Generator[SpectrumType, None, None]]:
     """Combines all spectra in multiple files into a list of spectra"""
-    all_spectra = []
+    # Just load spectra if it is a single file
     if isinstance(spectrum_files, str):
-        spectrum_files = [spectrum_files]
-    for spectrum_file in spectrum_files:
-        loaded_spectra = load_spectra(spectrum_file)
-        if isinstance(loaded_spectra, Generator):
-            loaded_spectra = list(loaded_spectra)
-        all_spectra.extend(loaded_spectra)
-    return all_spectra
+        return load_spectra(spectrum_files)
+    # If multiple files combine results into one generator
+    spectrum_generators = [load_spectra(spectrum_file) for
+                           spectrum_file in spectrum_files]
+
+    def chain(*iterables):
+        """Combines multiple iterators (and generators) into a single iterator"""
+        # chain('ABC', 'DEF') --> A B C D E F
+        for it in iterables:
+            for element in it:
+                yield element
+    return chain(*spectrum_generators)


Test patch:
diff --git a/tests/filtering/test_repair_not_matching_annotation.py b/tests/filtering/test_repair_not_matching_annotation.py
new file mode 100644
index 00000000..71aff25a
--- /dev/null
+++ b/tests/filtering/test_repair_not_matching_annotation.py
@@ -0,0 +1,47 @@
+import pytest
+from matchms.filtering import repair_not_matching_annotation
+from ..builder_Spectrum import SpectrumBuilder
+
+
+@pytest.mark.parametrize("smile, inchi, inchikey, correct_inchikey", [
+    ("CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"), #Already correct
+    ("CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWOOPSS-UHFFFAOYSA-N", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"), #Typo
+])
+def test_repair_not_matching_annotation_repair_inchikey(smile, inchi, inchikey, correct_inchikey):
+    """Checks if the inchikey is repaired, when smile and inchi already match"""
+    builder = SpectrumBuilder()
+    spectrum_in = builder.with_metadata({"smiles": smile,
+                                         "inchi": inchi,
+                                         "inchikey": inchikey}).build()
+    spectrum_out = repair_not_matching_annotation(spectrum_in)
+    assert spectrum_out.get("inchikey") == correct_inchikey
+    assert spectrum_out.get("smiles") == spectrum_in.get("smiles")
+    assert spectrum_out.get("inchi") == spectrum_in.get("inchi")
+
+
+@pytest.mark.parametrize("smile, inchi, inchikey, parent_mass, correct_smiles, correct_inchi, correct_inchikey", [
+    (None, "n/a", "wrong information", 100,
+     None, "n/a", "wrong information"),  # no annotation and wrong field entries
+    ("CCC", None, None, 100,
+     "CCC", None, None),  # missing inchi and inchikey
+    ("CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N", 100,
+     "CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),  # Already correct
+    ("CC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWOOPSS-UHFFFAOYSA-N", 44,  # smiles and inchikey wrong
+     "CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),
+    ("CCC", "InChI=1S/C4H10/c1-3-4-2/h3-4H2,1-2H3", "ATUOYWHBWOOPSS-UHFFFAOYSA-N", 44,  # inchi wrong
+     "CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),
+    ("C(=O)=O", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N", 44,  # Both match the parent mass
+     "C(=O)=O", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),  # So nothing should be changed
+])
+def test_repair_not_matching_annotation_repair_smiles_inchi(smile, inchi, inchikey, parent_mass,
+                                                     correct_smiles, correct_inchi, correct_inchikey):
+    # pylint: disable=too-many-arguments
+    builder = SpectrumBuilder()
+    spectrum_in = builder.with_metadata({"smiles": smile,
+                                         "inchi": inchi,
+                                         "inchikey": inchikey,
+                                         "parent_mass": parent_mass}).build()
+    spectrum_out = repair_not_matching_annotation(spectrum_in)
+    assert spectrum_out.get("inchikey") == correct_inchikey
+    assert spectrum_out.get("smiles") == correct_smiles
+    assert spectrum_out.get("inchi") == correct_inchi
diff --git a/tests/filtering/test_repair_smiles_from_compound_name.py b/tests/filtering/test_repair_smiles_from_compound_name.py
index ba6bb809..bf071f5f 100644
--- a/tests/filtering/test_repair_smiles_from_compound_name.py
+++ b/tests/filtering/test_repair_smiles_from_compound_name.py
@@ -1,5 +1,7 @@
 import csv
+import math
 import os
+from typing import List
 import pytest
 from matchms.filtering import derive_smiles_from_pubchem_compound_name_search
 from matchms.filtering.metadata_processing.derive_smiles_from_pubchem_compound_name_search import (
@@ -72,7 +74,7 @@ def test_repair_smiles_from_compound_name(compound_name, parent_mass, smiles,
     empty_csv_file = os.path.join(tmp_path, "test.csv")
     spectrum = derive_smiles_from_pubchem_compound_name_search(spectrum_in, empty_csv_file, mass_tolerance=0.1)
     assert spectrum.get("smiles") == expected_smiles, "Expected different smiles."
-    stored_in_csv_file = _load_compound_name_annotations(empty_csv_file, compound_name)
+    stored_in_csv_file = replace_nan_with_none(_load_compound_name_annotations(empty_csv_file, compound_name))
     assert len(stored_in_csv_file) == 1
     assert stored_in_csv_file[0]["smiles"] == expected_smiles
     assert stored_in_csv_file[0]["compound_name"] == compound_name
@@ -94,7 +96,7 @@ def test_write_compound_names_to_file(tmp_path):
     _write_compound_name_annotations(csv_file_name,
                                      annotation_2)
     assert _load_compound_name_annotations(csv_file_name, "glucose") == annotation_1
-    assert _load_compound_name_annotations(csv_file_name, "compound_2") == annotation_2
+    assert replace_nan_with_none(_load_compound_name_annotations(csv_file_name, "compound_2")) == annotation_2
 
 
 @pytest.mark.parametrize("compound_name, expected_output", [
@@ -106,7 +108,7 @@ def test_write_compound_names_to_file(tmp_path):
                      'inchikey': None, 'monoisotopic_mass': None}]), ])
 def test_load_compound_name_annotation(compound_name, expected_output, csv_file_annotated_compound_names):
     result = _load_compound_name_annotations(csv_file_annotated_compound_names, compound_name)
-    assert result == expected_output
+    assert replace_nan_with_none(result) == expected_output
 
 
 @pytest.mark.parametrize("compound_name, expected_output", [
@@ -132,4 +134,12 @@ def test_pubchem_name_search(compound_name, expected_output):
     ("does_not_exist", [])])
 def test_get_compound_name_annotation(compound_name, expected_output, csv_file_annotated_compound_names):
     result = _get_pubchem_compound_name_annotation(compound_name, csv_file_annotated_compound_names)
-    assert result == expected_output
+    assert replace_nan_with_none(result) == expected_output
+
+
+def replace_nan_with_none(matches_found: List[dict]):
+    for match in matches_found:
+        for key, value in match.items():
+            if isinstance(value, float) and math.isnan(value):
+                match[key] = None
+    return matches_found
diff --git a/tests/filtering/test_require_valid_annotation.py b/tests/filtering/test_require_valid_annotation.py
index 2e2f776b..fdedd987 100644
--- a/tests/filtering/test_require_valid_annotation.py
+++ b/tests/filtering/test_require_valid_annotation.py
@@ -1,6 +1,5 @@
 import pytest
-from matchms.filtering import (repair_not_matching_annotation,
-                               require_valid_annotation)
+from matchms.filtering import require_valid_annotation
 from ..builder_Spectrum import SpectrumBuilder
 
 
@@ -23,43 +22,3 @@ def test_require_valid_annotation(smile, inchi, inchikey, correct):
         assert spectrum_out == spectrum_in
     else:
         assert spectrum_out is None
-
-
-@pytest.mark.parametrize("smile, inchi, inchikey, correct_inchikey", [
-    ("CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"), #Already correct
-    ("CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWOOPSS-UHFFFAOYSA-N", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"), #Typo
-])
-def test_repair_valid_annotation_repair_inchikey(smile, inchi, inchikey, correct_inchikey):
-    """Checks if the inchikey is repaired, when smile and inchi already match"""
-    builder = SpectrumBuilder()
-    spectrum_in = builder.with_metadata({"smiles": smile,
-                                         "inchi": inchi,
-                                         "inchikey": inchikey}).build()
-    spectrum_out = repair_not_matching_annotation(spectrum_in)
-    assert spectrum_out.get("inchikey") == correct_inchikey
-    assert spectrum_out.get("smiles") == spectrum_in.get("smiles")
-    assert spectrum_out.get("inchi") == spectrum_in.get("inchi")
-
-
-@pytest.mark.parametrize("smile, inchi, inchikey, parent_mass, correct_smiles, correct_inchi, correct_inchikey", [
-    ("CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N", 100,
-     "CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),  # Already correct
-    ("CC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWOOPSS-UHFFFAOYSA-N", 44,  # smiles and inchikey wrong
-     "CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),
-    ("CCC", "InChI=1S/C4H10/c1-3-4-2/h3-4H2,1-2H3", "ATUOYWHBWOOPSS-UHFFFAOYSA-N", 44,  # inchi wrong
-     "CCC", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),
-    ("C(=O)=O", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N", 44,  # Both match the parent mass
-     "C(=O)=O", "InChI=1S/C3H8/c1-3-2/h3H2,1-2H3", "ATUOYWHBWRKTHZ-UHFFFAOYSA-N"),  # So nothing should be changed
-])
-def test_repair_valid_annotation_repair_smiles_inchi(smile, inchi, inchikey, parent_mass,
-                                                     correct_smiles, correct_inchi, correct_inchikey):
-    # pylint: disable=too-many-arguments
-    builder = SpectrumBuilder()
-    spectrum_in = builder.with_metadata({"smiles": smile,
-                                         "inchi": inchi,
-                                         "inchikey": inchikey,
-                                         "parent_mass": parent_mass}).build()
-    spectrum_out = repair_not_matching_annotation(spectrum_in)
-    assert spectrum_out.get("inchikey") == correct_inchikey
-    assert spectrum_out.get("smiles") == correct_smiles
-    assert spectrum_out.get("inchi") == correct_inchi
diff --git a/tests/filtering/test_spectrum_processor.py b/tests/filtering/test_spectrum_processor.py
index 25b39d39..4143f8df 100644
--- a/tests/filtering/test_spectrum_processor.py
+++ b/tests/filtering/test_spectrum_processor.py
@@ -7,16 +7,15 @@ from ..builder_Spectrum import SpectrumBuilder
 
 @pytest.fixture
 def spectrums():
-    metadata1 = {"charge": "+1",
-                 "pepmass": 100}
-    metadata2 = {"charge": "-1",
-                 "pepmass": 102}
-    metadata3 = {"charge": -1,
-                 "pepmass": 104}
-
-    s1 = SpectrumBuilder().with_metadata(metadata1).build()
-    s2 = SpectrumBuilder().with_metadata(metadata2).build()
-    s3 = SpectrumBuilder().with_metadata(metadata3).build()
+    s1 = SpectrumBuilder().\
+        with_metadata({"charge": "+1", "pepmass": 100}).\
+        with_mz([10, 20, 30]).with_intensities([0.1, 0.4, 10]).build()
+    s2 = SpectrumBuilder().with_metadata({"charge": "-1",
+                                          "pepmass": 102}).\
+        with_mz([10, 20, 30]).with_intensities([0.1, 0.2, 1]).build()
+    s3 = SpectrumBuilder().with_metadata({"charge": -1,
+                                          "pepmass": 104}).\
+        with_mz([10, ]).with_intensities([0.1, ]).build()
     return [s1, s2, s3]
 
 
@@ -135,18 +134,20 @@ def test_filter_spectrums(spectrums):
 
 def test_filter_spectrums_report(spectrums):
     processor = SpectrumProcessor("minimal")
+    processor.add_filter(filter_function=("require_minimum_number_of_peaks", {"n_required": 2}))
+    processor.add_filter(filter_function="add_losses")
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
-    assert len(spectrums) == 3
+    assert len(spectrums) == 2
     actual_masses = [s.get("precursor_mz") for s in spectrums]
-    expected_masses = [100, 102, 104]
+    expected_masses = [100, 102]
     assert actual_masses == expected_masses
     assert report.counter_number_processed == 3
-    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3, 'derive_ionmode': 3}
+    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3, 'derive_ionmode': 3}
     report_df = report.to_dataframe()
-    assert np.all(report_df.loc[["make_charge_int", "interpret_pepmass", "derive_ionmode"]].values == np.array(
-        [[0, 2],
-         [0, 3],
-         [0, 3]]))
+    assert np.all(report_df.loc[["require_minimum_number_of_peaks", "interpret_pepmass", "add_losses"]].values == np.array(
+        [[1, 0, 0],
+         [0, 3, 0],
+         [0, 0, 2]]))
 
 
 def test_processing_report_class(spectrums):
@@ -157,7 +158,7 @@ def test_processing_report_class(spectrums):
         processing_report.add_to_report(s, spectrum_processed, "test_filter")
 
     assert not processing_report.counter_removed_spectrums
-    assert processing_report.counter_changed_spectrum == {"test_filter": 3}
+    assert processing_report.counter_changed_metadata == {"test_filter": 3}
 
 
 def test_adding_custom_filter(spectrums):
@@ -172,7 +173,7 @@ def test_adding_custom_filter(spectrums):
     assert filters[-1].__name__ == "nonsense_inchikey"
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
     assert report.counter_number_processed == 3
-    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3,
+    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3,
                                                'derive_ionmode': 3, 'nonsense_inchikey': 3}
     assert spectrums[0].get("inchikey") == "NONSENSE", "Custom filter not executed properly"
 
@@ -189,7 +190,7 @@ def test_adding_custom_filter_with_parameters(spectrums):
     assert filters[-1].__name__ == "nonsense_inchikey_multiple"
     spectrums, report = processor.process_spectrums(spectrums, create_report=True)
     assert report.counter_number_processed == 3
-    assert report.counter_changed_spectrum == {'make_charge_int': 2, 'interpret_pepmass': 3,
+    assert report.counter_changed_metadata == {'make_charge_int': 2, 'interpret_pepmass': 3,
                                                'derive_ionmode': 3, 'nonsense_inchikey_multiple': 3}
     assert spectrums[0].get("inchikey") == "NONSENSENONSENSE", "Custom filter not executed properly"