Repository: matchms/matchms
Pull request #536 (2023-10-09 12:15:23)

Store cleaning settings
Currently the cleaning settings are not stored in the log file. 
Also we have a different way of generating a yaml file and generating the string format of cleaning. I will try to combine this into the same funciton in this PR


Patch:
diff --git a/matchms/Pipeline.py b/matchms/Pipeline.py
index a2b431b2..f8f3c342 100644
--- a/matchms/Pipeline.py
+++ b/matchms/Pipeline.py
@@ -2,7 +2,6 @@ import logging
 from collections import OrderedDict
 from datetime import datetime
 from typing import Iterable, List, Optional, Union
-import yaml
 import matchms.similarity as mssimilarity
 from matchms import calculate_scores
 from matchms.filtering.filter_order_and_default_pipelines import ALL_FILTERS
@@ -12,6 +11,8 @@ from matchms.logging_functions import (add_logging_to_file,
                                        reset_matchms_logger,
                                        set_matchms_logger_level)
 from matchms.typing import SpectrumType
+from matchms.yaml_file_functions import (load_workflow_from_yaml_file,
+                                         ordered_dump)
 
 
 _masking_functions = ["filter_by_range"]
@@ -382,38 +383,3 @@ def check_score_computation(score_computations: Iterable[Union[str, List[dict]]]
         if callable(computation[0]):
             continue
         raise ValueError(f"Unknown score computation: {computation[0]}.")
-
-
-def load_workflow_from_yaml_file(yaml_file: str) -> OrderedDict:
-    with open(yaml_file, 'r', encoding="utf-8") as file:
-        workflow = ordered_load(file, yaml.SafeLoader)
-    if workflow["reference_filters"] == "processing_queries":
-        workflow["reference_filters"] = workflow["query_filters"]
-    return workflow
-
-
-def ordered_load(stream, loader=yaml.SafeLoader, object_pairs_hook=OrderedDict) -> OrderedDict:
-    """ Code from https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts
-    """
-    class OrderedLoader(loader):
-        pass
-
-    def construct_mapping(loader, node):
-        loader.flatten_mapping(node)
-        return object_pairs_hook(loader.construct_pairs(node))
-    OrderedLoader.add_constructor(
-        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-        construct_mapping)
-    return yaml.load(stream, OrderedLoader)
-
-
-def ordered_dump(data: OrderedDict, stream=None, dumper=yaml.SafeDumper, **kwds):
-    class OrderedDumper(dumper):
-        pass
-
-    def _dict_representer(dumper, data):
-        return dumper.represent_mapping(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-            data.items())
-    OrderedDumper.add_representer(OrderedDict, _dict_representer)
-    return yaml.dump(data, stream, OrderedDumper, **kwds)
diff --git a/matchms/filtering/SpectrumProcessor.py b/matchms/filtering/SpectrumProcessor.py
index 5a09aba5..17df7490 100644
--- a/matchms/filtering/SpectrumProcessor.py
+++ b/matchms/filtering/SpectrumProcessor.py
@@ -1,6 +1,6 @@
 import inspect
 import logging
-from collections import defaultdict
+from collections import OrderedDict, defaultdict
 from functools import partial
 from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
 import numpy as np
@@ -9,6 +9,7 @@ from tqdm import tqdm
 from matchms import Spectrum
 from matchms.filtering.filter_order_and_default_pipelines import (
     ALL_FILTERS, FILTER_FUNCTION_NAMES, PREDEFINED_PIPELINES)
+from matchms.yaml_file_functions import ordered_dump
 
 
 logger = logging.getLogger("matchms")
@@ -215,17 +216,9 @@ class SpectrumProcessor:
         return filter_list
 
     def __str__(self):
-        summary_string = "SpectrumProcessor\nProcessing steps:"
-        for processing_step in self.processing_steps:
-            if isinstance(processing_step, str):
-                summary_string += "\n- " + processing_step
-            elif isinstance(processing_step, tuple):
-                filter_name = processing_step[0]
-                summary_string += "\n- - " + filter_name
-                filter_params = processing_step[1]
-                for filter_param in filter_params:
-                    summary_string += "\n  - " + str(filter_param)
-        return summary_string
+        workflow = OrderedDict()
+        workflow["Processing steps"] = self.processing_steps
+        return ordered_dump(workflow)
 
 
 def check_all_parameters_given(func):
@@ -304,13 +297,11 @@ class ProcessingReport:
     def __str__(self):
         pd.set_option('display.max_columns', 4)
         pd.set_option('display.width', 1000)
-        report_str = f"""\
------ Spectrum Processing Report -----
-Number of spectrums processed: {self.counter_number_processed}
-Number of spectrums removed: {sum(self.counter_removed_spectrums.values())}
-Changes during processing:
-{str(self.to_dataframe())}
-"""
+        report_str = ("----- Spectrum Processing Report -----\n"
+                      f"Number of spectrums processed: {self.counter_number_processed}\n"
+                      f"Number of spectrums removed: {sum(self.counter_removed_spectrums.values())}\n"
+                      "Changes during processing:\n"
+                      f"{str(self.to_dataframe())}")
         return report_str
 
     def __repr__(self):
diff --git a/matchms/filtering/metadata_processing/interpret_pepmass.py b/matchms/filtering/metadata_processing/interpret_pepmass.py
index 0e5ef752..ca221e03 100644
--- a/matchms/filtering/metadata_processing/interpret_pepmass.py
+++ b/matchms/filtering/metadata_processing/interpret_pepmass.py
@@ -58,8 +58,6 @@ def _interpret_pepmass_metadata(metadata):
         metadata["charge"] = charge
         logger.info("Added charge entry based on field 'pepmass'.")
 
-    del metadata["pepmass"]
-    logger.info("Removed pepmass, since the information was added to other fields")
     return metadata
 
 
diff --git a/matchms/yaml_file_functions.py b/matchms/yaml_file_functions.py
new file mode 100644
index 00000000..448cfce3
--- /dev/null
+++ b/matchms/yaml_file_functions.py
@@ -0,0 +1,37 @@
+from collections import OrderedDict
+import yaml
+
+
+def ordered_load(stream, loader=yaml.SafeLoader, object_pairs_hook=OrderedDict) -> OrderedDict:
+    """ Code from https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts
+    """
+    class OrderedLoader(loader):
+        pass
+
+    def construct_mapping(loader, node):
+        loader.flatten_mapping(node)
+        return object_pairs_hook(loader.construct_pairs(node))
+    OrderedLoader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        construct_mapping)
+    return yaml.load(stream, OrderedLoader)
+
+
+def ordered_dump(data: OrderedDict, stream=None, dumper=yaml.SafeDumper, **kwds):
+    class OrderedDumper(dumper):
+        pass
+
+    def _dict_representer(dumper, data):
+        return dumper.represent_mapping(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            data.items())
+    OrderedDumper.add_representer(OrderedDict, _dict_representer)
+    return yaml.dump(data, stream, OrderedDumper, **kwds)
+
+
+def load_workflow_from_yaml_file(yaml_file: str) -> OrderedDict:
+    with open(yaml_file, 'r', encoding="utf-8") as file:
+        workflow = ordered_load(file, yaml.SafeLoader)
+    if workflow["reference_filters"] == "processing_queries":
+        workflow["reference_filters"] = workflow["query_filters"]
+    return workflow


Test patch:
diff --git a/tests/filtering/test_spectrum_processor.py b/tests/filtering/test_spectrum_processor.py
index f716f3a6..7d0b9aa5 100644
--- a/tests/filtering/test_spectrum_processor.py
+++ b/tests/filtering/test_spectrum_processor.py
@@ -90,8 +90,8 @@ def test_incomplete_parameters():
 
 def test_string_output():
     processing = SpectrumProcessor("minimal")
-    expected_str = "SpectrumProcessor\nProcessing steps:\n- make_charge_int\n- interpret_pepmass" \
-                   "\n- derive_ionmode\n- correct_charge"
+    expected_str = "Processing steps:\n- make_charge_int\n- interpret_pepmass" \
+                   "\n- derive_ionmode\n- correct_charge\n"
     assert str(processing) == expected_str
 
 
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 54eaf2e5..b5a1fae0 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -3,8 +3,9 @@ import numpy as np
 import pytest
 from matchms import Pipeline
 from matchms.filtering import select_by_mz
-from matchms.Pipeline import create_workflow, load_workflow_from_yaml_file
+from matchms.Pipeline import create_workflow
 from matchms.similarity import ModifiedCosine
+from matchms.yaml_file_functions import load_workflow_from_yaml_file
 
 
 module_root = os.path.join(os.path.dirname(__file__), "..")