Repository: matchms/matchms
Pull request #674 (2024-06-03 09:57:08)

Add require correct ms level 2
Saving incrementally resulted in a significant reduction of speed, when I tested it. From 50 spectra/s to 3 spectra/s. 


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 59148c9e..b688c35c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [0.26.2] -2024-06-03
+### Added
+- Added require correct ms level
+
 ### Changed
 - Fixed bug in repair_adduct_and_parent_mass_based_on_smiles if mass from smiles is None
 
diff --git a/matchms/data/known_key_conversions.csv b/matchms/data/known_key_conversions.csv
index 5d532c02..b3c58610 100644
--- a/matchms/data/known_key_conversions.csv
+++ b/matchms/data/known_key_conversions.csv
@@ -1,205 +1,206 @@
-﻿known_synonym,matchms_default
-precursor_type,adduct
-ms$focused_ion:precursor_type,adduct
-precursortype,adduct
-comment:analysis_id,analysis_id
-comment:analysis_name,analysis_name
-authors,authors
-ms$focused_ion:base_peak,base_peak
-ms$focused_ion:base_peak_intensity,base_peak_intensity
-maximum_intensity,base_peak_intensity
-ac$chromatography:capillary_voltage,capillary_voltage
-ch$link:cas,cas
-related_cas#,cas
-precursor_charge,charge
-precursorcharge,charge
-ac$mass_spectrometry:charge,charge
-charge,charge
-ms$data_processing:charge_deconvoluted,charge_deconvolved
-ch$link:chebi,chebi
-ch$link:chempdb,chempdb
-ch$link:chemspider,chemspider
-ac$mass_spectrometry:date,collection_date
-collisionenergy,collision_energy
-colenergy,collision_energy
-ac$mass_spectrometry:collision_energy,collision_energy
-collision_energy,collision_energy
-collisiongas,collision_gas
-ac$mass_spectrometry:collsion_gas,collision_gas
-collision_gas,collision_gas
-ac$chromatogrpahy:collisional_cross_section,collisional_cross_section
-column,column_name
-columnname,column_name
-ac$chromatography:column_name,column_name
-pressure,column_pressure
-ac$chromatogrpahy:column_pressure,column_pressure
-ac$chromatography:column_temperature,column_temperature
-comment,comment
-comments,comment
-compound_type,compound_class
-ch$compound_class,compound_class
-ontology,compound_class
-name,compound_name
-compoundname,compound_name
-ch$name,compound_name
-comment:confidence,confidence
-libraryquality,confidence
-copyright,copyright
-datacollector,data_collector
-comment:datacollector,data_collector
-ms$data_processing:find_peak,data_processing_find_peak
-ms$data_processing:whole,data_processing_whole
-ms$focused_ion:derivative_form,derivative_formula
-ms$focused_ion:derivative_mass,derivative_mass
-ms$focused_ion:derivative_type,derivative_type
-ac$mass_spectrometry:desolvation_gas_flow,desolvation_gas
-ac$chromatogrpahy:drift_time,drift_time
-filename,file_name
-comment:filename,file_name
-ac$chromatography:flow_rate,flow_rate
-ch$formula,formula
-formula,formula
-ac$mass_spectrometry:fragmentation_mode,fragmentation_mode
-special_fragmentation,fragmentation_mode
-ac$mass_spectrometry:reagent_gas,gc_reagent_gas
-ac$chromatography:flow_gradient,gradient
-ch$iupac,inchi
-stdinchi,inchi
-inchi,inchi
-comment:inchiaux,inchi_aux
-inchiaux,inchi_aux
-ch$link:inchikey,inchikey
-inchikey,inchikey
-ac$mass_spectrometry:insource_voltage,insource_voltage
-cone_voltage,insource_voltage
-ac$instrument,instrument
-instrument,instrument
-ac$instrument_type,instrument_type
-instrument_type,instrument_type
-instrumenttype,instrument_type
-source_instrument,instrument_type
-ac$chromatogrpahy:inverse_reduced_mobility,inverse_reduced_mobility
-ac$mass_spectrometry:ion_mobility_drift_gas,ion_mobility_drift_gas
-ac$mass_spectrometry:ion_mobility_type,ion_mobility_type
-ac$mass_spectrometry:ion_mobility_voltage,ion_mobility_voltage
-ms$focused_ion:ion_type,ion_type
-ionimethod,ionization
-ac$mass_spectrometry:ionization,ionization
-ionization,ionization
-ac$mass_spectrometry:ionization_energy,ionization_energy
-ion_mode,ionmode
-ionpolarity,ionmode
-ionpol,ionmode
-ac$mass_spectrometry:ion_mode,ionmode
-ionmode,ionmode
-ms$focused_ion:isolation_window_lower_offset,isolation_window_lower_offset
-isolwidth,isolation_window_target_mz
-isolationwidth,isolation_window_target_mz
-isolation_width,isolation_window_target_mz
-ms$focused_ion:isolation_width,isolation_window_target_mz
-ms$focused_ion:isolation_window_upper_offset,isolation_window_upper_offset
-ch$link:kegg,kegg
-ch$link:knapsack,knapsack
-ac$mass_spectrometry:known_impurity,known_impurity
-known_impurity,known_impurity
-license,license
-ch$link:misc,link
-link,link
-ch$link:lipidbank,lipidbank
-ch$link:lipidmaps,lipidmaps
-ac$mass_spectrometry:laser,maldi_laser
-ac$mass_spectrometry:matrix,maldi_matrix
-ac$mass_spectrometry:mass_accuracy,mass_accuracy
-massrange,mass_range_mz
-massrangemz,mass_range_mz
-mass_range,mass_range_mz
-ac$mass_spectrometry:mass_range_m/z,mass_range_mz
-ac$mass_spectrometry:ms_type,ms_level
-spectrum_type,ms_level
-mslevel,ms_level
-ac$mass_spectrometry:msn_pathway,msn_pathway
-msn_pathway,msn_pathway
-ch$link:nist,nist_id
-ch$nominal_mass,nominal_mass
-mw,nominal_mass
-numpeaks,num_peaks
-pk$num_peak,num_peaks
-num peaks,num_peaks
-sp$lineage,organism_lineage
-sp$scientific_name,organism_name
-organism,organism_name
-parentmass,parent_mass
-exact_mass,parent_mass
-exactmass,parent_mass
-ch$exact_mass,parent_mass
-peakwidth,peak_width
-ms$focused_ion:peak_width,peak_width
-pk$annotation,peakannotations
-comment:peptide_modifications,peptide_modifications
-peptide_mods,peptide_modifications
-comment:peptide_sequence,peptide_sequence
-peptide_sequence,peptide_sequence
-seq,peptide_sequence
-ac$mass_spectrometry:post_source_decay,post_source_decay
-precursorintensity,precursor_intensity
-ms$focused_ion:precursor_ion_purity,precursor_ion_purity
-precursor_mass,precursor_mz
-precursormass,precursor_mz
-precursormz,precursor_mz
-precursor,precursor_mz
-ms$focused_ion:precursor_m/z,precursor_mz
-pi,principal_investigator
-comment:pi,principal_investigator
-ch$link:pubchem,pubchem
-publication,publication
-ch$link:pubmed,pubmed
-pubmed,pubmed
-reagentgaspressure,reagent_gas_pressure
-reagentgaspres,reagent_gas_pressure
-ac$mass_spectrometry:reagent_gas_pressure,reagent_gas_pressure
-reagention,reagent_ion
-ac$mass_spectrometry:reagent_ion,reagent_ion
-date,record_date
-ac$mass_spectrometry:reflector_mode,reflector_mode
-retentionindex,retention_index
-ri,retention_index
-ac$chromatography:retention_index,retention_index
-retention_index,retention_index
-retentiontime,retention_time
-rt,retention_time
-rt_query,retention_time
-rettime,retention_time
-rtinseconds,retention_time
-ac$chromatography:retention_time,retention_time
-comment:salt,salt
-salt,salt
-comment:salt_mix_cas,salt_mix_cas
-salt_mix_cas,salt_mix_cas
-sp$sample,sample
-ac$chromatogrpahy:sample_inlet,sample_inlet
-sample_inlet,sample_inlet
-sp$link,sample_link
-ac$mass_spectrometry:scanning,scanning
-comment:scans,scans
-scans,scans
-ac$mass_spectrometry:skim_1,skim_1_value
-ch$smiles,smiles
-smiles,smiles
-ms$data_processing:software_system_id,software_system_id
-ac$chromatogrpahy:solvent,solvent
-spectrumid,spectrum_id
-accession,spectrum_id
-db#,spectrum_id
-pk$splash,splash
-comment:structure,structure
-submituser,submit_user
-comment:submituser,submit_user
-targetgas,target_gas
-ac$mass_spectrometry:target_gas,target_gas
-targetgaspressure,target_gas_pressure
-targetgaspres,target_gas_pressure
-ac$mass_spectrometry:target_gas_pressure,target_gas_pressure
-record title,title
-title,title
-ac$mass_spectrometry:trap_drive,trap_drive_value
-ch$link:united_nations,united_nations_number
+﻿known_synonym,matchms_default
+precursor_type,adduct
+ms$focused_ion:precursor_type,adduct
+precursortype,adduct
+comment:analysis_id,analysis_id
+comment:analysis_name,analysis_name
+authors,authors
+ms$focused_ion:base_peak,base_peak
+ms$focused_ion:base_peak_intensity,base_peak_intensity
+maximum_intensity,base_peak_intensity
+ac$chromatography:capillary_voltage,capillary_voltage
+ch$link:cas,cas
+related_cas#,cas
+precursor_charge,charge
+precursorcharge,charge
+ac$mass_spectrometry:charge,charge
+charge,charge
+ms$data_processing:charge_deconvoluted,charge_deconvolved
+ch$link:chebi,chebi
+ch$link:chempdb,chempdb
+ch$link:chemspider,chemspider
+ac$mass_spectrometry:date,collection_date
+collisionenergy,collision_energy
+colenergy,collision_energy
+ac$mass_spectrometry:collision_energy,collision_energy
+collision_energy,collision_energy
+collisiongas,collision_gas
+ac$mass_spectrometry:collsion_gas,collision_gas
+collision_gas,collision_gas
+ac$chromatogrpahy:collisional_cross_section,collisional_cross_section
+column,column_name
+columnname,column_name
+ac$chromatography:column_name,column_name
+pressure,column_pressure
+ac$chromatogrpahy:column_pressure,column_pressure
+ac$chromatography:column_temperature,column_temperature
+comment,comment
+comments,comment
+compound_type,compound_class
+ch$compound_class,compound_class
+ontology,compound_class
+name,compound_name
+compoundname,compound_name
+ch$name,compound_name
+comment:confidence,confidence
+libraryquality,confidence
+copyright,copyright
+datacollector,data_collector
+comment:datacollector,data_collector
+ms$data_processing:find_peak,data_processing_find_peak
+ms$data_processing:whole,data_processing_whole
+ms$focused_ion:derivative_form,derivative_formula
+ms$focused_ion:derivative_mass,derivative_mass
+ms$focused_ion:derivative_type,derivative_type
+ac$mass_spectrometry:desolvation_gas_flow,desolvation_gas
+ac$chromatogrpahy:drift_time,drift_time
+filename,file_name
+comment:filename,file_name
+ac$chromatography:flow_rate,flow_rate
+ch$formula,formula
+formula,formula
+ac$mass_spectrometry:fragmentation_mode,fragmentation_mode
+special_fragmentation,fragmentation_mode
+ac$mass_spectrometry:reagent_gas,gc_reagent_gas
+ac$chromatography:flow_gradient,gradient
+ch$iupac,inchi
+stdinchi,inchi
+inchi,inchi
+comment:inchiaux,inchi_aux
+inchiaux,inchi_aux
+ch$link:inchikey,inchikey
+inchikey,inchikey
+ac$mass_spectrometry:insource_voltage,insource_voltage
+cone_voltage,insource_voltage
+ac$instrument,instrument
+instrument,instrument
+ac$instrument_type,instrument_type
+instrument_type,instrument_type
+instrumenttype,instrument_type
+source_instrument,instrument_type
+ac$chromatogrpahy:inverse_reduced_mobility,inverse_reduced_mobility
+ac$mass_spectrometry:ion_mobility_drift_gas,ion_mobility_drift_gas
+ac$mass_spectrometry:ion_mobility_type,ion_mobility_type
+ac$mass_spectrometry:ion_mobility_voltage,ion_mobility_voltage
+ms$focused_ion:ion_type,ion_type
+ionimethod,ionization
+ac$mass_spectrometry:ionization,ionization
+ionization,ionization
+ac$mass_spectrometry:ionization_energy,ionization_energy
+ion_mode,ionmode
+ionpolarity,ionmode
+ionpol,ionmode
+ac$mass_spectrometry:ion_mode,ionmode
+ionmode,ionmode
+ms$focused_ion:isolation_window_lower_offset,isolation_window_lower_offset
+isolwidth,isolation_window_target_mz
+isolationwidth,isolation_window_target_mz
+isolation_width,isolation_window_target_mz
+ms$focused_ion:isolation_width,isolation_window_target_mz
+ms$focused_ion:isolation_window_upper_offset,isolation_window_upper_offset
+ch$link:kegg,kegg
+ch$link:knapsack,knapsack
+ac$mass_spectrometry:known_impurity,known_impurity
+known_impurity,known_impurity
+license,license
+ch$link:misc,link
+link,link
+ch$link:lipidbank,lipidbank
+ch$link:lipidmaps,lipidmaps
+ac$mass_spectrometry:laser,maldi_laser
+ac$mass_spectrometry:matrix,maldi_matrix
+ac$mass_spectrometry:mass_accuracy,mass_accuracy
+massrange,mass_range_mz
+massrangemz,mass_range_mz
+mass_range,mass_range_mz
+ac$mass_spectrometry:mass_range_m/z,mass_range_mz
+ac$mass_spectrometry:ms_type,ms_level
+spectrum_type,ms_level
+mslevel,ms_level
+ms_type,ms_level
+ac$mass_spectrometry:msn_pathway,msn_pathway
+msn_pathway,msn_pathway
+ch$link:nist,nist_id
+ch$nominal_mass,nominal_mass
+mw,nominal_mass
+numpeaks,num_peaks
+pk$num_peak,num_peaks
+num peaks,num_peaks
+sp$lineage,organism_lineage
+sp$scientific_name,organism_name
+organism,organism_name
+parentmass,parent_mass
+exact_mass,parent_mass
+exactmass,parent_mass
+ch$exact_mass,parent_mass
+peakwidth,peak_width
+ms$focused_ion:peak_width,peak_width
+pk$annotation,peakannotations
+comment:peptide_modifications,peptide_modifications
+peptide_mods,peptide_modifications
+comment:peptide_sequence,peptide_sequence
+peptide_sequence,peptide_sequence
+seq,peptide_sequence
+ac$mass_spectrometry:post_source_decay,post_source_decay
+precursorintensity,precursor_intensity
+ms$focused_ion:precursor_ion_purity,precursor_ion_purity
+precursor_mass,precursor_mz
+precursormass,precursor_mz
+precursormz,precursor_mz
+precursor,precursor_mz
+ms$focused_ion:precursor_m/z,precursor_mz
+pi,principal_investigator
+comment:pi,principal_investigator
+ch$link:pubchem,pubchem
+publication,publication
+ch$link:pubmed,pubmed
+pubmed,pubmed
+reagentgaspressure,reagent_gas_pressure
+reagentgaspres,reagent_gas_pressure
+ac$mass_spectrometry:reagent_gas_pressure,reagent_gas_pressure
+reagention,reagent_ion
+ac$mass_spectrometry:reagent_ion,reagent_ion
+date,record_date
+ac$mass_spectrometry:reflector_mode,reflector_mode
+retentionindex,retention_index
+ri,retention_index
+ac$chromatography:retention_index,retention_index
+retention_index,retention_index
+retentiontime,retention_time
+rt,retention_time
+rt_query,retention_time
+rettime,retention_time
+rtinseconds,retention_time
+ac$chromatography:retention_time,retention_time
+comment:salt,salt
+salt,salt
+comment:salt_mix_cas,salt_mix_cas
+salt_mix_cas,salt_mix_cas
+sp$sample,sample
+ac$chromatogrpahy:sample_inlet,sample_inlet
+sample_inlet,sample_inlet
+sp$link,sample_link
+ac$mass_spectrometry:scanning,scanning
+comment:scans,scans
+scans,scans
+ac$mass_spectrometry:skim_1,skim_1_value
+ch$smiles,smiles
+smiles,smiles
+ms$data_processing:software_system_id,software_system_id
+ac$chromatogrpahy:solvent,solvent
+spectrumid,spectrum_id
+accession,spectrum_id
+db#,spectrum_id
+pk$splash,splash
+comment:structure,structure
+submituser,submit_user
+comment:submituser,submit_user
+targetgas,target_gas
+ac$mass_spectrometry:target_gas,target_gas
+targetgaspressure,target_gas_pressure
+targetgaspres,target_gas_pressure
+ac$mass_spectrometry:target_gas_pressure,target_gas_pressure
+record title,title
+title,title
+ac$mass_spectrometry:trap_drive,trap_drive_value
+ch$link:united_nations,united_nations_number
diff --git a/matchms/filtering/__init__.py b/matchms/filtering/__init__.py
index 82b92bb1..515df4fd 100644
--- a/matchms/filtering/__init__.py
+++ b/matchms/filtering/__init__.py
@@ -124,6 +124,8 @@ from matchms.filtering.metadata_processing.require_compound_name import \
     require_compound_name
 from matchms.filtering.metadata_processing.require_correct_ionmode import \
     require_correct_ionmode
+from matchms.filtering.metadata_processing.require_correct_ms_level import \
+    require_correct_ms_level
 from matchms.filtering.metadata_processing.require_formula import \
     require_formula
 from matchms.filtering.metadata_processing.require_matching_adduct_and_ionmode import \
@@ -212,6 +214,7 @@ __all__ = [
     "require_precursor_below_mz",
     "require_precursor_mz",
     "require_compound_name",
+    "require_correct_ms_level",
     "require_formula",
     "require_retention_time",
     "require_retention_index",
diff --git a/matchms/filtering/default_pipelines.py b/matchms/filtering/default_pipelines.py
index fd958579..6356673f 100644
--- a/matchms/filtering/default_pipelines.py
+++ b/matchms/filtering/default_pipelines.py
@@ -59,11 +59,12 @@ OTHER_FILTERS = [matchms.filtering.metadata_processing.require_precursor_mz.requ
                  msfilters.require_minimum_number_of_peaks,
                  msfilters.add_fingerprint,
                  msfilters.add_losses,
-                 msfilters.repair_parent_mass_match_smiles_wrapper, ]
+                 msfilters.repair_parent_mass_match_smiles_wrapper,
+                 ]
 
 BASIC_FILTERS = HARMONIZE_METADATA_FIELD_NAMES + DERIVE_METADATA_IN_WRONG_FIELD + HARMONIZE_METADATA_ENTRIES
 DEFAULT_FILTERS = BASIC_FILTERS + [msfilters.normalize_intensities, ] + REQUIRE_COMPLETE_METADATA + DERIVE_MISSING_METADATA
-LIBRARY_CLEANING = DEFAULT_FILTERS + REPAIR_ANNOTATION + REQUIRE_COMPLETE_ANNOTATION
+LIBRARY_CLEANING = DEFAULT_FILTERS + REPAIR_ANNOTATION + REQUIRE_COMPLETE_ANNOTATION + [msfilters.require_correct_ms_level]
 MS2DEEPSCORE_TRAINING = LIBRARY_CLEANING + CLEAN_PEAKS
 
 
diff --git a/matchms/filtering/filter_order.py b/matchms/filtering/filter_order.py
index a1fdc236..77f92330 100644
--- a/matchms/filtering/filter_order.py
+++ b/matchms/filtering/filter_order.py
@@ -5,7 +5,8 @@ from matchms import filtering as msfilters
 
 # IMPORTANT!! IF YOU CHANGE ANYTHING HERE PLEASE ADD A TEST to test_all_filter_order.py
 # to ensure it is not changed back by accident later.
-ALL_FILTERS = [msfilters.make_charge_int,
+ALL_FILTERS = [msfilters.require_correct_ms_level,
+               msfilters.make_charge_int,
                msfilters.add_compound_name,
                msfilters.derive_adduct_from_name,
                msfilters.derive_formula_from_name,
diff --git a/matchms/filtering/metadata_processing/require_correct_ms_level.py b/matchms/filtering/metadata_processing/require_correct_ms_level.py
new file mode 100644
index 00000000..c65b3414
--- /dev/null
+++ b/matchms/filtering/metadata_processing/require_correct_ms_level.py
@@ -0,0 +1,10 @@
+from typing import Optional
+from matchms.Spectrum import Spectrum
+
+
+def require_correct_ms_level(spectrum: Spectrum, required_ms_level: int = 2) -> Optional[Spectrum]:
+    if spectrum is None:
+        return None
+    if spectrum.get("ms_level") in (f"MS{required_ms_level}", str(required_ms_level)):
+        return spectrum
+    return None


Test patch:
diff --git a/tests/filtering/test_require_correct_ms_level.py b/tests/filtering/test_require_correct_ms_level.py
new file mode 100644
index 00000000..f51618ca
--- /dev/null
+++ b/tests/filtering/test_require_correct_ms_level.py
@@ -0,0 +1,23 @@
+import pytest
+from matchms.filtering.metadata_processing.require_correct_ms_level import \
+    require_correct_ms_level
+from ..builder_Spectrum import SpectrumBuilder
+
+
+@pytest.mark.parametrize("metadata, ms_level_to_keep, spectrum_removed", [
+    ({"ms_level": "2"}, 2, False),
+    ({"ms_level": "MS2"}, 2, False),
+    ({}, 2, True),
+    ({"ms_level": "MS3"}, 2, True),
+    ({"ms_level": "MS2"}, 3, True),
+    ({"ms_type": "MS2"}, 2, False), # Check that key conversions are used
+])
+def test_require_correct_ms_level(metadata, ms_level_to_keep, spectrum_removed):
+    builder = SpectrumBuilder()
+    spectrum_in = builder.with_metadata(metadata).build()
+    spectrum = require_correct_ms_level(spectrum_in, ms_level_to_keep)
+
+    if spectrum_removed is True:
+        assert spectrum is None, "Expected spectrum to be filtered out since it does not have the correct ionmode"
+    else:
+        assert spectrum == spectrum_in